diff options
149 files changed, 4085 insertions, 6059 deletions
diff --git a/Android.bp b/Android.bp index b1db8e937..682711d42 100644 --- a/Android.bp +++ b/Android.bp @@ -104,6 +104,7 @@ cc_genrule { // Instantiate the dex_bootjars singleton module. dex_bootjars { name: "dex_bootjars", + no_full_install: true, } // Pseudo-test that's run on checkbuilds to ensure that get_clang_version can @@ -121,12 +122,15 @@ dexpreopt_systemserver_check { } // buildinfo.prop contains common properties for system/build.prop, like ro.build.version.* +// TODO(b/322090587): merge this to gen_build_prop.py script. buildinfo_prop { name: "buildinfo.prop", // not installable because this will be included to system/build.prop installable: false, + product_config: ":product_config", + // Currently, only microdroid can refer to buildinfo.prop visibility: ["//packages/modules/Virtualization/microdroid"], } @@ -135,3 +139,8 @@ buildinfo_prop { all_apex_contributions { name: "all_apex_contributions", } + +product_config { + name: "product_config", + visibility: ["//device/google/cuttlefish/system_image"], +} @@ -594,19 +594,13 @@ modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced by all of the vendor's other modules using the normal namespace and visibility rules. -`soongConfigTraceMutator` enables modules affected by soong config variables to -write outputs into a hashed directory path. It does this by recording accesses -to soong config variables on each module, and then accumulating records of each -module's all dependencies. `m soong_config_trace` builds information about -hashes to `$OUT_DIR/soong/soong_config_trace.json`. - ## Build logic The build logic is written in Go using the -[blueprint](http://godoc.org/github.com/google/blueprint) framework. Build -logic receives module definitions parsed into Go structures using reflection -and produces build rules. The build rules are collected by blueprint and -written to a [ninja](http://ninja-build.org) build file. +[blueprint](https://android.googlesource.com/platform/build/blueprint) +framework. Build logic receives module definitions parsed into Go structures +using reflection and produces build rules. The build rules are collected by +blueprint and written to a [ninja](http://ninja-build.org) build file. ## Environment Variables Config File diff --git a/aconfig/aconfig_declarations.go b/aconfig/aconfig_declarations.go index 71a64dddc..9e3d291b6 100644 --- a/aconfig/aconfig_declarations.go +++ b/aconfig/aconfig_declarations.go @@ -15,7 +15,8 @@ package aconfig import ( - "fmt" + "path/filepath" + "slices" "strings" "android/soong/android" @@ -23,9 +24,15 @@ import ( "github.com/google/blueprint" ) +type AconfigReleaseConfigValue struct { + ReleaseConfig string + Values []string `blueprint:"mutated"` +} + type DeclarationsModule struct { android.ModuleBase android.DefaultableModuleBase + blueprint.IncrementalModule // Properties for "aconfig_declarations" properties struct { @@ -35,8 +42,10 @@ type DeclarationsModule struct { // Release config flag package Package string - // Values from TARGET_RELEASE / RELEASE_ACONFIG_VALUE_SETS - Values []string `blueprint:"mutated"` + // Values for release configs / RELEASE_ACONFIG_VALUE_SETS + // The current release config is `ReleaseConfig: ""`, others + // are from RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS. + ReleaseConfigValues []AconfigReleaseConfigValue // Container(system/vendor/apex) that this module belongs to Container string @@ -44,8 +53,6 @@ type DeclarationsModule struct { // The flags will only be repackaged if this prop is true. Exportable bool } - - intermediatePath android.WritablePath } func DeclarationsFactory() android.Module { @@ -60,6 +67,10 @@ func DeclarationsFactory() android.Module { type implicitValuesTagType struct { blueprint.BaseDependencyTag + + // The release config name for these values. + // Empty string for the actual current release config. + ReleaseConfig string } var implicitValuesTag = implicitValuesTagType{} @@ -84,17 +95,10 @@ func (module *DeclarationsModule) DepsMutator(ctx android.BottomUpMutatorContext if len(valuesFromConfig) > 0 { ctx.AddDependency(ctx.Module(), implicitValuesTag, valuesFromConfig...) } -} - -func (module *DeclarationsModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - // The default output of this module is the intermediates format, which is - // not installable and in a private format that no other rules can handle - // correctly. - return []android.Path{module.intermediatePath}, nil - default: - return nil, fmt.Errorf("unsupported aconfig_declarations module reference tag %q", tag) + for rcName, valueSets := range ctx.Config().ReleaseAconfigExtraReleaseConfigsValueSets() { + if len(valueSets) > 0 { + ctx.AddDependency(ctx.Module(), implicitValuesTagType{ReleaseConfig: rcName}, valueSets...) + } } } @@ -116,60 +120,115 @@ func optionalVariable(prefix string, value string) string { return sb.String() } +// Assemble the actual filename. +// If `rcName` is not empty, then insert "-{rcName}" into the path before the +// file extension. +func assembleFileName(rcName, path string) string { + if rcName == "" { + return path + } + dir, file := filepath.Split(path) + rcName = "-" + rcName + ext := filepath.Ext(file) + base := file[:len(file)-len(ext)] + return dir + base + rcName + ext +} + func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { - // Get the values that came from the global RELEASE_ACONFIG_VALUE_SETS flag - valuesFiles := make([]android.Path, 0) + // Determine which release configs we are processing. + // + // We always process the current release config (empty string). + // We may have been told to also create artifacts for some others. + configs := append([]string{""}, ctx.Config().ReleaseAconfigExtraReleaseConfigs()...) + slices.Sort(configs) + + values := make(map[string][]string) + valuesFiles := make(map[string][]android.Path, 0) + providerData := android.AconfigReleaseDeclarationsProviderData{} ctx.VisitDirectDeps(func(dep android.Module) { if depData, ok := android.OtherModuleProvider(ctx, dep, valueSetProviderKey); ok { - paths, ok := depData.AvailablePackages[module.properties.Package] - if ok { - valuesFiles = append(valuesFiles, paths...) - for _, path := range paths { - module.properties.Values = append(module.properties.Values, path.String()) + depTag := ctx.OtherModuleDependencyTag(dep) + for _, config := range configs { + tag := implicitValuesTagType{ReleaseConfig: config} + if depTag == tag { + paths, ok := depData.AvailablePackages[module.properties.Package] + if ok { + valuesFiles[config] = append(valuesFiles[config], paths...) + for _, path := range paths { + values[config] = append(values[config], path.String()) + } + } } } } }) - - // Intermediate format - declarationFiles := android.PathsForModuleSrc(ctx, module.properties.Srcs) - intermediateCacheFilePath := android.PathForModuleOut(ctx, "intermediate.pb") - defaultPermission := ctx.Config().ReleaseAconfigFlagDefaultPermission() - inputFiles := make([]android.Path, len(declarationFiles)) - copy(inputFiles, declarationFiles) - inputFiles = append(inputFiles, valuesFiles...) - args := map[string]string{ - "release_version": ctx.Config().ReleaseVersion(), - "package": module.properties.Package, - "declarations": android.JoinPathsWithPrefix(declarationFiles, "--declarations "), - "values": joinAndPrefix(" --values ", module.properties.Values), - "default-permission": optionalVariable(" --default-permission ", defaultPermission), - } - if len(module.properties.Container) > 0 { - args["container"] = "--container " + module.properties.Container + for _, config := range configs { + module.properties.ReleaseConfigValues = append(module.properties.ReleaseConfigValues, AconfigReleaseConfigValue{ + ReleaseConfig: config, + Values: values[config], + }) + + // Intermediate format + declarationFiles := android.PathsForModuleSrc(ctx, module.properties.Srcs) + intermediateCacheFilePath := android.PathForModuleOut(ctx, assembleFileName(config, "intermediate.pb")) + var defaultPermission string + defaultPermission = ctx.Config().ReleaseAconfigFlagDefaultPermission() + if config != "" { + if confPerm, ok := ctx.Config().GetBuildFlag("RELEASE_ACONFIG_FLAG_DEFAULT_PERMISSION_" + config); ok { + defaultPermission = confPerm + } + } + inputFiles := make([]android.Path, len(declarationFiles)) + copy(inputFiles, declarationFiles) + inputFiles = append(inputFiles, valuesFiles[config]...) + args := map[string]string{ + "release_version": ctx.Config().ReleaseVersion(), + "package": module.properties.Package, + "declarations": android.JoinPathsWithPrefix(declarationFiles, "--declarations "), + "values": joinAndPrefix(" --values ", values[config]), + "default-permission": optionalVariable(" --default-permission ", defaultPermission), + } + if len(module.properties.Container) > 0 { + args["container"] = "--container " + module.properties.Container + } + ctx.Build(pctx, android.BuildParams{ + Rule: aconfigRule, + Output: intermediateCacheFilePath, + Inputs: inputFiles, + Description: "aconfig_declarations", + Args: args, + }) + + intermediateDumpFilePath := android.PathForModuleOut(ctx, assembleFileName(config, "intermediate.txt")) + ctx.Build(pctx, android.BuildParams{ + Rule: aconfigTextRule, + Output: intermediateDumpFilePath, + Inputs: android.Paths{intermediateCacheFilePath}, + Description: "aconfig_text", + }) + + providerData[config] = android.AconfigDeclarationsProviderData{ + Package: module.properties.Package, + Container: module.properties.Container, + Exportable: module.properties.Exportable, + IntermediateCacheOutputPath: intermediateCacheFilePath, + IntermediateDumpOutputPath: intermediateDumpFilePath, + } } - ctx.Build(pctx, android.BuildParams{ - Rule: aconfigRule, - Output: intermediateCacheFilePath, - Inputs: inputFiles, - Description: "aconfig_declarations", - Args: args, - }) + android.SetProvider(ctx, android.AconfigDeclarationsProviderKey, providerData[""]) + android.SetProvider(ctx, android.AconfigReleaseDeclarationsProviderKey, providerData) +} - intermediateDumpFilePath := android.PathForModuleOut(ctx, "intermediate.txt") - ctx.Build(pctx, android.BuildParams{ - Rule: aconfigTextRule, - Output: intermediateDumpFilePath, - Inputs: android.Paths{intermediateCacheFilePath}, - Description: "aconfig_text", - }) +func (module *DeclarationsModule) BuildActionProviderKeys() []blueprint.AnyProviderKey { + return []blueprint.AnyProviderKey{android.AconfigDeclarationsProviderKey} +} - android.SetProvider(ctx, android.AconfigDeclarationsProviderKey, android.AconfigDeclarationsProviderData{ - Package: module.properties.Package, - Container: module.properties.Container, - Exportable: module.properties.Exportable, - IntermediateCacheOutputPath: intermediateCacheFilePath, - IntermediateDumpOutputPath: intermediateDumpFilePath, - }) +func (module *DeclarationsModule) PackageContextPath() string { + return pkgPath +} +func (module *DeclarationsModule) CachedRules() []blueprint.Rule { + return []blueprint.Rule{aconfigRule, aconfigTextRule} } + +var _ blueprint.Incremental = &DeclarationsModule{} diff --git a/aconfig/aconfig_declarations_test.go b/aconfig/aconfig_declarations_test.go index c37274c71..548329523 100644 --- a/aconfig/aconfig_declarations_test.go +++ b/aconfig/aconfig_declarations_test.go @@ -15,6 +15,7 @@ package aconfig import ( + "slices" "strings" "testing" @@ -134,3 +135,95 @@ func TestMandatoryProperties(t *testing.T) { }) } } + +func TestAssembleFileName(t *testing.T) { + testCases := []struct { + name string + releaseConfig string + path string + expectedValue string + }{ + { + name: "active release config", + path: "file.path", + expectedValue: "file.path", + }, + { + name: "release config FOO", + releaseConfig: "FOO", + path: "file.path", + expectedValue: "file-FOO.path", + }, + } + for _, test := range testCases { + actualValue := assembleFileName(test.releaseConfig, test.path) + if actualValue != test.expectedValue { + t.Errorf("Expected %q found %q", test.expectedValue, actualValue) + } + } +} + +func TestGenerateAndroidBuildActions(t *testing.T) { + testCases := []struct { + name string + buildFlags map[string]string + bp string + errorHandler android.FixtureErrorHandler + }{ + { + name: "generate extra", + buildFlags: map[string]string{ + "RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS": "config2", + "RELEASE_ACONFIG_VALUE_SETS": "aconfig_value_set-config1", + "RELEASE_ACONFIG_VALUE_SETS_config2": "aconfig_value_set-config2", + }, + bp: ` + aconfig_declarations { + name: "module_name", + package: "com.example.package", + container: "com.android.foo", + srcs: [ + "foo.aconfig", + "bar.aconfig", + ], + } + aconfig_value_set { + name: "aconfig_value_set-config1", + values: [] + } + aconfig_value_set { + name: "aconfig_value_set-config2", + values: [] + } + `, + }, + } + for _, test := range testCases { + fixture := PrepareForTest(t, addBuildFlagsForTest(test.buildFlags)) + if test.errorHandler != nil { + fixture = fixture.ExtendWithErrorHandler(test.errorHandler) + } + result := fixture.RunTestWithBp(t, test.bp) + module := result.ModuleForTests("module_name", "").Module().(*DeclarationsModule) + depData, _ := android.SingletonModuleProvider(result, module, android.AconfigReleaseDeclarationsProviderKey) + expectedKeys := []string{""} + for _, rc := range strings.Split(test.buildFlags["RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS"], " ") { + expectedKeys = append(expectedKeys, rc) + } + slices.Sort(expectedKeys) + actualKeys := []string{} + for rc := range depData { + actualKeys = append(actualKeys, rc) + } + slices.Sort(actualKeys) + android.AssertStringEquals(t, "provider keys", strings.Join(expectedKeys, " "), strings.Join(actualKeys, " ")) + for _, rc := range actualKeys { + if !strings.HasSuffix(depData[rc].IntermediateCacheOutputPath.String(), assembleFileName(rc, "/intermediate.pb")) { + t.Errorf("Incorrect intermediates proto path in provider for release config %s: %s", rc, depData[rc].IntermediateCacheOutputPath.String()) + } + if !strings.HasSuffix(depData[rc].IntermediateDumpOutputPath.String(), assembleFileName(rc, "/intermediate.txt")) { + t.Errorf("Incorrect intermediates text path in provider for release config %s: %s", rc, depData[rc].IntermediateDumpOutputPath.String()) + } + } + } +} diff --git a/aconfig/all_aconfig_declarations.go b/aconfig/all_aconfig_declarations.go index e771d0597..0437c266f 100644 --- a/aconfig/all_aconfig_declarations.go +++ b/aconfig/all_aconfig_declarations.go @@ -17,6 +17,7 @@ package aconfig import ( "android/soong/android" "fmt" + "slices" ) // A singleton module that collects all of the aconfig flags declared in the @@ -27,70 +28,90 @@ import ( // ones that are relevant to the product currently being built, so that that infra // doesn't need to pull from multiple builds and merge them. func AllAconfigDeclarationsFactory() android.Singleton { - return &allAconfigDeclarationsSingleton{} + return &allAconfigDeclarationsSingleton{releaseMap: make(map[string]allAconfigReleaseDeclarationsSingleton)} } -type allAconfigDeclarationsSingleton struct { +type allAconfigReleaseDeclarationsSingleton struct { intermediateBinaryProtoPath android.OutputPath intermediateTextProtoPath android.OutputPath } +type allAconfigDeclarationsSingleton struct { + releaseMap map[string]allAconfigReleaseDeclarationsSingleton +} + +func (this *allAconfigDeclarationsSingleton) sortedConfigNames() []string { + var names []string + for k := range this.releaseMap { + names = append(names, k) + } + slices.Sort(names) + return names +} + func (this *allAconfigDeclarationsSingleton) GenerateBuildActions(ctx android.SingletonContext) { - // Find all of the aconfig_declarations modules - var packages = make(map[string]int) - var cacheFiles android.Paths - ctx.VisitAllModules(func(module android.Module) { - decl, ok := android.SingletonModuleProvider(ctx, module, android.AconfigDeclarationsProviderKey) - if !ok { - return - } - cacheFiles = append(cacheFiles, decl.IntermediateCacheOutputPath) - packages[decl.Package]++ - }) + for _, rcName := range append([]string{""}, ctx.Config().ReleaseAconfigExtraReleaseConfigs()...) { + // Find all of the aconfig_declarations modules + var packages = make(map[string]int) + var cacheFiles android.Paths + ctx.VisitAllModules(func(module android.Module) { + decl, ok := android.SingletonModuleProvider(ctx, module, android.AconfigReleaseDeclarationsProviderKey) + if !ok { + return + } + cacheFiles = append(cacheFiles, decl[rcName].IntermediateCacheOutputPath) + packages[decl[rcName].Package]++ + }) - var numOffendingPkg = 0 - for pkg, cnt := range packages { - if cnt > 1 { - fmt.Printf("%d aconfig_declarations found for package %s\n", cnt, pkg) - numOffendingPkg++ + var numOffendingPkg = 0 + for pkg, cnt := range packages { + if cnt > 1 { + fmt.Printf("%d aconfig_declarations found for package %s\n", cnt, pkg) + numOffendingPkg++ + } } - } - if numOffendingPkg > 0 { - panic(fmt.Errorf("Only one aconfig_declarations allowed for each package.")) - } + if numOffendingPkg > 0 { + panic(fmt.Errorf("Only one aconfig_declarations allowed for each package.")) + } - // Generate build action for aconfig (binary proto output) - this.intermediateBinaryProtoPath = android.PathForIntermediates(ctx, "all_aconfig_declarations.pb") - ctx.Build(pctx, android.BuildParams{ - Rule: AllDeclarationsRule, - Inputs: cacheFiles, - Output: this.intermediateBinaryProtoPath, - Description: "all_aconfig_declarations", - Args: map[string]string{ - "cache_files": android.JoinPathsWithPrefix(cacheFiles, "--cache "), - }, - }) - ctx.Phony("all_aconfig_declarations", this.intermediateBinaryProtoPath) + // Generate build action for aconfig (binary proto output) + paths := allAconfigReleaseDeclarationsSingleton{ + intermediateBinaryProtoPath: android.PathForIntermediates(ctx, assembleFileName(rcName, "all_aconfig_declarations.pb")), + intermediateTextProtoPath: android.PathForIntermediates(ctx, assembleFileName(rcName, "all_aconfig_declarations.textproto")), + } + this.releaseMap[rcName] = paths + ctx.Build(pctx, android.BuildParams{ + Rule: AllDeclarationsRule, + Inputs: cacheFiles, + Output: this.releaseMap[rcName].intermediateBinaryProtoPath, + Description: "all_aconfig_declarations", + Args: map[string]string{ + "cache_files": android.JoinPathsWithPrefix(cacheFiles, "--cache "), + }, + }) + ctx.Phony("all_aconfig_declarations", this.releaseMap[rcName].intermediateBinaryProtoPath) - // Generate build action for aconfig (text proto output) - this.intermediateTextProtoPath = android.PathForIntermediates(ctx, "all_aconfig_declarations.textproto") - ctx.Build(pctx, android.BuildParams{ - Rule: AllDeclarationsRuleTextProto, - Inputs: cacheFiles, - Output: this.intermediateTextProtoPath, - Description: "all_aconfig_declarations_textproto", - Args: map[string]string{ - "cache_files": android.JoinPathsWithPrefix(cacheFiles, "--cache "), - }, - }) - ctx.Phony("all_aconfig_declarations_textproto", this.intermediateTextProtoPath) + // Generate build action for aconfig (text proto output) + ctx.Build(pctx, android.BuildParams{ + Rule: AllDeclarationsRuleTextProto, + Inputs: cacheFiles, + Output: this.releaseMap[rcName].intermediateTextProtoPath, + Description: "all_aconfig_declarations_textproto", + Args: map[string]string{ + "cache_files": android.JoinPathsWithPrefix(cacheFiles, "--cache "), + }, + }) + ctx.Phony("all_aconfig_declarations_textproto", this.releaseMap[rcName].intermediateTextProtoPath) + } } func (this *allAconfigDeclarationsSingleton) MakeVars(ctx android.MakeVarsContext) { - ctx.DistForGoal("droid", this.intermediateBinaryProtoPath) - for _, goal := range []string{"docs", "droid", "sdk"} { - ctx.DistForGoalWithFilename(goal, this.intermediateBinaryProtoPath, "flags.pb") - ctx.DistForGoalWithFilename(goal, this.intermediateTextProtoPath, "flags.textproto") + for _, rcName := range this.sortedConfigNames() { + ctx.DistForGoal("droid", this.releaseMap[rcName].intermediateBinaryProtoPath) + for _, goal := range []string{"docs", "droid", "sdk"} { + ctx.DistForGoalWithFilename(goal, this.releaseMap[rcName].intermediateBinaryProtoPath, assembleFileName(rcName, "flags.pb")) + ctx.DistForGoalWithFilename(goal, this.releaseMap[rcName].intermediateTextProtoPath, assembleFileName(rcName, "flags.textproto")) + } } } diff --git a/aconfig/build_flags/declarations.go b/aconfig/build_flags/declarations.go index f6a6ee1a8..e927db2cf 100644 --- a/aconfig/build_flags/declarations.go +++ b/aconfig/build_flags/declarations.go @@ -15,7 +15,6 @@ package build_flags import ( - "fmt" "strings" "android/soong/android" @@ -39,8 +38,6 @@ type DeclarationsModule struct { // aconfig files, relative to this Android.bp file Srcs []string `android:"path"` } - - intermediatePath android.WritablePath } func DeclarationsFactory() android.Module { @@ -53,18 +50,6 @@ func DeclarationsFactory() android.Module { return module } -func (module *DeclarationsModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - // The default output of this module is the intermediates format, which is - // not installable and in a private format that no other rules can handle - // correctly. - return []android.Path{module.intermediatePath}, nil - default: - return nil, fmt.Errorf("unsupported build_flags_declarations module reference tag %q", tag) - } -} - func joinAndPrefix(prefix string, values []string) string { var sb strings.Builder for _, v := range values { diff --git a/aconfig/codegen/Android.bp b/aconfig/codegen/Android.bp index 0c78b946b..5fac0a8d1 100644 --- a/aconfig/codegen/Android.bp +++ b/aconfig/codegen/Android.bp @@ -12,7 +12,6 @@ bootstrap_go_package { "soong", "soong-aconfig", "soong-android", - "soong-bazel", "soong-java", "soong-rust", ], diff --git a/aconfig/codegen/aconfig_declarations_group.go b/aconfig/codegen/aconfig_declarations_group.go index 1c91beeef..13daf470b 100644 --- a/aconfig/codegen/aconfig_declarations_group.go +++ b/aconfig/codegen/aconfig_declarations_group.go @@ -15,7 +15,6 @@ package codegen import ( - "fmt" "maps" "android/soong/android" @@ -40,11 +39,6 @@ type AconfigDeclarationsGroup struct { android.DefaultableModuleBase properties AconfigDeclarationsGroupProperties - - aconfigDeclarationNames []string - intermediateCacheOutputPaths android.Paths - javaSrcjars android.Paths - modeInfos map[string]android.ModeInfo } type AconfigDeclarationsGroupProperties struct { @@ -77,63 +71,45 @@ func (adg *AconfigDeclarationsGroup) DepsMutator(ctx android.BottomUpMutatorCont ctx.AddDependency(ctx.Module(), rustAconfigLibraryTag, adg.properties.Rust_aconfig_libraries...) } -func (adg *AconfigDeclarationsGroup) VisitDeps(ctx android.ModuleContext) { - adg.modeInfos = make(map[string]android.ModeInfo) +func (adg *AconfigDeclarationsGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { + modeInfos := make(map[string]android.ModeInfo) + var aconfigDeclarationNames []string + var intermediateCacheOutputPaths android.Paths + var javaSrcjars android.Paths ctx.VisitDirectDeps(func(dep android.Module) { tag := ctx.OtherModuleDependencyTag(dep) if provider, ok := android.OtherModuleProvider(ctx, dep, android.CodegenInfoProvider); ok { // aconfig declaration names and cache files are collected for all aconfig library dependencies - adg.aconfigDeclarationNames = append(adg.aconfigDeclarationNames, provider.AconfigDeclarations...) - adg.intermediateCacheOutputPaths = append(adg.intermediateCacheOutputPaths, provider.IntermediateCacheOutputPaths...) + aconfigDeclarationNames = append(aconfigDeclarationNames, provider.AconfigDeclarations...) + intermediateCacheOutputPaths = append(intermediateCacheOutputPaths, provider.IntermediateCacheOutputPaths...) switch tag { case aconfigDeclarationsGroupTag: // Will retrieve outputs from another language codegen modules when support is added - adg.javaSrcjars = append(adg.javaSrcjars, provider.Srcjars...) - maps.Copy(adg.modeInfos, provider.ModeInfos) + javaSrcjars = append(javaSrcjars, provider.Srcjars...) + maps.Copy(modeInfos, provider.ModeInfos) case javaAconfigLibraryTag: - adg.javaSrcjars = append(adg.javaSrcjars, provider.Srcjars...) - maps.Copy(adg.modeInfos, provider.ModeInfos) + javaSrcjars = append(javaSrcjars, provider.Srcjars...) + maps.Copy(modeInfos, provider.ModeInfos) case ccAconfigLibraryTag: - maps.Copy(adg.modeInfos, provider.ModeInfos) + maps.Copy(modeInfos, provider.ModeInfos) case rustAconfigLibraryTag: - maps.Copy(adg.modeInfos, provider.ModeInfos) + maps.Copy(modeInfos, provider.ModeInfos) } } }) -} -func (adg *AconfigDeclarationsGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { - adg.VisitDeps(ctx) - adg.aconfigDeclarationNames = android.FirstUniqueStrings(adg.aconfigDeclarationNames) - adg.intermediateCacheOutputPaths = android.FirstUniquePaths(adg.intermediateCacheOutputPaths) + aconfigDeclarationNames = android.FirstUniqueStrings(aconfigDeclarationNames) + intermediateCacheOutputPaths = android.FirstUniquePaths(intermediateCacheOutputPaths) android.SetProvider(ctx, android.CodegenInfoProvider, android.CodegenInfo{ - AconfigDeclarations: adg.aconfigDeclarationNames, - IntermediateCacheOutputPaths: adg.intermediateCacheOutputPaths, - Srcjars: adg.javaSrcjars, - ModeInfos: adg.modeInfos, + AconfigDeclarations: aconfigDeclarationNames, + IntermediateCacheOutputPaths: intermediateCacheOutputPaths, + Srcjars: javaSrcjars, + ModeInfos: modeInfos, }) -} - -var _ android.OutputFileProducer = (*AconfigDeclarationsGroup)(nil) - -func (adg *AconfigDeclarationsGroup) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return adg.intermediateCacheOutputPaths, nil - case ".srcjars": - return adg.javaSrcjars, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %s", tag) - } -} - -func (adg *AconfigDeclarationsGroup) Srcjars() android.Paths { - return adg.javaSrcjars -} -func (adg *AconfigDeclarationsGroup) AconfigDeclarations() []string { - return adg.aconfigDeclarationNames + ctx.SetOutputFiles(intermediateCacheOutputPaths, "") + ctx.SetOutputFiles(javaSrcjars, ".srcjars") } diff --git a/aconfig/codegen/java_aconfig_library.go b/aconfig/codegen/java_aconfig_library.go index 9f42e21de..673ac2afe 100644 --- a/aconfig/codegen/java_aconfig_library.go +++ b/aconfig/codegen/java_aconfig_library.go @@ -15,8 +15,6 @@ package codegen import ( - "fmt" - "android/soong/android" "android/soong/java" @@ -80,7 +78,7 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild // Get the values that came from the global RELEASE_ACONFIG_VALUE_SETS flag declarationsModules := ctx.GetDirectDepsWithTag(declarationsTag) if len(declarationsModules) != 1 { - panic(fmt.Errorf("Exactly one aconfig_declarations property required")) + panic("Exactly one aconfig_declarations property required") } declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], android.AconfigDeclarationsProviderKey) @@ -133,10 +131,6 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild return srcJarPath, declarations.IntermediateCacheOutputPath } -func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) AconfigDeclarations() *string { - return proptools.StringPtr(callbacks.properties.Aconfig_declarations) -} - func isModeSupported(mode string) bool { return android.InList(mode, aconfigSupportedModes) } diff --git a/aconfig/init.go b/aconfig/init.go index 46554676b..256b213cc 100644 --- a/aconfig/init.go +++ b/aconfig/init.go @@ -15,13 +15,16 @@ package aconfig import ( + "encoding/gob" + "android/soong/android" "github.com/google/blueprint" ) var ( - pctx = android.NewPackageContext("android/soong/aconfig") + pkgPath = "android/soong/aconfig" + pctx = android.NewPackageContext(pkgPath) // For aconfig_declarations: Generate cache file aconfigRule = pctx.AndroidStaticRule("aconfig", @@ -106,6 +109,9 @@ func init() { RegisterBuildComponents(android.InitRegistrationContext) pctx.HostBinToolVariable("aconfig", "aconfig") pctx.HostBinToolVariable("soong_zip", "soong_zip") + + gob.Register(android.AconfigDeclarationsProviderData{}) + gob.Register(android.ModuleOutPath{}) } func RegisterBuildComponents(ctx android.RegistrationContext) { diff --git a/aconfig/testing.go b/aconfig/testing.go index f6489ec3f..4ceb6b3e5 100644 --- a/aconfig/testing.go +++ b/aconfig/testing.go @@ -23,7 +23,25 @@ import ( var PrepareForTestWithAconfigBuildComponents = android.FixtureRegisterWithContext(RegisterBuildComponents) func runTest(t *testing.T, errorHandler android.FixtureErrorHandler, bp string) *android.TestResult { - return android.GroupFixturePreparers(PrepareForTestWithAconfigBuildComponents). + return PrepareForTest(t). ExtendWithErrorHandler(errorHandler). RunTestWithBp(t, bp) } + +func PrepareForTest(t *testing.T, preparers ...android.FixturePreparer) android.FixturePreparer { + preparers = append([]android.FixturePreparer{PrepareForTestWithAconfigBuildComponents}, preparers...) + return android.GroupFixturePreparers(preparers...) +} + +func addBuildFlagsForTest(buildFlags map[string]string) android.FixturePreparer { + return android.GroupFixturePreparers( + android.FixtureModifyProductVariables(func(vars android.FixtureProductVariables) { + if vars.BuildFlags == nil { + vars.BuildFlags = make(map[string]string) + } + for k, v := range buildFlags { + vars.BuildFlags[k] = v + } + }), + ) +} diff --git a/android/Android.bp b/android/Android.bp index 9ce8cdcd2..3c38148b6 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -38,7 +38,9 @@ bootstrap_go_package { "arch_list.go", "arch_module_context.go", "base_module_context.go", + "build_prop.go", "buildinfo_prop.go", + "compliance_metadata.go", "config.go", "test_config.go", "configurable_properties.go", @@ -83,6 +85,7 @@ bootstrap_go_package { "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", + "product_config.go", "proto.go", "provider.go", "raw_files.go", @@ -110,6 +113,7 @@ bootstrap_go_package { "androidmk_test.go", "apex_test.go", "arch_test.go", + "blueprint_e2e_test.go", "config_test.go", "configured_jars_test.go", "csuite_config_test.go", diff --git a/android/aconfig_providers.go b/android/aconfig_providers.go index ee9891df1..a47e80f03 100644 --- a/android/aconfig_providers.go +++ b/android/aconfig_providers.go @@ -43,6 +43,10 @@ type AconfigDeclarationsProviderData struct { var AconfigDeclarationsProviderKey = blueprint.NewProvider[AconfigDeclarationsProviderData]() +type AconfigReleaseDeclarationsProviderData map[string]AconfigDeclarationsProviderData + +var AconfigReleaseDeclarationsProviderKey = blueprint.NewProvider[AconfigReleaseDeclarationsProviderData]() + type ModeInfo struct { Container string Mode string @@ -112,6 +116,8 @@ func aconfigUpdateAndroidBuildActions(ctx ModuleContext) { if dep, ok := OtherModuleProvider(ctx, module, AconfigDeclarationsProviderKey); ok { mergedAconfigFiles[dep.Container] = append(mergedAconfigFiles[dep.Container], dep.IntermediateCacheOutputPath) } + // If we were generating on-device artifacts for other release configs, we would need to add code here to propagate + // those artifacts as well. See also b/298444886. if dep, ok := OtherModuleProvider(ctx, module, AconfigPropagatingProviderKey); ok { for container, v := range dep.AconfigFiles { mergedAconfigFiles[container] = append(mergedAconfigFiles[container], v...) diff --git a/android/androidmk.go b/android/androidmk.go index 66f42f97c..9699ce5b8 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -499,6 +499,7 @@ type fillInEntriesContext interface { Config() Config moduleProvider(module blueprint.Module, provider blueprint.AnyProviderKey) (any, bool) ModuleType(module blueprint.Module) string + OtherModulePropertyErrorf(module Module, property string, fmt string, args ...interface{}) } func (a *AndroidMkEntries) fillInEntries(ctx fillInEntriesContext, mod blueprint.Module) { @@ -514,7 +515,7 @@ func (a *AndroidMkEntries) fillInEntries(ctx fillInEntriesContext, mod blueprint if a.Include == "" { a.Include = "$(BUILD_PREBUILT)" } - a.Required = append(a.Required, amod.RequiredModuleNames()...) + a.Required = append(a.Required, amod.RequiredModuleNames(ctx)...) a.Host_required = append(a.Host_required, amod.HostRequiredModuleNames()...) a.Target_required = append(a.Target_required, amod.TargetRequiredModuleNames()...) diff --git a/android/apex.go b/android/apex.go index 2ac6ed08d..683e50162 100644 --- a/android/apex.go +++ b/android/apex.go @@ -84,6 +84,9 @@ type ApexInfo struct { // Returns the name of the test apexes that this module is included in. TestApexes []string + + // Returns the name of the overridden apex (com.android.foo) + BaseApexName string } // AllApexInfo holds the ApexInfo of all apexes that include this module. diff --git a/android/blueprint_e2e_test.go b/android/blueprint_e2e_test.go new file mode 100644 index 000000000..b27451218 --- /dev/null +++ b/android/blueprint_e2e_test.go @@ -0,0 +1,105 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "testing" +) + +var testCases []struct { + name string + fs MockFS + expectedError string +} = []struct { + name string + fs MockFS + expectedError string +}{ + { + name: "Can't reference variable before assignment", + fs: map[string][]byte{ + "Android.bp": []byte(` +x = foo +foo = "hello" +`), + }, + expectedError: "undefined variable foo", + }, + { + name: "Can't append to variable before assigned to", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo += "world" +foo = "hello" +`), + }, + expectedError: "modified non-existent variable \"foo\" with \\+=", + }, + { + name: "Can't reassign variable", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +foo = "world" +`), + }, + expectedError: "variable already set, previous assignment:", + }, + { + name: "Can't reassign variable in inherited scope", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +`), + "foo/Android.bp": []byte(` +foo = "world" +`), + }, + expectedError: "variable already set in inherited scope, previous assignment:", + }, + { + name: "Can't modify variable in inherited scope", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +`), + "foo/Android.bp": []byte(` +foo += "world" +`), + }, + expectedError: "modified non-local variable \"foo\" with \\+=", + }, + { + name: "Can't modify variable after referencing", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +x = foo +foo += "world" +`), + }, + expectedError: "modified variable \"foo\" with \\+= after referencing", + }, +} + +func TestBlueprintErrors(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fixtures := FixtureMergeMockFs(tc.fs) + fixtures = fixtures.ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(tc.expectedError)) + fixtures.RunTest(t) + }) + } +} diff --git a/android/build_prop.go b/android/build_prop.go new file mode 100644 index 000000000..45c17c35b --- /dev/null +++ b/android/build_prop.go @@ -0,0 +1,125 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "github.com/google/blueprint/proptools" +) + +func init() { + ctx := InitRegistrationContext + ctx.RegisterModuleType("build_prop", buildPropFactory) +} + +type buildPropProperties struct { + // Output file name. Defaults to "build.prop" + Stem *string + + // List of prop names to exclude. This affects not only common build properties but also + // properties in prop_files. + Block_list []string + + // Path to the input prop files. The contents of the files are directly + // emitted to the output + Prop_files []string `android:"path"` + + // Files to be appended at the end of build.prop. These files are appended after + // post_process_props without any further checking. + Footer_files []string `android:"path"` + + // Path to a JSON file containing product configs. + Product_config *string `android:"path"` +} + +type buildPropModule struct { + ModuleBase + + properties buildPropProperties + + outputFilePath OutputPath + installPath InstallPath +} + +func (p *buildPropModule) stem() string { + return proptools.StringDefault(p.properties.Stem, "build.prop") +} + +func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { + p.outputFilePath = PathForModuleOut(ctx, "build.prop").OutputPath + if !ctx.Config().KatiEnabled() { + WriteFileRule(ctx, p.outputFilePath, "# no build.prop if kati is disabled") + return + } + + partition := p.PartitionTag(ctx.DeviceConfig()) + if partition != "system" { + ctx.PropertyErrorf("partition", "unsupported partition %q: only \"system\" is supported", partition) + return + } + + rule := NewRuleBuilder(pctx, ctx) + + config := ctx.Config() + + cmd := rule.Command().BuiltTool("gen_build_prop") + + cmd.FlagWithInput("--build-hostname-file=", config.BuildHostnameFile(ctx)) + cmd.FlagWithInput("--build-number-file=", config.BuildNumberFile(ctx)) + // shouldn't depend on BuildFingerprintFile and BuildThumbprintFile to prevent from rebuilding + // on every incremental build. + cmd.FlagWithArg("--build-fingerprint-file=", config.BuildFingerprintFile(ctx).String()) + // Export build thumbprint only if the product has specified at least one oem fingerprint property + // b/17888863 + if shouldAddBuildThumbprint(config) { + // In the previous make implementation, a dependency was not added on the thumbprint file + cmd.FlagWithArg("--build-thumbprint-file=", config.BuildThumbprintFile(ctx).String()) + } + cmd.FlagWithArg("--build-username=", config.Getenv("BUILD_USERNAME")) + // shouldn't depend on BUILD_DATETIME_FILE to prevent from rebuilding on every incremental + // build. + cmd.FlagWithArg("--date-file=", ctx.Config().Getenv("BUILD_DATETIME_FILE")) + cmd.FlagWithInput("--platform-preview-sdk-fingerprint-file=", ApiFingerprintPath(ctx)) + cmd.FlagWithInput("--product-config=", PathForModuleSrc(ctx, proptools.String(p.properties.Product_config))) + cmd.FlagWithArg("--partition=", partition) + cmd.FlagWithOutput("--out=", p.outputFilePath) + + postProcessCmd := rule.Command().BuiltTool("post_process_props") + if ctx.DeviceConfig().BuildBrokenDupSysprop() { + postProcessCmd.Flag("--allow-dup") + } + postProcessCmd.FlagWithArg("--sdk-version ", config.PlatformSdkVersion().String()) + postProcessCmd.FlagWithInput("--kernel-version-file-for-uffd-gc ", PathForOutput(ctx, "dexpreopt/kernel_version_for_uffd_gc.txt")) + postProcessCmd.Text(p.outputFilePath.String()) + postProcessCmd.Flags(p.properties.Block_list) + + rule.Command().Text("echo").Text(proptools.NinjaAndShellEscape("# end of file")).FlagWithArg(">> ", p.outputFilePath.String()) + + rule.Build(ctx.ModuleName(), "generating build.prop") + + p.installPath = PathForModuleInstall(ctx) + ctx.InstallFile(p.installPath, p.stem(), p.outputFilePath) + + ctx.SetOutputFiles(Paths{p.outputFilePath}, "") +} + +// build_prop module generates {partition}/build.prop file. At first common build properties are +// printed based on Soong config variables. And then prop_files are printed as-is. Finally, +// post_process_props tool is run to check if the result build.prop is valid or not. +func buildPropFactory() Module { + module := &buildPropModule{} + module.AddProperties(&module.properties) + InitAndroidArchModule(module, DeviceSupported, MultilibCommon) + return module +} diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go index 083f3efcb..defbff0b0 100644 --- a/android/buildinfo_prop.go +++ b/android/buildinfo_prop.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "strings" "github.com/google/blueprint/proptools" ) @@ -29,6 +28,8 @@ func init() { type buildinfoPropProperties struct { // Whether this module is directly installable to one of the partitions. Default: true. Installable *bool + + Product_config *string `android:"path"` } type buildinfoPropModule struct { @@ -54,24 +55,6 @@ func (p *buildinfoPropModule) OutputFiles(tag string) (Paths, error) { return Paths{p.outputFilePath}, nil } -func getBuildVariant(config Config) string { - if config.Eng() { - return "eng" - } else if config.Debuggable() { - return "userdebug" - } else { - return "user" - } -} - -func getBuildFlavor(config Config) string { - buildFlavor := config.DeviceProduct() + "-" + getBuildVariant(config) - if InList("address", config.SanitizeDevice()) && !strings.Contains(buildFlavor, "_asan") { - buildFlavor += "_asan" - } - return buildFlavor -} - func shouldAddBuildThumbprint(config Config) bool { knownOemProperties := []string{ "ro.product.brand", @@ -101,63 +84,27 @@ func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { rule := NewRuleBuilder(pctx, ctx) config := ctx.Config() - buildVariant := getBuildVariant(config) - buildFlavor := getBuildFlavor(config) cmd := rule.Command().BuiltTool("buildinfo") - if config.BoardUseVbmetaDigestInFingerprint() { - cmd.Flag("--use-vbmeta-digest-in-fingerprint") - } - - cmd.FlagWithArg("--build-flavor=", buildFlavor) cmd.FlagWithInput("--build-hostname-file=", config.BuildHostnameFile(ctx)) - cmd.FlagWithArg("--build-id=", config.BuildId()) - cmd.FlagWithArg("--build-keys=", config.BuildKeys()) - // Note: depending on BuildNumberFile will cause the build.prop file to be rebuilt // every build, but that's intentional. cmd.FlagWithInput("--build-number-file=", config.BuildNumberFile(ctx)) + // Export build thumbprint only if the product has specified at least one oem fingerprint property + // b/17888863 if shouldAddBuildThumbprint(config) { // In the previous make implementation, a dependency was not added on the thumbprint file cmd.FlagWithArg("--build-thumbprint-file=", config.BuildThumbprintFile(ctx).String()) } - - cmd.FlagWithArg("--build-type=", config.BuildType()) cmd.FlagWithArg("--build-username=", config.Getenv("BUILD_USERNAME")) - cmd.FlagWithArg("--build-variant=", buildVariant) - cmd.FlagForEachArg("--cpu-abis=", config.DeviceAbi()) - // Technically we should also have a dependency on BUILD_DATETIME_FILE, // but it can be either an absolute or relative path, which is hard to turn into // a Path object. So just rely on the BuildNumberFile always changing to cause // us to rebuild. cmd.FlagWithArg("--date-file=", ctx.Config().Getenv("BUILD_DATETIME_FILE")) - - if len(config.ProductLocales()) > 0 { - cmd.FlagWithArg("--default-locale=", config.ProductLocales()[0]) - } - - cmd.FlagForEachArg("--default-wifi-channels=", config.ProductDefaultWifiChannels()) - cmd.FlagWithArg("--device=", config.DeviceName()) - if config.DisplayBuildNumber() { - cmd.Flag("--display-build-number") - } - - cmd.FlagWithArg("--platform-base-os=", config.PlatformBaseOS()) - cmd.FlagWithArg("--platform-display-version=", config.PlatformDisplayVersionName()) - cmd.FlagWithArg("--platform-min-supported-target-sdk-version=", config.PlatformMinSupportedTargetSdkVersion()) cmd.FlagWithInput("--platform-preview-sdk-fingerprint-file=", ApiFingerprintPath(ctx)) - cmd.FlagWithArg("--platform-preview-sdk-version=", config.PlatformPreviewSdkVersion()) - cmd.FlagWithArg("--platform-sdk-version=", config.PlatformSdkVersion().String()) - cmd.FlagWithArg("--platform-security-patch=", config.PlatformSecurityPatch()) - cmd.FlagWithArg("--platform-version=", config.PlatformVersionName()) - cmd.FlagWithArg("--platform-version-codename=", config.PlatformSdkCodename()) - cmd.FlagForEachArg("--platform-version-all-codenames=", config.PlatformVersionActiveCodenames()) - cmd.FlagWithArg("--platform-version-known-codenames=", config.PlatformVersionKnownCodenames()) - cmd.FlagWithArg("--platform-version-last-stable=", config.PlatformVersionLastStable()) - cmd.FlagWithArg("--product=", config.DeviceProduct()) - + cmd.FlagWithInput("--product-config=", PathForModuleSrc(ctx, proptools.String(p.properties.Product_config))) cmd.FlagWithOutput("--out=", p.outputFilePath) rule.Build(ctx.ModuleName(), "generating buildinfo props") diff --git a/android/compliance_metadata.go b/android/compliance_metadata.go new file mode 100644 index 000000000..6ea66541a --- /dev/null +++ b/android/compliance_metadata.go @@ -0,0 +1,314 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "bytes" + "encoding/csv" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/google/blueprint" +) + +var ( + // Constants of property names used in compliance metadata of modules + ComplianceMetadataProp = struct { + NAME string + PACKAGE string + MODULE_TYPE string + OS string + ARCH string + IS_PRIMARY_ARCH string + VARIANT string + IS_STATIC_LIB string + INSTALLED_FILES string + BUILT_FILES string + STATIC_DEPS string + STATIC_DEP_FILES string + WHOLE_STATIC_DEPS string + WHOLE_STATIC_DEP_FILES string + LICENSES string + + // module_type=package + PKG_DEFAULT_APPLICABLE_LICENSES string + + // module_type=license + LIC_LICENSE_KINDS string + LIC_LICENSE_TEXT string + LIC_PACKAGE_NAME string + + // module_type=license_kind + LK_CONDITIONS string + LK_URL string + }{ + "name", + "package", + "module_type", + "os", + "arch", + "is_primary_arch", + "variant", + "is_static_lib", + "installed_files", + "built_files", + "static_deps", + "static_dep_files", + "whole_static_deps", + "whole_static_dep_files", + "licenses", + + "pkg_default_applicable_licenses", + + "lic_license_kinds", + "lic_license_text", + "lic_package_name", + + "lk_conditions", + "lk_url", + } + + // A constant list of all property names in compliance metadata + // Order of properties here is the order of columns in the exported CSV file. + COMPLIANCE_METADATA_PROPS = []string{ + ComplianceMetadataProp.NAME, + ComplianceMetadataProp.PACKAGE, + ComplianceMetadataProp.MODULE_TYPE, + ComplianceMetadataProp.OS, + ComplianceMetadataProp.ARCH, + ComplianceMetadataProp.VARIANT, + ComplianceMetadataProp.IS_STATIC_LIB, + ComplianceMetadataProp.IS_PRIMARY_ARCH, + // Space separated installed files + ComplianceMetadataProp.INSTALLED_FILES, + // Space separated built files + ComplianceMetadataProp.BUILT_FILES, + // Space separated module names of static dependencies + ComplianceMetadataProp.STATIC_DEPS, + // Space separated file paths of static dependencies + ComplianceMetadataProp.STATIC_DEP_FILES, + // Space separated module names of whole static dependencies + ComplianceMetadataProp.WHOLE_STATIC_DEPS, + // Space separated file paths of whole static dependencies + ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, + ComplianceMetadataProp.LICENSES, + // module_type=package + ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES, + // module_type=license + ComplianceMetadataProp.LIC_LICENSE_KINDS, + ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths + ComplianceMetadataProp.LIC_PACKAGE_NAME, + // module_type=license_kind + ComplianceMetadataProp.LK_CONDITIONS, + ComplianceMetadataProp.LK_URL, + } +) + +// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license, +// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility +// methods to get/set properties' values. +type ComplianceMetadataInfo struct { + properties map[string]string +} + +func NewComplianceMetadataInfo() *ComplianceMetadataInfo { + return &ComplianceMetadataInfo{ + properties: map[string]string{}, + } +} + +func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) { + if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { + panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) + } + c.properties[propertyName] = value +} + +func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) { + c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " "))) +} + +func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string { + if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { + panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) + } + return c.properties[propertyName] +} + +func (c *ComplianceMetadataInfo) getAllValues() map[string]string { + return c.properties +} + +var ( + ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]() +) + +// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata +// for different module types without accessing their private fields but through android.Module interface +// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider. +func buildComplianceMetadataProvider(ctx ModuleContext, m *ModuleBase) { + complianceMetadataInfo := ctx.ComplianceMetadataInfo() + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType()) + + switch ctx.ModuleType() { + case "license": + licenseModule := m.module.(*licenseModule) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name)) + case "license_kind": + licenseKindModule := m.module.(*licenseKindModule) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url) + default: + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch())) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir()) + if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" { + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings()) + } + + var installed InstallPaths + installed = append(installed, m.module.FilesToInstall()...) + installed = append(installed, m.katiInstalls.InstallPaths()...) + installed = append(installed, m.katiSymlinks.InstallPaths()...) + installed = append(installed, m.katiInitRcInstalls.InstallPaths()...) + installed = append(installed, m.katiVintfInstalls.InstallPaths()...) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings())) + } + ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo) +} + +func init() { + RegisterComplianceMetadataSingleton(InitRegistrationContext) +} + +func RegisterComplianceMetadataSingleton(ctx RegistrationContext) { + ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory) +} + +var ( + // sqlite3 command line tool + sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3") + + // Command to import .csv files to sqlite3 database + importCsv = pctx.AndroidStaticRule("importCsv", + blueprint.RuleParams{ + Command: `rm -rf $out && ` + + `${sqlite3} $out ".import --csv $in modules" && ` + + `${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` + + `${sqlite3} $out ".import --csv ${make_modules} make_modules"`, + CommandDeps: []string{"${sqlite3}"}, + }, "make_metadata", "make_modules") +) + +func complianceMetadataSingletonFactory() Singleton { + return &complianceMetadataSingleton{} +} + +type complianceMetadataSingleton struct { +} + +func writerToCsv(csvWriter *csv.Writer, row []string) { + err := csvWriter.Write(row) + if err != nil { + panic(err) + } +} + +// Collect compliance metadata from all Soong modules, write to a CSV file and +// import compliance metadata from Make and Soong to a sqlite3 database. +func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) { + if !ctx.Config().HasDeviceProduct() { + return + } + var buffer bytes.Buffer + csvWriter := csv.NewWriter(&buffer) + + // Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file. + columnNames := []string{"id"} + columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...) + writerToCsv(csvWriter, columnNames) + + rowId := -1 + ctx.VisitAllModules(func(module Module) { + if !module.Enabled(ctx) { + return + } + moduleType := ctx.ModuleType(module) + if moduleType == "package" { + metadataMap := map[string]string{ + ComplianceMetadataProp.NAME: ctx.ModuleName(module), + ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module), + ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "), + } + rowId = rowId + 1 + metadata := []string{strconv.Itoa(rowId)} + for _, propertyName := range COMPLIANCE_METADATA_PROPS { + metadata = append(metadata, metadataMap[propertyName]) + } + writerToCsv(csvWriter, metadata) + return + } + if provider, ok := ctx.moduleProvider(module, ComplianceMetadataProvider); ok { + metadataInfo := provider.(*ComplianceMetadataInfo) + rowId = rowId + 1 + metadata := []string{strconv.Itoa(rowId)} + for _, propertyName := range COMPLIANCE_METADATA_PROPS { + metadata = append(metadata, metadataInfo.getStringValue(propertyName)) + } + writerToCsv(csvWriter, metadata) + return + } + }) + csvWriter.Flush() + + deviceProduct := ctx.Config().DeviceProduct() + modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv") + WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String()) + + // Metadata generated in Make + makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv") + makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv") + + // Import metadata from Make and Soong to sqlite3 database + complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db") + ctx.Build(pctx, BuildParams{ + Rule: importCsv, + Input: modulesCsv, + Implicits: []Path{ + makeMetadataCsv, + makeModulesCsv, + }, + Output: complianceMetadataDb, + Args: map[string]string{ + "make_metadata": makeMetadataCsv.String(), + "make_modules": makeModulesCsv.String(), + }, + }) + + // Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database. + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Inputs: []Path{complianceMetadataDb}, + Output: PathForPhony(ctx, "compliance-metadata.db"), + }) + +} diff --git a/android/config.go b/android/config.go index a18cb8be8..cda01f090 100644 --- a/android/config.go +++ b/android/config.go @@ -22,7 +22,6 @@ import ( "fmt" "os" "path/filepath" - "reflect" "runtime" "strconv" "strings" @@ -37,9 +36,7 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android/soongconfig" - "android/soong/bazel" "android/soong/remoteexec" - "android/soong/starlark_fmt" ) // Bool re-exports proptools.Bool for the android package. @@ -201,6 +198,33 @@ func (c Config) ReleaseAconfigValueSets() []string { return c.config.productVariables.ReleaseAconfigValueSets } +func (c Config) ReleaseAconfigExtraReleaseConfigs() []string { + result := []string{} + if val, ok := c.config.productVariables.BuildFlags["RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS"]; ok { + if len(val) > 0 { + // Remove any duplicates from the list. + found := make(map[string]bool) + for _, k := range strings.Split(val, " ") { + if !found[k] { + found[k] = true + result = append(result, k) + } + } + } + } + return result +} + +func (c Config) ReleaseAconfigExtraReleaseConfigsValueSets() map[string][]string { + result := make(map[string][]string) + for _, rcName := range c.ReleaseAconfigExtraReleaseConfigs() { + if value, ok := c.config.productVariables.BuildFlags["RELEASE_ACONFIG_VALUE_SETS_"+rcName]; ok { + result[rcName] = strings.Split(value, " ") + } + } + return result +} + // The flag default permission value passed to aconfig // derived from RELEASE_ACONFIG_FLAG_DEFAULT_PERMISSION func (c Config) ReleaseAconfigFlagDefaultPermission() string { @@ -413,7 +437,7 @@ func loadFromConfigFile(configurable *ProductVariables, filename string) error { proptools.StringPtr(String(configurable.Platform_sdk_codename)) } - return saveToBazelConfigFile(configurable, filepath.Dir(filename)) + return nil } // atomically writes the config file in case two copies of soong_build are running simultaneously @@ -447,81 +471,6 @@ func saveToConfigFile(config *ProductVariables, filename string) error { return nil } -type productVariableStarlarkRepresentation struct { - soongType string - selectable bool - archVariant bool -} - -func saveToBazelConfigFile(config *ProductVariables, outDir string) error { - dir := filepath.Join(outDir, bazel.SoongInjectionDirName, "product_config") - err := createDirIfNonexistent(dir, os.ModePerm) - if err != nil { - return fmt.Errorf("Could not create dir %s: %s", dir, err) - } - - allProductVariablesType := reflect.TypeOf((*ProductVariables)(nil)).Elem() - productVariablesInfo := make(map[string]productVariableStarlarkRepresentation) - p := variableProperties{} - t := reflect.TypeOf(p.Product_variables) - for i := 0; i < t.NumField(); i++ { - f := t.Field(i) - archVariant := proptools.HasTag(f, "android", "arch_variant") - if mainProductVariablesStructField, ok := allProductVariablesType.FieldByName(f.Name); ok { - productVariablesInfo[f.Name] = productVariableStarlarkRepresentation{ - soongType: stringRepresentationOfSimpleType(mainProductVariablesStructField.Type), - selectable: true, - archVariant: archVariant, - } - } else { - panic("Unknown variable " + f.Name) - } - } - - err = pathtools.WriteFileIfChanged(filepath.Join(dir, "product_variable_constants.bzl"), []byte(fmt.Sprintf(` -# product_var_constant_info is a map of product variables to information about them. The fields are: -# - soongType: The type of the product variable as it appears in soong's ProductVariables struct. -# examples are string, bool, int, *bool, *string, []string, etc. This may be an overly -# conservative estimation of the type, for example a *bool could oftentimes just be a -# bool that defaults to false. -# - selectable: if this product variable can be selected on in Android.bp/build files. This means -# it's listed in the "variableProperties" soong struct. Currently all variables in -# this list are selectable because we only need the selectable ones at the moment, -# but the list may be expanded later. -# - archVariant: If the variable is tagged as arch variant in the "variableProperties" struct. -product_var_constant_info = %s -product_var_constraints = [k for k, v in product_var_constant_info.items() if v.selectable] -arch_variant_product_var_constraints = [k for k, v in product_var_constant_info.items() if v.selectable and v.archVariant] -`, starlark_fmt.PrintAny(productVariablesInfo, 0))), 0644) - if err != nil { - return fmt.Errorf("Could not write .bzl config file %s", err) - } - err = pathtools.WriteFileIfChanged(filepath.Join(dir, "BUILD"), - []byte(bazel.GeneratedBazelFileWarning), 0644) - if err != nil { - return fmt.Errorf("Could not write BUILD config file %s", err) - } - - return nil -} - -func stringRepresentationOfSimpleType(ty reflect.Type) string { - switch ty.Kind() { - case reflect.String: - return "string" - case reflect.Bool: - return "bool" - case reflect.Int: - return "int" - case reflect.Slice: - return "[]" + stringRepresentationOfSimpleType(ty.Elem()) - case reflect.Pointer: - return "*" + stringRepresentationOfSimpleType(ty.Elem()) - default: - panic("unimplemented type: " + ty.Kind().String()) - } -} - // NullConfig returns a mostly empty Config for use by standalone tools like dexpreopt_gen that // use the android package. func NullConfig(outDir, soongOutDir string) Config { @@ -857,6 +806,17 @@ func (c *config) DisplayBuildNumber() bool { return Bool(c.productVariables.DisplayBuildNumber) } +// BuildFingerprintFile returns the path to a text file containing metadata +// representing the current build's fingerprint. +// +// Rules that want to reference the build fingerprint should read from this file +// without depending on it. They will run whenever their other dependencies +// require them to run and get the current build fingerprint. This ensures they +// don't rebuild on every incremental build when the build number changes. +func (c *config) BuildFingerprintFile(ctx PathContext) Path { + return PathForArbitraryOutput(ctx, "target", "product", c.DeviceName(), String(c.productVariables.BuildFingerprintFile)) +} + // BuildNumberFile returns the path to a text file containing metadata // representing the current build's number. // @@ -1392,10 +1352,6 @@ func (c *config) PrebuiltHiddenApiDir(_ PathContext) string { return String(c.productVariables.PrebuiltHiddenApiDir) } -func (c *config) IsVndkDeprecated() bool { - return !Bool(c.productVariables.KeepVndk) -} - func (c *config) VendorApiLevel() string { return String(c.productVariables.VendorApiLevel) } @@ -1960,6 +1916,10 @@ func (c *deviceConfig) BuildBrokenDontCheckSystemSdk() bool { return c.config.productVariables.BuildBrokenDontCheckSystemSdk } +func (c *deviceConfig) BuildBrokenDupSysprop() bool { + return c.config.productVariables.BuildBrokenDupSysprop +} + func (c *config) BuildWarningBadOptionalUsesLibsAllowlist() []string { return c.productVariables.BuildWarningBadOptionalUsesLibsAllowlist } diff --git a/android/config_test.go b/android/config_test.go index 7d327a27e..ca7c7f8b4 100644 --- a/android/config_test.go +++ b/android/config_test.go @@ -125,6 +125,43 @@ func assertStringEquals(t *testing.T, expected, actual string) { } } +func TestReleaseAconfigExtraReleaseConfigs(t *testing.T) { + testCases := []struct { + name string + flag string + expected []string + }{ + { + name: "empty", + flag: "", + expected: []string{}, + }, + { + name: "specified", + flag: "bar foo", + expected: []string{"bar", "foo"}, + }, + { + name: "duplicates", + flag: "foo bar foo", + expected: []string{"foo", "bar"}, + }, + } + + for _, tc := range testCases { + fixture := GroupFixturePreparers( + FixtureModifyProductVariables(func(vars FixtureProductVariables) { + if vars.BuildFlags == nil { + vars.BuildFlags = make(map[string]string) + } + vars.BuildFlags["RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS"] = tc.flag + }), + ) + actual := fixture.RunTest(t).Config.ReleaseAconfigExtraReleaseConfigs() + AssertArrayString(t, tc.name, tc.expected, actual) + } +} + func TestConfiguredJarList(t *testing.T) { list1 := CreateTestConfiguredJarList([]string{"apex1:jarA"}) diff --git a/android/image.go b/android/image.go index bc6b8cd86..c278dcdf9 100644 --- a/android/image.go +++ b/android/image.go @@ -19,6 +19,12 @@ type ImageInterface interface { // ImageMutatorBegin is called before any other method in the ImageInterface. ImageMutatorBegin(ctx BaseModuleContext) + // VendorVariantNeeded should return true if the module needs a vendor variant (installed on the vendor image). + VendorVariantNeeded(ctx BaseModuleContext) bool + + // ProductVariantNeeded should return true if the module needs a product variant (unstalled on the product image). + ProductVariantNeeded(ctx BaseModuleContext) bool + // CoreVariantNeeded should return true if the module needs a core variant (installed on the system image). CoreVariantNeeded(ctx BaseModuleContext) bool @@ -44,12 +50,19 @@ type ImageInterface interface { ExtraImageVariations(ctx BaseModuleContext) []string // SetImageVariation is called for each newly created image variant. The receiver is the original - // module, "variation" is the name of the newly created variant and "module" is the newly created - // variant itself. - SetImageVariation(ctx BaseModuleContext, variation string, module Module) + // module, "variation" is the name of the newly created variant. "variation" is set on the receiver. + SetImageVariation(ctx BaseModuleContext, variation string) } const ( + // VendorVariation is the variant name used for /vendor code that does not + // compile against the VNDK. + VendorVariation string = "vendor" + + // ProductVariation is the variant name used for /product code that does not + // compile against the VNDK. + ProductVariation string = "product" + // CoreVariation is the variant used for framework-private libraries, or // SDK libraries. (which framework-private libraries can use), which // will be installed to the system image. @@ -95,6 +108,12 @@ func imageMutator(ctx BottomUpMutatorContext) { if m.RecoveryVariantNeeded(ctx) { variations = append(variations, RecoveryVariation) } + if m.VendorVariantNeeded(ctx) { + variations = append(variations, VendorVariation) + } + if m.ProductVariantNeeded(ctx) { + variations = append(variations, ProductVariation) + } extraVariations := m.ExtraImageVariations(ctx) variations = append(variations, extraVariations...) @@ -106,7 +125,7 @@ func imageMutator(ctx BottomUpMutatorContext) { mod := ctx.CreateVariations(variations...) for i, v := range variations { mod[i].base().setImageVariation(v) - m.SetImageVariation(ctx, v, mod[i]) + mod[i].(ImageInterface).SetImageVariation(ctx, v) } } } diff --git a/android/module.go b/android/module.go index dc585d295..66f6e1bcd 100644 --- a/android/module.go +++ b/android/module.go @@ -15,9 +15,6 @@ package android import ( - "crypto/md5" - "encoding/hex" - "encoding/json" "fmt" "net/url" "path/filepath" @@ -26,8 +23,6 @@ import ( "sort" "strings" - "android/soong/bazel" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -113,7 +108,7 @@ type Module interface { // Get information about the properties that can contain visibility rules. visibilityProperties() []visibilityProperty - RequiredModuleNames() []string + RequiredModuleNames(ctx ConfigAndErrorContext) []string HostRequiredModuleNames() []string TargetRequiredModuleNames() []string @@ -249,31 +244,6 @@ func SortedUniqueNamedPaths(l NamedPaths) NamedPaths { return l[:k+1] } -// soongConfigTrace holds all references to VendorVars. Uses []string for blueprint:"mutated" -type soongConfigTrace struct { - Bools []string `json:",omitempty"` - Strings []string `json:",omitempty"` - IsSets []string `json:",omitempty"` -} - -func (c *soongConfigTrace) isEmpty() bool { - return len(c.Bools) == 0 && len(c.Strings) == 0 && len(c.IsSets) == 0 -} - -// Returns hash of serialized trace records (empty string if there's no trace recorded) -func (c *soongConfigTrace) hash() string { - // Use MD5 for speed. We don't care collision or preimage attack - if c.isEmpty() { - return "" - } - j, err := json.Marshal(c) - if err != nil { - panic(fmt.Errorf("json marshal of %#v failed: %#v", *c, err)) - } - hash := md5.Sum(j) - return hex.EncodeToString(hash[:]) -} - type nameProperties struct { // The name of the module. Must be unique across all modules. Name *string @@ -422,7 +392,7 @@ type commonProperties struct { Vintf_fragments []string `android:"path"` // names of other modules to install if this module is installed - Required []string `android:"arch_variant"` + Required proptools.Configurable[[]string] `android:"arch_variant"` // names of other modules to install on host if this module is installed Host_required []string `android:"arch_variant"` @@ -525,14 +495,6 @@ type commonProperties struct { // constants in image.go, but can also be set to a custom value by individual module types. ImageVariation string `blueprint:"mutated"` - // SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed - // and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental - // builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable), - // and there are variables other than soong_config, which isn't captured by soong config - // trace, but influence modules among products. - SoongConfigTrace soongConfigTrace `blueprint:"mutated"` - SoongConfigTraceHash string `blueprint:"mutated"` - // The team (defined by the owner/vendor) who owns the property. Team *string `android:"path"` } @@ -849,9 +811,6 @@ type ModuleBase struct { // archPropRoot that is filled with arch specific values by the arch mutator. archProperties [][]interface{} - // Properties specific to the Blueprint to BUILD migration. - bazelTargetModuleProperties bazel.BazelTargetModuleProperties - // Information about all the properties on the module that contains visibility rules that need // checking. visibilityPropertyInfo []visibilityProperty @@ -919,6 +878,10 @@ type ModuleBase struct { // outputFiles stores the output of a module by tag and is used to set // the OutputFilesProvider in GenerateBuildActions outputFiles OutputFilesInfo + + // complianceMetadataInfo is for different module types to dump metadata. + // See android.ModuleContext interface. + complianceMetadataInfo *ComplianceMetadataInfo } func (m *ModuleBase) AddJSONData(d *map[string]interface{}) { @@ -1101,7 +1064,7 @@ func addRequiredDeps(ctx BottomUpMutatorContext) { hostTargets = append(hostTargets, ctx.Config().BuildOSCommonTarget) if ctx.Device() { - for _, depName := range ctx.Module().RequiredModuleNames() { + for _, depName := range ctx.Module().RequiredModuleNames(ctx) { for _, target := range deviceTargets { addDep(target, depName) } @@ -1114,7 +1077,7 @@ func addRequiredDeps(ctx BottomUpMutatorContext) { } if ctx.Host() { - for _, depName := range ctx.Module().RequiredModuleNames() { + for _, depName := range ctx.Module().RequiredModuleNames(ctx) { for _, target := range hostTargets { // When a host module requires another host module, don't make a // dependency if they have different OSes (i.e. hostcross). @@ -1235,17 +1198,28 @@ func (m *ModuleBase) GenerateTaggedDistFiles(ctx BaseModuleContext) TaggedDistFi // the special tag name which represents that. tag := proptools.StringDefault(dist.Tag, DefaultDistTag) + distFileForTagFromProvider, err := outputFilesForModuleFromProvider(ctx, m.module, tag) + if err != OutputFilesProviderNotSet { + if err != nil && tag != DefaultDistTag { + ctx.PropertyErrorf("dist.tag", "%s", err.Error()) + } else { + distFiles = distFiles.addPathsForTag(tag, distFileForTagFromProvider...) + continue + } + } + + // if the tagged dist file cannot be obtained from OutputFilesProvider, + // fall back to use OutputFileProducer + // TODO: remove this part after OutputFilesProvider fully replaces OutputFileProducer if outputFileProducer, ok := m.module.(OutputFileProducer); ok { // Call the OutputFiles(tag) method to get the paths associated with the tag. distFilesForTag, err := outputFileProducer.OutputFiles(tag) - // If the tag was not supported and is not DefaultDistTag then it is an error. // Failing to find paths for DefaultDistTag is not an error. It just means // that the module type requires the legacy behavior. if err != nil && tag != DefaultDistTag { ctx.PropertyErrorf("dist.tag", "%s", err.Error()) } - distFiles = distFiles.addPathsForTag(tag, distFilesForTag...) } else if tag != DefaultDistTag { // If the tag was specified then it is an error if the module does not @@ -1619,8 +1593,8 @@ func (m *ModuleBase) InRecovery() bool { return m.base().commonProperties.ImageVariation == RecoveryVariation } -func (m *ModuleBase) RequiredModuleNames() []string { - return m.base().commonProperties.Required +func (m *ModuleBase) RequiredModuleNames(ctx ConfigAndErrorContext) []string { + return m.base().commonProperties.Required.GetOrDefault(m.ConfigurableEvaluator(ctx), nil) } func (m *ModuleBase) HostRequiredModuleNames() []string { @@ -1913,9 +1887,54 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) return } - m.module.GenerateAndroidBuildActions(ctx) - if ctx.Failed() { - return + incrementalAnalysis := false + incrementalEnabled := false + var cacheKey *blueprint.BuildActionCacheKey = nil + var incrementalModule *blueprint.Incremental = nil + if ctx.bp.GetIncrementalEnabled() { + if im, ok := m.module.(blueprint.Incremental); ok { + incrementalModule = &im + incrementalEnabled = im.IncrementalSupported() + incrementalAnalysis = ctx.bp.GetIncrementalAnalysis() && incrementalEnabled + } + } + if incrementalEnabled { + hash, err := proptools.CalculateHash(m.GetProperties()) + if err != nil { + ctx.ModuleErrorf("failed to calculate properties hash: %s", err) + return + } + cacheInput := new(blueprint.BuildActionCacheInput) + cacheInput.PropertiesHash = hash + ctx.VisitDirectDeps(func(module Module) { + cacheInput.ProvidersHash = + append(cacheInput.ProvidersHash, ctx.bp.OtherModuleProviderInitialValueHashes(module)) + }) + hash, err = proptools.CalculateHash(&cacheInput) + if err != nil { + ctx.ModuleErrorf("failed to calculate cache input hash: %s", err) + return + } + cacheKey = &blueprint.BuildActionCacheKey{ + Id: ctx.bp.ModuleId(), + InputHash: hash, + } + } + + restored := false + if incrementalAnalysis && cacheKey != nil { + restored = ctx.bp.RestoreBuildActions(cacheKey, incrementalModule) + } + + if !restored { + m.module.GenerateAndroidBuildActions(ctx) + if ctx.Failed() { + return + } + } + + if incrementalEnabled && cacheKey != nil { + ctx.bp.CacheBuildActions(cacheKey, incrementalModule) } // Create the set of tagged dist files after calling GenerateAndroidBuildActions @@ -1992,7 +2011,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) TargetDependencies: targetRequired, HostDependencies: hostRequired, Data: data, - Required: m.RequiredModuleNames(), + Required: m.RequiredModuleNames(ctx), } SetProvider(ctx, ModuleInfoJSONProvider, m.moduleInfoJSON) } @@ -2004,6 +2023,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) if m.outputFiles.DefaultOutputFiles != nil || m.outputFiles.TaggedOutputFiles != nil { SetProvider(ctx, OutputFilesProvider, m.outputFiles) } + + buildComplianceMetadataProvider(ctx, m) } func SetJarJarPrefixHandler(handler func(ModuleContext)) { @@ -2208,7 +2229,20 @@ func (e configurationEvalutor) EvaluateConfiguration(condition proptools.Configu variable := condition.Arg(1) if n, ok := ctx.Config().productVariables.VendorVars[namespace]; ok { if v, ok := n[variable]; ok { - return proptools.ConfigurableValueString(v) + ty := "" + if namespaces, ok := ctx.Config().productVariables.VendorVarTypes[namespace]; ok { + ty = namespaces[variable] + } + switch ty { + case "": + // strings are the default, we don't bother writing them to the soong variables json file + return proptools.ConfigurableValueString(v) + case "bool": + return proptools.ConfigurableValueBool(v == "true") + default: + panic("unhandled soong config variable type: " + ty) + } + } } return proptools.ConfigurableValueUndefined() @@ -2454,7 +2488,7 @@ func OutputFileForModule(ctx PathContext, module blueprint.Module, tag string) P func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { outputFilesFromProvider, err := outputFilesForModuleFromProvider(ctx, module, tag) - if outputFilesFromProvider != nil || err != nil { + if outputFilesFromProvider != nil || err != OutputFilesProviderNotSet { return outputFilesFromProvider, err } if outputFileProducer, ok := module.(OutputFileProducer); ok { @@ -2479,34 +2513,45 @@ func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) // *inter-module-communication*. // If mctx module is the same as the param module the output files are obtained // from outputFiles property of module base, to avoid both setting and -// reading OutputFilesProvider before GenerateBuildActions is finished. Also -// only empty-string-tag is supported in this case. +// reading OutputFilesProvider before GenerateBuildActions is finished. // If a module doesn't have the OutputFilesProvider, nil is returned. func outputFilesForModuleFromProvider(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { - // TODO: support OutputFilesProvider for singletons - mctx, ok := ctx.(ModuleContext) - if !ok { - return nil, nil - } - if mctx.Module() != module { - if outputFilesProvider, ok := OtherModuleProvider(mctx, module, OutputFilesProvider); ok { - if tag == "" { - return outputFilesProvider.DefaultOutputFiles, nil - } else if taggedOutputFiles, hasTag := outputFilesProvider.TaggedOutputFiles[tag]; hasTag { - return taggedOutputFiles, nil - } else { - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } + var outputFiles OutputFilesInfo + fromProperty := false + + if mctx, isMctx := ctx.(ModuleContext); isMctx { + if mctx.Module() != module { + outputFiles, _ = OtherModuleProvider(mctx, module, OutputFilesProvider) + } else { + outputFiles = mctx.Module().base().outputFiles + fromProperty = true } + } else if cta, isCta := ctx.(*singletonContextAdaptor); isCta { + providerData, _ := cta.moduleProvider(module, OutputFilesProvider) + outputFiles, _ = providerData.(OutputFilesInfo) + } + // TODO: Add a check for skipped context + + if outputFiles.isEmpty() { + // TODO: Add a check for param module not having OutputFilesProvider set + return nil, OutputFilesProviderNotSet + } + + if tag == "" { + return outputFiles.DefaultOutputFiles, nil + } else if taggedOutputFiles, hasTag := outputFiles.TaggedOutputFiles[tag]; hasTag { + return taggedOutputFiles, nil } else { - if tag == "" { - return mctx.Module().base().outputFiles.DefaultOutputFiles, nil - } else { + if fromProperty { return nil, fmt.Errorf("unsupported tag %q for module getting its own output files", tag) + } else { + return nil, fmt.Errorf("unsupported module reference tag %q", tag) } } - // TODO: Add a check for param module not having OutputFilesProvider set - return nil, nil +} + +func (o OutputFilesInfo) isEmpty() bool { + return o.DefaultOutputFiles == nil && o.TaggedOutputFiles == nil } type OutputFilesInfo struct { @@ -2519,6 +2564,9 @@ type OutputFilesInfo struct { var OutputFilesProvider = blueprint.NewProvider[OutputFilesInfo]() +// This is used to mark the case where OutputFilesProvider is not set on some modules. +var OutputFilesProviderNotSet = fmt.Errorf("No output files from provider") + // Modules can implement HostToolProvider and return a valid OptionalPath from HostToolPath() to // specify that they can be used as a tool by a genrule module. type HostToolProvider interface { @@ -2530,8 +2578,6 @@ type HostToolProvider interface { func init() { RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) - RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc) - FinalDepsMutators(registerSoongConfigTraceMutator) } func BuildTargetSingleton() Singleton { @@ -2693,54 +2739,3 @@ func CheckBlueprintSyntax(ctx BaseModuleContext, filename string, contents strin bpctx := ctx.blueprintBaseModuleContext() return blueprint.CheckBlueprintSyntax(bpctx.ModuleFactories(), filename, contents) } - -func registerSoongConfigTraceMutator(ctx RegisterMutatorsContext) { - ctx.BottomUp("soongconfigtrace", soongConfigTraceMutator).Parallel() -} - -// soongConfigTraceMutator accumulates recorded soong_config trace from children. Also it normalizes -// SoongConfigTrace to make it consistent. -func soongConfigTraceMutator(ctx BottomUpMutatorContext) { - trace := &ctx.Module().base().commonProperties.SoongConfigTrace - ctx.VisitDirectDeps(func(m Module) { - childTrace := &m.base().commonProperties.SoongConfigTrace - trace.Bools = append(trace.Bools, childTrace.Bools...) - trace.Strings = append(trace.Strings, childTrace.Strings...) - trace.IsSets = append(trace.IsSets, childTrace.IsSets...) - }) - trace.Bools = SortedUniqueStrings(trace.Bools) - trace.Strings = SortedUniqueStrings(trace.Strings) - trace.IsSets = SortedUniqueStrings(trace.IsSets) - - ctx.Module().base().commonProperties.SoongConfigTraceHash = trace.hash() -} - -// soongConfigTraceSingleton writes a map from each module's config hash value to trace data. -func soongConfigTraceSingletonFunc() Singleton { - return &soongConfigTraceSingleton{} -} - -type soongConfigTraceSingleton struct { -} - -func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) { - outFile := PathForOutput(ctx, "soong_config_trace.json") - - traces := make(map[string]*soongConfigTrace) - ctx.VisitAllModules(func(module Module) { - trace := &module.base().commonProperties.SoongConfigTrace - if !trace.isEmpty() { - hash := module.base().commonProperties.SoongConfigTraceHash - traces[hash] = trace - } - }) - - j, err := json.Marshal(traces) - if err != nil { - ctx.Errorf("json marshal to %q failed: %#v", outFile, err) - return - } - - WriteFileRule(ctx, outFile, string(j)) - ctx.Phony("soong_config_trace", outFile) -} diff --git a/android/module_context.go b/android/module_context.go index 591e270f0..253bebd3b 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -183,12 +183,11 @@ type ModuleContext interface { InstallInVendor() bool InstallForceOS() (*OsType, *ArchType) - RequiredModuleNames() []string + RequiredModuleNames(ctx ConfigAndErrorContext) []string HostRequiredModuleNames() []string TargetRequiredModuleNames() []string ModuleSubDir() string - SoongConfigTraceHash() string Variable(pctx PackageContext, name, value string) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule @@ -216,6 +215,11 @@ type ModuleContext interface { // SetOutputFiles stores the outputFiles to outputFiles property, which is used // to set the OutputFilesProvider later. SetOutputFiles(outputFiles Paths, tag string) + + // ComplianceMetadataInfo returns a ComplianceMetadataInfo instance for different module types to dump metadata, + // which usually happens in GenerateAndroidBuildActions() of a module type. + // See android.ModuleBase.complianceMetadataInfo + ComplianceMetadataInfo() *ComplianceMetadataInfo } type moduleContext struct { @@ -377,10 +381,6 @@ func (m *moduleContext) ModuleSubDir() string { return m.bp.ModuleSubDir() } -func (m *moduleContext) SoongConfigTraceHash() string { - return m.module.base().commonProperties.SoongConfigTraceHash -} - func (m *moduleContext) InstallInData() bool { return m.module.InstallInData() } @@ -729,6 +729,15 @@ func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) { } } +func (m *moduleContext) ComplianceMetadataInfo() *ComplianceMetadataInfo { + if complianceMetadataInfo := m.module.base().complianceMetadataInfo; complianceMetadataInfo != nil { + return complianceMetadataInfo + } + complianceMetadataInfo := NewComplianceMetadataInfo() + m.module.base().complianceMetadataInfo = complianceMetadataInfo + return complianceMetadataInfo +} + // Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must // be tagged with `android:"path" to support automatic source module dependency resolution. // @@ -755,8 +764,8 @@ func (m *moduleContext) ExpandOptionalSource(srcFile *string, _ string) Optional return OptionalPath{} } -func (m *moduleContext) RequiredModuleNames() []string { - return m.module.RequiredModuleNames() +func (m *moduleContext) RequiredModuleNames(ctx ConfigAndErrorContext) []string { + return m.module.RequiredModuleNames(ctx) } func (m *moduleContext) HostRequiredModuleNames() []string { diff --git a/android/paths.go b/android/paths.go index edc07000c..03772ebcb 100644 --- a/android/paths.go +++ b/android/paths.go @@ -15,6 +15,9 @@ package android import ( + "bytes" + "encoding/gob" + "errors" "fmt" "os" "path/filepath" @@ -1068,6 +1071,28 @@ type basePath struct { rel string } +func (p basePath) GobEncode() ([]byte, error) { + w := new(bytes.Buffer) + encoder := gob.NewEncoder(w) + err := errors.Join(encoder.Encode(p.path), encoder.Encode(p.rel)) + if err != nil { + return nil, err + } + + return w.Bytes(), nil +} + +func (p *basePath) GobDecode(data []byte) error { + r := bytes.NewBuffer(data) + decoder := gob.NewDecoder(r) + err := errors.Join(decoder.Decode(&p.path), decoder.Decode(&p.rel)) + if err != nil { + return err + } + + return nil +} + func (p basePath) Ext() string { return filepath.Ext(p.path) } @@ -1306,6 +1331,28 @@ type OutputPath struct { fullPath string } +func (p OutputPath) GobEncode() ([]byte, error) { + w := new(bytes.Buffer) + encoder := gob.NewEncoder(w) + err := errors.Join(encoder.Encode(p.basePath), encoder.Encode(p.soongOutDir), encoder.Encode(p.fullPath)) + if err != nil { + return nil, err + } + + return w.Bytes(), nil +} + +func (p *OutputPath) GobDecode(data []byte) error { + r := bytes.NewBuffer(data) + decoder := gob.NewDecoder(r) + err := errors.Join(decoder.Decode(&p.basePath), decoder.Decode(&p.soongOutDir), decoder.Decode(&p.fullPath)) + if err != nil { + return err + } + + return nil +} + func (p OutputPath) withRel(rel string) OutputPath { p.basePath = p.basePath.withRel(rel) p.fullPath = filepath.Join(p.fullPath, rel) @@ -1557,11 +1604,10 @@ type ModuleOutPathContext interface { ModuleName() string ModuleDir() string ModuleSubDir() string - SoongConfigTraceHash() string } func pathForModuleOut(ctx ModuleOutPathContext) OutputPath { - return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), ctx.SoongConfigTraceHash()) + return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir()) } // PathForModuleOut returns a Path representing the paths... under the module's diff --git a/android/product_config.go b/android/product_config.go new file mode 100644 index 000000000..20b29a7c4 --- /dev/null +++ b/android/product_config.go @@ -0,0 +1,58 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import "github.com/google/blueprint/proptools" + +func init() { + ctx := InitRegistrationContext + ctx.RegisterModuleType("product_config", productConfigFactory) +} + +type productConfigModule struct { + ModuleBase +} + +func (p *productConfigModule) GenerateAndroidBuildActions(ctx ModuleContext) { + if ctx.ModuleName() != "product_config" || ctx.ModuleDir() != "build/soong" { + ctx.ModuleErrorf("There can only be one product_config module in build/soong") + return + } + outputFilePath := PathForModuleOut(ctx, p.Name()+".json").OutputPath + + // DeviceProduct can be null so calling ctx.Config().DeviceProduct() may cause null dereference + targetProduct := proptools.String(ctx.Config().config.productVariables.DeviceProduct) + if targetProduct != "" { + targetProduct += "." + } + soongVariablesPath := PathForOutput(ctx, "soong."+targetProduct+"variables") + extraVariablesPath := PathForOutput(ctx, "soong."+targetProduct+"extra.variables") + + rule := NewRuleBuilder(pctx, ctx) + rule.Command().BuiltTool("merge_json"). + Output(outputFilePath). + Input(soongVariablesPath). + Input(extraVariablesPath). + rule.Build("product_config.json", "building product_config.json") + + ctx.SetOutputFiles(Paths{outputFilePath}, "") +} + +// product_config module exports product variables and extra variables as a JSON file. +func productConfigFactory() Module { + module := &productConfigModule{} + InitAndroidModule(module) + return module +} diff --git a/android/selects_test.go b/android/selects_test.go index 6f980ce4c..fc020a47d 100644 --- a/android/selects_test.go +++ b/android/selects_test.go @@ -25,12 +25,14 @@ import ( func TestSelects(t *testing.T) { testCases := []struct { - name string - bp string - provider selectsTestProvider - providers map[string]selectsTestProvider - vendorVars map[string]map[string]string - expectedError string + name string + bp string + fs MockFS + provider selectsTestProvider + providers map[string]selectsTestProvider + vendorVars map[string]map[string]string + vendorVarTypes map[string]map[string]string + expectedError string }{ { name: "basic string list", @@ -97,32 +99,38 @@ func TestSelects(t *testing.T) { }, }, { - name: "paths with module references", + name: "Expression in select", bp: ` my_module_type { name: "foo", - my_paths: select(soong_config_variable("my_namespace", "my_variable"), { - "a": [":a"], - "b": [":b"], - default: [":c"], + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + "a": "foo" + "bar", + default: "baz", }), } `, - expectedError: `"foo" depends on undefined module "c"`, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("foobar"), + }, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "a", + }, + }, }, { - name: "Differing types", + name: "paths with module references", bp: ` my_module_type { name: "foo", - my_string: select(soong_config_variable("my_namespace", "my_variable"), { - "a": "a.cpp", - "b": true, - default: "c.cpp", + my_paths: select(soong_config_variable("my_namespace", "my_variable"), { + "a": [":a"], + "b": [":b"], + default: [":c"], }), } `, - expectedError: `Android.bp:8:5: Found select statement with differing types "string" and "bool" in its cases`, + expectedError: `"foo" depends on undefined module "c"`, }, { name: "Select type doesn't match property type", @@ -136,7 +144,7 @@ func TestSelects(t *testing.T) { }), } `, - expectedError: `can't assign bool value to string property "my_string\[0\]"`, + expectedError: `can't assign bool value to string property`, }, { name: "String list non-default", @@ -584,6 +592,31 @@ func TestSelects(t *testing.T) { }, }, { + name: "Select on boolean soong config variable", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + true: "t", + false: "f", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "true", + }, + }, + vendorVarTypes: map[string]map[string]string{ + "my_namespace": { + "my_variable": "bool", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("t"), + }, + }, + { name: "Select on boolean false", bp: ` my_module_type { @@ -778,10 +811,215 @@ func TestSelects(t *testing.T) { my_string_list: &[]string{"a.cpp", "b.cpp", "c.cpp"}, }, }, + { + name: "Test AppendSimpleValue", + bp: ` + my_module_type { + name: "foo", + my_string_list: ["a.cpp"] + select(soong_config_variable("my_namespace", "my_variable"), { + "a": ["a.cpp"], + "b": ["b.cpp"], + default: ["c.cpp"], + }), + } + `, + vendorVars: map[string]map[string]string{ + "selects_test": { + "append_to_string_list": "foo.cpp", + }, + }, + provider: selectsTestProvider{ + my_string_list: &[]string{"a.cpp", "c.cpp", "foo.cpp"}, + }, + }, + { + name: "Arch variant bool", + bp: ` + my_variable = ["b.cpp"] + my_module_type { + name: "foo", + arch_variant_configurable_bool: false, + target: { + bionic_arm64: { + enabled: true, + }, + }, + } + `, + provider: selectsTestProvider{ + arch_variant_configurable_bool: proptools.BoolPtr(false), + }, + }, + { + name: "Simple string binding", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello " + my_binding, + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("hello world!"), + }, + }, + { + name: "Any branch with binding not taken", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello " + my_binding, + default: "goodbye", + }) + } + `, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("goodbye"), + }, + }, + { + name: "Any branch without binding", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any: "hello", + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("hello"), + }, + }, + { + name: "Binding conflicts with file-level variable", + bp: ` + my_binding = "asdf" + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello", + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + expectedError: "variable already set in inherited scope, previous assignment", + }, + { + name: "Binding in combination with file-level variable", + bp: ` + my_var = " there " + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello" + my_var + my_binding, + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("hello there world!"), + }, + }, + { + name: "Bindings in subdirectory inherits variable", + fs: map[string][]byte{ + "Android.bp": []byte(` +my_var = "abcd" +`), + "directoryB/Android.bp": []byte(` +my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "variable_a"), { + any @ my_binding: my_var + my_binding, + default: "", + }), +} +`), + }, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "variable_a": "e", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("abcde"), + }, + }, + { + name: "Cannot modify variable after referenced by select", + bp: ` +my_var = "foo" +my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "variable_a"), { + "a": my_var, + default: "", + }), +} +my_var += "bar" +`, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "variable_a": "b", // notably not the value that causes my_var to be referenced + }, + }, + expectedError: `modified variable "my_var" with \+= after referencing`, + }, + { + name: "Cannot shadow variable with binding", + bp: ` +my_var = "foo" +my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "variable_a"), { + any @ my_var: my_var, + default: "", + }), +} +`, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "variable_a": "a", + }, + }, + expectedError: `variable already set in inherited scope, previous assignment:`, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + fs := tc.fs + if fs == nil { + fs = make(MockFS) + } + if tc.bp != "" { + fs["Android.bp"] = []byte(tc.bp) + } fixtures := GroupFixturePreparers( PrepareForTestWithDefaults, PrepareForTestWithArchMutator, @@ -792,12 +1030,14 @@ func TestSelects(t *testing.T) { }), FixtureModifyProductVariables(func(variables FixtureProductVariables) { variables.VendorVars = tc.vendorVars + variables.VendorVarTypes = tc.vendorVarTypes }), + FixtureMergeMockFs(fs), ) if tc.expectedError != "" { fixtures = fixtures.ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(tc.expectedError)) } - result := fixtures.RunTestWithBp(t, tc.bp) + result := fixtures.RunTest(t) if tc.expectedError == "" { if len(tc.providers) == 0 { @@ -825,6 +1065,7 @@ type selectsTestProvider struct { my_string_list *[]string my_paths *[]string replacing_string_list *[]string + arch_variant_configurable_bool *bool my_nonconfigurable_bool *bool my_nonconfigurable_string *string my_nonconfigurable_string_list []string @@ -849,6 +1090,7 @@ func (p *selectsTestProvider) String() string { my_string_list: %s, my_paths: %s, replacing_string_list %s, + arch_variant_configurable_bool %v my_nonconfigurable_bool: %v, my_nonconfigurable_string: %s, my_nonconfigurable_string_list: %s, @@ -858,6 +1100,7 @@ func (p *selectsTestProvider) String() string { p.my_string_list, p.my_paths, p.replacing_string_list, + p.arch_variant_configurable_bool, p.my_nonconfigurable_bool, myNonconfigurableStringStr, p.my_nonconfigurable_string_list, @@ -872,6 +1115,7 @@ type selectsMockModuleProperties struct { My_string_list proptools.Configurable[[]string] My_paths proptools.Configurable[[]string] `android:"path"` Replacing_string_list proptools.Configurable[[]string] `android:"replace_instead_of_append,arch_variant"` + Arch_variant_configurable_bool proptools.Configurable[bool] `android:"replace_instead_of_append,arch_variant"` My_nonconfigurable_bool *bool My_nonconfigurable_string *string My_nonconfigurable_string_list []string @@ -892,12 +1136,17 @@ func optionalToPtr[T any](o proptools.ConfigurableOptional[T]) *T { } func (p *selectsMockModule) GenerateAndroidBuildActions(ctx ModuleContext) { + toAppend := ctx.Config().VendorConfig("selects_test").String("append_to_string_list") + if toAppend != "" { + p.properties.My_string_list.AppendSimpleValue([]string{toAppend}) + } SetProvider(ctx, selectsTestProviderKey, selectsTestProvider{ my_bool: optionalToPtr(p.properties.My_bool.Get(ctx)), my_string: optionalToPtr(p.properties.My_string.Get(ctx)), my_string_list: optionalToPtr(p.properties.My_string_list.Get(ctx)), my_paths: optionalToPtr(p.properties.My_paths.Get(ctx)), replacing_string_list: optionalToPtr(p.properties.Replacing_string_list.Get(ctx)), + arch_variant_configurable_bool: optionalToPtr(p.properties.Arch_variant_configurable_bool.Get(ctx)), my_nonconfigurable_bool: p.properties.My_nonconfigurable_bool, my_nonconfigurable_string: p.properties.My_nonconfigurable_string, my_nonconfigurable_string_list: p.properties.My_nonconfigurable_string_list, diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 38db92995..e0b1d7cbe 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -463,57 +463,6 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s }).(map[string]blueprint.ModuleFactory) } -// tracingConfig is a wrapper to soongconfig.SoongConfig which records all accesses to SoongConfig. -type tracingConfig struct { - config soongconfig.SoongConfig - boolSet map[string]bool - stringSet map[string]string - isSetSet map[string]bool -} - -func (c *tracingConfig) Bool(name string) bool { - c.boolSet[name] = c.config.Bool(name) - return c.boolSet[name] -} - -func (c *tracingConfig) String(name string) string { - c.stringSet[name] = c.config.String(name) - return c.stringSet[name] -} - -func (c *tracingConfig) IsSet(name string) bool { - c.isSetSet[name] = c.config.IsSet(name) - return c.isSetSet[name] -} - -func (c *tracingConfig) getTrace() soongConfigTrace { - ret := soongConfigTrace{} - - for k, v := range c.boolSet { - ret.Bools = append(ret.Bools, fmt.Sprintf("%q:%t", k, v)) - } - for k, v := range c.stringSet { - ret.Strings = append(ret.Strings, fmt.Sprintf("%q:%q", k, v)) - } - for k, v := range c.isSetSet { - ret.IsSets = append(ret.IsSets, fmt.Sprintf("%q:%t", k, v)) - } - - return ret -} - -func newTracingConfig(config soongconfig.SoongConfig) *tracingConfig { - c := tracingConfig{ - config: config, - boolSet: make(map[string]bool), - stringSet: make(map[string]string), - isSetSet: make(map[string]bool), - } - return &c -} - -var _ soongconfig.SoongConfig = (*tracingConfig)(nil) - // configModuleFactory takes an existing soongConfigModuleFactory and a // ModuleType to create a new ModuleFactory that uses a custom loadhook. func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType) blueprint.ModuleFactory { @@ -561,8 +510,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi // conditional on Soong config variables by reading the product // config variables from Make. AddLoadHook(module, func(ctx LoadHookContext) { - tracingConfig := newTracingConfig(ctx.Config().VendorConfig(moduleType.ConfigNamespace)) - newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, tracingConfig) + config := ctx.Config().VendorConfig(moduleType.ConfigNamespace) + newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config) if err != nil { ctx.ModuleErrorf("%s", err) return @@ -570,8 +519,6 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi for _, ps := range newProps { ctx.AppendProperties(ps) } - - module.(Module).base().commonProperties.SoongConfigTrace = tracingConfig.getTrace() }) return module, props } diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go index a6b2c51c6..04aafdeee 100644 --- a/android/soong_config_modules_test.go +++ b/android/soong_config_modules_test.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "path/filepath" "testing" ) @@ -506,197 +505,3 @@ func TestSoongConfigModuleSingletonModule(t *testing.T) { }) } } - -func TestSoongConfigModuleTrace(t *testing.T) { - bp := ` - soong_config_module_type { - name: "acme_test", - module_type: "test", - config_namespace: "acme", - variables: ["board", "feature1", "FEATURE3", "unused_string_var"], - bool_variables: ["feature2", "unused_feature", "always_true"], - value_variables: ["size", "unused_size"], - properties: ["cflags", "srcs", "defaults"], - } - - soong_config_module_type { - name: "acme_test_defaults", - module_type: "test_defaults", - config_namespace: "acme", - variables: ["board", "feature1", "FEATURE3", "unused_string_var"], - bool_variables: ["feature2", "unused_feature", "always_true"], - value_variables: ["size", "unused_size"], - properties: ["cflags", "srcs", "defaults"], - } - - soong_config_string_variable { - name: "board", - values: ["soc_a", "soc_b", "soc_c"], - } - - soong_config_string_variable { - name: "unused_string_var", - values: ["a", "b"], - } - - soong_config_bool_variable { - name: "feature1", - } - - soong_config_bool_variable { - name: "FEATURE3", - } - - test_defaults { - name: "test_defaults", - cflags: ["DEFAULT"], - } - - test { - name: "normal", - defaults: ["test_defaults"], - } - - acme_test { - name: "board_1", - defaults: ["test_defaults"], - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - }, - } - - acme_test { - name: "board_2", - defaults: ["test_defaults"], - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - }, - } - - acme_test { - name: "size", - defaults: ["test_defaults"], - soong_config_variables: { - size: { - cflags: ["-DSIZE=%s"], - }, - }, - } - - acme_test { - name: "board_and_size", - defaults: ["test_defaults"], - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - size: { - cflags: ["-DSIZE=%s"], - }, - }, - } - - acme_test_defaults { - name: "board_defaults", - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - }, - } - - acme_test_defaults { - name: "size_defaults", - soong_config_variables: { - size: { - cflags: ["-DSIZE=%s"], - }, - }, - } - - test { - name: "board_and_size_with_defaults", - defaults: ["board_defaults", "size_defaults"], - } - ` - - fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer { - return FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = vars - }) - } - - preparer := fixtureForVendorVars(map[string]map[string]string{ - "acme": { - "board": "soc_a", - "size": "42", - "feature1": "true", - "feature2": "false", - // FEATURE3 unset - "unused_feature": "true", // unused - "unused_size": "1", // unused - "unused_string_var": "a", // unused - "always_true": "true", - }, - }) - - t.Run("soong config trace hash", func(t *testing.T) { - result := GroupFixturePreparers( - preparer, - PrepareForTestWithDefaults, - PrepareForTestWithSoongConfigModuleBuildComponents, - prepareForSoongConfigTestModule, - FixtureRegisterWithContext(func(ctx RegistrationContext) { - ctx.FinalDepsMutators(registerSoongConfigTraceMutator) - }), - FixtureWithRootAndroidBp(bp), - ).RunTest(t) - - // Hashes of modules not using soong config should be empty - normal := result.ModuleForTests("normal", "").Module().(*soongConfigTestModule) - AssertDeepEquals(t, "normal hash", normal.base().commonProperties.SoongConfigTraceHash, "") - AssertDeepEquals(t, "normal hash out", normal.outputPath.RelativeToTop().String(), "out/soong/.intermediates/normal/test") - - board1 := result.ModuleForTests("board_1", "").Module().(*soongConfigTestModule) - board2 := result.ModuleForTests("board_2", "").Module().(*soongConfigTestModule) - size := result.ModuleForTests("size", "").Module().(*soongConfigTestModule) - - // Trace mutator sets soong config trace hash correctly - board1Hash := board1.base().commonProperties.SoongConfigTrace.hash() - board1Output := board1.outputPath.RelativeToTop().String() - AssertDeepEquals(t, "board hash calc", board1Hash, board1.base().commonProperties.SoongConfigTraceHash) - AssertDeepEquals(t, "board hash path", board1Output, filepath.Join("out/soong/.intermediates/board_1", board1Hash, "test")) - - sizeHash := size.base().commonProperties.SoongConfigTrace.hash() - sizeOutput := size.outputPath.RelativeToTop().String() - AssertDeepEquals(t, "size hash calc", sizeHash, size.base().commonProperties.SoongConfigTraceHash) - AssertDeepEquals(t, "size hash path", sizeOutput, filepath.Join("out/soong/.intermediates/size", sizeHash, "test")) - - // Trace should be identical for modules using the same set of variables - AssertDeepEquals(t, "board trace", board1.base().commonProperties.SoongConfigTrace, board2.base().commonProperties.SoongConfigTrace) - AssertDeepEquals(t, "board hash", board1.base().commonProperties.SoongConfigTraceHash, board2.base().commonProperties.SoongConfigTraceHash) - - // Trace hash should be different for different sets of soong variables - AssertBoolEquals(t, "board hash not equal to size hash", board1.base().commonProperties.SoongConfigTraceHash == size.commonProperties.SoongConfigTraceHash, false) - - boardSize := result.ModuleForTests("board_and_size", "").Module().(*soongConfigTestModule) - boardSizeDefaults := result.ModuleForTests("board_and_size_with_defaults", "").Module() - - // Trace should propagate - AssertDeepEquals(t, "board_size hash calc", boardSize.base().commonProperties.SoongConfigTrace.hash(), boardSize.base().commonProperties.SoongConfigTraceHash) - AssertDeepEquals(t, "board_size trace", boardSize.base().commonProperties.SoongConfigTrace, boardSizeDefaults.base().commonProperties.SoongConfigTrace) - AssertDeepEquals(t, "board_size hash", boardSize.base().commonProperties.SoongConfigTraceHash, boardSizeDefaults.base().commonProperties.SoongConfigTraceHash) - }) -} diff --git a/android/soongconfig/Android.bp b/android/soongconfig/Android.bp index 8fe1ff1eb..5a6df2684 100644 --- a/android/soongconfig/Android.bp +++ b/android/soongconfig/Android.bp @@ -9,7 +9,6 @@ bootstrap_go_package { "blueprint", "blueprint-parser", "blueprint-proptools", - "soong-bazel", "soong-starlark-format", ], srcs: [ diff --git a/android/soongconfig/modules.go b/android/soongconfig/modules.go index 87af774fd..f6046d00c 100644 --- a/android/soongconfig/modules.go +++ b/android/soongconfig/modules.go @@ -824,11 +824,16 @@ func (s *listVariable) printfIntoPropertyRecursive(fieldName []string, propStruc } field.Set(newField) case reflect.Struct: - fieldName = append(fieldName, propStruct.Type().Field(i).Name) - if err := s.printfIntoPropertyRecursive(fieldName, field, configValues); err != nil { - return err + if proptools.IsConfigurable(field.Type()) { + fieldName = append(fieldName, propStruct.Type().Field(i).Name) + return fmt.Errorf("soong_config_variables.%s.%s: list variables are not supported on configurable properties", s.variable, strings.Join(fieldName, ".")) + } else { + fieldName = append(fieldName, propStruct.Type().Field(i).Name) + if err := s.printfIntoPropertyRecursive(fieldName, field, configValues); err != nil { + return err + } + fieldName = fieldName[:len(fieldName)-1] } - fieldName = fieldName[:len(fieldName)-1] default: fieldName = append(fieldName, propStruct.Type().Field(i).Name) return fmt.Errorf("soong_config_variables.%s.%s: unsupported property type %q", s.variable, strings.Join(fieldName, "."), kind) diff --git a/android/testing.go b/android/testing.go index 6518f4a53..18fd3b31c 100644 --- a/android/testing.go +++ b/android/testing.go @@ -224,6 +224,10 @@ func (ctx *TestContext) OtherModuleProviderAdaptor() OtherModuleProviderContext }) } +func (ctx *TestContext) OtherModulePropertyErrorf(module Module, property string, fmt_ string, args ...interface{}) { + panic(fmt.Sprintf(fmt_, args...)) +} + // registeredComponentOrder defines the order in which a sortableComponent type is registered at // runtime and provides support for reordering the components registered for a test in the same // way. @@ -1014,10 +1018,21 @@ func (m TestingModule) VariablesForTestsRelativeToTop() map[string]string { return normalizeStringMapRelativeToTop(m.config, m.module.VariablesForTests()) } -// OutputFiles calls OutputFileProducer.OutputFiles on the encapsulated module, exits the test -// immediately if there is an error and otherwise returns the result of calling Paths.RelativeToTop +// OutputFiles first checks if module base outputFiles property has any output +// files can be used to return. +// If not, it calls OutputFileProducer.OutputFiles on the +// encapsulated module, exits the test immediately if there is an error and +// otherwise returns the result of calling Paths.RelativeToTop // on the returned Paths. func (m TestingModule) OutputFiles(t *testing.T, tag string) Paths { + // TODO: remove OutputFileProducer part + outputFiles := m.Module().base().outputFiles + if tag == "" && outputFiles.DefaultOutputFiles != nil { + return outputFiles.DefaultOutputFiles.RelativeToTop() + } else if taggedOutputFiles, hasTag := outputFiles.TaggedOutputFiles[tag]; hasTag { + return taggedOutputFiles + } + producer, ok := m.module.(OutputFileProducer) if !ok { t.Fatalf("%q must implement OutputFileProducer\n", m.module.Name()) diff --git a/android/updatable_modules.go b/android/updatable_modules.go index 1548170f9..dd7dc2c2f 100644 --- a/android/updatable_modules.go +++ b/android/updatable_modules.go @@ -33,4 +33,4 @@ package android // * AOSP - xx9990000 // * x-mainline-prod - xx9990000 // * master - 990090000 -const DefaultUpdatableModuleVersion = "990090000" +const DefaultUpdatableModuleVersion = "350090000" diff --git a/android/variable.go b/android/variable.go index 16338166b..a331439f6 100644 --- a/android/variable.go +++ b/android/variable.go @@ -199,11 +199,12 @@ type ProductVariables struct { // Suffix to add to generated Makefiles Make_suffix *string `json:",omitempty"` - BuildId *string `json:",omitempty"` - BuildNumberFile *string `json:",omitempty"` - BuildHostnameFile *string `json:",omitempty"` - BuildThumbprintFile *string `json:",omitempty"` - DisplayBuildNumber *bool `json:",omitempty"` + BuildId *string `json:",omitempty"` + BuildFingerprintFile *string `json:",omitempty"` + BuildNumberFile *string `json:",omitempty"` + BuildHostnameFile *string `json:",omitempty"` + BuildThumbprintFile *string `json:",omitempty"` + DisplayBuildNumber *bool `json:",omitempty"` Platform_display_version_name *string `json:",omitempty"` Platform_version_name *string `json:",omitempty"` @@ -398,7 +399,8 @@ type ProductVariables struct { PlatformSepolicyCompatVersions []string `json:",omitempty"` - VendorVars map[string]map[string]string `json:",omitempty"` + VendorVars map[string]map[string]string `json:",omitempty"` + VendorVarTypes map[string]map[string]string `json:",omitempty"` Ndk_abis *bool `json:",omitempty"` @@ -458,6 +460,7 @@ type ProductVariables struct { BuildBrokenIncorrectPartitionImages bool `json:",omitempty"` BuildBrokenInputDirModules []string `json:",omitempty"` BuildBrokenDontCheckSystemSdk bool `json:",omitempty"` + BuildBrokenDupSysprop bool `json:",omitempty"` BuildWarningBadOptionalUsesLibsAllowlist []string `json:",omitempty"` @@ -490,8 +493,6 @@ type ProductVariables struct { ReleaseDefaultModuleBuildFromSource *bool `json:",omitempty"` - KeepVndk *bool `json:",omitempty"` - CheckVendorSeappViolations *bool `json:",omitempty"` BuildFlags map[string]string `json:",omitempty"` diff --git a/androidmk/androidmk/android.go b/androidmk/androidmk/android.go index 8a8bb2eaa..570f36c16 100644 --- a/androidmk/androidmk/android.go +++ b/androidmk/androidmk/android.go @@ -341,9 +341,6 @@ func classifyLocalOrGlobalPath(value bpparser.Expression) (string, bpparser.Expr firstOperand := v.Args[0] secondOperand := v.Args[1] - if firstOperand.Type() != bpparser.StringType { - return "global", value, nil - } if _, ok := firstOperand.(*bpparser.Operator); ok { return "global", value, nil diff --git a/androidmk/androidmk/androidmk.go b/androidmk/androidmk/androidmk.go index 2e8810fe8..6fb20dcc1 100644 --- a/androidmk/androidmk/androidmk.go +++ b/androidmk/androidmk/androidmk.go @@ -493,7 +493,6 @@ func setVariable(file *bpFile, plusequals bool, prefix, name string, value bppar Name: name, NamePos: pos, Value: value, - OrigValue: value, EqualsPos: pos, Assigner: "+=", } @@ -506,7 +505,6 @@ func setVariable(file *bpFile, plusequals bool, prefix, name string, value bppar Name: name, NamePos: pos, Value: value, - OrigValue: value, EqualsPos: pos, Assigner: "=", } diff --git a/androidmk/androidmk/values.go b/androidmk/androidmk/values.go index 9618142fa..701c708e2 100644 --- a/androidmk/androidmk/values.go +++ b/androidmk/androidmk/values.go @@ -81,7 +81,7 @@ func makeToStringExpression(ms *mkparser.MakeString, file *bpFile) (bpparser.Exp } tmp := &bpparser.Variable{ Name: name, - Value: &bpparser.String{}, + Type_: bpparser.StringType, } if tmp.Name == "TOP" { @@ -150,7 +150,7 @@ func makeToListExpression(ms *mkparser.MakeString, file *bpFile) (bpparser.Expre } listOfListValues = append(listOfListValues, &bpparser.Variable{ Name: name, - Value: &bpparser.List{}, + Type_: bpparser.ListType, }) listValue = &bpparser.List{} } @@ -215,7 +215,7 @@ func makeToBoolExpression(ms *mkparser.MakeString, file *bpFile) (bpparser.Expre } return &bpparser.Variable{ Name: name, - Value: &bpparser.Bool{}, + Type_: bpparser.BoolType, }, nil } else { return nil, fmt.Errorf("non-const bool expression %s", ms.Dump()) diff --git a/apex/androidmk.go b/apex/androidmk.go index 619be8dc4..4112108dd 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -218,7 +218,7 @@ func (a *apexBundle) writeRequiredModules(w io.Writer, moduleNames []string) { var required []string var targetRequired []string var hostRequired []string - required = append(required, a.RequiredModuleNames()...) + required = append(required, a.required...) targetRequired = append(targetRequired, a.TargetRequiredModuleNames()...) hostRequired = append(hostRequired, a.HostRequiredModuleNames()...) for _, fi := range a.filesInfo { diff --git a/apex/apex.go b/apex/apex.go index 4c97fdb80..e6815bc29 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -18,7 +18,6 @@ package apex import ( "fmt" - "log" "path/filepath" "regexp" "sort" @@ -490,6 +489,9 @@ type apexBundle struct { javaApisUsedByModuleFile android.ModuleOutPath aconfigFiles []android.Path + + // Required modules, filled out during GenerateAndroidBuildActions and used in AndroidMk + required []string } // apexFileClass represents a type of file that can be included in APEX. @@ -568,7 +570,7 @@ func newApexFile(ctx android.BaseModuleContext, builtFile android.Path, androidM if module != nil { ret.moduleDir = ctx.OtherModuleDir(module) ret.partition = module.PartitionTag(ctx.DeviceConfig()) - ret.requiredModuleNames = module.RequiredModuleNames() + ret.requiredModuleNames = module.RequiredModuleNames(ctx) ret.targetRequiredModuleNames = module.TargetRequiredModuleNames() ret.hostRequiredModuleNames = module.HostRequiredModuleNames() ret.multilib = module.Target().Arch.ArchType.Multilib @@ -746,9 +748,9 @@ func (a *apexBundle) getImageVariationPair() (string, string) { prefix := android.CoreVariation if a.SocSpecific() || a.DeviceSpecific() { - prefix = cc.VendorVariation + prefix = android.VendorVariation } else if a.ProductSpecific() { - prefix = cc.ProductVariation + prefix = android.ProductVariation } return prefix, "" @@ -954,7 +956,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { // the non-system APEXes because the VNDK libraries won't be included (and duped) in the // APEX, but shared across APEXes via the VNDK APEX. useVndk := a.SocSpecific() || a.DeviceSpecific() || (a.ProductSpecific() && mctx.Config().EnforceProductPartitionInterface()) - excludeVndkLibs := useVndk && a.useVndkAsStable(mctx) if proptools.Bool(a.properties.Use_vndk_as_stable) { if !useVndk { mctx.PropertyErrorf("use_vndk_as_stable", "not supported for system/system_ext APEXes") @@ -962,11 +963,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { if a.minSdkVersionValue(mctx) != "" { mctx.PropertyErrorf("use_vndk_as_stable", "not supported when min_sdk_version is set") } - mctx.VisitDirectDepsWithTag(sharedLibTag, func(dep android.Module) { - if c, ok := dep.(*cc.Module); ok && c.IsVndk() { - mctx.PropertyErrorf("use_vndk_as_stable", "Trying to include a VNDK library(%s) while use_vndk_as_stable is true.", dep.Name()) - } - }) if mctx.Failed() { return } @@ -988,16 +984,9 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { if !android.IsDepInSameApex(mctx, parent, child) { return false } - if excludeVndkLibs { - if c, ok := child.(*cc.Module); ok && c.IsVndk() { - return false - } - } - //TODO: b/296491928 Vendor APEX should use libbinder.ndk instead of libbinder once VNDK is fully deprecated. - if useVndk && mctx.Config().IsVndkDeprecated() && child.Name() == "libbinder" { - log.Print("Libbinder is linked from Vendor APEX ", a.Name(), " with module ", parent.Name()) - return false + if useVndk && child.Name() == "libbinder" { + mctx.ModuleErrorf("Module %s in the vendor APEX %s should not use libbinder. Use libbinder_ndk instead.", parent.Name(), a.Name()) } // By default, all the transitive dependencies are collected, unless filtered out @@ -1054,6 +1043,7 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { InApexModules: []string{a.Name()}, // could be com.mycompany.android.foo ApexContents: []*android.ApexContents{apexContents}, TestApexes: testApexes, + BaseApexName: mctx.ModuleName(), } mctx.WalkDeps(func(child, parent android.Module) bool { if !continueApexDepsWalk(child, parent) { @@ -1182,6 +1172,7 @@ var ( "test_com.android.os.statsd", "test_com.android.permission", "test_com.android.wifi", + "test_imgdiag_com.android.art", "test_jitzygote_com.android.art", // go/keep-sorted end } @@ -1380,25 +1371,6 @@ func (a *apexBundle) DepIsInSameApex(_ android.BaseModuleContext, _ android.Modu return true } -var _ android.OutputFileProducer = (*apexBundle)(nil) - -// Implements android.OutputFileProducer -func (a *apexBundle) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "", android.DefaultDistTag: - // This is the default dist path. - return android.Paths{a.outputFile}, nil - case imageApexSuffix: - // uncompressed one - if a.outputApexFile != nil { - return android.Paths{a.outputApexFile}, nil - } - fallthrough - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - var _ multitree.Exportable = (*apexBundle)(nil) func (a *apexBundle) Exportable() bool { @@ -1677,12 +1649,12 @@ var _ javaModule = (*java.DexImport)(nil) var _ javaModule = (*java.SdkLibraryImport)(nil) // apexFileForJavaModule creates an apexFile for a java module's dex implementation jar. -func apexFileForJavaModule(ctx android.BaseModuleContext, module javaModule) apexFile { +func apexFileForJavaModule(ctx android.ModuleContext, module javaModule) apexFile { return apexFileForJavaModuleWithFile(ctx, module, module.DexJarBuildPath(ctx).PathOrNil()) } // apexFileForJavaModuleWithFile creates an apexFile for a java module with the supplied file. -func apexFileForJavaModuleWithFile(ctx android.BaseModuleContext, module javaModule, dexImplementationJar android.Path) apexFile { +func apexFileForJavaModuleWithFile(ctx android.ModuleContext, module javaModule, dexImplementationJar android.Path) apexFile { dirInApex := "javalib" af := newApexFile(ctx, dexImplementationJar, module.BaseModuleName(), dirInApex, javaSharedLib, module) af.jacocoReportClassesFile = module.JacocoReportClassesFile() @@ -1693,10 +1665,12 @@ func apexFileForJavaModuleWithFile(ctx android.BaseModuleContext, module javaMod if sdkLib, ok := module.(*java.SdkLibrary); ok { for _, install := range sdkLib.BuiltInstalledForApex() { af.requiredModuleNames = append(af.requiredModuleNames, install.FullModuleName()) + install.PackageFile(ctx) } } else if dexpreopter, ok := module.(java.DexpreopterInterface); ok { for _, install := range dexpreopter.DexpreoptBuiltInstalledForApex() { af.requiredModuleNames = append(af.requiredModuleNames, install.FullModuleName()) + install.PackageFile(ctx) } } return af @@ -2193,15 +2167,6 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, // tags used below are private (e.g. `cc.sharedDepTag`). if cc.IsSharedDepTag(depTag) || cc.IsRuntimeDepTag(depTag) { if ch, ok := child.(*cc.Module); ok { - if ch.UseVndk() && a.useVndkAsStable(ctx) && ch.IsVndk() { - vctx.requireNativeLibs = append(vctx.requireNativeLibs, ":vndk") - return false - } - - //TODO: b/296491928 Vendor APEX should use libbinder.ndk instead of libbinder once VNDK is fully deprecated. - if ch.InVendorOrProduct() && ctx.Config().IsVndkDeprecated() && child.Name() == "libbinder" { - return false - } af := apexFileForNativeLibrary(ctx, ch, vctx.handleSpecialLibs) af.transitiveDep = true @@ -2447,6 +2412,10 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.provideApexExportsInfo(ctx) a.providePrebuiltInfo(ctx) + + a.required = a.RequiredModuleNames(ctx) + + a.setOutputFiles(ctx) } // Set prebuiltInfoProvider. This will be used by `apex_prebuiltinfo_singleton` to print out a metadata file @@ -2475,6 +2444,18 @@ func (a *apexBundle) provideApexExportsInfo(ctx android.ModuleContext) { }) } +// Set output files to outputFiles property, which is later used to set the +// OutputFilesProvider +func (a *apexBundle) setOutputFiles(ctx android.ModuleContext) { + // default dist path + ctx.SetOutputFiles(android.Paths{a.outputFile}, "") + ctx.SetOutputFiles(android.Paths{a.outputFile}, android.DefaultDistTag) + // uncompressed one + if a.outputApexFile != nil { + ctx.SetOutputFiles(android.Paths{a.outputApexFile}, imageApexSuffix) + } +} + // apexBootclasspathFragmentFiles returns the list of apexFile structures defining the files that // the bootclasspath_fragment contributes to the apex. func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint.Module) []apexFile { @@ -3022,12 +3003,3 @@ func rBcpPackages() map[string][]string { func (a *apexBundle) IsTestApex() bool { return a.testApex } - -func (a *apexBundle) useVndkAsStable(ctx android.BaseModuleContext) bool { - // VNDK cannot be linked if it is deprecated - if ctx.Config().IsVndkDeprecated() { - return false - } - - return proptools.Bool(a.properties.Use_vndk_as_stable) -} diff --git a/apex/apex_test.go b/apex/apex_test.go index c60ee73cd..3bb396692 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -3785,32 +3785,31 @@ func TestVndkApexNameRule(t *testing.T) { } func TestVndkApexDoesntSupportNativeBridgeSupported(t *testing.T) { - testApexError(t, `module "com.android.vndk.current" .*: native_bridge_supported: .* doesn't support native bridge binary`, ` + testApexError(t, `module "com.android.vndk.v30" .*: native_bridge_supported: .* doesn't support native bridge binary`, ` apex_vndk { - name: "com.android.vndk.current", - key: "com.android.vndk.current.key", + name: "com.android.vndk.v30", + key: "com.android.vndk.v30.key", file_contexts: ":myapex-file_contexts", native_bridge_supported: true, } apex_key { - name: "com.android.vndk.current.key", + name: "com.android.vndk.v30.key", public_key: "testkey.avbpubkey", private_key: "testkey.pem", } - cc_library { + vndk_prebuilt_shared { name: "libvndk", + version: "30", + target_arch: "arm", srcs: ["mylib.cpp"], vendor_available: true, product_available: true, native_bridge_supported: true, - host_supported: true, vndk: { enabled: true, }, - system_shared_libs: [], - stl: "none", } `) } @@ -7126,6 +7125,46 @@ func TestJavaSDKLibrary(t *testing.T) { ensureMatches(t, contents, "<library\\n\\s+name=\\\"foo\\\"\\n\\s+file=\\\"/apex/myapex/javalib/foo.jar\\\"") } +func TestJavaSDKLibraryOverrideApexes(t *testing.T) { + ctx := testApex(t, ` + override_apex { + name: "mycompanyapex", + base: "myapex", + } + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo"], + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + apex_available: [ "myapex" ], + } + + prebuilt_apis { + name: "sdk", + api_dirs: ["100"], + } + `, withFiles(filesForSdkLibrary)) + + // Permission XML should point to the activated path of impl jar of java_sdk_library. + // Since override variants (com.mycompany.android.foo) are installed in the same package as the overridden variant + // (com.android.foo), the filepath should not contain override apex name. + sdkLibrary := ctx.ModuleForTests("foo.xml", "android_common_mycompanyapex").Output("foo.xml") + contents := android.ContentFromFileRuleForTests(t, ctx, sdkLibrary) + ensureMatches(t, contents, "<library\\n\\s+name=\\\"foo\\\"\\n\\s+file=\\\"/apex/myapex/javalib/foo.jar\\\"") +} + func TestJavaSDKLibrary_WithinApex(t *testing.T) { ctx := testApex(t, ` apex { @@ -9807,188 +9846,246 @@ func ensureDoesNotContainRequiredDeps(t *testing.T, ctx *android.TestContext, mo } } -// TODO(b/193460475): Re-enable this test -//func TestApexStrictUpdtabilityLint(t *testing.T) { -// bpTemplate := ` -// apex { -// name: "myapex", -// key: "myapex.key", -// java_libs: ["myjavalib"], -// updatable: %v, -// min_sdk_version: "29", -// } -// apex_key { -// name: "myapex.key", -// } -// java_library { -// name: "myjavalib", -// srcs: ["MyClass.java"], -// apex_available: [ "myapex" ], -// lint: { -// strict_updatability_linting: %v, -// }, -// sdk_version: "current", -// min_sdk_version: "29", -// } -// ` -// fs := android.MockFS{ -// "lint-baseline.xml": nil, -// } -// -// testCases := []struct { -// testCaseName string -// apexUpdatable bool -// javaStrictUpdtabilityLint bool -// lintFileExists bool -// disallowedFlagExpected bool -// }{ -// { -// testCaseName: "lint-baseline.xml does not exist, no disallowed flag necessary in lint cmd", -// apexUpdatable: true, -// javaStrictUpdtabilityLint: true, -// lintFileExists: false, -// disallowedFlagExpected: false, -// }, -// { -// testCaseName: "non-updatable apex respects strict_updatability of javalib", -// apexUpdatable: false, -// javaStrictUpdtabilityLint: false, -// lintFileExists: true, -// disallowedFlagExpected: false, -// }, -// { -// testCaseName: "non-updatable apex respects strict updatability of javalib", -// apexUpdatable: false, -// javaStrictUpdtabilityLint: true, -// lintFileExists: true, -// disallowedFlagExpected: true, -// }, -// { -// testCaseName: "updatable apex sets strict updatability of javalib to true", -// apexUpdatable: true, -// javaStrictUpdtabilityLint: false, // will be set to true by mutator -// lintFileExists: true, -// disallowedFlagExpected: true, -// }, -// } -// -// for _, testCase := range testCases { -// bp := fmt.Sprintf(bpTemplate, testCase.apexUpdatable, testCase.javaStrictUpdtabilityLint) -// fixtures := []android.FixturePreparer{} -// if testCase.lintFileExists { -// fixtures = append(fixtures, fs.AddToFixture()) -// } -// -// result := testApex(t, bp, fixtures...) -// myjavalib := result.ModuleForTests("myjavalib", "android_common_apex29") -// sboxProto := android.RuleBuilderSboxProtoForTests(t, myjavalib.Output("lint.sbox.textproto")) -// disallowedFlagActual := strings.Contains(*sboxProto.Commands[0].Command, "--baseline lint-baseline.xml --disallowed_issues NewApi") -// -// if disallowedFlagActual != testCase.disallowedFlagExpected { -// t.Errorf("Failed testcase: %v \nActual lint cmd: %v", testCase.testCaseName, *sboxProto.Commands[0].Command) -// } -// } -//} -// -//func TestUpdatabilityLintSkipLibcore(t *testing.T) { -// bp := ` -// apex { -// name: "myapex", -// key: "myapex.key", -// java_libs: ["myjavalib"], -// updatable: true, -// min_sdk_version: "29", -// } -// apex_key { -// name: "myapex.key", -// } -// java_library { -// name: "myjavalib", -// srcs: ["MyClass.java"], -// apex_available: [ "myapex" ], -// sdk_version: "current", -// min_sdk_version: "29", -// } -// ` -// -// testCases := []struct { -// testCaseName string -// moduleDirectory string -// disallowedFlagExpected bool -// }{ -// { -// testCaseName: "lintable module defined outside libcore", -// moduleDirectory: "", -// disallowedFlagExpected: true, -// }, -// { -// testCaseName: "lintable module defined in libcore root directory", -// moduleDirectory: "libcore/", -// disallowedFlagExpected: false, -// }, -// { -// testCaseName: "lintable module defined in libcore child directory", -// moduleDirectory: "libcore/childdir/", -// disallowedFlagExpected: true, -// }, -// } -// -// for _, testCase := range testCases { -// lintFileCreator := android.FixtureAddTextFile(testCase.moduleDirectory+"lint-baseline.xml", "") -// bpFileCreator := android.FixtureAddTextFile(testCase.moduleDirectory+"Android.bp", bp) -// result := testApex(t, "", lintFileCreator, bpFileCreator) -// myjavalib := result.ModuleForTests("myjavalib", "android_common_apex29") -// sboxProto := android.RuleBuilderSboxProtoForTests(t, myjavalib.Output("lint.sbox.textproto")) -// cmdFlags := fmt.Sprintf("--baseline %vlint-baseline.xml --disallowed_issues NewApi", testCase.moduleDirectory) -// disallowedFlagActual := strings.Contains(*sboxProto.Commands[0].Command, cmdFlags) -// -// if disallowedFlagActual != testCase.disallowedFlagExpected { -// t.Errorf("Failed testcase: %v \nActual lint cmd: %v", testCase.testCaseName, *sboxProto.Commands[0].Command) -// } -// } -//} -// -//// checks transtive deps of an apex coming from bootclasspath_fragment -//func TestApexStrictUpdtabilityLintBcpFragmentDeps(t *testing.T) { -// bp := ` -// apex { -// name: "myapex", -// key: "myapex.key", -// bootclasspath_fragments: ["mybootclasspathfragment"], -// updatable: true, -// min_sdk_version: "29", -// } -// apex_key { -// name: "myapex.key", -// } -// bootclasspath_fragment { -// name: "mybootclasspathfragment", -// contents: ["myjavalib"], -// apex_available: ["myapex"], -// hidden_api: { -// split_packages: ["*"], -// }, -// } -// java_library { -// name: "myjavalib", -// srcs: ["MyClass.java"], -// apex_available: [ "myapex" ], -// sdk_version: "current", -// min_sdk_version: "29", -// compile_dex: true, -// } -// ` -// fs := android.MockFS{ -// "lint-baseline.xml": nil, -// } -// -// result := testApex(t, bp, dexpreopt.FixtureSetApexBootJars("myapex:myjavalib"), fs.AddToFixture()) -// myjavalib := result.ModuleForTests("myjavalib", "android_common_apex29") -// sboxProto := android.RuleBuilderSboxProtoForTests(t, myjavalib.Output("lint.sbox.textproto")) -// if !strings.Contains(*sboxProto.Commands[0].Command, "--baseline lint-baseline.xml --disallowed_issues NewApi") { -// t.Errorf("Strict updabality lint missing in myjavalib coming from bootclasspath_fragment mybootclasspath-fragment\nActual lint cmd: %v", *sboxProto.Commands[0].Command) -// } -//} +func TestApexStrictUpdtabilityLint(t *testing.T) { + bpTemplate := ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["myjavalib"], + updatable: %v, + min_sdk_version: "29", + } + apex_key { + name: "myapex.key", + } + java_library { + name: "myjavalib", + srcs: ["MyClass.java"], + apex_available: [ "myapex" ], + lint: { + strict_updatability_linting: %v, + %s + }, + sdk_version: "current", + min_sdk_version: "29", + } + ` + fs := android.MockFS{ + "lint-baseline.xml": nil, + } + + testCases := []struct { + testCaseName string + apexUpdatable bool + javaStrictUpdtabilityLint bool + lintFileExists bool + disallowedFlagExpected bool + }{ + { + testCaseName: "lint-baseline.xml does not exist, no disallowed flag necessary in lint cmd", + apexUpdatable: true, + javaStrictUpdtabilityLint: true, + lintFileExists: false, + disallowedFlagExpected: false, + }, + { + testCaseName: "non-updatable apex respects strict_updatability of javalib", + apexUpdatable: false, + javaStrictUpdtabilityLint: false, + lintFileExists: true, + disallowedFlagExpected: false, + }, + { + testCaseName: "non-updatable apex respects strict updatability of javalib", + apexUpdatable: false, + javaStrictUpdtabilityLint: true, + lintFileExists: true, + disallowedFlagExpected: true, + }, + { + testCaseName: "updatable apex sets strict updatability of javalib to true", + apexUpdatable: true, + javaStrictUpdtabilityLint: false, // will be set to true by mutator + lintFileExists: true, + disallowedFlagExpected: true, + }, + } + + for _, testCase := range testCases { + fixtures := []android.FixturePreparer{} + baselineProperty := "" + if testCase.lintFileExists { + fixtures = append(fixtures, fs.AddToFixture()) + baselineProperty = "baseline_filename: \"lint-baseline.xml\"" + } + bp := fmt.Sprintf(bpTemplate, testCase.apexUpdatable, testCase.javaStrictUpdtabilityLint, baselineProperty) + + result := testApex(t, bp, fixtures...) + myjavalib := result.ModuleForTests("myjavalib", "android_common_apex29") + sboxProto := android.RuleBuilderSboxProtoForTests(t, result, myjavalib.Output("lint.sbox.textproto")) + disallowedFlagActual := strings.Contains(*sboxProto.Commands[0].Command, "--baseline lint-baseline.xml --disallowed_issues NewApi") + + if disallowedFlagActual != testCase.disallowedFlagExpected { + t.Errorf("Failed testcase: %v \nActual lint cmd: %v", testCase.testCaseName, *sboxProto.Commands[0].Command) + } + } +} + +func TestUpdatabilityLintSkipLibcore(t *testing.T) { + bp := ` + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["myjavalib"], + updatable: true, + min_sdk_version: "29", + } + apex_key { + name: "myapex.key", + } + java_library { + name: "myjavalib", + srcs: ["MyClass.java"], + apex_available: [ "myapex" ], + sdk_version: "current", + min_sdk_version: "29", + lint: { + baseline_filename: "lint-baseline.xml", + } + } + ` + + testCases := []struct { + testCaseName string + moduleDirectory string + disallowedFlagExpected bool + }{ + { + testCaseName: "lintable module defined outside libcore", + moduleDirectory: "", + disallowedFlagExpected: true, + }, + { + testCaseName: "lintable module defined in libcore root directory", + moduleDirectory: "libcore/", + disallowedFlagExpected: false, + }, + { + testCaseName: "lintable module defined in libcore child directory", + moduleDirectory: "libcore/childdir/", + disallowedFlagExpected: true, + }, + } + + for _, testCase := range testCases { + lintFileCreator := android.FixtureAddTextFile(testCase.moduleDirectory+"lint-baseline.xml", "") + bpFileCreator := android.FixtureAddTextFile(testCase.moduleDirectory+"Android.bp", bp) + result := testApex(t, "", lintFileCreator, bpFileCreator) + myjavalib := result.ModuleForTests("myjavalib", "android_common_apex29") + sboxProto := android.RuleBuilderSboxProtoForTests(t, result, myjavalib.Output("lint.sbox.textproto")) + cmdFlags := fmt.Sprintf("--baseline %vlint-baseline.xml --disallowed_issues NewApi", testCase.moduleDirectory) + disallowedFlagActual := strings.Contains(*sboxProto.Commands[0].Command, cmdFlags) + + if disallowedFlagActual != testCase.disallowedFlagExpected { + t.Errorf("Failed testcase: %v \nActual lint cmd: %v", testCase.testCaseName, *sboxProto.Commands[0].Command) + } + } +} + +// checks transtive deps of an apex coming from bootclasspath_fragment +func TestApexStrictUpdtabilityLintBcpFragmentDeps(t *testing.T) { + bp := ` + apex { + name: "myapex", + key: "myapex.key", + bootclasspath_fragments: ["mybootclasspathfragment"], + updatable: true, + min_sdk_version: "29", + } + apex_key { + name: "myapex.key", + } + bootclasspath_fragment { + name: "mybootclasspathfragment", + contents: ["myjavalib"], + apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, + } + java_library { + name: "myjavalib", + srcs: ["MyClass.java"], + apex_available: [ "myapex" ], + sdk_version: "current", + min_sdk_version: "29", + compile_dex: true, + lint: { + baseline_filename: "lint-baseline.xml", + } + } + ` + fs := android.MockFS{ + "lint-baseline.xml": nil, + } + + result := testApex(t, bp, dexpreopt.FixtureSetApexBootJars("myapex:myjavalib"), fs.AddToFixture()) + myjavalib := result.ModuleForTests("myjavalib", "android_common_apex29") + sboxProto := android.RuleBuilderSboxProtoForTests(t, result, myjavalib.Output("lint.sbox.textproto")) + if !strings.Contains(*sboxProto.Commands[0].Command, "--baseline lint-baseline.xml --disallowed_issues NewApi") { + t.Errorf("Strict updabality lint missing in myjavalib coming from bootclasspath_fragment mybootclasspath-fragment\nActual lint cmd: %v", *sboxProto.Commands[0].Command) + } +} + +func TestApexLintBcpFragmentSdkLibDeps(t *testing.T) { + bp := ` + apex { + name: "myapex", + key: "myapex.key", + bootclasspath_fragments: ["mybootclasspathfragment"], + min_sdk_version: "29", + } + apex_key { + name: "myapex.key", + } + bootclasspath_fragment { + name: "mybootclasspathfragment", + contents: ["foo"], + apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, + } + java_sdk_library { + name: "foo", + srcs: ["MyClass.java"], + apex_available: [ "myapex" ], + sdk_version: "current", + min_sdk_version: "29", + compile_dex: true, + } + ` + fs := android.MockFS{ + "lint-baseline.xml": nil, + } + + result := android.GroupFixturePreparers( + prepareForApexTest, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.PrepareForTestWithJacocoInstrumentation, + java.FixtureWithLastReleaseApis("foo"), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), + android.FixtureMergeMockFs(fs), + ).RunTestWithBp(t, bp) + + myapex := result.ModuleForTests("myapex", "android_common_myapex") + lintReportInputs := strings.Join(myapex.Output("lint-report-xml.zip").Inputs.Strings(), " ") + android.AssertStringDoesContain(t, + "myapex lint report expected to contain that of the sdk library impl lib as an input", + lintReportInputs, "foo.impl") +} // updatable apexes should propagate updatable=true to its apps func TestUpdatableApexEnforcesAppUpdatability(t *testing.T) { diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index af9123e70..919cb01ac 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -1366,4 +1366,89 @@ func TestBootclasspathFragment_AndroidNonUpdatable_AlwaysUsePrebuiltSdks(t *test android.AssertStringDoesContain(t, "test", command, "--test-stub-classpath="+nonUpdatableTestStubs) } +func TestBootclasspathFragmentProtoContainsMinSdkVersion(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForTestWithBootclasspathFragment, + prepareForTestWithMyapex, + // Configure bootclasspath jars to ensure that hidden API encoding is performed on them. + java.FixtureConfigureApexBootJars("myapex:foo", "myapex:bar"), + // Make sure that the frameworks/base/Android.bp file exists as otherwise hidden API encoding + // is disabled. + android.FixtureAddTextFile("frameworks/base/Android.bp", ""), + + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithLastReleaseApis("foo", "bar"), + ).RunTestWithBp(t, ` + apex { + name: "myapex", + key: "myapex.key", + bootclasspath_fragments: [ + "mybootclasspathfragment", + ], + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library { + name: "foo", + srcs: ["b.java"], + shared_library: false, + public: {enabled: true}, + apex_available: [ + "myapex", + ], + min_sdk_version: "33", + } + + java_sdk_library { + name: "bar", + srcs: ["b.java"], + shared_library: false, + public: {enabled: true}, + apex_available: [ + "myapex", + ], + min_sdk_version: "34", + } + + bootclasspath_fragment { + name: "mybootclasspathfragment", + contents: [ + "foo", + "bar", + ], + apex_available: [ + "myapex", + ], + hidden_api: { + split_packages: ["*"], + }, + } + `) + + fragment := result.ModuleForTests("mybootclasspathfragment", "android_common_apex10000") + classPathProtoContent := android.ContentFromFileRuleForTests(t, result.TestContext, fragment.Output("bootclasspath.pb.textproto")) + // foo + ensureContains(t, classPathProtoContent, `jars { +path: "/apex/myapex/javalib/foo.jar" +classpath: BOOTCLASSPATH +min_sdk_version: "33" +max_sdk_version: "" +} +`) + // bar + ensureContains(t, classPathProtoContent, `jars { +path: "/apex/myapex/javalib/bar.jar" +classpath: BOOTCLASSPATH +min_sdk_version: "34" +max_sdk_version: "" +} +`) +} + // TODO(b/177892522) - add test for host apex. diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 9ad5159b5..65c23d34b 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -197,6 +197,7 @@ func (p *prebuiltCommon) initApexFilesForAndroidMk(ctx android.ModuleContext) { // If this apex contains a system server jar, then the dexpreopt artifacts should be added as required for _, install := range p.Dexpreopter.DexpreoptBuiltInstalledForApex() { p.requiredModuleNames = append(p.requiredModuleNames, install.FullModuleName()) + install.PackageFile(ctx) } } @@ -587,15 +588,6 @@ func (a *Prebuilt) hasSanitizedSource(sanitizer string) bool { return false } -func (p *Prebuilt) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{p.outputApex}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - // prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. func PrebuiltFactory() android.Module { module := &Prebuilt{} @@ -894,6 +886,8 @@ func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.installedFile = ctx.InstallFile(p.installDir, p.installFilename, p.inputApex, p.compatSymlinks...) p.provenanceMetaDataFile = provenance.GenerateArtifactProvenanceMetaData(ctx, p.inputApex, p.installedFile) } + + ctx.SetOutputFiles(android.Paths{p.outputApex}, "") } func (p *Prebuilt) ProvenanceMetaDataFile() android.OutputPath { @@ -1009,15 +1003,6 @@ func (a *ApexSet) hasSanitizedSource(sanitizer string) bool { return false } -func (a *ApexSet) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{a.outputApex}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - // prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex. func apexSetFactory() android.Module { module := &ApexSet{} @@ -1121,6 +1106,8 @@ func (a *ApexSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { for _, overridden := range a.prebuiltCommonProperties.Overrides { a.compatSymlinks = append(a.compatSymlinks, makeCompatSymlinks(overridden, ctx)...) } + + ctx.SetOutputFiles(android.Paths{a.outputApex}, "") } type systemExtContext struct { diff --git a/bazel/Android.bp b/bazel/Android.bp index 4709f5cd3..f8273a847 100644 --- a/bazel/Android.bp +++ b/bazel/Android.bp @@ -6,22 +6,17 @@ bootstrap_go_package { name: "soong-bazel", pkgPath: "android/soong/bazel", srcs: [ - "aquery.go", - "bazel_proxy.go", "configurability.go", - "constants.go", "properties.go", "testing.go", ], testSrcs: [ - "aquery_test.go", "properties_test.go", ], pluginFor: [ "soong_build", ], deps: [ - "bazel_analysis_v2_proto", "blueprint", ], } diff --git a/bazel/aquery.go b/bazel/aquery.go deleted file mode 100644 index 35942bc32..000000000 --- a/bazel/aquery.go +++ /dev/null @@ -1,768 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "path/filepath" - "reflect" - "sort" - "strings" - "sync" - - analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" - - "github.com/google/blueprint/metrics" - "github.com/google/blueprint/proptools" - "google.golang.org/protobuf/proto" -) - -type artifactId int -type depsetId int -type pathFragmentId int - -// KeyValuePair represents Bazel's aquery proto, KeyValuePair. -type KeyValuePair struct { - Key string - Value string -} - -// AqueryDepset is a depset definition from Bazel's aquery response. This is -// akin to the `depSetOfFiles` in the response proto, except: -// - direct artifacts are enumerated by full path instead of by ID -// - it has a hash of the depset contents, instead of an int ID (for determinism) -// -// A depset is a data structure for efficient transitive handling of artifact -// paths. A single depset consists of one or more artifact paths and one or -// more "child" depsets. -type AqueryDepset struct { - ContentHash string - DirectArtifacts []string - TransitiveDepSetHashes []string -} - -// BuildStatement contains information to register a build statement corresponding (one to one) -// with a Bazel action from Bazel's action graph. -type BuildStatement struct { - Command string - Depfile *string - OutputPaths []string - SymlinkPaths []string - Env []*analysis_v2_proto.KeyValuePair - Mnemonic string - - // Inputs of this build statement, either as unexpanded depsets or expanded - // input paths. There should be no overlap between these fields; an input - // path should either be included as part of an unexpanded depset or a raw - // input path string, but not both. - InputDepsetHashes []string - InputPaths []string - FileContents string - // If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment - // and run the mixed build action there - ShouldRunInSbox bool - // A list of files to add as implicit deps to the outputs of this BuildStatement. - // Unlike most properties in BuildStatement, these paths must be relative to the root of - // the whole out/ folder, instead of relative to ctx.Config().BazelContext.OutputBase() - ImplicitDeps []string - IsExecutable bool -} - -// A helper type for aquery processing which facilitates retrieval of path IDs from their -// less readable Bazel structures (depset and path fragment). -type aqueryArtifactHandler struct { - // Maps depset id to AqueryDepset, a representation of depset which is - // post-processed for middleman artifact handling, unhandled artifact - // dropping, content hashing, etc. - depsetIdToAqueryDepset map[depsetId]AqueryDepset - emptyDepsetIds map[depsetId]struct{} - // Maps content hash to AqueryDepset. - depsetHashToAqueryDepset map[string]AqueryDepset - - // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening - // may be an expensive operation. - depsetHashToArtifactPathsCache sync.Map - // Maps artifact ids to fully expanded paths. - artifactIdToPath map[artifactId]string -} - -// The tokens should be substituted with the value specified here, instead of the -// one returned in 'substitutions' of TemplateExpand action. -var templateActionOverriddenTokens = map[string]string{ - // Uses "python3" for %python_binary% instead of the value returned by aquery - // which is "py3wrapper.sh". See removePy3wrapperScript. - "%python_binary%": "python3", -} - -const ( - middlemanMnemonic = "Middleman" - // The file name of py3wrapper.sh, which is used by py_binary targets. - py3wrapperFileName = "/py3wrapper.sh" -) - -func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V { - m := map[K]V{} - for _, v := range values { - m[keyFn(v)] = v - } - return m -} - -func newAqueryHandler(aqueryResult *analysis_v2_proto.ActionGraphContainer) (*aqueryArtifactHandler, error) { - pathFragments := indexBy(aqueryResult.PathFragments, func(pf *analysis_v2_proto.PathFragment) pathFragmentId { - return pathFragmentId(pf.Id) - }) - - artifactIdToPath := make(map[artifactId]string, len(aqueryResult.Artifacts)) - for _, artifact := range aqueryResult.Artifacts { - artifactPath, err := expandPathFragment(pathFragmentId(artifact.PathFragmentId), pathFragments) - if err != nil { - return nil, err - } - if artifact.IsTreeArtifact && - !strings.HasPrefix(artifactPath, "bazel-out/io_bazel_rules_go/") && - !strings.HasPrefix(artifactPath, "bazel-out/rules_java_builtin/") { - // Since we're using ninja as an executor, we can't use tree artifacts. Ninja only - // considers a file/directory "dirty" when it's mtime changes. Directories' mtimes will - // only change when a file in the directory is added/removed, but not when files in - // the directory are changed, or when files in subdirectories are changed/added/removed. - // Bazel handles this by walking the directory and generating a hash for it after the - // action runs, which we would have to do as well if we wanted to support these - // artifacts in mixed builds. - // - // However, there are some bazel built-in rules that use tree artifacts. Allow those, - // but keep in mind that they'll have incrementality issues. - return nil, fmt.Errorf("tree artifacts are currently not supported in mixed builds: " + artifactPath) - } - artifactIdToPath[artifactId(artifact.Id)] = artifactPath - } - - // Map middleman artifact ContentHash to input artifact depset ID. - // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, - // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then, - // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for - // that action instead. - middlemanIdToDepsetIds := map[artifactId][]uint32{} - for _, actionEntry := range aqueryResult.Actions { - if actionEntry.Mnemonic == middlemanMnemonic { - for _, outputId := range actionEntry.OutputIds { - middlemanIdToDepsetIds[artifactId(outputId)] = actionEntry.InputDepSetIds - } - } - } - - depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d *analysis_v2_proto.DepSetOfFiles) depsetId { - return depsetId(d.Id) - }) - - aqueryHandler := aqueryArtifactHandler{ - depsetIdToAqueryDepset: map[depsetId]AqueryDepset{}, - depsetHashToAqueryDepset: map[string]AqueryDepset{}, - depsetHashToArtifactPathsCache: sync.Map{}, - emptyDepsetIds: make(map[depsetId]struct{}, 0), - artifactIdToPath: artifactIdToPath, - } - - // Validate and adjust aqueryResult.DepSetOfFiles values. - for _, depset := range aqueryResult.DepSetOfFiles { - _, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset) - if err != nil { - return nil, err - } - } - - return &aqueryHandler, nil -} - -// Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given -// depset. -func (a *aqueryArtifactHandler) populateDepsetMaps(depset *analysis_v2_proto.DepSetOfFiles, middlemanIdToDepsetIds map[artifactId][]uint32, depsetIdToDepset map[depsetId]*analysis_v2_proto.DepSetOfFiles) (*AqueryDepset, error) { - if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depsetId(depset.Id)]; containsDepset { - return &aqueryDepset, nil - } - transitiveDepsetIds := depset.TransitiveDepSetIds - directArtifactPaths := make([]string, 0, len(depset.DirectArtifactIds)) - for _, id := range depset.DirectArtifactIds { - aId := artifactId(id) - path, pathExists := a.artifactIdToPath[aId] - if !pathExists { - return nil, fmt.Errorf("undefined input artifactId %d", aId) - } - // Filter out any inputs which are universally dropped, and swap middleman - // artifacts with their corresponding depsets. - if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[aId]; isMiddleman { - // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts. - transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...) - } else if strings.HasSuffix(path, py3wrapperFileName) || - strings.HasPrefix(path, "../bazel_tools") { - continue - // Drop these artifacts. - // See go/python-binary-host-mixed-build for more details. - // 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the - // TemplateExpandAction handles everything necessary to launch a Pythin application. - // 2) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the - // containing depset to always be considered newer than their outputs. - } else { - directArtifactPaths = append(directArtifactPaths, path) - } - } - - childDepsetHashes := make([]string, 0, len(transitiveDepsetIds)) - for _, id := range transitiveDepsetIds { - childDepsetId := depsetId(id) - childDepset, exists := depsetIdToDepset[childDepsetId] - if !exists { - if _, empty := a.emptyDepsetIds[childDepsetId]; empty { - continue - } else { - return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id) - } - } - if childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset); err != nil { - return nil, err - } else if childAqueryDepset == nil { - continue - } else { - childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash) - } - } - if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 { - a.emptyDepsetIds[depsetId(depset.Id)] = struct{}{} - return nil, nil - } - aqueryDepset := AqueryDepset{ - ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes), - DirectArtifacts: directArtifactPaths, - TransitiveDepSetHashes: childDepsetHashes, - } - a.depsetIdToAqueryDepset[depsetId(depset.Id)] = aqueryDepset - a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset - return &aqueryDepset, nil -} - -// getInputPaths flattens the depsets of the given IDs and returns all transitive -// input paths contained in these depsets. -// This is a potentially expensive operation, and should not be invoked except -// for actions which need specialized input handling. -func (a *aqueryArtifactHandler) getInputPaths(depsetIds []uint32) ([]string, error) { - var inputPaths []string - - for _, id := range depsetIds { - inputDepSetId := depsetId(id) - depset := a.depsetIdToAqueryDepset[inputDepSetId] - inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash) - if err != nil { - return nil, err - } - for _, inputPath := range inputArtifacts { - inputPaths = append(inputPaths, inputPath) - } - } - - return inputPaths, nil -} - -func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) { - if result, exists := a.depsetHashToArtifactPathsCache.Load(depsetHash); exists { - return result.([]string), nil - } - if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists { - result := depset.DirectArtifacts - for _, childHash := range depset.TransitiveDepSetHashes { - childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash) - if err != nil { - return nil, err - } - result = append(result, childArtifactIds...) - } - a.depsetHashToArtifactPathsCache.Store(depsetHash, result) - return result, nil - } else { - return nil, fmt.Errorf("undefined input depset hash %s", depsetHash) - } -} - -// AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset -// which should be registered (and output to a ninja file) to correspond with Bazel's -// action graph, as described by the given action graph json proto. -// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets -// are one-to-one with Bazel's depSetOfFiles objects. -func AqueryBuildStatements(aqueryJsonProto []byte, eventHandler *metrics.EventHandler) ([]*BuildStatement, []AqueryDepset, error) { - aqueryProto := &analysis_v2_proto.ActionGraphContainer{} - err := proto.Unmarshal(aqueryJsonProto, aqueryProto) - if err != nil { - return nil, nil, err - } - - var aqueryHandler *aqueryArtifactHandler - { - eventHandler.Begin("init_handler") - defer eventHandler.End("init_handler") - aqueryHandler, err = newAqueryHandler(aqueryProto) - if err != nil { - return nil, nil, err - } - } - - // allocate both length and capacity so each goroutine can write to an index independently without - // any need for synchronization for slice access. - buildStatements := make([]*BuildStatement, len(aqueryProto.Actions)) - { - eventHandler.Begin("build_statements") - defer eventHandler.End("build_statements") - wg := sync.WaitGroup{} - var errOnce sync.Once - id2targets := make(map[uint32]string, len(aqueryProto.Targets)) - for _, t := range aqueryProto.Targets { - id2targets[t.GetId()] = t.GetLabel() - } - for i, actionEntry := range aqueryProto.Actions { - wg.Add(1) - go func(i int, actionEntry *analysis_v2_proto.Action) { - if strings.HasPrefix(id2targets[actionEntry.TargetId], "@bazel_tools//") { - // bazel_tools are removed depsets in `populateDepsetMaps()` so skipping - // conversion to build statements as well - buildStatements[i] = nil - } else if buildStatement, aErr := aqueryHandler.actionToBuildStatement(actionEntry); aErr != nil { - errOnce.Do(func() { - aErr = fmt.Errorf("%s: [%s] [%s]", aErr.Error(), actionEntry.GetMnemonic(), id2targets[actionEntry.TargetId]) - err = aErr - }) - } else { - // set build statement at an index rather than appending such that each goroutine does not - // impact other goroutines - buildStatements[i] = buildStatement - } - wg.Done() - }(i, actionEntry) - } - wg.Wait() - } - if err != nil { - return nil, nil, err - } - - depsetsByHash := map[string]AqueryDepset{} - depsets := make([]AqueryDepset, 0, len(aqueryHandler.depsetIdToAqueryDepset)) - { - eventHandler.Begin("depsets") - defer eventHandler.End("depsets") - for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset { - if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey { - // Two depsets collide on hash. Ensure that their contents are identical. - if !reflect.DeepEqual(aqueryDepset, prevEntry) { - return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset) - } - } else { - depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset - depsets = append(depsets, aqueryDepset) - } - } - } - - eventHandler.Do("build_statement_sort", func() { - // Build Statements and depsets must be sorted by their content hash to - // preserve determinism between builds (this will result in consistent ninja file - // output). Note they are not sorted by their original IDs nor their Bazel ordering, - // as Bazel gives nondeterministic ordering / identifiers in aquery responses. - sort.Slice(buildStatements, func(i, j int) bool { - // Sort all nil statements to the end of the slice - if buildStatements[i] == nil { - return false - } else if buildStatements[j] == nil { - return true - } - //For build statements, compare output lists. In Bazel, each output file - // may only have one action which generates it, so this will provide - // a deterministic ordering. - outputs_i := buildStatements[i].OutputPaths - outputs_j := buildStatements[j].OutputPaths - if len(outputs_i) != len(outputs_j) { - return len(outputs_i) < len(outputs_j) - } - if len(outputs_i) == 0 { - // No outputs for these actions, so compare commands. - return buildStatements[i].Command < buildStatements[j].Command - } - // There may be multiple outputs, but the output ordering is deterministic. - return outputs_i[0] < outputs_j[0] - }) - }) - eventHandler.Do("depset_sort", func() { - sort.Slice(depsets, func(i, j int) bool { - return depsets[i].ContentHash < depsets[j].ContentHash - }) - }) - return buildStatements, depsets, nil -} - -// depsetContentHash computes and returns a SHA256 checksum of the contents of -// the given depset. This content hash may serve as the depset's identifier. -// Using a content hash for an identifier is superior for determinism. (For example, -// using an integer identifier which depends on the order in which the depsets are -// created would result in nondeterministic depset IDs.) -func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string { - h := sha256.New() - // Use newline as delimiter, as paths cannot contain newline. - h.Write([]byte(strings.Join(directPaths, "\n"))) - h.Write([]byte(strings.Join(transitiveDepsetHashes, ""))) - fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) - return fullHash -} - -func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []uint32) ([]string, error) { - var hashes []string - for _, id := range inputDepsetIds { - dId := depsetId(id) - if aqueryDepset, exists := a.depsetIdToAqueryDepset[dId]; !exists { - if _, empty := a.emptyDepsetIds[dId]; !empty { - return nil, fmt.Errorf("undefined (not even empty) input depsetId %d", dId) - } - } else { - hashes = append(hashes, aqueryDepset.ContentHash) - } - } - return hashes, nil -} - -// escapes the args received from aquery and creates a command string -func commandString(actionEntry *analysis_v2_proto.Action) string { - argsEscaped := make([]string, len(actionEntry.Arguments)) - for i, arg := range actionEntry.Arguments { - if arg == "" { - // If this is an empty string, add '' - // And not - // 1. (literal empty) - // 2. `''\'''\'''` (escaped version of '') - // - // If we had used (1), then this would appear as a whitespace when we strings.Join - argsEscaped[i] = "''" - } else { - argsEscaped[i] = proptools.ShellEscapeIncludingSpaces(arg) - } - } - return strings.Join(argsEscaped, " ") -} - -func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - command := commandString(actionEntry) - inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputDepsetHashes: inputDepsetHashes, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } - if buildStatement.Mnemonic == "GoToolchainBinaryBuild" { - // Unlike b's execution root, mixed build execution root contains a symlink to prebuilts/go - // This causes issues for `GOCACHE=$(mktemp -d) go build ...` - // To prevent this, sandbox this action in mixed builds as well - buildStatement.ShouldRunInSbox = true - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - if len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) - } - expandedTemplateContent := expandTemplateContent(actionEntry) - // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, - // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might - // change \n to space and mess up the format of Python programs. - // sed is used to convert \\n back to \n before saving to output file. - // See go/python-binary-host-mixed-build for more details. - command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, - escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) - inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputDepsetHashes: inputDepsetHashes, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, _, err := a.getOutputPaths(actionEntry) - var depsetHashes []string - if err == nil { - depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds) - } - if err != nil { - return nil, err - } - return &BuildStatement{ - Depfile: nil, - OutputPaths: outputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - InputDepsetHashes: depsetHashes, - FileContents: actionEntry.FileContents, - IsExecutable: actionEntry.IsExecutable, - }, nil -} - -func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, _, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - if len(inputPaths) != 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) - } - // The actual command is generated in bazelSingleton.GenerateBuildActions - return &BuildStatement{ - Depfile: nil, - OutputPaths: outputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - InputPaths: inputPaths, - }, nil -} - -type bazelSandwichJson struct { - Target string `json:"target"` - DependOnTarget *bool `json:"depend_on_target,omitempty"` - ImplicitDeps []string `json:"implicit_deps"` -} - -func (a *aqueryArtifactHandler) unresolvedSymlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - if len(actionEntry.InputDepSetIds) != 0 || len(outputPaths) != 1 { - return nil, fmt.Errorf("expected 0 inputs and 1 output to symlink action, got: input %q, output %q", actionEntry.InputDepSetIds, outputPaths) - } - target := actionEntry.UnresolvedSymlinkTarget - if target == "" { - return nil, fmt.Errorf("expected an unresolved_symlink_target, but didn't get one") - } - if filepath.Clean(target) != target { - return nil, fmt.Errorf("expected %q, got %q", filepath.Clean(target), target) - } - if strings.HasPrefix(target, "/") { - return nil, fmt.Errorf("no absolute symlinks allowed: %s", target) - } - - out := outputPaths[0] - outDir := filepath.Dir(out) - var implicitDeps []string - if strings.HasPrefix(target, "bazel_sandwich:") { - j := bazelSandwichJson{} - err := json.Unmarshal([]byte(target[len("bazel_sandwich:"):]), &j) - if err != nil { - return nil, err - } - if proptools.BoolDefault(j.DependOnTarget, true) { - implicitDeps = append(implicitDeps, j.Target) - } - implicitDeps = append(implicitDeps, j.ImplicitDeps...) - dotDotsToReachCwd := "" - if outDir != "." { - dotDotsToReachCwd = strings.Repeat("../", strings.Count(outDir, "/")+1) - } - target = proptools.ShellEscapeIncludingSpaces(j.Target) - target = "{DOTDOTS_TO_OUTPUT_ROOT}" + dotDotsToReachCwd + target - } else { - target = proptools.ShellEscapeIncludingSpaces(target) - } - - outDir = proptools.ShellEscapeIncludingSpaces(outDir) - out = proptools.ShellEscapeIncludingSpaces(out) - // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). - command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, target) - symlinkPaths := outputPaths[:] - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - SymlinkPaths: symlinkPaths, - ImplicitDeps: implicitDeps, - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - - inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - if len(inputPaths) != 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) - } - out := outputPaths[0] - outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) - out = proptools.ShellEscapeIncludingSpaces(out) - in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) - // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). - command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) - symlinkPaths := outputPaths[:] - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputPaths: inputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - SymlinkPaths: symlinkPaths, - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) getOutputPaths(actionEntry *analysis_v2_proto.Action) (outputPaths []string, depfile *string, err error) { - for _, outputId := range actionEntry.OutputIds { - outputPath, exists := a.artifactIdToPath[artifactId(outputId)] - if !exists { - err = fmt.Errorf("undefined outputId %d", outputId) - return - } - ext := filepath.Ext(outputPath) - if ext == ".d" { - if depfile != nil { - err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) - return - } else { - depfile = &outputPath - } - } else { - outputPaths = append(outputPaths, outputPath) - } - } - return -} - -// expandTemplateContent substitutes the tokens in a template. -func expandTemplateContent(actionEntry *analysis_v2_proto.Action) string { - replacerString := make([]string, len(actionEntry.Substitutions)*2) - for i, pair := range actionEntry.Substitutions { - value := pair.Value - if val, ok := templateActionOverriddenTokens[pair.Key]; ok { - value = val - } - replacerString[i*2] = pair.Key - replacerString[i*2+1] = value - } - replacer := strings.NewReplacer(replacerString...) - return replacer.Replace(actionEntry.TemplateContent) -} - -// \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"' -var commandLineArgumentReplacer = strings.NewReplacer( - `\`, `\\`, - `$`, `\$`, - "`", "\\`", - `"`, `\"`, - "\n", "\\n", - `'`, `'"'"'`, -) - -func escapeCommandlineArgument(str string) string { - return commandLineArgumentReplacer.Replace(str) -} - -func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - switch actionEntry.Mnemonic { - // Middleman actions are not handled like other actions; they are handled separately as a - // preparatory step so that their inputs may be relayed to actions depending on middleman - // artifacts. - case middlemanMnemonic: - return nil, nil - // PythonZipper is bogus action returned by aquery, ignore it (b/236198693) - case "PythonZipper": - return nil, nil - // Skip "Fail" actions, which are placeholder actions designed to always fail. - case "Fail": - return nil, nil - case "BaselineCoverage": - return nil, nil - case "Symlink", "SolibSymlink", "ExecutableSymlink": - return a.symlinkActionBuildStatement(actionEntry) - case "TemplateExpand": - if len(actionEntry.Arguments) < 1 { - return a.templateExpandActionBuildStatement(actionEntry) - } - case "FileWrite", "SourceSymlinkManifest", "RepoMappingManifest": - return a.fileWriteActionBuildStatement(actionEntry) - case "SymlinkTree": - return a.symlinkTreeActionBuildStatement(actionEntry) - case "UnresolvedSymlink": - return a.unresolvedSymlinkActionBuildStatement(actionEntry) - } - - if len(actionEntry.Arguments) < 1 { - return nil, errors.New("received action with no command") - } - return a.normalActionBuildStatement(actionEntry) - -} - -func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]*analysis_v2_proto.PathFragment) (string, error) { - var labels []string - currId := id - // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node. - for currId > 0 { - currFragment, ok := pathFragmentsMap[currId] - if !ok { - return "", fmt.Errorf("undefined path fragment id %d", currId) - } - labels = append([]string{currFragment.Label}, labels...) - parentId := pathFragmentId(currFragment.ParentId) - if currId == parentId { - return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment) - } - currId = parentId - } - return filepath.Join(labels...), nil -} diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go deleted file mode 100644 index cbd27919c..000000000 --- a/bazel/aquery_test.go +++ /dev/null @@ -1,1411 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "encoding/json" - "fmt" - "reflect" - "sort" - "testing" - - analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" - - "github.com/google/blueprint/metrics" - "google.golang.org/protobuf/proto" -) - -func TestAqueryMultiArchGenrule(t *testing.T) { - // This input string is retrieved from a real build of bionic-related genrules. - const inputString = ` -{ - "Artifacts": [ - { "Id": 1, "path_fragment_id": 1 }, - { "Id": 2, "path_fragment_id": 6 }, - { "Id": 3, "path_fragment_id": 8 }, - { "Id": 4, "path_fragment_id": 12 }, - { "Id": 5, "path_fragment_id": 19 }, - { "Id": 6, "path_fragment_id": 20 }, - { "Id": 7, "path_fragment_id": 21 }], - "Actions": [{ - "target_id": 1, - "action_key": "ab53f6ecbdc2ee8cb8812613b63205464f1f5083f6dca87081a0a398c0f1ecf7", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [1], - "output_ids": [4], - "primary_output_id": 4 - }, { - "target_id": 2, - "action_key": "9f4309ce165dac458498cb92811c18b0b7919782cc37b82a42d2141b8cc90826", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [2], - "output_ids": [5], - "primary_output_id": 5 - }, { - "target_id": 3, - "action_key": "50d6c586103ebeed3a218195540bcc30d329464eae36377eb82f8ce7c36ac342", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86_64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86_64.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [3], - "output_ids": [6], - "primary_output_id": 6 - }, { - "target_id": 4, - "action_key": "f30cbe442f5216f4223cf16a39112cad4ec56f31f49290d85cff587e48647ffa", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm64.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [4], - "output_ids": [7], - "primary_output_id": 7 - }], - "Targets": [ - { "Id": 1, "Label": "@sourceroot//bionic/libc:syscalls-arm", "rule_class_id": 1 }, - { "Id": 2, "Label": "@sourceroot//bionic/libc:syscalls-x86", "rule_class_id": 1 }, - { "Id": 3, "Label": "@sourceroot//bionic/libc:syscalls-x86_64", "rule_class_id": 1 }, - { "Id": 4, "Label": "@sourceroot//bionic/libc:syscalls-arm64", "rule_class_id": 1 }], - "dep_set_of_files": [ - { "Id": 1, "direct_artifact_ids": [1, 2, 3] }, - { "Id": 2, "direct_artifact_ids": [1, 2, 3] }, - { "Id": 3, "direct_artifact_ids": [1, 2, 3] }, - { "Id": 4, "direct_artifact_ids": [1, 2, 3] }], - "Configuration": [{ - "Id": 1, - "Mnemonic": "k8-fastbuild", - "platform_name": "k8", - "Checksum": "485c362832c178e367d972177f68e69e0981e51e67ef1c160944473db53fe046" - }], - "rule_classes": [{ "Id": 1, "Name": "genrule"}], - "path_fragments": [ - { "Id": 5, "Label": ".." }, - { "Id": 4, "Label": "sourceroot", "parent_id": 5 }, - { "Id": 3, "Label": "bionic", "parent_id": 4 }, - { "Id": 2, "Label": "libc", "parent_id": 3 }, - { "Id": 1, "Label": "SYSCALLS.TXT", "parent_id": 2 }, - { "Id": 7, "Label": "tools", "parent_id": 2 }, - { "Id": 6, "Label": "gensyscalls.py", "parent_id": 7 }, - { "Id": 11, "Label": "bazel_tools", "parent_id": 5 }, - { "Id": 10, "Label": "tools", "parent_id": 11 }, - { "Id": 9, "Label": "genrule", "parent_id": 10 }, - { "Id": 8, "Label": "genrule-setup.sh", "parent_id": 9 }, - { "Id": 18, "Label": "bazel-out" }, - { "Id": 17, "Label": "sourceroot", "parent_id": 18 }, - { "Id": 16, "Label": "k8-fastbuild", "parent_id": 17 }, - { "Id": 15, "Label": "bin", "parent_id": 16 }, - { "Id": 14, "Label": "bionic", "parent_id": 15 }, - { "Id": 13, "Label": "libc", "parent_id": 14 }, - { "Id": 12, "Label": "syscalls-arm.S", "parent_id": 13 }, - { "Id": 19, "Label": "syscalls-x86.S", "parent_id": 13 }, - { "Id": 20, "Label": "syscalls-x86_64.S", "parent_id": 13 }, - { "Id": 21, "Label": "syscalls-arm64.S", "parent_id": 13 }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - var expectedBuildStatements []*BuildStatement - for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} { - expectedBuildStatements = append(expectedBuildStatements, - &BuildStatement{ - Command: fmt.Sprintf( - "/bin/bash -c 'source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py %s ../sourceroot/bionic/libc/SYSCALLS.TXT > bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S'", - arch, arch), - OutputPaths: []string{ - fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch), - }, - Env: []*analysis_v2_proto.KeyValuePair{ - {Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, - }, - Mnemonic: "Genrule", - }) - } - assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) - - expectedFlattenedInputs := []string{ - "../sourceroot/bionic/libc/SYSCALLS.TXT", - "../sourceroot/bionic/libc/tools/gensyscalls.py", - } - // In this example, each depset should have the same expected inputs. - for _, actualDepset := range actualDepsets { - actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) - if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { - t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) - } - } -} - -func TestInvalidOutputId(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "X", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [3], - "primary_output_id": 3 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined outputId 3: [X] []") -} - -func TestInvalidInputDepsetIdFromAction(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "X", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [2], - "output_ids": [1], - "primary_output_id": 1 - }], - "targets": [{ - "id": 1, - "label": "target_x" - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined (not even empty) input depsetId 2: [X] [target_x]") -} - -func TestInvalidInputDepsetIdFromDepset(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2], "transitive_dep_set_ids": [42] }], - "path_fragments": [ - { "id": 1, "label": "one"}, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)") -} - -func TestInvalidInputArtifactId(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 3] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined input artifactId 3") -} - -func TestInvalidPathFragmentId(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two", "parent_id": 3 }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined path fragment id 3") -} - -func TestDepfiles(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }], - "actions": [{ - "target_Id": 1, - "action_Key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [2, 3], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_Artifact_Ids": [1, 2, 3] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }, - { "id": 3, "label": "two.d" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - if expected := 1; len(actual) != expected { - t.Fatalf("Expected %d build statements, got %d", expected, len(actual)) - return - } - - bs := actual[0] - expectedDepfile := "two.d" - if bs.Depfile == nil { - t.Errorf("Expected depfile %q, but there was none found", expectedDepfile) - } else if *bs.Depfile != expectedDepfile { - t.Errorf("Expected depfile %q, but got %q", expectedDepfile, *bs.Depfile) - } -} - -func TestMultipleDepfiles(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }, - { "id": 4, "path_fragment_id": 4 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "X", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [2,3,4], - "primary_output_id": 2 - }], - "dep_set_of_files": [{ - "id": 1, - "direct_artifact_ids": [1, 2, 3, 4] - }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }, - { "id": 3, "label": "two.d" }, - { "id": 4, "label": "other.d" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, `found multiple potential depfiles "two.d", "other.d": [X] []`) -} - -func TestTransitiveInputDepsets(t *testing.T) { - // The input aquery for this test comes from a proof-of-concept starlark rule which registers - // a single action with many inputs given via a deep depset. - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 7 }, - { "id": 3, "path_fragment_id": 8 }, - { "id": 4, "path_fragment_id": 9 }, - { "id": 5, "path_fragment_id": 10 }, - { "id": 6, "path_fragment_id": 11 }, - { "id": 7, "path_fragment_id": 12 }, - { "id": 8, "path_fragment_id": 13 }, - { "id": 9, "path_fragment_id": 14 }, - { "id": 10, "path_fragment_id": 15 }, - { "id": 11, "path_fragment_id": 16 }, - { "id": 12, "path_fragment_id": 17 }, - { "id": 13, "path_fragment_id": 18 }, - { "id": 14, "path_fragment_id": 19 }, - { "id": 15, "path_fragment_id": 20 }, - { "id": 16, "path_fragment_id": 21 }, - { "id": 17, "path_fragment_id": 22 }, - { "id": 18, "path_fragment_id": 23 }, - { "id": 19, "path_fragment_id": 24 }, - { "id": 20, "path_fragment_id": 25 }, - { "id": 21, "path_fragment_id": 26 }], - "actions": [{ - "target_id": 1, - "action_key": "3b826d17fadbbbcd8313e456b90ec47c078c438088891dd45b4adbcd8889dc50", - "mnemonic": "Action", - "configuration_id": 1, - "arguments": ["/bin/bash", "-c", "touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"], - "input_dep_set_ids": [1], - "output_ids": [21], - "primary_output_id": 21 - }], - "dep_set_of_files": [ - { "id": 3, "direct_artifact_ids": [1, 2, 3, 4, 5] }, - { "id": 4, "direct_artifact_ids": [6, 7, 8, 9, 10] }, - { "id": 2, "transitive_dep_set_ids": [3, 4], "direct_artifact_ids": [11, 12, 13, 14, 15] }, - { "id": 5, "direct_artifact_ids": [16, 17, 18, 19] }, - { "id": 1, "transitive_dep_set_ids": [2, 5], "direct_artifact_ids": [20] }], - "path_fragments": [ - { "id": 6, "label": "bazel-out" }, - { "id": 5, "label": "sourceroot", "parent_id": 6 }, - { "id": 4, "label": "k8-fastbuild", "parent_id": 5 }, - { "id": 3, "label": "bin", "parent_id": 4 }, - { "id": 2, "label": "testpkg", "parent_id": 3 }, - { "id": 1, "label": "test_1", "parent_id": 2 }, - { "id": 7, "label": "test_2", "parent_id": 2 }, - { "id": 8, "label": "test_3", "parent_id": 2 }, - { "id": 9, "label": "test_4", "parent_id": 2 }, - { "id": 10, "label": "test_5", "parent_id": 2 }, - { "id": 11, "label": "test_6", "parent_id": 2 }, - { "id": 12, "label": "test_7", "parent_id": 2 }, - { "id": 13, "label": "test_8", "parent_id": 2 }, - { "id": 14, "label": "test_9", "parent_id": 2 }, - { "id": 15, "label": "test_10", "parent_id": 2 }, - { "id": 16, "label": "test_11", "parent_id": 2 }, - { "id": 17, "label": "test_12", "parent_id": 2 }, - { "id": 18, "label": "test_13", "parent_id": 2 }, - { "id": 19, "label": "test_14", "parent_id": 2 }, - { "id": 20, "label": "test_15", "parent_id": 2 }, - { "id": 21, "label": "test_16", "parent_id": 2 }, - { "id": 22, "label": "test_17", "parent_id": 2 }, - { "id": 23, "label": "test_18", "parent_id": 2 }, - { "id": 24, "label": "test_19", "parent_id": 2 }, - { "id": 25, "label": "test_root", "parent_id": 2 }, - { "id": 26,"label": "test_out", "parent_id": 2 }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'", - OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, - Mnemonic: "Action", - SymlinkPaths: []string{}, - }, - } - assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) - - // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs - // are given via a deep depset, but the depset is flattened when returned as a - // BuildStatement slice. - var expectedFlattenedInputs []string - for i := 1; i < 20; i++ { - expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) - } - expectedFlattenedInputs = append(expectedFlattenedInputs, "bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root") - - actualDepsetHashes := actualbuildStatements[0].InputDepsetHashes - actualFlattenedInputs := flattenDepsets(actualDepsetHashes, actualDepsets) - if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { - t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) - } -} - -func TestSymlinkTree(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "SymlinkTree", - "configuration_id": 1, - "input_dep_set_ids": [1], - "output_ids": [2], - "primary_output_id": 2, - "execution_platform": "//build/bazel/platforms:linux_x86_64" - }], - "path_fragments": [ - { "id": 1, "label": "foo.manifest" }, - { "id": 2, "label": "foo.runfiles/MANIFEST" }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{ - &BuildStatement{ - Command: "", - OutputPaths: []string{"foo.runfiles/MANIFEST"}, - Mnemonic: "SymlinkTree", - InputPaths: []string{"foo.manifest"}, - SymlinkPaths: []string{}, - }, - }, actual) -} - -func TestBazelToolsRemovalFromInputDepsets(t *testing.T) { - const inputString = `{ - "artifacts": [ - { "id": 1, "path_fragment_id": 10 }, - { "id": 2, "path_fragment_id": 20 }, - { "id": 3, "path_fragment_id": 30 }, - { "id": 4, "path_fragment_id": 40 }], - "dep_set_of_files": [{ - "id": 1111, - "direct_artifact_ids": [3 , 4] - }, { - "id": 2222, - "direct_artifact_ids": [3] - }], - "actions": [{ - "target_id": 100, - "action_key": "x", - "input_dep_set_ids": [1111, 2222], - "mnemonic": "x", - "arguments": ["bogus", "command"], - "output_ids": [2], - "primary_output_id": 1 - }], - "path_fragments": [ - { "id": 10, "label": "input" }, - { "id": 20, "label": "output" }, - { "id": 30, "label": "dep1", "parent_id": 50 }, - { "id": 40, "label": "dep2", "parent_id": 60 }, - { "id": 50, "label": "bazel_tools", "parent_id": 60 }, - { "id": 60, "label": ".."} - ] -}` - /* depsets - 1111 2222 - / \ | - ../dep2 ../bazel_tools/dep1 - */ - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - if len(actualDepsets) != 1 { - t.Errorf("expected 1 depset but found %#v", actualDepsets) - return - } - dep2Found := false - for _, dep := range flattenDepsets([]string{actualDepsets[0].ContentHash}, actualDepsets) { - if dep == "../bazel_tools/dep1" { - t.Errorf("dependency %s expected to be removed but still exists", dep) - } else if dep == "../dep2" { - dep2Found = true - } - } - if !dep2Found { - t.Errorf("dependency ../dep2 expected but not found") - } - - expectedBuildStatement := &BuildStatement{ - Command: "bogus command", - OutputPaths: []string{"output"}, - Mnemonic: "x", - SymlinkPaths: []string{}, - } - buildStatementFound := false - for _, actualBuildStatement := range actualBuildStatements { - if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { - buildStatementFound = true - break - } - } - if !buildStatementFound { - t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements) - return - } -} - -func TestBazelToolsRemovalFromTargets(t *testing.T) { - const inputString = `{ - "artifacts": [{ "id": 1, "path_fragment_id": 10 }], - "targets": [ - { "id": 100, "label": "targetX" }, - { "id": 200, "label": "@bazel_tools//tool_y" } -], - "actions": [{ - "target_id": 100, - "action_key": "actionX", - "arguments": ["bogus", "command"], - "mnemonic" : "x", - "output_ids": [1] - }, { - "target_id": 200, - "action_key": "y" - }], - "path_fragments": [{ "id": 10, "label": "outputX"}] -}` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - if len(actualDepsets) != 0 { - t.Errorf("expected 0 depset but found %#v", actualDepsets) - return - } - expectedBuildStatement := &BuildStatement{ - Command: "bogus command", - OutputPaths: []string{"outputX"}, - Mnemonic: "x", - SymlinkPaths: []string{}, - } - buildStatementFound := false - for _, actualBuildStatement := range actualBuildStatements { - if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { - buildStatementFound = true - break - } - } - if !buildStatementFound { - t.Errorf("expected but missing %#v in %#v build statements", expectedBuildStatement, len(actualBuildStatements)) - return - } -} - -func TestBazelToolsRemovalFromTransitiveInputDepsets(t *testing.T) { - const inputString = `{ - "artifacts": [ - { "id": 1, "path_fragment_id": 10 }, - { "id": 2, "path_fragment_id": 20 }, - { "id": 3, "path_fragment_id": 30 }], - "dep_set_of_files": [{ - "id": 1111, - "transitive_dep_set_ids": [2222] - }, { - "id": 2222, - "direct_artifact_ids": [3] - }, { - "id": 3333, - "direct_artifact_ids": [3] - }, { - "id": 4444, - "transitive_dep_set_ids": [3333] - }], - "actions": [{ - "target_id": 100, - "action_key": "x", - "input_dep_set_ids": [1111, 4444], - "mnemonic": "x", - "arguments": ["bogus", "command"], - "output_ids": [2], - "primary_output_id": 1 - }], - "path_fragments": [ - { "id": 10, "label": "input" }, - { "id": 20, "label": "output" }, - { "id": 30, "label": "dep", "parent_id": 50 }, - { "id": 50, "label": "bazel_tools", "parent_id": 60 }, - { "id": 60, "label": ".."} - ] -}` - /* depsets - 1111 4444 - || || - 2222 3333 - | | - ../bazel_tools/dep - Note: in dep_set_of_files: - 1111 appears BEFORE its dependency,2222 while - 4444 appears AFTER its dependency 3333 - and this test shows that that order doesn't affect empty depset pruning - */ - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - if len(actualDepsets) != 0 { - t.Errorf("expected 0 depsets but found %#v", actualDepsets) - return - } - - expectedBuildStatement := &BuildStatement{ - Command: "bogus command", - OutputPaths: []string{"output"}, - Mnemonic: "x", - } - buildStatementFound := false - for _, actualBuildStatement := range actualBuildStatements { - if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { - buildStatementFound = true - break - } - } - if !buildStatementFound { - t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements) - return - } -} - -func TestMiddlemenAction(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }, - { "id": 4, "path_fragment_id": 4 }, - { "id": 5, "path_fragment_id": 5 }, - { "id": 6, "path_fragment_id": 6 }], - "path_fragments": [ - { "id": 1, "label": "middleinput_one" }, - { "id": 2, "label": "middleinput_two" }, - { "id": 3, "label": "middleman_artifact" }, - { "id": 4, "label": "maininput_one" }, - { "id": 5, "label": "maininput_two" }, - { "id": 6, "label": "output" }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }, - { "id": 2, "direct_artifact_ids": [3, 4, 5] }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "Middleman", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [3], - "primary_output_id": 3 - }, { - "target_id": 2, - "action_key": "y", - "mnemonic": "Main action", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [2], - "output_ids": [6], - "primary_output_id": 6 - }] -}` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - if expected := 2; len(actualBuildStatements) != expected { - t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements) - return - } - - expectedDepsetFiles := [][]string{ - {"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"}, - {"middleinput_one", "middleinput_two"}, - } - assertFlattenedDepsets(t, actualDepsets, expectedDepsetFiles) - - bs := actualBuildStatements[0] - if len(bs.InputPaths) > 0 { - t.Errorf("Expected main action raw inputs to be empty, but got %q", bs.InputPaths) - } - - expectedOutputs := []string{"output"} - if !reflect.DeepEqual(bs.OutputPaths, expectedOutputs) { - t.Errorf("Expected main action outputs %q, but got %q", expectedOutputs, bs.OutputPaths) - } - - expectedFlattenedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"} - actualFlattenedInputs := flattenDepsets(bs.InputDepsetHashes, actualDepsets) - - if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { - t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) - } - - bs = actualBuildStatements[1] - if bs != nil { - t.Errorf("Expected nil action for skipped") - } -} - -// Returns the contents of given depsets in concatenated post order. -func flattenDepsets(depsetHashesToFlatten []string, allDepsets []AqueryDepset) []string { - depsetsByHash := map[string]AqueryDepset{} - for _, depset := range allDepsets { - depsetsByHash[depset.ContentHash] = depset - } - var result []string - for _, depsetId := range depsetHashesToFlatten { - result = append(result, flattenDepset(depsetId, depsetsByHash)...) - } - return result -} - -// Returns the contents of a given depset in post order. -func flattenDepset(depsetHashToFlatten string, allDepsets map[string]AqueryDepset) []string { - depset := allDepsets[depsetHashToFlatten] - var result []string - for _, depsetId := range depset.TransitiveDepSetHashes { - result = append(result, flattenDepset(depsetId, allDepsets)...) - } - result = append(result, depset.DirectArtifacts...) - return result -} - -func assertFlattenedDepsets(t *testing.T, actualDepsets []AqueryDepset, expectedDepsetFiles [][]string) { - t.Helper() - if len(actualDepsets) != len(expectedDepsetFiles) { - t.Errorf("Expected %d depsets, but got %d depsets", len(expectedDepsetFiles), len(actualDepsets)) - } - for i, actualDepset := range actualDepsets { - actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) - if !reflect.DeepEqual(actualFlattenedInputs, expectedDepsetFiles[i]) { - t.Errorf("Expected depset files: %v, but got %v", expectedDepsetFiles[i], actualFlattenedInputs) - } - } -} - -func TestSimpleSymlink(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 3 }, - { "id": 2, "path_fragment_id": 5 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "Symlink", - "input_dep_set_ids": [1], - "output_ids": [2], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "file_subdir", "parent_id": 1 }, - { "id": 3, "label": "file", "parent_id": 2 }, - { "id": 4, "label": "symlink_subdir", "parent_id": 1 }, - { "id": 5, "label": "symlink", "parent_id": 4 }] -}` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "mkdir -p one/symlink_subdir && " + - "rm -f one/symlink_subdir/symlink && " + - "ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink", - InputPaths: []string{"one/file_subdir/file"}, - OutputPaths: []string{"one/symlink_subdir/symlink"}, - SymlinkPaths: []string{"one/symlink_subdir/symlink"}, - Mnemonic: "Symlink", - }, - } - assertBuildStatements(t, actual, expectedBuildStatements) -} - -func TestSymlinkQuotesPaths(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 3 }, - { "id": 2, "path_fragment_id": 5 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "SolibSymlink", - "input_dep_set_ids": [1], - "output_ids": [2], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "file subdir", "parent_id": 1 }, - { "id": 3, "label": "file", "parent_id": 2 }, - { "id": 4, "label": "symlink subdir", "parent_id": 1 }, - { "id": 5, "label": "symlink", "parent_id": 4 }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "mkdir -p 'one/symlink subdir' && " + - "rm -f 'one/symlink subdir/symlink' && " + - "ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'", - InputPaths: []string{"one/file subdir/file"}, - OutputPaths: []string{"one/symlink subdir/symlink"}, - SymlinkPaths: []string{"one/symlink subdir/symlink"}, - Mnemonic: "SolibSymlink", - }, - } - assertBuildStatements(t, expectedBuildStatements, actual) -} - -func TestSymlinkMultipleInputs(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "Symlink", - "input_dep_set_ids": [1], - "output_ids": [3], - "primary_output_id": 3 - }], - "dep_set_of_files": [{ "id": 1, "direct_artifact_ids": [1,2] }], - "path_fragments": [ - { "id": 1, "label": "file" }, - { "id": 2, "label": "other_file" }, - { "id": 3, "label": "symlink" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]: [Symlink] []`) -} - -func TestSymlinkMultipleOutputs(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 3, "path_fragment_id": 3 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "Symlink", - "input_dep_set_ids": [1], - "output_ids": [2,3], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }], - "path_fragments": [ - { "id": 1, "label": "file" }, - { "id": 2, "label": "symlink" }, - { "id": 3, "label": "other_symlink" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined outputId 2: [Symlink] []") -} - -func TestTemplateExpandActionSubstitutions(t *testing.T) { - const inputString = ` -{ - "artifacts": [{ - "id": 1, - "path_fragment_id": 1 - }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "TemplateExpand", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "template_content": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [ - { "key": "%token1%", "value": "abcd" }, - { "key": "%python_binary%", "value": "python3" }] - }], - "path_fragments": [ - { "id": 1, "label": "template_file" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " + - "chmod a+x template_file'", - OutputPaths: []string{"template_file"}, - Mnemonic: "TemplateExpand", - SymlinkPaths: []string{}, - }, - } - assertBuildStatements(t, expectedBuildStatements, actual) -} - -func TestTemplateExpandActionNoOutput(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "TemplateExpand", - "configuration_id": 1, - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "templateContent": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [ - { "key": "%token1%", "value": "abcd" }, - { "key": "%python_binary%", "value": "python3" }] - }], - "path_fragments": [ - { "id": 1, "label": "template_file" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, `Expect 1 output to template expand action, got: output []: [TemplateExpand] []`) -} - -func TestFileWrite(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "FileWrite", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "file_contents": "file data\n" - }], - "path_fragments": [ - { "id": 1, "label": "foo.manifest" }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{ - &BuildStatement{ - OutputPaths: []string{"foo.manifest"}, - Mnemonic: "FileWrite", - FileContents: "file data\n", - SymlinkPaths: []string{}, - }, - }, actual) -} - -func TestSourceSymlinkManifest(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "SourceSymlinkManifest", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "file_contents": "symlink target\n" - }], - "path_fragments": [ - { "id": 1, "label": "foo.manifest" }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{ - &BuildStatement{ - OutputPaths: []string{"foo.manifest"}, - Mnemonic: "SourceSymlinkManifest", - SymlinkPaths: []string{}, - }, - }, actual) -} - -func TestUnresolvedSymlink(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 } - ], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "UnresolvedSymlink", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "unresolved_symlink_target": "symlink/target" - }], - "path_fragments": [ - { "id": 1, "label": "path/to/symlink" } - ] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{{ - Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf symlink/target path/to/symlink", - OutputPaths: []string{"path/to/symlink"}, - Mnemonic: "UnresolvedSymlink", - SymlinkPaths: []string{"path/to/symlink"}, - }}, actual) -} - -func TestUnresolvedSymlinkBazelSandwich(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 } - ], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "UnresolvedSymlink", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "unresolved_symlink_target": "bazel_sandwich:{\"target\":\"target/product/emulator_x86_64/system\"}" - }], - "path_fragments": [ - { "id": 1, "label": "path/to/symlink" } - ] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{{ - Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink", - OutputPaths: []string{"path/to/symlink"}, - Mnemonic: "UnresolvedSymlink", - SymlinkPaths: []string{"path/to/symlink"}, - ImplicitDeps: []string{"target/product/emulator_x86_64/system"}, - }}, actual) -} - -func TestUnresolvedSymlinkBazelSandwichWithAlternativeDeps(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 } - ], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "UnresolvedSymlink", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "unresolved_symlink_target": "bazel_sandwich:{\"depend_on_target\":false,\"implicit_deps\":[\"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp\"],\"target\":\"target/product/emulator_x86_64/system\"}" - }], - "path_fragments": [ - { "id": 1, "label": "path/to/symlink" } - ] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{{ - Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink", - OutputPaths: []string{"path/to/symlink"}, - Mnemonic: "UnresolvedSymlink", - SymlinkPaths: []string{"path/to/symlink"}, - // Note that the target of the symlink, target/product/emulator_x86_64/system, is not listed here - ImplicitDeps: []string{"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp"}, - }}, actual) -} - -func assertError(t *testing.T, err error, expected string) { - t.Helper() - if err == nil { - t.Errorf("expected error '%s', but got no error", expected) - } else if err.Error() != expected { - t.Errorf("expected error:\n\t'%s', but got:\n\t'%s'", expected, err.Error()) - } -} - -// Asserts that the given actual build statements match the given expected build statements. -// Build statement equivalence is determined using buildStatementEquals. -func assertBuildStatements(t *testing.T, expected []*BuildStatement, actual []*BuildStatement) { - t.Helper() - if len(expected) != len(actual) { - t.Errorf("expected %d build statements, but got %d,\n expected: %#v,\n actual: %#v", - len(expected), len(actual), expected, actual) - return - } - type compareFn = func(i int, j int) bool - byCommand := func(slice []*BuildStatement) compareFn { - return func(i int, j int) bool { - if slice[i] == nil { - return false - } else if slice[j] == nil { - return false - } - return slice[i].Command < slice[j].Command - } - } - sort.SliceStable(expected, byCommand(expected)) - sort.SliceStable(actual, byCommand(actual)) - for i, actualStatement := range actual { - expectedStatement := expected[i] - if differingField := buildStatementEquals(actualStatement, expectedStatement); differingField != "" { - t.Errorf("%s differs\nunexpected build statement %#v.\nexpected: %#v", - differingField, actualStatement, expectedStatement) - return - } - } -} - -func buildStatementEquals(first *BuildStatement, second *BuildStatement) string { - if (first == nil) != (second == nil) { - return "Nil" - } - if first.Mnemonic != second.Mnemonic { - return "Mnemonic" - } - if first.Command != second.Command { - return "Command" - } - // Ordering is significant for environment variables. - if !reflect.DeepEqual(first.Env, second.Env) { - return "Env" - } - // Ordering is irrelevant for input and output paths, so compare sets. - if !reflect.DeepEqual(sortedStrings(first.InputPaths), sortedStrings(second.InputPaths)) { - return "InputPaths" - } - if !reflect.DeepEqual(sortedStrings(first.OutputPaths), sortedStrings(second.OutputPaths)) { - return "OutputPaths" - } - if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) { - return "SymlinkPaths" - } - if !reflect.DeepEqual(sortedStrings(first.ImplicitDeps), sortedStrings(second.ImplicitDeps)) { - return "ImplicitDeps" - } - if first.Depfile != second.Depfile { - return "Depfile" - } - return "" -} - -func sortedStrings(stringSlice []string) []string { - sorted := make([]string, len(stringSlice)) - copy(sorted, stringSlice) - sort.Strings(sorted) - return sorted -} - -// Transform the json format to ActionGraphContainer -func JsonToActionGraphContainer(inputString string) ([]byte, error) { - var aqueryProtoResult analysis_v2_proto.ActionGraphContainer - err := json.Unmarshal([]byte(inputString), &aqueryProtoResult) - if err != nil { - return []byte(""), err - } - data, _ := proto.Marshal(&aqueryProtoResult) - return data, err -} diff --git a/bazel/bazel_proxy.go b/bazel/bazel_proxy.go deleted file mode 100644 index 229818da0..000000000 --- a/bazel/bazel_proxy.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2023 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "bytes" - "encoding/gob" - "fmt" - "net" - os_lib "os" - "os/exec" - "path/filepath" - "strings" - "time" -) - -// Logs events of ProxyServer. -type ServerLogger interface { - Fatal(v ...interface{}) - Fatalf(format string, v ...interface{}) - Println(v ...interface{}) -} - -// CmdRequest is a request to the Bazel Proxy server. -type CmdRequest struct { - // Args to the Bazel command. - Argv []string - // Environment variables to pass to the Bazel invocation. Strings should be of - // the form "KEY=VALUE". - Env []string -} - -// CmdResponse is a response from the Bazel Proxy server. -type CmdResponse struct { - Stdout string - Stderr string - ErrorString string -} - -// ProxyClient is a client which can issue Bazel commands to the Bazel -// proxy server. Requests are issued (and responses received) via a unix socket. -// See ProxyServer for more details. -type ProxyClient struct { - outDir string -} - -// ProxyServer is a server which runs as a background goroutine. Each -// request to the server describes a Bazel command which the server should run. -// The server then issues the Bazel command, and returns a response describing -// the stdout/stderr of the command. -// Client-server communication is done via a unix socket under the output -// directory. -// The server is intended to circumvent sandboxing for subprocesses of the -// build. The build orchestrator (soong_ui) can launch a server to exist outside -// of sandboxing, and sandboxed processes (such as soong_build) can issue -// bazel commands through this socket tunnel. This allows a sandboxed process -// to issue bazel requests to a bazel that resides outside of sandbox. This -// is particularly useful to maintain a persistent Bazel server which lives -// past the duration of a single build. -// The ProxyServer will only live as long as soong_ui does; the -// underlying Bazel server will live past the duration of the build. -type ProxyServer struct { - logger ServerLogger - outDir string - workspaceDir string - bazeliskVersion string - // The server goroutine will listen on this channel and stop handling requests - // once it is written to. - done chan struct{} -} - -// NewProxyClient is a constructor for a ProxyClient. -func NewProxyClient(outDir string) *ProxyClient { - return &ProxyClient{ - outDir: outDir, - } -} - -func unixSocketPath(outDir string) string { - return filepath.Join(outDir, "bazelsocket.sock") -} - -// IssueCommand issues a request to the Bazel Proxy Server to issue a Bazel -// request. Returns a response describing the output from the Bazel process -// (if the Bazel process had an error, then the response will include an error). -// Returns an error if there was an issue with the connection to the Bazel Proxy -// server. -func (b *ProxyClient) IssueCommand(req CmdRequest) (CmdResponse, error) { - var resp CmdResponse - var err error - // Check for connections every 1 second. This is chosen to be a relatively - // short timeout, because the proxy server should accept requests quite - // quickly. - d := net.Dialer{Timeout: 1 * time.Second} - var conn net.Conn - conn, err = d.Dial("unix", unixSocketPath(b.outDir)) - if err != nil { - return resp, err - } - defer conn.Close() - - enc := gob.NewEncoder(conn) - if err = enc.Encode(req); err != nil { - return resp, err - } - dec := gob.NewDecoder(conn) - err = dec.Decode(&resp) - return resp, err -} - -// NewProxyServer is a constructor for a ProxyServer. -func NewProxyServer(logger ServerLogger, outDir string, workspaceDir string, bazeliskVersion string) *ProxyServer { - if len(bazeliskVersion) > 0 { - logger.Println("** Using Bazelisk for this build, due to env var USE_BAZEL_VERSION=" + bazeliskVersion + " **") - } - - return &ProxyServer{ - logger: logger, - outDir: outDir, - workspaceDir: workspaceDir, - done: make(chan struct{}), - bazeliskVersion: bazeliskVersion, - } -} - -func ExecBazel(bazelPath string, workspaceDir string, request CmdRequest) (stdout []byte, stderr []byte, cmdErr error) { - bazelCmd := exec.Command(bazelPath, request.Argv...) - bazelCmd.Dir = workspaceDir - bazelCmd.Env = request.Env - - stderrBuffer := &bytes.Buffer{} - bazelCmd.Stderr = stderrBuffer - - if output, err := bazelCmd.Output(); err != nil { - cmdErr = fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---", - err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderrBuffer) - } else { - stdout = output - } - stderr = stderrBuffer.Bytes() - return -} - -func (b *ProxyServer) handleRequest(conn net.Conn) error { - defer conn.Close() - - dec := gob.NewDecoder(conn) - var req CmdRequest - if err := dec.Decode(&req); err != nil { - return fmt.Errorf("Error decoding request: %s", err) - } - - if len(b.bazeliskVersion) > 0 { - req.Env = append(req.Env, "USE_BAZEL_VERSION="+b.bazeliskVersion) - } - stdout, stderr, cmdErr := ExecBazel("./build/bazel/bin/bazel", b.workspaceDir, req) - errorString := "" - if cmdErr != nil { - errorString = cmdErr.Error() - } - - resp := CmdResponse{string(stdout), string(stderr), errorString} - enc := gob.NewEncoder(conn) - if err := enc.Encode(&resp); err != nil { - return fmt.Errorf("Error encoding response: %s", err) - } - return nil -} - -func (b *ProxyServer) listenUntilClosed(listener net.Listener) error { - for { - // Check for connections every 1 second. This is a blocking operation, so - // if the server is closed, the goroutine will not fully close until this - // deadline is reached. Thus, this deadline is short (but not too short - // so that the routine churns). - listener.(*net.UnixListener).SetDeadline(time.Now().Add(time.Second)) - conn, err := listener.Accept() - - select { - case <-b.done: - return nil - default: - } - - if err != nil { - if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() { - // Timeout is normal and expected while waiting for client to establish - // a connection. - continue - } else { - b.logger.Fatalf("Listener error: %s", err) - } - } - - err = b.handleRequest(conn) - if err != nil { - b.logger.Fatal(err) - } - } -} - -// Start initializes the server unix socket and (in a separate goroutine) -// handles requests on the socket until the server is closed. Returns an error -// if a failure occurs during initialization. Will log any post-initialization -// errors to the server's logger. -func (b *ProxyServer) Start() error { - unixSocketAddr := unixSocketPath(b.outDir) - if err := os_lib.RemoveAll(unixSocketAddr); err != nil { - return fmt.Errorf("couldn't remove socket '%s': %s", unixSocketAddr, err) - } - listener, err := net.Listen("unix", unixSocketAddr) - - if err != nil { - return fmt.Errorf("error listening on socket '%s': %s", unixSocketAddr, err) - } - - go b.listenUntilClosed(listener) - return nil -} - -// Close shuts down the server. This will stop the server from listening for -// additional requests. -func (b *ProxyServer) Close() { - b.done <- struct{}{} -} diff --git a/bazel/constants.go b/bazel/constants.go deleted file mode 100644 index b10f256f0..000000000 --- a/bazel/constants.go +++ /dev/null @@ -1,30 +0,0 @@ -package bazel - -type RunName string - -// Below is a list bazel execution run names used through out the -// Platform Build systems. Each run name represents an unique key -// to query the bazel metrics. -const ( - // Perform a bazel build of the phony root to generate symlink forests - // for dependencies of the bazel build. - BazelBuildPhonyRootRunName = RunName("bazel-build-phony-root") - - // Perform aquery of the bazel build root to retrieve action information. - AqueryBuildRootRunName = RunName("aquery-buildroot") - - // Perform cquery of the Bazel build root and its dependencies. - CqueryBuildRootRunName = RunName("cquery-buildroot") - - // Run bazel as a ninja executer - BazelNinjaExecRunName = RunName("bazel-ninja-exec") - - SoongInjectionDirName = "soong_injection" - - GeneratedBazelFileWarning = "# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT." -) - -// String returns the name of the run. -func (c RunName) String() string { - return string(c) -} diff --git a/bazel/cquery/Android.bp b/bazel/cquery/Android.bp deleted file mode 100644 index 74f772184..000000000 --- a/bazel/cquery/Android.bp +++ /dev/null @@ -1,17 +0,0 @@ -package { - default_applicable_licenses: ["Android-Apache-2.0"], -} - -bootstrap_go_package { - name: "soong-cquery", - pkgPath: "android/soong/bazel/cquery", - srcs: [ - "request_type.go", - ], - pluginFor: [ - "soong_build", - ], - testSrcs: [ - "request_type_test.go", - ], -} diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go deleted file mode 100644 index 791c6bc23..000000000 --- a/bazel/cquery/request_type.go +++ /dev/null @@ -1,426 +0,0 @@ -package cquery - -import ( - "encoding/json" - "fmt" - "strings" -) - -var ( - GetOutputFiles = &getOutputFilesRequestType{} - GetCcInfo = &getCcInfoType{} - GetApexInfo = &getApexInfoType{} - GetCcUnstrippedInfo = &getCcUnstrippedInfoType{} - GetPrebuiltFileInfo = &getPrebuiltFileInfo{} -) - -type CcAndroidMkInfo struct { - LocalStaticLibs []string - LocalWholeStaticLibs []string - LocalSharedLibs []string -} - -type CcInfo struct { - CcAndroidMkInfo - OutputFiles []string - CcObjectFiles []string - CcSharedLibraryFiles []string - CcStaticLibraryFiles []string - Includes []string - SystemIncludes []string - Headers []string - // Archives owned by the current target (not by its dependencies). These will - // be a subset of OutputFiles. (or static libraries, this will be equal to OutputFiles, - // but general cc_library will also have dynamic libraries in output files). - RootStaticArchives []string - // Dynamic libraries (.so files) created by the current target. These will - // be a subset of OutputFiles. (or shared libraries, this will be equal to OutputFiles, - // but general cc_library will also have dynamic libraries in output files). - RootDynamicLibraries []string - TidyFiles []string - TocFile string - UnstrippedOutput string - AbiDiffFiles []string -} - -type getOutputFilesRequestType struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getOutputFilesRequestType) Name() string { - return "getOutputFiles" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getOutputFilesRequestType) StarlarkFunctionBody() string { - return "return ', '.join([f.path for f in target.files.to_list()])" -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getOutputFilesRequestType) ParseResult(rawString string) []string { - return splitOrEmpty(rawString, ", ") -} - -type getCcInfoType struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getCcInfoType) Name() string { - return "getCcInfo" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getCcInfoType) StarlarkFunctionBody() string { - return ` -outputFiles = [f.path for f in target.files.to_list()] -p = providers(target) -cc_info = p.get("CcInfo") -if not cc_info: - fail("%s did not provide CcInfo" % id_string) - -includes = cc_info.compilation_context.includes.to_list() -system_includes = cc_info.compilation_context.system_includes.to_list() -headers = [f.path for f in cc_info.compilation_context.headers.to_list()] - -ccObjectFiles = [] -staticLibraries = [] -rootStaticArchives = [] -linker_inputs = cc_info.linking_context.linker_inputs.to_list() - -static_info_tag = "//build/bazel/rules/cc:cc_library_static.bzl%CcStaticLibraryInfo" -if static_info_tag in p: - static_info = p[static_info_tag] - ccObjectFiles = [f.path for f in static_info.objects] - rootStaticArchives = [static_info.root_static_archive.path] -else: - for linker_input in linker_inputs: - for library in linker_input.libraries: - for object in library.objects: - ccObjectFiles += [object.path] - if library.static_library: - staticLibraries.append(library.static_library.path) - if linker_input.owner == target.label: - rootStaticArchives.append(library.static_library.path) - -sharedLibraries = [] -rootSharedLibraries = [] - -shared_info_tag = "//build/bazel/rules/cc:cc_library_shared.bzl%CcSharedLibraryOutputInfo" -stubs_tag = "//build/bazel/rules/cc:cc_stub_library.bzl%CcStubInfo" -unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo" -unstripped = "" - -if shared_info_tag in p: - shared_info = p[shared_info_tag] - path = shared_info.output_file.path - sharedLibraries.append(path) - rootSharedLibraries += [path] - unstripped = path - if unstripped_tag in p: - unstripped = p[unstripped_tag].unstripped.path -elif stubs_tag in p: - rootSharedLibraries.extend([f.path for f in target.files.to_list()]) -else: - for linker_input in linker_inputs: - for library in linker_input.libraries: - if library.dynamic_library: - path = library.dynamic_library.path - sharedLibraries.append(path) - if linker_input.owner == target.label: - rootSharedLibraries.append(path) - -toc_file = "" -toc_file_tag = "//build/bazel/rules/cc:generate_toc.bzl%CcTocInfo" -if toc_file_tag in p: - toc_file = p[toc_file_tag].toc.path -else: - # NOTE: It's OK if there's no ToC, as Soong just uses it for optimization - pass - -tidy_files = [] -clang_tidy_info = p.get("//build/bazel/rules/cc:clang_tidy.bzl%ClangTidyInfo") -if clang_tidy_info: - tidy_files = [v.path for v in clang_tidy_info.transitive_tidy_files.to_list()] - -abi_diff_files = [] -abi_diff_info = p.get("//build/bazel/rules/abi:abi_dump.bzl%AbiDiffInfo") -if abi_diff_info: - abi_diff_files = [f.path for f in abi_diff_info.diff_files.to_list()] - -local_static_libs = [] -local_whole_static_libs = [] -local_shared_libs = [] -androidmk_tag = "//build/bazel/rules/cc:cc_library_common.bzl%CcAndroidMkInfo" -if androidmk_tag in p: - androidmk_info = p[androidmk_tag] - local_static_libs = androidmk_info.local_static_libs - local_whole_static_libs = androidmk_info.local_whole_static_libs - local_shared_libs = androidmk_info.local_shared_libs - -return json.encode({ - "OutputFiles": outputFiles, - "CcObjectFiles": ccObjectFiles, - "CcSharedLibraryFiles": sharedLibraries, - "CcStaticLibraryFiles": staticLibraries, - "Includes": includes, - "SystemIncludes": system_includes, - "Headers": headers, - "RootStaticArchives": rootStaticArchives, - "RootDynamicLibraries": rootSharedLibraries, - "TidyFiles": [t for t in tidy_files], - "TocFile": toc_file, - "UnstrippedOutput": unstripped, - "AbiDiffFiles": abi_diff_files, - "LocalStaticLibs": [l for l in local_static_libs], - "LocalWholeStaticLibs": [l for l in local_whole_static_libs], - "LocalSharedLibs": [l for l in local_shared_libs], -})` - -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getCcInfoType) ParseResult(rawString string) (CcInfo, error) { - var ccInfo CcInfo - if err := parseJson(rawString, &ccInfo); err != nil { - return ccInfo, err - } - return ccInfo, nil -} - -// Query Bazel for the artifacts generated by the apex modules. -type getApexInfoType struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getApexInfoType) Name() string { - return "getApexInfo" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getApexInfoType) StarlarkFunctionBody() string { - return ` -info = providers(target).get("//build/bazel/rules/apex:apex_info.bzl%ApexInfo") -if not info: - fail("%s did not provide ApexInfo" % id_string) -bundle_key_info = info.bundle_key_info -container_key_info = info.container_key_info - -signed_compressed_output = "" # no .capex if the apex is not compressible, cannot be None as it needs to be json encoded. -if info.signed_compressed_output: - signed_compressed_output = info.signed_compressed_output.path - -mk_info = providers(target).get("//build/bazel/rules/apex:apex_info.bzl%ApexMkInfo") -if not mk_info: - fail("%s did not provide ApexMkInfo" % id_string) - -tidy_files = [] -clang_tidy_info = providers(target).get("//build/bazel/rules/cc:clang_tidy.bzl%ClangTidyInfo") -if clang_tidy_info: - tidy_files = [v.path for v in clang_tidy_info.transitive_tidy_files.to_list()] - -return json.encode({ - "signed_output": info.signed_output.path, - "signed_compressed_output": signed_compressed_output, - "unsigned_output": info.unsigned_output.path, - "provides_native_libs": [str(lib) for lib in info.provides_native_libs], - "requires_native_libs": [str(lib) for lib in info.requires_native_libs], - "bundle_key_info": [bundle_key_info.public_key.path, bundle_key_info.private_key.path], - "container_key_info": [container_key_info.pem.path, container_key_info.pk8.path, container_key_info.key_name], - "package_name": info.package_name, - "symbols_used_by_apex": info.symbols_used_by_apex.path, - "java_symbols_used_by_apex": info.java_symbols_used_by_apex.path, - "backing_libs": info.backing_libs.path, - "bundle_file": info.base_with_config_zip.path, - "installed_files": info.installed_files.path, - "make_modules_to_install": mk_info.make_modules_to_install, - "files_info": mk_info.files_info, - "tidy_files": [t for t in tidy_files], -})` -} - -type ApexInfo struct { - // From the ApexInfo provider - SignedOutput string `json:"signed_output"` - SignedCompressedOutput string `json:"signed_compressed_output"` - UnsignedOutput string `json:"unsigned_output"` - ProvidesLibs []string `json:"provides_native_libs"` - RequiresLibs []string `json:"requires_native_libs"` - BundleKeyInfo []string `json:"bundle_key_info"` - ContainerKeyInfo []string `json:"container_key_info"` - PackageName string `json:"package_name"` - SymbolsUsedByApex string `json:"symbols_used_by_apex"` - JavaSymbolsUsedByApex string `json:"java_symbols_used_by_apex"` - BackingLibs string `json:"backing_libs"` - BundleFile string `json:"bundle_file"` - InstalledFiles string `json:"installed_files"` - TidyFiles []string `json:"tidy_files"` - - // From the ApexMkInfo provider - MakeModulesToInstall []string `json:"make_modules_to_install"` - PayloadFilesInfo []map[string]string `json:"files_info"` -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getApexInfoType) ParseResult(rawString string) (ApexInfo, error) { - var info ApexInfo - err := parseJson(rawString, &info) - return info, err -} - -// getCcUnstrippedInfoType implements cqueryRequest interface. It handles the -// interaction with `bazel cquery` to retrieve CcUnstrippedInfo provided -// by the` cc_binary` and `cc_shared_library` rules. -type getCcUnstrippedInfoType struct{} - -func (g getCcUnstrippedInfoType) Name() string { - return "getCcUnstrippedInfo" -} - -func (g getCcUnstrippedInfoType) StarlarkFunctionBody() string { - return ` -p = providers(target) -output_path = target.files.to_list()[0].path - -unstripped = output_path -unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo" -if unstripped_tag in p: - unstripped_info = p[unstripped_tag] - unstripped = unstripped_info.unstripped[0].files.to_list()[0].path - -local_static_libs = [] -local_whole_static_libs = [] -local_shared_libs = [] -androidmk_tag = "//build/bazel/rules/cc:cc_library_common.bzl%CcAndroidMkInfo" -if androidmk_tag in p: - androidmk_info = p[androidmk_tag] - local_static_libs = androidmk_info.local_static_libs - local_whole_static_libs = androidmk_info.local_whole_static_libs - local_shared_libs = androidmk_info.local_shared_libs - -tidy_files = [] -clang_tidy_info = p.get("//build/bazel/rules/cc:clang_tidy.bzl%ClangTidyInfo") -if clang_tidy_info: - tidy_files = [v.path for v in clang_tidy_info.transitive_tidy_files.to_list()] - -return json.encode({ - "OutputFile": output_path, - "UnstrippedOutput": unstripped, - "LocalStaticLibs": [l for l in local_static_libs], - "LocalWholeStaticLibs": [l for l in local_whole_static_libs], - "LocalSharedLibs": [l for l in local_shared_libs], - "TidyFiles": [t for t in tidy_files], -}) -` -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getCcUnstrippedInfoType) ParseResult(rawString string) (CcUnstrippedInfo, error) { - var info CcUnstrippedInfo - err := parseJson(rawString, &info) - return info, err -} - -type CcUnstrippedInfo struct { - CcAndroidMkInfo - OutputFile string - UnstrippedOutput string - TidyFiles []string -} - -// splitOrEmpty is a modification of strings.Split() that returns an empty list -// if the given string is empty. -func splitOrEmpty(s string, sep string) []string { - if len(s) < 1 { - return []string{} - } else { - return strings.Split(s, sep) - } -} - -// parseJson decodes json string into the fields of the receiver. -// Unknown attribute name causes panic. -func parseJson(jsonString string, info interface{}) error { - decoder := json.NewDecoder(strings.NewReader(jsonString)) - decoder.DisallowUnknownFields() //useful to detect typos, e.g. in unit tests - err := decoder.Decode(info) - if err != nil { - return fmt.Errorf("cannot parse cquery result '%s': %s", jsonString, err) - } - return nil -} - -type getPrebuiltFileInfo struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getPrebuiltFileInfo) Name() string { - return "getPrebuiltFileInfo" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getPrebuiltFileInfo) StarlarkFunctionBody() string { - return ` -p = providers(target) -prebuilt_file_info = p.get("//build/bazel/rules:prebuilt_file.bzl%PrebuiltFileInfo") -if not prebuilt_file_info: - fail("%s did not provide PrebuiltFileInfo" % id_string) - -return json.encode({ - "Src": prebuilt_file_info.src.path, - "Dir": prebuilt_file_info.dir, - "Filename": prebuilt_file_info.filename, - "Installable": prebuilt_file_info.installable, -})` -} - -type PrebuiltFileInfo struct { - // TODO: b/207489266 - Fully support all properties in prebuilt_file - Src string - Dir string - Filename string - Installable bool -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getPrebuiltFileInfo) ParseResult(rawString string) (PrebuiltFileInfo, error) { - var info PrebuiltFileInfo - err := parseJson(rawString, &info) - return info, err -} diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go deleted file mode 100644 index e772bb7d6..000000000 --- a/bazel/cquery/request_type_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package cquery - -import ( - "encoding/json" - "reflect" - "strings" - "testing" -) - -func TestGetOutputFilesParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput []string - }{ - { - description: "no result", - input: "", - expectedOutput: []string{}, - }, - { - description: "one result", - input: "test", - expectedOutput: []string{"test"}, - }, - { - description: "splits on comma with space", - input: "foo, bar", - expectedOutput: []string{"foo", "bar"}, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput := GetOutputFiles.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetCcInfoParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - inputCcInfo CcInfo - expectedOutput CcInfo - }{ - { - description: "no result", - inputCcInfo: CcInfo{}, - expectedOutput: CcInfo{}, - }, - { - description: "all items set", - inputCcInfo: CcInfo{ - OutputFiles: []string{"out1", "out2"}, - CcObjectFiles: []string{"object1", "object2"}, - CcSharedLibraryFiles: []string{"shared_lib1", "shared_lib2"}, - CcStaticLibraryFiles: []string{"static_lib1", "static_lib2"}, - Includes: []string{".", "dir/subdir"}, - SystemIncludes: []string{"system/dir", "system/other/dir"}, - Headers: []string{"dir/subdir/hdr.h"}, - RootStaticArchives: []string{"rootstaticarchive1"}, - RootDynamicLibraries: []string{"rootdynamiclibrary1"}, - TocFile: "lib.so.toc", - }, - expectedOutput: CcInfo{ - OutputFiles: []string{"out1", "out2"}, - CcObjectFiles: []string{"object1", "object2"}, - CcSharedLibraryFiles: []string{"shared_lib1", "shared_lib2"}, - CcStaticLibraryFiles: []string{"static_lib1", "static_lib2"}, - Includes: []string{".", "dir/subdir"}, - SystemIncludes: []string{"system/dir", "system/other/dir"}, - Headers: []string{"dir/subdir/hdr.h"}, - RootStaticArchives: []string{"rootstaticarchive1"}, - RootDynamicLibraries: []string{"rootdynamiclibrary1"}, - TocFile: "lib.so.toc", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - jsonInput, _ := json.Marshal(tc.inputCcInfo) - actualOutput, err := GetCcInfo.ParseResult(string(jsonInput)) - if err != nil { - t.Errorf("error parsing result: %q", err) - } else if err == nil && !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v\n!= actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetCcInfoParseResultsError(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedError string - }{ - { - description: "not json", - input: ``, - expectedError: `cannot parse cquery result '': EOF`, - }, - { - description: "invalid field", - input: `{ - "toc_file": "dir/file.so.toc" -}`, - expectedError: `json: unknown field "toc_file"`, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - _, err := GetCcInfo.ParseResult(tc.input) - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) - } - }) - } -} - -func TestGetApexInfoParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput ApexInfo - }{ - { - description: "no result", - input: "{}", - expectedOutput: ApexInfo{}, - }, - { - description: "one result", - input: `{ - "signed_output":"my.apex", - "unsigned_output":"my.apex.unsigned", - "requires_native_libs":["//bionic/libc:libc","//bionic/libdl:libdl"], - "bundle_key_info":["foo.pem", "foo.privkey"], - "container_key_info":["foo.x509.pem", "foo.pk8", "foo"], - "package_name":"package.name", - "symbols_used_by_apex": "path/to/my.apex_using.txt", - "backing_libs":"path/to/backing.txt", - "bundle_file": "dir/bundlefile.zip", - "installed_files":"path/to/installed-files.txt", - "provides_native_libs":[], - "make_modules_to_install": ["foo","bar"] -}`, - expectedOutput: ApexInfo{ - // ApexInfo - SignedOutput: "my.apex", - UnsignedOutput: "my.apex.unsigned", - RequiresLibs: []string{"//bionic/libc:libc", "//bionic/libdl:libdl"}, - ProvidesLibs: []string{}, - BundleKeyInfo: []string{"foo.pem", "foo.privkey"}, - ContainerKeyInfo: []string{"foo.x509.pem", "foo.pk8", "foo"}, - PackageName: "package.name", - SymbolsUsedByApex: "path/to/my.apex_using.txt", - BackingLibs: "path/to/backing.txt", - BundleFile: "dir/bundlefile.zip", - InstalledFiles: "path/to/installed-files.txt", - - // ApexMkInfo - MakeModulesToInstall: []string{"foo", "bar"}, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput, err := GetApexInfo.ParseResult(tc.input) - if err != nil { - t.Errorf("Unexpected error %q", err) - } - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetApexInfoParseResultsError(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedError string - }{ - { - description: "not json", - input: ``, - expectedError: `cannot parse cquery result '': EOF`, - }, - { - description: "invalid field", - input: `{ - "fake_field": "path/to/file" -}`, - expectedError: `json: unknown field "fake_field"`, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - _, err := GetApexInfo.ParseResult(tc.input) - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) - } - }) - } -} - -func TestGetCcUnstrippedParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput CcUnstrippedInfo - }{ - { - description: "no result", - input: "{}", - expectedOutput: CcUnstrippedInfo{}, - }, - { - description: "one result", - input: `{"OutputFile":"myapp", "UnstrippedOutput":"myapp_unstripped"}`, - expectedOutput: CcUnstrippedInfo{ - OutputFile: "myapp", - UnstrippedOutput: "myapp_unstripped", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput, err := GetCcUnstrippedInfo.ParseResult(tc.input) - if err != nil { - t.Errorf("Unexpected error %q", err) - } - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetCcUnstrippedParseResultsErrors(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedError string - }{ - { - description: "not json", - input: ``, - expectedError: `cannot parse cquery result '': EOF`, - }, - { - description: "invalid field", - input: `{ - "fake_field": "path/to/file" -}`, - expectedError: `json: unknown field "fake_field"`, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - _, err := GetCcUnstrippedInfo.ParseResult(tc.input) - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) - } - }) - } -} diff --git a/bin/soongdbg b/bin/soongdbg index bfdbbde70..a73bdf90d 100755 --- a/bin/soongdbg +++ b/bin/soongdbg @@ -32,11 +32,13 @@ class Graph: dep.rdeps.add(node) node.dep_tags.setdefault(dep, list()).append(d) - def find_paths(self, id1, id2): + def find_paths(self, id1, id2, tag_filter): # Throws KeyError if one of the names isn't found def recurse(node1, node2, visited): result = set() for dep in node1.rdeps: + if not matches_tag(dep, node1, tag_filter): + continue if dep == node2: result.add(node2) if dep not in visited: @@ -214,6 +216,8 @@ def print_args(parser): help="jq query for each module metadata") parser.add_argument("--deptags", action="store_true", help="show dependency tags (makes the graph much more complex)") + parser.add_argument("--tag", action="append", + help="Limit output to these dependency tags.") group = parser.add_argument_group("output formats", "If no format is provided, a dot file will be written to" @@ -259,13 +263,21 @@ def print_nodes(args, nodes, module_formatter): sys.stdout.write(text) -def get_deps(nodes, root, maxdepth, reverse): +def matches_tag(node, dep, tag_filter): + if not tag_filter: + return True + return not tag_filter.isdisjoint([t.tag_type for t in node.dep_tags[dep]]) + + +def get_deps(nodes, root, maxdepth, reverse, tag_filter): if root in nodes: return nodes.add(root) if maxdepth != 0: for dep in (root.rdeps if reverse else root.deps): - get_deps(nodes, dep, maxdepth-1, reverse) + if not matches_tag(root, dep, tag_filter): + continue + get_deps(nodes, dep, maxdepth-1, reverse, tag_filter) def new_module_formatter(args): @@ -302,7 +314,7 @@ class BetweenCommand: def run(self, args): graph = load_graph() - print_nodes(args, graph.find_paths(args.module[0], args.module[1]), + print_nodes(args, graph.find_paths(args.module[0], args.module[1], set(args.tag)), new_module_formatter(args)) @@ -328,7 +340,7 @@ class DepsCommand: sys.stderr.write(f"error: Can't find root: {id}\n") err = True continue - get_deps(nodes, root, args.depth, args.reverse) + get_deps(nodes, root, args.depth, args.reverse, set(args.tag)) if err: sys.exit(1) print_nodes(args, nodes, new_module_formatter(args)) diff --git a/bpf/bpf.go b/bpf/bpf.go index 2eb869ebe..09262e507 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -104,6 +104,14 @@ var _ android.ImageInterface = (*bpf)(nil) func (bpf *bpf) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (bpf *bpf) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return proptools.Bool(bpf.properties.Vendor) +} + +func (bpf *bpf) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + func (bpf *bpf) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !proptools.Bool(bpf.properties.Vendor) } @@ -125,13 +133,10 @@ func (bpf *bpf) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { } func (bpf *bpf) ExtraImageVariations(ctx android.BaseModuleContext) []string { - if proptools.Bool(bpf.properties.Vendor) { - return []string{"vendor"} - } return nil } -func (bpf *bpf) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +func (bpf *bpf) SetImageVariation(ctx android.BaseModuleContext, variation string) { bpf.properties.VendorInternal = variation == "vendor" } diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go index ddaa98aa3..9163ab759 100644 --- a/bpfix/bpfix/bpfix.go +++ b/bpfix/bpfix/bpfix.go @@ -286,7 +286,7 @@ func (f *Fixer) reparse() ([]byte, error) { } func parse(name string, r io.Reader) (*parser.File, error) { - tree, errs := parser.Parse(name, r, parser.NewScope(nil)) + tree, errs := parser.Parse(name, r) if errs != nil { s := "parse error: " for _, err := range errs { diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index b5b49b1ab..f487d3c7c 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -46,7 +46,7 @@ func buildTree(local_include_dirs []string, export_include_dirs []string) (file } `, printListOfStrings(local_include_dirs), printListOfStrings(export_include_dirs)) - tree, errs := parser.Parse("", strings.NewReader(input), parser.NewScope(nil)) + tree, errs := parser.Parse("", strings.NewReader(input)) if len(errs) > 0 { errs = append([]error{fmt.Errorf("failed to parse:\n%s", input)}, errs...) } @@ -167,7 +167,7 @@ func preProcessIn(in string) (fixer *Fixer, err error) { return fixer, err } - tree, errs := parser.Parse("<testcase>", bytes.NewBufferString(in), parser.NewScope(nil)) + tree, errs := parser.Parse("<testcase>", bytes.NewBufferString(in)) if errs != nil { return fixer, err } diff --git a/bpfix/cmd_lib/bpfix.go b/bpfix/cmd_lib/bpfix.go index 1106d4af7..41430f8e4 100644 --- a/bpfix/cmd_lib/bpfix.go +++ b/bpfix/cmd_lib/bpfix.go @@ -66,7 +66,7 @@ func processFile(filename string, in io.Reader, out io.Writer, fixRequest bpfix. return err } r := bytes.NewBuffer(append([]byte(nil), src...)) - file, errs := parser.Parse(filename, r, parser.NewScope(nil)) + file, errs := parser.Parse(filename, r) if len(errs) > 0 { for _, err := range errs { fmt.Fprintln(os.Stderr, err) diff --git a/cc/androidmk.go b/cc/androidmk.go index 62ba4debd..143e86f09 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -104,16 +104,6 @@ func (c *Module) AndroidMkEntries() []android.AndroidMkEntries { entries.AddStrings("LOCAL_RUNTIME_LIBRARIES", c.Properties.AndroidMkRuntimeLibs...) } entries.SetString("LOCAL_SOONG_LINK_TYPE", c.makeLinkType) - if c.InVendorOrProduct() { - if c.IsVndk() && !c.static() { - entries.SetString("LOCAL_SOONG_VNDK_VERSION", c.VndkVersion()) - // VNDK libraries available to vendor are not installed because - // they are packaged in VNDK APEX and installed by APEX packages (apex/apex.go) - if !c.IsVndkExt() { - entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) - } - } - } if c.InVendor() { entries.SetBool("LOCAL_IN_VENDOR", true) } else if c.InProduct() { diff --git a/cc/binary.go b/cc/binary.go index 3ff35de56..2ac9a45bc 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -451,7 +451,7 @@ func (binary *binaryDecorator) unstrippedOutputFilePath() android.Path { } func (binary *binaryDecorator) strippedAllOutputFilePath() android.Path { - panic("Not implemented.") + return nil } func (binary *binaryDecorator) setSymlinkList(ctx ModuleContext) { diff --git a/cc/builder.go b/cc/builder.go index 8719d4f87..367bda380 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -798,9 +798,12 @@ func transformObjToStaticLib(ctx android.ModuleContext, // Generate a Rust staticlib from a list of rlibDeps. Returns nil if TransformRlibstoStaticlib is nil or rlibDeps is empty. func generateRustStaticlib(ctx android.ModuleContext, rlibDeps []RustRlibDep) android.Path { if TransformRlibstoStaticlib == nil && len(rlibDeps) > 0 { - // This should only be reachable if a module defines static_rlibs and + // This should only be reachable if a module defines Rust deps in static_libs and // soong-rust hasn't been loaded alongside soong-cc (e.g. in soong-cc tests). - panic(fmt.Errorf("TransformRlibstoStaticlib is not set and static_rlibs is defined in %s", ctx.ModuleName())) + panic(fmt.Errorf( + "TransformRlibstoStaticlib is not set and rust deps are defined in static_libs for %s", + ctx.ModuleName())) + } else if len(rlibDeps) == 0 { return nil } @@ -829,6 +832,7 @@ func generateRustStaticlib(ctx android.ModuleContext, rlibDeps []RustRlibDep) an func genRustStaticlibSrcFile(crateNames []string) string { lines := []string{ "// @Soong generated Source", + "#![no_std]", // pre-emptively set no_std to support both std and no_std. } for _, crate := range crateNames { lines = append(lines, fmt.Sprintf("extern crate %s;", crate)) @@ -49,7 +49,6 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { ctx.BottomUp("sdk", sdkMutator).Parallel() - ctx.BottomUp("vndk", VndkMutator).Parallel() ctx.BottomUp("llndk", llndkMutator).Parallel() ctx.BottomUp("link", LinkageMutator).Parallel() ctx.BottomUp("test_per_src", TestPerSrcMutator).Parallel() @@ -100,7 +99,6 @@ type Deps struct { StaticLibs, LateStaticLibs, WholeStaticLibs []string HeaderLibs []string RuntimeLibs []string - Rlibs []string // UnexportedStaticLibs are static libraries that are also passed to -Wl,--exclude-libs= to // prevent automatically exporting symbols. @@ -362,6 +360,8 @@ type BaseProperties struct { Recovery_available *bool // Used by imageMutator, set by ImageMutatorBegin() + VendorVariantNeeded bool `blueprint:"mutated"` + ProductVariantNeeded bool `blueprint:"mutated"` CoreVariantNeeded bool `blueprint:"mutated"` RamdiskVariantNeeded bool `blueprint:"mutated"` VendorRamdiskVariantNeeded bool `blueprint:"mutated"` @@ -480,19 +480,6 @@ type VendorProperties struct { // IsLLNDK is set to true for the vendor variant of a cc_library module that has LLNDK stubs. IsLLNDK bool `blueprint:"mutated"` - // IsVNDKCore is set if a VNDK module does not set the vndk.support_system_process property. - IsVNDKCore bool `blueprint:"mutated"` - - // IsVNDKSP is set if a VNDK module sets the vndk.support_system_process property. - IsVNDKSP bool `blueprint:"mutated"` - - // IsVNDKPrivate is set if a VNDK module sets the vndk.private property or an LLNDK - // module sets the llndk.private property. - IsVNDKPrivate bool `blueprint:"mutated"` - - // IsVNDKProduct is set if a VNDK module sets the product_available property. - IsVNDKProduct bool `blueprint:"mutated"` - // IsVendorPublicLibrary is set for the core and product variants of a library that has // vendor_public_library stubs. IsVendorPublicLibrary bool `blueprint:"mutated"` @@ -519,12 +506,7 @@ type ModuleContextIntf interface { useVndk() bool isNdk(config android.Config) bool IsLlndk() bool - IsLlndkPublic() bool isImplementationForLLNDKPublic() bool - IsVndkPrivate() bool - isVndk() bool - isVndkSp() bool - IsVndkExt() bool IsVendorPublicLibrary() bool inProduct() bool inVendor() bool @@ -534,7 +516,6 @@ type ModuleContextIntf interface { InVendorOrProduct() bool selectedStl() string baseModuleName() string - getVndkExtendsModuleName() string isAfdoCompile(ctx ModuleContext) bool isOrderfileCompile() bool isCfi() bool @@ -764,11 +745,6 @@ func (d libraryDependencyTag) static() bool { return d.Kind == staticLibraryDependency } -// rlib returns true if the libraryDependencyTag is tagging an rlib dependency. -func (d libraryDependencyTag) rlib() bool { - return d.Kind == rlibLibraryDependency -} - func (d libraryDependencyTag) LicenseAnnotations() []android.LicenseAnnotation { if d.shared() { return []android.LicenseAnnotation{android.LicenseAnnotationSharedDependency} @@ -900,7 +876,6 @@ type Module struct { coverage *coverage fuzzer *fuzzer sabi *sabi - vndkdep *vndkdep lto *lto afdo *afdo orderfile *orderfile @@ -936,6 +911,8 @@ type Module struct { hideApexVariantFromMake bool logtagsPaths android.Paths + + WholeRustStaticlib bool } func (c *Module) AddJSONData(d *map[string]interface{}) { @@ -974,12 +951,7 @@ func (c *Module) AddJSONData(d *map[string]interface{}) { "InstallInVendorRamdisk": c.InstallInVendorRamdisk(), "InstallInRecovery": c.InstallInRecovery(), "InstallInRoot": c.InstallInRoot(), - "IsVndk": c.IsVndk(), - "IsVndkExt": c.IsVndkExt(), - "IsVndkPrivate": c.IsVndkPrivate(), - "IsVndkSp": c.IsVndkSp(), "IsLlndk": c.IsLlndk(), - "IsLlndkPublic": c.IsLlndkPublic(), "IsVendorPublicLibrary": c.IsVendorPublicLibrary(), "ApexSdkVersion": c.apexSdkVersion, "TestFor": c.TestFor(), @@ -1007,8 +979,8 @@ func (c *Module) HiddenFromMake() bool { return c.Properties.HideFromMake } -func (c *Module) RequiredModuleNames() []string { - required := android.CopyOf(c.ModuleBase.RequiredModuleNames()) +func (c *Module) RequiredModuleNames(ctx android.ConfigAndErrorContext) []string { + required := android.CopyOf(c.ModuleBase.RequiredModuleNames(ctx)) if c.ImageVariation().Variation == android.CoreVariation { required = append(required, c.Properties.Target.Platform.Required...) required = removeListFromList(required, c.Properties.Target.Platform.Exclude_required) @@ -1216,6 +1188,16 @@ func (c *Module) BuildSharedVariant() bool { panic(fmt.Errorf("BuildSharedVariant called on non-library module: %q", c.BaseModuleName())) } +func (c *Module) BuildRlibVariant() bool { + // cc modules can never build rlib variants + return false +} + +func (c *Module) IsRustFFI() bool { + // cc modules are not Rust modules + return false +} + func (c *Module) Module() android.Module { return c } @@ -1289,9 +1271,6 @@ func (c *Module) Init() android.Module { if c.sabi != nil { c.AddProperties(c.sabi.props()...) } - if c.vndkdep != nil { - c.AddProperties(c.vndkdep.props()...) - } if c.lto != nil { c.AddProperties(c.lto.props()...) } @@ -1346,10 +1325,6 @@ func (c *Module) IsLlndk() bool { return c.VendorProperties.IsLLNDK } -func (c *Module) IsLlndkPublic() bool { - return c.VendorProperties.IsLLNDK && !c.VendorProperties.IsVNDKPrivate -} - func (m *Module) NeedsLlndkVariants() bool { lib := moduleLibraryInterface(m) return lib != nil && (lib.hasLLNDKStubs() || lib.hasLLNDKHeaders()) @@ -1396,31 +1371,6 @@ func (c *Module) isImplementationForLLNDKPublic() bool { !Bool(library.Properties.Llndk.Private) } -// Returns true for LLNDK-private, VNDK-SP-private, and VNDK-core-private. -func (c *Module) IsVndkPrivate() bool { - // Check if VNDK-core-private or VNDK-SP-private - if c.IsVndk() { - return Bool(c.vndkdep.Properties.Vndk.Private) - } - - // Check if LLNDK-private - if library, ok := c.library.(*libraryDecorator); ok && c.IsLlndk() { - return Bool(library.Properties.Llndk.Private) - } - - return false -} - -// IsVndk() returns true if this module has a vndk variant. -// Note that IsVndk() returns true for all variants of vndk-enabled libraries. Not only vendor variant, -// but also platform and product variants of vndk-enabled libraries return true for IsVndk(). -func (c *Module) IsVndk() bool { - if vndkdep := c.vndkdep; vndkdep != nil { - return vndkdep.isVndk() - } - return false -} - func (c *Module) isAfdoCompile(ctx ModuleContext) bool { if afdo := c.afdo; afdo != nil { return afdo.isAfdoCompile(ctx) @@ -1456,31 +1406,10 @@ func (c *Module) isNDKStubLibrary() bool { return false } -func (c *Module) IsVndkSp() bool { - if vndkdep := c.vndkdep; vndkdep != nil { - return vndkdep.isVndkSp() - } - return false -} - -func (c *Module) IsVndkExt() bool { - if vndkdep := c.vndkdep; vndkdep != nil { - return vndkdep.isVndkExt() - } - return false -} - func (c *Module) SubName() string { return c.Properties.SubName } -func (c *Module) getVndkExtendsModuleName() string { - if vndkdep := c.vndkdep; vndkdep != nil { - return vndkdep.getVndkExtendsModuleName() - } - return "" -} - func (c *Module) IsStubs() bool { if lib := c.library; lib != nil { return lib.buildStubs() @@ -1638,14 +1567,6 @@ func (ctx *moduleContextImpl) useSdk() bool { func (ctx *moduleContextImpl) sdkVersion() string { if ctx.ctx.Device() { - config := ctx.ctx.Config() - if !config.IsVndkDeprecated() && ctx.useVndk() { - vndkVer := ctx.mod.VndkVersion() - if inList(vndkVer, config.PlatformVersionActiveCodenames()) { - return "current" - } - return vndkVer - } return String(ctx.mod.Properties.Sdk_version) } return "" @@ -1662,7 +1583,7 @@ func (ctx *moduleContextImpl) minSdkVersion() string { if ctx.ctx.Device() { config := ctx.ctx.Config() - if config.IsVndkDeprecated() && ctx.inVendor() { + if ctx.inVendor() { // If building for vendor with final API, then use the latest _stable_ API as "current". if config.VendorApiLevelFrozen() && (ver == "" || ver == "current") { ver = config.PlatformSdkVersion().String() @@ -1722,22 +1643,10 @@ func (ctx *moduleContextImpl) IsLlndk() bool { return ctx.mod.IsLlndk() } -func (ctx *moduleContextImpl) IsLlndkPublic() bool { - return ctx.mod.IsLlndkPublic() -} - func (ctx *moduleContextImpl) isImplementationForLLNDKPublic() bool { return ctx.mod.isImplementationForLLNDKPublic() } -func (ctx *moduleContextImpl) IsVndkPrivate() bool { - return ctx.mod.IsVndkPrivate() -} - -func (ctx *moduleContextImpl) isVndk() bool { - return ctx.mod.IsVndk() -} - func (ctx *moduleContextImpl) isAfdoCompile(mctx ModuleContext) bool { return ctx.mod.isAfdoCompile(mctx) } @@ -1758,14 +1667,6 @@ func (ctx *moduleContextImpl) isNDKStubLibrary() bool { return ctx.mod.isNDKStubLibrary() } -func (ctx *moduleContextImpl) isVndkSp() bool { - return ctx.mod.IsVndkSp() -} - -func (ctx *moduleContextImpl) IsVndkExt() bool { - return ctx.mod.IsVndkExt() -} - func (ctx *moduleContextImpl) IsVendorPublicLibrary() bool { return ctx.mod.IsVendorPublicLibrary() } @@ -1785,10 +1686,6 @@ func (ctx *moduleContextImpl) baseModuleName() string { return ctx.mod.BaseModuleName() } -func (ctx *moduleContextImpl) getVndkExtendsModuleName() string { - return ctx.mod.getVndkExtendsModuleName() -} - func (ctx *moduleContextImpl) isForPlatform() bool { apexInfo, _ := android.ModuleProvider(ctx.ctx, android.ApexInfoProvider) return apexInfo.IsForPlatform() @@ -1853,7 +1750,6 @@ func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Mo module.coverage = &coverage{} module.fuzzer = &fuzzer{} module.sabi = &sabi{} - module.vndkdep = &vndkdep{} module.lto = <o{} module.afdo = &afdo{} module.orderfile = &orderfile{} @@ -2229,10 +2125,60 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake { moduleInfoJSON.Uninstallable = true } + } + buildComplianceMetadataInfo(ctx, c, deps) + + c.setOutputFiles(ctx) +} + +func (c *Module) setOutputFiles(ctx ModuleContext) { + if c.outputFile.Valid() { + ctx.SetOutputFiles(android.Paths{c.outputFile.Path()}, "") + } else { + ctx.SetOutputFiles(android.Paths{}, "") + } + if c.linker != nil { + ctx.SetOutputFiles(android.PathsIfNonNil(c.linker.unstrippedOutputFilePath()), "unstripped") + ctx.SetOutputFiles(android.PathsIfNonNil(c.linker.strippedAllOutputFilePath()), "stripped_all") } } +func buildComplianceMetadataInfo(ctx ModuleContext, c *Module, deps PathDeps) { + // Dump metadata that can not be done in android/compliance-metadata.go + complianceMetadataInfo := ctx.ComplianceMetadataInfo() + complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(ctx.static())) + complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, c.outputFile.String()) + + // Static deps + staticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(false)) + staticDepNames := make([]string, 0, len(staticDeps)) + for _, dep := range staticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + + staticDepPaths := make([]string, 0, len(deps.StaticLibs)) + for _, dep := range deps.StaticLibs { + staticDepPaths = append(staticDepPaths, dep.String()) + } + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths)) + + // Whole static deps + wholeStaticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(true)) + wholeStaticDepNames := make([]string, 0, len(wholeStaticDeps)) + for _, dep := range wholeStaticDeps { + wholeStaticDepNames = append(wholeStaticDepNames, dep.Name()) + } + + wholeStaticDepPaths := make([]string, 0, len(deps.WholeStaticLibs)) + for _, dep := range deps.WholeStaticLibs { + wholeStaticDepPaths = append(wholeStaticDepPaths, dep.String()) + } + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEPS, android.FirstUniqueStrings(wholeStaticDepNames)) + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, android.FirstUniqueStrings(wholeStaticDepPaths)) +} + func (c *Module) maybeUnhideFromMake() { // If a lib is directly included in any of the APEXes or is not available to the // platform (which is often the case when the stub is provided as a prebuilt), @@ -2340,7 +2286,6 @@ func (c *Module) deps(ctx DepsContext) Deps { deps.WholeStaticLibs = android.LastUniqueStrings(deps.WholeStaticLibs) deps.StaticLibs = android.LastUniqueStrings(deps.StaticLibs) - deps.Rlibs = android.LastUniqueStrings(deps.Rlibs) deps.LateStaticLibs = android.LastUniqueStrings(deps.LateStaticLibs) deps.SharedLibs = android.LastUniqueStrings(deps.SharedLibs) deps.LateSharedLibs = android.LastUniqueStrings(deps.LateSharedLibs) @@ -2621,7 +2566,7 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { if c.ImageVariation().Variation == android.CoreVariation && c.Device() && c.Target().NativeBridge == android.NativeBridgeDisabled { actx.AddVariationDependencies( - []blueprint.Variation{{Mutator: "image", Variation: VendorVariation}}, + []blueprint.Variation{{Mutator: "image", Variation: android.VendorVariation}}, llndkHeaderLibTag, deps.LlndkHeaderLibs...) } @@ -2635,28 +2580,20 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { } for _, lib := range deps.StaticLibs { + // Some dependencies listed in static_libs might actually be rust_ffi rlib variants. depTag := libraryDependencyTag{Kind: staticLibraryDependency} + if inList(lib, deps.ReexportStaticLibHeaders) { depTag.reexportFlags = true } if inList(lib, deps.ExcludeLibsForApex) { depTag.excludeInApex = true } - actx.AddVariationDependencies([]blueprint.Variation{ {Mutator: "link", Variation: "static"}, }, depTag, lib) } - for _, lib := range deps.Rlibs { - depTag := libraryDependencyTag{Kind: rlibLibraryDependency} - actx.AddVariationDependencies([]blueprint.Variation{ - {Mutator: "link", Variation: ""}, - {Mutator: "rust_libraries", Variation: "rlib"}, - {Mutator: "rust_stdlinkage", Variation: "rlib-std"}, - }, depTag, lib) - } - // staticUnwinderDep is treated as staticDep for Q apexes // so that native libraries/binaries are linked with static unwinder // because Q libc doesn't have unwinder APIs @@ -2785,15 +2722,6 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { {Mutator: "link", Variation: "shared"}, }, ndkLateStubDepTag, apiLateNdkLibs...) - if vndkdep := c.vndkdep; vndkdep != nil { - if vndkdep.isVndkExt() { - actx.AddVariationDependencies([]blueprint.Variation{ - c.ImageVariation(), - {Mutator: "link", Variation: "shared"}, - }, vndkExtDepTag, vndkdep.getVndkExtendsModuleName()) - } - } - if len(deps.AidlLibs) > 0 { actx.AddDependency( c, @@ -2831,20 +2759,6 @@ func checkLinkType(ctx android.BaseModuleContext, from LinkableInterface, to Lin return } - // VNDK is cc.Module supported only for now. - if ccFrom, ok := from.(*Module); ok && from.UseVndk() { - // Though allowed dependency is limited by the image mutator, - // each vendor and product module needs to check link-type - // for VNDK. - if ccTo, ok := to.(*Module); ok { - if ccFrom.vndkdep != nil { - ccFrom.vndkdep.vndkCheckLinkType(ctx, ccTo, tag) - } - } else if _, ok := to.(LinkableInterface); !ok { - ctx.ModuleErrorf("Attempting to link VNDK cc.Module with unsupported module type") - } - return - } // TODO(b/244244438) : Remove this once all variants are implemented if ccFrom, ok := from.(*Module); ok && ccFrom.isImportedApiLibrary() { return @@ -2999,7 +2913,7 @@ func checkDoubleLoadableLibraries(ctx android.TopDownMutatorContext) { return true } - if to.IsVndkSp() || to.IsLlndk() { + if to.IsLlndk() { return false } @@ -3267,78 +3181,86 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) } - case libDepTag.rlib(): - rlibDep := RustRlibDep{LibPath: linkFile.Path(), CrateName: ccDep.CrateName(), LinkDirs: ccDep.ExportedCrateLinkDirs()} - depPaths.ReexportedRustRlibDeps = append(depPaths.ReexportedRustRlibDeps, rlibDep) - depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, rlibDep) - depPaths.IncludeDirs = append(depPaths.IncludeDirs, depExporterInfo.IncludeDirs...) - depPaths.ReexportedDirs = append(depPaths.ReexportedDirs, depExporterInfo.IncludeDirs...) - case libDepTag.static(): - staticLibraryInfo, isStaticLib := android.OtherModuleProvider(ctx, dep, StaticLibraryInfoProvider) - if !isStaticLib { - if !ctx.Config().AllowMissingDependencies() { - ctx.ModuleErrorf("module %q is not a static library", depName) - } else { - ctx.AddMissingDependencies([]string{depName}) + if ccDep.RustLibraryInterface() { + rlibDep := RustRlibDep{LibPath: linkFile.Path(), CrateName: ccDep.CrateName(), LinkDirs: ccDep.ExportedCrateLinkDirs()} + depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, rlibDep) + depPaths.IncludeDirs = append(depPaths.IncludeDirs, depExporterInfo.IncludeDirs...) + if libDepTag.wholeStatic { + depPaths.ReexportedDirs = append(depPaths.ReexportedDirs, depExporterInfo.IncludeDirs...) + depPaths.ReexportedRustRlibDeps = append(depPaths.ReexportedRustRlibDeps, rlibDep) + + // If whole_static, track this as we want to make sure that in a final linkage for a shared library, + // exported functions from the rust generated staticlib still exported. + if c.CcLibrary() && c.Shared() { + c.WholeRustStaticlib = true + } } - return - } - // Stubs lib doesn't link to the static lib dependencies. Don't set - // linkFile, depFile, and ptr. - if c.IsStubs() { - break - } + } else { + staticLibraryInfo, isStaticLib := android.OtherModuleProvider(ctx, dep, StaticLibraryInfoProvider) + if !isStaticLib { + if !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf("module %q is not a static library", depName) + } else { + ctx.AddMissingDependencies([]string{depName}) + } + return + } - linkFile = android.OptionalPathForPath(staticLibraryInfo.StaticLibrary) - if libDepTag.wholeStatic { - ptr = &depPaths.WholeStaticLibs - if len(staticLibraryInfo.Objects.objFiles) > 0 { - depPaths.WholeStaticLibObjs = depPaths.WholeStaticLibObjs.Append(staticLibraryInfo.Objects) - } else { - // This case normally catches prebuilt static - // libraries, but it can also occur when - // AllowMissingDependencies is on and the - // dependencies has no sources of its own - // but has a whole_static_libs dependency - // on a missing library. We want to depend - // on the .a file so that there is something - // in the dependency tree that contains the - // error rule for the missing transitive - // dependency. - depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, linkFile.Path()) + // Stubs lib doesn't link to the static lib dependencies. Don't set + // linkFile, depFile, and ptr. + if c.IsStubs() { + break } - depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, - staticLibraryInfo.WholeStaticLibsFromPrebuilts...) - } else { - switch libDepTag.Order { - case earlyLibraryDependency: - panic(fmt.Errorf("early static libs not suppported")) - case normalLibraryDependency: - // static dependencies will be handled separately so they can be ordered - // using transitive dependencies. - ptr = nil - directStaticDeps = append(directStaticDeps, staticLibraryInfo) - case lateLibraryDependency: - ptr = &depPaths.LateStaticLibs - default: - panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) + + linkFile = android.OptionalPathForPath(staticLibraryInfo.StaticLibrary) + if libDepTag.wholeStatic { + ptr = &depPaths.WholeStaticLibs + if len(staticLibraryInfo.Objects.objFiles) > 0 { + depPaths.WholeStaticLibObjs = depPaths.WholeStaticLibObjs.Append(staticLibraryInfo.Objects) + } else { + // This case normally catches prebuilt static + // libraries, but it can also occur when + // AllowMissingDependencies is on and the + // dependencies has no sources of its own + // but has a whole_static_libs dependency + // on a missing library. We want to depend + // on the .a file so that there is something + // in the dependency tree that contains the + // error rule for the missing transitive + // dependency. + depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, linkFile.Path()) + } + depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, + staticLibraryInfo.WholeStaticLibsFromPrebuilts...) + } else { + switch libDepTag.Order { + case earlyLibraryDependency: + panic(fmt.Errorf("early static libs not supported")) + case normalLibraryDependency: + // static dependencies will be handled separately so they can be ordered + // using transitive dependencies. + ptr = nil + directStaticDeps = append(directStaticDeps, staticLibraryInfo) + case lateLibraryDependency: + ptr = &depPaths.LateStaticLibs + default: + panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) + } } - } - // We re-export the Rust static_rlibs so rlib dependencies don't need to be redeclared by cc_library_static dependents. - // E.g. libfoo (cc_library_static) depends on libfoo.ffi (a rust_ffi rlib), libbar depending on libfoo shouldn't have to also add libfoo.ffi to static_rlibs. - depPaths.ReexportedRustRlibDeps = append(depPaths.ReexportedRustRlibDeps, depExporterInfo.RustRlibDeps...) - depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, depExporterInfo.RustRlibDeps...) + // Collect any exported Rust rlib deps from static libraries which have been included as whole_static_libs + depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, depExporterInfo.RustRlibDeps...) - if libDepTag.unexportedSymbols { - depPaths.LdFlags = append(depPaths.LdFlags, - "-Wl,--exclude-libs="+staticLibraryInfo.StaticLibrary.Base()) + if libDepTag.unexportedSymbols { + depPaths.LdFlags = append(depPaths.LdFlags, + "-Wl,--exclude-libs="+staticLibraryInfo.StaticLibrary.Base()) + } } } - if libDepTag.static() && !libDepTag.wholeStatic { + if libDepTag.static() && !libDepTag.wholeStatic && !ccDep.RustLibraryInterface() { if !ccDep.CcLibraryInterface() || !ccDep.Static() { ctx.ModuleErrorf("module %q not a static library", depName) return @@ -3425,12 +3347,14 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { c.Properties.AndroidMkSharedLibs = append( c.Properties.AndroidMkSharedLibs, makeLibName) case libDepTag.static(): - if libDepTag.wholeStatic { - c.Properties.AndroidMkWholeStaticLibs = append( - c.Properties.AndroidMkWholeStaticLibs, makeLibName) - } else { - c.Properties.AndroidMkStaticLibs = append( - c.Properties.AndroidMkStaticLibs, makeLibName) + if !ccDep.RustLibraryInterface() { + if libDepTag.wholeStatic { + c.Properties.AndroidMkWholeStaticLibs = append( + c.Properties.AndroidMkWholeStaticLibs, makeLibName) + } else { + c.Properties.AndroidMkStaticLibs = append( + c.Properties.AndroidMkStaticLibs, makeLibName) + } } } } else if !c.IsStubs() { @@ -3711,28 +3635,6 @@ func (c *Module) IntermPathForModuleOut() android.OptionalPath { return c.outputFile } -func (c *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if c.outputFile.Valid() { - return android.Paths{c.outputFile.Path()}, nil - } - return android.Paths{}, nil - case "unstripped": - if c.linker != nil { - return android.PathsIfNonNil(c.linker.unstrippedOutputFilePath()), nil - } - return nil, nil - case "stripped_all": - if c.linker != nil { - return android.PathsIfNonNil(c.linker.strippedAllOutputFilePath()), nil - } - return nil, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (c *Module) static() bool { if static, ok := c.linker.(interface { static() bool @@ -3833,15 +3735,6 @@ func (m *Module) Rlib() bool { func GetMakeLinkType(actx android.ModuleContext, c LinkableInterface) string { if c.InVendorOrProduct() { if c.IsLlndk() { - if !c.IsLlndkPublic() { - return "native:vndk_private" - } - return "native:vndk" - } - if c.IsVndk() && !c.IsVndkExt() { - if c.IsVndkPrivate() { - return "native:vndk_private" - } return "native:vndk" } if c.InProduct() { @@ -4043,15 +3936,6 @@ func (c *Module) AlwaysRequiresPlatformApexVariant() bool { return c.IsStubs() || c.Target().NativeBridge == android.NativeBridgeEnabled } -// Overrides android.ApexModuleBase.UniqueApexVariations -func (c *Module) UniqueApexVariations() bool { - // When a vendor APEX needs a VNDK lib in it (use_vndk_as_stable: false), it should be a unique - // APEX variation. Otherwise, another vendor APEX with use_vndk_as_stable:true may use a wrong - // variation of the VNDK lib because APEX variations are merged/grouped. - // TODO(b/274401041) Find a way to merge APEX variations for vendor apexes. - return c.UseVndk() && c.IsVndk() -} - func (c *Module) overriddenModules() []string { if o, ok := c.linker.(overridable); ok { return o.overriddenModules() @@ -4159,7 +4043,6 @@ func DefaultsFactory(props ...interface{}) android.Module { &TidyProperties{}, &CoverageProperties{}, &SAbiProperties{}, - &VndkProperties{}, <OProperties{}, &AfdoProperties{}, &OrderfileProperties{}, diff --git a/cc/cc_test.go b/cc/cc_test.go index 026d291e0..ccdaae58f 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -300,13 +300,9 @@ func TestDataLibs(t *testing.T) { config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module() - testBinary := module.(*Module).linker.(*testBinary) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Errorf("Expected cc_test to produce output files, error: %s", err) - return - } + testingModule := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon") + testBinary := testingModule.Module().(*Module).linker.(*testBinary) + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Errorf("expected exactly one output file. output files: [%s]", outputFiles) return @@ -356,12 +352,10 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module() + testingModule := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon") + module := testingModule.Module() testBinary := module.(*Module).linker.(*testBinary) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected cc_test to produce output files, error: %s", err) - } + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } @@ -658,7 +652,7 @@ func TestMakeLinkType(t *testing.T) { }{ {vendorVariant, "libvendor", "native:vendor"}, {vendorVariant, "libllndk", "native:vndk"}, - {vendorVariant27, "prevndk.vndk.27.arm.binder32", "native:vndk"}, + {vendorVariant27, "prevndk.vndk.27.arm.binder32", "native:vendor"}, {coreVariant, "libllndk", "native:platform"}, } for _, test := range tests { @@ -1407,12 +1401,10 @@ func TestDataLibsPrebuiltSharedTestLibrary(t *testing.T) { config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module() + testingModule := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon") + module := testingModule.Module() testBinary := module.(*Module).linker.(*testBinary) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected cc_test to produce output files, error: %s", err) - } + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Errorf("expected exactly one output file. output files: [%s]", outputFiles) } @@ -2908,8 +2900,6 @@ func TestIncludeDirectoryOrdering(t *testing.T) { PrepareForIntegrationTestWithCc, android.FixtureAddTextFile("external/foo/Android.bp", bp), ).RunTest(t) - // Use the arm variant instead of the arm64 variant so that it gets headers from - // ndk_libandroid_support to test LateStaticLibs. cflags := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_sdk_static").Output("obj/external/foo/foo.o").Args["cFlags"] var includes []string @@ -3120,12 +3110,8 @@ func TestStrippedAllOutputFile(t *testing.T) { ` config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("test_lib", "android_arm_armv7-a-neon_shared").Module() - outputFile, err := module.(android.OutputFileProducer).OutputFiles("stripped_all") - if err != nil { - t.Errorf("Expected cc_library to produce output files, error: %s", err) - return - } + testingModule := ctx.ModuleForTests("test_lib", "android_arm_armv7-a-neon_shared") + outputFile := testingModule.OutputFiles(t, "stripped_all") if !strings.HasSuffix(outputFile.Strings()[0], "/stripped_all/test_lib.so") { t.Errorf("Unexpected output file: %s", outputFile.Strings()[0]) return diff --git a/cc/cmake_ext_add_aidl_library.txt b/cc/cmake_ext_add_aidl_library.txt index af5bdf6c0..aa3235e30 100644 --- a/cc/cmake_ext_add_aidl_library.txt +++ b/cc/cmake_ext_add_aidl_library.txt @@ -25,7 +25,7 @@ function(add_aidl_library NAME LANG AIDLROOT SOURCES AIDLFLAGS) endif() set(DEPFILE_ARG) - if (NOT ${CMAKE_GENERATOR} MATCHES "Unix Makefiles") + if (NOT ${CMAKE_GENERATOR} STREQUAL "Unix Makefiles") set(DEPFILE_ARG DEPFILE "${GEN_SOURCE}.d") endif() @@ -57,7 +57,7 @@ function(add_aidl_library NAME LANG AIDLROOT SOURCES AIDLFLAGS) "${GEN_DIR}/include" ) - if (${LANG} MATCHES "ndk") + if (${LANG} STREQUAL "ndk") set(BINDER_LIB_NAME "libbinder_ndk_sdk") else() set(BINDER_LIB_NAME "libbinder_sdk") diff --git a/cc/cmake_main.txt b/cc/cmake_main.txt index e9177d6e2..f6e21a6c6 100644 --- a/cc/cmake_main.txt +++ b/cc/cmake_main.txt @@ -11,7 +11,11 @@ if (NOT ANDROID_BUILD_TOP) set(ANDROID_BUILD_TOP "${CMAKE_CURRENT_SOURCE_DIR}") endif() -set(PREBUILTS_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prebuilts/host/linux-x86/bin") +if ("${CMAKE_HOST_SYSTEM_PROCESSOR}" MATCHES "^(arm|aarch)") + set(PREBUILTS_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prebuilts/host/linux_musl-arm64/bin") +else() + set(PREBUILTS_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prebuilts/host/linux-x86/bin") +endif() if (NOT AIDL_BIN) find_program(AIDL_BIN aidl REQUIRED HINTS "${PREBUILTS_BIN_DIR}") endif() diff --git a/cc/cmake_module_cc.txt b/cc/cmake_module_cc.txt index 693406d52..0dc45aed1 100644 --- a/cc/cmake_module_cc.txt +++ b/cc/cmake_module_cc.txt @@ -1,6 +1,6 @@ <<$srcs := getSources .M>> <<$includeDirs := getIncludeDirs .Ctx .M>> -<<$cflags := (getCompilerProperties .M).Cflags>> +<<$cflags := getCflagsProperty .Ctx .M>> <<$deps := mapLibraries .Ctx .M (concat5 (getLinkerProperties .M).Whole_static_libs (getLinkerProperties .M).Static_libs diff --git a/cc/cmake_snapshot.go b/cc/cmake_snapshot.go index ad7beedab..a5f870800 100644 --- a/cc/cmake_snapshot.go +++ b/cc/cmake_snapshot.go @@ -15,7 +15,6 @@ package cc import ( - "android/soong/android" "bytes" _ "embed" "fmt" @@ -25,6 +24,8 @@ import ( "strings" "text/template" + "android/soong/android" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -61,8 +62,13 @@ var defaultUnportableFlags []string = []string{ } var ignoredSystemLibs []string = []string{ + "crtbegin_dynamic", + "crtend_android", + "libc", "libc++", "libc++_static", + "libdl", + "libm", "prebuilt_libclang_rt.builtins", "prebuilt_libclang_rt.ubsan_minimal", } @@ -141,11 +147,7 @@ func parseTemplate(templateContents string) *template.Template { return list.String() }, "toStrings": func(files android.Paths) []string { - strings := make([]string, len(files)) - for idx, file := range files { - strings[idx] = file.String() - } - return strings + return files.Strings() }, "concat5": func(list1 []string, list2 []string, list3 []string, list4 []string, list5 []string) []string { return append(append(append(append(list1, list2...), list3...), list4...), list5...) @@ -187,6 +189,10 @@ func parseTemplate(templateContents string) *template.Template { "getCompilerProperties": func(m *Module) BaseCompilerProperties { return m.compiler.baseCompilerProps() }, + "getCflagsProperty": func(ctx android.ModuleContext, m *Module) []string { + cflags := m.compiler.baseCompilerProps().Cflags + return cflags.GetOrDefault(ctx, nil) + }, "getLinkerProperties": func(m *Module) BaseLinkerProperties { return m.linker.baseLinkerProps() }, @@ -262,12 +268,13 @@ func executeTemplate(templ *template.Template, buffer *bytes.Buffer, data any) s } func (m *CmakeSnapshot) DepsMutator(ctx android.BottomUpMutatorContext) { - variations := []blueprint.Variation{ - {"os", "linux_glibc"}, - {"arch", "x86_64"}, + hostVariations := ctx.Config().BuildOSTarget.Variations() + ctx.AddVariationDependencies(hostVariations, cmakeSnapshotModuleTag, m.Properties.Modules...) + + if len(m.Properties.Prebuilts) > 0 { + prebuilts := append(m.Properties.Prebuilts, "libc++") + ctx.AddVariationDependencies(hostVariations, cmakeSnapshotPrebuiltTag, prebuilts...) } - ctx.AddVariationDependencies(variations, cmakeSnapshotModuleTag, m.Properties.Modules...) - ctx.AddVariationDependencies(variations, cmakeSnapshotPrebuiltTag, m.Properties.Prebuilts...) } func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -385,7 +392,8 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Merging CMakeLists.txt contents for every module directory var makefilesList android.Paths - for moduleDir, fragments := range moduleDirs { + for _, moduleDir := range android.SortedKeys(moduleDirs) { + fragments := moduleDirs[moduleDir] moduleCmakePath := android.PathForModuleGen(ctx, moduleDir, "CMakeLists.txt") makefilesList = append(makefilesList, moduleCmakePath) sort.Strings(fragments) @@ -425,8 +433,9 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Packaging all sources into the zip file if m.Properties.Include_sources { var sourcesList android.Paths - for _, file := range sourceFiles { - sourcesList = append(sourcesList, file) + for _, file := range android.SortedKeys(sourceFiles) { + path := sourceFiles[file] + sourcesList = append(sourcesList, path) } sourcesRspFile := android.PathForModuleObj(ctx, ctx.ModuleName()+"_sources.rsp") @@ -458,15 +467,8 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Finish generating the final zip file zipRule.Build(m.zipPath.String(), "archiving "+ctx.ModuleName()) -} -func (m *CmakeSnapshot) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{m.zipPath}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } + ctx.SetOutputFiles(android.Paths{m.zipPath}, "") } func (m *CmakeSnapshot) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/cc/compiler.go b/cc/compiler.go index ede6a5d0a..03f9899d8 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -50,7 +50,7 @@ type BaseCompilerProperties struct { Exclude_srcs []string `android:"path,arch_variant"` // list of module-specific flags that will be used for C and C++ compiles. - Cflags []string `android:"arch_variant"` + Cflags proptools.Configurable[[]string] `android:"arch_variant"` // list of module-specific flags that will be used for C++ compiles Cppflags []string `android:"arch_variant"` @@ -98,7 +98,7 @@ type BaseCompilerProperties struct { // list of generated headers to add to the include path. These are the names // of genrule modules. - Generated_headers []string `android:"arch_variant,variant_prepend"` + Generated_headers proptools.Configurable[[]string] `android:"arch_variant,variant_prepend"` // pass -frtti instead of -fno-rtti Rtti *bool `android:"arch_variant"` @@ -172,12 +172,6 @@ type BaseCompilerProperties struct { Target_api *string } - Debug, Release struct { - // list of module-specific flags that will be used for C and C++ compiles in debug or - // release builds - Cflags []string `android:"arch_variant"` - } `android:"arch_variant"` - Target struct { Vendor, Product struct { // list of source files that should only be used in vendor or @@ -274,7 +268,7 @@ func (compiler *baseCompiler) Srcs() android.Paths { } func (compiler *baseCompiler) appendCflags(flags []string) { - compiler.Properties.Cflags = append(compiler.Properties.Cflags, flags...) + compiler.Properties.Cflags.AppendSimpleValue(flags) } func (compiler *baseCompiler) appendAsflags(flags []string) { @@ -302,7 +296,7 @@ func (compiler *baseCompiler) compilerInit(ctx BaseModuleContext) {} func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { deps.GeneratedSources = append(deps.GeneratedSources, compiler.Properties.Generated_sources...) deps.GeneratedSources = removeListFromList(deps.GeneratedSources, compiler.Properties.Exclude_generated_sources) - deps.GeneratedHeaders = append(deps.GeneratedHeaders, compiler.Properties.Generated_headers...) + deps.GeneratedHeaders = append(deps.GeneratedHeaders, compiler.Properties.Generated_headers.GetOrDefault(ctx, nil)...) deps.AidlLibs = append(deps.AidlLibs, compiler.Properties.Aidl.Libs...) android.ProtoDeps(ctx, &compiler.Proto) @@ -372,7 +366,8 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps compiler.srcsBeforeGen = android.PathsForModuleSrcExcludes(ctx, compiler.Properties.Srcs, compiler.Properties.Exclude_srcs) compiler.srcsBeforeGen = append(compiler.srcsBeforeGen, deps.GeneratedSources...) - CheckBadCompilerFlags(ctx, "cflags", compiler.Properties.Cflags) + cflags := compiler.Properties.Cflags.GetOrDefault(ctx, nil) + CheckBadCompilerFlags(ctx, "cflags", cflags) CheckBadCompilerFlags(ctx, "cppflags", compiler.Properties.Cppflags) CheckBadCompilerFlags(ctx, "conlyflags", compiler.Properties.Conlyflags) CheckBadCompilerFlags(ctx, "asflags", compiler.Properties.Asflags) @@ -385,7 +380,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps esc := proptools.NinjaAndShellEscapeList - flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Cflags)...) + flags.Local.CFlags = append(flags.Local.CFlags, esc(cflags)...) flags.Local.CppFlags = append(flags.Local.CppFlags, esc(compiler.Properties.Cppflags)...) flags.Local.ConlyFlags = append(flags.Local.ConlyFlags, esc(compiler.Properties.Conlyflags)...) flags.Local.AsFlags = append(flags.Local.AsFlags, esc(compiler.Properties.Asflags)...) @@ -478,11 +473,6 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps ctx.ModuleErrorf("%s", err) } - CheckBadCompilerFlags(ctx, "release.cflags", compiler.Properties.Release.Cflags) - - // TODO: debug - flags.Local.CFlags = append(flags.Local.CFlags, esc(compiler.Properties.Release.Cflags)...) - if !ctx.DeviceConfig().BuildBrokenClangCFlags() && len(compiler.Properties.Clang_cflags) != 0 { ctx.PropertyErrorf("clang_cflags", "property is deprecated, see Changes.md file") } else { @@ -549,12 +539,10 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.Global.CommonFlags = append(flags.Global.CommonFlags, "${config.ExternalCflags}") } - if tc.Bionic() { - if Bool(compiler.Properties.Rtti) { - flags.Local.CppFlags = append(flags.Local.CppFlags, "-frtti") - } else { - flags.Local.CppFlags = append(flags.Local.CppFlags, "-fno-rtti") - } + if Bool(compiler.Properties.Rtti) { + flags.Local.CppFlags = append(flags.Local.CppFlags, "-frtti") + } else { + flags.Local.CppFlags = append(flags.Local.CppFlags, "-fno-rtti") } flags.Global.AsFlags = append(flags.Global.AsFlags, "${config.CommonGlobalAsflags}") @@ -806,9 +794,6 @@ type RustBindgenClangProperties struct { // be added to the include path using -I Local_include_dirs []string `android:"arch_variant,variant_prepend"` - // list of Rust static libraries. - Static_rlibs []string `android:"arch_variant,variant_prepend"` - // list of static libraries that provide headers for this binding. Static_libs []string `android:"arch_variant,variant_prepend"` @@ -819,7 +804,7 @@ type RustBindgenClangProperties struct { Header_libs []string `android:"arch_variant,variant_prepend"` // list of clang flags required to correctly interpret the headers. - Cflags []string `android:"arch_variant"` + Cflags proptools.Configurable[[]string] `android:"arch_variant"` // list of c++ specific clang flags required to correctly interpret the headers. // This is provided primarily to make sure cppflags defined in cc_defaults are pulled in. diff --git a/cc/config/darwin_host.go b/cc/config/darwin_host.go index 47c61b08f..485666954 100644 --- a/cc/config/darwin_host.go +++ b/cc/config/darwin_host.go @@ -50,6 +50,8 @@ var ( darwinSupportedSdkVersions = []string{ "11", "12", + "13", + "14", } darwinAvailableLibraries = append( diff --git a/cc/config/riscv64_device.go b/cc/config/riscv64_device.go index 724676ae6..e5e95f3d6 100644 --- a/cc/config/riscv64_device.go +++ b/cc/config/riscv64_device.go @@ -35,8 +35,6 @@ var ( // (Note that we'll probably want to wait for berberis to be good enough // that most people don't care about qemu's V performance either!) "-mno-implicit-float", - // TODO: remove when clang default changed (https://github.com/google/android-riscv64/issues/124) - "-mllvm -jump-is-expensive=false", } riscv64ArchVariantCflags = map[string][]string{} diff --git a/cc/genrule.go b/cc/genrule.go index 431a01c98..fe3b127ea 100644 --- a/cc/genrule.go +++ b/cc/genrule.go @@ -62,6 +62,8 @@ func GenRuleFactory() android.Module { android.InitApexModule(module) + android.InitDefaultableModule(module) + return module } @@ -77,6 +79,14 @@ var _ android.ImageInterface = (*GenruleExtraProperties)(nil) func (g *GenruleExtraProperties) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (g *GenruleExtraProperties) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return Bool(g.Vendor_available) || Bool(g.Odm_available) || ctx.SocSpecific() || ctx.DeviceSpecific() +} + +func (g *GenruleExtraProperties) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return Bool(g.Product_available) || ctx.ProductSpecific() +} + func (g *GenruleExtraProperties) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !(ctx.SocSpecific() || ctx.DeviceSpecific() || ctx.ProductSpecific()) } @@ -100,19 +110,8 @@ func (g *GenruleExtraProperties) RecoveryVariantNeeded(ctx android.BaseModuleCon } func (g *GenruleExtraProperties) ExtraImageVariations(ctx android.BaseModuleContext) []string { - var variants []string - vendorVariantRequired := Bool(g.Vendor_available) || Bool(g.Odm_available) || ctx.SocSpecific() || ctx.DeviceSpecific() - productVariantRequired := Bool(g.Product_available) || ctx.ProductSpecific() - - if vendorVariantRequired { - variants = append(variants, VendorVariation) - } - if productVariantRequired { - variants = append(variants, ProductVariation) - } - - return variants + return nil } -func (g *GenruleExtraProperties) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +func (g *GenruleExtraProperties) SetImageVariation(ctx android.BaseModuleContext, variation string) { } diff --git a/cc/image.go b/cc/image.go index f8c5ca5f1..2e52ccc63 100644 --- a/cc/image.go +++ b/cc/image.go @@ -17,8 +17,6 @@ package cc // functions to determine where a module is installed, etc. import ( - "fmt" - "reflect" "strings" "android/soong/android" @@ -41,18 +39,10 @@ const ( ) const ( - // VendorVariation is the variant name used for /vendor code that does not - // compile against the VNDK. - VendorVariation = "vendor" - // VendorVariationPrefix is the variant prefix used for /vendor code that compiles // against the VNDK. VendorVariationPrefix = "vendor." - // ProductVariation is the variant name used for /product code that does not - // compile against the VNDK. - ProductVariation = "product" - // ProductVariationPrefix is the variant prefix used for /product code that compiles // against the VNDK. ProductVariationPrefix = "product." @@ -119,12 +109,12 @@ func (c *Module) HasNonSystemVariants() bool { // Returns true if the module is "product" variant. Usually these modules are installed in /product func (c *Module) InProduct() bool { - return c.Properties.ImageVariation == ProductVariation + return c.Properties.ImageVariation == android.ProductVariation } // Returns true if the module is "vendor" variant. Usually these modules are installed in /vendor func (c *Module) InVendor() bool { - return c.Properties.ImageVariation == VendorVariation + return c.Properties.ImageVariation == android.VendorVariation } // Returns true if the module is "vendor" or "product" variant. This replaces previous UseVndk usages @@ -157,52 +147,6 @@ func (c *Module) OnlyInRecovery() bool { return c.ModuleBase.InstallInRecovery() } -func visitPropsAndCompareVendorAndProductProps(v reflect.Value) bool { - if v.Kind() != reflect.Struct { - return true - } - for i := 0; i < v.NumField(); i++ { - prop := v.Field(i) - if prop.Kind() == reflect.Struct && v.Type().Field(i).Name == "Target" { - vendor_prop := prop.FieldByName("Vendor") - product_prop := prop.FieldByName("Product") - if vendor_prop.Kind() != reflect.Struct && product_prop.Kind() != reflect.Struct { - // Neither Target.Vendor nor Target.Product is defined - continue - } - if vendor_prop.Kind() != reflect.Struct || product_prop.Kind() != reflect.Struct || - !reflect.DeepEqual(vendor_prop.Interface(), product_prop.Interface()) { - // If only one of either Target.Vendor or Target.Product is - // defined or they have different values, it fails the build - // since VNDK must have the same properties for both vendor - // and product variants. - return false - } - } else if !visitPropsAndCompareVendorAndProductProps(prop) { - // Visit the substructures to find Target.Vendor and Target.Product - return false - } - } - return true -} - -// In the case of VNDK, vendor and product variants must have the same properties. -// VNDK installs only one file and shares it for both vendor and product modules on -// runtime. We may not define different versions of a VNDK lib for each partition. -// This function is used only for the VNDK modules that is available to both vendor -// and product partitions. -func (c *Module) compareVendorAndProductProps() bool { - if !c.IsVndk() && !Bool(c.VendorProperties.Product_available) { - panic(fmt.Errorf("This is only for product available VNDK libs. %q is not a VNDK library or not product available", c.Name())) - } - for _, properties := range c.GetProperties() { - if !visitPropsAndCompareVendorAndProductProps(reflect.ValueOf(properties).Elem()) { - return false - } - } - return true -} - // ImageMutatableModule provides a common image mutation interface for LinkableInterface modules. type ImageMutatableModule interface { android.Module @@ -255,67 +199,20 @@ type ImageMutatableModule interface { // SetCoreVariantNeeded sets whether the Core Variant is needed. SetCoreVariantNeeded(b bool) + + // SetProductVariantNeeded sets whether the Product Variant is needed. + SetProductVariantNeeded(b bool) + + // SetVendorVariantNeeded sets whether the Vendor Variant is needed. + SetVendorVariantNeeded(b bool) } var _ ImageMutatableModule = (*Module)(nil) func (m *Module) ImageMutatorBegin(mctx android.BaseModuleContext) { - m.CheckVndkProperties(mctx) MutateImage(mctx, m) } -// CheckVndkProperties checks whether the VNDK-related properties are set correctly. -// If properties are not set correctly, results in a module context property error. -func (m *Module) CheckVndkProperties(mctx android.BaseModuleContext) { - vendorSpecific := mctx.SocSpecific() || mctx.DeviceSpecific() - productSpecific := mctx.ProductSpecific() - - if vndkdep := m.vndkdep; vndkdep != nil { - if vndkdep.isVndk() { - if vendorSpecific || productSpecific { - if !vndkdep.isVndkExt() { - mctx.PropertyErrorf("vndk", - "must set `extends: \"...\"` to vndk extension") - } else if Bool(m.VendorProperties.Vendor_available) { - mctx.PropertyErrorf("vendor_available", - "must not set at the same time as `vndk: {extends: \"...\"}`") - } else if Bool(m.VendorProperties.Product_available) { - mctx.PropertyErrorf("product_available", - "must not set at the same time as `vndk: {extends: \"...\"}`") - } - } else { - if vndkdep.isVndkExt() { - mctx.PropertyErrorf("vndk", - "must set `vendor: true` or `product_specific: true` to set `extends: %q`", - m.getVndkExtendsModuleName()) - } - if !Bool(m.VendorProperties.Vendor_available) { - mctx.PropertyErrorf("vndk", - "vendor_available must be set to true when `vndk: {enabled: true}`") - } - if Bool(m.VendorProperties.Product_available) { - // If a VNDK module creates both product and vendor variants, they - // must have the same properties since they share a single VNDK - // library on runtime. - if !m.compareVendorAndProductProps() { - mctx.ModuleErrorf("product properties must have the same values with the vendor properties for VNDK modules") - } - } - } - } else { - if vndkdep.isVndkSp() { - mctx.PropertyErrorf("vndk", - "must set `enabled: true` to set `support_system_process: true`") - } - if vndkdep.isVndkExt() { - mctx.PropertyErrorf("vndk", - "must set `enabled: true` to set `extends: %q`", - m.getVndkExtendsModuleName()) - } - } - } -} - func (m *Module) VendorAvailable() bool { return Bool(m.VendorProperties.Vendor_available) } @@ -368,6 +265,14 @@ func (m *Module) SetCoreVariantNeeded(b bool) { m.Properties.CoreVariantNeeded = b } +func (m *Module) SetProductVariantNeeded(b bool) { + m.Properties.ProductVariantNeeded = b +} + +func (m *Module) SetVendorVariantNeeded(b bool) { + m.Properties.VendorVariantNeeded = b +} + func (m *Module) SnapshotVersion(mctx android.BaseModuleContext) string { if snapshot, ok := m.linker.(SnapshotInterface); ok { return snapshot.Version() @@ -420,43 +325,36 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { } } + var vendorVariantNeeded bool = false + var productVariantNeeded bool = false var coreVariantNeeded bool = false var ramdiskVariantNeeded bool = false var vendorRamdiskVariantNeeded bool = false var recoveryVariantNeeded bool = false - var vendorVariants []string - var productVariants []string - - needVndkVersionVendorVariantForLlndk := false - if m.NeedsLlndkVariants() { // This is an LLNDK library. The implementation of the library will be on /system, // and vendor and product variants will be created with LLNDK stubs. // The LLNDK libraries need vendor variants even if there is no VNDK. coreVariantNeeded = true - vendorVariants = append(vendorVariants, "") - productVariants = append(productVariants, "") - // Generate vendor variants for boardVndkVersion only if the VNDK snapshot does not - // provide the LLNDK stub libraries. - if needVndkVersionVendorVariantForLlndk { - vendorVariants = append(vendorVariants, "") - } + vendorVariantNeeded = true + productVariantNeeded = true + } else if m.NeedsVendorPublicLibraryVariants() { // A vendor public library has the implementation on /vendor, with stub variants // for system and product. coreVariantNeeded = true - vendorVariants = append(vendorVariants, "") - productVariants = append(productVariants, "") + vendorVariantNeeded = true + productVariantNeeded = true } else if m.IsSnapshotPrebuilt() { // Make vendor variants only for the versions in BOARD_VNDK_VERSION and // PRODUCT_EXTRA_VNDK_VERSIONS. if m.InstallInRecovery() { recoveryVariantNeeded = true } else { - vendorVariants = append(vendorVariants, m.SnapshotVersion(mctx)) + m.AppendExtraVariant(VendorVariationPrefix + m.SnapshotVersion(mctx)) } - } else if m.HasNonSystemVariants() && !m.IsVndkExt() { + } else if m.HasNonSystemVariants() { // This will be available to /system unless it is product_specific // which will be handled later. coreVariantNeeded = true @@ -465,16 +363,16 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { // BOARD_VNDK_VERSION. The other modules are regarded as AOSP, or // PLATFORM_VNDK_VERSION. if m.HasVendorVariant() { - vendorVariants = append(vendorVariants, "") + vendorVariantNeeded = true } // product_available modules are available to /product. if m.HasProductVariant() { - productVariants = append(productVariants, "") + productVariantNeeded = true } } else if vendorSpecific && m.SdkVersion() == "" { // This will be available in /vendor (or /odm) only - vendorVariants = append(vendorVariants, "") + vendorVariantNeeded = true } else { // This is either in /system (or similar: /data), or is a // module built with the NDK. Modules built with the NDK @@ -485,7 +383,7 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { if coreVariantNeeded && productSpecific && m.SdkVersion() == "" { // The module has "product_specific: true" that does not create core variant. coreVariantNeeded = false - productVariants = append(productVariants, "") + productVariantNeeded = true } if m.RamdiskAvailable() { @@ -515,36 +413,32 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { coreVariantNeeded = false } - for _, variant := range android.FirstUniqueStrings(vendorVariants) { - if variant == "" { - m.AppendExtraVariant(VendorVariation) - } else { - m.AppendExtraVariant(VendorVariationPrefix + variant) - } - } - - for _, variant := range android.FirstUniqueStrings(productVariants) { - if variant == "" { - m.AppendExtraVariant(ProductVariation) - } else { - m.AppendExtraVariant(ProductVariationPrefix + variant) - } - } - m.SetRamdiskVariantNeeded(ramdiskVariantNeeded) m.SetVendorRamdiskVariantNeeded(vendorRamdiskVariantNeeded) m.SetRecoveryVariantNeeded(recoveryVariantNeeded) m.SetCoreVariantNeeded(coreVariantNeeded) + m.SetProductVariantNeeded(productVariantNeeded) + m.SetVendorVariantNeeded(vendorVariantNeeded) // Disable the module if no variants are needed. if !ramdiskVariantNeeded && !recoveryVariantNeeded && !coreVariantNeeded && + !productVariantNeeded && + !vendorVariantNeeded && len(m.ExtraVariants()) == 0 { m.Disable() } } +func (c *Module) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.VendorVariantNeeded +} + +func (c *Module) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.ProductVariantNeeded +} + func (c *Module) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return c.Properties.CoreVariantNeeded } @@ -628,30 +522,29 @@ func squashRamdiskSrcs(m *Module) { } } -func (c *Module) SetImageVariation(ctx android.BaseModuleContext, variant string, module android.Module) { - m := module.(*Module) +func (c *Module) SetImageVariation(ctx android.BaseModuleContext, variant string) { if variant == android.RamdiskVariation { - m.MakeAsPlatform() - squashRamdiskSrcs(m) + c.MakeAsPlatform() + squashRamdiskSrcs(c) } else if variant == android.VendorRamdiskVariation { - m.MakeAsPlatform() - squashVendorRamdiskSrcs(m) + c.MakeAsPlatform() + squashVendorRamdiskSrcs(c) } else if variant == android.RecoveryVariation { - m.MakeAsPlatform() - squashRecoverySrcs(m) - } else if strings.HasPrefix(variant, VendorVariation) { - m.Properties.ImageVariation = VendorVariation + c.MakeAsPlatform() + squashRecoverySrcs(c) + } else if strings.HasPrefix(variant, android.VendorVariation) { + c.Properties.ImageVariation = android.VendorVariation if strings.HasPrefix(variant, VendorVariationPrefix) { - m.Properties.VndkVersion = strings.TrimPrefix(variant, VendorVariationPrefix) + c.Properties.VndkVersion = strings.TrimPrefix(variant, VendorVariationPrefix) } - squashVendorSrcs(m) - } else if strings.HasPrefix(variant, ProductVariation) { - m.Properties.ImageVariation = ProductVariation + squashVendorSrcs(c) + } else if strings.HasPrefix(variant, android.ProductVariation) { + c.Properties.ImageVariation = android.ProductVariation if strings.HasPrefix(variant, ProductVariationPrefix) { - m.Properties.VndkVersion = strings.TrimPrefix(variant, ProductVariationPrefix) + c.Properties.VndkVersion = strings.TrimPrefix(variant, ProductVariationPrefix) } - squashProductSrcs(m) + squashProductSrcs(c) } if c.NeedsVendorPublicLibraryVariants() && diff --git a/cc/library.go b/cc/library.go index 090908f79..4373b46b4 100644 --- a/cc/library.go +++ b/cc/library.go @@ -149,7 +149,7 @@ type StaticOrSharedProperties struct { Sanitized Sanitized `android:"arch_variant"` - Cflags []string `android:"arch_variant"` + Cflags proptools.Configurable[[]string] `android:"arch_variant"` Enabled *bool `android:"arch_variant"` Whole_static_libs []string `android:"arch_variant"` @@ -464,9 +464,9 @@ func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Fla } if library.static() { - flags.Local.CFlags = append(flags.Local.CFlags, library.StaticProperties.Static.Cflags...) + flags.Local.CFlags = append(flags.Local.CFlags, library.StaticProperties.Static.Cflags.GetOrDefault(ctx, nil)...) } else if library.shared() { - flags.Local.CFlags = append(flags.Local.CFlags, library.SharedProperties.Shared.Cflags...) + flags.Local.CFlags = append(flags.Local.CFlags, library.SharedProperties.Shared.Cflags.GetOrDefault(ctx, nil)...) } if library.shared() { @@ -1135,8 +1135,12 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, linkerDeps = append(linkerDeps, deps.SharedLibsDeps...) linkerDeps = append(linkerDeps, deps.LateSharedLibsDeps...) - if generatedLib := generateRustStaticlib(ctx, deps.RustRlibDeps); generatedLib != nil { - deps.StaticLibs = append(deps.StaticLibs, generatedLib) + if generatedLib := generateRustStaticlib(ctx, deps.RustRlibDeps); generatedLib != nil && !library.buildStubs() { + if ctx.Module().(*Module).WholeRustStaticlib { + deps.WholeStaticLibs = append(deps.WholeStaticLibs, generatedLib) + } else { + deps.StaticLibs = append(deps.StaticLibs, generatedLib) + } } transformObjToDynamicBinary(ctx, objs.objFiles, sharedLibs, @@ -1762,22 +1766,7 @@ func (library *libraryDecorator) installSymlinkToRuntimeApex(ctx ModuleContext, func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) { if library.shared() { - if ctx.Device() && ctx.useVndk() { - // set subDir for VNDK extensions - if ctx.IsVndkExt() { - if ctx.isVndkSp() { - library.baseInstaller.subDir = "vndk-sp" - } else { - library.baseInstaller.subDir = "vndk" - } - } - - // do not install vndk libs - // vndk libs are packaged into VNDK APEX - if ctx.isVndk() && !ctx.IsVndkExt() && !ctx.Config().IsVndkDeprecated() && !ctx.inProduct() { - return - } - } else if library.hasStubsVariants() && !ctx.Host() && ctx.directlyInAnyApex() { + if library.hasStubsVariants() && !ctx.Host() && ctx.directlyInAnyApex() { // Bionic libraries (e.g. libc.so) is installed to the bootstrap subdirectory. // The original path becomes a symlink to the corresponding file in the // runtime APEX. @@ -2081,8 +2070,8 @@ func reuseStaticLibrary(mctx android.BottomUpMutatorContext, static, shared *Mod // Check libraries in addition to cflags, since libraries may be exporting different // include directories. - if len(staticCompiler.StaticProperties.Static.Cflags) == 0 && - len(sharedCompiler.SharedProperties.Shared.Cflags) == 0 && + if len(staticCompiler.StaticProperties.Static.Cflags.GetOrDefault(mctx, nil)) == 0 && + len(sharedCompiler.SharedProperties.Shared.Cflags.GetOrDefault(mctx, nil)) == 0 && len(staticCompiler.StaticProperties.Static.Whole_static_libs) == 0 && len(sharedCompiler.SharedProperties.Shared.Whole_static_libs) == 0 && len(staticCompiler.StaticProperties.Static.Static_libs) == 0 && @@ -2164,7 +2153,6 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { modules := mctx.CreateLocalVariations(variations...) static := modules[0].(LinkableInterface) shared := modules[1].(LinkableInterface) - static.SetStatic() shared.SetShared() @@ -2188,6 +2176,12 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { mctx.CreateLocalVariations(variations...) mctx.AliasVariation(variations[0]) } + if library.BuildRlibVariant() && library.IsRustFFI() && !buildStatic { + // Rust modules do not build static libs, but rlibs are used as if they + // were via `static_libs`. Thus we need to alias the BuildRlibVariant + // to "static" for Rust FFI libraries. + mctx.CreateAliasVariation("static", "") + } } } diff --git a/cc/library_stub.go b/cc/library_stub.go index 746b95171..6f06333ac 100644 --- a/cc/library_stub.go +++ b/cc/library_stub.go @@ -494,6 +494,12 @@ func BuildApiVariantName(baseName string, variant string, version string) string // Implement ImageInterface to generate image variants func (v *CcApiVariant) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (v *CcApiVariant) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return String(v.properties.Variant) == "llndk" +} +func (v *CcApiVariant) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return String(v.properties.Variant) == "llndk" +} func (v *CcApiVariant) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return inList(String(v.properties.Variant), []string{"ndk", "apex"}) } @@ -501,15 +507,6 @@ func (v *CcApiVariant) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool func (v *CcApiVariant) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { return false } func (v *CcApiVariant) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { return false } func (v *CcApiVariant) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { return false } -func (v *CcApiVariant) ExtraImageVariations(ctx android.BaseModuleContext) []string { - var variations []string - - if String(v.properties.Variant) == "llndk" { - variations = append(variations, VendorVariation) - variations = append(variations, ProductVariation) - } - - return variations -} -func (v *CcApiVariant) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +func (v *CcApiVariant) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil } +func (v *CcApiVariant) SetImageVariation(ctx android.BaseModuleContext, variation string) { } diff --git a/cc/linkable.go b/cc/linkable.go index fecc6a2e6..1672366a7 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -94,12 +94,16 @@ type LinkableInterface interface { SelectedStl() string BuildStaticVariant() bool + BuildRlibVariant() bool BuildSharedVariant() bool SetStatic() SetShared() IsPrebuilt() bool Toc() android.OptionalPath + // IsRustFFI returns true if this is a Rust FFI library. + IsRustFFI() bool + // IsFuzzModule returns true if this a *_fuzz module. IsFuzzModule() bool @@ -136,9 +140,6 @@ type LinkableInterface interface { // IsLlndk returns true for both LLNDK (public) and LLNDK-private libs. IsLlndk() bool - // IsLlndkPublic returns true only for LLNDK (public) libs. - IsLlndkPublic() bool - // HasLlndkStubs returns true if this library has a variant that will build LLNDK stubs. HasLlndkStubs() bool @@ -162,12 +163,6 @@ type LinkableInterface interface { // Bootstrap tests if this module is allowed to use non-APEX version of libraries. Bootstrap() bool - // IsVndkSp returns true if this is a VNDK-SP module. - IsVndkSp() bool - - IsVndk() bool - IsVndkExt() bool - IsVndkPrivate() bool IsVendorPublicLibrary() bool IsVndkPrebuiltLibrary() bool HasVendorVariant() bool diff --git a/cc/linker.go b/cc/linker.go index 1675df698..d2974c204 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -39,9 +39,6 @@ type BaseLinkerProperties struct { // the dependency's .a file will be linked into this module using -Wl,--whole-archive. Whole_static_libs []string `android:"arch_variant,variant_prepend"` - // list of Rust libs that should be statically linked into this module. - Static_rlibs []string `android:"arch_variant"` - // list of modules that should be statically linked into this module. Static_libs []string `android:"arch_variant,variant_prepend"` @@ -127,10 +124,6 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Header_libs []string - // list of Rust libs that should be statically linked to build vendor or product - // variant. - Static_rlibs []string - // list of shared libs that should not be used to build vendor or // product variant of the C/C++ module. Exclude_shared_libs []string @@ -159,10 +152,6 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Static_libs []string - // list of Rust libs that should be statically linked to build the recovery - // variant. - Static_rlibs []string - // list of shared libs that should not be used to build // the recovery variant of the C/C++ module. Exclude_shared_libs []string @@ -184,10 +173,6 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Static_libs []string - // list of Rust libs that should be statically linked to build the ramdisk - // variant. - Static_rlibs []string - // list of shared libs that should not be used to build // the ramdisk variant of the C/C++ module. Exclude_shared_libs []string @@ -205,10 +190,6 @@ type BaseLinkerProperties struct { // the vendor ramdisk variant of the C/C++ module. Exclude_shared_libs []string - // list of Rust libs that should be statically linked to build the vendor ramdisk - // variant. - Static_rlibs []string - // list of static libs that should not be used to build // the vendor ramdisk variant of the C/C++ module. Exclude_static_libs []string @@ -224,10 +205,6 @@ type BaseLinkerProperties struct { // variants. Shared_libs []string - // list of Rust libs that should be statically linked to build the vendor ramdisk - // variant. - Static_rlibs []string - // list of ehader libs that only should be used to build platform variant of // the C/C++ module. Header_libs []string @@ -322,7 +299,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.WholeStaticLibs = append(deps.WholeStaticLibs, linker.Properties.Whole_static_libs...) deps.HeaderLibs = append(deps.HeaderLibs, linker.Properties.Header_libs...) deps.StaticLibs = append(deps.StaticLibs, linker.Properties.Static_libs...) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Static_rlibs...) deps.SharedLibs = append(deps.SharedLibs, linker.Properties.Shared_libs...) deps.RuntimeLibs = append(deps.RuntimeLibs, linker.Properties.Runtime_libs...) @@ -366,7 +342,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Vendor.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Vendor.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Vendor.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Vendor.Static_rlibs...) } if ctx.inProduct() { @@ -380,7 +355,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Product.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Product.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Product.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Product.Static_rlibs...) } if ctx.inRecovery() { @@ -394,7 +368,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Recovery.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Recovery.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Recovery.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Recovery.Static_rlibs...) } if ctx.inRamdisk() { @@ -405,7 +378,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Ramdisk.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Ramdisk.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Ramdisk.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Ramdisk.Static_rlibs...) } if ctx.inVendorRamdisk() { @@ -415,7 +387,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Vendor_ramdisk.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Vendor_ramdisk.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Vendor_ramdisk.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Vendor_ramdisk.Static_rlibs...) } if !ctx.useSdk() { diff --git a/cc/llndk_library.go b/cc/llndk_library.go index d612e9e72..632c76d90 100644 --- a/cc/llndk_library.go +++ b/cc/llndk_library.go @@ -15,6 +15,7 @@ package cc import ( + "fmt" "strings" "android/soong/android" @@ -122,6 +123,16 @@ func (txt *llndkLibrariesTxtModule) GenerateAndroidBuildActions(ctx android.Modu ctx.SetOutputFiles(android.Paths{txt.outputFile}, "") } +func getVndkFileName(m *Module) (string, error) { + if library, ok := m.linker.(*libraryDecorator); ok { + return library.getLibNameHelper(m.BaseModuleName(), true, false) + ".so", nil + } + if prebuilt, ok := m.linker.(*prebuiltLibraryLinker); ok { + return prebuilt.libraryDecorator.getLibNameHelper(m.BaseModuleName(), true, false) + ".so", nil + } + return "", fmt.Errorf("VNDK library should have libraryDecorator or prebuiltLibraryLinker as linker: %T", m.linker) +} + func (txt *llndkLibrariesTxtModule) GenerateSingletonBuildActions(ctx android.SingletonContext) { if txt.outputFile.String() == "" { // Skip if target file path is empty @@ -197,8 +208,10 @@ func llndkMutator(mctx android.BottomUpMutatorContext) { m.VendorProperties.IsLLNDK = true } - if m.IsVndkPrebuiltLibrary() && !m.IsVndk() { - m.VendorProperties.IsLLNDK = true + if vndkprebuilt, ok := m.linker.(*vndkPrebuiltLibraryDecorator); ok { + if !Bool(vndkprebuilt.properties.Vndk.Enabled) { + m.VendorProperties.IsLLNDK = true + } } } @@ -54,6 +54,9 @@ type LTOProperties struct { // Use -fwhole-program-vtables cflag. Whole_program_vtables *bool + + // Use --lto-O0 flag. + Lto_O0 *bool } type lto struct { @@ -106,12 +109,8 @@ func (lto *lto) flags(ctx ModuleContext, flags Flags) Flags { ltoCFlags := []string{"-flto=thin", "-fsplit-lto-unit"} var ltoLdFlags []string - // The module did not explicitly turn on LTO. Only leverage LTO's - // better dead code elimination and CFG simplification, but do - // not perform costly optimizations for a balance between compile - // time, binary size and performance. - // Apply the same for Eng builds as well. - if !lto.ThinLTO() || ctx.Config().Eng() { + // Do not perform costly LTO optimizations for Eng builds. + if Bool(lto.Properties.Lto_O0) || ctx.optimizeForSize() || ctx.Config().Eng() { ltoLdFlags = append(ltoLdFlags, "-Wl,--lto-O0") } diff --git a/cc/object.go b/cc/object.go index 6c0391f3b..8b23295c7 100644 --- a/cc/object.go +++ b/cc/object.go @@ -220,7 +220,7 @@ func (object *objectLinker) unstrippedOutputFilePath() android.Path { } func (object *objectLinker) strippedAllOutputFilePath() android.Path { - panic("Not implemented.") + return nil } func (object *objectLinker) nativeCoverage() bool { diff --git a/cc/sanitize.go b/cc/sanitize.go index 3abba8065..d72d7d36f 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -681,12 +681,6 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Integer_overflow = nil } - // Also disable CFI for VNDK variants of components - if ctx.isVndk() && ctx.useVndk() { - s.Cfi = nil - s.Diag.Cfi = nil - } - if ctx.inRamdisk() || ctx.inVendorRamdisk() || ctx.inRecovery() { // HWASan ramdisk (which is built from recovery) goes over some bootloader limit. // Keep libc instrumented so that ramdisk / vendor_ramdisk / recovery can run hwasan-instrumented code if necessary. diff --git a/cc/testing.go b/cc/testing.go index 989be0272..02f992426 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -555,7 +555,6 @@ var PrepareForTestWithCcBuildComponents = android.GroupFixturePreparers( ctx.RegisterModuleType("cc_test_library", TestLibraryFactory) ctx.RegisterModuleType("vndk_prebuilt_shared", VndkPrebuiltSharedFactory) - RegisterVndkLibraryTxtTypes(ctx) RegisterLlndkLibraryTxtType(ctx) }), @@ -704,7 +703,6 @@ func CreateTestContext(config android.Config) *android.TestContext { ctx.RegisterModuleType("filegroup", android.FileGroupFactory) ctx.RegisterModuleType("vndk_prebuilt_shared", VndkPrebuiltSharedFactory) - RegisterVndkLibraryTxtTypes(ctx) RegisterLlndkLibraryTxtType(ctx) ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) diff --git a/cc/vndk.go b/cc/vndk.go index 7141ea812..9d196a0f8 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -15,25 +15,17 @@ package cc import ( - "errors" - "fmt" "strings" "android/soong/android" - "android/soong/etc" - - "github.com/google/blueprint" - "github.com/google/blueprint/proptools" ) const ( - llndkLibrariesTxt = "llndk.libraries.txt" - llndkLibrariesTxtForApex = "llndk.libraries.txt.apex" - vndkCoreLibrariesTxt = "vndkcore.libraries.txt" - vndkSpLibrariesTxt = "vndksp.libraries.txt" - vndkPrivateLibrariesTxt = "vndkprivate.libraries.txt" - vndkProductLibrariesTxt = "vndkproduct.libraries.txt" - vndkUsingCoreVariantLibrariesTxt = "vndkcorevariant.libraries.txt" + llndkLibrariesTxt = "llndk.libraries.txt" + vndkCoreLibrariesTxt = "vndkcore.libraries.txt" + vndkSpLibrariesTxt = "vndksp.libraries.txt" + vndkPrivateLibrariesTxt = "vndkprivate.libraries.txt" + vndkProductLibrariesTxt = "vndkproduct.libraries.txt" ) func VndkLibrariesTxtModules(vndkVersion string, ctx android.BaseModuleContext) []string { @@ -83,440 +75,9 @@ type VndkProperties struct { } } -type vndkdep struct { - Properties VndkProperties -} - -func (vndk *vndkdep) props() []interface{} { - return []interface{}{&vndk.Properties} -} - -func (vndk *vndkdep) isVndk() bool { - return Bool(vndk.Properties.Vndk.Enabled) -} - -func (vndk *vndkdep) isVndkSp() bool { - return Bool(vndk.Properties.Vndk.Support_system_process) -} - -func (vndk *vndkdep) isVndkExt() bool { - return vndk.Properties.Vndk.Extends != nil -} - -func (vndk *vndkdep) getVndkExtendsModuleName() string { - return String(vndk.Properties.Vndk.Extends) -} - -func (vndk *vndkdep) typeName() string { - if !vndk.isVndk() { - return "native:vendor" - } - if !vndk.isVndkExt() { - if !vndk.isVndkSp() { - return "native:vendor:vndk" - } - return "native:vendor:vndksp" - } - if !vndk.isVndkSp() { - return "native:vendor:vndkext" - } - return "native:vendor:vndkspext" -} - -// VNDK link type check from a module with UseVndk() == true. -func (vndk *vndkdep) vndkCheckLinkType(ctx android.BaseModuleContext, to *Module, tag blueprint.DependencyTag) { - if to.linker == nil { - return - } - if !vndk.isVndk() { - // Non-VNDK modules those installed to /vendor, /system/vendor, - // /product or /system/product cannot depend on VNDK-private modules - // that include VNDK-core-private, VNDK-SP-private and LLNDK-private. - if to.IsVndkPrivate() { - ctx.ModuleErrorf("non-VNDK module should not link to %q which has `private: true`", to.Name()) - } - } - if lib, ok := to.linker.(*libraryDecorator); !ok || !lib.shared() { - // Check only shared libraries. - // Other (static) libraries are allowed to link. - return - } - - if to.IsLlndk() { - // LL-NDK libraries are allowed to link - return - } - - if !to.UseVndk() { - ctx.ModuleErrorf("(%s) should not link to %q which is not a vendor-available library", - vndk.typeName(), to.Name()) - return - } - if tag == vndkExtDepTag { - // Ensure `extends: "name"` property refers a vndk module that has vendor_available - // and has identical vndk properties. - if to.vndkdep == nil || !to.vndkdep.isVndk() { - ctx.ModuleErrorf("`extends` refers a non-vndk module %q", to.Name()) - return - } - if vndk.isVndkSp() != to.vndkdep.isVndkSp() { - ctx.ModuleErrorf( - "`extends` refers a module %q with mismatched support_system_process", - to.Name()) - return - } - if to.IsVndkPrivate() { - ctx.ModuleErrorf( - "`extends` refers module %q which has `private: true`", - to.Name()) - return - } - } - if to.vndkdep == nil { - return - } - - // Check the dependencies of VNDK shared libraries. - if err := vndkIsVndkDepAllowed(vndk, to.vndkdep); err != nil { - ctx.ModuleErrorf("(%s) should not link to %q (%s): %v", - vndk.typeName(), to.Name(), to.vndkdep.typeName(), err) - return - } -} - -func vndkIsVndkDepAllowed(from *vndkdep, to *vndkdep) error { - // Check the dependencies of VNDK, VNDK-Ext, VNDK-SP, VNDK-SP-Ext and vendor modules. - if from.isVndkExt() { - if from.isVndkSp() { - if to.isVndk() && !to.isVndkSp() { - return errors.New("VNDK-SP extensions must not depend on VNDK or VNDK extensions") - } - return nil - } - // VNDK-Ext may depend on VNDK, VNDK-Ext, VNDK-SP, VNDK-SP-Ext, or vendor libs. - return nil - } - if from.isVndk() { - if to.isVndkExt() { - return errors.New("VNDK-core and VNDK-SP must not depend on VNDK extensions") - } - if from.isVndkSp() { - if !to.isVndkSp() { - return errors.New("VNDK-SP must only depend on VNDK-SP") - } - return nil - } - if !to.isVndk() { - return errors.New("VNDK-core must only depend on VNDK-core or VNDK-SP") - } - return nil - } - // Vendor modules may depend on VNDK, VNDK-Ext, VNDK-SP, VNDK-SP-Ext, or vendor libs. - return nil -} - -type moduleListerFunc func(ctx android.SingletonContext) (moduleNames, fileNames []string) - -var ( - vndkSPLibraries = vndkModuleLister(func(m *Module) bool { return m.VendorProperties.IsVNDKSP }) - vndkCoreLibraries = vndkModuleLister(func(m *Module) bool { return m.VendorProperties.IsVNDKCore }) - vndkPrivateLibraries = vndkModuleLister(func(m *Module) bool { return m.VendorProperties.IsVNDKPrivate }) - vndkProductLibraries = vndkModuleLister(func(m *Module) bool { return m.VendorProperties.IsVNDKProduct }) -) - -// vndkModuleLister takes a predicate that operates on a Module and returns a moduleListerFunc -// that produces a list of module names and output file names for which the predicate returns true. -func vndkModuleLister(predicate func(*Module) bool) moduleListerFunc { - return func(ctx android.SingletonContext) (moduleNames, fileNames []string) { - ctx.VisitAllModules(func(m android.Module) { - if c, ok := m.(*Module); ok && predicate(c) && !c.IsVndkPrebuiltLibrary() { - filename, err := getVndkFileName(c) - if err != nil { - ctx.ModuleErrorf(m, "%s", err) - } - moduleNames = append(moduleNames, ctx.ModuleName(m)) - fileNames = append(fileNames, filename) - } - }) - moduleNames = android.SortedUniqueStrings(moduleNames) - fileNames = android.SortedUniqueStrings(fileNames) - return - } -} - -// vndkModuleListRemover takes a moduleListerFunc and a prefix and returns a moduleListerFunc -// that returns the same lists as the input moduleListerFunc, but with modules with the -// given prefix removed. -func vndkModuleListRemover(lister moduleListerFunc, prefix string) moduleListerFunc { - return func(ctx android.SingletonContext) (moduleNames, fileNames []string) { - moduleNames, fileNames = lister(ctx) - filter := func(in []string) []string { - out := make([]string, 0, len(in)) - for _, lib := range in { - if strings.HasPrefix(lib, prefix) { - continue - } - out = append(out, lib) - } - return out - } - return filter(moduleNames), filter(fileNames) - } -} - -func processVndkLibrary(mctx android.BottomUpMutatorContext, m *Module) { - if m.InProduct() { - // We may skip the steps for the product variants because they - // are already covered by the vendor variants. - return - } - - name := m.BaseModuleName() - - if lib := m.library; lib != nil && lib.hasStubsVariants() && name != "libz" { - // b/155456180 libz is the ONLY exception here. We don't want to make - // libz an LLNDK library because we in general can't guarantee that - // libz will behave consistently especially about the compression. - // i.e. the compressed output might be different across releases. - // As the library is an external one, it's risky to keep the compatibility - // promise if it becomes an LLNDK. - mctx.PropertyErrorf("vndk.enabled", "This library provides stubs. Shouldn't be VNDK. Consider making it as LLNDK") - } - - if m.vndkdep.isVndkSp() { - m.VendorProperties.IsVNDKSP = true - } else { - m.VendorProperties.IsVNDKCore = true - } - if m.IsVndkPrivate() { - m.VendorProperties.IsVNDKPrivate = true - } - if Bool(m.VendorProperties.Product_available) { - m.VendorProperties.IsVNDKProduct = true - } -} - -// Check for modules that mustn't be VNDK -func shouldSkipVndkMutator(ctx android.ConfigAndErrorContext, m *Module) bool { - if !m.Enabled(ctx) { - return true - } - if !m.Device() { - // Skip non-device modules - return true - } - if m.Target().NativeBridge == android.NativeBridgeEnabled { - // Skip native_bridge modules - return true - } - return false -} - -func IsForVndkApex(mctx android.BottomUpMutatorContext, m *Module) bool { - if shouldSkipVndkMutator(mctx, m) { - return false - } - - // TODO(b/142675459): Use enabled: to select target device in vndk_prebuilt_shared - // When b/142675459 is landed, remove following check - if p, ok := m.linker.(*vndkPrebuiltLibraryDecorator); ok { - // prebuilt vndk modules should match with device - if !p.MatchesWithDevice(mctx.DeviceConfig()) { - return false - } - } - - if lib, ok := m.linker.(libraryInterface); ok { - // VNDK APEX doesn't need stub variants - if lib.buildStubs() { - return false - } - return lib.shared() && m.InVendor() && m.IsVndk() && !m.IsVndkExt() - } - return false -} - -// gather list of vndk-core, vndk-sp, and ll-ndk libs -func VndkMutator(mctx android.BottomUpMutatorContext) { - m, ok := mctx.Module().(*Module) - if !ok { - return - } - - if shouldSkipVndkMutator(mctx, m) { - return - } - - lib, isLib := m.linker.(*libraryDecorator) - prebuiltLib, isPrebuiltLib := m.linker.(*prebuiltLibraryLinker) - - if m.InVendorOrProduct() && isLib && lib.hasLLNDKStubs() { - m.VendorProperties.IsVNDKPrivate = Bool(lib.Properties.Llndk.Private) - } - if m.InVendorOrProduct() && isPrebuiltLib && prebuiltLib.hasLLNDKStubs() { - m.VendorProperties.IsVNDKPrivate = Bool(prebuiltLib.Properties.Llndk.Private) - } - - if (isLib && lib.buildShared()) || (isPrebuiltLib && prebuiltLib.buildShared()) { - if m.vndkdep != nil && m.vndkdep.isVndk() && !m.vndkdep.isVndkExt() { - processVndkLibrary(mctx, m) - return - } - } -} - -func init() { - RegisterVndkLibraryTxtTypes(android.InitRegistrationContext) -} - -func RegisterVndkLibraryTxtTypes(ctx android.RegistrationContext) { - ctx.RegisterParallelSingletonModuleType("vndksp_libraries_txt", vndkSPLibrariesTxtFactory) - ctx.RegisterParallelSingletonModuleType("vndkcore_libraries_txt", vndkCoreLibrariesTxtFactory) - ctx.RegisterParallelSingletonModuleType("vndkprivate_libraries_txt", vndkPrivateLibrariesTxtFactory) - ctx.RegisterParallelSingletonModuleType("vndkproduct_libraries_txt", vndkProductLibrariesTxtFactory) -} - -type vndkLibrariesTxt struct { - android.SingletonModuleBase - - lister moduleListerFunc - makeVarName string - filterOutFromMakeVar string - - properties VndkLibrariesTxtProperties - - outputFile android.OutputPath - moduleNames []string - fileNames []string -} - -type VndkLibrariesTxtProperties struct { - Insert_vndk_version *bool - Stem *string -} - -var _ etc.PrebuiltEtcModule = &vndkLibrariesTxt{} - -// vndksp_libraries_txt is a singleton module whose content is a list of VNDKSP libraries -// generated by Soong but can be referenced by other modules. -// For example, apex_vndk can depend on these files as prebuilt. -func vndkSPLibrariesTxtFactory() android.SingletonModule { - return newVndkLibrariesTxt(vndkSPLibraries, "VNDK_SAMEPROCESS_LIBRARIES") -} - -// vndkcore_libraries_txt is a singleton module whose content is a list of VNDK core libraries -// generated by Soong but can be referenced by other modules. -// For example, apex_vndk can depend on these files as prebuilt. -func vndkCoreLibrariesTxtFactory() android.SingletonModule { - return newVndkLibrariesTxt(vndkCoreLibraries, "VNDK_CORE_LIBRARIES") -} - -// vndkprivate_libraries_txt is a singleton module whose content is a list of VNDK private libraries -// generated by Soong but can be referenced by other modules. -// For example, apex_vndk can depend on these files as prebuilt. -func vndkPrivateLibrariesTxtFactory() android.SingletonModule { - return newVndkLibrariesTxt(vndkPrivateLibraries, "VNDK_PRIVATE_LIBRARIES") -} - -// vndkproduct_libraries_txt is a singleton module whose content is a list of VNDK product libraries -// generated by Soong but can be referenced by other modules. -// For example, apex_vndk can depend on these files as prebuilt. -func vndkProductLibrariesTxtFactory() android.SingletonModule { - return newVndkLibrariesTxt(vndkProductLibraries, "VNDK_PRODUCT_LIBRARIES") -} - -func newVndkLibrariesWithMakeVarFilter(lister moduleListerFunc, makeVarName string, filter string) android.SingletonModule { - m := &vndkLibrariesTxt{ - lister: lister, - makeVarName: makeVarName, - filterOutFromMakeVar: filter, - } - m.AddProperties(&m.properties) - android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibCommon) - return m -} - -func newVndkLibrariesTxt(lister moduleListerFunc, makeVarName string) android.SingletonModule { - return newVndkLibrariesWithMakeVarFilter(lister, makeVarName, "") -} - func insertVndkVersion(filename string, vndkVersion string) string { if index := strings.LastIndex(filename, "."); index != -1 { return filename[:index] + "." + vndkVersion + filename[index:] } return filename } - -func (txt *vndkLibrariesTxt) GenerateAndroidBuildActions(ctx android.ModuleContext) { - filename := proptools.StringDefault(txt.properties.Stem, txt.Name()) - - txt.outputFile = android.PathForModuleOut(ctx, filename).OutputPath - - installPath := android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(installPath, filename, txt.outputFile) - - ctx.SetOutputFiles(android.Paths{txt.outputFile}, "") -} - -func (txt *vndkLibrariesTxt) GenerateSingletonBuildActions(ctx android.SingletonContext) { - txt.moduleNames, txt.fileNames = txt.lister(ctx) - android.WriteFileRule(ctx, txt.outputFile, strings.Join(txt.fileNames, "\n")) -} - -func (txt *vndkLibrariesTxt) AndroidMkEntries() []android.AndroidMkEntries { - return []android.AndroidMkEntries{android.AndroidMkEntries{ - Class: "ETC", - OutputFile: android.OptionalPathForPath(txt.outputFile), - ExtraEntries: []android.AndroidMkExtraEntriesFunc{ - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.SetString("LOCAL_MODULE_STEM", txt.outputFile.Base()) - }, - }, - }} -} - -func (txt *vndkLibrariesTxt) MakeVars(ctx android.MakeVarsContext) { - if txt.makeVarName == "" { - return - } - - filter := func(modules []string, prefix string) []string { - if prefix == "" { - return modules - } - var result []string - for _, module := range modules { - if strings.HasPrefix(module, prefix) { - continue - } else { - result = append(result, module) - } - } - return result - } - ctx.Strict(txt.makeVarName, strings.Join(filter(txt.moduleNames, txt.filterOutFromMakeVar), " ")) -} - -// PrebuiltEtcModule interface -func (txt *vndkLibrariesTxt) BaseDir() string { - return "etc" -} - -// PrebuiltEtcModule interface -func (txt *vndkLibrariesTxt) SubDir() string { - return "" -} - -func (txt *vndkLibrariesTxt) OutputFiles(tag string) (android.Paths, error) { - return android.Paths{txt.outputFile}, nil -} - -func getVndkFileName(m *Module) (string, error) { - if library, ok := m.linker.(*libraryDecorator); ok { - return library.getLibNameHelper(m.BaseModuleName(), true, false) + ".so", nil - } - if prebuilt, ok := m.linker.(*prebuiltLibraryLinker); ok { - return prebuilt.libraryDecorator.getLibNameHelper(m.BaseModuleName(), true, false) + ".so", nil - } - return "", fmt.Errorf("VNDK library should have libraryDecorator or prebuiltLibraryLinker as linker: %T", m.linker) -} diff --git a/cc/vndk_prebuilt.go b/cc/vndk_prebuilt.go index 4d2412fe5..e7dff4012 100644 --- a/cc/vndk_prebuilt.go +++ b/cc/vndk_prebuilt.go @@ -49,6 +49,8 @@ var ( // }, // } type vndkPrebuiltProperties struct { + VndkProperties + // VNDK snapshot version. Version *string @@ -268,3 +270,14 @@ func VndkPrebuiltSharedFactory() android.Module { func init() { android.RegisterModuleType("vndk_prebuilt_shared", VndkPrebuiltSharedFactory) } + +func IsForVndkApex(mctx android.BottomUpMutatorContext, m *Module) bool { + if !m.Enabled(mctx) { + return true + } + + if p, ok := m.linker.(*vndkPrebuiltLibraryDecorator); ok { + return p.MatchesWithDevice(mctx.DeviceConfig()) && Bool(p.properties.Vndk.Enabled) + } + return false +} diff --git a/cmd/release_config/release_config/main.go b/cmd/release_config/release_config/main.go index bd4ab4907..d06b2b796 100644 --- a/cmd/release_config/release_config/main.go +++ b/cmd/release_config/release_config/main.go @@ -88,7 +88,7 @@ func main() { return } // Write the makefile where release_config.mk is going to look for it. - err = configs.WriteMakefile(makefilePath, targetRelease) + err = config.WriteMakefile(makefilePath, targetRelease, configs) if err != nil { panic(err) } @@ -97,7 +97,7 @@ func main() { for _, c := range configs.GetSortedReleaseConfigs() { if c.Name != targetRelease { makefilePath = filepath.Join(outputDir, fmt.Sprintf("release_config-%s-%s.varmk", product, c.Name)) - err = configs.WriteMakefile(makefilePath, c.Name) + err = config.WriteMakefile(makefilePath, c.Name, configs) if err != nil { panic(err) } diff --git a/cmd/release_config/release_config_lib/Android.bp b/cmd/release_config/release_config_lib/Android.bp index 0c67e1106..17251bdba 100644 --- a/cmd/release_config/release_config_lib/Android.bp +++ b/cmd/release_config/release_config_lib/Android.bp @@ -24,6 +24,7 @@ bootstrap_go_package { "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", "soong-cmd-release_config-proto", + "blueprint-pathtools", ], srcs: [ "flag_artifact.go", diff --git a/cmd/release_config/release_config_lib/flag_artifact.go b/cmd/release_config/release_config_lib/flag_artifact.go index 6d365952c..cfac7d794 100644 --- a/cmd/release_config/release_config_lib/flag_artifact.go +++ b/cmd/release_config/release_config_lib/flag_artifact.go @@ -67,7 +67,7 @@ func FlagArtifactsFactory(artifactsPath string) *FlagArtifacts { if artifactsPath != "" { fas := &rc_proto.FlagArtifacts{} LoadMessage(artifactsPath, fas) - for _, fa_pb := range fas.FlagArtifacts { + for _, fa_pb := range fas.Flags { fa := &FlagArtifact{} fa.FlagDeclaration = fa_pb.GetFlagDeclaration() if val := fa_pb.GetValue(); val != nil { @@ -82,6 +82,15 @@ func FlagArtifactsFactory(artifactsPath string) *FlagArtifacts { return &ret } +func (fas *FlagArtifacts) SortedFlagNames() []string { + var names []string + for k, _ := range *fas { + names = append(names, k) + } + slices.Sort(names) + return names +} + func (fa *FlagArtifact) GenerateFlagDeclarationArtifact() *rc_proto.FlagDeclarationArtifact { ret := &rc_proto.FlagDeclarationArtifact{ Name: fa.FlagDeclaration.Name, @@ -135,9 +144,11 @@ func (src *FlagArtifact) Clone() *FlagArtifact { value := &rc_proto.Value{} proto.Merge(value, src.Value) return &FlagArtifact{ - FlagDeclaration: src.FlagDeclaration, - Traces: src.Traces, - Value: value, + FlagDeclaration: src.FlagDeclaration, + Traces: src.Traces, + Value: value, + DeclarationIndex: src.DeclarationIndex, + Redacted: src.Redacted, } } diff --git a/cmd/release_config/release_config_lib/release_config.go b/cmd/release_config/release_config_lib/release_config.go index 02b693c78..6d71d9343 100644 --- a/cmd/release_config/release_config_lib/release_config.go +++ b/cmd/release_config/release_config_lib/release_config.go @@ -17,6 +17,7 @@ package release_config_lib import ( "cmp" "fmt" + "os" "path/filepath" "regexp" "slices" @@ -169,8 +170,12 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro if err != nil { return err } - iConfig.GenerateReleaseConfig(configs) - if err := config.InheritConfig(iConfig); err != nil { + err = iConfig.GenerateReleaseConfig(configs) + if err != nil { + return err + } + err = config.InheritConfig(iConfig) + if err != nil { return err } } @@ -221,8 +226,16 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro config.PriorStagesMap[priorStage] = true } myDirsMap[contrib.DeclarationIndex] = true - if config.AconfigFlagsOnly && len(contrib.FlagValues) > 0 { - return fmt.Errorf("%s does not allow build flag overrides", config.Name) + if config.AconfigFlagsOnly { + // AconfigFlagsOnly allows very very few build flag values, all of them are part of aconfig flags. + allowedFlags := map[string]bool{ + "RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS": true, + } + for _, fv := range contrib.FlagValues { + if !allowedFlags[*fv.proto.Name] { + return fmt.Errorf("%s does not allow build flag overrides", config.Name) + } + } } for _, value := range contrib.FlagValues { name := *value.proto.Name @@ -251,7 +264,7 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro myAconfigValueSets := []string{} myAconfigValueSetsMap := map[string]bool{} for _, v := range strings.Split(releaseAconfigValueSets.Value.GetStringValue(), " ") { - if myAconfigValueSetsMap[v] { + if v == "" || myAconfigValueSetsMap[v] { continue } myAconfigValueSetsMap[v] = true @@ -277,13 +290,13 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro if _, ok := config.PartitionBuildFlags[container]; !ok { config.PartitionBuildFlags[container] = &rc_proto.FlagArtifacts{} } - config.PartitionBuildFlags[container].FlagArtifacts = append(config.PartitionBuildFlags[container].FlagArtifacts, artifact) + config.PartitionBuildFlags[container].Flags = append(config.PartitionBuildFlags[container].Flags, artifact) } } config.ReleaseConfigArtifact = &rc_proto.ReleaseConfigArtifact{ Name: proto.String(config.Name), OtherNames: config.OtherNames, - FlagArtifacts: func() []*rc_proto.FlagArtifact { + Flags: func() []*rc_proto.FlagArtifact { ret := []*rc_proto.FlagArtifact{} flagNames := []string{} for k := range config.FlagArtifacts { @@ -310,10 +323,95 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro return nil } +// Write the makefile for this targetRelease. +func (config *ReleaseConfig) WriteMakefile(outFile, targetRelease string, configs *ReleaseConfigs) error { + makeVars := make(map[string]string) + + myFlagArtifacts := config.FlagArtifacts.Clone() + + // Add any RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS variables. + var extraAconfigReleaseConfigs []string + if extraAconfigValueSetsValue, ok := config.FlagArtifacts["RELEASE_ACONFIG_EXTRA_RELEASE_CONFIGS"]; ok { + if val := MarshalValue(extraAconfigValueSetsValue.Value); len(val) > 0 { + extraAconfigReleaseConfigs = strings.Split(val, " ") + } + } + for _, rcName := range extraAconfigReleaseConfigs { + rc, err := configs.GetReleaseConfig(rcName) + if err != nil { + return err + } + myFlagArtifacts["RELEASE_ACONFIG_VALUE_SETS_"+rcName] = rc.FlagArtifacts["RELEASE_ACONFIG_VALUE_SETS"] + myFlagArtifacts["RELEASE_ACONFIG_FLAG_DEFAULT_PERMISSION_"+rcName] = rc.FlagArtifacts["RELEASE_ACONFIG_FLAG_DEFAULT_PERMISSION"] + } + + // Sort the flags by name first. + names := myFlagArtifacts.SortedFlagNames() + partitions := make(map[string][]string) + + vNames := []string{} + addVar := func(name, suffix, value string) { + fullName := fmt.Sprintf("_ALL_RELEASE_FLAGS.%s.%s", name, suffix) + vNames = append(vNames, fullName) + makeVars[fullName] = value + } + + for _, name := range names { + flag := myFlagArtifacts[name] + decl := flag.FlagDeclaration + + for _, container := range decl.Containers { + partitions[container] = append(partitions[container], name) + } + value := MarshalValue(flag.Value) + makeVars[name] = value + addVar(name, "TYPE", ValueType(flag.Value)) + addVar(name, "PARTITIONS", strings.Join(decl.Containers, " ")) + addVar(name, "DEFAULT", MarshalValue(decl.Value)) + addVar(name, "VALUE", value) + addVar(name, "DECLARED_IN", *flag.Traces[0].Source) + addVar(name, "SET_IN", *flag.Traces[len(flag.Traces)-1].Source) + addVar(name, "NAMESPACE", *decl.Namespace) + } + pNames := []string{} + for k := range partitions { + pNames = append(pNames, k) + } + slices.Sort(pNames) + + // Now sort the make variables, and output them. + slices.Sort(vNames) + + // Write the flags as: + // _ALL_RELELASE_FLAGS + // _ALL_RELEASE_FLAGS.PARTITIONS.* + // all _ALL_RELEASE_FLAGS.*, sorted by name + // Final flag values, sorted by name. + data := fmt.Sprintf("# TARGET_RELEASE=%s\n", config.Name) + if targetRelease != config.Name { + data += fmt.Sprintf("# User specified TARGET_RELEASE=%s\n", targetRelease) + } + // As it stands this list is not per-product, but conceptually it is, and will be. + data += fmt.Sprintf("ALL_RELEASE_CONFIGS_FOR_PRODUCT :=$= %s\n", strings.Join(configs.GetAllReleaseNames(), " ")) + data += fmt.Sprintf("_used_files := %s\n", strings.Join(config.GetSortedFileList(), " ")) + data += fmt.Sprintf("_ALL_RELEASE_FLAGS :=$= %s\n", strings.Join(names, " ")) + for _, pName := range pNames { + data += fmt.Sprintf("_ALL_RELEASE_FLAGS.PARTITIONS.%s :=$= %s\n", pName, strings.Join(partitions[pName], " ")) + } + for _, vName := range vNames { + data += fmt.Sprintf("%s :=$= %s\n", vName, makeVars[vName]) + } + data += "\n\n# Values for all build flags\n" + for _, name := range names { + data += fmt.Sprintf("%s :=$= %s\n", name, makeVars[name]) + } + return os.WriteFile(outFile, []byte(data), 0644) +} + func (config *ReleaseConfig) WritePartitionBuildFlags(outDir string) error { var err error for partition, flags := range config.PartitionBuildFlags { - slices.SortFunc(flags.FlagArtifacts, func(a, b *rc_proto.FlagArtifact) int { + slices.SortFunc(flags.Flags, func(a, b *rc_proto.FlagArtifact) int { return cmp.Compare(*a.FlagDeclaration.Name, *b.FlagDeclaration.Name) }) // The json file name must not be modified as this is read from diff --git a/cmd/release_config/release_config_lib/release_configs.go b/cmd/release_config/release_config_lib/release_configs.go index 052cde898..f2e138801 100644 --- a/cmd/release_config/release_config_lib/release_configs.go +++ b/cmd/release_config/release_config_lib/release_configs.go @@ -395,94 +395,10 @@ func (configs *ReleaseConfigs) GetAllReleaseNames() []string { allReleaseNames = append(allReleaseNames, v.Name) allReleaseNames = append(allReleaseNames, v.OtherNames...) } - slices.SortFunc(allReleaseNames, func(a, b string) int { - return cmp.Compare(a, b) - }) + slices.Sort(allReleaseNames) return allReleaseNames } -// Write the makefile for this targetRelease. -func (configs *ReleaseConfigs) WriteMakefile(outFile, targetRelease string) error { - makeVars := make(map[string]string) - config, err := configs.GetReleaseConfig(targetRelease) - if err != nil { - return err - } - - myFlagArtifacts := config.FlagArtifacts.Clone() - // Sort the flags by name first. - names := []string{} - for k, _ := range myFlagArtifacts { - names = append(names, k) - } - slices.SortFunc(names, func(a, b string) int { - return cmp.Compare(a, b) - }) - partitions := make(map[string][]string) - - vNames := []string{} - addVar := func(name, suffix, value string) { - fullName := fmt.Sprintf("_ALL_RELEASE_FLAGS.%s.%s", name, suffix) - vNames = append(vNames, fullName) - makeVars[fullName] = value - } - - for _, name := range names { - flag := myFlagArtifacts[name] - decl := flag.FlagDeclaration - - for _, container := range decl.Containers { - partitions[container] = append(partitions[container], name) - } - value := MarshalValue(flag.Value) - makeVars[name] = value - addVar(name, "TYPE", ValueType(flag.Value)) - addVar(name, "PARTITIONS", strings.Join(decl.Containers, " ")) - addVar(name, "DEFAULT", MarshalValue(decl.Value)) - addVar(name, "VALUE", value) - addVar(name, "DECLARED_IN", *flag.Traces[0].Source) - addVar(name, "SET_IN", *flag.Traces[len(flag.Traces)-1].Source) - addVar(name, "NAMESPACE", *decl.Namespace) - } - pNames := []string{} - for k := range partitions { - pNames = append(pNames, k) - } - slices.SortFunc(pNames, func(a, b string) int { - return cmp.Compare(a, b) - }) - - // Now sort the make variables, and output them. - slices.SortFunc(vNames, func(a, b string) int { - return cmp.Compare(a, b) - }) - - // Write the flags as: - // _ALL_RELELASE_FLAGS - // _ALL_RELEASE_FLAGS.PARTITIONS.* - // all _ALL_RELEASE_FLAGS.*, sorted by name - // Final flag values, sorted by name. - data := fmt.Sprintf("# TARGET_RELEASE=%s\n", config.Name) - if targetRelease != config.Name { - data += fmt.Sprintf("# User specified TARGET_RELEASE=%s\n", targetRelease) - } - // As it stands this list is not per-product, but conceptually it is, and will be. - data += fmt.Sprintf("ALL_RELEASE_CONFIGS_FOR_PRODUCT :=$= %s\n", strings.Join(configs.GetAllReleaseNames(), " ")) - data += fmt.Sprintf("_used_files := %s\n", strings.Join(config.GetSortedFileList(), " ")) - data += fmt.Sprintf("_ALL_RELEASE_FLAGS :=$= %s\n", strings.Join(names, " ")) - for _, pName := range pNames { - data += fmt.Sprintf("_ALL_RELEASE_FLAGS.PARTITIONS.%s :=$= %s\n", pName, strings.Join(partitions[pName], " ")) - } - for _, vName := range vNames { - data += fmt.Sprintf("%s :=$= %s\n", vName, makeVars[vName]) - } - data += "\n\n# Values for all build flags\n" - for _, name := range names { - data += fmt.Sprintf("%s :=$= %s\n", name, makeVars[name]) - } - return os.WriteFile(outFile, []byte(data), 0644) -} - func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) error { otherNames := make(map[string][]string) for aliasName, aliasTarget := range configs.Aliases { diff --git a/cmd/release_config/release_config_lib/util.go b/cmd/release_config/release_config_lib/util.go index 0a19efe01..9919c7081 100644 --- a/cmd/release_config/release_config_lib/util.go +++ b/cmd/release_config/release_config_lib/util.go @@ -25,6 +25,7 @@ import ( "slices" "strings" + "github.com/google/blueprint/pathtools" "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/proto" ) @@ -101,7 +102,7 @@ func WriteFormattedMessage(path, format string, message proto.Message) (err erro if err != nil { return err } - return os.WriteFile(path, data, 0644) + return pathtools.WriteFileIfChanged(path, data, 0644) } // Read a message from a file. diff --git a/cmd/release_config/release_config_proto/build_flags_common.pb.go b/cmd/release_config/release_config_proto/build_flags_common.pb.go index 1e927db6c..82fbcfaf9 100644 --- a/cmd/release_config/release_config_proto/build_flags_common.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_common.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.33.0 // protoc v3.21.12 // source: build_flags_common.proto diff --git a/cmd/release_config/release_config_proto/build_flags_declarations.pb.go b/cmd/release_config/release_config_proto/build_flags_declarations.pb.go index c0573edf3..73a7e8790 100644 --- a/cmd/release_config/release_config_proto/build_flags_declarations.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_declarations.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.33.0 // protoc v3.21.12 // source: build_flags_declarations.proto diff --git a/cmd/release_config/release_config_proto/build_flags_out.pb.go b/cmd/release_config/release_config_proto/build_flags_out.pb.go index 309ec3452..b246eb66d 100644 --- a/cmd/release_config/release_config_proto/build_flags_out.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_out.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.33.0 // protoc v3.21.12 // source: build_flags_out.proto @@ -163,7 +163,7 @@ type FlagArtifacts struct { unknownFields protoimpl.UnknownFields // The artifacts - FlagArtifacts []*FlagArtifact `protobuf:"bytes,1,rep,name=flag_artifacts,json=flagArtifacts" json:"flag_artifacts,omitempty"` + Flags []*FlagArtifact `protobuf:"bytes,1,rep,name=flags" json:"flags,omitempty"` } func (x *FlagArtifacts) Reset() { @@ -198,9 +198,9 @@ func (*FlagArtifacts) Descriptor() ([]byte, []int) { return file_build_flags_out_proto_rawDescGZIP(), []int{2} } -func (x *FlagArtifacts) GetFlagArtifacts() []*FlagArtifact { +func (x *FlagArtifacts) GetFlags() []*FlagArtifact { if x != nil { - return x.FlagArtifacts + return x.Flags } return nil } @@ -217,7 +217,7 @@ type ReleaseConfigArtifact struct { OtherNames []string `protobuf:"bytes,2,rep,name=other_names,json=otherNames" json:"other_names,omitempty"` // The complete set of build flags in this release config, after all // inheritance and other processing is complete. - FlagArtifacts []*FlagArtifact `protobuf:"bytes,3,rep,name=flag_artifacts,json=flagArtifacts" json:"flag_artifacts,omitempty"` + Flags []*FlagArtifact `protobuf:"bytes,3,rep,name=flags" json:"flags,omitempty"` // The (complete) list of aconfig_value_sets Soong modules to use. AconfigValueSets []string `protobuf:"bytes,4,rep,name=aconfig_value_sets,json=aconfigValueSets" json:"aconfig_value_sets,omitempty"` // The names of the release_config_artifacts from which we inherited. @@ -277,9 +277,9 @@ func (x *ReleaseConfigArtifact) GetOtherNames() []string { return nil } -func (x *ReleaseConfigArtifact) GetFlagArtifacts() []*FlagArtifact { +func (x *ReleaseConfigArtifact) GetFlags() []*FlagArtifact { if x != nil { - return x.FlagArtifacts + return x.Flags } return nil } @@ -406,67 +406,67 @@ var file_build_flags_out_proto_rawDesc = []byte{ 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x74, 0x72, 0x61, 0x63, 0x65, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x52, 0x06, 0x74, 0x72, 0x61, 0x63, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0e, 0x66, 0x6c, 0x61, - 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x52, 0x0a, 0x0e, 0x66, - 0x6c, 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x52, 0x0d, 0x66, 0x6c, 0x61, 0x67, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x22, - 0xb1, 0x02, 0x0a, 0x17, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x1f, 0x0a, 0x0b, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x73, - 0x12, 0x52, 0x0a, 0x0e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, - 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x0d, 0x66, 0x6c, 0x61, 0x67, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x65, - 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x20, - 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, - 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x65, 0x73, - 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x53, 0x74, 0x61, - 0x67, 0x65, 0x73, 0x22, 0xe8, 0x03, 0x0a, 0x18, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x12, 0x5c, 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, - 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, - 0x0d, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, - 0x0a, 0x15, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, - 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, - 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x52, 0x13, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x87, 0x01, 0x0a, 0x17, 0x72, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, - 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x61, 0x6e, + 0x74, 0x52, 0x06, 0x74, 0x72, 0x61, 0x63, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x0e, 0x66, 0x6c, 0x61, + 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x41, 0x0a, 0x05, 0x66, + 0x6c, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x61, 0x6e, 0x64, + 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x61, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x0e, + 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x22, 0xb0, + 0x02, 0x0a, 0x17, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, + 0x0a, 0x0b, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, + 0x41, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, + 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x66, 0x6c, + 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x05, 0x66, 0x6c, 0x61, + 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, + 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, + 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0b, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x21, + 0x0a, 0x0c, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x65, 0x73, 0x18, 0x07, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x67, 0x65, + 0x73, 0x52, 0x0e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x73, 0x22, 0xe8, 0x03, 0x0a, 0x18, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x5c, + 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x0d, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x69, 0x0a, 0x15, + 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x72, - 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, - 0x4d, 0x61, 0x70, 0x1a, 0x79, 0x0a, 0x19, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x30, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, - 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x6d, 0x61, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x33, - 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, - 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x72, - 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, + 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x52, 0x13, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x87, 0x01, 0x0a, 0x17, 0x72, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x73, 0x5f, + 0x6d, 0x61, 0x70, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x61, 0x6e, 0x64, 0x72, + 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, + 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, + 0x70, 0x1a, 0x79, 0x0a, 0x19, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x30, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, + 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x33, 0x5a, 0x31, + 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x72, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, } var ( @@ -498,8 +498,8 @@ var file_build_flags_out_proto_depIdxs = []int32{ 7, // 1: android.release_config_proto.flag_artifact.flag_declaration:type_name -> android.release_config_proto.flag_declaration 6, // 2: android.release_config_proto.flag_artifact.value:type_name -> android.release_config_proto.value 0, // 3: android.release_config_proto.flag_artifact.traces:type_name -> android.release_config_proto.tracepoint - 1, // 4: android.release_config_proto.flag_artifacts.flag_artifacts:type_name -> android.release_config_proto.flag_artifact - 1, // 5: android.release_config_proto.release_config_artifact.flag_artifacts:type_name -> android.release_config_proto.flag_artifact + 1, // 4: android.release_config_proto.flag_artifacts.flags:type_name -> android.release_config_proto.flag_artifact + 1, // 5: android.release_config_proto.release_config_artifact.flags:type_name -> android.release_config_proto.flag_artifact 3, // 6: android.release_config_proto.release_configs_artifact.release_config:type_name -> android.release_config_proto.release_config_artifact 3, // 7: android.release_config_proto.release_configs_artifact.other_release_configs:type_name -> android.release_config_proto.release_config_artifact 5, // 8: android.release_config_proto.release_configs_artifact.release_config_maps_map:type_name -> android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry diff --git a/cmd/release_config/release_config_proto/build_flags_out.proto b/cmd/release_config/release_config_proto/build_flags_out.proto index 0cbc15713..2f1715b39 100644 --- a/cmd/release_config/release_config_proto/build_flags_out.proto +++ b/cmd/release_config/release_config_proto/build_flags_out.proto @@ -58,7 +58,8 @@ message flag_artifact { message flag_artifacts { // The artifacts - repeated flag_artifact flag_artifacts = 1; + repeated flag_artifact flags = 1; + reserved "flag_artifacts"; } message release_config_artifact { @@ -71,7 +72,8 @@ message release_config_artifact { // The complete set of build flags in this release config, after all // inheritance and other processing is complete. - repeated flag_artifact flag_artifacts = 3; + repeated flag_artifact flags = 3; + reserved "flag_artifacts"; // The (complete) list of aconfig_value_sets Soong modules to use. repeated string aconfig_value_sets = 4; diff --git a/cmd/release_config/release_config_proto/build_flags_src.pb.go b/cmd/release_config/release_config_proto/build_flags_src.pb.go index 8de340ea1..bc5f5c013 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_src.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 +// protoc-gen-go v1.33.0 // protoc v3.21.12 // source: build_flags_src.proto diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 3dac8bdae..a8be7ec16 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -16,6 +16,7 @@ package main import ( "bytes" + "encoding/json" "errors" "flag" "fmt" @@ -28,11 +29,11 @@ import ( "android/soong/android/allowlists" "android/soong/bp2build" "android/soong/shared" - "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/deptools" "github.com/google/blueprint/metrics" + "github.com/google/blueprint/proptools" androidProtobuf "google.golang.org/protobuf/android" ) @@ -49,6 +50,14 @@ var ( cmdlineArgs android.CmdArgs ) +const configCacheFile = "config.cache" + +type ConfigCache struct { + EnvDepsHash uint64 + ProductVariableFileTimestamp int64 + SoongBuildFileTimestamp int64 +} + func init() { // Flags that make sense in every mode flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree") @@ -82,6 +91,7 @@ func init() { // Flags that probably shouldn't be flags of soong_build, but we haven't found // the time to remove them yet flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap") + flag.BoolVar(&cmdlineArgs.IncrementalBuildActions, "incremental-build-actions", false, "generate build actions incrementally") // Disable deterministic randomization in the protobuf package, so incremental // builds with unrelated Soong changes don't trigger large rebuilds (since we @@ -218,6 +228,60 @@ func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDe maybeQuit(err, "error writing depfile '%s'", depFile) } +// Check if there are changes to the environment file, product variable file and +// soong_build binary, in which case no incremental will be performed. +func incrementalValid(config android.Config, configCacheFile string) (*ConfigCache, bool) { + var newConfigCache ConfigCache + data, err := os.ReadFile(shared.JoinPath(topDir, usedEnvFile)) + if err != nil { + // Clean build + if os.IsNotExist(err) { + data = []byte{} + } else { + maybeQuit(err, "") + } + } + + newConfigCache.EnvDepsHash, err = proptools.CalculateHash(data) + newConfigCache.ProductVariableFileTimestamp = getFileTimestamp(filepath.Join(topDir, cmdlineArgs.SoongVariables)) + newConfigCache.SoongBuildFileTimestamp = getFileTimestamp(filepath.Join(topDir, config.HostToolDir(), "soong_build")) + //TODO(b/344917959): out/soong/dexpreopt.config might need to be checked as well. + + file, err := os.Open(configCacheFile) + if err != nil && os.IsNotExist(err) { + return &newConfigCache, false + } + maybeQuit(err, "") + defer file.Close() + + var configCache ConfigCache + decoder := json.NewDecoder(file) + err = decoder.Decode(&configCache) + maybeQuit(err, "") + + return &newConfigCache, newConfigCache == configCache +} + +func getFileTimestamp(file string) int64 { + stat, err := os.Stat(file) + if err == nil { + return stat.ModTime().UnixMilli() + } else if !os.IsNotExist(err) { + maybeQuit(err, "") + } + return 0 +} + +func writeConfigCache(configCache *ConfigCache, configCacheFile string) { + file, err := os.Create(configCacheFile) + maybeQuit(err, "") + defer file.Close() + + encoder := json.NewEncoder(file) + err = encoder.Encode(*configCache) + maybeQuit(err, "") +} + // runSoongOnlyBuild runs the standard Soong build in a number of different modes. func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string { ctx.EventHandler.Begin("soong_build") @@ -319,8 +383,26 @@ func main() { ctx := newContext(configuration) android.StartBackgroundMetrics(configuration) + var configCache *ConfigCache + configFile := filepath.Join(topDir, ctx.Config().OutDir(), configCacheFile) + incremental := false + ctx.SetIncrementalEnabled(cmdlineArgs.IncrementalBuildActions) + if cmdlineArgs.IncrementalBuildActions { + configCache, incremental = incrementalValid(ctx.Config(), configFile) + } + ctx.SetIncrementalAnalysis(incremental) + ctx.Register() finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps) + + if ctx.GetIncrementalEnabled() { + data, err := shared.EnvFileContents(configuration.EnvDeps()) + maybeQuit(err, "") + configCache.EnvDepsHash, err = proptools.CalculateHash(data) + maybeQuit(err, "") + writeConfigCache(configCache, configFile) + } + writeMetrics(configuration, ctx.EventHandler, metricsDir) writeUsedEnvironmentFile(configuration) diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 93351f1fc..d44cf7ea3 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -532,7 +532,7 @@ func OdexOnSystemOtherByName(name string, dexLocation string, global *GlobalConf } for _, f := range global.PatternsOnSystemOther { - if makefileMatch(filepath.Join(SystemPartition, f), dexLocation) { + if makefileMatch("/" + f, dexLocation) || makefileMatch(filepath.Join(SystemPartition, f), dexLocation) { return true } } diff --git a/dexpreopt/dexpreopt_test.go b/dexpreopt/dexpreopt_test.go index eff2416e5..6f7d3bb67 100644 --- a/dexpreopt/dexpreopt_test.go +++ b/dexpreopt/dexpreopt_test.go @@ -153,7 +153,7 @@ func TestDexPreoptSystemOther(t *testing.T) { moduleTests: []moduleTest{ {module: systemModule, expectedPartition: "system_other/system"}, {module: systemProductModule, expectedPartition: "system_other/system/product"}, - {module: productModule, expectedPartition: "product"}, + {module: productModule, expectedPartition: "system_other/product"}, }, }, } diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index c1a0b9c60..5a4818f4f 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -216,6 +216,14 @@ var _ android.ImageInterface = (*PrebuiltEtc)(nil) func (p *PrebuiltEtc) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (p *PrebuiltEtc) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + +func (p *PrebuiltEtc) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + func (p *PrebuiltEtc) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !p.ModuleBase.InstallInRecovery() && !p.ModuleBase.InstallInRamdisk() && !p.ModuleBase.InstallInVendorRamdisk() && !p.ModuleBase.InstallInDebugRamdisk() @@ -241,7 +249,7 @@ func (p *PrebuiltEtc) ExtraImageVariations(ctx android.BaseModuleContext) []stri return nil } -func (p *PrebuiltEtc) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +func (p *PrebuiltEtc) SetImageVariation(ctx android.BaseModuleContext, variation string) { } func (p *PrebuiltEtc) SourceFilePath(ctx android.ModuleContext) android.Path { @@ -268,17 +276,6 @@ func (p *PrebuiltEtc) OutputFile() android.OutputPath { return p.outputFilePaths[0] } -var _ android.OutputFileProducer = (*PrebuiltEtc)(nil) - -func (p *PrebuiltEtc) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return p.outputFilePaths.Paths(), nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (p *PrebuiltEtc) SubDir() string { if subDir := proptools.String(p.subdirProperties.Sub_dir); subDir != "" { return subDir @@ -420,6 +417,8 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { for _, ip := range installs { ip.addInstallRules(ctx) } + + ctx.SetOutputFiles(p.outputFilePaths.Paths(), "") } type installProperties struct { diff --git a/genrule/genrule.go b/genrule/genrule.go index 26dad01e7..c0942d3f1 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -213,21 +213,7 @@ func (g *Module) GeneratedDeps() android.Paths { return g.outputDeps } -func (g *Module) OutputFiles(tag string) (android.Paths, error) { - if tag == "" { - return append(android.Paths{}, g.outputFiles...), nil - } - // otherwise, tag should match one of outputs - for _, outputFile := range g.outputFiles { - if outputFile.Rel() == tag { - return android.Paths{outputFile}, nil - } - } - return nil, fmt.Errorf("unsupported module reference tag %q", tag) -} - var _ android.SourceFileProducer = (*Module)(nil) -var _ android.OutputFileProducer = (*Module)(nil) func toolDepsMutator(ctx android.BottomUpMutatorContext) { if g, ok := ctx.Module().(*Module); ok { @@ -585,6 +571,19 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { }) g.outputDeps = android.Paths{phonyFile} } + + g.setOutputFiles(ctx) +} + +func (g *Module) setOutputFiles(ctx android.ModuleContext) { + if len(g.outputFiles) == 0 { + return + } + ctx.SetOutputFiles(g.outputFiles, "") + // non-empty-string-tag should match one of the outputs + for _, files := range g.outputFiles { + ctx.SetOutputFiles(android.Paths{files}, files.Rel()) + } } // Collect information for opening IDE project files in java/jdeps.go. @@ -644,13 +643,15 @@ func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module { type noopImageInterface struct{} func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {} +func (x noopImageInterface) VendorVariantNeeded(android.BaseModuleContext) bool { return false } +func (x noopImageInterface) ProductVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) VendorRamdiskVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) DebugRamdiskVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) RecoveryVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil } -func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string) { } func NewGenSrcs() *Module { diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index 1df887b3e..fba9aec65 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -1254,12 +1254,6 @@ func (t *testOutputProducer) GenerateAndroidBuildActions(ctx android.ModuleConte t.outputFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "bin"), ctx.ModuleName(), android.PathForOutput(ctx, ctx.ModuleName())) } -func (t *testOutputProducer) OutputFiles(tag string) (android.Paths, error) { - return android.Paths{t.outputFile}, nil -} - -var _ android.OutputFileProducer = (*testOutputProducer)(nil) - type useSource struct { android.ModuleBase props struct { @@ -5,6 +5,5 @@ go 1.22 require ( github.com/google/blueprint v0.0.0 google.golang.org/protobuf v0.0.0 - prebuilts/bazel/common/proto/analysis_v2 v0.0.0 go.starlark.net v0.0.0 ) @@ -5,8 +5,6 @@ use ( ../../external/go-cmp ../../external/golang-protobuf ../../external/starlark-go - ../../prebuilts/bazel/common/proto/analysis_v2 - ../../prebuilts/bazel/common/proto/build ../blueprint ) @@ -15,7 +13,5 @@ replace ( github.com/google/blueprint v0.0.0 => ../blueprint github.com/google/go-cmp v0.0.0 => ../../external/go-cmp google.golang.org/protobuf v0.0.0 => ../../external/golang-protobuf - prebuilts/bazel/common/proto/analysis_v2 v0.0.0 => ../../prebuilts/bazel/common/proto/analysis_v2 - prebuilts/bazel/common/proto/build v0.0.0 => ../../prebuilts/bazel/common/proto/build go.starlark.net v0.0.0 => ../../external/starlark-go ) diff --git a/java/aar.go b/java/aar.go index 07392f6e5..3168d9bf4 100644 --- a/java/aar.go +++ b/java/aar.go @@ -831,12 +831,13 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) if a.usesLibrary.shouldDisableDexpreopt { a.dexpreopter.disableDexpreopt() } + aconfigTextFilePaths := getAconfigFilePaths(ctx) a.aapt.buildActions(ctx, aaptBuildActionOptions{ sdkContext: android.SdkContext(a), classLoaderContexts: a.classLoaderContexts, enforceDefaultTargetSdkVersion: false, - aconfigTextFiles: getAconfigFilePaths(ctx), + aconfigTextFiles: aconfigTextFilePaths, usesLibrary: &a.usesLibrary, }, ) @@ -906,6 +907,10 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) JniPackages: prebuiltJniPackages, }) } + + android.SetProvider(ctx, FlagsPackagesProvider, FlagsPackages{ + AconfigTextFiles: aconfigTextFilePaths, + }) } func (a *AndroidLibrary) IDEInfo(dpInfo *android.IdeInfo) { diff --git a/java/android_manifest.go b/java/android_manifest.go index 859900376..0c77968e6 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -71,12 +71,15 @@ func shouldReturnFinalOrFutureInt(ctx android.ModuleContext, targetSdkVersionLev return targetSdkVersionLevel.IsPreview() && (ctx.Config().UnbundledBuildApps() || includedInMts(ctx.Module())) } -// Helper function that casts android.Module to java.androidTestApp -// If this type conversion is possible, it queries whether the test app is included in an MTS suite +// Helper function that returns true if android_test, android_test_helper_app, java_test are in an MTS suite. func includedInMts(module android.Module) bool { if test, ok := module.(androidTestApp); ok { return test.includedInTestSuite("mts") } + // java_test + if test, ok := module.(*Test); ok { + return android.PrefixInList(test.testProperties.Test_suites, "mts") + } return false } diff --git a/java/app.go b/java/app.go index a24099c28..f35e4c3d4 100644 --- a/java/app.go +++ b/java/app.go @@ -47,6 +47,13 @@ var ( }, "packageName") ) +type FlagsPackages struct { + // Paths to the aconfig dump output text files that are consumed by aapt2 + AconfigTextFiles android.Paths +} + +var FlagsPackagesProvider = blueprint.NewProvider[FlagsPackages]() + func RegisterAppBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("android_app", AndroidAppFactory) ctx.RegisterModuleType("android_test", AndroidTestFactory) @@ -338,7 +345,35 @@ func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato } } +// TODO(b/156476221): Remove this allowlist +var ( + missingMinSdkVersionMtsAllowlist = []string{ + "CellBroadcastReceiverGoogleUnitTests", + "CellBroadcastReceiverUnitTests", + "CtsBatterySavingTestCases", + "CtsDeviceAndProfileOwnerApp23", + "CtsDeviceAndProfileOwnerApp30", + "CtsIntentSenderApp", + "CtsJobSchedulerTestCases", + "CtsMimeMapTestCases", + "CtsTareTestCases", + "LibStatsPullTests", + "MediaProviderClientTests", + "TeleServiceTests", + "TestExternalImsServiceApp", + "TestSmsRetrieverApp", + "TetheringPrivilegedTests", + } +) + +func checkMinSdkVersionMts(ctx android.ModuleContext, minSdkVersion android.ApiLevel) { + if includedInMts(ctx.Module()) && !minSdkVersion.Specified() && !android.InList(ctx.ModuleName(), missingMinSdkVersionMtsAllowlist) { + ctx.PropertyErrorf("min_sdk_version", "min_sdk_version is a required property for tests included in MTS") + } +} + func (a *AndroidTestHelperApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { + checkMinSdkVersionMts(ctx, a.MinSdkVersion(ctx)) applicationId := a.appTestHelperAppProperties.Manifest_values.ApplicationId if applicationId != nil { if a.overridableAppProperties.Package_name != nil { @@ -478,18 +513,27 @@ func (a *AndroidApp) renameResourcesPackage() bool { } func getAconfigFilePaths(ctx android.ModuleContext) (aconfigTextFilePaths android.Paths) { - ctx.VisitDirectDepsWithTag(aconfigDeclarationTag, func(dep android.Module) { - if provider, ok := android.OtherModuleProvider(ctx, dep, android.AconfigDeclarationsProviderKey); ok { - aconfigTextFilePaths = append(aconfigTextFilePaths, provider.IntermediateDumpOutputPath) - } else { - ctx.ModuleErrorf("Only aconfig_declarations module type is allowed for "+ - "flags_packages property, but %s is not aconfig_declarations module type", - dep.Name(), - ) + ctx.VisitDirectDeps(func(dep android.Module) { + tag := ctx.OtherModuleDependencyTag(dep) + switch tag { + case staticLibTag: + if flagPackages, ok := android.OtherModuleProvider(ctx, dep, FlagsPackagesProvider); ok { + aconfigTextFilePaths = append(aconfigTextFilePaths, flagPackages.AconfigTextFiles...) + } + + case aconfigDeclarationTag: + if provider, ok := android.OtherModuleProvider(ctx, dep, android.AconfigDeclarationsProviderKey); ok { + aconfigTextFilePaths = append(aconfigTextFilePaths, provider.IntermediateDumpOutputPath) + } else { + ctx.ModuleErrorf("Only aconfig_declarations module type is allowed for "+ + "flags_packages property, but %s is not aconfig_declarations module type", + dep.Name(), + ) + } } }) - return aconfigTextFilePaths + return android.FirstUniquePaths(aconfigTextFilePaths) } func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { @@ -544,6 +588,9 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { // Use non final ids if we are doing optimized shrinking and are using R8. nonFinalIds := a.dexProperties.optimizedResourceShrinkingEnabled(ctx) && a.dexer.effectiveOptimizeEnabled() + + aconfigTextFilePaths := getAconfigFilePaths(ctx) + a.aapt.buildActions(ctx, aaptBuildActionOptions{ sdkContext: android.SdkContext(a), @@ -552,13 +599,17 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { enforceDefaultTargetSdkVersion: a.enforceDefaultTargetSdkVersion(), forceNonFinalResourceIDs: nonFinalIds, extraLinkFlags: aaptLinkFlags, - aconfigTextFiles: getAconfigFilePaths(ctx), + aconfigTextFiles: aconfigTextFilePaths, usesLibrary: &a.usesLibrary, }, ) // apps manifests are handled by aapt, don't let Module see them a.properties.Manifest = nil + + android.SetProvider(ctx, FlagsPackagesProvider, FlagsPackages{ + AconfigTextFiles: aconfigTextFilePaths, + }) } func (a *AndroidApp) proguardBuildActions(ctx android.ModuleContext) { @@ -1343,6 +1394,7 @@ func (a *AndroidTestHelperApp) includedInTestSuite(searchPrefix string) bool { } func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { + checkMinSdkVersionMts(ctx, a.MinSdkVersion(ctx)) var configs []tradefed.Config if a.appTestProperties.Instrumentation_target_package != nil { a.additionalAaptFlags = append(a.additionalAaptFlags, diff --git a/java/app_test.go b/java/app_test.go index 804949435..9e2d19ee8 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -4146,6 +4146,7 @@ func TestTargetSdkVersionMtsTests(t *testing.T) { bpTemplate := ` %v { name: "mytest", + min_sdk_version: "34", target_sdk_version: "%v", test_suites: ["othersuite", "%v"], } @@ -4369,6 +4370,82 @@ func TestAppFlagsPackages(t *testing.T) { ) } +func TestAppFlagsPackagesPropagation(t *testing.T) { + ctx := testApp(t, ` + aconfig_declarations { + name: "foo", + package: "com.example.package.foo", + container: "com.android.foo", + srcs: [ + "foo.aconfig", + ], + } + aconfig_declarations { + name: "bar", + package: "com.example.package.bar", + container: "com.android.bar", + srcs: [ + "bar.aconfig", + ], + } + aconfig_declarations { + name: "baz", + package: "com.example.package.baz", + container: "com.android.baz", + srcs: [ + "baz.aconfig", + ], + } + android_library { + name: "foo_lib", + srcs: ["a.java"], + sdk_version: "current", + flags_packages: [ + "foo", + ], + } + android_library { + name: "bar_lib", + srcs: ["a.java"], + sdk_version: "current", + flags_packages: [ + "bar", + ], + } + android_app { + name: "baz_app", + srcs: ["a.java"], + sdk_version: "current", + flags_packages: [ + "baz", + ], + static_libs: [ + "bar_lib", + ], + libs: [ + "foo_lib", + ], + } + `) + + bazApp := ctx.ModuleForTests("baz_app", "android_common") + + // android_app module depends on aconfig_declarations listed in flags_packages + // and that of static libs, but not libs + aapt2LinkRule := bazApp.Rule("android/soong/java.aapt2Link") + linkInFlags := aapt2LinkRule.Args["inFlags"] + android.AssertStringDoesContain(t, + "aapt2 link command expected to pass feature flags arguments of flags_packages and that of its static libs", + linkInFlags, + "--feature-flags @out/soong/.intermediates/bar/intermediate.txt --feature-flags @out/soong/.intermediates/baz/intermediate.txt", + ) + android.AssertStringDoesNotContain(t, + "aapt2 link command expected to not pass feature flags arguments of flags_packages of its libs", + linkInFlags, + "--feature-flags @out/soong/.intermediates/foo/intermediate.txt", + ) +} + // Test that dexpreopt is disabled if an optional_uses_libs exists, but does not provide an implementation. func TestNoDexpreoptOptionalUsesLibDoesNotHaveImpl(t *testing.T) { bp := ` diff --git a/java/base.go b/java/base.go index 1a6584bfb..ee8df3e76 100644 --- a/java/base.go +++ b/java/base.go @@ -94,6 +94,9 @@ type CommonProperties struct { // if not blank, used as prefix to generate repackage rule Jarjar_prefix *string + // if set to true, skip the jarjar repackaging + Skip_jarjar_repackage *bool + // If not blank, set the java version passed to javac as -source and -target Java_version *string @@ -1109,11 +1112,13 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath jarjarProviderData := j.collectJarJarRules(ctx) if jarjarProviderData != nil { android.SetProvider(ctx, JarJarProvider, *jarjarProviderData) - text := getJarJarRuleText(jarjarProviderData) - if text != "" { - ruleTextFile := android.PathForModuleOut(ctx, "repackaged-jarjar", "repackaging.txt") - android.WriteFileRule(ctx, ruleTextFile, text) - j.repackageJarjarRules = ruleTextFile + if !proptools.Bool(j.properties.Skip_jarjar_repackage) { + text := getJarJarRuleText(jarjarProviderData) + if text != "" { + ruleTextFile := android.PathForModuleOut(ctx, "repackaged-jarjar", "repackaging.txt") + android.WriteFileRule(ctx, ruleTextFile, text) + j.repackageJarjarRules = ruleTextFile + } } } @@ -2544,7 +2549,7 @@ func collectDirectDepsProviders(ctx android.ModuleContext) (result *JarJarProvid case Implementation: return RenameUseInclude, "info" default: - //fmt.Printf("LJ: %v -> %v StubsLinkType unknown\n", module, m) + //fmt.Printf("collectDirectDepsProviders: %v -> %v StubsLinkType unknown\n", module, m) // Fall through to the heuristic logic. } switch reflect.TypeOf(m).String() { diff --git a/java/dex.go b/java/dex.go index 32546d985..c75e7749b 100644 --- a/java/dex.go +++ b/java/dex.go @@ -180,7 +180,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", "$r8Template": &remoteexec.REParams{ Labels: map[string]string{"type": "compile", "compiler": "r8"}, Inputs: []string{"$implicits", "${config.R8Jar}"}, - OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}", "${resourcesOutput}"}, + OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}", "${resourcesOutput}", "${outR8ArtProfile}"}, ExecStrategy: "${config.RER8ExecStrategy}", ToolchainInputs: []string{"${config.JavaCmd}"}, Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, @@ -200,7 +200,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, }, }, []string{"outDir", "outDict", "outConfig", "outUsage", "outUsageZip", "outUsageDir", - "r8Flags", "zipFlags", "mergeZipsFlags", "resourcesOutput"}, []string{"implicits"}) + "r8Flags", "zipFlags", "mergeZipsFlags", "resourcesOutput", "outR8ArtProfile"}, []string{"implicits"}) func (d *dexer) dexCommonFlags(ctx android.ModuleContext, dexParams *compileDexParams) (flags []string, deps android.Paths) { @@ -463,13 +463,6 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam proguardConfiguration, } r8Flags, r8Deps, r8ArtProfileOutputPath := d.r8Flags(ctx, dexParams) - if r8ArtProfileOutputPath != nil { - artProfileOutputPath = r8ArtProfileOutputPath - implicitOutputs = append( - implicitOutputs, - artProfileOutputPath, - ) - } rule := r8 args := map[string]string{ "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), @@ -482,6 +475,17 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam "outDir": outDir.String(), "mergeZipsFlags": mergeZipsFlags, } + if r8ArtProfileOutputPath != nil { + artProfileOutputPath = r8ArtProfileOutputPath + implicitOutputs = append( + implicitOutputs, + artProfileOutputPath, + ) + // Add the implicit r8 Art profile output to args so that r8RE knows + // about this implicit output + args["outR8ArtProfile"] = artProfileOutputPath.String() + } + if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_R8") { rule = r8RE args["implicits"] = strings.Join(r8Deps.Strings(), ",") diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 832b85036..794924401 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -96,6 +96,10 @@ func (install dexpreopterInstall) ToMakeEntries() android.AndroidMkEntries { } } +func (install dexpreopterInstall) PackageFile(ctx android.ModuleContext) android.PackagingSpec { + return ctx.PackageFile(install.installDirOnDevice, install.installFileOnDevice, install.outputPathOnHost) +} + type Dexpreopter struct { dexpreopter } @@ -312,10 +316,6 @@ func dexpreoptToolDepsMutator(ctx android.BottomUpMutatorContext) { dexpreopt.RegisterToolDeps(ctx) } -func (d *dexpreopter) odexOnSystemOther(ctx android.ModuleContext, libName string, installPath android.InstallPath) bool { - return dexpreopt.OdexOnSystemOtherByName(libName, android.InstallPathToOnDevicePath(ctx, installPath), dexpreopt.GetGlobalConfig(ctx)) -} - // Returns the install path of the dex jar of a module. // // Do not rely on `ApexInfo.ApexVariationName` because it can be something like "apex1000", rather @@ -545,12 +545,29 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, libName string, dexJa // Use the path of the dex file to determine the library name isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(dexJarStem) + dexpreoptPartition := d.installPath.Partition() + // dexpreoptPartition is set to empty for dexpreopts of system APEX and system_other. + // In case of system APEX, however, we can set it to "system" manually. + // TODO(b/346662300): Let dexpreopter generate the installPath for dexpreopt files instead of + // using the dex location to generate the installPath. + if isApexSystemServerJar { + dexpreoptPartition = "system" + } for _, install := range dexpreoptRule.Installs() { // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. installDir := strings.TrimPrefix(filepath.Dir(install.To), "/") + partition := dexpreoptPartition + if strings.HasPrefix(installDir, partition+"/") { + installDir = strings.TrimPrefix(installDir, partition+"/") + } else { + // If the partition for the installDir is different from the install partition, set the + // partition empty to install the dexpreopt files to the desired partition. + // TODO(b/346439786): Define and use the dexpreopt module type to avoid this mismatch. + partition = "" + } installBase := filepath.Base(install.To) arch := filepath.Base(installDir) - installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) + installPath := android.PathForModuleInPartitionInstall(ctx, partition, installDir) isProfile := strings.HasSuffix(installBase, ".prof") if isProfile { @@ -584,6 +601,37 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, libName string, dexJa } } +func getModuleInstallPathInfo(ctx android.ModuleContext, fullInstallPath string) (android.InstallPath, string, string) { + installPath := android.PathForModuleInstall(ctx) + installDir, installBase := filepath.Split(strings.TrimPrefix(fullInstallPath, "/")) + + if !strings.HasPrefix(installDir, installPath.Partition()+"/") { + // Return empty filename if the install partition is not for the target image. + return installPath, "", "" + } + relDir, err := filepath.Rel(installPath.Partition(), installDir) + if err != nil { + panic(err) + } + return installPath, relDir, installBase +} + +// RuleBuilder.Install() adds output-to-install copy pairs to a list for Make. To share this +// information with PackagingSpec in soong, call PackageFile for them. +// The install path and the target install partition of the module must be the same. +func packageFile(ctx android.ModuleContext, install android.RuleBuilderInstall) { + installPath, relDir, name := getModuleInstallPathInfo(ctx, install.To) + // Empty name means the install partition is not for the target image. + // For the system image, files for "apex" and "system_other" are skipped here. + // The skipped "apex" files are for testing only, for example, + // "/apex/art_boot_images/javalib/x86/boot.vdex". + // TODO(b/320196894): Files for "system_other" are skipped because soong creates the system + // image only for now. + if name != "" { + ctx.PackageFile(installPath.Join(ctx, relDir), name, install.From) + } +} + func (d *dexpreopter) DexpreoptBuiltInstalledForApex() []dexpreopterInstall { return d.builtInstalledForApex } diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 7229ca02d..defa82c0c 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -612,6 +612,9 @@ func (d *dexpreoptBootJars) GenerateAndroidBuildActions(ctx android.ModuleContex profileInstalls: profileInstalls, profileLicenseMetadataFile: android.OptionalPathForPath(ctx.LicenseMetadataFile()), }) + for _, install := range profileInstalls { + packageFile(ctx, install) + } } } @@ -929,6 +932,35 @@ func getApexNameToApexExportsInfoMap(ctx android.ModuleContext) apexNameToApexEx return apexNameToApexExportsInfoMap } +func packageFileForTargetImage(ctx android.ModuleContext, image *bootImageVariant) { + if image.target.Os != ctx.Os() { + // This is not for the target device. + return + } + + for _, install := range image.installs { + packageFile(ctx, install) + } + + for _, install := range image.vdexInstalls { + if image.target.Arch.ArchType.Name != ctx.DeviceConfig().DeviceArch() { + // Note that the vdex files are identical between architectures. If the target image is + // not for the primary architecture create symlinks to share the vdex of the primary + // architecture with the other architectures. + // + // Assuming that the install path has the architecture name with it, replace the + // architecture name with the primary architecture name to find the source vdex file. + installPath, relDir, name := getModuleInstallPathInfo(ctx, install.To) + if name != "" { + srcRelDir := strings.Replace(relDir, image.target.Arch.ArchType.Name, ctx.DeviceConfig().DeviceArch(), 1) + ctx.InstallSymlink(installPath.Join(ctx, relDir), name, installPath.Join(ctx, srcRelDir, name)) + } + } else { + packageFile(ctx, install) + } + } +} + // Generate boot image build rules for a specific target. func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, profile android.Path) bootImageVariantOutputs { @@ -1123,6 +1155,7 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p image.installs = rule.Installs() image.vdexInstalls = vdexInstalls image.unstrippedInstalls = unstrippedInstalls + packageFileForTargetImage(ctx, image) // Only set the licenseMetadataFile from the active module. if isActiveModule(ctx, ctx.Module()) { diff --git a/java/droiddoc.go b/java/droiddoc.go index 176779eb4..730f23696 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -223,17 +223,6 @@ type Javadoc struct { exportableStubsSrcJar android.WritablePath } -func (j *Javadoc) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{j.stubsSrcJar}, nil - case ".docs.zip": - return android.Paths{j.docZip}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - // javadoc converts .java source files to documentation using javadoc. func JavadocFactory() android.Module { module := &Javadoc{} @@ -254,8 +243,6 @@ func JavadocHostFactory() android.Module { return module } -var _ android.OutputFileProducer = (*Javadoc)(nil) - func (j *Javadoc) SdkVersion(ctx android.EarlyModuleContext) android.SdkSpec { return android.SdkSpecFrom(ctx, String(j.properties.Sdk_version)) } @@ -585,6 +572,9 @@ func (j *Javadoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { zipSyncCleanupCmd(rule, srcJarDir) rule.Build("javadoc", "javadoc") + + ctx.SetOutputFiles(android.Paths{j.stubsSrcJar}, "") + ctx.SetOutputFiles(android.Paths{j.docZip}, ".docs.zip") } // Droiddoc @@ -616,15 +606,6 @@ func DroiddocHostFactory() android.Module { return module } -func (d *Droiddoc) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "", ".docs.zip": - return android.Paths{d.Javadoc.docZip}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (d *Droiddoc) DepsMutator(ctx android.BottomUpMutatorContext) { d.Javadoc.addDeps(ctx) @@ -876,6 +857,9 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { zipSyncCleanupCmd(rule, srcJarDir) rule.Build("javadoc", desc) + + ctx.SetOutputFiles(android.Paths{d.Javadoc.docZip}, "") + ctx.SetOutputFiles(android.Paths{d.Javadoc.docZip}, ".docs.zip") } // Exported Droiddoc Directory diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index cab5402e9..4144de82b 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -1255,8 +1255,9 @@ func buildRuleToGenerateRemovedDexSignatures(ctx android.ModuleContext, suffix s rule := android.NewRuleBuilder(pctx, ctx) rule.Command(). BuiltTool("metalava"). + Text("signature-to-dex"). Inputs(removedTxtFiles). - FlagWithOutput("--dex-api ", output) + FlagWithOutput("--out ", output) rule.Build("modular-hiddenapi-removed-dex-signatures"+suffix, "modular hiddenapi removed dex signatures"+suffix) return android.OptionalPathForPath(output) } diff --git a/java/java.go b/java/java.go index 08fb6782c..a2fc5fbd1 100644 --- a/java/java.go +++ b/java/java.go @@ -1501,7 +1501,7 @@ func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { InstalledFiles: j.data, OutputFile: j.outputFile, TestConfig: j.testConfig, - RequiredModuleNames: j.RequiredModuleNames(), + RequiredModuleNames: j.RequiredModuleNames(ctx), TestSuites: j.testProperties.Test_suites, IsHost: true, LocalSdkVersion: j.sdkVersion.String(), @@ -1510,6 +1510,7 @@ func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { + checkMinSdkVersionMts(ctx, j.MinSdkVersion(ctx)) j.generateAndroidBuildActionsWithConfig(ctx, nil) android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } @@ -2285,10 +2286,10 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { al.stubsFlags(ctx, cmd, stubsDir) - migratingNullability := String(al.properties.Previous_api) != "" - if migratingNullability { - previousApi := android.PathForModuleSrc(ctx, String(al.properties.Previous_api)) - cmd.FlagWithInput("--migrate-nullness ", previousApi) + previousApi := String(al.properties.Previous_api) + if previousApi != "" { + previousApiFiles := android.PathsForModuleSrc(ctx, []string{previousApi}) + cmd.FlagForEachInput("--migrate-nullness ", previousApiFiles) } al.addValidation(ctx, cmd, al.validationPaths) diff --git a/java/lint.go b/java/lint.go index 82fac91cf..2eea07d31 100644 --- a/java/lint.go +++ b/java/lint.go @@ -319,25 +319,19 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru cmd.FlagWithInput("@", android.PathForSource(ctx, "build/soong/java/lint_defaults.txt")) - if l.compileSdkKind == android.SdkPublic { - cmd.FlagForEachArg("--error_check ", l.extraMainlineLintErrors) - } else { - // TODO(b/268261262): Remove this branch. We're demoting NewApi to a warning due to pre-existing issues that need to be fixed. - cmd.FlagForEachArg("--warning_check ", l.extraMainlineLintErrors) - } + cmd.FlagForEachArg("--error_check ", l.extraMainlineLintErrors) cmd.FlagForEachArg("--disable_check ", l.properties.Lint.Disabled_checks) cmd.FlagForEachArg("--warning_check ", l.properties.Lint.Warning_checks) cmd.FlagForEachArg("--error_check ", l.properties.Lint.Error_checks) cmd.FlagForEachArg("--fatal_check ", l.properties.Lint.Fatal_checks) - // TODO(b/193460475): Re-enable strict updatability linting - //if l.GetStrictUpdatabilityLinting() { - // // Verify the module does not baseline issues that endanger safe updatability. - // if baselinePath := l.getBaselineFilepath(ctx); baselinePath.Valid() { - // cmd.FlagWithInput("--baseline ", baselinePath.Path()) - // cmd.FlagForEachArg("--disallowed_issues ", updatabilityChecks) - // } - //} + if l.GetStrictUpdatabilityLinting() { + // Verify the module does not baseline issues that endanger safe updatability. + if l.properties.Lint.Baseline_filename != nil { + cmd.FlagWithInput("--baseline ", android.PathForModuleSrc(ctx, *l.properties.Lint.Baseline_filename)) + cmd.FlagForEachArg("--disallowed_issues ", updatabilityChecks) + } + } return lintPaths{ projectXML: projectXMLPath, diff --git a/java/lint_test.go b/java/lint_test.go index 751b139e7..b51753f71 100644 --- a/java/lint_test.go +++ b/java/lint_test.go @@ -91,9 +91,8 @@ func TestJavaLintUsesCorrectBpConfig(t *testing.T) { t.Error("did not use the correct file for baseline") } - if !strings.Contains(*sboxProto.Commands[0].Command, "--warning_check NewApi") { - // TODO(b/268261262): Change this to check for --error_check - t.Error("should check NewApi warnings") + if !strings.Contains(*sboxProto.Commands[0].Command, "--error_check NewApi") { + t.Error("should check NewApi errors") } if !strings.Contains(*sboxProto.Commands[0].Command, "--error_check SomeCheck") { @@ -153,52 +152,55 @@ func TestJavaLintBypassUpdatableChecks(t *testing.T) { } } -// TODO(b/193460475): Re-enable this test -//func TestJavaLintStrictUpdatabilityLinting(t *testing.T) { -// bp := ` -// java_library { -// name: "foo", -// srcs: [ -// "a.java", -// ], -// static_libs: ["bar"], -// min_sdk_version: "29", -// sdk_version: "current", -// lint: { -// strict_updatability_linting: true, -// }, -// } -// -// java_library { -// name: "bar", -// srcs: [ -// "a.java", -// ], -// min_sdk_version: "29", -// sdk_version: "current", -// } -// ` -// fs := android.MockFS{ -// "lint-baseline.xml": nil, -// } -// -// result := android.GroupFixturePreparers(PrepareForTestWithJavaDefaultModules, fs.AddToFixture()). -// RunTestWithBp(t, bp) -// -// foo := result.ModuleForTests("foo", "android_common") -// sboxProto := android.RuleBuilderSboxProtoForTests(t, foo.Output("lint.sbox.textproto")) -// if !strings.Contains(*sboxProto.Commands[0].Command, -// "--baseline lint-baseline.xml --disallowed_issues NewApi") { -// t.Error("did not restrict baselining NewApi") -// } -// -// bar := result.ModuleForTests("bar", "android_common") -// sboxProto = android.RuleBuilderSboxProtoForTests(t, bar.Output("lint.sbox.textproto")) -// if !strings.Contains(*sboxProto.Commands[0].Command, -// "--baseline lint-baseline.xml --disallowed_issues NewApi") { -// t.Error("did not restrict baselining NewApi") -// } -//} +func TestJavaLintStrictUpdatabilityLinting(t *testing.T) { + bp := ` + java_library { + name: "foo", + srcs: [ + "a.java", + ], + static_libs: ["bar"], + min_sdk_version: "29", + sdk_version: "current", + lint: { + strict_updatability_linting: true, + baseline_filename: "lint-baseline.xml", + }, + } + + java_library { + name: "bar", + srcs: [ + "a.java", + ], + min_sdk_version: "29", + sdk_version: "current", + lint: { + baseline_filename: "lint-baseline.xml", + } + } + ` + fs := android.MockFS{ + "lint-baseline.xml": nil, + } + + result := android.GroupFixturePreparers(PrepareForTestWithJavaDefaultModules, fs.AddToFixture()). + RunTestWithBp(t, bp) + + foo := result.ModuleForTests("foo", "android_common") + sboxProto := android.RuleBuilderSboxProtoForTests(t, result.TestContext, foo.Output("lint.sbox.textproto")) + if !strings.Contains(*sboxProto.Commands[0].Command, + "--baseline lint-baseline.xml --disallowed_issues NewApi") { + t.Error("did not restrict baselining NewApi") + } + + bar := result.ModuleForTests("bar", "android_common") + sboxProto = android.RuleBuilderSboxProtoForTests(t, result.TestContext, bar.Output("lint.sbox.textproto")) + if !strings.Contains(*sboxProto.Commands[0].Command, + "--baseline lint-baseline.xml --disallowed_issues NewApi") { + t.Error("did not restrict baselining NewApi") + } +} func TestJavaLintDatabaseSelectionFull(t *testing.T) { testCases := []struct { diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 8d4cf6823..38553a61b 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -15,8 +15,6 @@ package java import ( - "fmt" - "android/soong/android" "android/soong/dexpreopt" ) @@ -57,9 +55,6 @@ type platformBootclasspathModule struct { // Path to the monolithic hiddenapi-unsupported.csv file. hiddenAPIMetadataCSV android.OutputPath - - // Path to a srcjar containing all the transitive sources of the bootclasspath. - srcjar android.OutputPath } type platformBootclasspathProperties struct { @@ -76,8 +71,6 @@ func platformBootclasspathFactory() android.SingletonModule { return m } -var _ android.OutputFileProducer = (*platformBootclasspathModule)(nil) - func (b *platformBootclasspathModule) AndroidMkEntries() (entries []android.AndroidMkEntries) { entries = append(entries, android.AndroidMkEntries{ Class: "FAKE", @@ -89,22 +82,6 @@ func (b *platformBootclasspathModule) AndroidMkEntries() (entries []android.Andr return } -// Make the hidden API files available from the platform-bootclasspath module. -func (b *platformBootclasspathModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "hiddenapi-flags.csv": - return android.Paths{b.hiddenAPIFlagsCSV}, nil - case "hiddenapi-index.csv": - return android.Paths{b.hiddenAPIIndexCSV}, nil - case "hiddenapi-metadata.csv": - return android.Paths{b.hiddenAPIMetadataCSV}, nil - case ".srcjar": - return android.Paths{b.srcjar}, nil - } - - return nil, fmt.Errorf("unknown tag %s", tag) -} - func (b *platformBootclasspathModule) DepsMutator(ctx android.BottomUpMutatorContext) { // Create a dependency on all_apex_contributions to determine the selected mainline module ctx.AddDependency(ctx.Module(), apexContributionsMetadataDepTag, "all_apex_contributions") @@ -198,8 +175,8 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo } jarArgs := resourcePathsToJarArgs(transitiveSrcFiles) jarArgs = append(jarArgs, "-srcjar") // Move srcfiles to the right package - b.srcjar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-transitive.srcjar").OutputPath - TransformResourcesToJar(ctx, b.srcjar, jarArgs, transitiveSrcFiles) + srcjar := android.PathForModuleOut(ctx, ctx.ModuleName()+"-transitive.srcjar").OutputPath + TransformResourcesToJar(ctx, srcjar, jarArgs, transitiveSrcFiles) // Gather all the fragments dependencies. b.fragments = gatherApexModulePairDepsWithTag(ctx, bootclasspathFragmentDepTag) @@ -213,6 +190,11 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo bootDexJarByModule := b.generateHiddenAPIBuildActions(ctx, b.configuredModules, b.fragments) buildRuleForBootJarsPackageCheck(ctx, bootDexJarByModule) + + ctx.SetOutputFiles(android.Paths{b.hiddenAPIFlagsCSV}, "hiddenapi-flags.csv") + ctx.SetOutputFiles(android.Paths{b.hiddenAPIIndexCSV}, "hiddenapi-index.csv") + ctx.SetOutputFiles(android.Paths{b.hiddenAPIMetadataCSV}, "hiddenapi-metadata.csv") + ctx.SetOutputFiles(android.Paths{srcjar}, ".srcjar") } // Generate classpaths.proto config diff --git a/java/platform_compat_config.go b/java/platform_compat_config.go index 99fa092be..67ed84e1d 100644 --- a/java/platform_compat_config.go +++ b/java/platform_compat_config.go @@ -15,7 +15,6 @@ package java import ( - "fmt" "path/filepath" "android/soong/android" @@ -61,6 +60,8 @@ type platformCompatConfig struct { installDirPath android.InstallPath configFile android.OutputPath metadataFile android.OutputPath + + installConfigFile android.InstallPath } func (p *platformCompatConfig) compatConfigMetadata() android.Path { @@ -106,21 +107,15 @@ func (p *platformCompatConfig) GenerateAndroidBuildActions(ctx android.ModuleCon FlagWithOutput("--merged-config ", p.metadataFile) p.installDirPath = android.PathForModuleInstall(ctx, "etc", "compatconfig") + p.installConfigFile = android.PathForModuleInstall(ctx, "etc", "compatconfig", p.configFile.Base()) rule.Build(configFileName, "Extract compat/compat_config.xml and install it") - + ctx.InstallFile(p.installDirPath, p.configFile.Base(), p.configFile) } func (p *platformCompatConfig) AndroidMkEntries() []android.AndroidMkEntries { return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "ETC", OutputFile: android.OptionalPathForPath(p.configFile), - Include: "$(BUILD_PREBUILT)", - ExtraEntries: []android.AndroidMkExtraEntriesFunc{ - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.SetString("LOCAL_MODULE_PATH", p.installDirPath.String()) - entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.configFile.Base()) - }, - }, }} } @@ -266,7 +261,7 @@ func (p *platformCompatConfigSingleton) GenerateBuildActions(ctx android.Singlet func (p *platformCompatConfigSingleton) MakeVars(ctx android.MakeVarsContext) { if p.metadata != nil { - ctx.Strict("INTERNAL_PLATFORM_MERGED_COMPAT_CONFIG", p.metadata.String()) + ctx.DistForGoal("droidcore", p.metadata) } } @@ -284,32 +279,23 @@ type globalCompatConfig struct { android.ModuleBase properties globalCompatConfigProperties - - outputFilePath android.OutputPath } func (c *globalCompatConfig) GenerateAndroidBuildActions(ctx android.ModuleContext) { filename := String(c.properties.Filename) inputPath := platformCompatConfigPath(ctx) - c.outputFilePath = android.PathForModuleOut(ctx, filename).OutputPath + outputFilePath := android.PathForModuleOut(ctx, filename).OutputPath // This ensures that outputFilePath has the correct name for others to // use, as the source file may have a different name. ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, - Output: c.outputFilePath, + Output: outputFilePath, Input: inputPath, }) -} -func (h *globalCompatConfig) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{h.outputFilePath}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } + ctx.SetOutputFiles(android.Paths{outputFilePath}, "") } // global_compat_config provides access to the merged compat config xml file generated by the build. diff --git a/java/rro.go b/java/rro.go index 72170fc6d..0fc6e1c6f 100644 --- a/java/rro.go +++ b/java/rro.go @@ -150,12 +150,13 @@ func (r *RuntimeResourceOverlay) GenerateAndroidBuildActions(ctx android.ModuleC aaptLinkFlags = append(aaptLinkFlags, "--rename-overlay-category "+*r.overridableProperties.Category) } + aconfigTextFilePaths := getAconfigFilePaths(ctx) r.aapt.buildActions(ctx, aaptBuildActionOptions{ sdkContext: r, enforceDefaultTargetSdkVersion: false, extraLinkFlags: aaptLinkFlags, - aconfigTextFiles: getAconfigFilePaths(ctx), + aconfigTextFiles: aconfigTextFilePaths, }, ) @@ -176,6 +177,10 @@ func (r *RuntimeResourceOverlay) GenerateAndroidBuildActions(ctx android.ModuleC partition := rroPartition(ctx) r.installDir = android.PathForModuleInPartitionInstall(ctx, partition, "overlay", String(r.properties.Theme)) ctx.InstallFile(r.installDir, r.outputFile.Base(), r.outputFile) + + android.SetProvider(ctx, FlagsPackagesProvider, FlagsPackages{ + AconfigTextFiles: aconfigTextFilePaths, + }) } func (r *RuntimeResourceOverlay) SdkVersion(ctx android.EarlyModuleContext) android.SdkSpec { diff --git a/java/sdk_library.go b/java/sdk_library.go index e9fa83ae7..e6cb6c49b 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -118,9 +118,6 @@ type apiScope struct { // The tag to use to depend on the stubs source module (if separate from the API module). stubsSourceTag scopeDependencyTag - // The tag to use to depend on the API file generating module (if separate from the stubs source module). - apiFileTag scopeDependencyTag - // The tag to use to depend on the stubs source and API module. stubsSourceAndApiTag scopeDependencyTag @@ -195,11 +192,6 @@ func initApiScope(scope *apiScope) *apiScope { apiScope: scope, depInfoExtractor: (*scopePaths).extractStubsSourceInfoFromDep, } - scope.apiFileTag = scopeDependencyTag{ - name: name + "-api", - apiScope: scope, - depInfoExtractor: (*scopePaths).extractApiInfoFromDep, - } scope.stubsSourceAndApiTag = scopeDependencyTag{ name: name + "-stubs-source-and-api", apiScope: scope, @@ -804,12 +796,6 @@ func (paths *scopePaths) extractApiInfoFromApiStubsProvider(provider ApiStubsPro return combinedError } -func (paths *scopePaths) extractApiInfoFromDep(ctx android.ModuleContext, dep android.Module) error { - return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) error { - return paths.extractApiInfoFromApiStubsProvider(provider, Everything) - }) -} - func (paths *scopePaths) extractStubsSourceInfoFromApiStubsProviders(provider ApiStubsSrcProvider, stubsType StubsType) error { stubsSrcJar, err := provider.StubsSrcJar(stubsType) if err == nil { @@ -819,22 +805,23 @@ func (paths *scopePaths) extractStubsSourceInfoFromApiStubsProviders(provider Ap } func (paths *scopePaths) extractStubsSourceInfoFromDep(ctx android.ModuleContext, dep android.Module) error { + stubsType := Everything + if ctx.Config().ReleaseHiddenApiExportableStubs() { + stubsType = Exportable + } return paths.treatDepAsApiStubsSrcProvider(dep, func(provider ApiStubsSrcProvider) error { - return paths.extractStubsSourceInfoFromApiStubsProviders(provider, Everything) + return paths.extractStubsSourceInfoFromApiStubsProviders(provider, stubsType) }) } func (paths *scopePaths) extractStubsSourceAndApiInfoFromApiStubsProvider(ctx android.ModuleContext, dep android.Module) error { + stubsType := Everything if ctx.Config().ReleaseHiddenApiExportableStubs() { - return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) error { - extractApiInfoErr := paths.extractApiInfoFromApiStubsProvider(provider, Exportable) - extractStubsSourceInfoErr := paths.extractStubsSourceInfoFromApiStubsProviders(provider, Exportable) - return errors.Join(extractApiInfoErr, extractStubsSourceInfoErr) - }) + stubsType = Exportable } return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) error { - extractApiInfoErr := paths.extractApiInfoFromApiStubsProvider(provider, Everything) - extractStubsSourceInfoErr := paths.extractStubsSourceInfoFromApiStubsProviders(provider, Everything) + extractApiInfoErr := paths.extractApiInfoFromApiStubsProvider(provider, stubsType) + extractStubsSourceInfoErr := paths.extractStubsSourceInfoFromApiStubsProviders(provider, stubsType) return errors.Join(extractApiInfoErr, extractStubsSourceInfoErr) }) } @@ -1679,6 +1666,7 @@ func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) module.dexer.proguardDictionary = module.implLibraryModule.dexer.proguardDictionary module.dexer.proguardUsageZip = module.implLibraryModule.dexer.proguardUsageZip module.linter.reports = module.implLibraryModule.linter.reports + module.linter.outputs.depSets = module.implLibraryModule.LintDepSets() if !module.Host() { module.hostdexInstallFile = module.implLibraryModule.hostdexInstallFile @@ -3279,7 +3267,7 @@ func (module *sdkLibraryXml) implPath(ctx android.ModuleContext) string { // TODO(b/146468504): ApexVariationName() is only a soong module name, not apex name. // In most cases, this works fine. But when apex_name is set or override_apex is used // this can be wrong. - return fmt.Sprintf("/apex/%s/javalib/%s.jar", apexInfo.ApexVariationName, implName) + return fmt.Sprintf("/apex/%s/javalib/%s.jar", apexInfo.BaseApexName, implName) } partition := "system" if module.SocSpecific() { diff --git a/multitree/api_surface.go b/multitree/api_surface.go index f739a2430..0f605d84a 100644 --- a/multitree/api_surface.go +++ b/multitree/api_surface.go @@ -16,8 +16,6 @@ package multitree import ( "android/soong/android" - "fmt" - "github.com/google/blueprint" ) @@ -40,7 +38,6 @@ type ApiSurface struct { ExportableModuleBase properties apiSurfaceProperties - allOutputs android.Paths taggedOutputs map[string]android.Paths } @@ -86,15 +83,9 @@ func (surface *ApiSurface) GenerateAndroidBuildActions(ctx android.ModuleContext Inputs: allOutputs, }) - surface.allOutputs = allOutputs surface.taggedOutputs = contributionFiles -} -func (surface *ApiSurface) OutputFiles(tag string) (android.Paths, error) { - if tag != "" { - return nil, fmt.Errorf("unknown tag: %q", tag) - } - return surface.allOutputs, nil + ctx.SetOutputFiles(allOutputs, "") } func (surface *ApiSurface) TaggedOutputs() map[string]android.Paths { @@ -105,7 +96,6 @@ func (surface *ApiSurface) Exportable() bool { return true } -var _ android.OutputFileProducer = (*ApiSurface)(nil) var _ Exportable = (*ApiSurface)(nil) type ApiContribution interface { diff --git a/multitree/export.go b/multitree/export.go index aecade58d..8be8f7058 100644 --- a/multitree/export.go +++ b/multitree/export.go @@ -50,7 +50,6 @@ type Exportable interface { type ExportableModule interface { android.Module - android.OutputFileProducer Exportable } diff --git a/phony/phony.go b/phony/phony.go index 54692386a..b421176be 100644 --- a/phony/phony.go +++ b/phony/phony.go @@ -49,7 +49,7 @@ func PhonyFactory() android.Module { } func (p *phony) GenerateAndroidBuildActions(ctx android.ModuleContext) { - p.requiredModuleNames = ctx.RequiredModuleNames() + p.requiredModuleNames = ctx.RequiredModuleNames(ctx) p.hostRequiredModuleNames = ctx.HostRequiredModuleNames() p.targetRequiredModuleNames = ctx.TargetRequiredModuleNames() } diff --git a/python/binary.go b/python/binary.go index b935aba45..5f60761be 100644 --- a/python/binary.go +++ b/python/binary.go @@ -103,6 +103,7 @@ func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleConte p.buildBinary(ctx) p.installedDest = ctx.InstallFile(installDir(ctx, "bin", "", ""), p.installSource.Base(), p.installSource) + ctx.SetOutputFiles(android.Paths{p.installSource}, "") } func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { @@ -187,16 +188,6 @@ func (p *PythonBinaryModule) HostToolPath() android.OptionalPath { return android.OptionalPathForPath(p.installedDest) } -// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. -func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{p.installSource}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool { return BoolDefault(p.properties.Embedded_launcher, true) } diff --git a/rust/binary.go b/rust/binary.go index 996951366..cba29a023 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -134,6 +134,9 @@ func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps Path ret := buildOutput{outputFile: outputFile} crateRootPath := crateRootPath(ctx, binary) + // Ensure link dirs are not duplicated + deps.linkDirs = android.FirstUniqueStrings(deps.linkDirs) + flags.RustFlags = append(flags.RustFlags, deps.depFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.linkObjects...) diff --git a/rust/bindgen.go b/rust/bindgen.go index 427775351..dbc369762 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -29,7 +29,7 @@ var ( defaultBindgenFlags = []string{""} // bindgen should specify its own Clang revision so updating Clang isn't potentially blocked on bindgen failures. - bindgenClangVersion = "clang-r510928" + bindgenClangVersion = "clang-r522817" _ = pctx.VariableFunc("bindgenClangVersion", func(ctx android.PackageVarContext) string { if override := ctx.Config().Getenv("LLVM_BINDGEN_PREBUILTS_VERSION"); override != "" { @@ -236,7 +236,8 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr esc := proptools.NinjaAndShellEscapeList // Filter out invalid cflags - for _, flag := range b.ClangProperties.Cflags { + cflagsProp := b.ClangProperties.Cflags.GetOrDefault(ctx, nil) + for _, flag := range cflagsProp { if flag == "-x c++" || flag == "-xc++" { ctx.PropertyErrorf("cflags", "-x c++ should not be specified in cflags; setting cpp_std specifies this is a C++ header, or change the file extension to '.hpp' or '.hh'") @@ -248,7 +249,7 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr } // Module defined clang flags and include paths - cflags = append(cflags, esc(b.ClangProperties.Cflags)...) + cflags = append(cflags, esc(cflagsProp)...) for _, include := range b.ClangProperties.Local_include_dirs { cflags = append(cflags, "-I"+android.PathForModuleSrc(ctx, include).String()) implicits = append(implicits, android.PathForModuleSrc(ctx, include)) diff --git a/rust/builder_test.go b/rust/builder_test.go index c093ac4df..ae5ccde27 100644 --- a/rust/builder_test.go +++ b/rust/builder_test.go @@ -65,6 +65,11 @@ func TestCompilationOutputFiles(t *testing.T) { crate_name: "rust_ffi", srcs: ["lib.rs"], } + rust_ffi_static { + name: "librust_ffi_static", + crate_name: "rust_ffi", + srcs: ["lib.rs"], + } `) testcases := []struct { testName string @@ -118,14 +123,14 @@ func TestCompilationOutputFiles(t *testing.T) { }, }, { - testName: "rust_ffi static", - moduleName: "librust_ffi", - variant: "android_arm64_armv8-a_static", + testName: "rust_ffi_static rlib", + moduleName: "librust_ffi_static", + variant: "android_arm64_armv8-a_rlib_rlib-std", expectedFiles: []string{ - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/librust_ffi.a", - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/librust_ffi.a.clippy", - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/meta_lic", - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/rustdoc.timestamp", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/librust_ffi_static.rlib", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/librust_ffi_static.rlib.clippy", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/meta_lic", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/rustdoc.timestamp", }, }, { @@ -148,6 +153,7 @@ func TestCompilationOutputFiles(t *testing.T) { "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/unstripped/librust_ffi.so", "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/unstripped/librust_ffi.so.toc", "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/meta_lic", + "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/rustdoc.timestamp", "out/soong/target/product/test_device/system/lib64/librust_ffi.so", }, }, diff --git a/rust/fuzz_test.go b/rust/fuzz_test.go index 0d4622a7e..6cb8b9319 100644 --- a/rust/fuzz_test.go +++ b/rust/fuzz_test.go @@ -114,17 +114,23 @@ func TestCCFuzzDepBundling(t *testing.T) { srcs: ["foo.rs"], shared_libs: ["libcc_transitive_dep"], } + rust_ffi_static { + name: "libtest_fuzzing_static", + crate_name: "test_fuzzing", + srcs: ["foo.rs"], + shared_libs: ["libcc_transitive_dep"], + } cc_fuzz { name: "fuzz_shared_libtest", shared_libs: ["libtest_fuzzing"], } cc_fuzz { name: "fuzz_static_libtest", - static_rlibs: ["libtest_fuzzing"], + static_libs: ["libtest_fuzzing"], } cc_fuzz { name: "fuzz_staticffi_libtest", - static_libs: ["libtest_fuzzing"], + static_libs: ["libtest_fuzzing_static"], } `) diff --git a/rust/image.go b/rust/image.go index e0d267d61..26929b1ac 100644 --- a/rust/image.go +++ b/rust/image.go @@ -77,6 +77,14 @@ func (mod *Module) SetCoreVariantNeeded(b bool) { mod.Properties.CoreVariantNeeded = b } +func (mod *Module) SetProductVariantNeeded(b bool) { + mod.Properties.ProductVariantNeeded = b +} + +func (mod *Module) SetVendorVariantNeeded(b bool) { + mod.Properties.VendorVariantNeeded = b +} + func (mod *Module) SnapshotVersion(mctx android.BaseModuleContext) string { if snapshot, ok := mod.compiler.(cc.SnapshotInterface); ok { return snapshot.Version() @@ -86,6 +94,14 @@ func (mod *Module) SnapshotVersion(mctx android.BaseModuleContext) string { } } +func (mod *Module) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return mod.Properties.VendorVariantNeeded +} + +func (mod *Module) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return mod.Properties.ProductVariantNeeded +} + func (mod *Module) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { return mod.Properties.VendorRamdiskVariantNeeded } @@ -184,12 +200,12 @@ func (mod *Module) HasNonSystemVariants() bool { } func (mod *Module) InProduct() bool { - return mod.Properties.ImageVariation == cc.ProductVariation + return mod.Properties.ImageVariation == android.ProductVariation } // Returns true if the module is "vendor" variant. Usually these modules are installed in /vendor func (mod *Module) InVendor() bool { - return mod.Properties.ImageVariation == cc.VendorVariation + return mod.Properties.ImageVariation == android.VendorVariation } // Returns true if the module is "vendor" or "product" variant. @@ -197,21 +213,20 @@ func (mod *Module) InVendorOrProduct() bool { return mod.InVendor() || mod.InProduct() } -func (mod *Module) SetImageVariation(ctx android.BaseModuleContext, variant string, module android.Module) { - m := module.(*Module) +func (mod *Module) SetImageVariation(ctx android.BaseModuleContext, variant string) { if variant == android.VendorRamdiskVariation { - m.MakeAsPlatform() + mod.MakeAsPlatform() } else if variant == android.RecoveryVariation { - m.MakeAsPlatform() - } else if strings.HasPrefix(variant, cc.VendorVariation) { - m.Properties.ImageVariation = cc.VendorVariation + mod.MakeAsPlatform() + } else if strings.HasPrefix(variant, android.VendorVariation) { + mod.Properties.ImageVariation = android.VendorVariation if strings.HasPrefix(variant, cc.VendorVariationPrefix) { - m.Properties.VndkVersion = strings.TrimPrefix(variant, cc.VendorVariationPrefix) + mod.Properties.VndkVersion = strings.TrimPrefix(variant, cc.VendorVariationPrefix) } - } else if strings.HasPrefix(variant, cc.ProductVariation) { - m.Properties.ImageVariation = cc.ProductVariation + } else if strings.HasPrefix(variant, android.ProductVariation) { + mod.Properties.ImageVariation = android.ProductVariation if strings.HasPrefix(variant, cc.ProductVariationPrefix) { - m.Properties.VndkVersion = strings.TrimPrefix(variant, cc.ProductVariationPrefix) + mod.Properties.VndkVersion = strings.TrimPrefix(variant, cc.ProductVariationPrefix) } } } diff --git a/rust/image_test.go b/rust/image_test.go index 71e271c89..d84eb10c5 100644 --- a/rust/image_test.go +++ b/rust/image_test.go @@ -22,18 +22,20 @@ import ( "android/soong/cc" ) -// Test that cc modules can link against vendor_available rust_ffi_rlib/rust_ffi_static libraries. +// Test that cc modules can depend on vendor_available rust_ffi_rlib/rust_ffi_static libraries. func TestVendorLinkage(t *testing.T) { ctx := testRust(t, ` cc_binary { name: "fizz_vendor_available", - static_libs: ["libfoo_vendor_static"], - static_rlibs: ["libfoo_vendor"], + static_libs: [ + "libfoo_vendor", + "libfoo_vendor_static" + ], vendor_available: true, } cc_binary { name: "fizz_soc_specific", - static_rlibs: ["libfoo_vendor"], + static_libs: ["libfoo_vendor"], soc_specific: true, } rust_ffi_rlib { @@ -52,8 +54,8 @@ func TestVendorLinkage(t *testing.T) { vendorBinary := ctx.ModuleForTests("fizz_vendor_available", "android_vendor_arm64_armv8-a").Module().(*cc.Module) - if !android.InList("libfoo_vendor_static.vendor", vendorBinary.Properties.AndroidMkStaticLibs) { - t.Errorf("vendorBinary should have a dependency on libfoo_vendor_static.vendor: %#v", vendorBinary.Properties.AndroidMkStaticLibs) + if android.InList("libfoo_vendor_static.vendor", vendorBinary.Properties.AndroidMkStaticLibs) { + t.Errorf("vendorBinary should not have a staticlib dependency on libfoo_vendor_static.vendor: %#v", vendorBinary.Properties.AndroidMkStaticLibs) } } @@ -110,8 +112,10 @@ func TestVendorRamdiskLinkage(t *testing.T) { ctx := testRust(t, ` cc_library_shared { name: "libcc_vendor_ramdisk", - static_rlibs: ["libfoo_vendor_ramdisk"], - static_libs: ["libfoo_static_vendor_ramdisk"], + static_libs: [ + "libfoo_vendor_ramdisk", + "libfoo_static_vendor_ramdisk" + ], system_shared_libs: [], vendor_ramdisk_available: true, } @@ -131,8 +135,8 @@ func TestVendorRamdiskLinkage(t *testing.T) { vendorRamdiskLibrary := ctx.ModuleForTests("libcc_vendor_ramdisk", "android_vendor_ramdisk_arm64_armv8-a_shared").Module().(*cc.Module) - if !android.InList("libfoo_static_vendor_ramdisk.vendor_ramdisk", vendorRamdiskLibrary.Properties.AndroidMkStaticLibs) { - t.Errorf("libcc_vendor_ramdisk should have a dependency on libfoo_static_vendor_ramdisk") + if android.InList("libfoo_static_vendor_ramdisk.vendor_ramdisk", vendorRamdiskLibrary.Properties.AndroidMkStaticLibs) { + t.Errorf("libcc_vendor_ramdisk should not have a dependency on the libfoo_static_vendor_ramdisk static library") } } diff --git a/rust/library.go b/rust/library.go index 2a21263bd..96c02c832 100644 --- a/rust/library.go +++ b/rust/library.go @@ -43,9 +43,9 @@ func init() { android.RegisterModuleType("rust_ffi_host_rlib", RustFFIRlibHostFactory) // TODO: Remove when all instances of rust_ffi_static have been switched to rust_ffi_rlib - // Alias rust_ffi_static to the combined rust_ffi_rlib factory - android.RegisterModuleType("rust_ffi_static", RustFFIStaticRlibFactory) - android.RegisterModuleType("rust_ffi_host_static", RustFFIStaticRlibHostFactory) + // Alias rust_ffi_static to the rust_ffi_rlib factory + android.RegisterModuleType("rust_ffi_static", RustFFIRlibFactory) + android.RegisterModuleType("rust_ffi_host_static", RustFFIRlibHostFactory) } type VariantLibraryProperties struct { @@ -353,7 +353,7 @@ func RustFFISharedHostFactory() android.Module { // type "rlib"). func RustFFIRlibHostFactory() android.Module { module, library := NewRustLibrary(android.HostSupported) - library.BuildOnlyRlibStatic() + library.BuildOnlyRlib() library.isFFI = true return module.Init() @@ -368,30 +368,12 @@ func RustFFIRlibFactory() android.Module { return module.Init() } -// rust_ffi_static produces a staticlib and an rlib variant -func RustFFIStaticRlibFactory() android.Module { - module, library := NewRustLibrary(android.HostAndDeviceSupported) - library.BuildOnlyRlibStatic() - - library.isFFI = true - return module.Init() -} - -// rust_ffi_static_host produces a staticlib and an rlib variant for the host -func RustFFIStaticRlibHostFactory() android.Module { - module, library := NewRustLibrary(android.HostSupported) - library.BuildOnlyRlibStatic() - - library.isFFI = true - return module.Init() -} - func (library *libraryDecorator) BuildOnlyFFI() { library.MutatedProperties.BuildDylib = false // we build rlibs for later static ffi linkage. library.MutatedProperties.BuildRlib = true library.MutatedProperties.BuildShared = true - library.MutatedProperties.BuildStatic = true + library.MutatedProperties.BuildStatic = false library.isFFI = true } @@ -417,14 +399,6 @@ func (library *libraryDecorator) BuildOnlyRlib() { library.MutatedProperties.BuildStatic = false } -func (library *libraryDecorator) BuildOnlyRlibStatic() { - library.MutatedProperties.BuildDylib = false - library.MutatedProperties.BuildRlib = true - library.MutatedProperties.BuildShared = false - library.MutatedProperties.BuildStatic = true - library.isFFI = true -} - func (library *libraryDecorator) BuildOnlyStatic() { library.MutatedProperties.BuildRlib = false library.MutatedProperties.BuildDylib = false @@ -766,10 +740,13 @@ func LibraryMutator(mctx android.BottomUpMutatorContext) { // The order of the variations (modules) matches the variant names provided. Iterate // through the new variation modules and set their mutated properties. + var emptyVariant = false + var rlibVariant = false for i, v := range modules { switch variants[i] { case rlibVariation: v.(*Module).compiler.(libraryInterface).setRlib() + rlibVariant = true case dylibVariation: v.(*Module).compiler.(libraryInterface).setDylib() if v.(*Module).ModuleBase.ImageVariation().Variation == android.VendorRamdiskVariation { @@ -784,11 +761,19 @@ func LibraryMutator(mctx android.BottomUpMutatorContext) { // Disable the compilation steps. v.(*Module).compiler.SetDisabled() case "": - // if there's an empty variant, alias it so it is the default variant - mctx.AliasVariation("") + emptyVariant = true } } + if rlibVariant && library.isFFILibrary() { + // If an rlib variant is set and this is an FFI library, make it the + // default variant so CC can link against it appropriately. + mctx.AliasVariation(rlibVariation) + } else if emptyVariant { + // If there's an empty variant, alias it so it is the default variant + mctx.AliasVariation("") + } + // If a source variant is created, add an inter-variant dependency // between the other variants and the source variant. if sourceVariant { @@ -817,6 +802,7 @@ func LibstdMutator(mctx android.BottomUpMutatorContext) { rlib := modules[0].(*Module) rlib.compiler.(libraryInterface).setRlibStd() rlib.Properties.RustSubName += RlibStdlibSuffix + mctx.AliasVariation("rlib-std") } else { variants := []string{"rlib-std", "dylib-std"} modules := mctx.CreateLocalVariations(variants...) diff --git a/rust/library_test.go b/rust/library_test.go index 1133c28ed..35a420cd5 100644 --- a/rust/library_test.go +++ b/rust/library_test.go @@ -34,10 +34,14 @@ func TestLibraryVariants(t *testing.T) { name: "libfoo.ffi", srcs: ["foo.rs"], crate_name: "foo" + } + rust_ffi_host_static { + name: "libfoo.ffi_static", + srcs: ["foo.rs"], + crate_name: "foo" }`) // Test all variants are being built. - libfooStatic := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_static").Rule("rustc") libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib_rlib-std").Rule("rustc") libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Rule("rustc") libfooFFIRlib := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_rlib_rlib-std").Rule("rustc") @@ -46,7 +50,6 @@ func TestLibraryVariants(t *testing.T) { rlibCrateType := "rlib" dylibCrateType := "dylib" sharedCrateType := "cdylib" - staticCrateType := "staticlib" // Test crate type for rlib is correct. if !strings.Contains(libfooRlib.Args["rustcFlags"], "crate-type="+rlibCrateType) { @@ -58,11 +61,6 @@ func TestLibraryVariants(t *testing.T) { t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", dylibCrateType, libfooDylib.Args["rustcFlags"]) } - // Test crate type for C static libraries is correct. - if !strings.Contains(libfooStatic.Args["rustcFlags"], "crate-type="+staticCrateType) { - t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", staticCrateType, libfooStatic.Args["rustcFlags"]) - } - // Test crate type for FFI rlibs is correct if !strings.Contains(libfooFFIRlib.Args["rustcFlags"], "crate-type="+rlibCrateType) { t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", rlibCrateType, libfooFFIRlib.Args["rustcFlags"]) @@ -188,23 +186,18 @@ func TestSharedLibraryToc(t *testing.T) { func TestStaticLibraryLinkage(t *testing.T) { ctx := testRust(t, ` - rust_ffi { + rust_ffi_static { name: "libfoo", srcs: ["foo.rs"], crate_name: "foo", }`) libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_rlib_rlib-std") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") if !android.InList("libstd", libfoo.Module().(*Module).Properties.AndroidMkRlibs) { t.Errorf("Static libstd rlib expected to be a dependency of Rust rlib libraries. Rlib deps are: %#v", libfoo.Module().(*Module).Properties.AndroidMkDylibs) } - if !android.InList("libstd", libfooStatic.Module().(*Module).Properties.AndroidMkRlibs) { - t.Errorf("Static libstd rlib expected to be a dependency of Rust static libraries. Rlib deps are: %#v", - libfoo.Module().(*Module).Properties.AndroidMkDylibs) - } } func TestNativeDependencyOfRlib(t *testing.T) { @@ -215,39 +208,31 @@ func TestNativeDependencyOfRlib(t *testing.T) { rlibs: ["librust_rlib"], srcs: ["foo.rs"], } - rust_ffi_static { - name: "libffi_static", - crate_name: "ffi_static", - rlibs: ["librust_rlib"], - srcs: ["foo.rs"], - } rust_library_rlib { name: "librust_rlib", crate_name: "rust_rlib", srcs: ["foo.rs"], - shared_libs: ["shared_cc_dep"], - static_libs: ["static_cc_dep"], + shared_libs: ["libshared_cc_dep"], + static_libs: ["libstatic_cc_dep"], } cc_library_shared { - name: "shared_cc_dep", + name: "libshared_cc_dep", srcs: ["foo.cpp"], } cc_library_static { - name: "static_cc_dep", + name: "libstatic_cc_dep", srcs: ["foo.cpp"], } `) rustRlibRlibStd := ctx.ModuleForTests("librust_rlib", "android_arm64_armv8-a_rlib_rlib-std") rustRlibDylibStd := ctx.ModuleForTests("librust_rlib", "android_arm64_armv8-a_rlib_dylib-std") - ffiStatic := ctx.ModuleForTests("libffi_static", "android_arm64_armv8-a_static") ffiRlib := ctx.ModuleForTests("libffi_rlib", "android_arm64_armv8-a_rlib_rlib-std") modules := []android.TestingModule{ rustRlibRlibStd, rustRlibDylibStd, ffiRlib, - ffiStatic, } // librust_rlib specifies -L flag to cc deps output directory on rustc command @@ -258,17 +243,17 @@ func TestNativeDependencyOfRlib(t *testing.T) { // TODO: We could consider removing these flags for _, module := range modules { if !strings.Contains(module.Rule("rustc").Args["libFlags"], - "-L out/soong/.intermediates/shared_cc_dep/android_arm64_armv8-a_shared/") { + "-L out/soong/.intermediates/libshared_cc_dep/android_arm64_armv8-a_shared/") { t.Errorf( - "missing -L flag for shared_cc_dep, rustcFlags: %#v", - rustRlibRlibStd.Rule("rustc").Args["libFlags"], + "missing -L flag for libshared_cc_dep of %s, rustcFlags: %#v", + module.Module().Name(), rustRlibRlibStd.Rule("rustc").Args["libFlags"], ) } if !strings.Contains(module.Rule("rustc").Args["libFlags"], - "-L out/soong/.intermediates/static_cc_dep/android_arm64_armv8-a_static/") { + "-L out/soong/.intermediates/libstatic_cc_dep/android_arm64_armv8-a_static/") { t.Errorf( - "missing -L flag for static_cc_dep, rustcFlags: %#v", - rustRlibRlibStd.Rule("rustc").Args["libFlags"], + "missing -L flag for libstatic_cc_dep of %s, rustcFlags: %#v", + module.Module().Name(), rustRlibRlibStd.Rule("rustc").Args["libFlags"], ) } } @@ -305,15 +290,23 @@ func TestAutoDeps(t *testing.T) { "libbar", "librlib_only", ], + } + rust_ffi_host_static { + name: "libfoo.ffi.static", + srcs: ["foo.rs"], + crate_name: "foo", + rustlibs: [ + "libbar", + "librlib_only", + ], }`) libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib_rlib-std") libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib") libfooFFIRlib := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_rlib_rlib-std") - libfooStatic := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_static") libfooShared := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_shared") - for _, static := range []android.TestingModule{libfooRlib, libfooStatic, libfooFFIRlib} { + for _, static := range []android.TestingModule{libfooRlib, libfooFFIRlib} { if !android.InList("libbar.rlib-std", static.Module().(*Module).Properties.AndroidMkRlibs) { t.Errorf("libbar not present as rlib dependency in static lib: %s", static.Module().Name()) } @@ -381,6 +374,12 @@ func TestLibstdLinkage(t *testing.T) { crate_name: "bar", rustlibs: ["libfoo"], } + rust_ffi_static { + name: "libbar_static", + srcs: ["foo.rs"], + crate_name: "bar", + rustlibs: ["libfoo"], + } rust_ffi { name: "libbar.prefer_rlib", srcs: ["foo.rs"], @@ -394,7 +393,6 @@ func TestLibstdLinkage(t *testing.T) { libfooRlibDynamic := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_rlib_dylib-std").Module().(*Module) libbarShared := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_shared").Module().(*Module) - libbarStatic := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_static").Module().(*Module) libbarFFIRlib := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_rlib_rlib-std").Module().(*Module) // prefer_rlib works the same for both rust_library and rust_ffi, so a single check is sufficient here. @@ -413,12 +411,6 @@ func TestLibstdLinkage(t *testing.T) { if !android.InList("libstd", libbarShared.Properties.AndroidMkDylibs) { t.Errorf("Device rust_ffi_shared does not link libstd as an dylib") } - if !android.InList("libstd", libbarStatic.Properties.AndroidMkRlibs) { - t.Errorf("Device rust_ffi_static does not link libstd as an rlib") - } - if !android.InList("libfoo.rlib-std", libbarStatic.Properties.AndroidMkRlibs) { - t.Errorf("Device rust_ffi_static does not link dependent rustlib rlib-std variant") - } if !android.InList("libstd", libbarFFIRlib.Properties.AndroidMkRlibs) { t.Errorf("Device rust_ffi_rlib does not link libstd as an rlib") } diff --git a/rust/rust.go b/rust/rust.go index 5790dd6e4..6b7b2fbae 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -16,6 +16,7 @@ package rust import ( "fmt" + "strconv" "strings" "android/soong/bloaty" @@ -80,6 +81,8 @@ type BaseProperties struct { RustSubName string `blueprint:"mutated"` // Set by imageMutator + ProductVariantNeeded bool `blueprint:"mutated"` + VendorVariantNeeded bool `blueprint:"mutated"` CoreVariantNeeded bool `blueprint:"mutated"` VendorRamdiskVariantNeeded bool `blueprint:"mutated"` RamdiskVariantNeeded bool `blueprint:"mutated"` @@ -206,27 +209,6 @@ func (mod *Module) IsPrebuilt() bool { return false } -func (mod *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if mod.sourceProvider != nil && (mod.compiler == nil || mod.compiler.Disabled()) { - return mod.sourceProvider.Srcs(), nil - } else { - if mod.OutputFile().Valid() { - return android.Paths{mod.OutputFile().Path()}, nil - } - return android.Paths{}, nil - } - case "unstripped": - if mod.compiler != nil { - return android.PathsIfNonNil(mod.compiler.unstrippedOutputFilePath()), nil - } - return nil, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (mod *Module) SelectedStl() string { return "" } @@ -345,19 +327,6 @@ func (mod *Module) SubName() string { return mod.Properties.SubName } -func (mod *Module) IsVndk() bool { - // TODO(b/165791368) - return false -} - -func (mod *Module) IsVndkExt() bool { - return false -} - -func (mod *Module) IsVndkSp() bool { - return false -} - func (mod *Module) IsVndkPrebuiltLibrary() bool { // Rust modules do not provide VNDK prebuilts return false @@ -380,10 +349,6 @@ func (c *Module) IsLlndk() bool { return false } -func (c *Module) IsLlndkPublic() bool { - return false -} - func (mod *Module) KernelHeadersDecorator() bool { return false } @@ -455,7 +420,7 @@ type PathDeps struct { depFlags []string depLinkFlags []string - // linkDirs are link paths passed via -L to rustc. linkObjects are objects passed directly to the linker. + // linkDirs are link paths passed via -L to rustc. linkObjects are objects passed directly to the linker // Both of these are exported and propagate to dependencies. linkDirs []string linkObjects []string @@ -477,9 +442,6 @@ type PathDeps struct { // Paths to generated source files SrcDeps android.Paths srcProviderFiles android.Paths - - // Used by Generated Libraries - depExportedRlibs []cc.RustRlibDep } type RustLibraries []RustLibrary @@ -500,6 +462,7 @@ type xref interface { type flagExporter struct { linkDirs []string + ccLinkDirs []string linkObjects []string } @@ -693,6 +656,24 @@ func (mod *Module) BuildStaticVariant() bool { panic(fmt.Errorf("BuildStaticVariant called on non-library module: %q", mod.BaseModuleName())) } +func (mod *Module) BuildRlibVariant() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.buildRlib() + } + } + panic(fmt.Errorf("BuildRlibVariant called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) IsRustFFI() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.isFFILibrary() + } + } + return false +} + func (mod *Module) BuildSharedVariant() bool { if mod.compiler != nil { if library, ok := mod.compiler.(libraryInterface); ok { @@ -1003,6 +984,61 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if mod.testModule { android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } + + mod.setOutputFiles(ctx) + + buildComplianceMetadataInfo(ctx, mod, deps) +} + +func (mod *Module) setOutputFiles(ctx ModuleContext) { + if mod.sourceProvider != nil && (mod.compiler == nil || mod.compiler.Disabled()) { + ctx.SetOutputFiles(mod.sourceProvider.Srcs(), "") + } else if mod.OutputFile().Valid() { + ctx.SetOutputFiles(android.Paths{mod.OutputFile().Path()}, "") + } else { + ctx.SetOutputFiles(android.Paths{}, "") + } + if mod.compiler != nil { + ctx.SetOutputFiles(android.PathsIfNonNil(mod.compiler.unstrippedOutputFilePath()), "unstripped") + } +} + +func buildComplianceMetadataInfo(ctx *moduleContext, mod *Module, deps PathDeps) { + // Dump metadata that can not be done in android/compliance-metadata.go + metadataInfo := ctx.ComplianceMetadataInfo() + metadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(mod.Static())) + metadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, mod.outputFile.String()) + + // Static libs + staticDeps := ctx.GetDirectDepsWithTag(rlibDepTag) + staticDepNames := make([]string, 0, len(staticDeps)) + for _, dep := range staticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + ccStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(false)) + for _, dep := range ccStaticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + + staticDepPaths := make([]string, 0, len(deps.StaticLibs)+len(deps.RLibs)) + // C static libraries + for _, dep := range deps.StaticLibs { + staticDepPaths = append(staticDepPaths, dep.String()) + } + // Rust static libraries + for _, dep := range deps.RLibs { + staticDepPaths = append(staticDepPaths, dep.Path.String()) + } + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths)) + + // C Whole static libs + ccWholeStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(true)) + wholeStaticDepNames := make([]string, 0, len(ccWholeStaticDeps)) + for _, dep := range ccStaticDeps { + wholeStaticDepNames = append(wholeStaticDepNames, dep.Name()) + } + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) } func (mod *Module) deps(ctx DepsContext) Deps { @@ -1217,7 +1253,6 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { ctx.VisitDirectDeps(func(dep android.Module) { depName := ctx.OtherModuleName(dep) depTag := ctx.OtherModuleDependencyTag(dep) - if _, exists := skipModuleList[depName]; exists { return } @@ -1230,8 +1265,8 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { //Handle Rust Modules makeLibName := rustMakeLibName(ctx, mod, rustDep, depName+rustDep.Properties.RustSubName) - switch depTag { - case dylibDepTag: + switch { + case depTag == dylibDepTag: dylib, ok := rustDep.compiler.(libraryInterface) if !ok || !dylib.dylib() { ctx.ModuleErrorf("mod %q not an dylib library", depName) @@ -1241,8 +1276,7 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { mod.Properties.AndroidMkDylibs = append(mod.Properties.AndroidMkDylibs, makeLibName) mod.Properties.SnapshotDylibs = append(mod.Properties.SnapshotDylibs, cc.BaseLibName(depName)) - case rlibDepTag: - + case depTag == rlibDepTag: rlib, ok := rustDep.compiler.(libraryInterface) if !ok || !rlib.rlib() { ctx.ModuleErrorf("mod %q not an rlib library", makeLibName) @@ -1257,16 +1291,25 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { depPaths.depIncludePaths = append(depPaths.depIncludePaths, exportedInfo.IncludeDirs...) depPaths.exportedLinkDirs = append(depPaths.exportedLinkDirs, linkPathFromFilePath(rustDep.OutputFile().Path())) - case procMacroDepTag: + case depTag == procMacroDepTag: directProcMacroDeps = append(directProcMacroDeps, rustDep) mod.Properties.AndroidMkProcMacroLibs = append(mod.Properties.AndroidMkProcMacroLibs, makeLibName) // proc_macro link dirs need to be exported, so collect those here. depPaths.exportedLinkDirs = append(depPaths.exportedLinkDirs, linkPathFromFilePath(rustDep.OutputFile().Path())) - case sourceDepTag: + case depTag == sourceDepTag: if _, ok := mod.sourceProvider.(*protobufDecorator); ok { collectIncludedProtos(mod, rustDep) } + case cc.IsStaticDepTag(depTag): + // Rust FFI rlibs should not be declared in a Rust modules + // "static_libs" list as we can't handle them properly at the + // moment (for example, they only produce an rlib-std variant). + // Instead, a normal rust_library variant should be used. + ctx.PropertyErrorf("static_libs", + "found '%s' in static_libs; use a rust_library module in rustlibs instead of a rust_ffi module in static_libs", + depName) + } transitiveAndroidMkSharedLibs = append(transitiveAndroidMkSharedLibs, rustDep.transitiveAndroidMkSharedLibs) @@ -1486,7 +1529,7 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { var srcProviderDepFiles android.Paths for _, dep := range directSrcProvidersDeps { - srcs, _ := dep.OutputFiles("") + srcs := android.OutputFilesForModule(ctx, dep, "") srcProviderDepFiles = append(srcProviderDepFiles, srcs...) } for _, dep := range directSrcDeps { @@ -1873,5 +1916,3 @@ var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String var StringPtr = proptools.StringPtr - -var _ android.OutputFileProducer = (*Module)(nil) diff --git a/rust/rust_test.go b/rust/rust_test.go index 8b96df8b3..0d005d0a2 100644 --- a/rust/rust_test.go +++ b/rust/rust_test.go @@ -447,23 +447,30 @@ func TestRustRlibs(t *testing.T) { export_include_dirs: ["foo_includes"] } + rust_ffi_rlib { + name: "libbuzz", + crate_name: "buzz", + srcs: ["src/lib.rs"], + export_include_dirs: ["buzz_includes"] + } + cc_library_shared { name: "libcc_shared", srcs:["foo.c"], - static_rlibs: ["libbar"], + static_libs: ["libbar"], } cc_library_static { name: "libcc_static", srcs:["foo.c"], - static_rlibs: ["libfoo"], + static_libs: ["libbuzz"], + whole_static_libs: ["libfoo"], } cc_binary { name: "ccBin", srcs:["foo.c"], - static_rlibs: ["libbar"], - static_libs: ["libcc_static"], + static_libs: ["libcc_static", "libbar"], } `) @@ -514,10 +521,13 @@ func TestRustRlibs(t *testing.T) { "-Ibar_includes", ccbin_cc.Args) } - // Make sure that direct dependencies and indirect dependencies are + // Make sure that direct dependencies and indirect whole static dependencies are // propagating correctly to the generated rlib. if !strings.Contains(ccbin_rustc.Args["libFlags"], "--extern foo=") { - t.Errorf("Missing indirect dependency libfoo when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) + t.Errorf("Missing indirect whole_static_lib dependency libfoo when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) + } + if strings.Contains(ccbin_rustc.Args["libFlags"], "--extern buzz=") { + t.Errorf("Indirect static_lib dependency libbuzz found when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) } if !strings.Contains(ccbin_rustc.Args["libFlags"], "--extern bar=") { t.Errorf("Missing direct dependency libbar when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) diff --git a/rust/test_test.go b/rust/test_test.go index 6d0ebcf28..dc796c8a4 100644 --- a/rust/test_test.go +++ b/rust/test_test.go @@ -106,12 +106,9 @@ func TestDataLibs(t *testing.T) { ctx := testRust(t, bp) - module := ctx.ModuleForTests("main_test", "android_arm64_armv8-a").Module() - testBinary := module.(*Module).compiler.(*testDecorator) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected rust_test to produce output files, error: %s", err) - } + testingModule := ctx.ModuleForTests("main_test", "android_arm64_armv8-a") + testBinary := testingModule.Module().(*Module).compiler.(*testDecorator) + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } @@ -168,12 +165,10 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { ` ctx := testRust(t, bp) - module := ctx.ModuleForTests("main_test", "android_arm64_armv8-a").Module() + testingModule := ctx.ModuleForTests("main_test", "android_arm64_armv8-a") + module := testingModule.Module() testBinary := module.(*Module).compiler.(*testDecorator) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected rust_test to produce output files, error: %s", err) - } + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } diff --git a/rust/testing.go b/rust/testing.go index f31c59188..6ee49a971 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -189,11 +189,11 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.RegisterModuleType("rust_ffi", RustFFIFactory) ctx.RegisterModuleType("rust_ffi_shared", RustFFISharedFactory) ctx.RegisterModuleType("rust_ffi_rlib", RustFFIRlibFactory) - ctx.RegisterModuleType("rust_ffi_static", RustFFIStaticRlibFactory) + ctx.RegisterModuleType("rust_ffi_static", RustFFIRlibFactory) ctx.RegisterModuleType("rust_ffi_host", RustFFIHostFactory) ctx.RegisterModuleType("rust_ffi_host_shared", RustFFISharedHostFactory) ctx.RegisterModuleType("rust_ffi_host_rlib", RustFFIRlibHostFactory) - ctx.RegisterModuleType("rust_ffi_host_static", RustFFIStaticRlibHostFactory) + ctx.RegisterModuleType("rust_ffi_host_static", RustFFIRlibHostFactory) ctx.RegisterModuleType("rust_proc_macro", ProcMacroFactory) ctx.RegisterModuleType("rust_protobuf", RustProtobufFactory) ctx.RegisterModuleType("rust_protobuf_host", RustProtobufHostFactory) diff --git a/scripts/Android.bp b/scripts/Android.bp index 80cd93579..91aa19560 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -292,6 +292,20 @@ python_binary_host { } python_binary_host { + name: "merge_json", + main: "merge_json.py", + srcs: [ + "merge_json.py", + ], +} + +python_binary_host { + name: "gen_build_prop", + main: "gen_build_prop.py", + srcs: ["gen_build_prop.py"], +} + +python_binary_host { name: "buildinfo", main: "buildinfo.py", srcs: ["buildinfo.py"], diff --git a/scripts/buildinfo.py b/scripts/buildinfo.py index e4fb0da09..db9920918 100755 --- a/scripts/buildinfo.py +++ b/scripts/buildinfo.py @@ -18,55 +18,90 @@ import argparse import contextlib +import json +import os import subprocess +TEST_KEY_DIR = "build/make/target/product/security" + +def get_build_variant(product_config): + if product_config["Eng"]: + return "eng" + elif product_config["Debuggable"]: + return "userdebug" + else: + return "user" + +def get_build_flavor(product_config): + build_flavor = product_config["DeviceProduct"] + "-" + get_build_variant(product_config) + if "address" in product_config.get("SanitizeDevice", []) and "_asan" not in build_flavor: + build_flavor += "_asan" + return build_flavor + +def get_build_keys(product_config): + default_cert = product_config.get("DefaultAppCertificate", "") + if default_cert == "" or default_cert == os.path.join(TEST_KEY_DIR, "testKey"): + return "test-keys" + return "dev-keys" + def parse_args(): """Parse commandline arguments.""" parser = argparse.ArgumentParser() - parser.add_argument('--use-vbmeta-digest-in-fingerprint', action='store_true') - parser.add_argument('--build-flavor', required=True) parser.add_argument('--build-hostname-file', required=True, type=argparse.FileType('r')), - parser.add_argument('--build-id', required=True) - parser.add_argument('--build-keys', required=True) parser.add_argument('--build-number-file', required=True, type=argparse.FileType('r')) parser.add_argument('--build-thumbprint-file', type=argparse.FileType('r')) - parser.add_argument('--build-type', required=True) parser.add_argument('--build-username', required=True) - parser.add_argument('--build-variant', required=True) - parser.add_argument('--cpu-abis', action='append', required=True) parser.add_argument('--date-file', required=True, type=argparse.FileType('r')) - parser.add_argument('--default-locale') - parser.add_argument('--default-wifi-channels', action='append', default=[]) - parser.add_argument('--device', required=True) - parser.add_argument("--display-build-number", action='store_true') - parser.add_argument('--platform-base-os', required=True) - parser.add_argument('--platform-display-version', required=True) - parser.add_argument('--platform-min-supported-target-sdk-version', required=True) parser.add_argument('--platform-preview-sdk-fingerprint-file', required=True, type=argparse.FileType('r')) - parser.add_argument('--platform-preview-sdk-version', required=True) - parser.add_argument('--platform-sdk-version', required=True) - parser.add_argument('--platform-security-patch', required=True) - parser.add_argument('--platform-version', required=True) - parser.add_argument('--platform-version-codename',required=True) - parser.add_argument('--platform-version-all-codenames', action='append', required=True) - parser.add_argument('--platform-version-known-codenames', required=True) - parser.add_argument('--platform-version-last-stable', required=True) - parser.add_argument('--product', required=True) - + parser.add_argument('--product-config', required=True, type=argparse.FileType('r')) parser.add_argument('--out', required=True, type=argparse.FileType('w')) - return parser.parse_args() + option = parser.parse_args() + + product_config = json.load(option.product_config) + build_flags = product_config["BuildFlags"] + + option.build_flavor = get_build_flavor(product_config) + option.build_keys = get_build_keys(product_config) + option.build_id = product_config["BuildId"] + option.build_type = product_config["BuildType"] + option.build_variant = get_build_variant(product_config) + option.build_version_tags = product_config["BuildVersionTags"] + option.cpu_abis = product_config["DeviceAbi"] + option.default_locale = None + if len(product_config.get("ProductLocales", [])) > 0: + option.default_locale = product_config["ProductLocales"][0] + option.default_wifi_channels = product_config.get("ProductDefaultWifiChannels", []) + option.device = product_config["DeviceName"] + option.display_build_number = product_config["DisplayBuildNumber"] + option.platform_base_os = product_config["Platform_base_os"] + option.platform_display_version = product_config["Platform_display_version_name"] + option.platform_min_supported_target_sdk_version = build_flags["RELEASE_PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION"] + option.platform_preview_sdk_version = product_config["Platform_preview_sdk_version"] + option.platform_sdk_version = product_config["Platform_sdk_version"] + option.platform_security_patch = product_config["Platform_security_patch"] + option.platform_version = product_config["Platform_version_name"] + option.platform_version_codename = product_config["Platform_sdk_codename"] + option.platform_version_all_codenames = product_config["Platform_version_active_codenames"] + option.platform_version_known_codenames = product_config["Platform_version_known_codenames"] + option.platform_version_last_stable = product_config["Platform_version_last_stable"] + option.product = product_config["DeviceProduct"] + option.use_vbmeta_digest_in_fingerprint = product_config["BoardUseVbmetaDigestInFingerprint"] + + return option def main(): option = parse_args() build_hostname = option.build_hostname_file.read().strip() build_number = option.build_number_file.read().strip() - build_version_tags = option.build_keys + build_version_tags_list = option.build_version_tags if option.build_type == "debug": - build_version_tags = "debug," + build_version_tags + build_version_tags_list.append("debug") + build_version_tags_list.append(option.build_keys) + build_version_tags = ",".join(sorted(set(build_version_tags_list))) raw_date = option.date_file.read().strip() date = subprocess.check_output(["date", "-d", f"@{raw_date}"], text=True).strip() diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt index 47eae0769..bb88ccebf 100644 --- a/scripts/check_boot_jars/package_allowed_list.txt +++ b/scripts/check_boot_jars/package_allowed_list.txt @@ -3,52 +3,7 @@ ################################################### # core-libart.jar & core-oj.jar -java\.awt\.font -java\.beans -java\.io -java\.lang -java\.lang\.annotation -java\.lang\.constant -java\.lang\.invoke -java\.lang\.ref -java\.lang\.reflect -java\.lang\.runtime -java\.math -java\.net -java\.nio -java\.nio\.file -java\.nio\.file\.spi -java\.nio\.file\.attribute -java\.nio\.channels -java\.nio\.channels\.spi -java\.nio\.charset -java\.nio\.charset\.spi -java\.security -java\.security\.acl -java\.security\.cert -java\.security\.interfaces -java\.security\.spec -java\.sql -java\.text -java\.text\.spi -java\.time -java\.time\.chrono -java\.time\.format -java\.time\.temporal -java\.time\.zone -java\.util -java\.util\.concurrent -java\.util\.concurrent\.atomic -java\.util\.concurrent\.locks -java\.util\.function -java\.util\.jar -java\.util\.logging -java\.util\.prefs -java\.util\.random -java\.util\.regex -java\.util\.spi -java\.util\.stream -java\.util\.zip +java(\..*)? # TODO: Remove javax.annotation.processing if possible, see http://b/132338110: javax\.annotation\.processing javax\.crypto @@ -72,18 +27,7 @@ javax\.xml\.transform\.sax javax\.xml\.transform\.stream javax\.xml\.validation javax\.xml\.xpath -jdk\.internal -jdk\.internal\.access -jdk\.internal\.math -jdk\.internal\.misc -jdk\.internal\.ref -jdk\.internal\.reflect -jdk\.internal\.util -jdk\.internal\.util\.jar -jdk\.internal\.util\.random -jdk\.internal\.vm\.annotation -jdk\.net -jdk\.random +jdk\..* org\.w3c\.dom org\.w3c\.dom\.ls org\.w3c\.dom\.traversal diff --git a/scripts/gen_build_prop.py b/scripts/gen_build_prop.py new file mode 100644 index 000000000..42e320728 --- /dev/null +++ b/scripts/gen_build_prop.py @@ -0,0 +1,562 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for generating {partition}/build.prop""" + +import argparse +import contextlib +import json +import subprocess +import sys + +def get_build_variant(product_config): + if product_config["Eng"]: + return "eng" + elif product_config["Debuggable"]: + return "userdebug" + else: + return "user" + +def get_build_flavor(product_config): + build_flavor = product_config["DeviceProduct"] + "-" + get_build_variant(product_config) + if "address" in product_config.get("SanitizeDevice", []) and "_asan" not in build_flavor: + build_flavor += "_asan" + return build_flavor + +def get_build_keys(product_config): + default_cert = product_config.get("DefaultAppCertificate", "") + if default_cert == "" or default_cert == os.path.join(TEST_KEY_DIR, "testKey"): + return "test-keys" + return "dev-keys" + +def parse_args(): + """Parse commandline arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument("--build-fingerprint-file", required=True, type=argparse.FileType("r")) + parser.add_argument("--build-hostname-file", required=True, type=argparse.FileType("r")) + parser.add_argument("--build-number-file", required=True, type=argparse.FileType("r")) + parser.add_argument("--build-thumbprint-file", type=argparse.FileType("r")) + parser.add_argument("--build-username", required=True) + parser.add_argument("--date-file", required=True, type=argparse.FileType("r")) + parser.add_argument("--platform-preview-sdk-fingerprint-file", required=True, type=argparse.FileType("r")) + parser.add_argument("--prop-files", action="append", type=argparse.FileType("r"), default=[]) + parser.add_argument("--product-config", required=True, type=argparse.FileType("r")) + parser.add_argument("--partition", required=True) + parser.add_argument("--build-broken-dup-sysprop", action="store_true", default=False) + + parser.add_argument("--out", required=True, type=argparse.FileType("w")) + + args = parser.parse_args() + + # post process parse_args requiring manual handling + args.config = json.load(args.product_config) + config = args.config + + config["BuildFlavor"] = get_build_flavor(config) + config["BuildKeys"] = get_build_keys(config) + config["BuildVariant"] = get_build_variant(config) + + config["BuildFingerprint"] = args.build_fingerprint_file.read().strip() + config["BuildHostname"] = args.build_hostname_file.read().strip() + config["BuildNumber"] = args.build_number_file.read().strip() + config["BuildUsername"] = args.build_username + + build_version_tags_list = config["BuildVersionTags"] + if config["BuildType"] == "debug": + build_version_tags_list.append("debug") + build_version_tags_list.append(config["BuildKeys"]) + build_version_tags = ",".join(sorted(set(build_version_tags_list))) + config["BuildVersionTags"] = build_version_tags + + raw_date = args.date_file.read().strip() + config["Date"] = subprocess.check_output(["date", "-d", f"@{raw_date}"], text=True).strip() + config["DateUtc"] = subprocess.check_output(["date", "-d", f"@{raw_date}", "+%s"], text=True).strip() + + # build_desc is human readable strings that describe this build. This has the same info as the + # build fingerprint. + # e.g. "aosp_cf_x86_64_phone-userdebug VanillaIceCream MAIN eng.20240319.143939 test-keys" + config["BuildDesc"] = f"{config['DeviceProduct']}-{config['BuildVariant']} " \ + f"{config['Platform_version_name']} {config['BuildId']} " \ + f"{config['BuildNumber']} {config['BuildVersionTags']}" + + config["PlatformPreviewSdkFingerprint"] = args.platform_preview_sdk_fingerprint_file.read().strip() + + if args.build_thumbprint_file: + config["BuildThumbprint"] = args.build_thumbprint_file.read().strip() + + append_additional_system_props(args) + append_additional_vendor_props(args) + append_additional_product_props(args) + + return args + +def generate_common_build_props(args): + print("####################################") + print("# from generate_common_build_props") + print("# These properties identify this partition image.") + print("####################################") + + config = args.config + partition = args.partition + + if partition == "system": + print(f"ro.product.{partition}.brand={config['SystemBrand']}") + print(f"ro.product.{partition}.device={config['SystemDevice']}") + print(f"ro.product.{partition}.manufacturer={config['SystemManufacturer']}") + print(f"ro.product.{partition}.model={config['SystemModel']}") + print(f"ro.product.{partition}.name={config['SystemName']}") + else: + print(f"ro.product.{partition}.brand={config['ProductBrand']}") + print(f"ro.product.{partition}.device={config['DeviceName']}") + print(f"ro.product.{partition}.manufacturer={config['ProductManufacturer']}") + print(f"ro.product.{partition}.model={config['ProductModel']}") + print(f"ro.product.{partition}.name={config['DeviceProduct']}") + + if partition != "system": + if config["ModelForAttestation"]: + print(f"ro.product.model_for_attestation={config['ModelForAttestation']}") + if config["BrandForAttestation"]: + print(f"ro.product.brand_for_attestation={config['BrandForAttestation']}") + if config["NameForAttestation"]: + print(f"ro.product.name_for_attestation={config['NameForAttestation']}") + if config["DeviceForAttestation"]: + print(f"ro.product.device_for_attestation={config['DeviceForAttestation']}") + if config["ManufacturerForAttestation"]: + print(f"ro.product.manufacturer_for_attestation={config['ManufacturerForAttestation']}") + + if config["ZygoteForce64"]: + if partition == "vendor": + print(f"ro.{partition}.product.cpu.abilist={config['DeviceAbiList64']}") + print(f"ro.{partition}.product.cpu.abilist32=") + print(f"ro.{partition}.product.cpu.abilist64={config['DeviceAbiList64']}") + else: + if partition == "system" or partition == "vendor" or partition == "odm": + print(f"ro.{partition}.product.cpu.abilist={config['DeviceAbiList']}") + print(f"ro.{partition}.product.cpu.abilist32={config['DeviceAbiList32']}") + print(f"ro.{partition}.product.cpu.abilist64={config['DeviceAbiList64']}") + + print(f"ro.{partition}.build.date={config['Date']}") + print(f"ro.{partition}.build.date.utc={config['DateUtc']}") + # Allow optional assignments for ARC forward-declarations (b/249168657) + # TODO: Remove any tag-related inconsistencies once the goals from + # go/arc-android-sigprop-changes have been achieved. + print(f"ro.{partition}.build.fingerprint?={config['BuildFingerprint']}") + print(f"ro.{partition}.build.id?={config['BuildId']}") + print(f"ro.{partition}.build.tags?={config['BuildVersionTags']}") + print(f"ro.{partition}.build.type={config['BuildVariant']}") + print(f"ro.{partition}.build.version.incremental={config['BuildNumber']}") + print(f"ro.{partition}.build.version.release={config['Platform_version_last_stable']}") + print(f"ro.{partition}.build.version.release_or_codename={config['Platform_version_name']}") + print(f"ro.{partition}.build.version.sdk={config['Platform_sdk_version']}") + +def generate_build_info(args): + print() + print("####################################") + print("# from gen_build_prop.py:generate_build_info") + print("####################################") + print("# begin build properties") + + config = args.config + build_flags = config["BuildFlags"] + + # The ro.build.id will be set dynamically by init, by appending the unique vbmeta digest. + if config["BoardUseVbmetaDigestInFingerprint"]: + print(f"ro.build.legacy.id={config['BuildId']}") + else: + print(f"ro.build.id?={config['BuildId']}") + + # ro.build.display.id is shown under Settings -> About Phone + if config["BuildVariant"] == "user": + # User builds should show: + # release build number or branch.buld_number non-release builds + + # Dev. branches should have DISPLAY_BUILD_NUMBER set + if config["DisplayBuildNumber"]: + print(f"ro.build.display.id?={config['BuildId']} {config['BuildNumber']} {config['BuildKeys']}") + else: + print(f"ro.build.display.id?={config['BuildId']} {config['BuildKeys']}") + else: + # Non-user builds should show detailed build information (See build desc above) + print(f"ro.build.display.id?={config['BuildDesc']}") + print(f"ro.build.version.incremental={config['BuildNumber']}") + print(f"ro.build.version.sdk={config['Platform_sdk_version']}") + print(f"ro.build.version.preview_sdk={config['Platform_preview_sdk_version']}") + print(f"ro.build.version.preview_sdk_fingerprint={config['PlatformPreviewSdkFingerprint']}") + print(f"ro.build.version.codename={config['Platform_sdk_codename']}") + print(f"ro.build.version.all_codenames={','.join(config['Platform_version_active_codenames'])}") + print(f"ro.build.version.known_codenames={config['Platform_version_known_codenames']}") + print(f"ro.build.version.release={config['Platform_version_last_stable']}") + print(f"ro.build.version.release_or_codename={config['Platform_version_name']}") + print(f"ro.build.version.release_or_preview_display={config['Platform_display_version_name']}") + print(f"ro.build.version.security_patch={config['Platform_security_patch']}") + print(f"ro.build.version.base_os={config['Platform_base_os']}") + print(f"ro.build.version.min_supported_target_sdk={build_flags['RELEASE_PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION']}") + print(f"ro.build.date={config['Date']}") + print(f"ro.build.date.utc={config['DateUtc']}") + print(f"ro.build.type={config['BuildVariant']}") + print(f"ro.build.user={config['BuildUsername']}") + print(f"ro.build.host={config['BuildHostname']}") + # TODO: Remove any tag-related optional property declarations once the goals + # from go/arc-android-sigprop-changes have been achieved. + print(f"ro.build.tags?={config['BuildVersionTags']}") + # ro.build.flavor are used only by the test harness to distinguish builds. + # Only add _asan for a sanitized build if it isn't already a part of the + # flavor (via a dedicated lunch config for example). + print(f"ro.build.flavor={config['BuildFlavor']}") + + # These values are deprecated, use "ro.product.cpu.abilist" + # instead (see below). + print(f"# ro.product.cpu.abi and ro.product.cpu.abi2 are obsolete,") + print(f"# use ro.product.cpu.abilist instead.") + print(f"ro.product.cpu.abi={config['DeviceAbi'][0]}") + if len(config["DeviceAbi"]) > 1: + print(f"ro.product.cpu.abi2={config['DeviceAbi'][1]}") + + if config["ProductLocales"]: + print(f"ro.product.locale={config['ProductLocales'][0]}") + print(f"ro.wifi.channels={' '.join(config['ProductDefaultWifiChannels'])}") + + print(f"# ro.build.product is obsolete; use ro.product.device") + print(f"ro.build.product={config['DeviceName']}") + + print(f"# Do not try to parse description or thumbprint") + print(f"ro.build.description?={config['BuildDesc']}") + if "build_thumbprint" in config: + print(f"ro.build.thumbprint={config['BuildThumbprint']}") + + print(f"# end build properties") + +def write_properties_from_file(file): + print() + print("####################################") + print(f"# from {file.name}") + print("####################################") + print(file.read(), end="") + +def write_properties_from_variable(name, props, build_broken_dup_sysprop): + print() + print("####################################") + print(f"# from variable {name}") + print("####################################") + + # Implement the legacy behavior when BUILD_BROKEN_DUP_SYSPROP is on. + # Optional assignments are all converted to normal assignments and + # when their duplicates the first one wins. + if build_broken_dup_sysprop: + processed_props = [] + seen_props = set() + for line in props: + line = line.replace("?=", "=") + key, value = line.split("=", 1) + if key in seen_props: + continue + seen_props.add(key) + processed_props.append(line) + props = processed_props + + for line in props: + print(line) + +def append_additional_system_props(args): + props = [] + + config = args.config + + # Add the product-defined properties to the build properties. + if config["PropertySplitEnabled"] or config["VendorImageFileSystemType"]: + if "PRODUCT_PROPERTY_OVERRIDES" in config: + props += config["PRODUCT_PROPERTY_OVERRIDES"] + + props.append(f"ro.treble.enabled={'true' if config['FullTreble'] else 'false'}") + # Set ro.llndk.api_level to show the maximum vendor API level that the LLNDK + # in the system partition supports. + if config["VendorApiLevel"]: + props.append(f"ro.llndk.api_level={config['VendorApiLevel']}") + + # Sets ro.actionable_compatible_property.enabled to know on runtime whether + # the allowed list of actionable compatible properties is enabled or not. + props.append("ro.actionable_compatible_property.enabled=true") + + # Enable core platform API violation warnings on userdebug and eng builds. + if config["BuildVariant"] != "user": + props.append("persist.debug.dalvik.vm.core_platform_api_policy=just-warn") + + # Define ro.sanitize.<name> properties for all global sanitizers. + for sanitize_target in config["SanitizeDevice"]: + props.append(f"ro.sanitize.{sanitize_target}=true") + + # Sets the default value of ro.postinstall.fstab.prefix to /system. + # Device board config should override the value to /product when needed by: + # + # PRODUCT_PRODUCT_PROPERTIES += ro.postinstall.fstab.prefix=/product + # + # It then uses ${ro.postinstall.fstab.prefix}/etc/fstab.postinstall to + # mount system_other partition. + props.append("ro.postinstall.fstab.prefix=/system") + + enable_target_debugging = True + if config["BuildVariant"] == "user" or config["BuildVariant"] == "userdebug": + # Target is secure in user builds. + props.append("ro.secure=1") + props.append("security.perf_harden=1") + + if config["BuildVariant"] == "user": + # Disable debugging in plain user builds. + props.append("ro.adb.secure=1") + enable_target_debugging = False + + # Disallow mock locations by default for user builds + props.append("ro.allow.mock.location=0") + else: + # Turn on checkjni for non-user builds. + props.append("ro.kernel.android.checkjni=1") + # Set device insecure for non-user builds. + props.append("ro.secure=0") + # Allow mock locations by default for non user builds + props.append("ro.allow.mock.location=1") + + if enable_target_debugging: + # Enable Dalvik lock contention logging. + props.append("dalvik.vm.lockprof.threshold=500") + + # Target is more debuggable and adbd is on by default + props.append("ro.debuggable=1") + else: + # Target is less debuggable and adbd is off by default + props.append("ro.debuggable=0") + + if config["BuildVariant"] == "eng": + if "ro.setupwizard.mode=ENABLED" in props: + # Don't require the setup wizard on eng builds + props = list(filter(lambda x: not x.startswith("ro.setupwizard.mode="), props)) + props.append("ro.setupwizard.mode=OPTIONAL") + + if not config["SdkBuild"]: + # To speedup startup of non-preopted builds, don't verify or compile the boot image. + props.append("dalvik.vm.image-dex2oat-filter=extract") + # b/323566535 + props.append("init.svc_debug.no_fatal.zygote=true") + + if config["SdkBuild"]: + props.append("xmpp.auto-presence=true") + props.append("ro.config.nocheckin=yes") + + props.append("net.bt.name=Android") + + # This property is set by flashing debug boot image, so default to false. + props.append("ro.force.debuggable=0") + + config["ADDITIONAL_SYSTEM_PROPERTIES"] = props + +def append_additional_vendor_props(args): + props = [] + + config = args.config + build_flags = config["BuildFlags"] + + # Add cpu properties for bionic and ART. + props.append(f"ro.bionic.arch={config['DeviceArch']}") + props.append(f"ro.bionic.cpu_variant={config['DeviceCpuVariantRuntime']}") + props.append(f"ro.bionic.2nd_arch={config['DeviceSecondaryArch']}") + props.append(f"ro.bionic.2nd_cpu_variant={config['DeviceSecondaryCpuVariantRuntime']}") + + props.append(f"persist.sys.dalvik.vm.lib.2=libart.so") + props.append(f"dalvik.vm.isa.{config['DeviceArch']}.variant={config['Dex2oatTargetCpuVariantRuntime']}") + if config["Dex2oatTargetInstructionSetFeatures"]: + props.append(f"dalvik.vm.isa.{config['DeviceArch']}.features={config['Dex2oatTargetInstructionSetFeatures']}") + + if config["DeviceSecondaryArch"]: + props.append(f"dalvik.vm.isa.{config['DeviceSecondaryArch']}.variant={config['SecondaryDex2oatCpuVariantRuntime']}") + if config["SecondaryDex2oatInstructionSetFeatures"]: + props.append(f"dalvik.vm.isa.{config['DeviceSecondaryArch']}.features={config['SecondaryDex2oatInstructionSetFeatures']}") + + # Although these variables are prefixed with TARGET_RECOVERY_, they are also needed under charger + # mode (via libminui). + if config["RecoveryDefaultRotation"]: + props.append(f"ro.minui.default_rotation={config['RecoveryDefaultRotation']}") + + if config["RecoveryOverscanPercent"]: + props.append(f"ro.minui.overscan_percent={config['RecoveryOverscanPercent']}") + + if config["RecoveryPixelFormat"]: + props.append(f"ro.minui.pixel_format={config['RecoveryPixelFormat']}") + + if "UseDynamicPartitions" in config: + props.append(f"ro.boot.dynamic_partitions={'true' if config['UseDynamicPartitions'] else 'false'}") + + if "RetrofitDynamicPartitions" in config: + props.append(f"ro.boot.dynamic_partitions_retrofit={'true' if config['RetrofitDynamicPartitions'] else 'false'}") + + if config["ShippingApiLevel"]: + props.append(f"ro.product.first_api_level={config['ShippingApiLevel']}") + + if config["ShippingVendorApiLevel"]: + props.append(f"ro.vendor.api_level={config['ShippingVendorApiLevel']}") + + if config["BuildVariant"] != "user" and config["BuildDebugfsRestrictionsEnabled"]: + props.append(f"ro.product.debugfs_restrictions.enabled=true") + + # Vendors with GRF must define BOARD_SHIPPING_API_LEVEL for the vendor API level. + # This must not be defined for the non-GRF devices. + # The values of the GRF properties will be verified by post_process_props.py + if config["BoardShippingApiLevel"]: + props.append(f"ro.board.first_api_level={config['ProductShippingApiLevel']}") + + # Build system set BOARD_API_LEVEL to show the api level of the vendor API surface. + # This must not be altered outside of build system. + if config["VendorApiLevel"]: + props.append(f"ro.board.api_level={config['VendorApiLevel']}") + + # RELEASE_BOARD_API_LEVEL_FROZEN is true when the vendor API surface is frozen. + if build_flags["RELEASE_BOARD_API_LEVEL_FROZEN"]: + props.append(f"ro.board.api_frozen=true") + + # Set build prop. This prop is read by ota_from_target_files when generating OTA, + # to decide if VABC should be disabled. + if config["DontUseVabcOta"]: + props.append(f"ro.vendor.build.dont_use_vabc=true") + + # Set the flag in vendor. So VTS would know if the new fingerprint format is in use when + # the system images are replaced by GSI. + if config["BoardUseVbmetaDigestInFingerprint"]: + props.append(f"ro.vendor.build.fingerprint_has_digest=1") + + props.append(f"ro.vendor.build.security_patch={config['VendorSecurityPatch']}") + props.append(f"ro.product.board={config['BootloaderBoardName']}") + props.append(f"ro.board.platform={config['BoardPlatform']}") + props.append(f"ro.hwui.use_vulkan={'true' if config['UsesVulkan'] else 'false'}") + + if config["ScreenDensity"]: + props.append(f"ro.sf.lcd_density={config['ScreenDensity']}") + + if "AbOtaUpdater" in config: + props.append(f"ro.build.ab_update={'true' if config['AbOtaUpdater'] else 'false'}") + if config["AbOtaUpdater"]: + props.append(f"ro.vendor.build.ab_ota_partitions={config['AbOtaPartitions']}") + + config["ADDITIONAL_VENDOR_PROPERTIES"] = props + +def append_additional_product_props(args): + props = [] + + config = args.config + + # Add the system server compiler filter if they are specified for the product. + if config["SystemServerCompilerFilter"]: + props.append(f"dalvik.vm.systemservercompilerfilter={config['SystemServerCompilerFilter']}") + + # Add the 16K developer args if it is defined for the product. + props.append(f"ro.product.build.16k_page.enabled={'true' if config['Product16KDeveloperOption'] else 'false'}") + + props.append(f"ro.build.characteristics={config['AAPTCharacteristics']}") + + if "AbOtaUpdater" in config and config["AbOtaUpdater"]: + props.append(f"ro.product.ab_ota_partitions={config['AbOtaPartitions']}") + + # Set this property for VTS to skip large page size tests on unsupported devices. + props.append(f"ro.product.cpu.pagesize.max={config['DeviceMaxPageSizeSupported']}") + + if config["NoBionicPageSizeMacro"]: + props.append(f"ro.product.build.no_bionic_page_size_macro=true") + + # If the value is "default", it will be mangled by post_process_props.py. + props.append(f"ro.dalvik.vm.enable_uffd_gc={config['EnableUffdGc']}") + + config["ADDITIONAL_PRODUCT_PROPERTIES"] = props + +def build_system_prop(args): + config = args.config + + # Order matters here. When there are duplicates, the last one wins. + # TODO(b/117892318): don't allow duplicates so that the ordering doesn't matter + variables = [ + "ADDITIONAL_SYSTEM_PROPERTIES", + "PRODUCT_SYSTEM_PROPERTIES", + # TODO(b/117892318): deprecate this + "PRODUCT_SYSTEM_DEFAULT_PROPERTIES", + ] + + if not config["PropertySplitEnabled"]: + variables += [ + "ADDITIONAL_VENDOR_PROPERTIES", + "PRODUCT_VENDOR_PROPERTIES", + ] + + build_prop(args, gen_build_info=True, gen_common_build_props=True, variables=variables) + +''' +def build_vendor_prop(args): + config = args.config + + # Order matters here. When there are duplicates, the last one wins. + # TODO(b/117892318): don't allow duplicates so that the ordering doesn't matter + variables = [] + if config["PropertySplitEnabled"]: + variables += [ + "ADDITIONAL_VENDOR_PROPERTIES", + "PRODUCT_VENDOR_PROPERTIES", + # TODO(b/117892318): deprecate this + "PRODUCT_DEFAULT_PROPERTY_OVERRIDES", + "PRODUCT_PROPERTY_OVERRIDES", + ] + + build_prop(args, gen_build_info=False, gen_common_build_props=True, variables=variables) + +def build_product_prop(args): + config = args.config + + # Order matters here. When there are duplicates, the last one wins. + # TODO(b/117892318): don't allow duplicates so that the ordering doesn't matter + variables = [ + "ADDITIONAL_PRODUCT_PROPERTIES", + "PRODUCT_PRODUCT_PROPERTIES", + ] + build_prop(args, gen_build_info=False, gen_common_build_props=True, variables=variables) +''' + +def build_prop(args, gen_build_info, gen_common_build_props, variables): + config = args.config + + if gen_common_build_props: + generate_common_build_props(args) + + if gen_build_info: + generate_build_info(args) + + for prop_file in args.prop_files: + write_properties_from_file(prop_file) + + for variable in variables: + if variable in config: + write_properties_from_variable(variable, config[variable], args.build_broken_dup_sysprop) + +def main(): + args = parse_args() + + with contextlib.redirect_stdout(args.out): + if args.partition == "system": + build_system_prop(args) + ''' + elif args.partition == "vendor": + build_vendor_prop(args) + elif args.partition == "product": + build_product_prop(args) + ''' + else: + sys.exit(f"not supported partition {args.partition}") + +if __name__ == "__main__": + main() diff --git a/scripts/merge_json.py b/scripts/merge_json.py new file mode 100644 index 000000000..7e2f6eb51 --- /dev/null +++ b/scripts/merge_json.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# +# Copyright 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for merging two or more JSON files.""" + +import argparse +import logging +import json +import sys + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument("output", help="output JSON file", type=argparse.FileType("w")) + parser.add_argument("input", help="input JSON files", nargs="+", type=argparse.FileType("r")) + return parser.parse_args() + +def main(): + """Program entry point.""" + args = parse_args() + merged_dict = {} + has_error = False + logger = logging.getLogger(__name__) + + for json_file in args.input: + try: + data = json.load(json_file) + except json.JSONDecodeError as e: + logger.error(f"Error parsing JSON in file: {json_file.name}. Reason: {e}") + has_error = True + continue + + for key, value in data.items(): + if key not in merged_dict: + merged_dict[key] = value + elif merged_dict[key] == value: + logger.warning(f"Duplicate key '{key}' with identical values found.") + else: + logger.error(f"Conflicting values for key '{key}': {merged_dict[key]} != {value}") + has_error = True + + if has_error: + sys.exit(1) + + json.dump(merged_dict, args.output) + +if __name__ == "__main__": + main() diff --git a/sdk/sdk.go b/sdk/sdk.go index fd16ab63f..5b644fd85 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -193,6 +193,10 @@ func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Generate the snapshot from the member info. s.buildSnapshot(ctx, sdkVariants) } + + if s.snapshotFile.Valid() { + ctx.SetOutputFiles([]android.Path{s.snapshotFile.Path()}, "") + } } func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { @@ -222,18 +226,6 @@ func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { }} } -func (s *sdk) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if s.snapshotFile.Valid() { - return []android.Path{s.snapshotFile.Path()}, nil - } - return nil, fmt.Errorf("snapshot file not defined. This is most likely because this isn't the common_os variant of this module") - default: - return nil, fmt.Errorf("unknown tag %q", tag) - } -} - // gatherTraits gathers the traits from the dynamically generated trait specific properties. // // Returns a map from member name to the set of required traits. diff --git a/sh/sh_binary.go b/sh/sh_binary.go index 3a4adc6f3..2e48d83e6 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -99,6 +99,12 @@ type shBinaryProperties struct { // Make this module available when building for recovery. Recovery_available *bool + + // The name of the image this module is built for + ImageVariation string `blueprint:"mutated"` + + // Suffix for the name of Android.mk entries generated by this module + SubName string `blueprint:"mutated"` } type TestProperties struct { @@ -206,16 +212,24 @@ var _ android.ImageInterface = (*ShBinary)(nil) func (s *ShBinary) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (s *ShBinary) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return s.InstallInVendor() +} + +func (s *ShBinary) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return s.InstallInProduct() +} + func (s *ShBinary) CoreVariantNeeded(ctx android.BaseModuleContext) bool { - return !s.ModuleBase.InstallInRecovery() && !s.ModuleBase.InstallInRamdisk() + return !s.InstallInRecovery() && !s.InstallInRamdisk() && !s.InstallInVendorRamdisk() && !s.ModuleBase.InstallInVendor() } func (s *ShBinary) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool { - return proptools.Bool(s.properties.Ramdisk_available) || s.ModuleBase.InstallInRamdisk() + return proptools.Bool(s.properties.Ramdisk_available) || s.InstallInRamdisk() } func (s *ShBinary) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { - return proptools.Bool(s.properties.Vendor_ramdisk_available) || s.ModuleBase.InstallInVendorRamdisk() + return proptools.Bool(s.properties.Vendor_ramdisk_available) || s.InstallInVendorRamdisk() } func (s *ShBinary) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { @@ -223,14 +237,36 @@ func (s *ShBinary) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) bool } func (s *ShBinary) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { - return proptools.Bool(s.properties.Recovery_available) || s.ModuleBase.InstallInRecovery() + return proptools.Bool(s.properties.Recovery_available) || s.InstallInRecovery() } func (s *ShBinary) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil } -func (s *ShBinary) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { +func (s *ShBinary) SetImageVariation(ctx android.BaseModuleContext, variation string) { + s.properties.ImageVariation = variation +} + +// Overrides ModuleBase.InstallInRamdisk() so that the install rule respects +// Ramdisk_available property for ramdisk variant +func (s *ShBinary) InstallInRamdisk() bool { + return s.ModuleBase.InstallInRamdisk() || + (proptools.Bool(s.properties.Ramdisk_available) && s.properties.ImageVariation == android.RamdiskVariation) +} + +// Overrides ModuleBase.InstallInVendorRamdisk() so that the install rule respects +// Vendor_ramdisk_available property for vendor ramdisk variant +func (s *ShBinary) InstallInVendorRamdisk() bool { + return s.ModuleBase.InstallInVendorRamdisk() || + (proptools.Bool(s.properties.Vendor_ramdisk_available) && s.properties.ImageVariation == android.VendorRamdiskVariation) +} + +// Overrides ModuleBase.InstallInRecovery() so that the install rule respects +// Recovery_available property for recovery variant +func (s *ShBinary) InstallInRecovery() bool { + return s.ModuleBase.InstallInRecovery() || + (proptools.Bool(s.properties.Recovery_available) && s.properties.ImageVariation == android.RecoveryVariation) } func (s *ShBinary) generateAndroidBuildActions(ctx android.ModuleContext) { @@ -260,11 +296,24 @@ func (s *ShBinary) generateAndroidBuildActions(ctx android.ModuleContext) { Output: s.outputFilePath, Input: s.sourceFilePath, }) + + s.properties.SubName = s.GetSubname(ctx) + android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: []string{s.sourceFilePath.String()}}) ctx.SetOutputFiles(android.Paths{s.outputFilePath}, "") } +func (s *ShBinary) GetSubname(ctx android.ModuleContext) string { + ret := "" + if s.properties.ImageVariation != "" { + if s.properties.ImageVariation != android.VendorVariation { + ret = "." + s.properties.ImageVariation + } + } + return ret +} + func (s *ShBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { s.generateAndroidBuildActions(ctx) installDir := android.PathForModuleInstall(ctx, "bin", proptools.String(s.properties.Sub_dir)) @@ -278,7 +327,7 @@ func (s *ShBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (s *ShBinary) AndroidMkEntries() []android.AndroidMkEntries { - return []android.AndroidMkEntries{android.AndroidMkEntries{ + return []android.AndroidMkEntries{{ Class: "EXECUTABLES", OutputFile: android.OptionalPathForPath(s.outputFilePath), Include: "$(BUILD_SYSTEM)/soong_cc_rust_prebuilt.mk", @@ -289,6 +338,7 @@ func (s *ShBinary) AndroidMkEntries() []android.AndroidMkEntries { entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !s.Installable()) }, }, + SubName: s.properties.SubName, }} } diff --git a/shared/Android.bp b/shared/Android.bp index 3c84f5532..d5e86146f 100644 --- a/shared/Android.bp +++ b/shared/Android.bp @@ -15,7 +15,6 @@ bootstrap_go_package { "paths_test.go", ], deps: [ - "soong-bazel", "golang-protobuf-proto", ], } diff --git a/shared/paths.go b/shared/paths.go index fca8b4c15..1ee66d581 100644 --- a/shared/paths.go +++ b/shared/paths.go @@ -18,8 +18,6 @@ package shared import ( "path/filepath" - - "android/soong/bazel" ) // A SharedPaths represents a list of paths that are shared between @@ -49,11 +47,3 @@ func JoinPath(base string, rest ...string) string { func TempDirForOutDir(outDir string) (tempPath string) { return filepath.Join(outDir, ".temp") } - -// BazelMetricsFilename returns the bazel profile filename based -// on the action name. This is to help to store a set of bazel -// profiles since bazel may execute multiple times during a single -// build. -func BazelMetricsFilename(s SharedPaths, actionName bazel.RunName) string { - return filepath.Join(s.BazelMetricsDir(), actionName.String()+"_bazel_profile.gz") -} diff --git a/snapshot/host_fake_snapshot.go b/snapshot/host_fake_snapshot.go index b416ebdd4..278247e10 100644 --- a/snapshot/host_fake_snapshot.go +++ b/snapshot/host_fake_snapshot.go @@ -129,12 +129,12 @@ func (c *hostFakeSingleton) GenerateBuildActions(ctx android.SingletonContext) { if !seen[outFile] { seen[outFile] = true outputs = append(outputs, WriteStringToFileRule(ctx, "", outFile)) - jsonData = append(jsonData, hostSnapshotFakeJsonFlags{*hostJsonDesc(module), false}) + jsonData = append(jsonData, hostSnapshotFakeJsonFlags{*hostJsonDesc(ctx, module), false}) } } }) // Update any module prebuilt information - for idx, _ := range jsonData { + for idx := range jsonData { if _, ok := prebuilts[jsonData[idx].ModuleName]; ok { // Prebuilt exists for this module jsonData[idx].Prebuilt = true diff --git a/snapshot/host_snapshot.go b/snapshot/host_snapshot.go index edcc16348..1ecab7ddc 100644 --- a/snapshot/host_snapshot.go +++ b/snapshot/host_snapshot.go @@ -101,7 +101,7 @@ func (f *hostSnapshot) CreateMetaData(ctx android.ModuleContext, fileName string // Create JSON file based on the direct dependencies ctx.VisitDirectDeps(func(dep android.Module) { - desc := hostJsonDesc(dep) + desc := hostJsonDesc(ctx, dep) if desc != nil { jsonData = append(jsonData, *desc) } @@ -209,7 +209,7 @@ func hostRelativePathString(m android.Module) string { // Create JSON description for given module, only create descriptions for binary modules // and rust_proc_macro modules which provide a valid HostToolPath -func hostJsonDesc(m android.Module) *SnapshotJsonFlags { +func hostJsonDesc(ctx android.ConfigAndErrorContext, m android.Module) *SnapshotJsonFlags { path := hostToolPath(m) relPath := hostRelativePathString(m) procMacro := false @@ -226,7 +226,7 @@ func hostJsonDesc(m android.Module) *SnapshotJsonFlags { props := &SnapshotJsonFlags{ ModuleStemName: moduleStem, Filename: path.String(), - Required: append(m.HostRequiredModuleNames(), m.RequiredModuleNames()...), + Required: append(m.HostRequiredModuleNames(), m.RequiredModuleNames(ctx)...), RelativeInstallPath: relPath, RustProcMacro: procMacro, CrateName: crateName, diff --git a/sysprop/sysprop_library.go b/sysprop/sysprop_library.go index b9b68be2d..84f20c555 100644 --- a/sysprop/sysprop_library.go +++ b/sysprop/sysprop_library.go @@ -49,8 +49,6 @@ type syspropJavaGenRule struct { android.ModuleBase properties syspropGenProperties - - genSrcjars android.Paths } type syspropRustGenRule struct { @@ -59,7 +57,6 @@ type syspropRustGenRule struct { properties rustLibraryProperties } -var _ android.OutputFileProducer = (*syspropJavaGenRule)(nil) var _ rust.SourceProvider = (*syspropRustGenRule)(nil) var ( @@ -100,6 +97,7 @@ func (g *syspropJavaGenRule) GenerateAndroidBuildActions(ctx android.ModuleConte } }) + var genSrcjars android.Paths for _, syspropFile := range android.PathsForModuleSrc(ctx, g.properties.Srcs) { srcJarFile := android.GenPathWithExt(ctx, "sysprop", syspropFile, "srcjar") @@ -114,8 +112,10 @@ func (g *syspropJavaGenRule) GenerateAndroidBuildActions(ctx android.ModuleConte }, }) - g.genSrcjars = append(g.genSrcjars, srcJarFile) + genSrcjars = append(genSrcjars, srcJarFile) } + + ctx.SetOutputFiles(genSrcjars, "") } func (g *syspropJavaGenRule) DepsMutator(ctx android.BottomUpMutatorContext) { @@ -124,15 +124,6 @@ func (g *syspropJavaGenRule) DepsMutator(ctx android.BottomUpMutatorContext) { ctx.AddFarVariationDependencies(nil, nil, proptools.String(g.properties.Check_api)) } -func (g *syspropJavaGenRule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return g.genSrcjars, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func syspropJavaGenFactory() android.Module { g := &syspropJavaGenRule{} g.AddProperties(&g.properties) diff --git a/ui/build/androidmk_denylist.go b/ui/build/androidmk_denylist.go index a29f413ae..bbac2db17 100644 --- a/ui/build/androidmk_denylist.go +++ b/ui/build/androidmk_denylist.go @@ -19,10 +19,13 @@ import ( ) var androidmk_denylist []string = []string{ + "bionic/", "chained_build_config/", "cts/", "dalvik/", "developers/", + "development/", + "device/sample/", "frameworks/", // Do not block other directories in kernel/, see b/319658303. "kernel/configs/", @@ -31,6 +34,10 @@ var androidmk_denylist []string = []string{ "libcore/", "libnativehelper/", "pdk/", + "prebuilts/", + "sdk/", + "test/", + "trusty/", // Add back toolchain/ once defensive Android.mk files are removed //"toolchain/", } diff --git a/ui/build/build.go b/ui/build/build.go index 9a9eccd7d..03d839237 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -21,7 +21,6 @@ import ( "path/filepath" "sync" "text/template" - "time" "android/soong/ui/metrics" ) @@ -66,9 +65,12 @@ func SetupOutDir(ctx Context, config Config) { // (to allow for source control that uses something other than numbers), // but must be a single word and a valid file name. // - // If no BUILD_NUMBER is set, create a useful "I am an engineering build - // from this date/time" value. Make it start with a non-digit so that - // anyone trying to parse it as an integer will probably get "0". + // If no BUILD_NUMBER is set, create a useful "I am an engineering build" + // value. Make it start with a non-digit so that anyone trying to parse + // it as an integer will probably get "0". This value used to contain + // a timestamp, but now that more dependencies are tracked in order to + // reduce the importance of `m installclean`, changing it every build + // causes unnecessary rebuilds for local development. buildNumber, ok := config.environ.Get("BUILD_NUMBER") if ok { writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) @@ -77,7 +79,7 @@ func SetupOutDir(ctx Context, config Config) { if username, ok = config.environ.Get("BUILD_USERNAME"); !ok { ctx.Fatalln("Missing BUILD_USERNAME") } - buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */)) + buildNumber = fmt.Sprintf("eng.%.6s.00000000.000000", username) writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username) } // Write the build number to a file so it can be read back in diff --git a/ui/build/config.go b/ui/build/config.go index feded1c85..c4a67978f 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -85,6 +85,7 @@ type configImpl struct { skipMetricsUpload bool buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time buildFromSourceStub bool + incrementalBuildActions bool ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built // From the product config @@ -811,6 +812,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { } } else if arg == "--build-from-source-stub" { c.buildFromSourceStub = true + } else if arg == "--incremental-build-actions" { + c.incrementalBuildActions = true } else if strings.HasPrefix(arg, "--build-command=") { buildCmd := strings.TrimPrefix(arg, "--build-command=") // remove quotations @@ -1243,6 +1246,11 @@ func (c *configImpl) StartGoma() bool { } func (c *configImpl) canSupportRBE() bool { + // Only supported on linux + if runtime.GOOS != "linux" { + return false + } + // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since // its unlikely that we will be able to obtain necessary creds without stubby. authType, _ := c.rbeAuth() @@ -1488,6 +1496,15 @@ func (c *configImpl) SoongVarsFile() string { } } +func (c *configImpl) SoongExtraVarsFile() string { + targetProduct, err := c.TargetProductOrErr() + if err != nil { + return filepath.Join(c.SoongOutDir(), "soong.extra.variables") + } else { + return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".extra.variables") + } +} + func (c *configImpl) SoongNinjaFile() string { targetProduct, err := c.TargetProductOrErr() if err != nil { diff --git a/ui/build/soong.go b/ui/build/soong.go index 2f3150d03..77fee0ac6 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -27,7 +27,6 @@ import ( "android/soong/ui/tracer" - "android/soong/bazel" "android/soong/ui/metrics" "android/soong/ui/metrics/metrics_proto" "android/soong/ui/status" @@ -315,6 +314,9 @@ func bootstrapBlueprint(ctx Context, config Config) { if config.ensureAllowlistIntegrity { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity") } + if config.incrementalBuildActions { + mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--incremental-build-actions") + } queryviewDir := filepath.Join(config.SoongOutDir(), "queryview") @@ -600,10 +602,6 @@ func runSoong(ctx Context, config Config) { checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag)) - // Remove bazel files in the event that bazel is disabled for the build. - // These files may have been left over from a previous bazel-enabled build. - cleanBazelFiles(config) - if config.JsonModuleGraph() { checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag)) } @@ -694,6 +692,7 @@ func runSoong(ctx Context, config Config) { } } distFile(ctx, config, config.SoongVarsFile(), "soong") + distFile(ctx, config, config.SoongExtraVarsFile(), "soong") if !config.SkipKati() { distGzipFile(ctx, config, config.SoongAndroidMk(), "soong") @@ -754,18 +753,6 @@ func loadSoongBuildMetrics(ctx Context, config Config, oldTimestamp time.Time) { } } -func cleanBazelFiles(config Config) { - files := []string{ - shared.JoinPath(config.SoongOutDir(), "bp2build"), - shared.JoinPath(config.SoongOutDir(), "workspace"), - shared.JoinPath(config.SoongOutDir(), bazel.SoongInjectionDirName), - shared.JoinPath(config.OutDir(), "bazel")} - - for _, f := range files { - os.RemoveAll(f) - } -} - func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) { ctx.BeginTrace(metrics.RunSoong, name) defer ctx.EndTrace() diff --git a/ui/build/test_build.go b/ui/build/test_build.go index 24ad08284..687ad6f61 100644 --- a/ui/build/test_build.go +++ b/ui/build/test_build.go @@ -64,7 +64,8 @@ func testForDanglingRules(ctx Context, config Config) { outDir := config.OutDir() modulePathsDir := filepath.Join(outDir, ".module_paths") rawFilesDir := filepath.Join(outDir, "soong", "raw") - variablesFilePath := filepath.Join(outDir, "soong", "soong.variables") + variablesFilePath := config.SoongVarsFile() + extraVariablesFilePath := config.SoongExtraVarsFile() // dexpreopt.config is an input to the soong_docs action, which runs the // soong_build primary builder. However, this file is created from $(shell) @@ -95,6 +96,7 @@ func testForDanglingRules(ctx Context, config Config) { if strings.HasPrefix(line, modulePathsDir) || strings.HasPrefix(line, rawFilesDir) || line == variablesFilePath || + line == extraVariablesFilePath || line == dexpreoptConfigFilePath || line == buildDatetimeFilePath || line == bpglob || |