diff options
72 files changed, 1440 insertions, 529 deletions
diff --git a/aconfig/aconfig_declarations.go b/aconfig/aconfig_declarations.go index 80d36afd5..b55d7bf5c 100644 --- a/aconfig/aconfig_declarations.go +++ b/aconfig/aconfig_declarations.go @@ -112,24 +112,6 @@ func optionalVariable(prefix string, value string) string { return sb.String() } -// Provider published by aconfig_value_set -type DeclarationsProviderData struct { - Package string - Container string - IntermediateCacheOutputPath android.WritablePath - IntermediateDumpOutputPath android.WritablePath -} - -var DeclarationsProviderKey = blueprint.NewProvider[DeclarationsProviderData]() - -// This is used to collect the aconfig declarations info on the transitive closure, -// the data is keyed on the container. -type TransitiveDeclarationsInfo struct { - AconfigFiles map[string]android.Paths -} - -var TransitiveDeclarationsInfoProvider = blueprint.NewProvider[TransitiveDeclarationsInfo]() - func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Get the values that came from the global RELEASE_ACONFIG_VALUE_SETS flag valuesFiles := make([]android.Path, 0) @@ -174,7 +156,7 @@ func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.Module Description: "aconfig_text", }) - android.SetProvider(ctx, DeclarationsProviderKey, DeclarationsProviderData{ + android.SetProvider(ctx, android.AconfigDeclarationsProviderKey, android.AconfigDeclarationsProviderData{ Package: module.properties.Package, Container: module.properties.Container, IntermediateCacheOutputPath: intermediateCacheFilePath, @@ -182,57 +164,18 @@ func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.Module }) } -func CollectDependencyAconfigFiles(ctx android.ModuleContext, mergedAconfigFiles *map[string]android.Paths) { - if *mergedAconfigFiles == nil { - *mergedAconfigFiles = make(map[string]android.Paths) - } - ctx.VisitDirectDeps(func(module android.Module) { - if dep, _ := android.OtherModuleProvider(ctx, module, DeclarationsProviderKey); dep.IntermediateCacheOutputPath != nil { - (*mergedAconfigFiles)[dep.Container] = append((*mergedAconfigFiles)[dep.Container], dep.IntermediateCacheOutputPath) - return - } - if dep, _ := android.OtherModuleProvider(ctx, module, TransitiveDeclarationsInfoProvider); len(dep.AconfigFiles) > 0 { - for container, v := range dep.AconfigFiles { - (*mergedAconfigFiles)[container] = append((*mergedAconfigFiles)[container], v...) - } - } - }) - - for container, aconfigFiles := range *mergedAconfigFiles { - (*mergedAconfigFiles)[container] = mergeAconfigFiles(ctx, aconfigFiles) - } - - android.SetProvider(ctx, TransitiveDeclarationsInfoProvider, TransitiveDeclarationsInfo{ - AconfigFiles: *mergedAconfigFiles, - }) -} - -func mergeAconfigFiles(ctx android.ModuleContext, inputs android.Paths) android.Paths { - inputs = android.LastUniquePaths(inputs) - if len(inputs) == 1 { - return android.Paths{inputs[0]} - } - - output := android.PathForModuleOut(ctx, "aconfig_merged.pb") - - ctx.Build(pctx, android.BuildParams{ - Rule: mergeAconfigFilesRule, - Description: "merge aconfig files", - Inputs: inputs, - Output: output, - Args: map[string]string{ - "flags": android.JoinWithPrefix(inputs.Strings(), "--cache "), - }, - }) - - return android.Paths{output} -} func SetAconfigFileMkEntries(m *android.ModuleBase, entries *android.AndroidMkEntries, aconfigFiles map[string]android.Paths) { - if m.InstallInVendor() { - entries.SetPaths("LOCAL_ACONFIG_FILES", aconfigFiles["vendor"]) - } else { - // TODO(b/311155208): The container here should be system. - entries.SetPaths("LOCAL_ACONFIG_FILES", aconfigFiles[""]) + // TODO(b/311155208): The default container here should be system. + container := "" + + if m.SocSpecific() { + container = "vendor" + } else if m.ProductSpecific() { + container = "product" + } else if m.SystemExtSpecific() { + container = "system_ext" } + + entries.SetPaths("LOCAL_ACONFIG_FILES", aconfigFiles[container]) } diff --git a/aconfig/aconfig_declarations_test.go b/aconfig/aconfig_declarations_test.go index 1b4acabeb..d508af7a4 100644 --- a/aconfig/aconfig_declarations_test.go +++ b/aconfig/aconfig_declarations_test.go @@ -38,7 +38,7 @@ func TestAconfigDeclarations(t *testing.T) { module := result.ModuleForTests("module_name", "").Module().(*DeclarationsModule) // Check that the provider has the right contents - depData, _ := android.SingletonModuleProvider(result, module, DeclarationsProviderKey) + depData, _ := android.SingletonModuleProvider(result, module, android.AconfigDeclarationsProviderKey) android.AssertStringEquals(t, "package", depData.Package, "com.example.package") android.AssertStringEquals(t, "container", depData.Container, "com.android.foo") if !strings.HasSuffix(depData.IntermediateCacheOutputPath.String(), "/intermediate.pb") { diff --git a/aconfig/all_aconfig_declarations.go b/aconfig/all_aconfig_declarations.go index d8604985b..36bea0e3f 100644 --- a/aconfig/all_aconfig_declarations.go +++ b/aconfig/all_aconfig_declarations.go @@ -37,7 +37,7 @@ func (this *allAconfigDeclarationsSingleton) GenerateBuildActions(ctx android.Si // Find all of the aconfig_declarations modules var cacheFiles android.Paths ctx.VisitAllModules(func(module android.Module) { - decl, ok := android.SingletonModuleProvider(ctx, module, DeclarationsProviderKey) + decl, ok := android.SingletonModuleProvider(ctx, module, android.AconfigDeclarationsProviderKey) if !ok { return } diff --git a/aconfig/codegen/cc_aconfig_library.go b/aconfig/codegen/cc_aconfig_library.go index 2c8369be0..12c2dea3a 100644 --- a/aconfig/codegen/cc_aconfig_library.go +++ b/aconfig/codegen/cc_aconfig_library.go @@ -15,7 +15,6 @@ package codegen import ( - "android/soong/aconfig" "android/soong/android" "android/soong/cc" @@ -92,7 +91,7 @@ func (this *CcAconfigLibraryCallbacks) GeneratorSources(ctx cc.ModuleContext) cc if len(declarationsModules) != 1 { panic(fmt.Errorf("Exactly one aconfig_declarations property required")) } - declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], aconfig.DeclarationsProviderKey) + declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], android.AconfigDeclarationsProviderKey) // Figure out the generated file paths. This has to match aconfig's codegen_cpp.rs. this.generatedDir = android.PathForModuleGen(ctx) @@ -122,7 +121,7 @@ func (this *CcAconfigLibraryCallbacks) GeneratorBuildActions(ctx cc.ModuleContex if len(declarationsModules) != 1 { panic(fmt.Errorf("Exactly one aconfig_declarations property required")) } - declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], aconfig.DeclarationsProviderKey) + declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], android.AconfigDeclarationsProviderKey) mode := proptools.StringDefault(this.properties.Mode, "production") if !isModeSupported(mode) { diff --git a/aconfig/codegen/java_aconfig_library.go b/aconfig/codegen/java_aconfig_library.go index b33481b20..c027815db 100644 --- a/aconfig/codegen/java_aconfig_library.go +++ b/aconfig/codegen/java_aconfig_library.go @@ -17,7 +17,6 @@ package codegen import ( "fmt" - "android/soong/aconfig" "android/soong/android" "android/soong/java" @@ -62,10 +61,18 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) DepsMutator(module *ja ctx.AddDependency(ctx.Module(), declarationsTag, declarations) } - // Add aconfig-annotations-lib as a dependency for the optimization / code stripping annotations - module.AddSharedLibrary("aconfig-annotations-lib") - // TODO(b/303773055): Remove the annotation after access issue is resolved. - module.AddSharedLibrary("unsupportedappusage") + // "libcore_aconfig_flags_lib" module has a circular dependency because the shared libraries + // are built on core_current and the module is used to flag the APIs in the core_current. + // http://b/316554963#comment2 has the details of the circular dependency chain. + // If a java_aconfig_library uses "none" sdk_version, it should include and build these + // annotation files as the shared library themselves. + var addLibraries bool = module.Library.Module.SdkVersion(ctx).Kind != android.SdkNone + if addLibraries { + // Add aconfig-annotations-lib as a dependency for the optimization / code stripping annotations + module.AddSharedLibrary("aconfig-annotations-lib") + // TODO(b/303773055): Remove the annotation after access issue is resolved. + module.AddSharedLibrary("unsupportedappusage") + } } func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuildActions(module *java.GeneratedJavaLibraryModule, ctx android.ModuleContext) android.Path { @@ -74,7 +81,7 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild if len(declarationsModules) != 1 { panic(fmt.Errorf("Exactly one aconfig_declarations property required")) } - declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], aconfig.DeclarationsProviderKey) + declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], android.AconfigDeclarationsProviderKey) // Generate the action to build the srcjar srcJarPath := android.PathForModuleGen(ctx, ctx.ModuleName()+".srcjar") diff --git a/aconfig/codegen/rust_aconfig_library.go b/aconfig/codegen/rust_aconfig_library.go index 88f5b452c..73b6fece2 100644 --- a/aconfig/codegen/rust_aconfig_library.go +++ b/aconfig/codegen/rust_aconfig_library.go @@ -3,7 +3,6 @@ package codegen import ( "fmt" - "android/soong/aconfig" "android/soong/android" "android/soong/rust" @@ -65,7 +64,7 @@ func (a *aconfigDecorator) GenerateSource(ctx rust.ModuleContext, deps rust.Path if len(declarationsModules) != 1 { panic(fmt.Errorf("Exactly one aconfig_declarations property required")) } - declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], aconfig.DeclarationsProviderKey) + declarations, _ := android.OtherModuleProvider(ctx, declarationsModules[0], android.AconfigDeclarationsProviderKey) mode := proptools.StringDefault(a.Properties.Mode, "production") if !isModeSupported(mode) { diff --git a/aconfig/exported_java_aconfig_library.go b/aconfig/exported_java_aconfig_library.go index 864481065..291938fa8 100644 --- a/aconfig/exported_java_aconfig_library.go +++ b/aconfig/exported_java_aconfig_library.go @@ -30,7 +30,7 @@ func (this *exportedJavaDeclarationsLibrarySingleton) GenerateBuildActions(ctx a // Find all of the aconfig_declarations modules var cacheFiles android.Paths ctx.VisitAllModules(func(module android.Module) { - decl, ok := android.SingletonModuleProvider(ctx, module, DeclarationsProviderKey) + decl, ok := android.SingletonModuleProvider(ctx, module, android.AconfigDeclarationsProviderKey) if !ok { return } diff --git a/aconfig/init.go b/aconfig/init.go index 52d075572..3e9d29760 100644 --- a/aconfig/init.go +++ b/aconfig/init.go @@ -43,7 +43,7 @@ var ( // For create-device-config-sysprops: Generate aconfig flag value map text file aconfigTextRule = pctx.AndroidStaticRule("aconfig_text", blueprint.RuleParams{ - Command: `${aconfig} dump --format bool` + + Command: `${aconfig} dump-cache --format='{fully_qualified_name}={state:bool}'` + ` --cache ${in}` + ` --out ${out}.tmp` + ` && ( if cmp -s ${out}.tmp ${out} ; then rm ${out}.tmp ; else mv ${out}.tmp ${out} ; fi )`, @@ -56,17 +56,12 @@ var ( // For all_aconfig_declarations: Combine all parsed_flags proto files AllDeclarationsRule = pctx.AndroidStaticRule("All_aconfig_declarations_dump", blueprint.RuleParams{ - Command: `${aconfig} dump --format protobuf --out ${out} ${cache_files}`, + Command: `${aconfig} dump-cache --format protobuf --out ${out} ${cache_files}`, CommandDeps: []string{ "${aconfig}", }, }, "cache_files") - mergeAconfigFilesRule = pctx.AndroidStaticRule("mergeAconfigFilesRule", - blueprint.RuleParams{ - Command: `${aconfig} dump --dedup --format protobuf --out $out $flags`, - CommandDeps: []string{"${aconfig}"}, - }, "flags") // For exported_java_aconfig_library: Generate a JAR from all // java_aconfig_libraries to be consumed by apps built outside the // platform @@ -78,7 +73,7 @@ var ( blueprint.RuleParams{ Command: `rm -rf ${out}.tmp` + `&& for cache in ${cache_files}; do ` + - ` if [[ -n "$$(${aconfig} dump --cache $$cache --filter=is_exported:true --format='{fully_qualified_name}')" ]]; then ` + + ` if [ -n "$$(${aconfig} dump-cache --cache $$cache --filter=is_exported:true --format='{fully_qualified_name}')" ]; then ` + ` ${aconfig} create-java-lib --cache $$cache --mode=exported --out ${out}.tmp; ` + ` fi ` + `done` + diff --git a/android/Android.bp b/android/Android.bp index 3a7ffd09c..e40e462cc 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -27,6 +27,7 @@ bootstrap_go_package { "androidmk-parser", ], srcs: [ + "aconfig_providers.go", "androidmk.go", "apex.go", "apex_contributions.go", @@ -81,6 +82,7 @@ bootstrap_go_package { "prebuilt_build_tool.go", "proto.go", "provider.go", + "raw_files.go", "register.go", "rule_builder.go", "sandbox.go", diff --git a/android/aconfig_providers.go b/android/aconfig_providers.go new file mode 100644 index 000000000..ddebec343 --- /dev/null +++ b/android/aconfig_providers.go @@ -0,0 +1,92 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "github.com/google/blueprint" +) + +var ( + mergeAconfigFilesRule = pctx.AndroidStaticRule("mergeAconfigFilesRule", + blueprint.RuleParams{ + Command: `${aconfig} dump --dedup --format protobuf --out $out $flags`, + CommandDeps: []string{"${aconfig}"}, + }, "flags") + _ = pctx.HostBinToolVariable("aconfig", "aconfig") +) + +// Provider published by aconfig_value_set +type AconfigDeclarationsProviderData struct { + Package string + Container string + IntermediateCacheOutputPath WritablePath + IntermediateDumpOutputPath WritablePath +} + +var AconfigDeclarationsProviderKey = blueprint.NewProvider[AconfigDeclarationsProviderData]() + +// This is used to collect the aconfig declarations info on the transitive closure, +// the data is keyed on the container. +type AconfigTransitiveDeclarationsInfo struct { + AconfigFiles map[string]Paths +} + +var AconfigTransitiveDeclarationsInfoProvider = blueprint.NewProvider[AconfigTransitiveDeclarationsInfo]() + +func CollectDependencyAconfigFiles(ctx ModuleContext, mergedAconfigFiles *map[string]Paths) { + if *mergedAconfigFiles == nil { + *mergedAconfigFiles = make(map[string]Paths) + } + ctx.VisitDirectDeps(func(module Module) { + if dep, _ := OtherModuleProvider(ctx, module, AconfigDeclarationsProviderKey); dep.IntermediateCacheOutputPath != nil { + (*mergedAconfigFiles)[dep.Container] = append((*mergedAconfigFiles)[dep.Container], dep.IntermediateCacheOutputPath) + return + } + if dep, _ := OtherModuleProvider(ctx, module, AconfigTransitiveDeclarationsInfoProvider); len(dep.AconfigFiles) > 0 { + for container, v := range dep.AconfigFiles { + (*mergedAconfigFiles)[container] = append((*mergedAconfigFiles)[container], v...) + } + } + }) + + for container, aconfigFiles := range *mergedAconfigFiles { + (*mergedAconfigFiles)[container] = mergeAconfigFiles(ctx, aconfigFiles) + } + + SetProvider(ctx, AconfigTransitiveDeclarationsInfoProvider, AconfigTransitiveDeclarationsInfo{ + AconfigFiles: *mergedAconfigFiles, + }) +} + +func mergeAconfigFiles(ctx ModuleContext, inputs Paths) Paths { + inputs = LastUniquePaths(inputs) + if len(inputs) == 1 { + return Paths{inputs[0]} + } + + output := PathForModuleOut(ctx, "aconfig_merged.pb") + + ctx.Build(pctx, BuildParams{ + Rule: mergeAconfigFilesRule, + Description: "merge aconfig files", + Inputs: inputs, + Output: output, + Args: map[string]string{ + "flags": JoinWithPrefix(inputs.Strings(), "--cache "), + }, + }) + + return Paths{output} +} diff --git a/android/androidmk.go b/android/androidmk.go index a3334dc2e..f65e084be 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -893,6 +893,7 @@ func translateAndroidModule(ctx SingletonContext, w io.Writer, moduleInfoJSONs * case "*java.SystemModules": // doesn't go through base_rules case "*java.systemModulesImport": // doesn't go through base_rules case "*phony.phony": // license properties written + case "*phony.PhonyRule": // writes phony deps and acts like `.PHONY` case "*selinux.selinuxContextsModule": // license properties written case "*sysprop.syspropLibrary": // license properties written default: diff --git a/android/apex.go b/android/apex.go index c0907a758..c1e7a5cd0 100644 --- a/android/apex.go +++ b/android/apex.go @@ -954,3 +954,18 @@ type ApexTestInterface interface { // Return true if the apex bundle is an apex_test IsTestApex() bool } + +var ApexExportsInfoProvider = blueprint.NewProvider[ApexExportsInfo]() + +// ApexExportsInfo contains information about the artifacts provided by apexes to dexpreopt and hiddenapi +type ApexExportsInfo struct { + // Canonical name of this APEX. Used to determine the path to the activated APEX on + // device (/apex/<apex_name>) + ApexName string + + // Path to the image profile file on host (or empty, if profile is not generated). + ProfilePathOnHost Path + + // Map from the apex library name (without prebuilt_ prefix) to the dex file path on host + LibraryNameToDexJarPathOnHost map[string]Path +} diff --git a/android/apex_contributions.go b/android/apex_contributions.go index 34941c091..a30964080 100644 --- a/android/apex_contributions.go +++ b/android/apex_contributions.go @@ -164,6 +164,18 @@ func (p *PrebuiltSelectionInfoMap) IsSelected(baseModuleName, name string) bool } } +// Return the list of soong modules selected for this api domain +// In the case of apexes, it is the canonical name of the apex on device (/apex/<apex_name>) +func (p *PrebuiltSelectionInfoMap) GetSelectedModulesForApiDomain(apiDomain string) []string { + selected := []string{} + for _, entry := range *p { + if entry.apiDomain == apiDomain { + selected = append(selected, entry.selectedModuleName) + } + } + return selected +} + // This module type does not have any build actions. func (a *allApexContributions) GenerateAndroidBuildActions(ctx ModuleContext) { } diff --git a/android/base_module_context.go b/android/base_module_context.go index 2a4b12ee5..3dfe1234b 100644 --- a/android/base_module_context.go +++ b/android/base_module_context.go @@ -75,34 +75,28 @@ type BaseModuleContext interface { // It is intended for use inside the visit functions of Visit* and WalkDeps. OtherModuleType(m blueprint.Module) string - // OtherModuleProvider returns the value for a provider for the given module. If the value is - // not set it returns the zero value of the type of the provider, so the return value can always - // be type asserted to the type of the provider. The value returned may be a deep copy of the - // value originally passed to SetProvider. - OtherModuleProvider(m blueprint.Module, provider blueprint.AnyProviderKey) any - - // OtherModuleHasProvider returns true if the provider for the given module has been set. - OtherModuleHasProvider(m blueprint.Module, provider blueprint.AnyProviderKey) bool - + // otherModuleProvider returns the value for a provider for the given module. If the value is + // not set it returns nil and false. The value returned may be a deep copy of the value originally + // passed to SetProvider. + // + // This method shouldn't be used directly, prefer the type-safe android.OtherModuleProvider instead. otherModuleProvider(m blueprint.Module, provider blueprint.AnyProviderKey) (any, bool) // Provider returns the value for a provider for the current module. If the value is - // not set it returns the zero value of the type of the provider, so the return value can always - // be type asserted to the type of the provider. It panics if called before the appropriate + // not set it returns nil and false. It panics if called before the appropriate // mutator or GenerateBuildActions pass for the provider. The value returned may be a deep // copy of the value originally passed to SetProvider. - Provider(provider blueprint.AnyProviderKey) any - - // HasProvider returns true if the provider for the current module has been set. - HasProvider(provider blueprint.AnyProviderKey) bool - + // + // This method shouldn't be used directly, prefer the type-safe android.ModuleProvider instead. provider(provider blueprint.AnyProviderKey) (any, bool) - // SetProvider sets the value for a provider for the current module. It panics if not called + // setProvider sets the value for a provider for the current module. It panics if not called // during the appropriate mutator or GenerateBuildActions pass for the provider, if the value // is not of the appropriate type, or if the value has already been set. The value should not // be modified after being passed to SetProvider. - SetProvider(provider blueprint.AnyProviderKey, value interface{}) + // + // This method shouldn't be used directly, prefer the type-safe android.SetProvider instead. + setProvider(provider blueprint.AnyProviderKey, value any) GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module @@ -264,35 +258,16 @@ func (b *baseModuleContext) OtherModuleReverseDependencyVariantExists(name strin func (b *baseModuleContext) OtherModuleType(m blueprint.Module) string { return b.bp.OtherModuleType(m) } -func (b *baseModuleContext) OtherModuleProvider(m blueprint.Module, provider blueprint.AnyProviderKey) any { - value, _ := b.bp.OtherModuleProvider(m, provider) - return value -} - -func (b *baseModuleContext) OtherModuleHasProvider(m blueprint.Module, provider blueprint.AnyProviderKey) bool { - _, ok := b.bp.OtherModuleProvider(m, provider) - return ok -} func (b *baseModuleContext) otherModuleProvider(m blueprint.Module, provider blueprint.AnyProviderKey) (any, bool) { return b.bp.OtherModuleProvider(m, provider) } -func (b *baseModuleContext) Provider(provider blueprint.AnyProviderKey) any { - value, _ := b.bp.Provider(provider) - return value -} - -func (b *baseModuleContext) HasProvider(provider blueprint.AnyProviderKey) bool { - _, ok := b.bp.Provider(provider) - return ok -} - func (b *baseModuleContext) provider(provider blueprint.AnyProviderKey) (any, bool) { return b.bp.Provider(provider) } -func (b *baseModuleContext) SetProvider(provider blueprint.AnyProviderKey, value any) { +func (b *baseModuleContext) setProvider(provider blueprint.AnyProviderKey, value any) { b.bp.SetProvider(provider, value) } diff --git a/android/config.go b/android/config.go index 312a5da49..24b9b8a62 100644 --- a/android/config.go +++ b/android/config.go @@ -18,6 +18,7 @@ package android // product variables necessary for soong_build's operation. import ( + "android/soong/shared" "encoding/json" "fmt" "os" @@ -118,6 +119,11 @@ func (c Config) SoongOutDir() string { return c.soongOutDir } +// tempDir returns the path to out/soong/.temp, which is cleared at the beginning of every build. +func (c Config) tempDir() string { + return shared.TempDirForOutDir(c.soongOutDir) +} + func (c Config) OutDir() string { return c.outDir } diff --git a/android/deapexer.go b/android/deapexer.go index de933d1a4..2704b3ef7 100644 --- a/android/deapexer.go +++ b/android/deapexer.go @@ -15,6 +15,7 @@ package android import ( + "fmt" "strings" "github.com/google/blueprint" @@ -78,6 +79,10 @@ type DeapexerInfo struct { // // See Prebuilt.ApexInfoMutator for more information. exports map[string]WritablePath + + // name of the java libraries exported from the apex + // e.g. core-libart + exportedModuleNames []string } // ApexModuleName returns the name of the APEX module that provided the info. @@ -96,6 +101,10 @@ func (i DeapexerInfo) PrebuiltExportPath(apexRelativePath string) WritablePath { return path } +func (i DeapexerInfo) GetExportedModuleNames() []string { + return i.exportedModuleNames +} + // Provider that can be used from within the `GenerateAndroidBuildActions` of a module that depends // on a `deapexer` module to retrieve its `DeapexerInfo`. var DeapexerProvider = blueprint.NewProvider[DeapexerInfo]() @@ -104,10 +113,11 @@ var DeapexerProvider = blueprint.NewProvider[DeapexerInfo]() // for use with a prebuilt_apex module. // // See apex/deapexer.go for more information. -func NewDeapexerInfo(apexModuleName string, exports map[string]WritablePath) DeapexerInfo { +func NewDeapexerInfo(apexModuleName string, exports map[string]WritablePath, moduleNames []string) DeapexerInfo { return DeapexerInfo{ - apexModuleName: apexModuleName, - exports: exports, + apexModuleName: apexModuleName, + exports: exports, + exportedModuleNames: moduleNames, } } @@ -146,10 +156,16 @@ type RequiresFilesFromPrebuiltApexTag interface { // FindDeapexerProviderForModule searches through the direct dependencies of the current context // module for a DeapexerTag dependency and returns its DeapexerInfo. If a single nonambiguous -// deapexer module isn't found then errors are reported with ctx.ModuleErrorf and nil is returned. -func FindDeapexerProviderForModule(ctx ModuleContext) *DeapexerInfo { +// deapexer module isn't found then it returns it an error +// clients should check the value of error and call ctx.ModuleErrof if a non nil error is received +func FindDeapexerProviderForModule(ctx ModuleContext) (*DeapexerInfo, error) { var di *DeapexerInfo + var err error ctx.VisitDirectDepsWithTag(DeapexerTag, func(m Module) { + if err != nil { + // An err has been found. Do not visit further. + return + } c, _ := OtherModuleProvider(ctx, m, DeapexerProvider) p := &c if di != nil { @@ -159,17 +175,18 @@ func FindDeapexerProviderForModule(ctx ModuleContext) *DeapexerInfo { di = selected return } - ctx.ModuleErrorf("Multiple installable prebuilt APEXes provide ambiguous deapexers: %s and %s", - di.ApexModuleName(), p.ApexModuleName()) + err = fmt.Errorf("Multiple installable prebuilt APEXes provide ambiguous deapexers: %s and %s", di.ApexModuleName(), p.ApexModuleName()) } di = p }) + if err != nil { + return nil, err + } if di != nil { - return di + return di, nil } ai, _ := ModuleProvider(ctx, ApexInfoProvider) - ctx.ModuleErrorf("No prebuilt APEX provides a deapexer module for APEX variant %s", ai.ApexVariationName) - return nil + return nil, fmt.Errorf("No prebuilt APEX provides a deapexer module for APEX variant %s", ai.ApexVariationName) } // removeCompressedApexSuffix removes the _compressed suffix from the name if present. diff --git a/android/defs.go b/android/defs.go index 03968c10d..a9889648b 100644 --- a/android/defs.go +++ b/android/defs.go @@ -15,13 +15,8 @@ package android import ( - "fmt" - "strings" - "testing" - "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" - "github.com/google/blueprint/proptools" ) var ( @@ -72,8 +67,7 @@ var ( Command: "if ! cmp -s $in $out; then cp $in $out; fi", Description: "cp if changed $out", Restat: true, - }, - "cpFlags") + }) CpExecutable = pctx.AndroidStaticRule("CpExecutable", blueprint.RuleParams{ @@ -146,106 +140,6 @@ func BazelCcToolchainVars(config Config) string { return BazelToolchainVars(config, exportedVars) } -var ( - // echoEscaper escapes a string such that passing it to "echo -e" will produce the input value. - echoEscaper = strings.NewReplacer( - `\`, `\\`, // First escape existing backslashes so they aren't interpreted by `echo -e`. - "\n", `\n`, // Then replace newlines with \n - ) - - // echoEscaper reverses echoEscaper. - echoUnescaper = strings.NewReplacer( - `\n`, "\n", - `\\`, `\`, - ) - - // shellUnescaper reverses the replacer in proptools.ShellEscape - shellUnescaper = strings.NewReplacer(`'\''`, `'`) -) - -func buildWriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { - content = echoEscaper.Replace(content) - content = proptools.NinjaEscape(proptools.ShellEscapeIncludingSpaces(content)) - if content == "" { - content = "''" - } - ctx.Build(pctx, BuildParams{ - Rule: writeFile, - Output: outputFile, - Description: "write " + outputFile.Base(), - Args: map[string]string{ - "content": content, - }, - }) -} - -// WriteFileRule creates a ninja rule to write contents to a file. The contents will be escaped -// so that the file contains exactly the contents passed to the function, plus a trailing newline. -func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { - WriteFileRuleVerbatim(ctx, outputFile, content+"\n") -} - -// WriteFileRuleVerbatim creates a ninja rule to write contents to a file. The contents will be -// escaped so that the file contains exactly the contents passed to the function. -func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { - // This is MAX_ARG_STRLEN subtracted with some safety to account for shell escapes - const SHARD_SIZE = 131072 - 10000 - - if len(content) > SHARD_SIZE { - var chunks WritablePaths - for i, c := range ShardString(content, SHARD_SIZE) { - tempPath := outputFile.ReplaceExtension(ctx, fmt.Sprintf("%s.%d", outputFile.Ext(), i)) - buildWriteFileRule(ctx, tempPath, c) - chunks = append(chunks, tempPath) - } - ctx.Build(pctx, BuildParams{ - Rule: Cat, - Inputs: chunks.Paths(), - Output: outputFile, - Description: "Merging to " + outputFile.Base(), - }) - return - } - buildWriteFileRule(ctx, outputFile, content) -} - -// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result -func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { - intermediate := PathForIntermediates(ctx, "write_executable_file_intermediates").Join(ctx, outputFile.String()) - WriteFileRuleVerbatim(ctx, intermediate, content) - ctx.Build(pctx, BuildParams{ - Rule: CpExecutable, - Output: outputFile, - Input: intermediate, - }) -} - -// shellUnescape reverses proptools.ShellEscape -func shellUnescape(s string) string { - // Remove leading and trailing quotes if present - if len(s) >= 2 && s[0] == '\'' { - s = s[1 : len(s)-1] - } - s = shellUnescaper.Replace(s) - return s -} - -// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use -// in tests. -func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string { - t.Helper() - if g, w := params.Rule, writeFile; g != w { - t.Errorf("expected params.Rule to be %q, was %q", w, g) - return "" - } - - content := params.Args["content"] - content = shellUnescape(content) - content = echoUnescaper.Replace(content) - - return content -} - // GlobToListFileRule creates a rule that writes a list of files matching a pattern to a file. func GlobToListFileRule(ctx ModuleContext, pattern string, excludes []string, file WritablePath) { bootstrap.GlobFile(ctx.blueprintModuleContext(), pattern, excludes, file.String()) diff --git a/android/packaging.go b/android/packaging.go index 503bb97e0..2506378d7 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -17,6 +17,7 @@ package android import ( "fmt" "path/filepath" + "strings" "github.com/google/blueprint" ) @@ -240,6 +241,9 @@ func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]Packa // entries into the specified directory. func (p *PackagingBase) CopySpecsToDir(ctx ModuleContext, builder *RuleBuilder, specs map[string]PackagingSpec, dir WritablePath) (entries []string) { seenDir := make(map[string]bool) + preparerPath := PathForModuleOut(ctx, "preparer.sh") + cmd := builder.Command().Tool(preparerPath) + var sb strings.Builder for _, k := range SortedKeys(specs) { ps := specs[k] destPath := filepath.Join(dir.String(), ps.relPathInPackage) @@ -247,18 +251,21 @@ func (p *PackagingBase) CopySpecsToDir(ctx ModuleContext, builder *RuleBuilder, entries = append(entries, ps.relPathInPackage) if _, ok := seenDir[destDir]; !ok { seenDir[destDir] = true - builder.Command().Text("mkdir").Flag("-p").Text(destDir) + sb.WriteString(fmt.Sprintf("mkdir -p %s\n", destDir)) } if ps.symlinkTarget == "" { - builder.Command().Text("cp").Input(ps.srcPath).Text(destPath) + cmd.Implicit(ps.srcPath) + sb.WriteString(fmt.Sprintf("cp %s %s\n", ps.srcPath, destPath)) } else { - builder.Command().Text("ln").Flag("-sf").Text(ps.symlinkTarget).Text(destPath) + sb.WriteString(fmt.Sprintf("ln -sf %s %s\n", ps.symlinkTarget, destPath)) } if ps.executable { - builder.Command().Text("chmod").Flag("a+x").Text(destPath) + sb.WriteString(fmt.Sprintf("chmod a+x %s\n", destPath)) } } + WriteExecutableFileRuleVerbatim(ctx, preparerPath, sb.String()) + return entries } diff --git a/android/provider.go b/android/provider.go index b2cc7c06d..3b9c5d2ba 100644 --- a/android/provider.go +++ b/android/provider.go @@ -79,7 +79,7 @@ func SingletonModuleProvider[K any](ctx SingletonModuleProviderContext, module b // SetProviderContext is a helper interface that is a subset of ModuleContext, BottomUpMutatorContext, or // TopDownMutatorContext for use in SetProvider. type SetProviderContext interface { - SetProvider(provider blueprint.AnyProviderKey, value any) + setProvider(provider blueprint.AnyProviderKey, value any) } var _ SetProviderContext = BaseModuleContext(nil) @@ -95,7 +95,7 @@ var _ SetProviderContext = TopDownMutatorContext(nil) // SetProviderContext is a helper interface that accepts ModuleContext, BottomUpMutatorContext, or // TopDownMutatorContext. func SetProvider[K any](ctx SetProviderContext, provider blueprint.ProviderKey[K], value K) { - ctx.SetProvider(provider, value) + ctx.setProvider(provider, value) } var _ OtherModuleProviderContext = (*otherModuleProviderAdaptor)(nil) diff --git a/android/raw_files.go b/android/raw_files.go new file mode 100644 index 000000000..9d7f5e82d --- /dev/null +++ b/android/raw_files.go @@ -0,0 +1,279 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "github.com/google/blueprint" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/blueprint/proptools" +) + +// WriteFileRule creates a ninja rule to write contents to a file by immediately writing the +// contents, plus a trailing newline, to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating +// a ninja rule to copy the file into place. +func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { + writeFileRule(ctx, outputFile, content, true, false) +} + +// WriteFileRuleVerbatim creates a ninja rule to write contents to a file by immediately writing the +// contents to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating a ninja rule to copy the file into place. +func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { + writeFileRule(ctx, outputFile, content, false, false) +} + +// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result +func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { + writeFileRule(ctx, outputFile, content, false, true) +} + +// tempFile provides a testable wrapper around a file in out/soong/.temp. It writes to a temporary file when +// not in tests, but writes to a buffer in memory when used in tests. +type tempFile struct { + // tempFile contains wraps an io.Writer, which will be file if testMode is false, or testBuf if it is true. + io.Writer + + file *os.File + testBuf *strings.Builder +} + +func newTempFile(ctx BuilderContext, pattern string, testMode bool) *tempFile { + if testMode { + testBuf := &strings.Builder{} + return &tempFile{ + Writer: testBuf, + testBuf: testBuf, + } + } else { + f, err := os.CreateTemp(absolutePath(ctx.Config().tempDir()), pattern) + if err != nil { + panic(fmt.Errorf("failed to open temporary raw file: %w", err)) + } + return &tempFile{ + Writer: f, + file: f, + } + } +} + +func (t *tempFile) close() error { + if t.file != nil { + return t.file.Close() + } + return nil +} + +func (t *tempFile) name() string { + if t.file != nil { + return t.file.Name() + } + return "temp_file_in_test" +} + +func (t *tempFile) rename(to string) { + if t.file != nil { + os.MkdirAll(filepath.Dir(to), 0777) + err := os.Rename(t.file.Name(), to) + if err != nil { + panic(fmt.Errorf("failed to rename %s to %s: %w", t.file.Name(), to, err)) + } + } +} + +func (t *tempFile) remove() error { + if t.file != nil { + return os.Remove(t.file.Name()) + } + return nil +} + +func writeContentToTempFileAndHash(ctx BuilderContext, content string, newline bool) (*tempFile, string) { + tempFile := newTempFile(ctx, "raw", ctx.Config().captureBuild) + defer tempFile.close() + + hash := sha1.New() + w := io.MultiWriter(tempFile, hash) + + _, err := io.WriteString(w, content) + if err == nil && newline { + _, err = io.WriteString(w, "\n") + } + if err != nil { + panic(fmt.Errorf("failed to write to temporary raw file %s: %w", tempFile.name(), err)) + } + return tempFile, hex.EncodeToString(hash.Sum(nil)) +} + +func writeFileRule(ctx BuilderContext, outputFile WritablePath, content string, newline bool, executable bool) { + // Write the contents to a temporary file while computing its hash. + tempFile, hash := writeContentToTempFileAndHash(ctx, content, newline) + + // Shard the final location of the raw file into a subdirectory based on the first two characters of the + // hash to avoid making the raw directory too large and slowing down accesses. + relPath := filepath.Join(hash[0:2], hash) + + // These files are written during soong_build. If something outside the build deleted them there would be no + // trigger to rerun soong_build, and the build would break with dependencies on missing files. Writing them + // to their final locations would risk having them deleted when cleaning a module, and would also pollute the + // output directory with files for modules that have never been built. + // Instead, the files are written to a separate "raw" directory next to the build.ninja file, and a ninja + // rule is created to copy the files into their final location as needed. + // Obsolete files written by previous runs of soong_build must be cleaned up to avoid continually growing + // disk usage as the hashes of the files change over time. The cleanup must not remove files that were + // created by previous runs of soong_build for other products, as the build.ninja files for those products + // may still exist and still reference those files. The raw files from different products are kept + // separate by appending the Make_suffix to the directory name. + rawPath := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix), relPath) + + rawFileInfo := rawFileInfo{ + relPath: relPath, + } + + if ctx.Config().captureBuild { + // When running tests tempFile won't write to disk, instead store the contents for later retrieval by + // ContentFromFileRuleForTests. + rawFileInfo.contentForTests = tempFile.testBuf.String() + } + + rawFileSet := getRawFileSet(ctx.Config()) + if _, exists := rawFileSet.LoadOrStore(hash, rawFileInfo); exists { + // If a raw file with this hash has already been created delete the temporary file. + tempFile.remove() + } else { + // If this is the first time this hash has been seen then move it from the temporary directory + // to the raw directory. If the file already exists in the raw directory assume it has the correct + // contents. + absRawPath := absolutePath(rawPath.String()) + _, err := os.Stat(absRawPath) + if os.IsNotExist(err) { + tempFile.rename(absRawPath) + } else if err != nil { + panic(fmt.Errorf("failed to stat %q: %w", absRawPath, err)) + } else { + tempFile.remove() + } + } + + // Emit a rule to copy the file from raw directory to the final requested location in the output tree. + // Restat is used to ensure that two different products that produce identical files copied from their + // own raw directories they don't cause everything downstream to rebuild. + rule := rawFileCopy + if executable { + rule = rawFileCopyExecutable + } + ctx.Build(pctx, BuildParams{ + Rule: rule, + Input: rawPath, + Output: outputFile, + Description: "raw " + outputFile.Base(), + }) +} + +var ( + rawFileCopy = pctx.AndroidStaticRule("rawFileCopy", + blueprint.RuleParams{ + Command: "if ! cmp -s $in $out; then cp $in $out; fi", + Description: "copy raw file $out", + Restat: true, + }) + rawFileCopyExecutable = pctx.AndroidStaticRule("rawFileCopyExecutable", + blueprint.RuleParams{ + Command: "if ! cmp -s $in $out; then cp $in $out; fi && chmod +x $out", + Description: "copy raw exectuable file $out", + Restat: true, + }) +) + +type rawFileInfo struct { + relPath string + contentForTests string +} + +var rawFileSetKey OnceKey = NewOnceKey("raw file set") + +func getRawFileSet(config Config) *SyncMap[string, rawFileInfo] { + return config.Once(rawFileSetKey, func() any { + return &SyncMap[string, rawFileInfo]{} + }).(*SyncMap[string, rawFileInfo]) +} + +// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use +// in tests. +func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string { + t.Helper() + if params.Rule != rawFileCopy && params.Rule != rawFileCopyExecutable { + t.Errorf("expected params.Rule to be rawFileCopy or rawFileCopyExecutable, was %q", params.Rule) + return "" + } + + key := filepath.Base(params.Input.String()) + rawFileSet := getRawFileSet(ctx.Config()) + rawFileInfo, _ := rawFileSet.Load(key) + + return rawFileInfo.contentForTests +} + +func rawFilesSingletonFactory() Singleton { + return &rawFilesSingleton{} +} + +type rawFilesSingleton struct{} + +func (rawFilesSingleton) GenerateBuildActions(ctx SingletonContext) { + if ctx.Config().captureBuild { + // Nothing to do when running in tests, no temporary files were created. + return + } + rawFileSet := getRawFileSet(ctx.Config()) + rawFilesDir := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix)).String() + absRawFilesDir := absolutePath(rawFilesDir) + err := filepath.WalkDir(absRawFilesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Ignore obsolete directories for now. + return nil + } + + // Assume the basename of the file is a hash + key := filepath.Base(path) + relPath, err := filepath.Rel(absRawFilesDir, path) + if err != nil { + return err + } + + // Check if a file with the same hash was written by this run of soong_build. If the file was not written, + // or if a file with the same hash was written but to a different path in the raw directory, then delete it. + // Checking that the path matches allows changing the structure of the raw directory, for example to increase + // the sharding. + rawFileInfo, written := rawFileSet.Load(key) + if !written || rawFileInfo.relPath != relPath { + os.Remove(path) + } + return nil + }) + if err != nil { + panic(fmt.Errorf("failed to clean %q: %w", rawFilesDir, err)) + } +} diff --git a/android/register.go b/android/register.go index cd968cd02..d00c15fd0 100644 --- a/android/register.go +++ b/android/register.go @@ -191,8 +191,9 @@ func collateGloballyRegisteredSingletons() sortableComponents { // Register makevars after other singletons so they can export values through makevars singleton{parallel: false, name: "makevars", factory: makeVarsSingletonFunc}, - // Register env and ninjadeps last so that they can track all used environment variables and + // Register rawfiles and ninjadeps last so that they can track all used environment variables and // Ninja file dependencies stored in the config. + singleton{parallel: false, name: "rawfiles", factory: rawFilesSingletonFactory}, singleton{parallel: false, name: "ninjadeps", factory: ninjaDepsSingletonFactory}, ) diff --git a/android/rule_builder.go b/android/rule_builder.go index 1a491f708..1454357ec 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -590,7 +590,7 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b To: proto.String(sboxOutSubDir), }, { - From: proto.String(PathForOutput(r.ctx).String()), + From: proto.String(r.ctx.Config().OutDir()), To: proto.String(sboxOutSubDir), }, }, @@ -891,7 +891,7 @@ func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox boo // When sandboxing inputs all inputs have to be copied into the sandbox. Input files that // are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they // will be copied to relative paths under __SBOX_OUT_DIR__/out. - rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String()) + rel, isRelOut, _ := maybeRelErr(r.ctx.Config().OutDir(), path.String()) if isRelOut { return filepath.Join(sboxOutSubDir, rel), true } diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index 63c35272a..9e5f12dff 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -474,7 +474,7 @@ func TestRuleBuilder(t *testing.T) { wantCommands := []string{ "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile " + "FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output " + - "FlagWithRspFileInputList=__SBOX_SANDBOX_DIR__/out/rsp Input __SBOX_SANDBOX_DIR__/out/Output " + + "FlagWithRspFileInputList=__SBOX_SANDBOX_DIR__/out/soong/rsp Input __SBOX_SANDBOX_DIR__/out/Output " + "__SBOX_SANDBOX_DIR__/out/SymlinkOutput Text __SBOX_SANDBOX_DIR__/tools/src/Tool after command2 old cmd", "command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/tools/src/tool2", "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3 input3 __SBOX_SANDBOX_DIR__/out/output2", @@ -816,13 +816,13 @@ func TestRuleBuilderHashInputs(t *testing.T) { func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) { bp := ` rule_builder_test { - name: "foo_sbox_escaped_ninja", + name: "foo_sbox_escaped", flags: ["${cmdFlags}"], sbox: true, sbox_inputs: true, } rule_builder_test { - name: "foo_sbox", + name: "foo_sbox_unescaped", flags: ["${cmdFlags}"], sbox: true, sbox_inputs: true, @@ -834,15 +834,16 @@ func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) { FixtureWithRootAndroidBp(bp), ).RunTest(t) - escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped_ninja", "").Rule("writeFile") + escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped", "").Output("sbox.textproto") + AssertStringEquals(t, "expected rule", "android/soong/android.rawFileCopy", escapedNinjaMod.Rule.String()) AssertStringDoesContain( t, "", - escapedNinjaMod.BuildParams.Args["content"], - "$${cmdFlags}", + ContentFromFileRuleForTests(t, result.TestContext, escapedNinjaMod), + "${cmdFlags}", ) - unescapedNinjaMod := result.ModuleForTests("foo_sbox", "").Rule("unescapedWriteFile") + unescapedNinjaMod := result.ModuleForTests("foo_sbox_unescaped", "").Rule("unescapedWriteFile") AssertStringDoesContain( t, "", diff --git a/android/testing.go b/android/testing.go index 39a268b23..3d0300a01 100644 --- a/android/testing.go +++ b/android/testing.go @@ -203,16 +203,6 @@ func (ctx *TestContext) HardCodedPreArchMutators(f RegisterMutatorFunc) { ctx.PreArchMutators(f) } -func (ctx *TestContext) ModuleProvider(m blueprint.Module, p blueprint.AnyProviderKey) any { - value, _ := ctx.Context.ModuleProvider(m, p) - return value -} - -func (ctx *TestContext) ModuleHasProvider(m blueprint.Module, p blueprint.AnyProviderKey) bool { - _, ok := ctx.Context.ModuleProvider(m, p) - return ok -} - func (ctx *TestContext) moduleProvider(m blueprint.Module, p blueprint.AnyProviderKey) (any, bool) { return ctx.Context.ModuleProvider(m, p) } diff --git a/android/util.go b/android/util.go index ae1c65756..51313ceec 100644 --- a/android/util.go +++ b/android/util.go @@ -22,6 +22,7 @@ import ( "runtime" "sort" "strings" + "sync" ) // CopyOf returns a new slice that has the same contents as s. @@ -597,3 +598,32 @@ func AddToStringSet(set map[string]bool, items []string) { set[item] = true } } + +// SyncMap is a wrapper around sync.Map that provides type safety via generics. +type SyncMap[K comparable, V any] struct { + sync.Map +} + +// Load returns the value stored in the map for a key, or the zero value if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *SyncMap[K, V]) Load(key K) (value V, ok bool) { + v, ok := m.Map.Load(key) + if !ok { + return *new(V), false + } + return v.(V), true +} + +// Store sets the value for a key. +func (m *SyncMap[K, V]) Store(key K, value V) { + m.Map.Store(key, value) +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *SyncMap[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { + v, loaded := m.Map.LoadOrStore(key, value) + return v.(V), loaded +} diff --git a/apex/apex.go b/apex/apex.go index 56559b16b..c4545d292 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -24,7 +24,6 @@ import ( "sort" "strings" - "android/soong/aconfig" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -2272,7 +2271,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, } func addAconfigFiles(vctx *visitorContext, ctx android.ModuleContext, module blueprint.Module) { - dep, _ := android.OtherModuleProvider(ctx, module, aconfig.TransitiveDeclarationsInfoProvider) + dep, _ := android.OtherModuleProvider(ctx, module, android.AconfigTransitiveDeclarationsInfoProvider) if len(dep.AconfigFiles) > 0 && dep.AconfigFiles[ctx.ModuleName()] != nil { vctx.aconfigFiles = append(vctx.aconfigFiles, dep.AconfigFiles[ctx.ModuleName()]...) } @@ -2371,6 +2370,25 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.buildApex(ctx) a.buildApexDependencyInfo(ctx) a.buildLintReports(ctx) + + // Set a provider for dexpreopt of bootjars + a.provideApexExportsInfo(ctx) +} + +// Set a provider containing information about the jars and .prof provided by the apex +// Apexes built from source retrieve this information by visiting `bootclasspath_fragments` +// Used by dex_bootjars to generate the boot image +func (a *apexBundle) provideApexExportsInfo(ctx android.ModuleContext) { + ctx.VisitDirectDepsWithTag(bcpfTag, func(child android.Module) { + if info, ok := android.OtherModuleProvider(ctx, child, java.BootclasspathFragmentApexContentInfoProvider); ok { + exports := android.ApexExportsInfo{ + ApexName: a.ApexVariationName(), + ProfilePathOnHost: info.ProfilePathOnHost(), + LibraryNameToDexJarPathOnHost: info.DexBootJarPathMap(), + } + android.SetProvider(ctx, android.ApexExportsInfoProvider, exports) + } + }) } // apexBootclasspathFragmentFiles returns the list of apexFile structures defining the files that @@ -2867,14 +2885,6 @@ func makeApexAvailableBaseline() map[string][]string { // // Module separator // - m["com.android.tethering"] = []string{ - "android.hardware.tetheroffload.config-V1.0-java", - "android.hardware.tetheroffload.control-V1.0-java", - "net-utils-framework-common", - } - // - // Module separator - // m["com.android.wifi"] = []string{ "PlatformProperties", "android.hardware.wifi-V1.0-java", @@ -2898,15 +2908,6 @@ func makeApexAvailableBaseline() map[string][]string { "wifi-nano-protos", "wifi-service-pre-jarjar", } - // - // Module separator - // - m[android.AvailableToAnyApex] = []string{ - "libprofile-clang-extras", - "libprofile-clang-extras_ndk", - "libprofile-extras", - "libprofile-extras_ndk", - } return m } diff --git a/apex/apex_test.go b/apex/apex_test.go index abf6b1534..1b9fa19f0 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -3725,7 +3725,7 @@ func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName, var } func ensureExactDeapexedContents(t *testing.T, ctx *android.TestContext, moduleName string, variant string, files []string) { - deapexer := ctx.ModuleForTests(moduleName+".deapexer", variant).Rule("deapexer") + deapexer := ctx.ModuleForTests(moduleName+".deapexer", variant).Description("deapex") outputs := make([]string, 0, len(deapexer.ImplicitOutputs)+1) if deapexer.Output != nil { outputs = append(outputs, deapexer.Output.String()) @@ -8408,30 +8408,39 @@ func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, preparer android.F func TestDuplicateDeapexersFromPrebuiltApexes(t *testing.T) { preparers := android.GroupFixturePreparers( java.PrepareForTestWithJavaDefaultModules, + prepareForTestWithBootclasspathFragment, + dexpreopt.FixtureSetTestOnlyArtBootImageJars("com.android.art:libfoo"), PrepareForTestWithApexBuildComponents, ). ExtendWithErrorHandler(android.FixtureExpectsAtLeastOneErrorMatchingPattern( - "Multiple installable prebuilt APEXes provide ambiguous deapexers: com.android.myapex and com.mycompany.android.myapex")) + "Multiple installable prebuilt APEXes provide ambiguous deapexers: com.android.art and com.mycompany.android.art")) bpBase := ` apex_set { - name: "com.android.myapex", + name: "com.android.art", installable: true, - exported_bootclasspath_fragments: ["my-bootclasspath-fragment"], + exported_bootclasspath_fragments: ["art-bootclasspath-fragment"], set: "myapex.apks", } apex_set { - name: "com.mycompany.android.myapex", - apex_name: "com.android.myapex", + name: "com.mycompany.android.art", + apex_name: "com.android.art", installable: true, - exported_bootclasspath_fragments: ["my-bootclasspath-fragment"], + exported_bootclasspath_fragments: ["art-bootclasspath-fragment"], set: "company-myapex.apks", } prebuilt_bootclasspath_fragment { - name: "my-bootclasspath-fragment", - apex_available: ["com.android.myapex"], + name: "art-bootclasspath-fragment", + apex_available: ["com.android.art"], + hidden_api: { + annotation_flags: "my-bootclasspath-fragment/annotation-flags.csv", + metadata: "my-bootclasspath-fragment/metadata.csv", + index: "my-bootclasspath-fragment/index.csv", + stub_flags: "my-bootclasspath-fragment/stub-flags.csv", + all_flags: "my-bootclasspath-fragment/all-flags.csv", + }, %s } ` @@ -8441,7 +8450,7 @@ func TestDuplicateDeapexersFromPrebuiltApexes(t *testing.T) { java_import { name: "libfoo", jars: ["libfoo.jar"], - apex_available: ["com.android.myapex"], + apex_available: ["com.android.art"], } `) }) @@ -8453,7 +8462,8 @@ func TestDuplicateDeapexersFromPrebuiltApexes(t *testing.T) { public: { jars: ["libbar.jar"], }, - apex_available: ["com.android.myapex"], + shared_library: false, + apex_available: ["com.android.art"], } `) }) @@ -8468,7 +8478,8 @@ func TestDuplicateDeapexersFromPrebuiltApexes(t *testing.T) { public: { jars: ["libbar.jar"], }, - apex_available: ["com.android.myapex"], + shared_library: false, + apex_available: ["com.android.art"], } `) }) @@ -11404,3 +11415,181 @@ func TestAconfigFilesRemoveDuplicates(t *testing.T) { android.EnsureListContainsSuffix(t, buildParams.Inputs.Strings(), "my_aconfig_declarations_foo/intermediate.pb") ensureContains(t, buildParams.Output.String(), "android_common_myapex/aconfig_flags.pb") } + +// Test that the boot jars come from the _selected_ apex prebuilt +// RELEASE_APEX_CONTIRBUTIONS_* build flags will be used to select the correct prebuilt for a specific release config +func TestBootDexJarsMultipleApexPrebuilts(t *testing.T) { + checkBootDexJarPath := func(t *testing.T, ctx *android.TestContext, stem string, bootDexJarPath string) { + t.Helper() + s := ctx.ModuleForTests("dex_bootjars", "android_common") + foundLibfooJar := false + base := stem + ".jar" + for _, output := range s.AllOutputs() { + if filepath.Base(output) == base { + foundLibfooJar = true + buildRule := s.Output(output) + android.AssertStringEquals(t, "boot dex jar path", bootDexJarPath, buildRule.Input.String()) + } + } + if !foundLibfooJar { + t.Errorf("Rule for libfoo.jar missing in dex_bootjars singleton outputs %q", android.StringPathsRelativeToTop(ctx.Config().SoongOutDir(), s.AllOutputs())) + } + } + + // Check that the boot jars of the selected apex are run through boot_jars_package_check + // This validates that the jars on the bootclasspath do not contain packages outside an allowlist + checkBootJarsPackageCheck := func(t *testing.T, ctx *android.TestContext, expectedBootJar string) { + platformBcp := ctx.ModuleForTests("platform-bootclasspath", "android_common") + bootJarsCheckRule := platformBcp.Rule("boot_jars_package_check") + android.AssertStringMatches(t, "Could not find the correct boot dex jar in package check rule", bootJarsCheckRule.RuleParams.Command, "build/soong/scripts/check_boot_jars/package_allowed_list.txt.*"+expectedBootJar) + } + + // Check that the boot jars used to generate the monolithic hiddenapi flags come from the selected apex + checkBootJarsForMonolithicHiddenapi := func(t *testing.T, ctx *android.TestContext, expectedBootJar string) { + monolithicHiddenapiFlagsCmd := ctx.ModuleForTests("platform-bootclasspath", "android_common").Output("out/soong/hiddenapi/hiddenapi-stub-flags.txt").RuleParams.Command + android.AssertStringMatches(t, "Could not find the correct boot dex jar in monolithic hiddenapi flags generation command", monolithicHiddenapiFlagsCmd, "--boot-dex="+expectedBootJar) + } + + bp := ` + // Source APEX. + + java_library { + name: "framework-foo", + srcs: ["foo.java"], + installable: true, + apex_available: [ + "com.android.foo", + ], + } + + bootclasspath_fragment { + name: "foo-bootclasspath-fragment", + contents: ["framework-foo"], + apex_available: [ + "com.android.foo", + ], + hidden_api: { + split_packages: ["*"], + }, + } + + apex_key { + name: "com.android.foo.key", + public_key: "com.android.foo.avbpubkey", + private_key: "com.android.foo.pem", + } + + apex { + name: "com.android.foo", + key: "com.android.foo.key", + bootclasspath_fragments: ["foo-bootclasspath-fragment"], + updatable: false, + } + + // Prebuilt APEX. + + java_sdk_library_import { + name: "framework-foo", + public: { + jars: ["foo.jar"], + }, + apex_available: ["com.android.foo"], + shared_library: false, + } + + prebuilt_bootclasspath_fragment { + name: "foo-bootclasspath-fragment", + contents: ["framework-foo"], + hidden_api: { + annotation_flags: "my-bootclasspath-fragment/annotation-flags.csv", + metadata: "my-bootclasspath-fragment/metadata.csv", + index: "my-bootclasspath-fragment/index.csv", + stub_flags: "my-bootclasspath-fragment/stub-flags.csv", + all_flags: "my-bootclasspath-fragment/all-flags.csv", + }, + apex_available: [ + "com.android.foo", + ], + } + + prebuilt_apex { + name: "com.android.foo", + apex_name: "com.android.foo", + src: "com.android.foo-arm.apex", + exported_bootclasspath_fragments: ["foo-bootclasspath-fragment"], + } + + // Another Prebuilt ART APEX + prebuilt_apex { + name: "com.android.foo.v2", + apex_name: "com.android.foo", // Used to determine the API domain + src: "com.android.foo-arm.apex", + exported_bootclasspath_fragments: ["foo-bootclasspath-fragment"], + } + + // APEX contribution modules + + apex_contributions { + name: "foo.source.contributions", + api_domain: "com.android.foo", + contents: ["com.android.foo"], + } + + apex_contributions { + name: "foo.prebuilt.contributions", + api_domain: "com.android.foo", + contents: ["prebuilt_com.android.foo"], + } + + apex_contributions { + name: "foo.prebuilt.v2.contributions", + api_domain: "com.android.foo", + contents: ["com.android.foo.v2"], // prebuilt_ prefix is missing because of prebuilt_rename mutator + } + ` + + testCases := []struct { + desc string + selectedApexContributions string + expectedBootJar string + }{ + { + desc: "Source apex com.android.foo is selected, bootjar should come from source java library", + selectedApexContributions: "foo.source.contributions", + expectedBootJar: "out/soong/.intermediates/foo-bootclasspath-fragment/android_common_apex10000/hiddenapi-modular/encoded/framework-foo.jar", + }, + { + desc: "Prebuilt apex prebuilt_com.android.foo is selected, profile should come from .prof deapexed from the prebuilt", + selectedApexContributions: "foo.prebuilt.contributions", + expectedBootJar: "out/soong/.intermediates/com.android.foo.deapexer/android_common/deapexer/javalib/framework-foo.jar", + }, + { + desc: "Prebuilt apex prebuilt_com.android.foo.v2 is selected, profile should come from .prof deapexed from the prebuilt", + selectedApexContributions: "foo.prebuilt.v2.contributions", + expectedBootJar: "out/soong/.intermediates/com.android.foo.v2.deapexer/android_common/deapexer/javalib/framework-foo.jar", + }, + } + + fragment := java.ApexVariantReference{ + Apex: proptools.StringPtr("com.android.foo"), + Module: proptools.StringPtr("foo-bootclasspath-fragment"), + } + + for _, tc := range testCases { + preparer := android.GroupFixturePreparers( + java.FixtureConfigureApexBootJars("com.android.foo:framework-foo"), + android.FixtureMergeMockFs(map[string][]byte{ + "system/sepolicy/apex/com.android.foo-file_contexts": nil, + }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": tc.selectedApexContributions, + } + }), + ) + ctx := testDexpreoptWithApexes(t, bp, "", preparer, fragment) + checkBootDexJarPath(t, ctx, "framework-foo", tc.expectedBootJar) + checkBootJarsPackageCheck(t, ctx, tc.expectedBootJar) + checkBootJarsForMonolithicHiddenapi(t, ctx, tc.expectedBootJar) + } +} diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 42f5cd444..159e9e1c4 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -530,6 +530,8 @@ func TestBootclasspathFragmentInPrebuiltArtApex(t *testing.T) { java.CheckModuleDependencies(t, result.TestContext, "com.android.art", "android_common_com.android.art", []string{ `com.android.art.apex.selector`, + `com.android.art.deapexer`, + `dex2oatd`, `prebuilt_art-bootclasspath-fragment`, }) diff --git a/apex/deapexer.go b/apex/deapexer.go index 5aeea63d1..5ff622c5b 100644 --- a/apex/deapexer.go +++ b/apex/deapexer.go @@ -98,6 +98,7 @@ func privateDeapexerFactory() android.Module { func (p *Deapexer) DepsMutator(ctx android.BottomUpMutatorContext) { // Add dependencies from the java modules to which this exports files from the `.apex` file onto // this module so that they can access the `DeapexerInfo` object that this provides. + // TODO: b/308174306 - Once all the mainline modules have been flagged, drop this dependency edge for _, lib := range p.properties.CommonModules { dep := prebuiltApexExportedModuleName(ctx, lib) ctx.AddReverseDependency(ctx.Module(), android.DeapexerTag, dep) @@ -126,7 +127,7 @@ func (p *Deapexer) GenerateAndroidBuildActions(ctx android.ModuleContext) { // apex relative path to extracted file path available for other modules. if len(exports) > 0 { // Make the information available for other modules. - di := android.NewDeapexerInfo(apexModuleName(ctx.ModuleName()), exports) + di := android.NewDeapexerInfo(apexModuleName(ctx.ModuleName()), exports, p.properties.CommonModules) android.SetProvider(ctx, android.DeapexerProvider, di) // Create a sorted list of the files that this exports. diff --git a/apex/dexpreopt_bootjars_test.go b/apex/dexpreopt_bootjars_test.go index 2e828cade..34ccdd713 100644 --- a/apex/dexpreopt_bootjars_test.go +++ b/apex/dexpreopt_bootjars_test.go @@ -252,3 +252,162 @@ func TestDexpreoptBootZip(t *testing.T) { testDexpreoptBoot(t, ruleFile, expectedInputs, expectedOutputs, false) } + +// Multiple ART apexes might exist in the tree. +// The profile should correspond to the apex selected using release build flags +func TestDexpreoptProfileWithMultiplePrebuiltArtApexes(t *testing.T) { + ruleFile := "out/soong/dexpreopt_arm64/dex_bootjars/android/system/framework/arm64/boot.art" + bp := ` + // Platform. + + platform_bootclasspath { + name: "platform-bootclasspath", + fragments: [ + { + apex: "com.android.art", + module: "art-bootclasspath-fragment", + }, + ], + } + + // Source ART APEX. + + java_library { + name: "core-oj", + srcs: ["core-oj.java"], + installable: true, + apex_available: [ + "com.android.art", + ], + } + + bootclasspath_fragment { + name: "art-bootclasspath-fragment", + image_name: "art", + contents: ["core-oj"], + apex_available: [ + "com.android.art", + ], + hidden_api: { + split_packages: ["*"], + }, + } + + apex_key { + name: "com.android.art.key", + public_key: "com.android.art.avbpubkey", + private_key: "com.android.art.pem", + } + + apex { + name: "com.android.art", + key: "com.android.art.key", + bootclasspath_fragments: ["art-bootclasspath-fragment"], + updatable: false, + } + + // Prebuilt ART APEX. + + java_import { + name: "core-oj", + jars: ["core-oj.jar"], + apex_available: [ + "com.android.art", + ], + } + + prebuilt_bootclasspath_fragment { + name: "art-bootclasspath-fragment", + image_name: "art", + contents: ["core-oj"], + hidden_api: { + annotation_flags: "my-bootclasspath-fragment/annotation-flags.csv", + metadata: "my-bootclasspath-fragment/metadata.csv", + index: "my-bootclasspath-fragment/index.csv", + stub_flags: "my-bootclasspath-fragment/stub-flags.csv", + all_flags: "my-bootclasspath-fragment/all-flags.csv", + }, + apex_available: [ + "com.android.art", + ], + } + + prebuilt_apex { + name: "com.android.art", + apex_name: "com.android.art", + src: "com.android.art-arm.apex", + exported_bootclasspath_fragments: ["art-bootclasspath-fragment"], + } + + // Another Prebuilt ART APEX + prebuilt_apex { + name: "com.android.art.v2", + apex_name: "com.android.art", // Used to determine the API domain + src: "com.android.art-arm.apex", + exported_bootclasspath_fragments: ["art-bootclasspath-fragment"], + } + + // APEX contribution modules + + apex_contributions { + name: "art.source.contributions", + api_domain: "com.android.art", + contents: ["com.android.art"], + } + + apex_contributions { + name: "art.prebuilt.contributions", + api_domain: "com.android.art", + contents: ["prebuilt_com.android.art"], + } + + apex_contributions { + name: "art.prebuilt.v2.contributions", + api_domain: "com.android.art", + contents: ["com.android.art.v2"], // prebuilt_ prefix is missing because of prebuilt_rename mutator + } + + ` + + testCases := []struct { + desc string + selectedArtApexContributions string + expectedProfile string + }{ + { + desc: "Source apex com.android.art is selected, profile should come from source java library", + selectedArtApexContributions: "art.source.contributions", + expectedProfile: "out/soong/.intermediates/art-bootclasspath-fragment/android_common_apex10000/art-bootclasspath-fragment/boot.prof", + }, + { + desc: "Prebuilt apex prebuilt_com.android.art is selected, profile should come from .prof deapexed from the prebuilt", + selectedArtApexContributions: "art.prebuilt.contributions", + expectedProfile: "out/soong/.intermediates/com.android.art.deapexer/android_common/deapexer/etc/boot-image.prof", + }, + { + desc: "Prebuilt apex prebuilt_com.android.art.v2 is selected, profile should come from .prof deapexed from the prebuilt", + selectedArtApexContributions: "art.prebuilt.v2.contributions", + expectedProfile: "out/soong/.intermediates/com.android.art.v2.deapexer/android_common/deapexer/etc/boot-image.prof", + }, + } + for _, tc := range testCases { + result := android.GroupFixturePreparers( + java.PrepareForTestWithDexpreopt, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureConfigureBootJars("com.android.art:core-oj"), + PrepareForTestWithApexBuildComponents, + prepareForTestWithArtApex, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ART": tc.selectedArtApexContributions, + } + }), + ).RunTestWithBp(t, bp) + + dexBootJars := result.ModuleForTests("dex_bootjars", "android_common") + rule := dexBootJars.Output(ruleFile) + + inputs := rule.Implicits.Strings() + android.AssertStringListContains(t, tc.desc, inputs, tc.expectedProfile) + } +} diff --git a/apex/platform_bootclasspath_test.go b/apex/platform_bootclasspath_test.go index b741963c8..01b616bb6 100644 --- a/apex/platform_bootclasspath_test.go +++ b/apex/platform_bootclasspath_test.go @@ -382,6 +382,9 @@ func TestPlatformBootclasspathDependencies(t *testing.T) { // Make sure that the myplatform-bootclasspath has the correct dependencies. CheckModuleDependencies(t, result.TestContext, "myplatform-bootclasspath", "android_common", []string{ + // source vs prebuilt selection metadata module + `platform:all_apex_contributions`, + // The following are stubs. `platform:android_stubs_current`, `platform:android_system_stubs_current`, @@ -534,6 +537,9 @@ func TestPlatformBootclasspath_AlwaysUsePrebuiltSdks(t *testing.T) { // Make sure that the myplatform-bootclasspath has the correct dependencies. CheckModuleDependencies(t, result.TestContext, "myplatform-bootclasspath", "android_common", []string{ + // source vs prebuilt selection metadata module + `platform:all_apex_contributions`, + // The following are stubs. "platform:prebuilt_sdk_public_current_android", "platform:prebuilt_sdk_system_current_android", diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 179d90b17..188875ac9 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -22,6 +22,7 @@ import ( "strings" "android/soong/android" + "android/soong/dexpreopt" "android/soong/java" "android/soong/provenance" @@ -50,6 +51,7 @@ type prebuilt interface { type prebuiltCommon struct { android.ModuleBase + java.Dexpreopter prebuilt android.Prebuilt // Properties common to both prebuilt_apex and apex_set. @@ -170,50 +172,42 @@ func (p *prebuiltCommon) installable() bool { return proptools.BoolDefault(p.prebuiltCommonProperties.Installable, true) } -// initApexFilesForAndroidMk initializes the prebuiltCommon.apexFilesForAndroidMk field from the -// modules that this depends upon. +// To satisfy java.DexpreopterInterface +func (p *prebuiltCommon) IsInstallable() bool { + return p.installable() +} + +// initApexFilesForAndroidMk initializes the prebuiltCommon.requiredModuleNames field with the install only deps of the prebuilt apex func (p *prebuiltCommon) initApexFilesForAndroidMk(ctx android.ModuleContext) { - // Walk the dependencies of this module looking for the java modules that it exports. - ctx.WalkDeps(func(child, parent android.Module) bool { - tag := ctx.OtherModuleDependencyTag(child) + // If this apex contains a system server jar, then the dexpreopt artifacts should be added as required + for _, install := range p.Dexpreopter.DexpreoptBuiltInstalledForApex() { + p.requiredModuleNames = append(p.requiredModuleNames, install.FullModuleName()) + } +} - name := android.RemoveOptionalPrebuiltPrefix(ctx.OtherModuleName(child)) - if java.IsBootclasspathFragmentContentDepTag(tag) || - java.IsSystemServerClasspathFragmentContentDepTag(tag) || tag == exportedJavaLibTag { - // If the exported java module provides a dex jar path then add it to the list of apexFiles. - path := child.(interface { - DexJarBuildPath() java.OptionalDexJarPath - }).DexJarBuildPath() - if path.IsSet() { - af := apexFile{ - module: child, - moduleDir: ctx.OtherModuleDir(child), - androidMkModuleName: name, - builtFile: path.Path(), - class: javaSharedLib, - } - if module, ok := child.(java.DexpreopterInterface); ok { - for _, install := range module.DexpreoptBuiltInstalledForApex() { - af.requiredModuleNames = append(af.requiredModuleNames, install.FullModuleName()) - } - } - p.apexFilesForAndroidMk = append(p.apexFilesForAndroidMk, af) - } - } else if tag == exportedBootclasspathFragmentTag { - _, ok := child.(*java.PrebuiltBootclasspathFragmentModule) - if !ok { - ctx.PropertyErrorf("exported_bootclasspath_fragments", "%q is not a prebuilt_bootclasspath_fragment module", name) - return false - } - // Visit the children of the bootclasspath_fragment. - return true - } else if tag == exportedSystemserverclasspathFragmentTag { - // Visit the children of the systemserver_fragment. - return true +// If this prebuilt has system server jar, create the rules to dexpreopt it and install it alongside the prebuilt apex +func (p *prebuiltCommon) dexpreoptSystemServerJars(ctx android.ModuleContext) { + // If this apex does not export anything, return + if !p.hasExportedDeps() { + return + } + // Use apex_name to determine the api domain of this prebuilt apex + apexName := p.ApexVariationName() + di, err := android.FindDeapexerProviderForModule(ctx) + if err != nil { + ctx.ModuleErrorf(err.Error()) + } + dc := dexpreopt.GetGlobalConfig(ctx) + systemServerJarList := dc.AllApexSystemServerJars(ctx) + + for i := 0; i < systemServerJarList.Len(); i++ { + sscpApex := systemServerJarList.Apex(i) + sscpJar := systemServerJarList.Jar(i) + if apexName != sscpApex { + continue } - - return false - }) + p.Dexpreopter.DexpreoptPrebuiltApexSystemServerJars(ctx, sscpJar, di) + } } func (p *prebuiltCommon) addRequiredModules(entries *android.AndroidMkEntries) { @@ -248,6 +242,11 @@ func (p *prebuiltCommon) AndroidMkEntries() []android.AndroidMkEntries { }, } + // Add the dexpreopt artifacts to androidmk + for _, install := range p.Dexpreopter.DexpreoptBuiltInstalledForApex() { + entriesList = append(entriesList, install.ToMakeEntries()) + } + // Iterate over the apexFilesForAndroidMk list and create an AndroidMkEntries struct for each // file. This provides similar behavior to that provided in apexBundle.AndroidMk() as it makes the // apex specific variants of the exported java modules available for use from within make. @@ -756,12 +755,47 @@ func (p *Prebuilt) ComponentDepsMutator(ctx android.BottomUpMutatorContext) { p.prebuiltApexContentsDeps(ctx) } +func (p *prebuiltCommon) DepsMutator(ctx android.BottomUpMutatorContext) { + if p.hasExportedDeps() { + // Create a dependency from the prebuilt apex (prebuilt_apex/apex_set) to the internal deapexer module + // The deapexer will return a provider that will be bubbled up to the rdeps of apexes (e.g. dex_bootjars) + ctx.AddDependency(ctx.Module(), android.DeapexerTag, deapexerModuleName(p.BaseModuleName())) + } +} + var _ ApexInfoMutator = (*Prebuilt)(nil) func (p *Prebuilt) ApexInfoMutator(mctx android.TopDownMutatorContext) { p.apexInfoMutator(mctx) } +// Set a provider containing information about the jars and .prof provided by the apex +// Apexes built from prebuilts retrieve this information by visiting its internal deapexer module +// Used by dex_bootjars to generate the boot image +func (p *prebuiltCommon) provideApexExportsInfo(ctx android.ModuleContext) { + if !p.hasExportedDeps() { + // nothing to do + return + } + if di, err := android.FindDeapexerProviderForModule(ctx); err == nil { + javaModuleToDexPath := map[string]android.Path{} + for _, commonModule := range di.GetExportedModuleNames() { + if dex := di.PrebuiltExportPath(java.ApexRootRelativePathToJavaLib(commonModule)); dex != nil { + javaModuleToDexPath[commonModule] = dex + } + } + + exports := android.ApexExportsInfo{ + ApexName: p.ApexVariationName(), + ProfilePathOnHost: di.PrebuiltExportPath(java.ProfileInstallPathInApex), + LibraryNameToDexJarPathOnHost: javaModuleToDexPath, + } + android.SetProvider(ctx, android.ApexExportsInfoProvider, exports) + } else { + ctx.ModuleErrorf(err.Error()) + } +} + func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.apexKeysPath = writeApexKeys(ctx, p) // TODO(jungjw): Check the key validity. @@ -783,6 +817,12 @@ func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { return } + // dexpreopt any system server jars if present + p.dexpreoptSystemServerJars(ctx) + + // provide info used for generating the boot image + p.provideApexExportsInfo(ctx) + // Save the files that need to be made available to Make. p.initApexFilesForAndroidMk(ctx) @@ -999,6 +1039,12 @@ func (a *ApexSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { return } + // dexpreopt any system server jars if present + a.dexpreoptSystemServerJars(ctx) + + // provide info used for generating the boot image + a.provideApexExportsInfo(ctx) + // Save the files that need to be made available to Make. a.initApexFilesForAndroidMk(ctx) diff --git a/apex/systemserver_classpath_fragment_test.go b/apex/systemserver_classpath_fragment_test.go index 40d05814e..90fd2caa9 100644 --- a/apex/systemserver_classpath_fragment_test.go +++ b/apex/systemserver_classpath_fragment_test.go @@ -272,7 +272,9 @@ func TestPrebuiltSystemserverclasspathFragmentContents(t *testing.T) { ctx := result.TestContext java.CheckModuleDependencies(t, ctx, "myapex", "android_common_myapex", []string{ + `dex2oatd`, `myapex.apex.selector`, + `myapex.deapexer`, `prebuilt_mysystemserverclasspathfragment`, }) @@ -28,7 +28,6 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/proptools" - "android/soong/aconfig" "android/soong/aidl_library" "android/soong/android" "android/soong/cc/config" @@ -2137,7 +2136,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { } android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: deps.GeneratedSources.Strings()}) - aconfig.CollectDependencyAconfigFiles(ctx, &c.mergedAconfigFiles) + android.CollectDependencyAconfigFiles(ctx, &c.mergedAconfigFiles) c.maybeInstall(ctx, apexInfo) } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 7742492bf..472756698 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -28,6 +28,7 @@ import ( "android/soong/android/allowlists" "android/soong/bp2build" "android/soong/shared" + "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/deptools" @@ -196,7 +197,7 @@ func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArg ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile) } -func writeBuildGlobsNinjaFile(ctx *android.Context) []string { +func writeBuildGlobsNinjaFile(ctx *android.Context) { ctx.EventHandler.Begin("globs_ninja_file") defer ctx.EventHandler.End("globs_ninja_file") @@ -208,7 +209,6 @@ func writeBuildGlobsNinjaFile(ctx *android.Context) []string { SrcDir: ctx.SrcDir(), }, ctx.Config()) maybeQuit(err, "") - return bootstrap.GlobFileListFiles(globDir) } func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) { @@ -238,8 +238,7 @@ func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string { maybeQuit(err, "") ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - globListFiles := writeBuildGlobsNinjaFile(ctx) - ninjaDeps = append(ninjaDeps, globListFiles...) + writeBuildGlobsNinjaFile(ctx) // Convert the Soong module graph into Bazel BUILD files. switch ctx.Config().BuildMode { diff --git a/docs/map_files.md b/docs/map_files.md index 35e8cbbfc..37f91ecb5 100644 --- a/docs/map_files.md +++ b/docs/map_files.md @@ -134,6 +134,9 @@ both APEX and the LL-NDK. Historically this annotation was spelled `vndk`, but it has always meant LL-NDK. +When an llndk API is deprecated, the `llndk` tag is dropped and +`llndk-deprecate=<V>` is added. + ### platform-only Indicates that the version or symbol is public in the implementation library but diff --git a/genrule/allowlists.go b/genrule/allowlists.go index dc2d9e687..cbefa4585 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -23,9 +23,7 @@ var ( SandboxingDenyModuleList = []string{ // go/keep-sorted start - "CtsApkVerityTestDebugFiles", "aidl_camera_build_version", - "chre_atoms_log.h", // go/keep-sorted end } diff --git a/java/androidmk.go b/java/androidmk.go index 809f9b563..cbf9abb96 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -207,11 +207,7 @@ func (j *TestHelperLibrary) AndroidMkEntries() []android.AndroidMkEntries { func (prebuilt *Import) AndroidMkEntries() []android.AndroidMkEntries { if prebuilt.hideApexVariantFromMake { - // For a library imported from a prebuilt APEX, we don't need a Make module for itself, as we - // don't need to install it. However, we need to add its dexpreopt outputs as sub-modules, if it - // is preopted. - dexpreoptEntries := prebuilt.dexpreopter.AndroidMkEntriesForApex() - return append(dexpreoptEntries, android.AndroidMkEntries{Disabled: true}) + return []android.AndroidMkEntries{} } return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", diff --git a/java/app.go b/java/app.go index 7f0303a6b..8b28dac78 100755 --- a/java/app.go +++ b/java/app.go @@ -22,7 +22,6 @@ import ( "path/filepath" "strings" - "android/soong/aconfig" "android/soong/testing" "github.com/google/blueprint" @@ -509,7 +508,7 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { var aconfigTextFilePaths android.Paths ctx.VisitDirectDepsWithTag(aconfigDeclarationTag, func(dep android.Module) { - if provider, ok := android.OtherModuleProvider(ctx, dep, aconfig.DeclarationsProviderKey); ok { + if provider, ok := android.OtherModuleProvider(ctx, dep, android.AconfigDeclarationsProviderKey); ok { aconfigTextFilePaths = append(aconfigTextFilePaths, provider.IntermediateDumpOutputPath) } else { ctx.ModuleErrorf("Only aconfig_declarations module type is allowed for "+ diff --git a/java/app_import_test.go b/java/app_import_test.go index 8f29bb373..ef4626e26 100644 --- a/java/app_import_test.go +++ b/java/app_import_test.go @@ -40,8 +40,8 @@ func TestAndroidAppImport(t *testing.T) { variant := ctx.ModuleForTests("foo", "android_common") // Check dexpreopt outputs. - if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || - variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + if variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.odex").Rule == nil { t.Errorf("can't find dexpreopt outputs") } @@ -74,8 +74,8 @@ func TestAndroidAppImport_NoDexPreopt(t *testing.T) { variant := ctx.ModuleForTests("foo", "android_common") // Check dexpreopt outputs. They shouldn't exist. - if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule != nil || - variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule != nil { + if variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.vdex").Rule != nil || + variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.odex").Rule != nil { t.Errorf("dexpreopt shouldn't have run.") } @@ -101,8 +101,8 @@ func TestAndroidAppImport_Presigned(t *testing.T) { variant := ctx.ModuleForTests("foo", "android_common") // Check dexpreopt outputs. - if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || - variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + if variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.odex").Rule == nil { t.Errorf("can't find dexpreopt outputs") } // Make sure signing was skipped and aligning was done. @@ -210,8 +210,8 @@ func TestAndroidAppImport_DefaultDevCert(t *testing.T) { variant := ctx.ModuleForTests("foo", "android_common") // Check dexpreopt outputs. - if variant.MaybeOutput("dexpreopt/oat/arm64/package.vdex").Rule == nil || - variant.MaybeOutput("dexpreopt/oat/arm64/package.odex").Rule == nil { + if variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.vdex").Rule == nil || + variant.MaybeOutput("dexpreopt/foo/oat/arm64/package.odex").Rule == nil { t.Errorf("can't find dexpreopt outputs") } diff --git a/java/app_test.go b/java/app_test.go index 0936b281a..861c04761 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3357,7 +3357,7 @@ func TestUsesLibraries(t *testing.T) { cmd := app.Rule("dexpreopt").RuleParams.Command android.AssertStringDoesContain(t, "dexpreopt app cmd context", cmd, "--context-json=") android.AssertStringDoesContain(t, "dexpreopt app cmd product_packages", cmd, - "--product-packages=out/soong/.intermediates/app/android_common/dexpreopt/product_packages.txt") + "--product-packages=out/soong/.intermediates/app/android_common/dexpreopt/app/product_packages.txt") } func TestDexpreoptBcp(t *testing.T) { diff --git a/java/base.go b/java/base.go index 41f2fcc96..0d3e4dbe3 100644 --- a/java/base.go +++ b/java/base.go @@ -24,7 +24,6 @@ import ( "github.com/google/blueprint/pathtools" "github.com/google/blueprint/proptools" - "android/soong/aconfig" "android/soong/android" "android/soong/dexpreopt" "android/soong/java/config" @@ -1694,7 +1693,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath ctx.CheckbuildFile(outputFile) - aconfig.CollectDependencyAconfigFiles(ctx, &j.mergedAconfigFiles) + android.CollectDependencyAconfigFiles(ctx, &j.mergedAconfigFiles) android.SetProvider(ctx, JavaInfoProvider, JavaInfo{ HeaderJars: android.PathsIfNonNil(j.headerJarFile), diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index d2bb52315..ae2440466 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -240,7 +240,8 @@ type BootclasspathFragmentModule struct { sourceOnlyProperties SourceOnlyBootclasspathProperties // Path to the boot image profile. - profilePath android.WritablePath + profilePath android.WritablePath + profilePathErr error } // commonBootclasspathFragment defines the methods that are implemented by both source and prebuilt @@ -384,6 +385,10 @@ func (i BootclasspathFragmentApexContentInfo) DexBootJarPathForContentModule(mod } } +func (i BootclasspathFragmentApexContentInfo) DexBootJarPathMap() bootDexJarByModule { + return i.contentModuleDexJarPaths +} + func (i BootclasspathFragmentApexContentInfo) ProfilePathOnHost() android.Path { return i.profilePathOnHost } @@ -533,7 +538,7 @@ func (b *BootclasspathFragmentModule) provideApexContentInfo(ctx android.ModuleC if profile != nil { info.profilePathOnHost = profile - info.profileInstallPathInApex = profileInstallPathInApex + info.profileInstallPathInApex = ProfileInstallPathInApex } // Make the apex content info available for other modules. @@ -1033,10 +1038,6 @@ func (module *PrebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx an return android.PathForModuleSrc(ctx, *src) } - // Retrieve the dex files directly from the content modules. They in turn should retrieve the - // encoded dex jars from the prebuilt .apex files. - encodedBootDexJarsByModule := extractEncodedDexJarsFromModules(ctx, contents) - output := HiddenAPIOutput{ HiddenAPIFlagOutput: HiddenAPIFlagOutput{ AnnotationFlagsPath: pathForSrc("hidden_api.annotation_flags", module.prebuiltProperties.Hidden_api.Annotation_flags), @@ -1047,8 +1048,6 @@ func (module *PrebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx an StubFlagsPath: pathForOptionalSrc(module.prebuiltProperties.Hidden_api.Stub_flags, nil), AllFlagsPath: pathForOptionalSrc(module.prebuiltProperties.Hidden_api.All_flags, nil), }, - - EncodedBootDexFilesByModule: encodedBootDexJarsByModule, } // TODO: Temporarily fallback to stub_flags/all_flags properties until prebuilts have been updated. @@ -1065,15 +1064,21 @@ func (module *PrebuiltBootclasspathFragmentModule) produceBootImageProfile(ctx a return nil } - di := android.FindDeapexerProviderForModule(ctx) - if di == nil { + di, err := android.FindDeapexerProviderForModule(ctx) + if err != nil { + // An error was found, possibly due to multiple apexes in the tree that export this library + // Defer the error till a client tries to call getProfilePath + module.profilePathErr = err return nil // An error has been reported by FindDeapexerProviderForModule. } - return di.PrebuiltExportPath(profileInstallPathInApex) + return di.PrebuiltExportPath(ProfileInstallPathInApex) } func (b *PrebuiltBootclasspathFragmentModule) getProfilePath() android.Path { + if b.profilePathErr != nil { + panic(b.profilePathErr.Error()) + } return b.profilePath } @@ -1087,7 +1092,7 @@ var _ commonBootclasspathFragment = (*PrebuiltBootclasspathFragmentModule)(nil) func (module *PrebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex(ctx android.BaseModuleContext) []string { for _, apex := range module.ApexProperties.Apex_available { if isProfileProviderApex(ctx, apex) { - return []string{profileInstallPathInApex} + return []string{ProfileInstallPathInApex} } } return nil diff --git a/java/code_metadata_test.go b/java/code_metadata_test.go index 509e70112..0ef348afe 100644 --- a/java/code_metadata_test.go +++ b/java/code_metadata_test.go @@ -25,12 +25,10 @@ func TestCodeMetadata(t *testing.T) { }` result := runCodeMetadataTest(t, android.FixtureExpectsNoErrors, bp) - module := result.ModuleForTests( - "module-name", "", - ).Module().(*soongTesting.CodeMetadataModule) + module := result.ModuleForTests("module-name", "") // Check that the provider has the right contents - data, _ := android.SingletonModuleProvider(result, module, soongTesting.CodeMetadataProviderKey) + data, _ := android.SingletonModuleProvider(result, module.Module(), soongTesting.CodeMetadataProviderKey) if !strings.HasSuffix( data.IntermediatePath.String(), "/intermediateCodeMetadata.pb", ) { @@ -40,13 +38,8 @@ func TestCodeMetadata(t *testing.T) { ) } - buildParamsSlice := module.BuildParamsForTests() - var metadata = "" - for _, params := range buildParamsSlice { - if params.Rule.String() == "android/soong/android.writeFile" { - metadata = params.Args["content"] - } - } + metadata := android.ContentFromFileRuleForTests(t, result.TestContext, + module.Output(data.IntermediatePath.String())) metadataList := make([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership, 0, 2) teamId := "12345" @@ -63,9 +56,7 @@ func TestCodeMetadata(t *testing.T) { CodeMetadataMetadata := code_metadata_internal_proto.CodeMetadataInternal{TargetOwnershipList: metadataList} protoData, _ := proto.Marshal(&CodeMetadataMetadata) - rawData := string(protoData) - formattedData := strings.ReplaceAll(rawData, "\n", "\\n") - expectedMetadata := "'" + formattedData + "\\n'" + expectedMetadata := string(protoData) if metadata != expectedMetadata { t.Errorf( diff --git a/java/config/config.go b/java/config/config.go index d80ed4142..6a945ac9c 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -95,13 +95,11 @@ func init() { "-JXX:TieredStopAtLevel=1", "-JDcom.android.tools.r8.emitRecordAnnotationsInDex", "-JDcom.android.tools.r8.emitPermittedSubclassesAnnotationsInDex", - "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ "-JXmx4096M", "-JDcom.android.tools.r8.emitRecordAnnotationsInDex", "-JDcom.android.tools.r8.emitPermittedSubclassesAnnotationsInDex", - "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{ diff --git a/java/dex.go b/java/dex.go index cdae0a2df..fbb841860 100644 --- a/java/dex.go +++ b/java/dex.go @@ -223,6 +223,13 @@ func (d *dexer) dexCommonFlags(ctx android.ModuleContext, if err != nil { ctx.PropertyErrorf("min_sdk_version", "%s", err) } + if effectiveVersion.FinalOrFutureInt() >= 35 { + // V is 35, but we have not bumped the SDK version yet, so check for both. + if ctx.Config().PlatformSdkVersion().FinalInt() >= 35 || + ctx.Config().PlatformSdkCodename() == "VanillaIceCream" { + flags = append([]string{"-JDcom.android.tools.r8.dexContainerExperiment"}, flags...) + } + } // If the specified SDK level is 10000, then configure the compiler to use the // current platform SDK level and to compile the build as a platform build. diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 0f69dc3a4..bd3cce412 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -79,18 +79,25 @@ func (install *dexpreopterInstall) SubModuleName() string { func (install dexpreopterInstall) ToMakeEntries() android.AndroidMkEntries { return android.AndroidMkEntries{ Class: "ETC", - SubName: install.SubModuleName(), OutputFile: android.OptionalPathForPath(install.outputPathOnHost), ExtraEntries: []android.AndroidMkExtraEntriesFunc{ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE", install.FullModuleName()) entries.SetString("LOCAL_MODULE_PATH", install.installDirOnDevice.String()) entries.SetString("LOCAL_INSTALLED_MODULE_STEM", install.installFileOnDevice) entries.SetString("LOCAL_NOT_AVAILABLE_FOR_PLATFORM", "false") + // Unset LOCAL_SOONG_INSTALLED_MODULE so that this does not default to the primary .apex file + // Without this, installation of the dexpreopt artifacts get skipped + entries.SetString("LOCAL_SOONG_INSTALLED_MODULE", "") }, }, } } +type Dexpreopter struct { + dexpreopter +} + type dexpreopter struct { dexpreoptProperties DexpreoptProperties importDexpreoptProperties ImportDexpreoptProperties @@ -258,6 +265,17 @@ func (d *dexpreopter) getInstallPath( return defaultInstallPath } +// DexpreoptPrebuiltApexSystemServerJars generates the dexpreopt artifacts from a jar file that has been deapexed from a prebuilt apex +func (d *Dexpreopter) DexpreoptPrebuiltApexSystemServerJars(ctx android.ModuleContext, libraryName string, di *android.DeapexerInfo) { + // A single prebuilt apex can have multiple apex system jars + // initialize the output path for this dex jar + dc := dexpreopt.GetGlobalConfig(ctx) + d.installPath = android.PathForModuleInPartitionInstall(ctx, "", strings.TrimPrefix(dexpreopt.GetSystemServerDexLocation(ctx, dc, libraryName), "/")) + // generate the rules for creating the .odex and .vdex files for this system server jar + dexJarFile := di.PrebuiltExportPath(ApexRootRelativePathToJavaLib(libraryName)) + d.dexpreopt(ctx, dexJarFile) +} + func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.WritablePath) { global := dexpreopt.GetGlobalConfig(ctx) @@ -346,11 +364,15 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr d.dexpreoptProperties.Dex_preopt_result.Profile_guided = profileClassListing.Valid() + // A single apex can have multiple system server jars + // Use the dexJar to create a unique scope for each + dexJarStem := strings.TrimSuffix(dexJarFile.Base(), dexJarFile.Ext()) + // Full dexpreopt config, used to create dexpreopt build rules. dexpreoptConfig := &dexpreopt.ModuleConfig{ Name: moduleName(ctx), DexLocation: dexLocation, - BuildPath: android.PathForModuleOut(ctx, "dexpreopt", moduleName(ctx)+".jar").OutputPath, + BuildPath: android.PathForModuleOut(ctx, "dexpreopt", dexJarStem, moduleName(ctx)+".jar").OutputPath, DexPath: dexJarFile, ManifestPath: android.OptionalPathForPath(d.manifestFile), UncompressedDex: d.uncompressedDex, @@ -380,7 +402,7 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr PresignedPrebuilt: d.isPresignedPrebuilt, } - d.configPath = android.PathForModuleOut(ctx, "dexpreopt", "dexpreopt.config") + d.configPath = android.PathForModuleOut(ctx, "dexpreopt", dexJarStem, "dexpreopt.config") dexpreopt.WriteModuleConfig(ctx, dexpreoptConfig, d.configPath) if d.dexpreoptDisabled(ctx) { @@ -394,7 +416,7 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr // dependencies to create a per-app list, and use `rsync --checksum` to prevent the file's mtime // from being changed if the contents don't change. This avoids unnecessary dexpreopt reruns. productPackages := android.PathForModuleInPartitionInstall(ctx, "", "product_packages.txt") - appProductPackages := android.PathForModuleOut(ctx, "dexpreopt", "product_packages.txt") + appProductPackages := android.PathForModuleOut(ctx, "dexpreopt", dexJarStem, "product_packages.txt") appProductPackagesStaging := appProductPackages.ReplaceExtension(ctx, "txt.tmp") clcNames, _ := dexpreopt.ComputeClassLoaderContextDependencies(dexpreoptConfig.ClassLoaderContexts) sort.Strings(clcNames) // The order needs to be deterministic. @@ -416,7 +438,7 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr Text("rsync --checksum"). Input(appProductPackagesStaging). Output(appProductPackages) - productPackagesRule.Restat().Build("product_packages", "dexpreopt product_packages") + productPackagesRule.Restat().Build("product_packages."+dexJarStem, "dexpreopt product_packages") dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule( ctx, globalSoong, global, dexpreoptConfig, appProductPackages) @@ -425,9 +447,11 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr return } - dexpreoptRule.Build("dexpreopt", "dexpreopt") + dexpreoptRule.Build("dexpreopt"+"."+dexJarStem, "dexpreopt") - isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + // The current ctx might be of a deapexer module created by a prebuilt apex + // Use the path of the dex file to determine the library name + isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(dexJarStem) for _, install := range dexpreoptRule.Installs() { // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. @@ -452,7 +476,7 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr // The installs will be handled by Make as sub-modules of the java library. d.builtInstalledForApex = append(d.builtInstalledForApex, dexpreopterInstall{ name: arch + "-" + installBase, - moduleName: moduleName(ctx), + moduleName: dexJarStem, outputPathOnHost: install.From, installDirOnDevice: installPath, installFileOnDevice: installBase, diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 5a19945b4..82cece346 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -21,6 +21,7 @@ import ( "android/soong/android" "android/soong/dexpreopt" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -224,8 +225,9 @@ var artApexNames = []string{ } var ( - dexpreoptBootJarDepTag = bootclasspathDependencyTag{name: "dexpreopt-boot-jar"} - dexBootJarsFragmentsKey = android.NewOnceKey("dexBootJarsFragments") + dexpreoptBootJarDepTag = bootclasspathDependencyTag{name: "dexpreopt-boot-jar"} + dexBootJarsFragmentsKey = android.NewOnceKey("dexBootJarsFragments") + apexContributionsMetadataDepTag = dependencyTag{name: "all_apex_contributions"} ) func init() { @@ -502,6 +504,11 @@ type dexpreoptBootJars struct { dexpreoptConfigForMake android.WritablePath } +func (dbj *dexpreoptBootJars) DepsMutator(ctx android.BottomUpMutatorContext) { + // Create a dependency on all_apex_contributions to determine the selected mainline module + ctx.AddDependency(ctx.Module(), apexContributionsMetadataDepTag, "all_apex_contributions") +} + func DexpreoptBootJarsMutator(ctx android.BottomUpMutatorContext) { if _, ok := ctx.Module().(*dexpreoptBootJars); !ok { return @@ -520,6 +527,14 @@ func DexpreoptBootJarsMutator(ctx android.BottomUpMutatorContext) { } // For accessing the boot jars. addDependenciesOntoBootImageModules(ctx, config.modules, dexpreoptBootJarDepTag) + // Create a dependency on the apex selected using RELEASE_APEX_CONTRIBUTIONS_* + // TODO: b/308174306 - Remove the direct depedendency edge to the java_library (source/prebuilt) once all mainline modules + // have been flagged using RELEASE_APEX_CONTRIBUTIONS_* + apexes := []string{} + for i := 0; i < config.modules.Len(); i++ { + apexes = append(apexes, config.modules.Apex(i)) + } + addDependenciesOntoSelectedBootImageApexes(ctx, android.FirstUniqueStrings(apexes)...) } if ctx.OtherModuleExists("platform-bootclasspath") { @@ -532,6 +547,28 @@ func DexpreoptBootJarsMutator(ctx android.BottomUpMutatorContext) { } } +// Create a dependency from dex_bootjars to the specific apexes selected using all_apex_contributions +// This dependency will be used to get the path to the deapexed dex boot jars and profile (via a provider) +func addDependenciesOntoSelectedBootImageApexes(ctx android.BottomUpMutatorContext, apexes ...string) { + psi := android.PrebuiltSelectionInfoMap{} + ctx.VisitDirectDepsWithTag(apexContributionsMetadataDepTag, func(am android.Module) { + if info, exists := android.OtherModuleProvider(ctx, am, android.PrebuiltSelectionInfoProvider); exists { + psi = info + } + }) + for _, apex := range apexes { + for _, selected := range psi.GetSelectedModulesForApiDomain(apex) { + // We need to add a dep on only the apex listed in `contents` of the selected apex_contributions module + // This is not available in a structured format in `apex_contributions`, so this hack adds a dep on all `contents` + // (some modules like art.module.public.api do not have an apex variation since it is a pure stub module that does not get installed) + apexVariationOfSelected := append(ctx.Target().Variations(), blueprint.Variation{Mutator: "apex", Variation: apex}) + if ctx.OtherModuleDependencyVariantExists(apexVariationOfSelected, selected) { + ctx.AddFarVariationDependencies(apexVariationOfSelected, dexpreoptBootJarDepTag, selected) + } + } + } +} + func gatherBootclasspathFragments(ctx android.ModuleContext) map[string]android.Module { return ctx.Config().Once(dexBootJarsFragmentsKey, func() interface{} { fragments := make(map[string]android.Module) @@ -662,36 +699,67 @@ func getModulesForImage(ctx android.ModuleContext, imageConfig *bootImageConfig) // extractEncodedDexJarsFromModulesOrBootclasspathFragments gets the hidden API encoded dex jars for // the given modules. func extractEncodedDexJarsFromModulesOrBootclasspathFragments(ctx android.ModuleContext, apexJarModulePairs []apexJarModulePair) bootDexJarByModule { + apexNameToApexExportInfoMap := getApexNameToApexExportsInfoMap(ctx) encodedDexJarsByModuleName := bootDexJarByModule{} for _, pair := range apexJarModulePairs { - var path android.Path - if android.IsConfiguredJarForPlatform(pair.apex) || android.IsModulePrebuilt(pair.jarModule) { - // This gives us the dex jar with the hidden API flags encoded from the monolithic hidden API - // files or the dex jar extracted from a prebuilt APEX. We can't use this for a boot jar for - // a source APEX because there is no guarantee that it is the same as the jar packed into the - // APEX. In practice, they are the same when we are building from a full source tree, but they - // are different when we are building from a thin manifest (e.g., master-art), where there is - // no monolithic hidden API files at all. - path = retrieveEncodedBootDexJarFromModule(ctx, pair.jarModule) + dexJarPath := getDexJarForApex(ctx, pair, apexNameToApexExportInfoMap) + encodedDexJarsByModuleName.addPath(pair.jarModule, dexJarPath) + } + return encodedDexJarsByModuleName +} + +type apexNameToApexExportsInfoMap map[string]android.ApexExportsInfo + +// javaLibraryPathOnHost returns the path to the java library which is exported by the apex for hiddenapi and dexpreopt and a boolean indicating whether the java library exists +// For prebuilt apexes, this is created by deapexing the prebuilt apex +func (m *apexNameToApexExportsInfoMap) javaLibraryDexPathOnHost(ctx android.ModuleContext, apex string, javalib string) (android.Path, bool) { + if info, exists := (*m)[apex]; exists { + if dex, exists := info.LibraryNameToDexJarPathOnHost[javalib]; exists { + return dex, true } else { - // Use exactly the same jar that is packed into the APEX. - fragment := getBootclasspathFragmentByApex(ctx, pair.apex) - if fragment == nil { - ctx.ModuleErrorf("Boot jar '%[1]s' is from APEX '%[2]s', but a bootclasspath_fragment for "+ - "APEX '%[2]s' doesn't exist or is not added as a dependency of dex_bootjars", - pair.jarModule.Name(), - pair.apex) - } - bootclasspathFragmentInfo, _ := android.OtherModuleProvider(ctx, fragment, BootclasspathFragmentApexContentInfoProvider) - jar, err := bootclasspathFragmentInfo.DexBootJarPathForContentModule(pair.jarModule) - if err != nil { - ctx.ModuleErrorf("%s", err) - } - path = jar + ctx.ModuleErrorf("Apex %s does not provide a dex boot jar for library %s\n", apex, javalib) } - encodedDexJarsByModuleName.addPath(pair.jarModule, path) } - return encodedDexJarsByModuleName + // An apex entry could not be found. Return false. + // TODO: b/308174306 - When all the mainline modules have been flagged, make this a hard error + return nil, false +} + +// Returns the java libraries exported by the apex for hiddenapi and dexpreopt +// This information can come from two mechanisms +// 1. New: Direct deps to _selected_ apexes. The apexes return a ApexExportsInfo +// 2. Legacy: An edge to java_library or java_import (java_sdk_library) module. For prebuilt apexes, this serves as a hook and is populated by deapexers of prebuilt apxes +// TODO: b/308174306 - Once all mainline modules have been flagged, drop (2) +func getDexJarForApex(ctx android.ModuleContext, pair apexJarModulePair, apexNameToApexExportsInfoMap apexNameToApexExportsInfoMap) android.Path { + if dex, found := apexNameToApexExportsInfoMap.javaLibraryDexPathOnHost(ctx, pair.apex, android.RemoveOptionalPrebuiltPrefix(pair.jarModule.Name())); found { + return dex + } + // TODO: b/308174306 - Remove the legacy mechanism + if android.IsConfiguredJarForPlatform(pair.apex) || android.IsModulePrebuilt(pair.jarModule) { + // This gives us the dex jar with the hidden API flags encoded from the monolithic hidden API + // files or the dex jar extracted from a prebuilt APEX. We can't use this for a boot jar for + // a source APEX because there is no guarantee that it is the same as the jar packed into the + // APEX. In practice, they are the same when we are building from a full source tree, but they + // are different when we are building from a thin manifest (e.g., master-art), where there is + // no monolithic hidden API files at all. + return retrieveEncodedBootDexJarFromModule(ctx, pair.jarModule) + } else { + // Use exactly the same jar that is packed into the APEX. + fragment := getBootclasspathFragmentByApex(ctx, pair.apex) + if fragment == nil { + ctx.ModuleErrorf("Boot jar '%[1]s' is from APEX '%[2]s', but a bootclasspath_fragment for "+ + "APEX '%[2]s' doesn't exist or is not added as a dependency of dex_bootjars", + pair.jarModule.Name(), + pair.apex) + } + bootclasspathFragmentInfo, _ := android.OtherModuleProvider(ctx, fragment, BootclasspathFragmentApexContentInfoProvider) + jar, err := bootclasspathFragmentInfo.DexBootJarPathForContentModule(pair.jarModule) + if err != nil { + ctx.ModuleErrorf("%s", err) + } + return jar + } + return nil } // copyBootJarsToPredefinedLocations generates commands that will copy boot jars to predefined @@ -823,6 +891,37 @@ type bootImageVariantOutputs struct { config *bootImageVariant } +// Returns the profile file for an apex +// This information can come from two mechanisms +// 1. New: Direct deps to _selected_ apexes. The apexes return a BootclasspathFragmentApexContentInfo +// 2. Legacy: An edge to bootclasspath_fragment module. For prebuilt apexes, this serves as a hook and is populated by deapexers of prebuilt apxes +// TODO: b/308174306 - Once all mainline modules have been flagged, drop (2) +func getProfilePathForApex(ctx android.ModuleContext, apexName string, apexNameToBcpInfoMap map[string]android.ApexExportsInfo) android.Path { + if info, exists := apexNameToBcpInfoMap[apexName]; exists { + return info.ProfilePathOnHost + } + // TODO: b/308174306 - Remove the legacy mechanism + fragment := getBootclasspathFragmentByApex(ctx, apexName) + if fragment == nil { + ctx.ModuleErrorf("Boot image config imports profile from '%[2]s', but a "+ + "bootclasspath_fragment for APEX '%[2]s' doesn't exist or is not added as a "+ + "dependency of dex_bootjars", + apexName) + return nil + } + return fragment.(commonBootclasspathFragment).getProfilePath() +} + +func getApexNameToApexExportsInfoMap(ctx android.ModuleContext) apexNameToApexExportsInfoMap { + apexNameToApexExportsInfoMap := apexNameToApexExportsInfoMap{} + ctx.VisitDirectDepsWithTag(dexpreoptBootJarDepTag, func(am android.Module) { + if info, exists := android.OtherModuleProvider(ctx, am, android.ApexExportsInfoProvider); exists { + apexNameToApexExportsInfoMap[info.ApexName] = info + } + }) + return apexNameToApexExportsInfoMap +} + // Generate boot image build rules for a specific target. func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, profile android.Path) bootImageVariantOutputs { @@ -865,6 +964,8 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p invocationPath := outputPath.ReplaceExtension(ctx, "invocation") + apexNameToApexExportsInfoMap := getApexNameToApexExportsInfoMap(ctx) + cmd.Tool(globalSoong.Dex2oat). Flag("--avoid-storing-invocation"). FlagWithOutput("--write-invocation-to=", invocationPath).ImplicitOutput(invocationPath). @@ -877,16 +978,7 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p } for _, apex := range image.profileImports { - fragment := getBootclasspathFragmentByApex(ctx, apex) - if fragment == nil { - ctx.ModuleErrorf("Boot image config '%[1]s' imports profile from '%[2]s', but a "+ - "bootclasspath_fragment for APEX '%[2]s' doesn't exist or is not added as a "+ - "dependency of dex_bootjars", - image.name, - apex) - return bootImageVariantOutputs{} - } - importedProfile := fragment.(commonBootclasspathFragment).getProfilePath() + importedProfile := getProfilePathForApex(ctx, apex, apexNameToApexExportsInfoMap) if importedProfile == nil { ctx.ModuleErrorf("Boot image config '%[1]s' imports profile from '%[2]s', but '%[2]s' "+ "doesn't provide a profile", diff --git a/java/dexpreopt_config.go b/java/dexpreopt_config.go index 2bd696c3d..254b2c1b0 100644 --- a/java/dexpreopt_config.go +++ b/java/dexpreopt_config.go @@ -45,7 +45,7 @@ var ( frameworkBootImageName = "boot" mainlineBootImageName = "mainline" bootImageStem = "boot" - profileInstallPathInApex = "etc/boot-image.prof" + ProfileInstallPathInApex = "etc/boot-image.prof" ) // getImageNames returns an ordered list of image names. The order doesn't matter but needs to be diff --git a/java/dexpreopt_test.go b/java/dexpreopt_test.go index fedd5640e..73e33f4fb 100644 --- a/java/dexpreopt_test.go +++ b/java/dexpreopt_test.go @@ -410,7 +410,7 @@ func TestAndroidMkEntriesForApex(t *testing.T) { verifyEntries(t, "entriesList[0]", "service-foo-dexpreopt-arm64-apex@com.android.apex1@javalib@service-foo.jar@classes.odex", - "/dexpreopt/oat/arm64/javalib.odex", + "/dexpreopt/service-foo/oat/arm64/javalib.odex", "/system/framework/oat/arm64", "apex@com.android.apex1@javalib@service-foo.jar@classes.odex", entriesList[0]) @@ -418,7 +418,7 @@ func TestAndroidMkEntriesForApex(t *testing.T) { verifyEntries(t, "entriesList[1]", "service-foo-dexpreopt-arm64-apex@com.android.apex1@javalib@service-foo.jar@classes.vdex", - "/dexpreopt/oat/arm64/javalib.vdex", + "/dexpreopt/service-foo/oat/arm64/javalib.vdex", "/system/framework/oat/arm64", "apex@com.android.apex1@javalib@service-foo.jar@classes.vdex", entriesList[1]) @@ -459,7 +459,7 @@ func TestGenerateProfileEvenIfDexpreoptIsDisabled(t *testing.T) { ctx := result.TestContext dexpreopt := ctx.ModuleForTests("foo", "android_common").MaybeRule("dexpreopt") - expected := []string{"out/soong/.intermediates/foo/android_common/dexpreopt/profile.prof"} + expected := []string{"out/soong/.intermediates/foo/android_common/dexpreopt/foo/profile.prof"} android.AssertArrayString(t, "outputs", expected, dexpreopt.AllOutputs()) } diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go index 17cad8958..7bcaca1d4 100644 --- a/java/droidstubs_test.go +++ b/java/droidstubs_test.go @@ -213,7 +213,7 @@ func TestDroidstubsSandbox(t *testing.T) { } manifest := android.RuleBuilderSboxProtoForTests(t, ctx, m.Output("metalava.sbox.textproto")) - if g, w := manifest.Commands[0].GetCommand(), "reference __SBOX_SANDBOX_DIR__/out/.intermediates/foo/gen/foo.txt"; !strings.Contains(g, w) { + if g, w := manifest.Commands[0].GetCommand(), "reference __SBOX_SANDBOX_DIR__/out/soong/.intermediates/foo/gen/foo.txt"; !strings.Contains(g, w) { t.Errorf("Expected command to contain %q, got %q", w, g) } } diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index 8011f343b..c3caa084f 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -19,6 +19,7 @@ import ( "strings" "android/soong/android" + "android/soong/dexpreopt" "github.com/google/blueprint" ) @@ -947,6 +948,7 @@ type HiddenAPIOutput struct { HiddenAPIFlagOutput // The map from base module name to the path to the encoded boot dex file. + // This field is not available in prebuilt apexes EncodedBootDexFilesByModule bootDexJarByModule } @@ -1249,9 +1251,27 @@ func buildRuleToGenerateRemovedDexSignatures(ctx android.ModuleContext, suffix s } // extractBootDexJarsFromModules extracts the boot dex jars from the supplied modules. +// This information can come from two mechanisms +// 1. New: Direct deps to _selected_ apexes. The apexes contain a ApexExportsInfo +// 2. Legacy: An edge to java_sdk_library(_import) module. For prebuilt apexes, this serves as a hook and is populated by deapexers of prebuilt apxes +// TODO: b/308174306 - Once all mainline modules have been flagged, drop (2) func extractBootDexJarsFromModules(ctx android.ModuleContext, contents []android.Module) bootDexJarByModule { bootDexJars := bootDexJarByModule{} + + apexNameToApexExportsInfoMap := getApexNameToApexExportsInfoMap(ctx) + // For ART and mainline module jars, query apexNameToApexExportsInfoMap to get the dex file + apexJars := dexpreopt.GetGlobalConfig(ctx).ArtApexJars.AppendList(&dexpreopt.GetGlobalConfig(ctx).ApexBootJars) + for i := 0; i < apexJars.Len(); i++ { + if dex, found := apexNameToApexExportsInfoMap.javaLibraryDexPathOnHost(ctx, apexJars.Apex(i), apexJars.Jar(i)); found { + bootDexJars[apexJars.Jar(i)] = dex + } + } + + // TODO - b/308174306: Drop the legacy mechanism for _, module := range contents { + if _, exists := bootDexJars[android.RemoveOptionalPrebuiltPrefix(module.Name())]; exists { + continue + } hiddenAPIModule := hiddenAPIModuleFromModule(ctx, module) if hiddenAPIModule == nil { continue diff --git a/java/java.go b/java/java.go index 630318e6d..2a4fafa8b 100644 --- a/java/java.go +++ b/java/java.go @@ -2100,6 +2100,7 @@ type Import struct { // output file containing classes.dex and resources dexJarFile OptionalDexJarPath + dexJarFileErr error dexJarInstallFile android.Path combinedClasspathFile android.Path @@ -2250,15 +2251,18 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { ai, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) if ai.ForPrebuiltApex { // Get the path of the dex implementation jar from the `deapexer` module. - di := android.FindDeapexerProviderForModule(ctx) - if di == nil { - return // An error has been reported by FindDeapexerProviderForModule. + di, err := android.FindDeapexerProviderForModule(ctx) + if err != nil { + // An error was found, possibly due to multiple apexes in the tree that export this library + // Defer the error till a client tries to call DexJarBuildPath + j.dexJarFileErr = err + return } - dexJarFileApexRootRelative := apexRootRelativePathToJavaLib(j.BaseModuleName()) + dexJarFileApexRootRelative := ApexRootRelativePathToJavaLib(j.BaseModuleName()) if dexOutputPath := di.PrebuiltExportPath(dexJarFileApexRootRelative); dexOutputPath != nil { dexJarFile := makeDexJarPathFromPath(dexOutputPath) j.dexJarFile = dexJarFile - installPath := android.PathForModuleInPartitionInstall(ctx, "apex", ai.ApexVariationName, apexRootRelativePathToJavaLib(j.BaseModuleName())) + installPath := android.PathForModuleInPartitionInstall(ctx, "apex", ai.ApexVariationName, ApexRootRelativePathToJavaLib(j.BaseModuleName())) j.dexJarInstallFile = installPath j.dexpreopter.installPath = j.dexpreopter.getInstallPath(ctx, installPath) @@ -2375,6 +2379,9 @@ func (j *Import) ImplementationAndResourcesJars() android.Paths { } func (j *Import) DexJarBuildPath() OptionalDexJarPath { + if j.dexJarFileErr != nil { + panic(j.dexJarFileErr.Error()) + } return j.dexJarFile } @@ -2415,7 +2422,7 @@ func (j *Import) ShouldSupportSdkVersion(ctx android.BaseModuleContext, // java_sdk_library_import with the specified base module name requires to be exported from a // prebuilt_apex/apex_set. func requiredFilesFromPrebuiltApexForImport(name string, d *dexpreopter) []string { - dexJarFileApexRootRelative := apexRootRelativePathToJavaLib(name) + dexJarFileApexRootRelative := ApexRootRelativePathToJavaLib(name) // Add the dex implementation jar to the set of exported files. files := []string{ dexJarFileApexRootRelative, @@ -2426,9 +2433,9 @@ func requiredFilesFromPrebuiltApexForImport(name string, d *dexpreopter) []strin return files } -// apexRootRelativePathToJavaLib returns the path, relative to the root of the apex's contents, for +// ApexRootRelativePathToJavaLib returns the path, relative to the root of the apex's contents, for // the java library with the specified name. -func apexRootRelativePathToJavaLib(name string) string { +func ApexRootRelativePathToJavaLib(name string) string { return filepath.Join("javalib", name+".jar") } diff --git a/java/java_test.go b/java/java_test.go index e21018c39..b9dc453b3 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -618,8 +618,6 @@ func TestPrebuilts(t *testing.T) { android.AssertStringEquals(t, "unexpected LOCAL_SOONG_MODULE_TYPE", "java_library", entries.EntryMap["LOCAL_SOONG_MODULE_TYPE"][0]) entries = android.AndroidMkEntriesForTest(t, ctx, barModule.Module())[0] android.AssertStringEquals(t, "unexpected LOCAL_SOONG_MODULE_TYPE", "java_import", entries.EntryMap["LOCAL_SOONG_MODULE_TYPE"][0]) - entries = android.AndroidMkEntriesForTest(t, ctx, ctx.ModuleForTests("sdklib", "android_common").Module())[0] - android.AssertStringEquals(t, "unexpected LOCAL_SOONG_MODULE_TYPE", "java_sdk_library_import", entries.EntryMap["LOCAL_SOONG_MODULE_TYPE"][0]) } func assertDeepEquals(t *testing.T, message string, expected interface{}, actual interface{}) { @@ -2433,7 +2431,7 @@ func TestSdkLibraryProvidesSystemModulesToApiLibrary(t *testing.T) { manifest := m.Output("metalava.sbox.textproto") sboxProto := android.RuleBuilderSboxProtoForTests(t, result.TestContext, manifest) manifestCommand := sboxProto.Commands[0].GetCommand() - classPathFlag := "--classpath __SBOX_SANDBOX_DIR__/out/.intermediates/bar/android_common/turbine-combined/bar.jar" + classPathFlag := "--classpath __SBOX_SANDBOX_DIR__/out/soong/.intermediates/bar/android_common/turbine-combined/bar.jar" android.AssertStringDoesContain(t, "command expected to contain classpath flag", manifestCommand, classPathFlag) } diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 88d1ae8c0..4db426e0d 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -106,6 +106,9 @@ func (b *platformBootclasspathModule) OutputFiles(tag string) (android.Paths, er } func (b *platformBootclasspathModule) DepsMutator(ctx android.BottomUpMutatorContext) { + // Create a dependency on all_apex_contributions to determine the selected mainline module + ctx.AddDependency(ctx.Module(), apexContributionsMetadataDepTag, "all_apex_contributions") + b.hiddenAPIDepsMutator(ctx) if !dexpreopt.IsDex2oatNeeded(ctx) { @@ -130,6 +133,8 @@ func (b *platformBootclasspathModule) hiddenAPIDepsMutator(ctx android.BottomUpM func (b *platformBootclasspathModule) BootclasspathDepsMutator(ctx android.BottomUpMutatorContext) { // Add dependencies on all the ART jars. global := dexpreopt.GetGlobalConfig(ctx) + addDependenciesOntoSelectedBootImageApexes(ctx, "com.android.art") + // TODO: b/308174306 - Remove the mechanism of depending on the java_sdk_library(_import) directly addDependenciesOntoBootImageModules(ctx, global.ArtApexJars, platformBootclasspathArtBootJarDepTag) // Add dependencies on all the non-updatable jars, which are on the platform or in non-updatable @@ -138,6 +143,12 @@ func (b *platformBootclasspathModule) BootclasspathDepsMutator(ctx android.Botto // Add dependencies on all the updatable jars, except the ART jars. apexJars := dexpreopt.GetGlobalConfig(ctx).ApexBootJars + apexes := []string{} + for i := 0; i < apexJars.Len(); i++ { + apexes = append(apexes, apexJars.Apex(i)) + } + addDependenciesOntoSelectedBootImageApexes(ctx, android.FirstUniqueStrings(apexes)...) + // TODO: b/308174306 - Remove the mechanism of depending on the java_sdk_library(_import) directly addDependenciesOntoBootImageModules(ctx, apexJars, platformBootclasspathApexBootJarDepTag) // Add dependencies on all the fragments. diff --git a/java/sdk_library.go b/java/sdk_library.go index 0584281a5..ef34fb6cd 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -2378,7 +2378,8 @@ type SdkLibraryImport struct { xmlPermissionsFileModule *sdkLibraryXml // Build path to the dex implementation jar obtained from the prebuilt_apex, if any. - dexJarFile OptionalDexJarPath + dexJarFile OptionalDexJarPath + dexJarFileErr error // Expected install file path of the source module(sdk_library) // or dex implementation jar obtained from the prebuilt_apex, if any. @@ -2591,14 +2592,6 @@ func (module *SdkLibraryImport) DepsMutator(ctx android.BottomUpMutatorContext) } } -func (module *SdkLibraryImport) AndroidMkEntries() []android.AndroidMkEntries { - // For an SDK library imported from a prebuilt APEX, we don't need a Make module for itself, as we - // don't need to install it. However, we need to add its dexpreopt outputs as sub-modules, if it - // is preopted. - dexpreoptEntries := module.dexpreopter.AndroidMkEntriesForApex() - return append(dexpreoptEntries, android.AndroidMkEntries{Disabled: true}) -} - var _ android.ApexModule = (*SdkLibraryImport)(nil) // Implements android.ApexModule @@ -2695,11 +2688,14 @@ func (module *SdkLibraryImport) GenerateAndroidBuildActions(ctx android.ModuleCo ai, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) if ai.ForPrebuiltApex { // Get the path of the dex implementation jar from the `deapexer` module. - di := android.FindDeapexerProviderForModule(ctx) - if di == nil { - return // An error has been reported by FindDeapexerProviderForModule. + di, err := android.FindDeapexerProviderForModule(ctx) + if err != nil { + // An error was found, possibly due to multiple apexes in the tree that export this library + // Defer the error till a client tries to call DexJarBuildPath + module.dexJarFileErr = err + return } - dexJarFileApexRootRelative := apexRootRelativePathToJavaLib(module.BaseModuleName()) + dexJarFileApexRootRelative := ApexRootRelativePathToJavaLib(module.BaseModuleName()) if dexOutputPath := di.PrebuiltExportPath(dexJarFileApexRootRelative); dexOutputPath != nil { dexJarFile := makeDexJarPathFromPath(dexOutputPath) module.dexJarFile = dexJarFile @@ -2759,6 +2755,9 @@ func (module *SdkLibraryImport) SdkImplementationJars(ctx android.BaseModuleCont func (module *SdkLibraryImport) DexJarBuildPath() OptionalDexJarPath { // The dex implementation jar extracted from the .apex file should be used in preference to the // source. + if module.dexJarFileErr != nil { + panic(module.dexJarFileErr.Error()) + } if module.dexJarFile.IsSet() { return module.dexJarFile } diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 0965fc2c7..63419d6b1 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -950,6 +950,7 @@ func TestJavaSdkLibraryImport_WithSource(t *testing.T) { }) CheckModuleDependencies(t, result.TestContext, "prebuilt_sdklib", "android_common", []string{ + `all_apex_contributions`, `prebuilt_sdklib.stubs`, `sdklib.impl`, // This should be prebuilt_sdklib.stubs but is set to sdklib.stubs because the @@ -1022,6 +1023,7 @@ func testJavaSdkLibraryImport_Preferred(t *testing.T, prefer string, preparer an }) CheckModuleDependencies(t, result.TestContext, "prebuilt_sdklib", "android_common", []string{ + `all_apex_contributions`, `dex2oatd`, `prebuilt_sdklib.stubs`, `prebuilt_sdklib.stubs.source`, @@ -1085,9 +1087,6 @@ func TestSdkLibraryImport_MetadataModuleSupersedesPreferred(t *testing.T) { "prebuilt_sdklib.source_preferred_using_legacy_flags", ], } - all_apex_contributions { - name: "all_apex_contributions", - } java_sdk_library { name: "sdklib.prebuilt_preferred_using_legacy_flags", srcs: ["a.java"], @@ -1169,9 +1168,6 @@ func TestSdkLibraryImport_MetadataModuleSupersedesPreferred(t *testing.T) { prepareForJavaTest, PrepareForTestWithJavaSdkLibraryFiles, FixtureWithLastReleaseApis("sdklib.source_preferred_using_legacy_flags", "sdklib.prebuilt_preferred_using_legacy_flags"), - android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { - android.RegisterApexContributionsBuildComponents(ctx) - }), android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.BuildFlags = map[string]string{ "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contributions", diff --git a/java/system_modules.go b/java/system_modules.go index 1c7917109..f3446483b 100644 --- a/java/system_modules.go +++ b/java/system_modules.go @@ -55,7 +55,8 @@ var ( `${config.MergeZipsCmd} -j ${workDir}/module.jar ${workDir}/classes.jar $in && ` + // Note: The version of the java.base module created must match the version // of the jlink tool which consumes it. - `${config.JmodCmd} create --module-version ${config.JlinkVersion} --target-platform android ` + + // Use LINUX-OTHER to be compatible with JDK 21+ (b/294137077) + `${config.JmodCmd} create --module-version ${config.JlinkVersion} --target-platform LINUX-OTHER ` + ` --class-path ${workDir}/module.jar ${workDir}/jmod/java.base.jmod && ` + `${config.JlinkCmd} --module-path ${workDir}/jmod --add-modules java.base --output ${outDir} ` + // Note: The system-modules jlink plugin is disabled because (a) it is not diff --git a/java/test_spec_test.go b/java/test_spec_test.go index f628b4b74..4144dad69 100644 --- a/java/test_spec_test.go +++ b/java/test_spec_test.go @@ -29,12 +29,10 @@ func TestTestSpec(t *testing.T) { }` result := runTestSpecTest(t, android.FixtureExpectsNoErrors, bp) - module := result.ModuleForTests( - "module-name", "", - ).Module().(*soongTesting.TestSpecModule) + module := result.ModuleForTests("module-name", "") // Check that the provider has the right contents - data, _ := android.SingletonModuleProvider(result, module, soongTesting.TestSpecProviderKey) + data, _ := android.SingletonModuleProvider(result, module.Module(), soongTesting.TestSpecProviderKey) if !strings.HasSuffix( data.IntermediatePath.String(), "/intermediateTestSpecMetadata.pb", ) { @@ -44,13 +42,8 @@ func TestTestSpec(t *testing.T) { ) } - buildParamsSlice := module.BuildParamsForTests() - var metadata = "" - for _, params := range buildParamsSlice { - if params.Rule.String() == "android/soong/android.writeFile" { - metadata = params.Args["content"] - } - } + metadata := android.ContentFromFileRuleForTests(t, result.TestContext, + module.Output(data.IntermediatePath.String())) metadataList := make([]*test_spec_proto.TestSpec_OwnershipMetadata, 0, 2) teamId := "12345" @@ -70,9 +63,7 @@ func TestTestSpec(t *testing.T) { } testSpecMetadata := test_spec_proto.TestSpec{OwnershipMetadataList: metadataList} protoData, _ := proto.Marshal(&testSpecMetadata) - rawData := string(protoData) - formattedData := strings.ReplaceAll(rawData, "\n", "\\n") - expectedMetadata := "'" + formattedData + "\\n'" + expectedMetadata := string(protoData) if metadata != expectedMetadata { t.Errorf( diff --git a/java/testing.go b/java/testing.go index d55cffc40..5959c49e4 100644 --- a/java/testing.go +++ b/java/testing.go @@ -383,6 +383,7 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { RegisterSystemModulesBuildComponents(ctx) registerSystemserverClasspathBuildComponents(ctx) registerLintBuildComponents(ctx) + android.RegisterApexContributionsBuildComponents(ctx) } // gatherRequiredDepsForTest gathers the module definitions used by @@ -570,6 +571,11 @@ func gatherRequiredDepsForTest() string { } ` + bp += ` + all_apex_contributions { + name: "all_apex_contributions", + } +` return bp } diff --git a/phony/phony.go b/phony/phony.go index a8b651aa8..bb4878899 100644 --- a/phony/phony.go +++ b/phony/phony.go @@ -24,6 +24,7 @@ import ( func init() { android.RegisterModuleType("phony", PhonyFactory) + android.RegisterModuleType("phony_rule", PhonyRuleFactory) } type phony struct { @@ -71,3 +72,40 @@ func (p *phony) AndroidMk() android.AndroidMkData { }, } } + +type PhonyRule struct { + android.ModuleBase + + properties PhonyProperties +} + +type PhonyProperties struct { + // The Phony_deps is the set of all dependencies for this target, + // and it can function similarly to .PHONY in a makefile. + // Additionally, dependencies within it can even include genrule. + Phony_deps []string +} + +// The phony_rule provides functionality similar to the .PHONY in a makefile. +// It can create a phony target and include relevant dependencies associated with it. +func PhonyRuleFactory() android.Module { + module := &PhonyRule{} + android.InitAndroidModule(module) + module.AddProperties(&module.properties) + return module +} + +func (p *PhonyRule) GenerateAndroidBuildActions(ctx android.ModuleContext) { +} + +func (p *PhonyRule) AndroidMk() android.AndroidMkData { + return android.AndroidMkData{ + Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + if len(p.properties.Phony_deps) > 0 { + depModulesStr := strings.Join(p.properties.Phony_deps, " ") + fmt.Fprintln(w, ".PHONY:", name) + fmt.Fprintln(w, name, ":", depModulesStr) + } + }, + } +} diff --git a/rust/protobuf.go b/rust/protobuf.go index d021076eb..0b26b80fa 100644 --- a/rust/protobuf.go +++ b/rust/protobuf.go @@ -54,11 +54,6 @@ type ProtobufProperties struct { // List of libraries which export include paths required for this module Header_libs []string `android:"arch_variant,variant_prepend"` - // Use protobuf version 3.x. This will be deleted once we migrate all current users - // of protobuf off of 2.x. - // ludovicb@: DEPRECATED, to be removed - Use_protobuf3 *bool - // List of exported include paths containing proto files for dependent rust_protobuf modules. Exported_include_dirs []string } diff --git a/rust/protobuf_test.go b/rust/protobuf_test.go index b375a6463..cae071b8f 100644 --- a/rust/protobuf_test.go +++ b/rust/protobuf_test.go @@ -28,7 +28,6 @@ func TestRustProtobuf3(t *testing.T) { protos: ["buf.proto", "proto.proto"], crate_name: "rust_proto", source_stem: "buf", - use_protobuf3: true, shared_libs: ["libfoo_shared"], static_libs: ["libfoo_static"], } @@ -77,7 +76,6 @@ func TestRustProtobufInclude(t *testing.T) { protos: ["proto.proto"], crate_name: "rust_proto", source_stem: "proto", - use_protobuf3: true, rustlibs: ["librust_exported_proto", "libfoo"], } rust_protobuf { @@ -85,7 +83,6 @@ func TestRustProtobufInclude(t *testing.T) { protos: ["proto.proto"], crate_name: "rust_exported_proto", source_stem: "exported_proto", - use_protobuf3: true, exported_include_dirs: ["proto"] } rust_library { diff --git a/rust/rust.go b/rust/rust.go index 521f624a2..6f4631de7 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -23,7 +23,6 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/proptools" - "android/soong/aconfig" "android/soong/android" "android/soong/cc" cc_config "android/soong/cc/config" @@ -1007,7 +1006,7 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } - aconfig.CollectDependencyAconfigFiles(ctx, &mod.mergedAconfigFiles) + android.CollectDependencyAconfigFiles(ctx, &mod.mergedAconfigFiles) } func (mod *Module) deps(ctx DepsContext) Deps { diff --git a/scripts/Android.bp b/scripts/Android.bp index 97f6ab424..7baaadb77 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -254,3 +254,8 @@ python_test_host { "modify_permissions_allowlist.py", ], } + +sh_binary_host { + name: "keep-flagged-apis", + src: "keep-flagged-apis.sh", +} diff --git a/scripts/keep-flagged-apis.sh b/scripts/keep-flagged-apis.sh new file mode 100755 index 000000000..9c48fdbad --- /dev/null +++ b/scripts/keep-flagged-apis.sh @@ -0,0 +1,46 @@ +#!/bin/bash -e +# +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Convert a list of flags in the input file to a list of metalava options +# that will keep the APIs for those flags will hiding all other flagged +# APIs. + +FLAGS="$1" + +FLAGGED="android.annotation.FlaggedApi" + +# Convert the list of feature flags in the input file to Metalava options +# of the form `--revert-annotation !android.annotation.FlaggedApi("<flag>")` +# to prevent the annotated APIs from being hidden, i.e. include the annotated +# APIs in the SDK snapshots. This also preserves the line comments, they will +# be ignored by Metalava but might be useful when debugging. +while read -r line; do + key=$(echo "$line" | cut -d= -f1) + value=$(echo "$line" | cut -d= -f2) + + # Skip if value is not true and line does not start with '#' + if [[ ( $value != "true" ) && ( $line =~ ^[^#] )]]; then + continue + fi + + # Escape and quote the key for sed + escaped_key=$(echo "$key" | sed "s/'/\\\'/g; s/ /\\ /g") + + echo $line | sed "s|^[^#].*$|--revert-annotation '!$FLAGGED(\"$escaped_key\")'|" +done < "$FLAGS" + +# Revert all flagged APIs, unless listed above. +echo "--revert-annotation $FLAGGED" diff --git a/testing/code_metadata.go b/testing/code_metadata.go index 3cf7c5965..11ba43037 100644 --- a/testing/code_metadata.go +++ b/testing/code_metadata.go @@ -128,7 +128,7 @@ func (module *CodeMetadataModule) GenerateAndroidBuildActions(ctx android.Module intermediatePath := android.PathForModuleOut( ctx, "intermediateCodeMetadata.pb", ) - android.WriteFileRule(ctx, intermediatePath, string(protoData)) + android.WriteFileRuleVerbatim(ctx, intermediatePath, string(protoData)) android.SetProvider(ctx, CodeMetadataProviderKey, diff --git a/testing/test_spec.go b/testing/test_spec.go index d25961229..4d885c6de 100644 --- a/testing/test_spec.go +++ b/testing/test_spec.go @@ -117,7 +117,7 @@ func (module *TestSpecModule) GenerateAndroidBuildActions(ctx android.ModuleCont if err != nil { ctx.ModuleErrorf("Error: %s", err.Error()) } - android.WriteFileRule(ctx, intermediatePath, string(protoData)) + android.WriteFileRuleVerbatim(ctx, intermediatePath, string(protoData)) android.SetProvider(ctx, TestSpecProviderKey, TestSpecProviderData{ diff --git a/ui/build/soong.go b/ui/build/soong.go index 0bf886205..90c3bfc10 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -36,6 +36,7 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/microfactory" + "github.com/google/blueprint/pathtools" "google.golang.org/protobuf/proto" ) @@ -181,7 +182,15 @@ func getGlobPathName(config Config) string { return globPathName } -func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation { +func getGlobPathNameFromPrimaryBuilderFactory(config Config, pb PrimaryBuilderFactory) string { + if pb.name == soongBuildTag { + // Glob path for soong build would be separated per product target + return getGlobPathName(config) + } + return pb.name +} + +func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstrap.PrimaryBuilderInvocation { commonArgs := make([]string, 0, 0) if !pb.config.skipSoongTests { @@ -215,11 +224,7 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuil var allArgs []string allArgs = append(allArgs, pb.specificArgs...) - globPathName := pb.name - // Glob path for soong build would be separated per product target - if pb.name == soongBuildTag { - globPathName = getGlobPathName(pb.config) - } + globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb) allArgs = append(allArgs, "--globListDir", globPathName, "--globFile", pb.config.NamedGlobFile(globPathName)) @@ -234,8 +239,11 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuil } allArgs = append(allArgs, "Android.bp") + globfiles := bootstrap.GlobFileListFiles(bootstrap.GlobDirectory(config.SoongOutDir(), globPathName)) + return bootstrap.PrimaryBuilderInvocation{ Inputs: []string{"Android.bp"}, + Implicits: globfiles, Outputs: []string{pb.output}, Args: allArgs, Description: pb.description, @@ -376,17 +384,10 @@ func bootstrapBlueprint(ctx Context, config Config) { if debuggedInvocations[pbf.name] { pbf.debugPort = delvePort } - pbi := pbf.primaryBuilderInvocation() + pbi := pbf.primaryBuilderInvocation(config) invocations = append(invocations, pbi) } - // The glob .ninja files are subninja'd. However, they are generated during - // the build itself so we write an empty file if the file does not exist yet - // so that the subninja doesn't fail on clean builds - for _, globFile := range bootstrapGlobFileList(config) { - writeEmptyFile(ctx, globFile) - } - blueprintArgs := bootstrap.Args{ ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"), OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"), @@ -408,6 +409,28 @@ func bootstrapBlueprint(ctx Context, config Config) { primaryBuilderInvocations: invocations, } + // The glob ninja files are generated during the main build phase. However, the + // primary buildifer invocation depends on all of its glob files, even before + // it's been run. Generate a "empty" glob ninja file on the first run, + // so that the files can be there to satisfy the dependency. + for _, pb := range pbfs { + globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb) + globNinjaFile := config.NamedGlobFile(globPathName) + if _, err := os.Stat(globNinjaFile); os.IsNotExist(err) { + err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{ + GlobLister: func() pathtools.MultipleGlobResults { return nil }, + GlobFile: globNinjaFile, + GlobDir: bootstrap.GlobDirectory(config.SoongOutDir(), globPathName), + SrcDir: ".", + }, blueprintConfig) + if err != nil { + ctx.Fatal(err) + } + } else if err != nil { + ctx.Fatal(err) + } + } + // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little // reason to write a `bootstrap.ninja.d` file _, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig) diff --git a/ui/build/test_build.go b/ui/build/test_build.go index c5dc4c53f..309513919 100644 --- a/ui/build/test_build.go +++ b/ui/build/test_build.go @@ -63,6 +63,7 @@ func testForDanglingRules(ctx Context, config Config) { outDir := config.OutDir() modulePathsDir := filepath.Join(outDir, ".module_paths") + rawFilesDir := filepath.Join(outDir, "soong", "raw") variablesFilePath := filepath.Join(outDir, "soong", "soong.variables") // dexpreopt.config is an input to the soong_docs action, which runs the @@ -88,6 +89,7 @@ func testForDanglingRules(ctx Context, config Config) { continue } if strings.HasPrefix(line, modulePathsDir) || + strings.HasPrefix(line, rawFilesDir) || line == variablesFilePath || line == dexpreoptConfigFilePath || line == buildDatetimeFilePath || diff --git a/ui/terminal/format.go b/ui/terminal/format.go index 539102390..241a1ddf7 100644 --- a/ui/terminal/format.go +++ b/ui/terminal/format.go @@ -25,7 +25,6 @@ import ( type formatter struct { format string quiet bool - smart bool start time.Time } @@ -33,11 +32,10 @@ type formatter struct { // the terminal in a format similar to Ninja. // format takes nearly all the same options as NINJA_STATUS. // %c is currently unsupported. -func newFormatter(format string, quiet bool, smart bool) formatter { +func newFormatter(format string, quiet bool) formatter { return formatter{ format: format, quiet: quiet, - smart: smart, start: time.Now(), } } @@ -63,9 +61,8 @@ func remainingTimeString(t time.Time) string { func (s formatter) progress(counts status.Counts) string { if s.format == "" { output := fmt.Sprintf("[%3d%% %d/%d", 100*counts.FinishedActions/counts.TotalActions, counts.FinishedActions, counts.TotalActions) - // Not to break parsing logic in the build bot - // TODO(b/313981966): make buildbot more flexible for output format - if s.smart && !counts.EstimatedTime.IsZero() { + + if !counts.EstimatedTime.IsZero() { output += fmt.Sprintf(" %s remaining", remainingTimeString(counts.EstimatedTime)) } output += "] " diff --git a/ui/terminal/status.go b/ui/terminal/status.go index 810e3c93d..2ad174fee 100644 --- a/ui/terminal/status.go +++ b/ui/terminal/status.go @@ -27,10 +27,9 @@ import ( // statusFormat takes nearly all the same options as NINJA_STATUS. // %c is currently unsupported. func NewStatusOutput(w io.Writer, statusFormat string, forceSimpleOutput, quietBuild, forceKeepANSI bool) status.StatusOutput { - useSmartStatus := !forceSimpleOutput && isSmartTerminal(w) - formatter := newFormatter(statusFormat, quietBuild, useSmartStatus) + formatter := newFormatter(statusFormat, quietBuild) - if useSmartStatus { + if !forceSimpleOutput && isSmartTerminal(w) { return NewSmartStatusOutput(w, formatter) } else { return NewSimpleStatusOutput(w, formatter, forceKeepANSI) |