diff options
39 files changed, 508 insertions, 119 deletions
diff --git a/android/Android.bp b/android/Android.bp index 774d24a20..ce27241ad 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -93,6 +93,7 @@ bootstrap_go_package { "register.go", "rule_builder.go", "sandbox.go", + "sbom.go", "sdk.go", "sdk_version.go", "shared_properties.go", diff --git a/android/config.go b/android/config.go index eff9fdd5b..cadc929c9 100644 --- a/android/config.go +++ b/android/config.go @@ -1195,6 +1195,10 @@ func (c *config) UseGoma() bool { return Bool(c.productVariables.UseGoma) } +func (c *config) UseABFS() bool { + return Bool(c.productVariables.UseABFS) +} + func (c *config) UseRBE() bool { return Bool(c.productVariables.UseRBE) } diff --git a/android/defaults.go b/android/defaults.go index ff7900232..c0a2fc68f 100644 --- a/android/defaults.go +++ b/android/defaults.go @@ -28,7 +28,7 @@ type defaultsDependencyTag struct { var DefaultsDepTag defaultsDependencyTag type defaultsProperties struct { - Defaults []string + Defaults proptools.Configurable[[]string] } type DefaultableModuleBase struct { @@ -278,13 +278,14 @@ func RegisterDefaultsPreArchMutators(ctx RegisterMutatorsContext) { func defaultsDepsMutator(ctx BottomUpMutatorContext) { if defaultable, ok := ctx.Module().(Defaultable); ok { - ctx.AddDependency(ctx.Module(), DefaultsDepTag, defaultable.defaults().Defaults...) + ctx.AddDependency(ctx.Module(), DefaultsDepTag, defaultable.defaults().Defaults.GetOrDefault(ctx, nil)...) } } func defaultsMutator(ctx TopDownMutatorContext) { if defaultable, ok := ctx.Module().(Defaultable); ok { - if len(defaultable.defaults().Defaults) > 0 { + defaults := defaultable.defaults().Defaults.GetOrDefault(ctx, nil) + if len(defaults) > 0 { var defaultsList []Defaults seen := make(map[Defaults]bool) @@ -294,7 +295,7 @@ func defaultsMutator(ctx TopDownMutatorContext) { if !seen[defaults] { seen[defaults] = true defaultsList = append(defaultsList, defaults) - return len(defaults.defaults().Defaults) > 0 + return len(defaults.defaults().Defaults.GetOrDefault(ctx, nil)) > 0 } } else { ctx.PropertyErrorf("defaults", "module %s is not an defaults module", diff --git a/android/module.go b/android/module.go index 37e26f9b8..5c2b1e1ea 100644 --- a/android/module.go +++ b/android/module.go @@ -389,7 +389,7 @@ type commonProperties struct { Init_rc []string `android:"arch_variant,path"` // VINTF manifest fragments to be installed if this module is installed - Vintf_fragments []string `android:"path"` + Vintf_fragments proptools.Configurable[[]string] `android:"path"` // names of other modules to install if this module is installed Required proptools.Configurable[[]string] `android:"arch_variant"` @@ -1853,7 +1853,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) } } - m.vintfFragmentsPaths = PathsForModuleSrc(ctx, m.commonProperties.Vintf_fragments) + m.vintfFragmentsPaths = PathsForModuleSrc(ctx, m.commonProperties.Vintf_fragments.GetOrDefault(ctx, nil)) vintfDir := PathForModuleInstall(ctx, "etc", "vintf", "manifest") for _, src := range m.vintfFragmentsPaths { installedVintfFragment := vintfDir.Join(ctx, src.Base()) diff --git a/android/module_test.go b/android/module_test.go index 922ea21fe..829c07987 100644 --- a/android/module_test.go +++ b/android/module_test.go @@ -722,7 +722,6 @@ test { propInfo{Name: "Arch.X86_64.A", Type: "string", Value: "x86_64 a"}, propInfo{Name: "B", Type: "bool", Value: "true"}, propInfo{Name: "C", Type: "string slice", Values: []string{"default_c", "c"}}, - propInfo{Name: "Defaults", Type: "string slice", Values: []string{"foo_defaults"}}, propInfo{Name: "Embedded_prop", Type: "string", Value: "a"}, propInfo{Name: "Name", Type: "string", Value: "foo"}, propInfo{Name: "Nested.E", Type: "string", Value: "nested e"}, @@ -746,7 +745,6 @@ test { foo := result.ModuleForTests("foo", "").Module().base() AssertDeepEquals(t, "foo ", tc.expectedProps, foo.propertiesWithValues()) - }) } } diff --git a/android/sbom.go b/android/sbom.go new file mode 100644 index 000000000..dd2d2fa26 --- /dev/null +++ b/android/sbom.go @@ -0,0 +1,100 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "io" + "path/filepath" + "strings" + + "github.com/google/blueprint" +) + +var ( + // Command line tool to generate SBOM in Soong + genSbom = pctx.HostBinToolVariable("genSbom", "gen_sbom") + + // Command to generate SBOM in Soong. + genSbomRule = pctx.AndroidStaticRule("genSbomRule", blueprint.RuleParams{ + Command: "rm -rf $out && ${genSbom} --output_file ${out} --metadata ${in} --product_out ${productOut} --soong_out ${soongOut} --build_version \"$$(cat ${buildFingerprintFile})\" --product_mfr \"${productManufacturer}\" --json", + CommandDeps: []string{"${genSbom}"}, + }, "productOut", "soongOut", "buildFingerprintFile", "productManufacturer") +) + +func init() { + RegisterSbomSingleton(InitRegistrationContext) +} + +func RegisterSbomSingleton(ctx RegistrationContext) { + ctx.RegisterParallelSingletonType("sbom_singleton", sbomSingletonFactory) +} + +// sbomSingleton is used to generate build actions of generating SBOM of products. +type sbomSingleton struct{} + +func sbomSingletonFactory() Singleton { + return &sbomSingleton{} +} + +// Generates SBOM of products +func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) { + if !ctx.Config().HasDeviceProduct() { + return + } + // Get all METADATA files and add them as implicit input + metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list") + f, err := ctx.Config().fs.Open(metadataFileListFile.String()) + if err != nil { + panic(err) + } + b, err := io.ReadAll(f) + if err != nil { + panic(err) + } + allMetadataFiles := strings.Split(string(b), "\n") + implicits := []Path{metadataFileListFile} + for _, path := range allMetadataFiles { + implicits = append(implicits, PathForSource(ctx, path)) + } + prodVars := ctx.Config().productVariables + buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt") + implicits = append(implicits, buildFingerprintFile) + + // Add installed_files.stamp as implicit input, which depends on all installed files of the product. + installedFilesStamp := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "installed_files.stamp") + implicits = append(implicits, installedFilesStamp) + + metadataDb := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "compliance-metadata.db") + sbomFile := PathForOutput(ctx, "sbom", ctx.Config().DeviceProduct(), "sbom.spdx.json") + ctx.Build(pctx, BuildParams{ + Rule: genSbomRule, + Input: metadataDb, + Implicits: implicits, + Output: sbomFile, + Args: map[string]string{ + "productOut": filepath.Join(ctx.Config().OutDir(), "target", "product", String(prodVars.DeviceName)), + "soongOut": ctx.Config().soongOutDir, + "buildFingerprintFile": buildFingerprintFile.String(), + "productManufacturer": ctx.Config().ProductVariables().ProductManufacturer, + }, + }) + + // Phony rule "soong-sbom". "m soong-sbom" to generate product SBOM in Soong. + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Inputs: []Path{sbomFile}, + Output: PathForPhony(ctx, "soong-sbom"), + }) +} diff --git a/android/variable.go b/android/variable.go index eb0e210c5..df0e59ce1 100644 --- a/android/variable.go +++ b/android/variable.go @@ -294,6 +294,7 @@ type ProductVariables struct { HostStaticBinaries *bool `json:",omitempty"` Binder32bit *bool `json:",omitempty"` UseGoma *bool `json:",omitempty"` + UseABFS *bool `json:",omitempty"` UseRBE *bool `json:",omitempty"` UseRBEJAVAC *bool `json:",omitempty"` UseRBER8 *bool `json:",omitempty"` diff --git a/apex/apex.go b/apex/apex.go index c1a9d7430..fc0500aec 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -86,7 +86,7 @@ type apexBundleProperties struct { // AndroidManifest.xml file used for the zip container of this APEX bundle. If unspecified, // a default one is automatically generated. - AndroidManifest *string `android:"path"` + AndroidManifest proptools.Configurable[string] `android:"path,replace_instead_of_append"` // Determines the file contexts file for setting the security contexts to files in this APEX // bundle. For platform APEXes, this should points to a file under /system/sepolicy Default: @@ -104,7 +104,7 @@ type apexBundleProperties struct { // path_or_glob is a path or glob pattern for a file or set of files, // uid/gid are numerial values of user ID and group ID, mode is octal value // for the file mode, and cap is hexadecimal value for the capability. - Canned_fs_config *string `android:"path"` + Canned_fs_config proptools.Configurable[string] `android:"path,replace_instead_of_append"` ApexNativeDependencies @@ -117,7 +117,8 @@ type apexBundleProperties struct { Bootclasspath_fragments []string // List of systemserverclasspath fragments that are embedded inside this APEX bundle. - Systemserverclasspath_fragments []string + Systemserverclasspath_fragments proptools.Configurable[[]string] + ResolvedSystemserverclasspathFragments []string `blueprint:"mutated"` // List of java libraries that are embedded inside this APEX bundle. Java_libs []string @@ -221,7 +222,8 @@ type ApexNativeDependencies struct { Rust_dyn_libs []string // List of native executables that are embedded inside this APEX. - Binaries []string + Binaries proptools.Configurable[[]string] + ResolvedBinaries []string `blueprint:"mutated"` // List of native tests that are embedded inside this APEX. Tests []string @@ -230,7 +232,8 @@ type ApexNativeDependencies struct { Filesystems []string // List of prebuilt_etcs that are embedded inside this APEX bundle. - Prebuilts []string + Prebuilts proptools.Configurable[[]string] + ResolvedPrebuilts []string `blueprint:"mutated"` // List of native libraries to exclude from this APEX. Exclude_native_shared_libs []string @@ -255,14 +258,14 @@ type ApexNativeDependencies struct { } // Merge combines another ApexNativeDependencies into this one -func (a *ApexNativeDependencies) Merge(b ApexNativeDependencies) { +func (a *ApexNativeDependencies) Merge(ctx android.BaseMutatorContext, b ApexNativeDependencies) { a.Native_shared_libs = append(a.Native_shared_libs, b.Native_shared_libs...) a.Jni_libs = append(a.Jni_libs, b.Jni_libs...) a.Rust_dyn_libs = append(a.Rust_dyn_libs, b.Rust_dyn_libs...) - a.Binaries = append(a.Binaries, b.Binaries...) + a.ResolvedBinaries = append(a.ResolvedBinaries, b.Binaries.GetOrDefault(ctx, nil)...) a.Tests = append(a.Tests, b.Tests...) a.Filesystems = append(a.Filesystems, b.Filesystems...) - a.Prebuilts = append(a.Prebuilts, b.Prebuilts...) + a.ResolvedPrebuilts = append(a.ResolvedPrebuilts, b.Prebuilts.GetOrDefault(ctx, nil)...) a.Exclude_native_shared_libs = append(a.Exclude_native_shared_libs, b.Exclude_native_shared_libs...) a.Exclude_jni_libs = append(a.Exclude_jni_libs, b.Exclude_jni_libs...) @@ -338,10 +341,10 @@ type apexArchBundleProperties struct { // base apex. type overridableProperties struct { // List of APKs that are embedded inside this APEX. - Apps []string + Apps proptools.Configurable[[]string] // List of prebuilt files that are embedded inside this APEX bundle. - Prebuilts []string + Prebuilts proptools.Configurable[[]string] // List of BPF programs inside this APEX bundle. Bpfs []string @@ -715,7 +718,7 @@ func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, nativeM // this module. This is required since arch variant of an APEX bundle is 'common' but it is // 'arm' or 'arm64' for native shared libs. ctx.AddFarVariationDependencies(binVariations, executableTag, - android.RemoveListFromList(nativeModules.Binaries, nativeModules.Exclude_binaries)...) + android.RemoveListFromList(nativeModules.ResolvedBinaries, nativeModules.Exclude_binaries)...) ctx.AddFarVariationDependencies(binVariations, testTag, android.RemoveListFromList(nativeModules.Tests, nativeModules.Exclude_tests)...) ctx.AddFarVariationDependencies(libVariations, jniLibTag, @@ -727,7 +730,7 @@ func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, nativeM ctx.AddFarVariationDependencies(target.Variations(), fsTag, android.RemoveListFromList(nativeModules.Filesystems, nativeModules.Exclude_filesystems)...) ctx.AddFarVariationDependencies(target.Variations(), prebuiltTag, - android.RemoveListFromList(nativeModules.Prebuilts, nativeModules.Exclude_prebuilts)...) + android.RemoveListFromList(nativeModules.ResolvedPrebuilts, nativeModules.Exclude_prebuilts)...) } func (a *apexBundle) combineProperties(ctx android.BottomUpMutatorContext) { @@ -782,20 +785,19 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { // Add native modules targeting both ABIs. When multilib.* is omitted for // native_shared_libs/jni_libs/tests, it implies multilib.both - deps.Merge(a.properties.Multilib.Both) - deps.Merge(ApexNativeDependencies{ + deps.Merge(ctx, a.properties.Multilib.Both) + deps.Merge(ctx, ApexNativeDependencies{ Native_shared_libs: a.properties.Native_shared_libs, Tests: a.properties.Tests, Jni_libs: a.properties.Jni_libs, - Binaries: nil, }) // Add native modules targeting the first ABI When multilib.* is omitted for // binaries, it implies multilib.first isPrimaryAbi := i == 0 if isPrimaryAbi { - deps.Merge(a.properties.Multilib.First) - deps.Merge(ApexNativeDependencies{ + deps.Merge(ctx, a.properties.Multilib.First) + deps.Merge(ctx, ApexNativeDependencies{ Native_shared_libs: nil, Tests: nil, Jni_libs: nil, @@ -806,27 +808,27 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { // Add native modules targeting either 32-bit or 64-bit ABI switch target.Arch.ArchType.Multilib { case "lib32": - deps.Merge(a.properties.Multilib.Lib32) - deps.Merge(a.properties.Multilib.Prefer32) + deps.Merge(ctx, a.properties.Multilib.Lib32) + deps.Merge(ctx, a.properties.Multilib.Prefer32) case "lib64": - deps.Merge(a.properties.Multilib.Lib64) + deps.Merge(ctx, a.properties.Multilib.Lib64) if !has32BitTarget { - deps.Merge(a.properties.Multilib.Prefer32) + deps.Merge(ctx, a.properties.Multilib.Prefer32) } } // Add native modules targeting a specific arch variant switch target.Arch.ArchType { case android.Arm: - deps.Merge(a.archProperties.Arch.Arm.ApexNativeDependencies) + deps.Merge(ctx, a.archProperties.Arch.Arm.ApexNativeDependencies) case android.Arm64: - deps.Merge(a.archProperties.Arch.Arm64.ApexNativeDependencies) + deps.Merge(ctx, a.archProperties.Arch.Arm64.ApexNativeDependencies) case android.Riscv64: - deps.Merge(a.archProperties.Arch.Riscv64.ApexNativeDependencies) + deps.Merge(ctx, a.archProperties.Arch.Riscv64.ApexNativeDependencies) case android.X86: - deps.Merge(a.archProperties.Arch.X86.ApexNativeDependencies) + deps.Merge(ctx, a.archProperties.Arch.X86.ApexNativeDependencies) case android.X86_64: - deps.Merge(a.archProperties.Arch.X86_64.ApexNativeDependencies) + deps.Merge(ctx, a.archProperties.Arch.X86_64.ApexNativeDependencies) default: panic(fmt.Errorf("unsupported arch %v\n", ctx.Arch().ArchType)) } @@ -840,11 +842,13 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { } } + a.properties.ResolvedSystemserverclasspathFragments = a.properties.Systemserverclasspath_fragments.GetOrDefault(ctx, nil) + // Common-arch dependencies come next commonVariation := ctx.Config().AndroidCommonTarget.Variations() ctx.AddFarVariationDependencies(commonVariation, rroTag, a.properties.Rros...) ctx.AddFarVariationDependencies(commonVariation, bcpfTag, a.properties.Bootclasspath_fragments...) - ctx.AddFarVariationDependencies(commonVariation, sscpfTag, a.properties.Systemserverclasspath_fragments...) + ctx.AddFarVariationDependencies(commonVariation, sscpfTag, a.properties.ResolvedSystemserverclasspathFragments...) ctx.AddFarVariationDependencies(commonVariation, javaLibTag, a.properties.Java_libs...) ctx.AddFarVariationDependencies(commonVariation, fsTag, a.properties.Filesystems...) ctx.AddFarVariationDependencies(commonVariation, compatConfigTag, a.properties.Compat_configs...) @@ -857,9 +861,9 @@ func (a *apexBundle) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato } commonVariation := ctx.Config().AndroidCommonTarget.Variations() - ctx.AddFarVariationDependencies(commonVariation, androidAppTag, a.overridableProperties.Apps...) + ctx.AddFarVariationDependencies(commonVariation, androidAppTag, a.overridableProperties.Apps.GetOrDefault(ctx, nil)...) ctx.AddFarVariationDependencies(commonVariation, bpfTag, a.overridableProperties.Bpfs...) - if prebuilts := a.overridableProperties.Prebuilts; len(prebuilts) > 0 { + if prebuilts := a.overridableProperties.Prebuilts.GetOrDefault(ctx, nil); len(prebuilts) > 0 { // For prebuilt_etc, use the first variant (64 on 64/32bit device, 32 on 32bit device) // regardless of the TARGET_PREFER_* setting. See b/144532908 arches := ctx.DeviceConfig().Arches() @@ -1492,7 +1496,6 @@ func (a *apexBundle) AddSanitizerDependencies(ctx android.BottomUpMutatorContext Native_shared_libs: []string{"libclang_rt.hwasan"}, Tests: nil, Jni_libs: nil, - Binaries: nil, }, target, imageVariation) break } @@ -2806,7 +2809,7 @@ func isStaticExecutableAllowed(apex string, exec string) bool { func (a *apexBundle) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, a.properties.Java_libs...) dpInfo.Deps = append(dpInfo.Deps, a.properties.Bootclasspath_fragments...) - dpInfo.Deps = append(dpInfo.Deps, a.properties.Systemserverclasspath_fragments...) + dpInfo.Deps = append(dpInfo.Deps, a.properties.ResolvedSystemserverclasspathFragments...) } var ( diff --git a/apex/apex_test.go b/apex/apex_test.go index f62ee680b..261d2ce59 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -5244,7 +5244,7 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { myApex := ctx.ModuleForTests("myapex", "android_common_myapex").Module() overrideNames := []string{ - "myapex", + "", "myjavalib.myapex", "libfoo.myapex", "libbar.myapex", @@ -11294,13 +11294,6 @@ func TestBootDexJarsMultipleApexPrebuilts(t *testing.T) { // Test that product packaging installs the selected mainline module (either source or a specific prebuilt) // RELEASE_APEX_CONTIRBUTIONS_* build flags will be used to select the correct prebuilt for a specific release config func TestInstallationRulesForMultipleApexPrebuilts(t *testing.T) { - // check that the LOCAL_MODULE in the generated mk file matches the name used in PRODUCT_PACKAGES - // Since the name used in PRODUCT_PACKAGES does not contain prebuilt_ prefix, LOCAL_MODULE should not contain any prefix either - checkLocalModuleName := func(t *testing.T, ctx *android.TestContext, soongApexModuleName string, expectedLocalModuleName string) { - // Variations are created based on apex_name - entries := android.AndroidMkEntriesForTest(t, ctx, ctx.ModuleForTests(soongApexModuleName, "android_common_com.android.foo").Module()) - android.AssertStringEquals(t, "LOCAL_MODULE of the prebuilt apex must match the name listed in PRODUCT_PACKAGES", expectedLocalModuleName, entries[0].EntryMap["LOCAL_MODULE"][0]) - } // for a mainline module family, check that only the flagged soong module is visible to make checkHideFromMake := func(t *testing.T, ctx *android.TestContext, visibleModuleName string, hiddenModuleNames []string) { variation := func(moduleName string) string { @@ -11355,7 +11348,7 @@ func TestInstallationRulesForMultipleApexPrebuilts(t *testing.T) { prebuilt_apex { name: "com.google.android.foo.v2", apex_name: "com.android.foo", - source_apex_name: "com.google.android.foo", // source_apex_name becomes LOCAL_MODULE in the generated mk file + source_apex_name: "com.google.android.foo", src: "com.android.foo-arm.apex", prefer: true, // prefer is set to true on both the prebuilts to induce an error if flagging is not present } @@ -11441,11 +11434,6 @@ func TestInstallationRulesForMultipleApexPrebuilts(t *testing.T) { } ctx := testApex(t, bp, preparer) - // Check that the LOCAL_MODULE of the two prebuilts is com.android.foo - // This ensures that product packaging can pick them for installation if it has been flagged by apex_contributions - checkLocalModuleName(t, ctx, "prebuilt_com.google.android.foo", "com.google.android.foo") - checkLocalModuleName(t, ctx, "prebuilt_com.google.android.foo.v2", "com.google.android.foo") - // Check that // 1. The contents of the selected apex_contributions are visible to make // 2. The rest of the apexes in the mainline module family (source or other prebuilt) is hidden from make diff --git a/apex/builder.go b/apex/builder.go index 763ce4d20..bfe16922d 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -704,8 +704,9 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { optFlags = append(optFlags, "--override_apk_package_name "+manifestPackageName) } - if a.properties.AndroidManifest != nil { - androidManifestFile := android.PathForModuleSrc(ctx, proptools.String(a.properties.AndroidManifest)) + androidManifest := a.properties.AndroidManifest.GetOrDefault(ctx, "") + if androidManifest != "" { + androidManifestFile := android.PathForModuleSrc(ctx, androidManifest) if a.testApex { androidManifestFile = markManifestTestOnly(ctx, androidManifestFile) @@ -1195,8 +1196,9 @@ func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext, defaultReadO } // Custom fs_config is "appended" to the last so that entries from the file are preferred // over default ones set above. - if a.properties.Canned_fs_config != nil { - cmd.Text("cat").Input(android.PathForModuleSrc(ctx, *a.properties.Canned_fs_config)) + customFsConfig := a.properties.Canned_fs_config.GetOrDefault(ctx, "") + if customFsConfig != "" { + cmd.Text("cat").Input(android.PathForModuleSrc(ctx, customFsConfig)) } cmd.Text(")").FlagWithOutput("> ", cannedFsConfig) builder.Build("generateFsConfig", fmt.Sprintf("Generating canned fs config for %s", a.BaseModuleName())) diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 65c23d34b..b9cc09ba7 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -246,7 +246,6 @@ func (p *prebuiltCommon) AndroidMkEntries() []android.AndroidMkEntries { OutputFile: android.OptionalPathForPath(p.outputApex), Include: "$(BUILD_PREBUILT)", Host_required: p.hostRequired, - OverrideName: p.BaseModuleName(), ExtraEntries: []android.AndroidMkExtraEntriesFunc{ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.SetString("LOCAL_MODULE_PATH", p.installDir.String()) diff --git a/bin/installmod b/bin/installmod index 1d0d836ff..1ad5b8412 100755 --- a/bin/installmod +++ b/bin/installmod @@ -28,7 +28,6 @@ if [[ $# -eq 0 ]]; then return 1 fi -local _path _path=$(outmod ${@:$#:1}) if [ $? -ne 0 ]; then return 1 @@ -39,7 +38,7 @@ if [ -z "$_path" ]; then echo "Module '$1' does not produce a file ending with .apk (try 'refreshmod' if there have been build changes?)" >&2 return 1 fi -local serial_device="" +serial_device="" if [[ "$1" == "-s" ]]; then if [[ $# -le 2 ]]; then echo "-s requires an argument" >&2 @@ -48,7 +47,7 @@ if [[ "$1" == "-s" ]]; then serial_device="-s $2" shift 2 fi -local length=$(( $# - 1 )) +length=$(( $# - 1 )) echo adb $serial_device install ${@:1:$length} $_path adb $serial_device install ${@:1:$length} $_path diff --git a/cc/compiler.go b/cc/compiler.go index 03f9899d8..ed1053334 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -53,7 +53,7 @@ type BaseCompilerProperties struct { Cflags proptools.Configurable[[]string] `android:"arch_variant"` // list of module-specific flags that will be used for C++ compiles - Cppflags []string `android:"arch_variant"` + Cppflags proptools.Configurable[[]string] `android:"arch_variant"` // list of module-specific flags that will be used for C compiles Conlyflags []string `android:"arch_variant"` @@ -367,8 +367,9 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps compiler.srcsBeforeGen = append(compiler.srcsBeforeGen, deps.GeneratedSources...) cflags := compiler.Properties.Cflags.GetOrDefault(ctx, nil) + cppflags := compiler.Properties.Cppflags.GetOrDefault(ctx, nil) CheckBadCompilerFlags(ctx, "cflags", cflags) - CheckBadCompilerFlags(ctx, "cppflags", compiler.Properties.Cppflags) + CheckBadCompilerFlags(ctx, "cppflags", cppflags) CheckBadCompilerFlags(ctx, "conlyflags", compiler.Properties.Conlyflags) CheckBadCompilerFlags(ctx, "asflags", compiler.Properties.Asflags) CheckBadCompilerFlags(ctx, "vendor.cflags", compiler.Properties.Target.Vendor.Cflags) @@ -381,7 +382,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps esc := proptools.NinjaAndShellEscapeList flags.Local.CFlags = append(flags.Local.CFlags, esc(cflags)...) - flags.Local.CppFlags = append(flags.Local.CppFlags, esc(compiler.Properties.Cppflags)...) + flags.Local.CppFlags = append(flags.Local.CppFlags, esc(cppflags)...) flags.Local.ConlyFlags = append(flags.Local.ConlyFlags, esc(compiler.Properties.Conlyflags)...) flags.Local.AsFlags = append(flags.Local.AsFlags, esc(compiler.Properties.Asflags)...) flags.Local.YasmFlags = append(flags.Local.YasmFlags, esc(compiler.Properties.Asflags)...) @@ -808,7 +809,7 @@ type RustBindgenClangProperties struct { // list of c++ specific clang flags required to correctly interpret the headers. // This is provided primarily to make sure cppflags defined in cc_defaults are pulled in. - Cppflags []string `android:"arch_variant"` + Cppflags proptools.Configurable[[]string] `android:"arch_variant"` // C standard version to use. Can be a specific version (such as "gnu11"), // "experimental" (which will use draft versions like C1x when available), diff --git a/cc/library.go b/cc/library.go index ff21cc3f6..092b17779 100644 --- a/cc/library.go +++ b/cc/library.go @@ -64,7 +64,7 @@ type LibraryProperties struct { Stubs struct { // Relative path to the symbol map. The symbol map provides the list of // symbols that are exported for stubs variant of this library. - Symbol_file *string `android:"path"` + Symbol_file *string `android:"path,arch_variant"` // List versions to generate stubs libs for. The version name "current" is always // implicitly added. @@ -75,7 +75,7 @@ type LibraryProperties struct { // implementation is made available by some other means, e.g. in a Microdroid // virtual machine. Implementation_installable *bool - } + } `android:"arch_variant"` // set the name of the output Stem *string `android:"arch_variant"` @@ -118,7 +118,7 @@ type LibraryProperties struct { // If this is an LLNDK library, properties to describe the LLNDK stubs. Will be copied from // the module pointed to by llndk_stubs if it is set. - Llndk llndkLibraryProperties + Llndk llndkLibraryProperties `android:"arch_variant"` // If this is a vendor public library, properties to describe the vendor public library stubs. Vendor_public_library vendorPublicLibraryProperties diff --git a/cc/llndk_library.go b/cc/llndk_library.go index 85c3edf8b..5ece78a83 100644 --- a/cc/llndk_library.go +++ b/cc/llndk_library.go @@ -30,7 +30,7 @@ var ( type llndkLibraryProperties struct { // Relative path to the symbol map. // An example file can be seen here: TODO(danalbert): Make an example. - Symbol_file *string + Symbol_file *string `android:"path,arch_variant"` // Whether to export any headers as -isystem instead of -I. Mainly for use by // bionic/libc. diff --git a/cc/proto.go b/cc/proto.go index 4d72f2665..93142b9fe 100644 --- a/cc/proto.go +++ b/cc/proto.go @@ -19,6 +19,8 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" + + "strings" ) const ( @@ -35,13 +37,21 @@ func genProto(ctx android.ModuleContext, protoFile android.Path, flags builderFl srcSuffix = ".c" } + srcInfix := "pb" + for _, value := range flags.proto.Flags { + if strings.HasPrefix(value, "--plugin=") && strings.HasSuffix(value, "protoc-gen-grpc-cpp-plugin") { + srcInfix = "grpc.pb" + break + } + } + if flags.proto.CanonicalPathFromRoot { - ccFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb"+srcSuffix) - headerFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb.h") + ccFile = android.GenPathWithExt(ctx, "proto", protoFile, srcInfix+srcSuffix) + headerFile = android.GenPathWithExt(ctx, "proto", protoFile, srcInfix+".h") } else { rel := protoFile.Rel() - ccFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb"+srcSuffix)) - headerFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb.h")) + ccFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, srcInfix+srcSuffix)) + headerFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, srcInfix+".h")) } protoDeps := flags.proto.Deps diff --git a/cc/proto_test.go b/cc/proto_test.go index abcb27304..a905ea889 100644 --- a/cc/proto_test.go +++ b/cc/proto_test.go @@ -68,4 +68,36 @@ func TestProto(t *testing.T) { } }) + t.Run("grpc-cpp-plugin", func(t *testing.T) { + ctx := testCc(t, ` + cc_binary_host { + name: "protoc-gen-grpc-cpp-plugin", + stl: "none", + } + + cc_library_shared { + name: "libgrpc", + srcs: ["a.proto"], + proto: { + plugin: "grpc-cpp-plugin", + }, + }`) + + buildOS := ctx.Config().BuildOS.String() + + proto := ctx.ModuleForTests("libgrpc", "android_arm_armv7-a-neon_shared").Output("proto/a.grpc.pb.cc") + grpcCppPlugin := ctx.ModuleForTests("protoc-gen-grpc-cpp-plugin", buildOS+"_x86_64") + + cmd := proto.RuleParams.Command + if w := "--grpc-cpp-plugin_out="; !strings.Contains(cmd, w) { + t.Errorf("expected %q in %q", w, cmd) + } + + grpcCppPluginPath := grpcCppPlugin.Module().(android.HostToolProvider).HostToolPath().RelativeToTop().String() + + if w := "--plugin=protoc-gen-grpc-cpp-plugin=" + grpcCppPluginPath; !strings.Contains(cmd, w) { + t.Errorf("expected %q in %q", w, cmd) + } + }) + } diff --git a/filesystem/vbmeta.go b/filesystem/vbmeta.go index 0c6e7f428..3a9a64daa 100644 --- a/filesystem/vbmeta.go +++ b/filesystem/vbmeta.go @@ -59,7 +59,7 @@ type vbmetaProperties struct { // List of filesystem modules that this vbmeta has descriptors for. The filesystem modules // have to be signed (use_avb: true). - Partitions []string + Partitions proptools.Configurable[[]string] // List of chained partitions that this vbmeta deletages the verification. Chained_partitions []chainedPartitionProperties @@ -110,7 +110,7 @@ type vbmetaDep struct { var vbmetaPartitionDep = vbmetaDep{kind: "partition"} func (v *vbmeta) DepsMutator(ctx android.BottomUpMutatorContext) { - ctx.AddDependency(ctx.Module(), vbmetaPartitionDep, v.properties.Partitions...) + ctx.AddDependency(ctx.Module(), vbmetaPartitionDep, v.properties.Partitions.GetOrDefault(ctx, nil)...) } func (v *vbmeta) installFileName() string { diff --git a/genrule/genrule.go b/genrule/genrule.go index 5b40768b4..25579220d 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -139,7 +139,8 @@ type generatorProperties struct { Export_include_dirs []string // list of input files - Srcs []string `android:"path,arch_variant"` + Srcs proptools.Configurable[[]string] `android:"path,arch_variant"` + ResolvedSrcs []string `blueprint:"mutated"` // input files to exclude Exclude_srcs []string `android:"path,arch_variant"` @@ -382,7 +383,8 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { } return srcFiles } - srcFiles := addLabelsForInputs("srcs", g.properties.Srcs, g.properties.Exclude_srcs) + g.properties.ResolvedSrcs = g.properties.Srcs.GetOrDefault(ctx, nil) + srcFiles := addLabelsForInputs("srcs", g.properties.ResolvedSrcs, g.properties.Exclude_srcs) android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: srcFiles.Strings()}) var copyFrom android.Paths @@ -589,7 +591,7 @@ func (g *Module) setOutputFiles(ctx android.ModuleContext) { // Collect information for opening IDE project files in java/jdeps.go. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...) - for _, src := range g.properties.Srcs { + for _, src := range g.properties.ResolvedSrcs { if strings.HasPrefix(src, ":") { src = strings.Trim(src, ":") dpInfo.Deps = append(dpInfo.Deps, src) diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index fba9aec65..444aedb90 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -694,7 +694,7 @@ func TestGenruleDefaults(t *testing.T) { android.AssertStringEquals(t, "cmd", expectedCmd, gen.rawCommands[0]) expectedSrcs := []string{"in1"} - android.AssertDeepEquals(t, "srcs", expectedSrcs, gen.properties.Srcs) + android.AssertDeepEquals(t, "srcs", expectedSrcs, gen.properties.ResolvedSrcs) } func TestGenruleAllowMissingDependencies(t *testing.T) { diff --git a/java/app_import.go b/java/app_import.go index fa87997cf..045a89a34 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -431,6 +431,9 @@ func (a *AndroidAppImport) validatePresignedApk(ctx android.ModuleContext, srcAp var extraArgs []string if a.Privileged() { extraArgs = append(extraArgs, "--privileged") + if ctx.Config().UncompressPrivAppDex() { + extraArgs = append(extraArgs, "--uncompress-priv-app-dex") + } } if proptools.Bool(a.properties.Skip_preprocessed_apk_checks) { extraArgs = append(extraArgs, "--skip-preprocessed-apk-checks") diff --git a/java/app_import_test.go b/java/app_import_test.go index 496fc1308..54a5e7518 100644 --- a/java/app_import_test.go +++ b/java/app_import_test.go @@ -777,30 +777,79 @@ func TestAndroidTestImport_Preprocessed(t *testing.T) { } func TestAndroidAppImport_Preprocessed(t *testing.T) { - ctx, _ := testJava(t, ` - android_app_import { - name: "foo", - apk: "prebuilts/apk/app.apk", - presigned: true, - preprocessed: true, - } - `) + for _, dontUncompressPrivAppDexs := range []bool{false, true} { + name := fmt.Sprintf("dontUncompressPrivAppDexs:%t", dontUncompressPrivAppDexs) + t.Run(name, func(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.UncompressPrivAppDex = proptools.BoolPtr(!dontUncompressPrivAppDexs) + }), + ).RunTestWithBp(t, ` + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + presigned: true, + preprocessed: true, + } - apkName := "foo.apk" - variant := ctx.ModuleForTests("foo", "android_common") - outputBuildParams := variant.Output(apkName).BuildParams - if outputBuildParams.Rule.String() != android.Cp.String() { - t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String()) - } + android_app_import { + name: "bar", + apk: "prebuilts/apk/app.apk", + presigned: true, + privileged: true, + preprocessed: true, + } + `) + + // non-privileged app + apkName := "foo.apk" + variant := result.ModuleForTests("foo", "android_common") + outputBuildParams := variant.Output(apkName).BuildParams + if outputBuildParams.Rule.String() != android.Cp.String() { + t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String()) + } - // Make sure compression and aligning were validated. - if outputBuildParams.Validation == nil { - t.Errorf("Expected validation rule, but was not found") - } + // Make sure compression and aligning were validated. + if outputBuildParams.Validation == nil { + t.Errorf("Expected validation rule, but was not found") + } + + validationBuildParams := variant.Output("validated-prebuilt/check.stamp").BuildParams + if validationBuildParams.Rule.String() != checkPresignedApkRule.String() { + t.Errorf("Unexpected validation rule: " + validationBuildParams.Rule.String()) + } - validationBuildParams := variant.Output("validated-prebuilt/check.stamp").BuildParams - if validationBuildParams.Rule.String() != checkPresignedApkRule.String() { - t.Errorf("Unexpected validation rule: " + validationBuildParams.Rule.String()) + expectedScriptArgs := "--preprocessed" + actualScriptArgs := validationBuildParams.Args["extraArgs"] + android.AssertStringEquals(t, "check script extraArgs", expectedScriptArgs, actualScriptArgs) + + // privileged app + apkName = "bar.apk" + variant = result.ModuleForTests("bar", "android_common") + outputBuildParams = variant.Output(apkName).BuildParams + if outputBuildParams.Rule.String() != android.Cp.String() { + t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String()) + } + + // Make sure compression and aligning were validated. + if outputBuildParams.Validation == nil { + t.Errorf("Expected validation rule, but was not found") + } + + validationBuildParams = variant.Output("validated-prebuilt/check.stamp").BuildParams + if validationBuildParams.Rule.String() != checkPresignedApkRule.String() { + t.Errorf("Unexpected validation rule: " + validationBuildParams.Rule.String()) + } + + expectedScriptArgs = "--privileged" + if !dontUncompressPrivAppDexs { + expectedScriptArgs += " --uncompress-priv-app-dex" + } + expectedScriptArgs += " --preprocessed" + actualScriptArgs = validationBuildParams.Args["extraArgs"] + android.AssertStringEquals(t, "check script extraArgs", expectedScriptArgs, actualScriptArgs) + }) } } diff --git a/java/ravenwood.go b/java/ravenwood.go index 908619d5f..84c285cc7 100644 --- a/java/ravenwood.go +++ b/java/ravenwood.go @@ -33,6 +33,8 @@ func RegisterRavenwoodBuildComponents(ctx android.RegistrationContext) { var ravenwoodLibContentTag = dependencyTag{name: "ravenwoodlibcontent"} var ravenwoodUtilsTag = dependencyTag{name: "ravenwoodutils"} var ravenwoodRuntimeTag = dependencyTag{name: "ravenwoodruntime"} +var ravenwoodDataTag = dependencyTag{name: "ravenwooddata"} +var ravenwoodTestResourceApkTag = dependencyTag{name: "ravenwoodtestresapk"} const ravenwoodUtilsName = "ravenwood-utils" const ravenwoodRuntimeName = "ravenwood-runtime" @@ -53,6 +55,13 @@ func getLibPath(archType android.ArchType) string { type ravenwoodTestProperties struct { Jni_libs []string + + // Specify another android_app module here to copy it to the test directory, so that + // the ravenwood test can access it. + // TODO: For now, we simply refer to another android_app module and copy it to the + // test directory. Eventually, android_ravenwood_test should support all the resource + // related properties and build resources from the `res/` directory. + Resource_apk *string } type ravenwoodTest struct { @@ -114,6 +123,11 @@ func (r *ravenwoodTest) DepsMutator(ctx android.BottomUpMutatorContext) { for _, lib := range r.ravenwoodTestProperties.Jni_libs { ctx.AddVariationDependencies(ctx.Config().BuildOSTarget.Variations(), jniLibTag, lib) } + + // Resources APK + if resourceApk := proptools.String(r.ravenwoodTestProperties.Resource_apk); resourceApk != "" { + ctx.AddVariationDependencies(nil, ravenwoodTestResourceApkTag, resourceApk) + } } func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -175,6 +189,14 @@ func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { installDeps = append(installDeps, installJni) } + resApkInstallPath := installPath.Join(ctx, "ravenwood-res-apks") + if resApk := ctx.GetDirectDepsWithTag(ravenwoodTestResourceApkTag); len(resApk) > 0 { + for _, installFile := range resApk[0].FilesToInstall() { + installResApk := ctx.InstallFile(resApkInstallPath, "ravenwood-res.apk", installFile) + installDeps = append(installDeps, installResApk) + } + } + // Install our JAR with all dependencies ctx.InstallFile(installPath, ctx.ModuleName()+".jar", r.outputFile, installDeps...) } @@ -198,6 +220,9 @@ type ravenwoodLibgroupProperties struct { Libs []string Jni_libs []string + + // We use this to copy framework-res.apk to the ravenwood runtime directory. + Data []string } type ravenwoodLibgroup struct { @@ -236,6 +261,9 @@ func (r *ravenwoodLibgroup) DepsMutator(ctx android.BottomUpMutatorContext) { for _, lib := range r.ravenwoodLibgroupProperties.Jni_libs { ctx.AddVariationDependencies(ctx.Config().BuildOSTarget.Variations(), jniLibTag, lib) } + for _, data := range r.ravenwoodLibgroupProperties.Data { + ctx.AddVariationDependencies(nil, ravenwoodDataTag, data) + } } func (r *ravenwoodLibgroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -266,6 +294,13 @@ func (r *ravenwoodLibgroup) GenerateAndroidBuildActions(ctx android.ModuleContex ctx.InstallFile(soInstallPath, jniLib.path.Base(), jniLib.path) } + dataInstallPath := installPath.Join(ctx, "ravenwood-data") + for _, data := range r.ravenwoodLibgroupProperties.Data { + libModule := ctx.GetDirectDepWithTag(data, ravenwoodDataTag) + file := android.OutputFileForModule(ctx, libModule, "") + ctx.InstallFile(dataInstallPath, file.Base(), file) + } + // Normal build should perform install steps ctx.Phony(r.BaseModuleName(), android.PathForPhony(ctx, r.BaseModuleName()+"-install")) } diff --git a/java/ravenwood_test.go b/java/ravenwood_test.go index 59612645c..d26db930d 100644 --- a/java/ravenwood_test.go +++ b/java/ravenwood_test.go @@ -57,6 +57,14 @@ var prepareRavenwoodRuntime = android.GroupFixturePreparers( name: "framework-rules.ravenwood", srcs: ["Rules.java"], } + android_app { + name: "app1", + sdk_version: "current", + } + android_app { + name: "app2", + sdk_version: "current", + } android_ravenwood_libgroup { name: "ravenwood-runtime", libs: [ @@ -67,6 +75,9 @@ var prepareRavenwoodRuntime = android.GroupFixturePreparers( "ravenwood-runtime-jni1", "ravenwood-runtime-jni2", ], + data: [ + "app1", + ], } android_ravenwood_libgroup { name: "ravenwood-utils", @@ -102,6 +113,7 @@ func TestRavenwoodRuntime(t *testing.T) { runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/ravenwood-runtime-jni1.so") runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/libred.so") runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/ravenwood-runtime-jni3.so") + runtime.Output(installPathPrefix + "/ravenwood-runtime/ravenwood-data/app1.apk") utils := ctx.ModuleForTests("ravenwood-utils", "android_common") utils.Output(installPathPrefix + "/ravenwood-utils/framework-rules.ravenwood.jar") } @@ -143,6 +155,7 @@ func TestRavenwoodTest(t *testing.T) { "jni-lib2", "ravenwood-runtime-jni2", ], + resource_apk: "app2", sdk_version: "test_current", } `) @@ -169,6 +182,7 @@ func TestRavenwoodTest(t *testing.T) { module.Output(installPathPrefix + "/ravenwood-test/lib64/jni-lib1.so") module.Output(installPathPrefix + "/ravenwood-test/lib64/libblue.so") module.Output(installPathPrefix + "/ravenwood-test/lib64/libpink.so") + module.Output(installPathPrefix + "/ravenwood-test/ravenwood-res-apks/ravenwood-res.apk") // ravenwood-runtime*.so are included in the runtime, so it shouldn't be emitted. for _, o := range module.AllOutputs() { diff --git a/phony/phony.go b/phony/phony.go index b421176be..807b95b32 100644 --- a/phony/phony.go +++ b/phony/phony.go @@ -20,6 +20,8 @@ import ( "strings" "android/soong/android" + + "github.com/google/blueprint/proptools" ) func init() { @@ -88,14 +90,15 @@ type PhonyRule struct { android.ModuleBase android.DefaultableModuleBase - properties PhonyProperties + phonyDepsModuleNames []string + properties PhonyProperties } type PhonyProperties struct { // The Phony_deps is the set of all dependencies for this target, // and it can function similarly to .PHONY in a makefile. // Additionally, dependencies within it can even include genrule. - Phony_deps []string + Phony_deps proptools.Configurable[[]string] } // The phony_rule provides functionality similar to the .PHONY in a makefile. @@ -109,13 +112,14 @@ func PhonyRuleFactory() android.Module { } func (p *PhonyRule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + p.phonyDepsModuleNames = p.properties.Phony_deps.GetOrDefault(ctx, nil) } func (p *PhonyRule) AndroidMk() android.AndroidMkData { return android.AndroidMkData{ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { - if len(p.properties.Phony_deps) > 0 { - depModulesStr := strings.Join(p.properties.Phony_deps, " ") + if len(p.phonyDepsModuleNames) > 0 { + depModulesStr := strings.Join(p.phonyDepsModuleNames, " ") fmt.Fprintln(w, ".PHONY:", name) fmt.Fprintln(w, name, ":", depModulesStr) } diff --git a/rust/bindgen.go b/rust/bindgen.go index f1579ccbe..d412ea1cc 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -285,7 +285,7 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr if isCpp { cflags = append(cflags, "-x c++") // Add any C++ only flags. - cflags = append(cflags, esc(b.ClangProperties.Cppflags)...) + cflags = append(cflags, esc(b.ClangProperties.Cppflags.GetOrDefault(ctx, nil))...) } else { cflags = append(cflags, "-x c") } diff --git a/scripts/check_prebuilt_presigned_apk.py b/scripts/check_prebuilt_presigned_apk.py index abedfb770..abab2e146 100755 --- a/scripts/check_prebuilt_presigned_apk.py +++ b/scripts/check_prebuilt_presigned_apk.py @@ -37,7 +37,7 @@ def has_preprocessed_issues(args, *, fail=False): sys.exit(args.apk + ': Contains compressed JNI libraries') return True # It's ok for non-privileged apps to have compressed dex files, see go/gms-uncompressed-jni-slides - if args.privileged: + if args.privileged and args.uncompress_priv_app_dex: if info.filename.endswith('.dex') and info.compress_type != zipfile.ZIP_STORED: if fail: sys.exit(args.apk + ': Contains compressed dex files and is privileged') @@ -52,6 +52,7 @@ def main(): parser.add_argument('--skip-preprocessed-apk-checks', action = 'store_true', help = "the value of the soong property with the same name") parser.add_argument('--preprocessed', action = 'store_true', help = "the value of the soong property with the same name") parser.add_argument('--privileged', action = 'store_true', help = "the value of the soong property with the same name") + parser.add_argument('--uncompress-priv-app-dex', action = 'store_true', help = "the value of the product variable with the same name") parser.add_argument('apk', help = "the apk to check") parser.add_argument('stampfile', help = "a file to touch if successful") args = parser.parse_args() diff --git a/tests/run_tool_with_logging_test.py b/tests/run_tool_with_logging_test.py index 57a6d6296..1a946a1f8 100644 --- a/tests/run_tool_with_logging_test.py +++ b/tests/run_tool_with_logging_test.py @@ -193,7 +193,7 @@ class RunToolWithLoggingTest(unittest.TestCase): logger_path = self._import_executable("tool_event_logger") self._run_script_and_wait(f""" - TMPDIR="{self.working_dir.name}" + export TMPDIR="{self.working_dir.name}" export ANDROID_TOOL_LOGGER="{logger_path}" export ANDROID_TOOL_LOGGER_EXTRA_ARGS="--dry_run" {self.logging_script_path} "FAKE_TOOL" {test_tool.executable} arg1 arg2 @@ -206,7 +206,7 @@ class RunToolWithLoggingTest(unittest.TestCase): logger_path = self._import_executable("tool_event_logger") self._run_script_and_wait(f""" - TMPDIR="{self.working_dir.name}" + export TMPDIR="{self.working_dir.name}" export ANDROID_TOOL_LOGGER="{logger_path}" export ANDROID_TOOL_LOGGER_EXTRA_ARGS="--dry_run" {self.logging_script_path} "FAKE_TOOL" {test_tool.executable} --tool-arg1 diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh index 8dc16301c..794003de8 100755 --- a/tests/sbom_test.sh +++ b/tests/sbom_test.sh @@ -70,13 +70,14 @@ function test_sbom_aosp_cf_x86_64_phone { # m droid, build sbom later in case additional dependencies might be built and included in partition images. run_soong "${out_dir}" "droid dump.erofs lz4" + soong_sbom_out=$out_dir/soong/sbom/$target_product product_out=$out_dir/target/product/vsoc_x86_64 sbom_test=$product_out/sbom_test mkdir -p $sbom_test cp $product_out/*.img $sbom_test - # m sbom - run_soong "${out_dir}" sbom + # m sbom soong-sbom + run_soong "${out_dir}" "sbom soong-sbom" # Generate installed file list from .img files in PRODUCT_OUT dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs @@ -118,6 +119,7 @@ function test_sbom_aosp_cf_x86_64_phone { partition_name=$(basename $f | cut -d. -f1) file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" + files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt" rm "$file_list_file" > /dev/null 2>&1 || true all_dirs="/" while [ ! -z "$all_dirs" ]; do @@ -145,6 +147,7 @@ function test_sbom_aosp_cf_x86_64_phone { done sort -n -o "$file_list_file" "$file_list_file" + # Diff the file list from image and file list in SBOM created by Make grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file" if [ "$partition_name" = "system" ]; then # system partition is mounted to /, so include FileName starts with /root/ too. @@ -154,6 +157,17 @@ function test_sbom_aosp_cf_x86_64_phone { echo ============ Diffing files in $f and SBOM diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" "" + + # Diff the file list from image and file list in SBOM created by Soong + grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file" + if [ "$partition_name" = "system" ]; then + # system partition is mounted to /, so include FileName starts with /root/ too. + grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file" + fi + sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file" + + echo ============ Diffing files in $f and SBOM created by Soong + diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" "" done RAMDISK_IMAGES="$product_out/ramdisk.img" @@ -161,6 +175,7 @@ function test_sbom_aosp_cf_x86_64_phone { partition_name=$(basename $f | cut -d. -f1) file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" + files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt" # lz4 decompress $f to stdout # cpio list all entries like ls -l # grep filter normal files and symlinks @@ -170,11 +185,19 @@ function test_sbom_aosp_cf_x86_64_phone { grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file" + grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file" + echo ============ Diffing files in $f and SBOM diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" "" + + echo ============ Diffing files in $f and SBOM created by Soong + diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" "" done verify_package_verification_code "$product_out/sbom.spdx" + verify_package_verification_code "$soong_sbom_out/sbom.spdx" + + verify_packages_licenses "$soong_sbom_out/sbom.spdx" # Teardown cleanup "${out_dir}" @@ -213,6 +236,41 @@ function verify_package_verification_code { fi } +function verify_packages_licenses { + local sbom_file="$1"; shift + + num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l) + num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l) + if [ "$num_of_packages" = "$num_of_declared_licenses" ] + then + echo "Number of packages with declared license is correct." + else + echo "Number of packages with declared license is WRONG." + exit 1 + fi + + # PRODUCT and 7 prebuilt packages have "PackageLicenseDeclared: NOASSERTION" + # All other packages have declared licenses + num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l) + if [ $num_of_packages_with_noassertion_license = 15 ] + then + echo "Number of packages with NOASSERTION license is correct." + else + echo "Number of packages with NOASSERTION license is WRONG." + exit 1 + fi + + num_of_files=$(grep 'FileName:' $sbom_file | wc -l) + num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l) + if [ "$num_of_files" = "$num_of_concluded_licenses" ] + then + echo "Number of files with concluded license is correct." + else + echo "Number of files with concluded license is WRONG." + exit 1 + fi +} + function test_sbom_unbundled_apex { # Setup out_dir="$(setup)" @@ -274,7 +332,7 @@ function test_sbom_unbundled_apk { target_product=aosp_cf_x86_64_phone target_release=trunk_staging -target_build_variant=userdebug +target_build_variant=eng for i in "$@"; do case $i in TARGET_PRODUCT=*) diff --git a/ui/build/build.go b/ui/build/build.go index c7319ed33..49ac791ce 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -211,9 +211,38 @@ func checkRAM(ctx Context, config Config) { } } +func abfsBuildStarted(ctx Context, config Config) { + abfsBox := config.PrebuiltBuildTool("abfsbox") + cmdArgs := []string{"build-started", "--"} + cmdArgs = append(cmdArgs, config.Arguments()...) + cmd := Command(ctx, config, "abfsbox", abfsBox, cmdArgs...) + cmd.Sandbox = noSandbox + cmd.RunAndPrintOrFatal() +} + +func abfsBuildFinished(ctx Context, config Config, finished bool) { + var errMsg string + if !finished { + errMsg = "build was interrupted" + } + abfsBox := config.PrebuiltBuildTool("abfsbox") + cmdArgs := []string{"build-finished", "-e", errMsg, "--"} + cmdArgs = append(cmdArgs, config.Arguments()...) + cmd := Command(ctx, config, "abfsbox", abfsBox, cmdArgs...) + cmd.RunAndPrintOrFatal() +} + // Build the tree. Various flags in `config` govern which components of // the build to run. func Build(ctx Context, config Config) { + done := false + if config.UseABFS() { + abfsBuildStarted(ctx, config) + defer func() { + abfsBuildFinished(ctx, config, done) + }() + } + ctx.Verboseln("Starting build with args:", config.Arguments()) ctx.Verboseln("Environment:", config.Environment().Environ()) @@ -351,6 +380,7 @@ func Build(ctx Context, config Config) { if what&RunDistActions != 0 { runDistActions(ctx, config) } + done = true } func updateBuildIdDir(ctx Context, config Config) { diff --git a/ui/build/config.go b/ui/build/config.go index 2470f843d..631b76f03 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -1270,9 +1270,25 @@ func (c *configImpl) canSupportRBE() bool { if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") { return false } + if c.UseABFS() { + return false + } return true } +func (c *configImpl) UseABFS() bool { + if v, ok := c.environ.Get("NO_ABFS"); ok { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "true" || v == "1" { + return false + } + } + + abfsBox := c.PrebuiltBuildTool("abfsbox") + err := exec.Command(abfsBox, "hash", srcDirFileCheck).Run() + return err == nil +} + func (c *configImpl) UseRBE() bool { // These alternate modes of running Soong do not use RBE / reclient. if c.Queryview() || c.JsonModuleGraph() { @@ -1585,6 +1601,23 @@ func (c *configImpl) HostPrebuiltTag() string { } } +func (c *configImpl) KatiBin() string { + binName := "ckati" + if c.UseABFS() { + binName = "ckati-wrap" + } + + return c.PrebuiltBuildTool(binName) +} + +func (c *configImpl) NinjaBin() string { + binName := "ninja" + if c.UseABFS() { + binName = "ninjago" + } + return c.PrebuiltBuildTool(binName) +} + func (c *configImpl) PrebuiltBuildTool(name string) string { if v, ok := c.environ.Get("SANITIZE_HOST"); ok { if sanitize := strings.Fields(v); inList("address", sanitize) { diff --git a/ui/build/config_test.go b/ui/build/config_test.go index b1222fe93..b42edb0c6 100644 --- a/ui/build/config_test.go +++ b/ui/build/config_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "strings" "testing" @@ -1043,12 +1044,13 @@ func TestBuildConfig(t *testing.T) { }, }, { + // RBE is only supported on linux. name: "use rbe", environ: Environment{"USE_RBE=1"}, expectedBuildConfig: &smpb.BuildConfig{ ForceUseGoma: proto.Bool(false), UseGoma: proto.Bool(false), - UseRbe: proto.Bool(true), + UseRbe: proto.Bool(runtime.GOOS == "linux"), NinjaWeightListSource: smpb.BuildConfig_NOT_USED.Enum(), }, }, diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index e77df44a5..5df3a959b 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -93,7 +93,7 @@ func dumpMakeVars(ctx Context, config Config, goals, vars []string, write_soong_ defer tool.Finish() cmd := Command(ctx, config, "dumpvars", - config.PrebuiltBuildTool("ckati"), + config.KatiBin(), "-f", "build/make/core/config.mk", "--color_warnings", "--kati_stats", diff --git a/ui/build/kati.go b/ui/build/kati.go index d599c99a7..a0efd2c88 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -84,7 +84,7 @@ func writeValueIfChanged(ctx Context, config Config, dir string, filename string // arguments, and a custom function closure to mutate the environment Kati runs // in. func runKati(ctx Context, config Config, extraSuffix string, args []string, envFunc func(*Environment)) { - executable := config.PrebuiltBuildTool("ckati") + executable := config.KatiBin() // cKati arguments. args = append([]string{ // Instead of executing commands directly, generate a Ninja file. diff --git a/ui/build/ninja.go b/ui/build/ninja.go index 1935e7210..b5e74b422 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -49,7 +49,7 @@ func runNinjaForBuild(ctx Context, config Config) { nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo) defer nr.Close() - executable := config.PrebuiltBuildTool("ninja") + executable := config.NinjaBin() args := []string{ "-d", "keepdepfile", "-d", "keeprsp", diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index 266f76b35..d1b8e2644 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" "testing" @@ -26,6 +27,10 @@ import ( ) func TestDumpRBEMetrics(t *testing.T) { + // RBE is only supported on linux. + if runtime.GOOS != "linux" { + t.Skip("RBE is only supported on linux") + } ctx := testContext() tests := []struct { description string @@ -82,6 +87,10 @@ func TestDumpRBEMetrics(t *testing.T) { } func TestDumpRBEMetricsErrors(t *testing.T) { + // RBE is only supported on linux. + if runtime.GOOS != "linux" { + t.Skip("RBE is only supported on linux") + } ctx := testContext() tests := []struct { description string diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go index edb3b66d4..5c3fec15e 100644 --- a/ui/build/sandbox_linux.go +++ b/ui/build/sandbox_linux.go @@ -200,6 +200,9 @@ func (c *Cmd) wrapSandbox() { // Only log important warnings / errors "-q", } + if c.config.UseABFS() { + sandboxArgs = append(sandboxArgs, "-B", "{ABFS_DIR}") + } // Mount srcDir RW allowlists as Read-Write if len(c.config.sandboxConfig.SrcDirRWAllowlist()) > 0 && !c.config.sandboxConfig.SrcDirIsRO() { diff --git a/ui/build/soong.go b/ui/build/soong.go index e18cc2579..2ccdfecd8 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -661,7 +661,7 @@ func runSoong(ctx Context, config Config) { } ninjaArgs = append(ninjaArgs, targets...) - ninjaCmd := config.PrebuiltBuildTool("ninja") + ninjaCmd := config.NinjaBin() if config.useN2 { ninjaCmd = config.PrebuiltBuildTool("n2") } diff --git a/ui/build/test_build.go b/ui/build/test_build.go index 687ad6f61..3faa94d95 100644 --- a/ui/build/test_build.go +++ b/ui/build/test_build.go @@ -15,14 +15,16 @@ package build import ( - "android/soong/ui/metrics" - "android/soong/ui/status" "bufio" "fmt" "path/filepath" + "regexp" "runtime" "sort" "strings" + + "android/soong/ui/metrics" + "android/soong/ui/status" ) // Checks for files in the out directory that have a rule that depends on them but no rule to @@ -84,6 +86,10 @@ func testForDanglingRules(ctx Context, config Config) { // before running soong and ninja. releaseConfigDir := filepath.Join(outDir, "soong", "release-config") + // out/target/product/<xxxxx>/build_fingerprint.txt is a source file created in sysprop.mk + // ^out/target/product/[^/]+/build_fingerprint.txt$ + buildFingerPrintFilePattern := regexp.MustCompile("^" + filepath.Join(outDir, "target", "product") + "/[^/]+/build_fingerprint.txt$") + danglingRules := make(map[string]bool) scanner := bufio.NewScanner(stdout) @@ -100,7 +106,8 @@ func testForDanglingRules(ctx Context, config Config) { line == dexpreoptConfigFilePath || line == buildDatetimeFilePath || line == bpglob || - strings.HasPrefix(line, releaseConfigDir) { + strings.HasPrefix(line, releaseConfigDir) || + buildFingerPrintFilePattern.MatchString(line) { // Leaf node is in one of Soong's bootstrap directories, which do not have // full build rules in the primary build.ninja file. continue |