diff options
50 files changed, 1456 insertions, 315 deletions
@@ -10,10 +10,12 @@ dacek@google.com delmerico@google.com dwillemsen@google.com eakammer@google.com +jihoonkang@google.com jobredeaux@google.com joeo@google.com juu@google.com lamontjones@google.com +mrziwang@google.com spandandas@google.com tradical@google.com usta@google.com @@ -565,6 +565,12 @@ modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced by all of the vendor's other modules using the normal namespace and visibility rules. +`soongConfigTraceMutator` enables modules affected by soong config variables to +write outputs into a hashed directory path. It does this by recording accesses +to soong config variables on each module, and then accumulating records of each +module's all dependencies. `m soong_config_trace` builds information about +hashes to `$OUT_DIR/soong/soong_config_trace.json`. + ## Build logic The build logic is written in Go using the diff --git a/aidl_library/aidl_library.go b/aidl_library/aidl_library.go index 8a84e6bfa..598510386 100644 --- a/aidl_library/aidl_library.go +++ b/aidl_library/aidl_library.go @@ -107,8 +107,10 @@ func (lib *AidlLibrary) ConvertWithBp2build(ctx android.TopDownMutatorContext) { type AidlLibraryInfo struct { // The direct aidl files of the module Srcs android.Paths - // The include dirs to the direct aidl files and those provided from aidl_library deps + // The include dirs to the direct aidl files and those provided from transitive aidl_library deps IncludeDirs android.DepSet + // The direct hdrs and hdrs from transitive deps + Hdrs android.DepSet } // AidlLibraryProvider provides the srcs and the transitive include dirs @@ -116,37 +118,48 @@ var AidlLibraryProvider = blueprint.NewProvider(AidlLibraryInfo{}) func (lib *AidlLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { includeDirsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) + hdrsDepSetBuilder := android.NewDepSetBuilder(android.PREORDER) if len(lib.properties.Srcs) == 0 && len(lib.properties.Hdrs) == 0 { ctx.ModuleErrorf("at least srcs or hdrs prop must be non-empty") } srcs := android.PathsForModuleSrc(ctx, lib.properties.Srcs) + hdrs := android.PathsForModuleSrc(ctx, lib.properties.Hdrs) + if lib.properties.Strip_import_prefix != nil { srcs = android.PathsWithModuleSrcSubDir( ctx, srcs, - android.String(lib.properties.Strip_import_prefix)) + android.String(lib.properties.Strip_import_prefix), + ) + + hdrs = android.PathsWithModuleSrcSubDir( + ctx, + hdrs, + android.String(lib.properties.Strip_import_prefix), + ) } + hdrsDepSetBuilder.Direct(hdrs...) includeDir := android.PathForModuleSrc( ctx, proptools.StringDefault(lib.properties.Strip_import_prefix, ""), ) - includeDirsDepSetBuilder.Direct(includeDir) for _, dep := range ctx.GetDirectDepsWithTag(aidlLibraryTag) { if ctx.OtherModuleHasProvider(dep, AidlLibraryProvider) { info := ctx.OtherModuleProvider(dep, AidlLibraryProvider).(AidlLibraryInfo) includeDirsDepSetBuilder.Transitive(&info.IncludeDirs) + hdrsDepSetBuilder.Transitive(&info.Hdrs) } } - // TODO(b/279960133) Propagate direct and transitive headers/srcs when aidl action sandboxes inputs ctx.SetProvider(AidlLibraryProvider, AidlLibraryInfo{ Srcs: srcs, IncludeDirs: *includeDirsDepSetBuilder.Build(), + Hdrs: *hdrsDepSetBuilder.Build(), }) } diff --git a/aidl_library/aidl_library_test.go b/aidl_library/aidl_library_test.go index d9b410acc..d9dd24515 100644 --- a/aidl_library/aidl_library_test.go +++ b/aidl_library/aidl_library_test.go @@ -37,7 +37,7 @@ func TestAidlLibrary(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], - hdrs: ["Header.aidl"], + hdrs: ["a/Header.aidl"], strip_import_prefix: "a", deps: ["bar"], } @@ -61,6 +61,13 @@ func TestAidlLibrary(t *testing.T) { []string{"package_foo/a/b/Foo.aidl"}, actualInfo.Srcs, ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl hdrs paths", + []string{"package_foo/a/Header.aidl"}, + actualInfo.Hdrs.ToList(), + ) } func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { @@ -72,6 +79,7 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { aidl_library { name: "bar", srcs: ["x/y/Bar.aidl"], + hdrs: ["BarHeader.aidl"], } `), }.AddToFixture(), @@ -80,7 +88,6 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], - hdrs: ["Header.aidl"], deps: ["bar"], } `), @@ -103,6 +110,13 @@ func TestAidlLibraryWithoutStripImportPrefix(t *testing.T) { []string{"package_foo/a/b/Foo.aidl"}, actualInfo.Srcs, ) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl hdrs paths", + []string{"package_bar/BarHeader.aidl"}, + actualInfo.Hdrs.ToList(), + ) } func TestAidlLibraryWithNoSrcsHdrsDeps(t *testing.T) { diff --git a/android/Android.bp b/android/Android.bp index 118087db7..94d2c04f6 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -77,6 +77,7 @@ bootstrap_go_package { "path_properties.go", "paths.go", "phony.go", + "plugin.go", "prebuilt.go", "prebuilt_build_tool.go", "proto.go", diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 751a4cb83..6405e9fe9 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -761,6 +761,11 @@ var ( // aidl "aidl", "libaidl-common", + + // java_resources containing only a single filegroup + "libauto_value_plugin", + "auto_value_plugin_resources", + "auto_value_extension", } Bp2buildModuleTypeAlwaysConvertList = []string{ @@ -835,7 +840,6 @@ var ( "libprotobuf-internal-python-srcs", // TODO(b/210751803), we don't handle path property for filegroups "libprotobuf-java-full", // TODO(b/210751803), we don't handle path property for filegroups "libprotobuf-java-util-full", // TODO(b/210751803), we don't handle path property for filegroups - "auto_value_plugin_resources", // TODO(b/210751803), we don't handle path property for filegroups // go deps: "analyze_bcpf", // depends on bpmodify a blueprint_go_binary. diff --git a/android/bazel.go b/android/bazel.go index 114b1f5c2..d32663483 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -46,6 +46,10 @@ const ( // that is not a platform incompatibility. Example: the module-type is not // enabled, or is not bp2build-converted. ModuleIncompatibility + + // Missing dependencies. We can't query Bazel for modules if it has missing dependencies, there + // will be failures. + ModuleMissingDeps ) // FileGroupAsLibrary describes a filegroup module that is converted to some library @@ -367,16 +371,26 @@ func GetBp2BuildAllowList() Bp2BuildConversionAllowlist { // As a side effect, calling this method will also log whether this module is // mixed build enabled for metrics reporting. func MixedBuildsEnabled(ctx BaseModuleContext) MixedBuildEnabledStatus { - module := ctx.Module() - apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) - withinApex := !apexInfo.IsForPlatform() - platformIncompatible := isPlatformIncompatible(ctx.Os(), ctx.Arch().ArchType) if platformIncompatible { ctx.Config().LogMixedBuild(ctx, false) return TechnicalIncompatibility } + if ctx.Config().AllowMissingDependencies() { + missingDeps := ctx.getMissingDependencies() + // If there are missing dependencies, querying Bazel will fail. Soong instead fails at execution + // time, not loading/analysis. disable mixed builds and fall back to Soong to maintain that + // behavior. + if len(missingDeps) > 0 { + ctx.Config().LogMixedBuild(ctx, false) + return ModuleMissingDeps + } + } + + module := ctx.Module() + apexInfo := ctx.Provider(ApexInfoProvider).(ApexInfo) + withinApex := !apexInfo.IsForPlatform() mixedBuildEnabled := ctx.Config().IsMixedBuildsEnabled() && module.Enabled() && convertedToBazel(ctx, module) && diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 10cf60a0a..47dd16180 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -74,14 +74,12 @@ var ( } ) -func init() { - RegisterMixedBuildsMutator(InitRegistrationContext) +func registerMixedBuildsMutator(ctx RegisterMutatorsContext) { + ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() } func RegisterMixedBuildsMutator(ctx RegistrationContext) { - ctx.FinalDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() - }) + ctx.FinalDepsMutators(registerMixedBuildsMutator) } func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { @@ -183,10 +181,6 @@ type BazelContext interface { // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order). GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) - // Returns the executable binary resultant from building together the python sources - // TODO(b/232976601): Remove. - GetPythonBinary(label string, cfgKey configKey) (string, error) - // Returns the results of the GetApexInfo query (including output files) GetApexInfo(label string, cfgkey configKey) (cquery.ApexInfo, error) @@ -315,14 +309,6 @@ func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcIn return result, nil } -func (m MockBazelContext) GetPythonBinary(label string, _ configKey) (string, error) { - result, ok := m.LabelToPythonBinary[label] - if !ok { - return "", fmt.Errorf("no target with label %q in LabelToPythonBinary", label) - } - return result, nil -} - func (m MockBazelContext) GetApexInfo(label string, _ configKey) (cquery.ApexInfo, error) { result, ok := m.LabelToApexInfo[label] if !ok { @@ -431,15 +417,6 @@ func (bazelCtx *mixedBuildBazelContext) GetCcInfo(label string, cfgKey configKey return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key) } -func (bazelCtx *mixedBuildBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { - key := makeCqueryKey(label, cquery.GetPythonBinary, cfgKey) - if rawString, ok := bazelCtx.results[key]; ok { - bazelOutput := strings.TrimSpace(rawString) - return cquery.GetPythonBinary.ParseResult(bazelOutput), nil - } - return "", fmt.Errorf("no bazel response found for %v", key) -} - func (bazelCtx *mixedBuildBazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) { key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey) if rawString, ok := bazelCtx.results[key]; ok { @@ -468,10 +445,6 @@ func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error panic("unimplemented") } -func (n noopBazelContext) GetPythonBinary(_ string, _ configKey) (string, error) { - panic("unimplemented") -} - func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexInfo, error) { panic("unimplemented") } @@ -505,7 +478,7 @@ func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset { return []bazel.AqueryDepset{} } -func addToStringSet(set map[string]bool, items []string) { +func AddToStringSet(set map[string]bool, items []string) { for _, item := range items { set[item] = true } @@ -517,19 +490,19 @@ func GetBazelEnabledAndDisabledModules(buildMode SoongBuildMode, forceEnabled ma switch buildMode { case BazelProdMode: - addToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) + AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) for enabledAdHocModule := range forceEnabled { enabledModules[enabledAdHocModule] = true } case BazelStagingMode: // Staging mode includes all prod modules plus all staging modules. - addToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) - addToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList) + AddToStringSet(enabledModules, allowlists.ProdMixedBuildsEnabledList) + AddToStringSet(enabledModules, allowlists.StagingMixedBuildsEnabledList) for enabledAdHocModule := range forceEnabled { enabledModules[enabledAdHocModule] = true } case BazelDevMode: - addToStringSet(disabledModules, allowlists.MixedBuildsDisabledList) + AddToStringSet(disabledModules, allowlists.MixedBuildsDisabledList) default: panic("Expected BazelProdMode, BazelStagingMode, or BazelDevMode") } @@ -609,7 +582,7 @@ func NewBazelContext(c *config) (BazelContext, error) { allowlists.StagingDclaMixedBuildsEnabledList...) } dclaEnabledModules := map[string]bool{} - addToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList) + AddToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList) return &mixedBuildBazelContext{ bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)}, paths: &paths, diff --git a/android/bazel_test.go b/android/bazel_test.go index 77e251575..13fd40849 100644 --- a/android/bazel_test.go +++ b/android/bazel_test.go @@ -436,3 +436,150 @@ func TestShouldKeepExistingBuildFileForDir(t *testing.T) { } } } + +type mixedBuildModule struct { + ModuleBase + BazelModuleBase + props struct { + Deps []string + Mixed_build_incompatible *bool + QueuedBazelCall bool `blueprint:"mutated"` + } +} + +type mixedBuildModuleInfo struct { + QueuedBazelCall bool +} + +var mixedBuildModuleProvider = blueprint.NewProvider(mixedBuildModuleInfo{}) + +func mixedBuildModuleFactory() Module { + m := &mixedBuildModule{} + m.AddProperties(&m.props) + InitAndroidArchModule(m, HostAndDeviceDefault, MultilibBoth) + InitBazelModule(m) + + return m +} + +func (m *mixedBuildModule) ConvertWithBp2build(ctx TopDownMutatorContext) { +} + +func (m *mixedBuildModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), installDepTag{}, m.props.Deps...) +} + +func (m *mixedBuildModule) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (m *mixedBuildModule) IsMixedBuildSupported(ctx BaseModuleContext) bool { + return !proptools.Bool(m.props.Mixed_build_incompatible) +} + +func (m *mixedBuildModule) QueueBazelCall(ctx BaseModuleContext) { + m.props.QueuedBazelCall = true +} + +func (m *mixedBuildModule) ProcessBazelQueryResponse(ctx ModuleContext) { + ctx.SetProvider(mixedBuildModuleProvider, mixedBuildModuleInfo{ + QueuedBazelCall: m.props.QueuedBazelCall, + }) +} + +var prepareForMixedBuildTests = FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterModuleType("deps", mixedBuildModuleFactory) + RegisterMixedBuildsMutator(ctx) +}) + +func TestMixedBuildsEnabledForType(t *testing.T) { + baseBp := ` + deps { + name: "foo", + deps: ["bar"], + target: { windows: { enabled: true } }, + %s + } +` + depBp := ` + deps { + name: "bar", + target: { + windows: { + enabled: true, + }, + }, + } +` + testCases := []struct { + desc string + variant *string + missingDeps bool + extraBpInfo string + mixedBuildsEnabled bool + }{ + { + desc: "mixed builds works", + mixedBuildsEnabled: true, + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "missing deps", + missingDeps: true, + mixedBuildsEnabled: false, + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "windows no mixed builds", + mixedBuildsEnabled: false, + variant: proptools.StringPtr("windows_x86"), + extraBpInfo: `bazel_module: { bp2build_available: true },`, + }, + { + desc: "mixed builds disabled by type", + mixedBuildsEnabled: false, + extraBpInfo: `mixed_build_incompatible: true, + bazel_module: { bp2build_available: true },`, + }, + { + desc: "mixed builds not bp2build available", + mixedBuildsEnabled: false, + extraBpInfo: `bazel_module: { bp2build_available: false },`, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + handlers := GroupFixturePreparers( + prepareForMixedBuildTests, + PrepareForTestWithArchMutator, + FixtureModifyConfig(func(config Config) { + config.BazelContext = MockBazelContext{ + OutputBaseDir: "base", + } + config.Targets[Windows] = []Target{ + {Windows, Arch{ArchType: X86_64}, NativeBridgeDisabled, "", "", true}, + {Windows, Arch{ArchType: X86}, NativeBridgeDisabled, "", "", true}, + } + }), + ) + bp := fmt.Sprintf(baseBp, tc.extraBpInfo) + if tc.missingDeps { + handlers = GroupFixturePreparers( + handlers, + PrepareForTestWithAllowMissingDependencies, + ) + } else { + bp += depBp + } + result := handlers.RunTestWithBp(t, bp) + + variant := proptools.StringDefault(tc.variant, "android_arm64_armv8-a") + + m := result.ModuleForTests("foo", variant) + mixedBuildModuleInfo := result.TestContext.ModuleProvider(m.Module(), mixedBuildModuleProvider).(mixedBuildModuleInfo) + if w, g := tc.mixedBuildsEnabled, mixedBuildModuleInfo.QueuedBazelCall; w != g { + t.Errorf("Expected mixed builds enabled %t, got mixed builds enabled %t", w, g) + } + }) + } +} diff --git a/android/config.go b/android/config.go index ddf2d746c..bed57e3b5 100644 --- a/android/config.go +++ b/android/config.go @@ -757,6 +757,14 @@ func (c *config) HostJavaToolPath(ctx PathContext, tool string) Path { return path } +func (c *config) HostCcSharedLibPath(ctx PathContext, lib string) Path { + libDir := "lib" + if ctx.Config().BuildArch.Multilib == "lib64" { + libDir = "lib64" + } + return pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, libDir, false, lib+".so") +} + // PrebuiltOS returns the name of the host OS used in prebuilts directories. func (c *config) PrebuiltOS() string { switch runtime.GOOS { @@ -1881,6 +1889,10 @@ func (c *deviceConfig) ShippingApiLevel() ApiLevel { return uncheckedFinalApiLevel(apiLevel) } +func (c *deviceConfig) BuildBrokenPluginValidation() []string { + return c.config.productVariables.BuildBrokenPluginValidation +} + func (c *deviceConfig) BuildBrokenClangAsFlags() bool { return c.config.productVariables.BuildBrokenClangAsFlags } @@ -1917,8 +1929,8 @@ func (c *deviceConfig) BuildBrokenInputDir(name string) bool { return InList(name, c.config.productVariables.BuildBrokenInputDirModules) } -func (c *deviceConfig) BuildBrokenDepfile() bool { - return Bool(c.config.productVariables.BuildBrokenDepfile) +func (c *deviceConfig) GenruleSandboxing() bool { + return Bool(c.config.productVariables.GenruleSandboxing) } func (c *deviceConfig) RequiresInsecureExecmemForSwiftshader() bool { diff --git a/android/filegroup.go b/android/filegroup.go index f30ee5143..c042ff18e 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -177,6 +177,17 @@ func (fg *fileGroup) ConvertWithBp2build(ctx TopDownMutatorContext) { } } +type FileGroupPath interface { + GetPath(ctx TopDownMutatorContext) string +} + +func (fg *fileGroup) GetPath(ctx TopDownMutatorContext) string { + if fg.properties.Path != nil { + return *fg.properties.Path + } + return "" +} + type fileGroupProperties struct { // srcs lists files that will be included in this filegroup Srcs []string `android:"path"` @@ -207,6 +218,7 @@ type fileGroup struct { BazelModuleBase DefaultableModuleBase FileGroupAsLibrary + FileGroupPath properties fileGroupProperties srcs Paths } @@ -214,6 +226,7 @@ type fileGroup struct { var _ MixedBuildBuildable = (*fileGroup)(nil) var _ SourceFileProducer = (*fileGroup)(nil) var _ FileGroupAsLibrary = (*fileGroup)(nil) +var _ FileGroupPath = (*fileGroup)(nil) // filegroup contains a list of files that are referenced by other modules // properties (such as "srcs") using the syntax ":<name>". filegroup are diff --git a/android/module.go b/android/module.go index 9024896e5..98084f359 100644 --- a/android/module.go +++ b/android/module.go @@ -15,6 +15,9 @@ package android import ( + "crypto/md5" + "encoding/hex" + "encoding/json" "fmt" "net/url" "os" @@ -354,6 +357,10 @@ type BaseModuleContext interface { AddMissingDependencies(missingDeps []string) + // getMissingDependencies returns the list of missing dependencies. + // Calling this function prevents adding new dependencies. + getMissingDependencies() []string + // AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build AddUnconvertedBp2buildDep(dep string) @@ -710,6 +717,31 @@ func SortedUniqueNamedPaths(l NamedPaths) NamedPaths { return l[:k+1] } +// soongConfigTrace holds all references to VendorVars. Uses []string for blueprint:"mutated" +type soongConfigTrace struct { + Bools []string `json:",omitempty"` + Strings []string `json:",omitempty"` + IsSets []string `json:",omitempty"` +} + +func (c *soongConfigTrace) isEmpty() bool { + return len(c.Bools) == 0 && len(c.Strings) == 0 && len(c.IsSets) == 0 +} + +// Returns hash of serialized trace records (empty string if there's no trace recorded) +func (c *soongConfigTrace) hash() string { + // Use MD5 for speed. We don't care collision or preimage attack + if c.isEmpty() { + return "" + } + j, err := json.Marshal(c) + if err != nil { + panic(fmt.Errorf("json marshal of %#v failed: %#v", *c, err)) + } + hash := md5.Sum(j) + return hex.EncodeToString(hash[:]) +} + type nameProperties struct { // The name of the module. Must be unique across all modules. Name *string @@ -939,7 +971,8 @@ type commonProperties struct { NamespaceExportedToMake bool `blueprint:"mutated"` - MissingDeps []string `blueprint:"mutated"` + MissingDeps []string `blueprint:"mutated"` + CheckedMissingDeps bool `blueprint:"mutated"` // Name and variant strings stored by mutators to enable Module.String() DebugName string `blueprint:"mutated"` @@ -953,6 +986,10 @@ type commonProperties struct { // Bazel conversion status BazelConversionStatus BazelConversionStatus `blueprint:"mutated"` + + // SoongConfigTrace records accesses to VendorVars (soong_config) + SoongConfigTrace soongConfigTrace `blueprint:"mutated"` + SoongConfigTraceHash string `blueprint:"mutated"` } // CommonAttributes represents the common Bazel attributes from which properties @@ -2862,6 +2899,20 @@ func (b *baseModuleContext) AddMissingDependencies(deps []string) { } } +func (b *baseModuleContext) checkedMissingDeps() bool { + return b.Module().base().commonProperties.CheckedMissingDeps +} + +func (b *baseModuleContext) getMissingDependencies() []string { + checked := &b.Module().base().commonProperties.CheckedMissingDeps + *checked = true + var missingDeps []string + missingDeps = append(missingDeps, b.Module().base().commonProperties.MissingDeps...) + missingDeps = append(missingDeps, b.bp.EarlyGetMissingDependencies()...) + missingDeps = FirstUniqueStrings(missingDeps) + return missingDeps +} + type AllowDisabledModuleDependency interface { blueprint.DependencyTag AllowDisabledModuleDependency(target Module) bool @@ -3141,6 +3192,10 @@ func (m *moduleContext) ModuleSubDir() string { return m.bp.ModuleSubDir() } +func (m *moduleContext) ModuleSoongConfigHash() string { + return m.module.base().commonProperties.SoongConfigTraceHash +} + func (b *baseModuleContext) Target() Target { return b.target } @@ -3725,6 +3780,8 @@ func (m *moduleContext) TargetRequiredModuleNames() []string { func init() { RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) + RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc) + FinalDepsMutators(registerSoongConfigTraceMutator) } func BuildTargetSingleton() Singleton { @@ -3906,3 +3963,54 @@ func (d *installPathsDepSet) ToList() InstallPaths { } return d.depSet.ToList().(InstallPaths) } + +func registerSoongConfigTraceMutator(ctx RegisterMutatorsContext) { + ctx.BottomUp("soongconfigtrace", soongConfigTraceMutator).Parallel() +} + +// soongConfigTraceMutator accumulates recorded soong_config trace from children. Also it normalizes +// SoongConfigTrace to make it consistent. +func soongConfigTraceMutator(ctx BottomUpMutatorContext) { + trace := &ctx.Module().base().commonProperties.SoongConfigTrace + ctx.VisitDirectDeps(func(m Module) { + childTrace := &m.base().commonProperties.SoongConfigTrace + trace.Bools = append(trace.Bools, childTrace.Bools...) + trace.Strings = append(trace.Strings, childTrace.Strings...) + trace.IsSets = append(trace.IsSets, childTrace.IsSets...) + }) + trace.Bools = SortedUniqueStrings(trace.Bools) + trace.Strings = SortedUniqueStrings(trace.Strings) + trace.IsSets = SortedUniqueStrings(trace.IsSets) + + ctx.Module().base().commonProperties.SoongConfigTraceHash = trace.hash() +} + +// soongConfigTraceSingleton writes a map from each module's config hash value to trace data. +func soongConfigTraceSingletonFunc() Singleton { + return &soongConfigTraceSingleton{} +} + +type soongConfigTraceSingleton struct { +} + +func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) { + outFile := PathForOutput(ctx, "soong_config_trace.json") + + traces := make(map[string]*soongConfigTrace) + ctx.VisitAllModules(func(module Module) { + trace := &module.base().commonProperties.SoongConfigTrace + if !trace.isEmpty() { + hash := module.base().commonProperties.SoongConfigTraceHash + traces[hash] = trace + } + }) + + j, err := json.Marshal(traces) + if err != nil { + ctx.Errorf("json marshal to %q failed: %#v", outFile, err) + return + } + + WriteFileRule(ctx, outFile, string(j)) + ctx.Phony("soong_config_trace", outFile) +} diff --git a/android/mutator.go b/android/mutator.go index 0a091eb6f..41853154f 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -67,6 +67,8 @@ func registerMutatorsForBazelConversion(ctx *Context, bp2buildMutators []Registe // collateGloballyRegisteredMutators constructs the list of mutators that have been registered // with the InitRegistrationContext and will be used at runtime. func collateGloballyRegisteredMutators() sortableComponents { + // ensure mixed builds mutator is the last mutator + finalDeps = append(finalDeps, registerMixedBuildsMutator) return collateRegisteredMutators(preArch, preDeps, postDeps, finalDeps) } @@ -885,10 +887,16 @@ func (b *bottomUpMutatorContext) Rename(name string) { } func (b *bottomUpMutatorContext) AddDependency(module blueprint.Module, tag blueprint.DependencyTag, name ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddDependency(module, tag, name...) } func (b *bottomUpMutatorContext) AddReverseDependency(module blueprint.Module, tag blueprint.DependencyTag, name string) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.AddReverseDependency(module, tag, name) } @@ -938,11 +946,17 @@ func (b *bottomUpMutatorContext) SetDefaultDependencyVariation(variation *string func (b *bottomUpMutatorContext) AddVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddVariationDependencies(variations, tag, names...) } func (b *bottomUpMutatorContext) AddFarVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } return b.bp.AddFarVariationDependencies(variations, tag, names...) } @@ -952,10 +966,16 @@ func (b *bottomUpMutatorContext) AddInterVariantDependency(tag blueprint.Depende } func (b *bottomUpMutatorContext) ReplaceDependencies(name string) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.ReplaceDependencies(name) } func (b *bottomUpMutatorContext) ReplaceDependenciesIf(name string, predicate blueprint.ReplaceDependencyPredicate) { + if b.baseModuleContext.checkedMissingDeps() { + panic("Adding deps not allowed after checking for missing deps") + } b.bp.ReplaceDependenciesIf(name, predicate) } diff --git a/android/paths.go b/android/paths.go index eaa6a8d0b..0f3d97232 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1475,7 +1475,11 @@ type ModuleOutPathContext interface { } func pathForModuleOut(ctx ModuleOutPathContext) OutputPath { - return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir()) + soongConfigHash := "" + if i, ok := ctx.(interface{ ModuleSoongConfigHash() string }); ok { + soongConfigHash = i.ModuleSoongConfigHash() + } + return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), soongConfigHash) } // PathForModuleOut returns a Path representing the paths... under the module's diff --git a/android/plugin.go b/android/plugin.go new file mode 100644 index 000000000..4672453ee --- /dev/null +++ b/android/plugin.go @@ -0,0 +1,140 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/google/blueprint" +) + +func init() { + RegisterPluginSingletonBuildComponents(InitRegistrationContext) +} + +func RegisterPluginSingletonBuildComponents(ctx RegistrationContext) { + ctx.RegisterParallelSingletonType("plugins", pluginSingletonFactory) +} + +// pluginSingleton is a singleton to handle allowlisting of the final Android-<product_name>.mk file +// output. +func pluginSingletonFactory() Singleton { + return &pluginSingleton{} +} + +type pluginSingleton struct{} + +var allowedPluginsByName = map[string]bool{ + "aidl-soong-rules": true, + "arm_compute_library_nn_driver": true, + "cuttlefish-soong-rules": true, + "gki-soong-rules": true, + "hidl-soong-rules": true, + "kernel-config-soong-rules": true, + "soong-angle-codegen": true, + "soong-api": true, + "soong-art": true, + "soong-ca-certificates": true, + "soong-ca-certificates-apex": true, + "soong-clang": true, + "soong-clang-prebuilts": true, + "soong-csuite": true, + "soong-fluoride": true, + "soong-fs_config": true, + "soong-icu": true, + "soong-java-config-error_prone": true, + "soong-libchrome": true, + "soong-llvm": true, + "soong-robolectric": true, + "soong-rust-prebuilts": true, + "soong-selinux": true, + "soong-wayland-protocol-codegen": true, + "treble_report_app": true, + "treble_report_local": true, + "treble_report_module": true, + "vintf-compatibility-matrix-soong-rules": true, + "xsdc-soong-rules": true, +} + +const ( + internalPluginsPath = "vendor/google/build/soong/internal_plugins.json" +) + +type pluginProvider interface { + IsPluginFor(string) bool +} + +func maybeAddInternalPluginsToAllowlist(ctx SingletonContext) { + if path := ExistentPathForSource(ctx, internalPluginsPath); path.Valid() { + ctx.AddNinjaFileDeps(path.String()) + absPath := absolutePath(path.String()) + var moreAllowed map[string]bool + data, err := ioutil.ReadFile(absPath) + if err != nil { + ctx.Errorf("Failed to open internal plugins path %q %q", internalPluginsPath, err) + } + if err := json.Unmarshal(data, &moreAllowed); err != nil { + fmt.Fprintf(os.Stderr, "Internal plugins file %q did not parse correctly: %q", data, err) + } + for k, v := range moreAllowed { + allowedPluginsByName[k] = v + } + } +} + +func (p *pluginSingleton) GenerateBuildActions(ctx SingletonContext) { + for _, p := range ctx.DeviceConfig().BuildBrokenPluginValidation() { + allowedPluginsByName[p] = true + } + maybeAddInternalPluginsToAllowlist(ctx) + + disallowedPlugins := map[string]bool{} + ctx.VisitAllModulesBlueprint(func(module blueprint.Module) { + if ctx.ModuleType(module) != "bootstrap_go_package" { + return + } + + p, ok := module.(pluginProvider) + if !ok || !p.IsPluginFor("soong_build") { + return + } + + name := ctx.ModuleName(module) + if _, ok := allowedPluginsByName[name]; ok { + return + } + + dir := ctx.ModuleDir(module) + + // allow use of plugins within Soong to not allowlist everything + if strings.HasPrefix(dir, "build/soong") { + return + } + + // allow third party users outside of external to create new plugins, i.e. non-google paths + // under vendor or hardware + if !strings.HasPrefix(dir, "external/") && IsThirdPartyPath(dir) { + return + } + disallowedPlugins[name] = true + }) + if len(disallowedPlugins) > 0 { + ctx.Errorf("New plugins are not supported; however %q were found. Please reach out to the build team or use BUILD_BROKEN_PLUGIN_VALIDATION (see Changes.md for more info).", SortedStringKeys(disallowedPlugins)) + } +} diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 5fa60124e..0246a08d8 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -421,6 +421,57 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s }).(map[string]blueprint.ModuleFactory) } +// tracingConfig is a wrapper to soongconfig.SoongConfig which records all accesses to SoongConfig. +type tracingConfig struct { + config soongconfig.SoongConfig + boolSet map[string]bool + stringSet map[string]string + isSetSet map[string]bool +} + +func (c *tracingConfig) Bool(name string) bool { + c.boolSet[name] = c.config.Bool(name) + return c.boolSet[name] +} + +func (c *tracingConfig) String(name string) string { + c.stringSet[name] = c.config.String(name) + return c.stringSet[name] +} + +func (c *tracingConfig) IsSet(name string) bool { + c.isSetSet[name] = c.config.IsSet(name) + return c.isSetSet[name] +} + +func (c *tracingConfig) getTrace() soongConfigTrace { + ret := soongConfigTrace{} + + for k, v := range c.boolSet { + ret.Bools = append(ret.Bools, fmt.Sprintf("%q:%t", k, v)) + } + for k, v := range c.stringSet { + ret.Strings = append(ret.Strings, fmt.Sprintf("%q:%q", k, v)) + } + for k, v := range c.isSetSet { + ret.IsSets = append(ret.IsSets, fmt.Sprintf("%q:%t", k, v)) + } + + return ret +} + +func newTracingConfig(config soongconfig.SoongConfig) *tracingConfig { + c := tracingConfig{ + config: config, + boolSet: make(map[string]bool), + stringSet: make(map[string]string), + isSetSet: make(map[string]bool), + } + return &c +} + +var _ soongconfig.SoongConfig = (*tracingConfig)(nil) + // configModuleFactory takes an existing soongConfigModuleFactory and a // ModuleType to create a new ModuleFactory that uses a custom loadhook. func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType, bp2build bool) blueprint.ModuleFactory { @@ -485,8 +536,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi // conditional on Soong config variables by reading the product // config variables from Make. AddLoadHook(module, func(ctx LoadHookContext) { - config := ctx.Config().VendorConfig(moduleType.ConfigNamespace) - newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config) + tracingConfig := newTracingConfig(ctx.Config().VendorConfig(moduleType.ConfigNamespace)) + newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, tracingConfig) if err != nil { ctx.ModuleErrorf("%s", err) return @@ -494,6 +545,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi for _, ps := range newProps { ctx.AppendProperties(ps) } + + module.(Module).base().commonProperties.SoongConfigTrace = tracingConfig.getTrace() }) } return module, props diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go index cab3e2d6b..79bdeb829 100644 --- a/android/soong_config_modules_test.go +++ b/android/soong_config_modules_test.go @@ -16,6 +16,7 @@ package android import ( "fmt" + "path/filepath" "testing" ) @@ -34,7 +35,8 @@ func soongConfigTestDefaultsModuleFactory() Module { type soongConfigTestModule struct { ModuleBase DefaultableModuleBase - props soongConfigTestModuleProperties + props soongConfigTestModuleProperties + outputPath ModuleOutPath } type soongConfigTestModuleProperties struct { @@ -49,7 +51,9 @@ func soongConfigTestModuleFactory() Module { return m } -func (t soongConfigTestModule) GenerateAndroidBuildActions(ModuleContext) {} +func (t *soongConfigTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + t.outputPath = PathForModuleOut(ctx, "test") +} var prepareForSoongConfigTestModule = FixtureRegisterWithContext(func(ctx RegistrationContext) { ctx.RegisterModuleType("test_defaults", soongConfigTestDefaultsModuleFactory) @@ -503,3 +507,197 @@ func TestSoongConfigModuleSingletonModule(t *testing.T) { }) } } + +func TestSoongConfigModuleTrace(t *testing.T) { + bp := ` + soong_config_module_type { + name: "acme_test", + module_type: "test", + config_namespace: "acme", + variables: ["board", "feature1", "FEATURE3", "unused_string_var"], + bool_variables: ["feature2", "unused_feature", "always_true"], + value_variables: ["size", "unused_size"], + properties: ["cflags", "srcs", "defaults"], + } + + soong_config_module_type { + name: "acme_test_defaults", + module_type: "test_defaults", + config_namespace: "acme", + variables: ["board", "feature1", "FEATURE3", "unused_string_var"], + bool_variables: ["feature2", "unused_feature", "always_true"], + value_variables: ["size", "unused_size"], + properties: ["cflags", "srcs", "defaults"], + } + + soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b", "soc_c"], + } + + soong_config_string_variable { + name: "unused_string_var", + values: ["a", "b"], + } + + soong_config_bool_variable { + name: "feature1", + } + + soong_config_bool_variable { + name: "FEATURE3", + } + + test_defaults { + name: "test_defaults", + cflags: ["DEFAULT"], + } + + test { + name: "normal", + defaults: ["test_defaults"], + } + + acme_test { + name: "board_1", + defaults: ["test_defaults"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + }, + } + + acme_test { + name: "board_2", + defaults: ["test_defaults"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + }, + } + + acme_test { + name: "size", + defaults: ["test_defaults"], + soong_config_variables: { + size: { + cflags: ["-DSIZE=%s"], + }, + }, + } + + acme_test { + name: "board_and_size", + defaults: ["test_defaults"], + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + size: { + cflags: ["-DSIZE=%s"], + }, + }, + } + + acme_test_defaults { + name: "board_defaults", + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + }, + }, + } + + acme_test_defaults { + name: "size_defaults", + soong_config_variables: { + size: { + cflags: ["-DSIZE=%s"], + }, + }, + } + + test { + name: "board_and_size_with_defaults", + defaults: ["board_defaults", "size_defaults"], + } + ` + + fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer { + return FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.VendorVars = vars + }) + } + + preparer := fixtureForVendorVars(map[string]map[string]string{ + "acme": { + "board": "soc_a", + "size": "42", + "feature1": "true", + "feature2": "false", + // FEATURE3 unset + "unused_feature": "true", // unused + "unused_size": "1", // unused + "unused_string_var": "a", // unused + "always_true": "true", + }, + }) + + t.Run("soong config trace hash", func(t *testing.T) { + result := GroupFixturePreparers( + preparer, + PrepareForTestWithDefaults, + PrepareForTestWithSoongConfigModuleBuildComponents, + prepareForSoongConfigTestModule, + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.FinalDepsMutators(registerSoongConfigTraceMutator) + }), + FixtureWithRootAndroidBp(bp), + ).RunTest(t) + + // Hashes of modules not using soong config should be empty + normal := result.ModuleForTests("normal", "").Module().(*soongConfigTestModule) + AssertDeepEquals(t, "normal hash", normal.base().commonProperties.SoongConfigTraceHash, "") + AssertDeepEquals(t, "normal hash out", normal.outputPath.RelativeToTop().String(), "out/soong/.intermediates/normal/test") + + board1 := result.ModuleForTests("board_1", "").Module().(*soongConfigTestModule) + board2 := result.ModuleForTests("board_2", "").Module().(*soongConfigTestModule) + size := result.ModuleForTests("size", "").Module().(*soongConfigTestModule) + + // Trace mutator sets soong config trace hash correctly + board1Hash := board1.base().commonProperties.SoongConfigTrace.hash() + board1Output := board1.outputPath.RelativeToTop().String() + AssertDeepEquals(t, "board hash calc", board1Hash, board1.base().commonProperties.SoongConfigTraceHash) + AssertDeepEquals(t, "board hash path", board1Output, filepath.Join("out/soong/.intermediates/board_1", board1Hash, "test")) + + sizeHash := size.base().commonProperties.SoongConfigTrace.hash() + sizeOutput := size.outputPath.RelativeToTop().String() + AssertDeepEquals(t, "size hash calc", sizeHash, size.base().commonProperties.SoongConfigTraceHash) + AssertDeepEquals(t, "size hash path", sizeOutput, filepath.Join("out/soong/.intermediates/size", sizeHash, "test")) + + // Trace should be identical for modules using the same set of variables + AssertDeepEquals(t, "board trace", board1.base().commonProperties.SoongConfigTrace, board2.base().commonProperties.SoongConfigTrace) + AssertDeepEquals(t, "board hash", board1.base().commonProperties.SoongConfigTraceHash, board2.base().commonProperties.SoongConfigTraceHash) + + // Trace hash should be different for different sets of soong variables + AssertBoolEquals(t, "board hash not equal to size hash", board1.base().commonProperties.SoongConfigTraceHash == size.commonProperties.SoongConfigTraceHash, false) + + boardSize := result.ModuleForTests("board_and_size", "").Module().(*soongConfigTestModule) + boardSizeDefaults := result.ModuleForTests("board_and_size_with_defaults", "").Module() + + // Trace should propagate + AssertDeepEquals(t, "board_size hash calc", boardSize.base().commonProperties.SoongConfigTrace.hash(), boardSize.base().commonProperties.SoongConfigTraceHash) + AssertDeepEquals(t, "board_size trace", boardSize.base().commonProperties.SoongConfigTrace, boardSizeDefaults.base().commonProperties.SoongConfigTrace) + AssertDeepEquals(t, "board_size hash", boardSize.base().commonProperties.SoongConfigTraceHash, boardSizeDefaults.base().commonProperties.SoongConfigTraceHash) + }) +} diff --git a/android/test_asserts.go b/android/test_asserts.go index 4143f150d..5cc7e4ac0 100644 --- a/android/test_asserts.go +++ b/android/test_asserts.go @@ -17,6 +17,7 @@ package android import ( "fmt" "reflect" + "regexp" "strings" "testing" ) @@ -137,6 +138,20 @@ func AssertStringContainsEquals(t *testing.T, message string, s string, substrin } } +// AssertStringMatches checks if the string matches the given regular expression. If it does not match, +// then an error is reported with the supplied message including a reason for why it failed. +func AssertStringMatches(t *testing.T, message, s, expectedRex string) { + t.Helper() + ok, err := regexp.MatchString(expectedRex, s) + if err != nil { + t.Fatalf("regexp failure trying to match %s against `%s` expression: %s", s, expectedRex, err) + return + } + if !ok { + t.Errorf("%s does not match regular expression %s", s, expectedRex) + } +} + // AssertStringListContains checks if the list of strings contains the expected string. If it does // not then it reports an error prefixed with the supplied message and including a reason for why it // failed. diff --git a/android/variable.go b/android/variable.go index 97171e7b9..77888e586 100644 --- a/android/variable.go +++ b/android/variable.go @@ -440,10 +440,11 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` + BuildBrokenPluginValidation []string `json:",omitempty"` BuildBrokenClangAsFlags bool `json:",omitempty"` BuildBrokenClangCFlags bool `json:",omitempty"` BuildBrokenClangProperty bool `json:",omitempty"` - BuildBrokenDepfile *bool `json:",omitempty"` + GenruleSandboxing *bool `json:",omitempty"` BuildBrokenEnforceSyspropOwner bool `json:",omitempty"` BuildBrokenTrebleSyspropNeverallow bool `json:",omitempty"` BuildBrokenUsesSoongPython2Modules bool `json:",omitempty"` @@ -847,6 +848,9 @@ func (productConfigProperties *ProductConfigProperties) AddSoongConfigProperties // indirections to extract the struct from the reflect.Value. if v, ok := maybeExtractConfigVarProp(variableStruct); ok { variableStruct = v + } else if !v.IsValid() { + // Skip invalid variables which may not used, else leads to panic + continue } for j := 0; j < variableStruct.NumField(); j++ { diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index 61058df09..9623a8bc4 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -242,7 +242,7 @@ func (s *sdkRepoHost) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", name, "sdk_repo", "sdk-repo-"+name) fmt.Fprintln(w, "sdk_repo", "sdk-repo-"+name+":", strings.Join(s.FilesToInstall().Strings(), " ")) - fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-$(FILE_NAME_TAG).zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) + fmt.Fprintf(w, "$(call dist-for-goals,sdk_repo sdk-repo-%s,%s:%s-FILE_NAME_TAG_PLACEHOLDER.zip)\n\n", s.BaseModuleName(), s.outputFile.String(), s.outputBaseName) }, } } diff --git a/apex/apex.go b/apex/apex.go index 1c794639c..f49492e84 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1871,14 +1871,17 @@ func apexFilesForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) []ap }); ok { af.overriddenPackageName = app.OverriddenManifestPackageName() } - apexFiles := []apexFile{af} + + apexFiles := []apexFile{} if allowlist := aapp.PrivAppAllowlist(); allowlist.Valid() { dirInApex := filepath.Join("etc", "permissions") - privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"privapp", dirInApex, etc, aapp) + privAppAllowlist := newApexFile(ctx, allowlist.Path(), aapp.BaseModuleName()+"_privapp", dirInApex, etc, aapp) apexFiles = append(apexFiles, privAppAllowlist) } + apexFiles = append(apexFiles, af) + return apexFiles } diff --git a/apex/apex_test.go b/apex/apex_test.go index c1d80a39f..38e24e844 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -6260,8 +6260,7 @@ func TestApexWithApps(t *testing.T) { sdk_version: "current", system_modules: "none", privileged: true, - privapp_allowlist: "perms.xml", - package_name: "com.android.AppFooPriv", + privapp_allowlist: "privapp_allowlist_com.android.AppFooPriv.xml", stl: "none", apex_available: [ "myapex" ], } @@ -6306,6 +6305,18 @@ func TestApexWithApps(t *testing.T) { // ... and not directly inside the APEX ensureNotContains(t, copyCmds, "image.apex/lib64/"+jni+".so") } + + apexBundle := module.Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, ctx, apexBundle) + var builder strings.Builder + data.Custom(&builder, apexBundle.Name(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := AppFooPriv.myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := AppFoo.myapex") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFooPriv.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALLED_MODULE := \\S+AppFoo.apk") + ensureMatches(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := \\S+AppFooPriv.apk") + ensureContains(t, androidMk, "LOCAL_SOONG_INSTALL_PAIRS := privapp_allowlist_com.android.AppFooPriv.xml:$(PRODUCT_OUT)/apex/myapex/etc/permissions/privapp_allowlist_com.android.AppFooPriv.xml") } func TestApexWithAppImportBuildId(t *testing.T) { diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 6a3b3c82c..ca96f2339 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -8,7 +8,6 @@ import ( var ( GetOutputFiles = &getOutputFilesRequestType{} - GetPythonBinary = &getPythonBinaryRequestType{} GetCcInfo = &getCcInfoType{} GetApexInfo = &getApexInfoType{} GetCcUnstrippedInfo = &getCcUnstrippedInfoType{} @@ -45,8 +44,6 @@ type CcInfo struct { type getOutputFilesRequestType struct{} -type getPythonBinaryRequestType struct{} - // Name returns a string name for this request type. Such request type names must be unique, // and must only consist of alphanumeric characters. func (g getOutputFilesRequestType) Name() string { @@ -72,31 +69,6 @@ func (g getOutputFilesRequestType) ParseResult(rawString string) []string { return splitOrEmpty(rawString, ", ") } -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getPythonBinaryRequestType) Name() string { - return "getPythonBinary" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getPythonBinaryRequestType) StarlarkFunctionBody() string { - return "return providers(target)['FilesToRunProvider'].executable.path" -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getPythonBinaryRequestType) ParseResult(rawString string) string { - return rawString -} - type getCcInfoType struct{} // Name returns a string name for this request type. Such request type names must be unique, diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go index 7003ce193..e772bb7d6 100644 --- a/bazel/cquery/request_type_test.go +++ b/bazel/cquery/request_type_test.go @@ -40,34 +40,6 @@ func TestGetOutputFilesParseResults(t *testing.T) { } } -func TestGetPythonBinaryParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput string - }{ - { - description: "no result", - input: "", - expectedOutput: "", - }, - { - description: "one result", - input: "test", - expectedOutput: "test", - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput := GetPythonBinary.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - func TestGetCcInfoParseResults(t *testing.T) { t.Parallel() testCases := []struct { diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index 60766f42a..fd92e95c6 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -826,3 +826,43 @@ func TestJavaLibraryArchVariantSrcsWithExcludes(t *testing.T) { }, }) } + +func TestJavaLibraryJavaResourcesSingleFilegroup(t *testing.T) { + runJavaLibraryTestCaseWithRegistrationCtxFunc(t, Bp2buildTestCase{ + Filesystem: map[string]string{ + "res/a.res": "", + "res/b.res": "", + "res/dir1/b.res": "", + }, + Description: "java_library", + Blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java"], + java_resources: [":filegroup1"], + bazel_module: { bp2build_available: true }, +} + +filegroup { + name: "filegroup1", + path: "foo", + srcs: ["foo/a", "foo/b"], +} + +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("java_library", "java-lib-1", AttrNameToString{ + "srcs": `["a.java"]`, + "resources": `[":filegroup1"]`, + "resource_strip_prefix": `"foo"`, + }), + MakeNeverlinkDuplicateTarget("java_library", "java-lib-1"), + MakeBazelTargetNoRestrictions("filegroup", "filegroup1", AttrNameToString{ + "srcs": `[ + "foo/a", + "foo/b", + ]`}), + }, + }, func(ctx android.RegistrationContext) { + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + }) +} diff --git a/bp2build/soong_config_module_type_conversion_test.go b/bp2build/soong_config_module_type_conversion_test.go index ad07f6833..813773d00 100644 --- a/bp2build/soong_config_module_type_conversion_test.go +++ b/bp2build/soong_config_module_type_conversion_test.go @@ -200,6 +200,53 @@ custom_cc_library_static { )`}}) } +func TestSoongConfigModuleType_MultipleBoolVar_PartialUseNotPanic(t *testing.T) { + bp := ` +soong_config_bool_variable { + name: "feature1", +} + +soong_config_bool_variable { + name: "feature2", +} + +soong_config_module_type { + name: "custom_cc_library_static", + module_type: "cc_library_static", + config_namespace: "acme", + variables: ["feature1", "feature2",], + properties: ["cflags"], +} + +custom_cc_library_static { + name: "foo", + bazel_module: { bp2build_available: true }, + host_supported: true, + soong_config_variables: { + feature1: { + conditions_default: { + cflags: ["-DDEFAULT1"], + }, + cflags: ["-DFEATURE1"], + }, + }, +}` + + runSoongConfigModuleTypeTest(t, Bp2buildTestCase{ + Description: "soong config variables - used part of multiple bool variable do not panic", + ModuleTypeUnderTest: "cc_library_static", + ModuleTypeUnderTestFactory: cc.LibraryStaticFactory, + Blueprint: bp, + ExpectedBazelTargets: []string{`cc_library_static( + name = "foo", + copts = select({ + "//build/bazel/product_variables:acme__feature1": ["-DFEATURE1"], + "//conditions:default": ["-DDEFAULT1"], + }), + local_includes = ["."], +)`}}) +} + func TestSoongConfigModuleType_StringAndBoolVar(t *testing.T) { bp := ` soong_config_bool_variable { diff --git a/cc/cc_test.go b/cc/cc_test.go index 422df7310..173911bc4 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -39,7 +39,6 @@ func TestMain(m *testing.M) { var prepareForCcTest = android.GroupFixturePreparers( PrepareForTestWithCcIncludeVndk, - aidl_library.PrepareForTestWithAidlLibrary, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.DeviceVndkVersion = StringPtr("current") variables.ProductVndkVersion = StringPtr("current") @@ -4420,7 +4419,7 @@ func TestStubsLibReexportsHeaders(t *testing.T) { } } -func TestAidlLibraryWithHeader(t *testing.T) { +func TestAidlLibraryWithHeaders(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( prepareForCcTest, @@ -4430,6 +4429,7 @@ func TestAidlLibraryWithHeader(t *testing.T) { aidl_library { name: "bar", srcs: ["x/y/Bar.aidl"], + hdrs: ["x/HeaderBar.aidl"], strip_import_prefix: "x", } `)}.AddToFixture(), @@ -4438,6 +4438,7 @@ func TestAidlLibraryWithHeader(t *testing.T) { aidl_library { name: "foo", srcs: ["a/b/Foo.aidl"], + hdrs: ["a/HeaderFoo.aidl"], strip_import_prefix: "a", deps: ["bar"], } @@ -4452,7 +4453,20 @@ func TestAidlLibraryWithHeader(t *testing.T) { ).RunTest(t).TestContext libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") - manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl.sbox.textproto")) + + android.AssertPathsRelativeToTopEquals( + t, + "aidl headers", + []string{ + "package_bar/x/HeaderBar.aidl", + "package_foo/a/HeaderFoo.aidl", + "package_foo/a/b/Foo.aidl", + "out/soong/.intermediates/package_foo/libfoo/android_arm64_armv8-a_static/gen/aidl_library.sbox.textproto", + }, + libfoo.Rule("aidl_library").Implicits, + ) + + manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl_library.sbox.textproto")) aidlCommand := manifest.Commands[0].GetCommand() expectedAidlFlags := "-Ipackage_foo/a -Ipackage_bar/x" @@ -4462,14 +4476,14 @@ func TestAidlLibraryWithHeader(t *testing.T) { outputs := strings.Join(libfoo.AllOutputs(), " ") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BpFoo.h") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/BnFoo.h") - android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl/b/Foo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/BpFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/BnFoo.h") + android.AssertStringDoesContain(t, "aidl-generated header", outputs, "gen/aidl_library/b/Foo.h") android.AssertStringDoesContain(t, "aidl-generated cpp", outputs, "b/Foo.cpp") // Confirm that the aidl header doesn't get compiled to cpp and h files - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BpBar.h") - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/BnBar.h") - android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl/y/Bar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/BpBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/BnBar.h") + android.AssertStringDoesNotContain(t, "aidl-generated header", outputs, "gen/aidl_library/y/Bar.h") android.AssertStringDoesNotContain(t, "aidl-generated cpp", outputs, "y/Bar.cpp") } @@ -4548,6 +4562,55 @@ func TestAidlFlagsWithMinSdkVersion(t *testing.T) { } } +func TestInvalidAidlProp(t *testing.T) { + t.Parallel() + + testCases := []struct { + description string + bp string + }{ + { + description: "Invalid use of aidl.libs and aidl.include_dirs", + bp: ` + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl"], + include_dirs: ["bar/include"], + } + } + `, + }, + { + description: "Invalid use of aidl.libs and aidl.local_include_dirs", + bp: ` + cc_library { + name: "foo", + aidl: { + libs: ["foo_aidl"], + local_include_dirs: ["include"], + } + } + `, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + bp := ` + aidl_library { + name: "foo_aidl", + srcs: ["Foo.aidl"], + } ` + testCase.bp + android.GroupFixturePreparers( + prepareForCcTest, + aidl_library.PrepareForTestWithAidlLibrary. + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern("For aidl headers, please only use aidl.libs prop")), + ).RunTestWithBp(t, bp) + }) + } +} + func TestMinSdkVersionInClangTriple(t *testing.T) { t.Parallel() ctx := testCc(t, ` @@ -4789,23 +4852,24 @@ func TestIncludeDirsExporting(t *testing.T) { checkIncludeDirs(t, ctx, foo, expectedIncludeDirs(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library `), expectedSystemIncludeDirs(``), expectedGeneratedHeaders(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BpBar.h `), expectedOrderOnlyDeps(` .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/b.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bnb.h .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/Bpb.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/Bar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BnBar.h - .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl/y/BpBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/Bar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BnBar.h + .intermediates/libfoo/android_arm64_armv8-a_shared/gen/aidl_library/y/BpBar.h `), ) }) diff --git a/cc/compiler.go b/cc/compiler.go index 5da745e02..16f4a6ec0 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -565,6 +565,11 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps "-I"+android.PathForModuleGen(ctx, "yacc", ctx.ModuleDir()).String()) } + if len(compiler.Properties.Aidl.Libs) > 0 && + (len(compiler.Properties.Aidl.Include_dirs) > 0 || len(compiler.Properties.Aidl.Local_include_dirs) > 0) { + ctx.ModuleErrorf("aidl.libs and (aidl.include_dirs or aidl.local_include_dirs) can't be set at the same time. For aidl headers, please only use aidl.libs prop") + } + if compiler.hasAidl(deps) { flags.aidlFlags = append(flags.aidlFlags, compiler.Properties.Aidl.Flags...) if len(compiler.Properties.Aidl.Local_include_dirs) > 0 { @@ -594,8 +599,14 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } flags.aidlFlags = append(flags.aidlFlags, "--min_sdk_version="+aidlMinSdkVersion) - flags.Local.CommonFlags = append(flags.Local.CommonFlags, - "-I"+android.PathForModuleGen(ctx, "aidl").String()) + if compiler.hasSrcExt(".aidl") { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, + "-I"+android.PathForModuleGen(ctx, "aidl").String()) + } + if len(deps.AidlLibraryInfos) > 0 { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, + "-I"+android.PathForModuleGen(ctx, "aidl_library").String()) + } } if compiler.hasSrcExt(".rscript") || compiler.hasSrcExt(".fs") { @@ -107,7 +107,14 @@ func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile andr return ret } -func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile android.Path, aidlFlags string) (cppFile android.OutputPath, headerFiles android.Paths) { +func genAidl( + ctx android.ModuleContext, + rule *android.RuleBuilder, + outDirBase string, + aidlFile android.Path, + aidlHdrs android.Paths, + aidlFlags string, +) (cppFile android.OutputPath, headerFiles android.Paths) { aidlPackage := strings.TrimSuffix(aidlFile.Rel(), aidlFile.Base()) baseName := strings.TrimSuffix(aidlFile.Base(), aidlFile.Ext()) shortName := baseName @@ -119,7 +126,7 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr shortName = strings.TrimPrefix(baseName, "I") } - outDir := android.PathForModuleGen(ctx, "aidl") + outDir := android.PathForModuleGen(ctx, outDirBase) cppFile = outDir.Join(ctx, aidlPackage, baseName+".cpp") depFile := outDir.Join(ctx, aidlPackage, baseName+".cpp.d") headerI := outDir.Join(ctx, aidlPackage, baseName+".h") @@ -128,6 +135,8 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr cmd := rule.Command() cmd.BuiltTool("aidl-cpp"). + // libc++ is default stl for aidl-cpp (a cc_binary_host module) + ImplicitTool(ctx.Config().HostCcSharedLibPath(ctx, "libc++")). FlagWithDepFile("-d", depFile). Flag("--ninja"). Flag(aidlFlags). @@ -140,6 +149,10 @@ func genAidl(ctx android.ModuleContext, rule *android.RuleBuilder, aidlFile andr headerBp, }) + if aidlHdrs != nil { + cmd.Implicits(aidlHdrs) + } + return cppFile, android.Paths{ headerI, headerBn, @@ -283,14 +296,19 @@ func genSources( ctx android.ModuleContext, aidlLibraryInfos []aidl_library.AidlLibraryInfo, srcFiles android.Paths, - buildFlags builderFlags) (android.Paths, android.Paths, generatedSourceInfo) { + buildFlags builderFlags, +) (android.Paths, android.Paths, generatedSourceInfo) { var info generatedSourceInfo var deps android.Paths var rsFiles android.Paths + // aidlRule supports compiling aidl files from srcs prop while aidlLibraryRule supports + // compiling aidl files from aidl_library modules specified in aidl.libs prop. + // The rules are separated so that they don't wipe out the other's outputDir var aidlRule *android.RuleBuilder + var aidlLibraryRule *android.RuleBuilder var yaccRule_ *android.RuleBuilder yaccRule := func() *android.RuleBuilder { @@ -331,7 +349,14 @@ func genSources( android.PathForModuleGen(ctx, "aidl.sbox.textproto")) } baseDir := strings.TrimSuffix(srcFile.String(), srcFile.Rel()) - cppFile, aidlHeaders := genAidl(ctx, aidlRule, srcFile, buildFlags.aidlFlags+" -I"+baseDir) + cppFile, aidlHeaders := genAidl( + ctx, + aidlRule, + "aidl", + srcFile, + nil, + buildFlags.aidlFlags+" -I"+baseDir, + ) srcFiles[i] = cppFile info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -354,13 +379,21 @@ func genSources( } for _, aidlLibraryInfo := range aidlLibraryInfos { + if aidlLibraryRule == nil { + aidlLibraryRule = android.NewRuleBuilder(pctx, ctx).Sbox( + android.PathForModuleGen(ctx, "aidl_library"), + android.PathForModuleGen(ctx, "aidl_library.sbox.textproto"), + ).SandboxInputs() + } for _, aidlSrc := range aidlLibraryInfo.Srcs { - if aidlRule == nil { - // TODO(b/279960133): Sandbox inputs to ensure aidl headers are explicitly specified - aidlRule = android.NewRuleBuilder(pctx, ctx).Sbox(android.PathForModuleGen(ctx, "aidl"), - android.PathForModuleGen(ctx, "aidl.sbox.textproto")) - } - cppFile, aidlHeaders := genAidl(ctx, aidlRule, aidlSrc, buildFlags.aidlFlags) + cppFile, aidlHeaders := genAidl( + ctx, + aidlLibraryRule, + "aidl_library", + aidlSrc, + aidlLibraryInfo.Hdrs.ToList(), + buildFlags.aidlFlags, + ) srcFiles = append(srcFiles, cppFile) info.aidlHeaders = append(info.aidlHeaders, aidlHeaders...) @@ -375,6 +408,10 @@ func genSources( aidlRule.Build("aidl", "gen aidl") } + if aidlLibraryRule != nil { + aidlLibraryRule.Build("aidl_library", "gen aidl_library") + } + if yaccRule_ != nil { yaccRule_.Build("yacc", "gen yacc") } diff --git a/cc/library.go b/cc/library.go index 09a7253ab..98096a8b6 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2118,8 +2118,14 @@ func (library *libraryDecorator) link(ctx ModuleContext, // Optionally export aidl headers. if Bool(library.Properties.Aidl.Export_aidl_headers) { if library.baseCompiler.hasAidl(deps) { - dir := android.PathForModuleGen(ctx, "aidl") - library.reexportDirs(dir) + if library.baseCompiler.hasSrcExt(".aidl") { + dir := android.PathForModuleGen(ctx, "aidl") + library.reexportDirs(dir) + } + if len(deps.AidlLibraryInfos) > 0 { + dir := android.PathForModuleGen(ctx, "aidl_library") + library.reexportDirs(dir) + } library.reexportDeps(library.baseCompiler.aidlOrderOnlyDeps...) library.addExportedGeneratedHeaders(library.baseCompiler.aidlHeaders...) diff --git a/genrule/Android.bp b/genrule/Android.bp index 8fb5c402c..b201cae9a 100644 --- a/genrule/Android.bp +++ b/genrule/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "soong-shared", ], srcs: [ + "allowlists.go", "genrule.go", "locations.go", ], diff --git a/genrule/allowlists.go b/genrule/allowlists.go new file mode 100644 index 000000000..875dbabed --- /dev/null +++ b/genrule/allowlists.go @@ -0,0 +1,147 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package genrule + +import ( + "android/soong/android" +) + +var ( + DepfileAllowList = []string{ + "depfile_allowed_for_test", + "tflite_support_spm_config", + "tflite_support_spm_encoder_config", + "gen_uwb_core_proto", + "libtextclassifier_fbgen_utils_flatbuffers_flatbuffers_test", + "libtextclassifier_fbgen_utils_lua_utils_tests", + "libtextclassifier_fbgen_lang_id_common_flatbuffers_model", + "libtextclassifier_fbgen_lang_id_common_flatbuffers_embedding-network", + "libtextclassifier_fbgen_annotator_datetime_datetime", + "libtextclassifier_fbgen_annotator_model", + "libtextclassifier_fbgen_annotator_experimental_experimental", + "libtextclassifier_fbgen_annotator_entity-data", + "libtextclassifier_fbgen_annotator_person_name_person_name_model", + "libtextclassifier_fbgen_utils_tflite_text_encoder_config", + "libtextclassifier_fbgen_utils_codepoint-range", + "libtextclassifier_fbgen_utils_intents_intent-config", + "libtextclassifier_fbgen_utils_flatbuffers_flatbuffers", + "libtextclassifier_fbgen_utils_zlib_buffer", + "libtextclassifier_fbgen_utils_tokenizer", + "libtextclassifier_fbgen_utils_grammar_rules", + "libtextclassifier_fbgen_utils_grammar_semantics_expression", + "libtextclassifier_fbgen_utils_resources", + "libtextclassifier_fbgen_utils_i18n_language-tag", + "libtextclassifier_fbgen_utils_normalization", + "libtextclassifier_fbgen_utils_container_bit-vector", + "libtextclassifier_fbgen_actions_actions-entity-data", + "libtextclassifier_fbgen_actions_actions_model", + "libtextclassifier_fbgen_utils_grammar_testing_value", + } + + SandboxingDenyModuleList = []string{ + "framework-javastream-protos", + "RsBalls-rscript", + "CtsRsBlasTestCases-rscript", + "pvmfw_fdt_template_rs", + "RSTest_v14-rscript", + "com.android.apex.test.bar_stripped", + "com.android.apex.test.sharedlibs_secondary_generated", + "ImageProcessingJB-rscript", + "RSTest-rscript", + "BluetoothGeneratedDumpsysBinarySchema_bfbs", + "WmediumdServerProto_h", + "TracingVMProtoStub_h", + "FrontendStub_h", + "VehicleServerProtoStub_cc", + "AudioFocusControlProtoStub_cc", + "AudioFocusControlProtoStub_h", + "TracingVMProtoStub_cc", + "VehicleServerProtoStub_h", + "hidl2aidl_translate_cpp_test_gen_headers", + "hidl2aidl_translate_cpp_test_gen_src", + "hidl2aidl_translate_java_test_gen_src", + "hidl2aidl_translate_ndk_test_gen_headers", + "hidl2aidl_translate_ndk_test_gen_src", + "hidl_hash_test_gen", + "nos_app_avb_service_genc++", + "nos_app_avb_service_genc++_headers", + "nos_app_avb_service_genc++_mock", + "nos_app_identity_service_genc++", + "nos_app_keymaster_service_genc++", + "nos_generator_test_service_genc++_headers", + "nos_generator_test_service_genc++_mock", + "r8retrace-run-retrace", + "ltp_config_arm", + "ltp_config_arm_64_hwasan", + "ltp_config_arm_lowmem", + "ltp_config_arm_64", + "ltp_config_riscv_64", + "ltp_config_x86_64", + "vm-tests-tf-lib", + "hidl_cpp_impl_test_gen-headers", + "pandora_experimental-python-gen-src", + "framework-cppstream-protos", + "Refocus-rscript", + "RSTest_v11-rscript", + "RSTest_v16-rscript", + "ScriptGroupTest-rscript", + "ImageProcessing2-rscript", + "ImageProcessing-rscript", + "com.android.apex.test.pony_stripped", + "com.android.apex.test.baz_stripped", + "com.android.apex.test.foo_stripped", + "com.android.apex.test.sharedlibs_generated", + "CtsRenderscriptTestCases-rscript", + "BlueberryFacadeAndCertGeneratedStub_py", + "BlueberryFacadeGeneratedStub_cc", + "BlueberryFacadeGeneratedStub_h", + "BluetoothGeneratedDumpsysDataSchema_h", + "FrontendStub_cc", + "OpenwrtControlServerProto_cc", + "OpenwrtControlServerProto_h", + "WmediumdServerProto_cc", + "c2hal_test_genc++", + "c2hal_test_genc++_headers", + "hidl2aidl_test_gen_aidl", + "hidl_error_test_gen", + "hidl_export_test_gen-headers", + "hidl_format_test_diff", + "hidl_hash_version_gen", + "libbt_topshim_facade_py_proto", + "nos_app_identity_service_genc++_headers", + "nos_app_identity_service_genc++_mock", + "nos_app_keymaster_service_genc++_headers", + "nos_app_keymaster_service_genc++_mock", + "nos_app_weaver_service_genc++", + "nos_app_weaver_service_genc++_headers", + "nos_app_weaver_service_genc++_mock", + "nos_generator_test_service_genc++", + "pandora-python-gen-src", + } + + SandboxingDenyPathList = []string{ + "art/test", + "external/perfetto", + } +) +var DepfileAllowSet = map[string]bool{} +var SandboxingDenyModuleSet = map[string]bool{} +var SandboxingDenyPathSet = map[string]bool{} + +func init() { + android.AddToStringSet(DepfileAllowSet, DepfileAllowList) + android.AddToStringSet(SandboxingDenyModuleSet, append(DepfileAllowList, SandboxingDenyModuleList...)) + android.AddToStringSet(SandboxingDenyPathSet, SandboxingDenyPathList) +} diff --git a/genrule/genrule.go b/genrule/genrule.go index 00adb7025..c830fcc43 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -452,7 +452,7 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { manifestPath := android.PathForModuleOut(ctx, manifestName) // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox. - rule := android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath).SandboxTools() + rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(task.genDir, manifestPath)) cmd := rule.Command() for _, out := range task.out { @@ -594,14 +594,16 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Allowlist genrule to use depfile until we have a solution to remove it. // TODO(b/235582219): Remove allowlist for genrule - if ctx.ModuleType() == "gensrcs" && - !ctx.DeviceConfig().BuildBrokenDepfile() && - Bool(g.properties.Depfile) { - ctx.PropertyErrorf( - "depfile", - "Deprecated to ensure the module type is convertible to Bazel. "+ - "Try specifying the dependencies explicitly so that there is no need to use depfile. "+ - "If not possible, the escape hatch is to use BUILD_BROKEN_DEPFILE to bypass the error.") + if Bool(g.properties.Depfile) { + // TODO(b/283852474): Checking the GenruleSandboxing flag is temporary in + // order to pass the presubmit before internal master is updated. + if ctx.DeviceConfig().GenruleSandboxing() && !DepfileAllowSet[g.Name()] { + ctx.PropertyErrorf( + "depfile", + "Deprecated to ensure the module type is convertible to Bazel. "+ + "Try specifying the dependencies explicitly so that there is no need to use depfile. "+ + "If not possible, the escape hatch is to add the module to allowlists.go to bypass the error.") + } } g.generateCommonBuildActions(ctx) @@ -737,7 +739,7 @@ func NewGenSrcs() *Module { // TODO(ccross): this RuleBuilder is a hack to be able to call // rule.Command().PathForOutput. Replace this with passing the rule into the // generator. - rule := android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil).SandboxTools() + rule := getSandboxedRuleBuilder(ctx, android.NewRuleBuilder(pctx, ctx).Sbox(genDir, nil)) for _, in := range shard { outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension)) @@ -1020,3 +1022,11 @@ func DefaultsFactory(props ...interface{}) android.Module { return module } + +func getSandboxedRuleBuilder(ctx android.ModuleContext, r *android.RuleBuilder) *android.RuleBuilder { + if !ctx.DeviceConfig().GenruleSandboxing() || SandboxingDenyPathSet[ctx.ModuleDir()] || + SandboxingDenyModuleSet[ctx.ModuleName()] { + return r.SandboxTools() + } + return r.SandboxInputs() +} diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index 63f8fa9b8..370fe3d46 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -97,8 +97,9 @@ func testGenruleBp() string { func TestGenruleCmd(t *testing.T) { testcases := []struct { - name string - prop string + name string + moduleName string + prop string allowMissingDependencies bool @@ -285,7 +286,8 @@ func TestGenruleCmd(t *testing.T) { expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out2", }, { - name: "depfile", + name: "depfile", + moduleName: "depfile_allowed_for_test", prop: ` out: ["out"], depfile: true, @@ -397,7 +399,8 @@ func TestGenruleCmd(t *testing.T) { err: "$(depfile) used without depfile property", }, { - name: "error no depfile", + name: "error no depfile", + moduleName: "depfile_allowed_for_test", prop: ` out: ["out"], depfile: true, @@ -440,11 +443,15 @@ func TestGenruleCmd(t *testing.T) { for _, test := range testcases { t.Run(test.name, func(t *testing.T) { - bp := "genrule {\n" - bp += "name: \"gen\",\n" - bp += test.prop - bp += "}\n" - + moduleName := "gen" + if test.moduleName != "" { + moduleName = test.moduleName + } + bp := fmt.Sprintf(` + genrule { + name: "%s", + %s + }`, moduleName, test.prop) var expectedErrors []string if test.err != "" { expectedErrors = append(expectedErrors, regexp.QuoteMeta(test.err)) @@ -455,6 +462,9 @@ func TestGenruleCmd(t *testing.T) { android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { variables.Allow_missing_dependencies = proptools.BoolPtr(test.allowMissingDependencies) }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.GenruleSandboxing = proptools.BoolPtr(true) + }), android.FixtureModifyContext(func(ctx *android.TestContext) { ctx.SetAllowMissingDependencies(test.allowMissingDependencies) }), @@ -466,7 +476,7 @@ func TestGenruleCmd(t *testing.T) { return } - gen := result.Module("gen", "").(*Module) + gen := result.Module(moduleName, "").(*Module) android.AssertStringEquals(t, "raw commands", test.expect, gen.rawCommands[0]) }) } @@ -627,53 +637,42 @@ func TestGenSrcs(t *testing.T) { } } -func TestGensrcsBuildBrokenDepfile(t *testing.T) { +func TestGenruleAllowlistingDepfile(t *testing.T) { tests := []struct { - name string - prop string - BuildBrokenDepfile *bool - err string + name string + prop string + err string + moduleName string }{ { - name: `error when BuildBrokenDepfile is set to false`, + name: `error when module is not allowlisted`, prop: ` depfile: true, cmd: "cat $(in) > $(out) && cat $(depfile)", `, - BuildBrokenDepfile: proptools.BoolPtr(false), - err: "depfile: Deprecated to ensure the module type is convertible to Bazel", + err: "depfile: Deprecated to ensure the module type is convertible to Bazel", }, { - name: `error when BuildBrokenDepfile is not set`, + name: `no error when module is allowlisted`, prop: ` depfile: true, cmd: "cat $(in) > $(out) && cat $(depfile)", `, - err: "depfile: Deprecated to ensure the module type is convertible to Bazel.", - }, - { - name: `no error when BuildBrokenDepfile is explicitly set to true`, - prop: ` - depfile: true, - cmd: "cat $(in) > $(out) && cat $(depfile)", - `, - BuildBrokenDepfile: proptools.BoolPtr(true), - }, - { - name: `no error if depfile is not set`, - prop: ` - cmd: "cat $(in) > $(out)", - `, + moduleName: `depfile_allowed_for_test`, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + moduleName := "foo" + if test.moduleName != "" { + moduleName = test.moduleName + } bp := fmt.Sprintf(` gensrcs { - name: "foo", + name: "%s", srcs: ["data.txt"], %s - }`, test.prop) + }`, moduleName, test.prop) var expectedErrors []string if test.err != "" { @@ -682,9 +681,7 @@ func TestGensrcsBuildBrokenDepfile(t *testing.T) { android.GroupFixturePreparers( prepareForGenRuleTest, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - if test.BuildBrokenDepfile != nil { - variables.BuildBrokenDepfile = test.BuildBrokenDepfile - } + variables.GenruleSandboxing = proptools.BoolPtr(true) }), ). ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(expectedErrors)). diff --git a/java/app.go b/java/app.go index 366005ce3..561ce1d17 100755 --- a/java/app.go +++ b/java/app.go @@ -613,7 +613,6 @@ func processMainCert(m android.ModuleBase, certPropValue string, certificates [] } } - return mainCertificate, certificates } @@ -621,13 +620,21 @@ func (a *AndroidApp) InstallApkName() string { return a.installApkName } -func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) *android.OutputPath { +func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.Path { if a.appProperties.Privapp_allowlist == nil { return nil } + + isOverrideApp := a.GetOverriddenBy() != "" + if !isOverrideApp { + // if this is not an override, we don't need to rewrite the existing privapp allowlist + return android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist) + } + if a.overridableAppProperties.Package_name == nil { ctx.PropertyErrorf("privapp_allowlist", "package_name must be set to use privapp_allowlist") } + packageName := *a.overridableAppProperties.Package_name fileName := "privapp_allowlist_" + packageName + ".xml" outPath := android.PathForModuleOut(ctx, fileName).OutputPath @@ -787,17 +794,17 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Install the app package. shouldInstallAppPackage := (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && !a.appProperties.PreventInstall if shouldInstallAppPackage { + if a.privAppAllowlist.Valid() { + installPath := android.PathForModuleInstall(ctx, "etc", "permissions") + ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) + } + var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) } ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) - - if a.privAppAllowlist.Valid() { - installPath := android.PathForModuleInstall(ctx, "etc", "permissions") - ctx.InstallFile(installPath, a.privAppAllowlist.Path().Base(), a.privAppAllowlist.Path()) - } } a.buildAppDependencyInfo(ctx) diff --git a/java/app_test.go b/java/app_test.go index c485478eb..9293da48e 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3563,9 +3563,8 @@ func TestPrivappAllowlist(t *testing.T) { android_app { name: "foo", srcs: ["a.java"], - privapp_allowlist: "perms.xml", + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", privileged: true, - package_name: "com.android.foo", sdk_version: "current", } override_android_app { @@ -3578,20 +3577,94 @@ func TestPrivappAllowlist(t *testing.T) { app := result.ModuleForTests("foo", "android_common") overrideApp := result.ModuleForTests("foo", "android_common_bar") - // verify that privapp allowlist is created - app.Output("out/soong/.intermediates/foo/android_common/privapp_allowlist_com.android.foo.xml") + // verify that privapp allowlist is created for override apps overrideApp.Output("out/soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml") - expectedAllowlist := "perms.xml" - actualAllowlist := app.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != actualAllowlist { - t.Errorf("expected allowlist to be %q; got %q", expectedAllowlist, actualAllowlist) - } - overrideActualAllowlist := overrideApp.Rule("modifyAllowlist").Input.String() - if expectedAllowlist != overrideActualAllowlist { - t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlist, overrideActualAllowlist) + expectedAllowlistInput := "privapp_allowlist_com.android.foo.xml" + overrideActualAllowlistInput := overrideApp.Rule("modifyAllowlist").Input.String() + if expectedAllowlistInput != overrideActualAllowlistInput { + t.Errorf("expected override allowlist to be %q; got %q", expectedAllowlistInput, overrideActualAllowlistInput) } // verify that permissions are copied to device app.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml") overrideApp.Output("out/soong/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml") } + +func TestPrivappAllowlistAndroidMk(t *testing.T) { + result := android.GroupFixturePreparers( + PrepareForTestWithJavaDefaultModules, + android.PrepareForTestWithAndroidMk, + ).RunTestWithBp( + t, + ` + android_app { + name: "foo", + srcs: ["a.java"], + privapp_allowlist: "privapp_allowlist_com.android.foo.xml", + privileged: true, + sdk_version: "current", + } + override_android_app { + name: "bar", + base: "foo", + package_name: "com.google.android.foo", + } + `, + ) + baseApp := result.ModuleForTests("foo", "android_common") + overrideApp := result.ModuleForTests("foo", "android_common_bar") + + baseAndroidApp := baseApp.Module().(*AndroidApp) + baseEntries := android.AndroidMkEntriesForTest(t, result.TestContext, baseAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include foo.apk", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+foo.apk:\\S+/target/product/test_device/system/priv-app/foo/foo.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + baseEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "privapp_allowlist_com.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.android.foo.xml", + ) + + overrideAndroidApp := overrideApp.Module().(*AndroidApp) + overrideEntries := android.AndroidMkEntriesForTest(t, result.TestContext, overrideAndroidApp)[0] + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALLED_MODULE; expected to find bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALLED_MODULE"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include bar.apk", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include app", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+bar.apk:\\S+/target/product/test_device/system/priv-app/bar/bar.apk", + ) + android.AssertStringMatches( + t, + "androidmk has incorrect LOCAL_SOONG_INSTALL_PAIRS; expected to it to include privapp_allowlist", + overrideEntries.EntryMap["LOCAL_SOONG_INSTALL_PAIRS"][0], + "\\S+soong/.intermediates/foo/android_common_bar/privapp_allowlist_com.google.android.foo.xml:\\S+/target/product/test_device/system/etc/permissions/privapp_allowlist_com.google.android.foo.xml", + ) +} diff --git a/java/config/config.go b/java/config/config.go index b82a137e7..195dae16b 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -98,7 +98,7 @@ func init() { "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ - "-JXmx2048M", + "-JXmx4096M", "-JDcom.android.tools.r8.emitRecordAnnotationsInDex", "-JDcom.android.tools.r8.emitPermittedSubclassesAnnotationsInDex", "-JDcom.android.tools.r8.emitRecordAnnotationsExInDex", diff --git a/java/core-libraries/TxtStubLibraries.bp b/java/core-libraries/TxtStubLibraries.bp index 813187e54..0cf0f360b 100644 --- a/java/core-libraries/TxtStubLibraries.bp +++ b/java/core-libraries/TxtStubLibraries.bp @@ -22,8 +22,6 @@ java_system_modules { libs: [ "core-current-stubs-for-system-modules-no-annotations.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -36,8 +34,6 @@ java_library { "core.current.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } // Same as core-module-lib-stubs-system-modules, but the stubs are generated from .txt files @@ -47,8 +43,6 @@ java_system_modules { libs: [ "core-module-lib-stubs-for-system-modules-no-annotations.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -61,8 +55,6 @@ java_library { "core.module_lib.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -79,8 +71,6 @@ java_library { sdk_version: "none", system_modules: "none", visibility: ["//visibility:private"], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } // Same as legacy-core-platform-api-stubs-system-modules, but the stubs are generated from .txt files @@ -91,8 +81,6 @@ java_system_modules { "legacy.core.platform.api.no.annotations.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -108,8 +96,6 @@ java_library { "legacy.core.platform.api.stubs.from-text", ], patch_module: "java.base", - // TODO: Enable after stub generation from .txt file is available - enabled: false, } // Same as stable-core-platform-api-stubs-system-modules, but the stubs are generated from .txt files @@ -120,8 +106,6 @@ java_system_modules { "stable.core.platform.api.no.annotations.stubs.from-text", "core-lambda-stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_library { @@ -137,8 +121,6 @@ java_library { "stable.core.platform.api.stubs.from-text", ], patch_module: "java.base", - // TODO: Enable after stub generation from .txt file is available - enabled: false, } java_api_library { @@ -151,6 +133,4 @@ java_api_library { // LambdaMetaFactory depends on CallSite etc. which is part of the Core API surface "core.current.stubs.from-text", ], - // TODO: Enable after stub generation from .txt file is available - enabled: false, } diff --git a/java/fuzz.go b/java/fuzz.go index 5c5f769a5..5dfaacfb3 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -177,7 +177,11 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder) // Add .jar - files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.implementationJarFile}) + if !javaFuzzModule.Host() { + files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.implementationJarFile, DestinationPathPrefix: "classes"}) + } + + files = append(files, fuzz.FileToZip{SourceFilePath: javaFuzzModule.outputFile}) // Add jni .so files for _, fPath := range javaFuzzModule.jniFilePaths { diff --git a/java/java.go b/java/java.go index 33846bec6..aa9f936d0 100644 --- a/java/java.go +++ b/java/java.go @@ -1813,7 +1813,7 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { case javaApiContributionTag: provider := ctx.OtherModuleProvider(dep, JavaApiImportProvider).(JavaApiImportInfo) providerApiFile := provider.ApiFile - if providerApiFile == nil { + if providerApiFile == nil && !ctx.Config().AllowMissingDependencies() { ctx.ModuleErrorf("Error: %s has an empty api file.", dep.Name()) } srcFiles = append(srcFiles, android.PathForSource(ctx, providerApiFile.String())) @@ -1835,7 +1835,7 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { srcFiles = append(srcFiles, android.PathForModuleSrc(ctx, api)) } - if srcFiles == nil { + if srcFiles == nil && !ctx.Config().AllowMissingDependencies() { ctx.ModuleErrorf("Error: %s has an empty api file.", ctx.ModuleName()) } @@ -2606,6 +2606,7 @@ func DefaultsFactory() android.Module { &appProperties{}, &appTestProperties{}, &overridableAppProperties{}, + &hostTestProperties{}, &testProperties{}, &ImportProperties{}, &AARImportProperties{}, @@ -2703,6 +2704,15 @@ type javaResourcesAttributes struct { Resource_strip_prefix *string } +func (m *Library) javaResourcesGetSingleFilegroupStripPrefix(ctx android.TopDownMutatorContext) (string, bool) { + if otherM, ok := ctx.ModuleFromName(m.properties.Java_resources[0]); ok && len(m.properties.Java_resources) == 1 { + if fg, isFilegroup := otherM.(android.FileGroupPath); isFilegroup { + return filepath.Join(ctx.OtherModuleDir(otherM), fg.GetPath(ctx)), true + } + } + return "", false +} + func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorContext) *javaResourcesAttributes { var resources bazel.LabelList var resourceStripPrefix *string @@ -2712,8 +2722,12 @@ func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorConte } if m.properties.Java_resources != nil { + if prefix, ok := m.javaResourcesGetSingleFilegroupStripPrefix(ctx); ok { + resourceStripPrefix = proptools.StringPtr(prefix) + } else { + resourceStripPrefix = proptools.StringPtr(ctx.ModuleDir()) + } resources.Append(android.BazelLabelForModuleSrc(ctx, m.properties.Java_resources)) - resourceStripPrefix = proptools.StringPtr(ctx.ModuleDir()) } //TODO(b/179889880) handle case where glob includes files outside package diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 8225df60d..c0ae0c0fe 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -870,7 +870,7 @@ func (ctx *parseContext) handleSubConfig( } // Safeguard against $(call inherit-product,$(PRODUCT_PATH)) - const maxMatchingFiles = 150 + const maxMatchingFiles = 155 // temporarily increased to 155 for b/284854738 if len(matchingPaths) > maxMatchingFiles { return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)} } diff --git a/rust/fuzz_test.go b/rust/fuzz_test.go index 865665eef..7fa9f5c8f 100644 --- a/rust/fuzz_test.go +++ b/rust/fuzz_test.go @@ -46,18 +46,16 @@ func TestRustFuzz(t *testing.T) { // Check that compiler flags are set appropriately . fuzz_libtest := ctx.ModuleForTests("fuzz_libtest", "android_arm64_armv8-a_fuzzer").Rule("rustc") - if !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-Z sanitizer=hwaddress") || - !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-C passes='sancov-module'") || + if !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-C passes='sancov-module'") || !strings.Contains(fuzz_libtest.Args["rustcFlags"], "--cfg fuzzing") { - t.Errorf("rust_fuzz module does not contain the expected flags (sancov-module, cfg fuzzing, hwaddress sanitizer).") + t.Errorf("rust_fuzz module does not contain the expected flags (sancov-module, cfg fuzzing).") } // Check that dependencies have 'fuzzer' variants produced for them as well. libtest_fuzzer := ctx.ModuleForTests("libtest_fuzzing", "android_arm64_armv8-a_rlib_rlib-std_fuzzer").Output("libtest_fuzzing.rlib") - if !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "-Z sanitizer=hwaddress") || - !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "-C passes='sancov-module'") || + if !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "-C passes='sancov-module'") || !strings.Contains(libtest_fuzzer.Args["rustcFlags"], "--cfg fuzzing") { - t.Errorf("rust_fuzz dependent library does not contain the expected flags (sancov-module, cfg fuzzing, hwaddress sanitizer).") + t.Errorf("rust_fuzz dependent library does not contain the expected flags (sancov-module, cfg fuzzing).") } } diff --git a/rust/sanitize.go b/rust/sanitize.go index c68137ecd..83cf05552 100644 --- a/rust/sanitize.go +++ b/rust/sanitize.go @@ -226,11 +226,6 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags, deps PathDeps) ( } if Bool(sanitize.Properties.Sanitize.Fuzzer) { flags.RustFlags = append(flags.RustFlags, fuzzerFlags...) - if ctx.Arch().ArchType == android.Arm64 && ctx.Os().Bionic() { - flags.RustFlags = append(flags.RustFlags, hwasanFlags...) - } else { - flags.RustFlags = append(flags.RustFlags, asanFlags...) - } } else if Bool(sanitize.Properties.Sanitize.Hwaddress) { flags.RustFlags = append(flags.RustFlags, hwasanFlags...) } else if Bool(sanitize.Properties.Sanitize.Address) { @@ -424,14 +419,6 @@ func (mod *Module) IsSanitizerExplicitlyDisabled(t cc.SanitizerType) bool { return true } - // TODO(b/178365482): Rust/CC interop doesn't work just yet; don't sanitize rust_ffi modules until - // linkage issues are resolved. - if lib, ok := mod.compiler.(libraryInterface); ok { - if lib.shared() || lib.static() { - return true - } - } - return mod.sanitize.isSanitizerExplicitlyDisabled(t) } diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 59352477c..5fc05f8b1 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -207,8 +207,8 @@ EOF function test_soong_build_rerun_iff_environment_changes() { setup - mkdir -p cherry - cat > cherry/Android.bp <<'EOF' + mkdir -p build/soong/cherry + cat > build/soong/cherry/Android.bp <<'EOF' bootstrap_go_package { name: "cherry", pkgPath: "android/soong/cherry", @@ -224,7 +224,7 @@ bootstrap_go_package { } EOF - cat > cherry/cherry.go <<'EOF' + cat > build/soong/cherry/cherry.go <<'EOF' package cherry import ( @@ -317,8 +317,8 @@ function test_add_file_to_soong_build() { run_soong local -r mtime1=$(stat -c "%y" out/soong/build.ninja) - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p vendor/foo/picard + cat > vendor/foo/picard/Android.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -334,7 +334,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > vendor/foo/picard/picard.go <<'EOF' package picard import ( @@ -390,11 +390,11 @@ EOF function test_glob_during_bootstrapping() { setup - mkdir -p a - cat > a/Android.bp <<'EOF' + mkdir -p build/soong/picard + cat > build/soong/picard/Android.bp <<'EOF' build=["foo*.bp"] EOF - cat > a/fooa.bp <<'EOF' + cat > build/soong/picard/fooa.bp <<'EOF' bootstrap_go_package { name: "picard-soong-rules", pkgPath: "android/soong/picard", @@ -410,7 +410,7 @@ bootstrap_go_package { } EOF - cat > a/picard.go <<'EOF' + cat > build/soong/picard/picard.go <<'EOF' package picard import ( @@ -459,7 +459,7 @@ EOF grep -q "Make it so" out/soong/build.ninja || fail "Original action not present" - cat > a/foob.bp <<'EOF' + cat > build/soong/picard/foob.bp <<'EOF' bootstrap_go_package { name: "worf-soong-rules", pkgPath: "android/soong/worf", @@ -476,7 +476,7 @@ bootstrap_go_package { } EOF - cat > a/worf.go <<'EOF' + cat > build/soong/picard/worf.go <<'EOF' package worf import "android/soong/picard" diff --git a/tests/lib.sh b/tests/lib.sh index 715eac143..7f3970eb7 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -99,6 +99,7 @@ function create_mock_soong { symlink_directory external/python symlink_directory external/sqlite symlink_directory external/spdx-tools + symlink_directory libcore touch "$MOCK_TOP/Android.bp" } diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh index 2f154cde5..94fe51d2d 100755 --- a/tests/sbom_test.sh +++ b/tests/sbom_test.sh @@ -90,10 +90,12 @@ diff_excludes[system]=\ -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \ -I /system/lib64/android.security.compat-ndk.so \ -I /system/lib64/libkeymaster4_1support.so \ + -I /system/lib64/libkeymaster4support.so \ -I /system/lib64/libkeymint.so \ -I /system/lib64/libkeystore2_aaid.so \ -I /system/lib64/libkeystore2_apc_compat.so \ -I /system/lib64/libkeystore2_crypto.so \ + -I /system/lib64/libkeystore-attestation-application-id.so \ -I /system/lib64/libkm_compat_service.so \ -I /system/lib64/libkm_compat.so \ -I /system/lib64/vndk-29 \ diff --git a/tests/soong_test.sh b/tests/soong_test.sh index f7bee4008..6779d8a95 100755 --- a/tests/soong_test.sh +++ b/tests/soong_test.sh @@ -9,12 +9,8 @@ source "$(dirname "$0")/lib.sh" function test_m_clean_works { setup - # Create a directory with files that cannot be removed - mkdir -p out/bad_directory_permissions - touch out/bad_directory_permissions/unremovable_file - # File permissions are fine but directory permissions are bad - chmod a+rwx out/bad_directory_permissions/unremovable_file - chmod a-rwx out/bad_directory_permissions + mkdir -p out/some_directory + touch out/some_directory/some_file run_soong clean } diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go index fd601772c..ee5332735 100644 --- a/ui/build/cleanbuild.go +++ b/ui/build/cleanbuild.go @@ -17,7 +17,6 @@ package build import ( "bytes" "fmt" - "io/fs" "io/ioutil" "os" "path/filepath" @@ -59,37 +58,9 @@ const ( FILEMODE_USER_EXECUTE = FILEMODE_EXECUTE << FILEMODE_USER_SHIFT ) -// Ensures that files and directories in the out dir can be deleted. -// For example, Bazen can generate output directories where the write bit isn't set, causing 'm' clean' to fail. -func ensureOutDirRemovable(ctx Context, config Config) { - err := filepath.WalkDir(config.OutDir(), func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - info, err := d.Info() - if err != nil { - return err - } - // Equivalent to running chmod u+rwx on each directory - newMode := info.Mode() | FILEMODE_USER_READ | FILEMODE_USER_WRITE | FILEMODE_USER_EXECUTE - if err := os.Chmod(path, newMode); err != nil { - return err - } - } - // Continue walking the out dir... - return nil - }) - if err != nil && !os.IsNotExist(err) { - // Display the error, but don't crash. - ctx.Println(err.Error()) - } -} - // Remove everything under the out directory. Don't remove the out directory // itself in case it's a symlink. func clean(ctx Context, config Config) { - ensureOutDirRemovable(ctx, config) removeGlobs(ctx, filepath.Join(config.OutDir(), "*")) ctx.Println("Entire build directory removed.") } diff --git a/ui/build/kati.go b/ui/build/kati.go index dad68fac2..aea56d31a 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -22,6 +22,7 @@ import ( "os/user" "path/filepath" "strings" + "time" "android/soong/ui/metrics" "android/soong/ui/status" @@ -66,6 +67,21 @@ func genKatiSuffix(ctx Context, config Config) { } } +func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) { + filePath := filepath.Join(dir, filename) + previousValue := "" + rawPreviousValue, err := ioutil.ReadFile(filePath) + if err == nil { + previousValue = string(rawPreviousValue) + } + + if previousValue != value { + if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil { + ctx.Fatalf("Failed to write: %v", err) + } + } +} + // Base function to construct and run the Kati command line with additional // arguments, and a custom function closure to mutate the environment Kati runs // in. @@ -157,28 +173,60 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF } cmd.Stderr = cmd.Stdout - // Apply the caller's function closure to mutate the environment variables. - envFunc(cmd.Environment) - + var username string // Pass on various build environment metadata to Kati. - if _, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { - username := "unknown" + if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { + username = "unknown" if u, err := user.Current(); err == nil { username = u.Username } else { ctx.Println("Failed to get current user:", err) } cmd.Environment.Set("BUILD_USERNAME", username) + } else { + username = usernameFromEnv } - if _, ok := cmd.Environment.Get("BUILD_HOSTNAME"); !ok { - hostname, err := os.Hostname() + hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") + // Unset BUILD_HOSTNAME during kati run to avoid kati rerun, kati will use BUILD_HOSTNAME from a file. + cmd.Environment.Unset("BUILD_HOSTNAME") + if !ok { + hostname, err = os.Hostname() if err != nil { ctx.Println("Failed to read hostname:", err) hostname = "unknown" } - cmd.Environment.Set("BUILD_HOSTNAME", hostname) } + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) + + // BUILD_NUMBER should be set to the source control value that + // represents the current state of the source code. E.g., a + // perforce changelist number or a git hash. Can be an arbitrary string + // (to allow for source control that uses something other than numbers), + // but must be a single word and a valid file name. + // + // If no BUILD_NUMBER is set, create a useful "I am an engineering build + // from this date/time" value. Make it start with a non-digit so that + // anyone trying to parse it as an integer will probably get "0". + cmd.Environment.Unset("HAS_BUILD_NUMBER") + buildNumber, ok := cmd.Environment.Get("BUILD_NUMBER") + // Unset BUILD_NUMBER during kati run to avoid kati rerun, kati will use BUILD_NUMBER from a file. + cmd.Environment.Unset("BUILD_NUMBER") + if ok { + cmd.Environment.Set("HAS_BUILD_NUMBER", "true") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", buildNumber) + } else { + buildNumber = fmt.Sprintf("eng.%.6s.%s", username, time.Now().Format("20060102.150405" /* YYYYMMDD.HHMMSS */)) + cmd.Environment.Set("HAS_BUILD_NUMBER", "false") + writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username) + } + // Write the build number to a file so it can be read back in + // without changing the command line every time. Avoids rebuilds + // when using ninja. + writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_number.txt", buildNumber) + + // Apply the caller's function closure to mutate the environment variables. + envFunc(cmd.Environment) cmd.StartOrFatal() // Set up the ToolStatus command line reader for Kati for a consistent UI @@ -336,6 +384,7 @@ func runKatiPackage(ctx Context, config Config) { "ANDROID_BUILD_SHELL", "DIST_DIR", "OUT_DIR", + "FILE_NAME_TAG", }...) if config.Dist() { diff --git a/ui/metrics/BUILD.bazel b/ui/metrics/BUILD.bazel index 15ebb8864..2dc1ab633 100644 --- a/ui/metrics/BUILD.bazel +++ b/ui/metrics/BUILD.bazel @@ -16,7 +16,7 @@ load("//build/bazel/rules/python:py_proto.bzl", "py_proto_library") py_proto_library( name = "metrics-py-proto", - visibility = ["//build/bazel/scripts/incremental_build:__pkg__"], + visibility = ["//build/bazel/scripts:__subpackages__"], deps = [":metrics-proto"], ) |