diff options
60 files changed, 1323 insertions, 814 deletions
diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 16f144e93..bf3940405 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -347,6 +347,7 @@ var ( ".":/*recursive = */ false, "build/bazel":/* recursive = */ true, + "build/make/core":/* recursive = */ false, "build/bazel_common_rules":/* recursive = */ true, // build/make/tools/signapk BUILD file is generated, so build/make/tools is not recursive. "build/make/tools":/* recursive = */ false, @@ -382,6 +383,8 @@ var ( "prebuilts/sdk":/* recursive = */ false, "prebuilts/sdk/tools":/* recursive = */ false, "prebuilts/r8":/* recursive = */ false, + + "tools/asuite/atest/":/* recursive = */ true, } Bp2buildModuleAlwaysConvertList = []string{ @@ -647,6 +650,10 @@ var ( "libcodec2_soft_avcenc", "libcodec2_soft_aacdec", "libcodec2_soft_common", + + // kotlin srcs in java libs + "CtsPkgInstallerConstants", + "kotlinx_atomicfu", } Bp2buildModuleTypeAlwaysConvertList = []string{ diff --git a/android/arch.go b/android/arch.go index 086e945da..6acf9cf51 100644 --- a/android/arch.go +++ b/android/arch.go @@ -16,6 +16,7 @@ package android import ( "encoding" + "encoding/json" "fmt" "reflect" "runtime" @@ -1681,10 +1682,10 @@ func hasArmAndroidArch(targets []Target) bool { // archConfig describes a built-in configuration. type archConfig struct { - arch string - archVariant string - cpuVariant string - abi []string + Arch string `json:"arch"` + ArchVariant string `json:"arch_variant"` + CpuVariant string `json:"cpu_variant"` + Abi []string `json:"abis"` } // getNdkAbisConfig returns the list of archConfigs that are used for building @@ -1713,8 +1714,8 @@ func decodeAndroidArchSettings(archConfigs []archConfig) ([]Target, error) { var ret []Target for _, config := range archConfigs { - arch, err := decodeArch(Android, config.arch, &config.archVariant, - &config.cpuVariant, config.abi) + arch, err := decodeArch(Android, config.Arch, &config.ArchVariant, + &config.CpuVariant, config.Abi) if err != nil { return nil, err } @@ -2284,6 +2285,14 @@ func printArchTypeNestedStarlarkDict(dict map[ArchType]map[string][]string) stri return starlark_fmt.PrintDict(valDict, 0) } +func printArchConfigList(arches []archConfig) string { + jsonOut, err := json.MarshalIndent(arches, "", starlark_fmt.Indention(1)) + if err != nil { + panic(fmt.Errorf("Error converting arch configs %#v to json: %q", arches, err)) + } + return fmt.Sprintf("json.decode('''%s''')", string(jsonOut)) +} + func StarlarkArchConfigurations() string { return fmt.Sprintf(` _arch_to_variants = %s @@ -2294,13 +2303,21 @@ _arch_to_features = %s _android_arch_feature_for_arch_variant = %s +_aml_arches = %s + +_ndk_arches = %s + arch_to_variants = _arch_to_variants arch_to_cpu_variants = _arch_to_cpu_variants arch_to_features = _arch_to_features android_arch_feature_for_arch_variants = _android_arch_feature_for_arch_variant +aml_arches = _aml_arches +ndk_arches = _ndk_arches `, printArchTypeStarlarkDict(archVariants), printArchTypeStarlarkDict(cpuVariants), printArchTypeStarlarkDict(archFeatures), printArchTypeNestedStarlarkDict(androidArchFeatureMap), + printArchConfigList(getAmlAbisConfig()), + printArchConfigList(getNdkAbisConfig()), ) } diff --git a/android/bazel_handler.go b/android/bazel_handler.go index d014fc62b..e2751d6c9 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -150,8 +150,9 @@ type BazelContext interface { // ** end Cquery Results Retrieval Functions // Issues commands to Bazel to receive results for all cquery requests - // queued in the BazelContext. - InvokeBazel(config Config) error + // queued in the BazelContext. The ctx argument is optional and is only + // used for performance data collection + InvokeBazel(config Config, ctx *Context) error // Returns true if Bazel handling is enabled for the module with the given name. // Note that this only implies "bazel mixed build" allowlisting. The caller @@ -259,7 +260,7 @@ func (m MockBazelContext) GetCcUnstrippedInfo(label string, _ configKey) (cquery return result, nil } -func (m MockBazelContext) InvokeBazel(_ Config) error { +func (m MockBazelContext) InvokeBazel(_ Config, ctx *Context) error { panic("unimplemented") } @@ -317,7 +318,7 @@ func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (s func (bazelCtx *bazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexInfo, error) { key := makeCqueryKey(label, cquery.GetApexInfo, cfgKey) if rawString, ok := bazelCtx.results[key]; ok { - return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString)), nil + return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString)) } return cquery.ApexInfo{}, fmt.Errorf("no bazel response found for %v", key) } @@ -325,7 +326,7 @@ func (bazelCtx *bazelContext) GetApexInfo(label string, cfgKey configKey) (cquer func (bazelCtx *bazelContext) GetCcUnstrippedInfo(label string, cfgKey configKey) (cquery.CcUnstrippedInfo, error) { key := makeCqueryKey(label, cquery.GetCcUnstrippedInfo, cfgKey) if rawString, ok := bazelCtx.results[key]; ok { - return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString)), nil + return cquery.GetCcUnstrippedInfo.ParseResult(strings.TrimSpace(rawString)) } return cquery.CcUnstrippedInfo{}, fmt.Errorf("no bazel response for %s", key) } @@ -355,7 +356,7 @@ func (n noopBazelContext) GetCcUnstrippedInfo(_ string, _ configKey) (cquery.CcU panic("implement me") } -func (n noopBazelContext) InvokeBazel(_ Config) error { +func (n noopBazelContext) InvokeBazel(_ Config, ctx *Context) error { panic("unimplemented") } @@ -865,51 +866,78 @@ func (p *bazelPaths) outDir() string { return filepath.Dir(p.soongOutDir) } +const buildrootLabel = "@soong_injection//mixed_builds:buildroot" + +var ( + cqueryCmd = bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)} + aqueryCmd = bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)} + buildCmd = bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"} +) + // Issues commands to Bazel to receive results for all cquery requests // queued in the BazelContext. -func (context *bazelContext) InvokeBazel(config Config) error { +func (context *bazelContext) InvokeBazel(config Config, ctx *Context) error { + if ctx != nil { + ctx.EventHandler.Begin("bazel") + defer ctx.EventHandler.End("bazel") + } + + if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" { + if err := os.MkdirAll(metricsDir, 0777); err != nil { + return err + } + } context.results = make(map[cqueryKey]string) + if err := context.runCquery(ctx); err != nil { + return err + } + if err := context.runAquery(config, ctx); err != nil { + return err + } + if err := context.generateBazelSymlinks(ctx); err != nil { + return err + } - var err error + // Clear requests. + context.requests = map[cqueryKey]bool{} + return nil +} +func (context *bazelContext) runCquery(ctx *Context) error { + if ctx != nil { + ctx.EventHandler.Begin("cquery") + defer ctx.EventHandler.End("cquery") + } soongInjectionPath := absolutePath(context.paths.injectedFilesDir()) mixedBuildsPath := filepath.Join(soongInjectionPath, "mixed_builds") if _, err := os.Stat(mixedBuildsPath); os.IsNotExist(err) { err = os.MkdirAll(mixedBuildsPath, 0777) - } - if err != nil { - return err - } - if metricsDir := context.paths.BazelMetricsDir(); metricsDir != "" { - err = os.MkdirAll(metricsDir, 0777) if err != nil { return err } } - if err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil { return err } - if err = ioutil.WriteFile(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil { return err } - if err = ioutil.WriteFile(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil { return err } cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery") - if err = ioutil.WriteFile(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil { + if err := ioutil.WriteFile(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil { return err } - const buildrootLabel = "@soong_injection//mixed_builds:buildroot" - cqueryCmd := bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)} cqueryCommandWithFlag := context.createBazelCommand(context.paths, bazel.CqueryBuildRootRunName, cqueryCmd, "--output=starlark", "--starlark:file="+absolutePath(cqueryFileRelpath)) - cqueryOutput, cqueryErr, err := context.issueBazelCommand(cqueryCommandWithFlag) - if err != nil { - return err + cqueryOutput, cqueryErrorMessage, cqueryErr := context.issueBazelCommand(cqueryCommandWithFlag) + if cqueryErr != nil { + return cqueryErr } cqueryCommandPrint := fmt.Sprintf("cquery command line:\n %s \n\n\n", printableCqueryCommand(cqueryCommandWithFlag)) - if err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryCommandPrint+cqueryOutput), 0666); err != nil { return err } cqueryResults := map[string]string{} @@ -924,10 +952,17 @@ func (context *bazelContext) InvokeBazel(config Config) error { context.results[val] = cqueryResult } else { return fmt.Errorf("missing result for bazel target %s. query output: [%s], cquery err: [%s]", - getCqueryId(val), cqueryOutput, cqueryErr) + getCqueryId(val), cqueryOutput, cqueryErrorMessage) } } + return nil +} +func (context *bazelContext) runAquery(config Config, ctx *Context) error { + if ctx != nil { + ctx.EventHandler.Begin("aquery") + defer ctx.EventHandler.End("aquery") + } // Issue an aquery command to retrieve action information about the bazel build tree. // // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's @@ -937,6 +972,12 @@ func (context *bazelContext) InvokeBazel(config Config) error { extraFlags = append(extraFlags, "--collect_code_coverage") paths := make([]string, 0, 2) if p := config.productVariables.NativeCoveragePaths; len(p) > 0 { + for i, _ := range p { + // TODO(b/259404593) convert path wildcard to regex values + if p[i] == "*" { + p[i] = ".*" + } + } paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ",")) } if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 { @@ -946,26 +987,25 @@ func (context *bazelContext) InvokeBazel(config Config) error { extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ",")) } } - aqueryCmd := bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)} - if aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(context.paths, bazel.AqueryBuildRootRunName, aqueryCmd, - extraFlags...)); err == nil { - context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) - } + aqueryOutput, _, err := context.issueBazelCommand(context.createBazelCommand(context.paths, bazel.AqueryBuildRootRunName, aqueryCmd, + extraFlags...)) if err != nil { return err } + context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) + return err +} +func (context *bazelContext) generateBazelSymlinks(ctx *Context) error { + if ctx != nil { + ctx.EventHandler.Begin("symlinks") + defer ctx.EventHandler.End("symlinks") + } // Issue a build command of the phony root to generate symlink forests for dependencies of the // Bazel build. This is necessary because aquery invocations do not generate this symlink forest, // but some of symlinks may be required to resolve source dependencies of the build. - buildCmd := bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"} - if _, _, err = context.issueBazelCommand(context.createBazelCommand(context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd)); err != nil { - return err - } - - // Clear requests. - context.requests = map[cqueryKey]bool{} - return nil + _, _, err := context.issueBazelCommand(context.createBazelCommand(context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd)) + return err } func (context *bazelContext) BuildStatementsToRegister() []bazel.BuildStatement { diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index 6e3acd5f7..bc16cb53a 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -23,7 +23,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) { bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `@//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`, }) bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg) - err := bazelContext.InvokeBazel(testConfig) + err := bazelContext.InvokeBazel(testConfig, nil) if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } @@ -37,7 +37,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) { func TestInvokeBazelWritesBazelFiles(t *testing.T) { bazelContext, baseDir := testBazelContext(t, map[bazelCommand]string{}) - err := bazelContext.InvokeBazel(testConfig) + err := bazelContext.InvokeBazel(testConfig, nil) if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } @@ -118,7 +118,7 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: string(data)}) - err = bazelContext.InvokeBazel(testConfig) + err = bazelContext.InvokeBazel(testConfig, nil) if err != nil { t.Fatalf("testCase #%d: did not expect error invoking Bazel, but got %s", i+1, err) } @@ -155,6 +155,10 @@ func TestCoverageFlagsAfterInvokeBazel(t *testing.T) { testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=-bar1`) + testConfig.productVariables.NativeCoveragePaths = []string{"*"} + testConfig.productVariables.NativeCoverageExcludePaths = nil + verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+.*`) + testConfig.productVariables.ClangCoverage = boolPtr(false) actual := verifyExtraFlags(t, testConfig, ``) if strings.Contains(actual, "--collect_code_coverage") || @@ -166,7 +170,7 @@ func TestCoverageFlagsAfterInvokeBazel(t *testing.T) { func verifyExtraFlags(t *testing.T, config Config, expected string) string { bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) - err := bazelContext.InvokeBazel(config) + err := bazelContext.InvokeBazel(config, nil) if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } @@ -193,7 +197,7 @@ func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) } aqueryCommand := bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"} if _, exists := bazelCommandResults[aqueryCommand]; !exists { - bazelCommandResults[aqueryCommand] = "{}\n" + bazelCommandResults[aqueryCommand] = "" } runner := &mockBazelRunner{bazelCommandResults: bazelCommandResults} return &bazelContext{ diff --git a/android/config.go b/android/config.go index 0c3e2d1f1..1deb7d4ae 100644 --- a/android/config.go +++ b/android/config.go @@ -1272,10 +1272,6 @@ func (c *deviceConfig) NativeCoverageEnabledForPath(path string) bool { return coverage } -func (c *deviceConfig) AfdoAdditionalProfileDirs() []string { - return c.config.productVariables.AfdoAdditionalProfileDirs -} - func (c *deviceConfig) PgoAdditionalProfileDirs() []string { return c.config.productVariables.PgoAdditionalProfileDirs } diff --git a/android/defs.go b/android/defs.go index 2a28e98a6..9ae360e36 100644 --- a/android/defs.go +++ b/android/defs.go @@ -25,7 +25,8 @@ import ( ) var ( - pctx = NewPackageContext("android/soong/android") + pctx = NewPackageContext("android/soong/android") + exportedVars = NewExportedVariables(pctx) cpPreserveSymlinks = pctx.VariableConfigMethod("cpPreserveSymlinks", Config.CpPreserveSymlinksFlags) @@ -128,6 +129,13 @@ func init() { pctx.VariableFunc("RBEWrapper", func(ctx PackageVarContext) string { return ctx.Config().RBEWrapper() }) + + exportedVars.ExportStringList("NeverAllowNotInIncludeDir", neverallowNotInIncludeDir) + exportedVars.ExportStringList("NeverAllowNoUseIncludeDir", neverallowNoUseIncludeDir) +} + +func BazelCcToolchainVars(config Config) string { + return BazelToolchainVars(config, exportedVars) } var ( diff --git a/android/fixture.go b/android/fixture.go index f33e7189d..3f01f5a0d 100644 --- a/android/fixture.go +++ b/android/fixture.go @@ -658,7 +658,7 @@ func (t *TestPathContext) AddNinjaFileDeps(deps ...string) { func createFixture(t *testing.T, buildDir string, preparers []*simpleFixturePreparer) Fixture { config := TestConfig(buildDir, nil, "", nil) - ctx := NewTestContext(config) + ctx := newTestContextForFixture(config) fixture := &fixture{ preparers: preparers, t: t, @@ -790,6 +790,16 @@ func (f *fixture) RunTest() *TestResult { } } + // Create and set the Context's NameInterface. It needs to be created here as it depends on the + // configuration that has been prepared for this fixture. + resolver := NewNameResolver(ctx.config) + + // Set the NameInterface in the main Context. + ctx.SetNameInterface(resolver) + + // Set the NameResolver in the TestContext. + ctx.NameResolver = resolver + ctx.Register() var ninjaDeps []string extraNinjaDeps, errs := ctx.ParseBlueprintsFiles("ignored") diff --git a/android/gen_notice.go b/android/gen_notice.go index 008aac5e2..28fddbcee 100644 --- a/android/gen_notice.go +++ b/android/gen_notice.go @@ -61,6 +61,9 @@ func (s *genNoticeBuildRules) GenerateBuildActions(ctx SingletonContext) { if mod == nil { continue } + if !mod.Enabled() { // don't depend on variants without build rules + continue + } modules = append(modules, mod) } } diff --git a/android/namespace.go b/android/namespace.go index a3ff76125..b43ffdf19 100644 --- a/android/namespace.go +++ b/android/namespace.go @@ -91,7 +91,27 @@ type NameResolver struct { namespaceExportFilter func(*Namespace) bool } -func NewNameResolver(namespaceExportFilter func(*Namespace) bool) *NameResolver { +// NameResolverConfig provides the subset of the Config interface needed by the +// NewNameResolver function. +type NameResolverConfig interface { + // ExportedNamespaces is the list of namespaces that Soong must export to + // make. + ExportedNamespaces() []string +} + +func NewNameResolver(config NameResolverConfig) *NameResolver { + namespacePathsToExport := make(map[string]bool) + + for _, namespaceName := range config.ExportedNamespaces() { + namespacePathsToExport[namespaceName] = true + } + + namespacePathsToExport["."] = true // always export the root namespace + + namespaceExportFilter := func(namespace *Namespace) bool { + return namespacePathsToExport[namespace.Path] + } + r := &NameResolver{ namespacesByDir: sync.Map{}, namespaceExportFilter: namespaceExportFilter, diff --git a/android/namespace_test.go b/android/namespace_test.go index 87d13206b..7a387a0bd 100644 --- a/android/namespace_test.go +++ b/android/namespace_test.go @@ -602,6 +602,36 @@ func TestRename(t *testing.T) { // RunTest will report any errors } +func TestNamespace_Exports(t *testing.T) { + result := GroupFixturePreparers( + prepareForTestWithNamespace, + FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.NamespacesToExport = []string{"dir1"} + }), + dirBpToPreparer(map[string]string{ + "dir1": ` + soong_namespace { + } + test_module { + name: "a", + } + `, + "dir2": ` + soong_namespace { + } + test_module { + name: "b", + } + `, + }), + ).RunTest(t) + + aModule := result.Module("a", "") + AssertBoolEquals(t, "a exported", true, aModule.ExportedToMake()) + bModule := result.Module("b", "") + AssertBoolEquals(t, "b not exported", false, bModule.ExportedToMake()) +} + // some utils to support the tests var prepareForTestWithNamespace = GroupFixturePreparers( diff --git a/android/neverallow.go b/android/neverallow.go index d28843995..293bac865 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -74,8 +74,8 @@ func createBp2BuildRule() Rule { "supported for custom conversion, use allowlists.go instead.") } -func createIncludeDirsRules() []Rule { - notInIncludeDir := []string{ +var ( + neverallowNotInIncludeDir = []string{ "art", "art/libnativebridge", "art/libnativeloader", @@ -91,7 +91,7 @@ func createIncludeDirsRules() []Rule { "external/vixl", "external/wycheproof", } - noUseIncludeDir := []string{ + neverallowNoUseIncludeDir = []string{ "frameworks/av/apex", "frameworks/av/tools", "frameworks/native/cmds", @@ -103,10 +103,12 @@ func createIncludeDirsRules() []Rule { "system/libfmq", "system/libvintf", } +) - rules := make([]Rule, 0, len(notInIncludeDir)+len(noUseIncludeDir)) +func createIncludeDirsRules() []Rule { + rules := make([]Rule, 0, len(neverallowNotInIncludeDir)+len(neverallowNoUseIncludeDir)) - for _, path := range notInIncludeDir { + for _, path := range neverallowNotInIncludeDir { rule := NeverAllow(). WithMatcher("include_dirs", StartsWith(path+"/")). @@ -116,7 +118,7 @@ func createIncludeDirsRules() []Rule { rules = append(rules, rule) } - for _, path := range noUseIncludeDir { + for _, path := range neverallowNoUseIncludeDir { rule := NeverAllow().In(path+"/").WithMatcher("include_dirs", isSetMatcherInstance). Because("include_dirs is deprecated, all usages of them in '" + path + "' have been migrated" + " to use alternate mechanisms and so can no longer be used.") diff --git a/android/sdk.go b/android/sdk.go index bd2f5d13f..fc0a84eb8 100644 --- a/android/sdk.go +++ b/android/sdk.go @@ -232,12 +232,6 @@ type SnapshotBuilder interface { // relative path) and add the dest to the zip. CopyToSnapshot(src Path, dest string) - // EmptyFile returns the path to an empty file. - // - // This can be used by sdk member types that need to create an empty file in the snapshot, simply - // pass the value returned from this to the CopyToSnapshot() method. - EmptyFile() Path - // UnzipToSnapshot generates a rule that will unzip the supplied zip into the snapshot relative // directory destDir. UnzipToSnapshot(zipPath Path, destDir string) @@ -264,6 +258,14 @@ type SnapshotBuilder interface { // See sdk/update.go for more information. AddPrebuiltModule(member SdkMember, moduleType string) BpModule + // AddInternalModule creates a new module in the generated Android.bp file that can only be + // referenced by one of the other modules in the snapshot. + // + // The created module's name is constructed by concatenating the name of this member and the + // nameSuffix, separated by "-". It also has the visibility property set to "//visibility:private" + // to prevent it from being inadvertently accessed from outside the snapshot. + AddInternalModule(properties SdkMemberProperties, moduleType string, nameSuffix string) BpModule + // SdkMemberReferencePropertyTag returns a property tag to use when adding a property to a // BpModule that contains references to other sdk members. // @@ -922,6 +924,12 @@ func RegisterSdkMemberType(memberType SdkMemberType) { // // Contains common properties that apply across many different member types. type SdkMemberPropertiesBase struct { + // The name of the member. + // + // Ignore this property during optimization. This is needed because this property is the same for + // all variants of a member and so would be optimized away if it was not ignored. + MemberName string `sdk:"ignore"` + // The number of unique os types supported by the member variants. // // If a member has a variant with more than one os type then it will need to differentiate @@ -945,6 +953,10 @@ type SdkMemberPropertiesBase struct { Compile_multilib string `android:"arch_variant"` } +func (b *SdkMemberPropertiesBase) Name() string { + return b.MemberName +} + // OsPrefix returns the os prefix to use for any file paths in the sdk. // // Is an empty string if the member only provides variants for a single os type, otherwise @@ -970,6 +982,8 @@ type SdkMemberProperties interface { // Base returns the base structure. Base() *SdkMemberPropertiesBase + Name() string + // PopulateFromVariant populates this structure with information from a module variant. // // It will typically be called once for each variant of a member module that the SDK depends upon. diff --git a/android/testing.go b/android/testing.go index 2256c96bc..8fcf4409b 100644 --- a/android/testing.go +++ b/android/testing.go @@ -30,19 +30,11 @@ import ( "github.com/google/blueprint/proptools" ) -func NewTestContext(config Config) *TestContext { - namespaceExportFilter := func(namespace *Namespace) bool { - return true - } - - nameResolver := NewNameResolver(namespaceExportFilter) +func newTestContextForFixture(config Config) *TestContext { ctx := &TestContext{ - Context: &Context{blueprint.NewContext(), config}, - NameResolver: nameResolver, + Context: &Context{blueprint.NewContext(), config}, } - ctx.SetNameInterface(nameResolver) - ctx.postDeps = append(ctx.postDeps, registerPathDepsMutator) ctx.SetFs(ctx.config.fs) @@ -53,6 +45,16 @@ func NewTestContext(config Config) *TestContext { return ctx } +func NewTestContext(config Config) *TestContext { + ctx := newTestContextForFixture(config) + + nameResolver := NewNameResolver(config) + ctx.NameResolver = nameResolver + ctx.SetNameInterface(nameResolver) + + return ctx +} + var PrepareForTestWithArchMutator = GroupFixturePreparers( // Configure architecture targets in the fixture config. FixtureModifyConfig(modifyTestConfigToSupportArchMutator), diff --git a/android/variable.go b/android/variable.go index 37ecab51d..28f22c9b3 100644 --- a/android/variable.go +++ b/android/variable.go @@ -336,8 +336,7 @@ type productVariables struct { NamespacesToExport []string `json:",omitempty"` - AfdoAdditionalProfileDirs []string `json:",omitempty"` - PgoAdditionalProfileDirs []string `json:",omitempty"` + PgoAdditionalProfileDirs []string `json:",omitempty"` VndkUseCoreVariant *bool `json:",omitempty"` VndkSnapshotBuildArtifacts *bool `json:",omitempty"` diff --git a/apex/apex.go b/apex/apex.go index 72403f9ba..04808c13b 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -455,9 +455,6 @@ type apexBundle struct { // Processed file_contexts files fileContexts android.WritablePath - // Path to notice file in html.gz format. - htmlGzNotice android.WritablePath - // The built APEX file. This is the main product. // Could be .apex or .capex outputFile android.WritablePath @@ -1903,14 +1900,12 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) { apexType := a.properties.ApexType switch apexType { case imageApex: - - // TODO(b/190817312): Generate the notice file from the apex rule. - a.htmlGzNotice = android.PathForBazelOut(ctx, "NOTICE.html.gz") a.bundleModuleFile = android.PathForBazelOut(ctx, outputs.BundleFile) a.nativeApisUsedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, outputs.SymbolsUsedByApex)) a.nativeApisBackedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, outputs.BackingLibs)) // TODO(b/239084755): Generate the java api using.xml file from Bazel. a.javaApisUsedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, outputs.JavaSymbolsUsedByApex)) + a.installedFilesFile = android.ModuleOutPath(android.PathForBazelOut(ctx, outputs.InstalledFiles)) installSuffix := imageApexSuffix if a.isCompressed { installSuffix = imageCapexSuffix @@ -2507,7 +2502,8 @@ func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint. filesToAdd = append(filesToAdd, *af) } - if pathInApex := bootclasspathFragmentInfo.ProfileInstallPathInApex(); pathInApex != "" { + pathInApex := bootclasspathFragmentInfo.ProfileInstallPathInApex() + if pathInApex != "" && !java.SkipDexpreoptBootJars(ctx) { pathOnHost := bootclasspathFragmentInfo.ProfilePathOnHost() tempPath := android.PathForModuleOut(ctx, "boot_image_profile", pathInApex) diff --git a/apex/apex_test.go b/apex/apex_test.go index 985ad591f..ea3e73470 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -9775,6 +9775,7 @@ apex { SymbolsUsedByApex: "foo_using.txt", JavaSymbolsUsedByApex: "foo_using.xml", BundleFile: "apex_bundle.zip", + InstalledFiles: "installed-files.txt", // unused PackageName: "pkg_name", @@ -9820,6 +9821,10 @@ apex { t.Errorf("Expected output file %q, got %q", w, g) } + if w, g := "out/bazel/execroot/__main__/installed-files.txt", ab.installedFilesFile.String(); w != g { + t.Errorf("Expected installed-files.txt %q, got %q", w, g) + } + mkData := android.AndroidMkDataForTest(t, result.TestContext, m) var builder strings.Builder mkData.Custom(&builder, "foo", "BAZEL_TARGET_", "", mkData) @@ -9828,4 +9833,7 @@ apex { if w := "ALL_MODULES.$(my_register_name).BUNDLE := out/bazel/execroot/__main__/apex_bundle.zip"; !strings.Contains(data, w) { t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) } + if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/installed-files.txt:foo-installed-files.txt)"; !strings.Contains(data, w) { + t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) + } } diff --git a/apex/builder.go b/apex/builder.go index e4c1673c4..9e368b604 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -650,9 +650,9 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { } // Create a NOTICE file, and embed it as an asset file in the APEX. - a.htmlGzNotice = android.PathForModuleOut(ctx, "NOTICE.html.gz") + htmlGzNotice := android.PathForModuleOut(ctx, "NOTICE.html.gz") android.BuildNoticeHtmlOutputFromLicenseMetadata( - ctx, a.htmlGzNotice, "", "", + ctx, htmlGzNotice, "", "", []string{ android.PathForModuleInstall(ctx).String() + "/", android.PathForModuleInPartitionInstall(ctx, "apex").String() + "/", @@ -660,7 +660,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { noticeAssetPath := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("cp"). - Input(a.htmlGzNotice). + Input(htmlGzNotice). Output(noticeAssetPath) builder.Build("notice_dir", "Building notice dir") implicitInputs = append(implicitInputs, noticeAssetPath) diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 5bf9fb255..b675f1710 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -194,7 +194,9 @@ return json_encode({ // Starlark given in StarlarkFunctionBody. func (g getCcInfoType) ParseResult(rawString string) (CcInfo, error) { var ccInfo CcInfo - parseJson(rawString, &ccInfo) + if err := parseJson(rawString, &ccInfo); err != nil { + return ccInfo, err + } return ccInfo, nil } @@ -230,6 +232,7 @@ return json_encode({ "java_symbols_used_by_apex": info.java_symbols_used_by_apex.path, "backing_libs": info.backing_libs.path, "bundle_file": info.base_with_config_zip.path, + "installed_files": info.installed_files.path, })` } @@ -245,15 +248,16 @@ type ApexInfo struct { JavaSymbolsUsedByApex string `json:"java_symbols_used_by_apex"` BackingLibs string `json:"backing_libs"` BundleFile string `json:"bundle_file"` + InstalledFiles string `json:"installed_files"` } // ParseResult returns a value obtained by parsing the result of the request's Starlark function. // The given rawString must correspond to the string output which was created by evaluating the // Starlark given in StarlarkFunctionBody. -func (g getApexInfoType) ParseResult(rawString string) ApexInfo { +func (g getApexInfoType) ParseResult(rawString string) (ApexInfo, error) { var info ApexInfo - parseJson(rawString, &info) - return info + err := parseJson(rawString, &info) + return info, err } // getCcUnstrippedInfoType implements cqueryRequest interface. It handles the @@ -282,10 +286,10 @@ return json_encode({ // ParseResult returns a value obtained by parsing the result of the request's Starlark function. // The given rawString must correspond to the string output which was created by evaluating the // Starlark given in StarlarkFunctionBody. -func (g getCcUnstippedInfoType) ParseResult(rawString string) CcUnstrippedInfo { +func (g getCcUnstippedInfoType) ParseResult(rawString string) (CcUnstrippedInfo, error) { var info CcUnstrippedInfo - parseJson(rawString, &info) - return info + err := parseJson(rawString, &info) + return info, err } type CcUnstrippedInfo struct { @@ -305,10 +309,12 @@ func splitOrEmpty(s string, sep string) []string { // parseJson decodes json string into the fields of the receiver. // Unknown attribute name causes panic. -func parseJson(jsonString string, info interface{}) { +func parseJson(jsonString string, info interface{}) error { decoder := json.NewDecoder(strings.NewReader(jsonString)) decoder.DisallowUnknownFields() //useful to detect typos, e.g. in unit tests - if err := decoder.Decode(info); err != nil { - panic(fmt.Errorf("cannot parse cquery result '%s': %s", jsonString, err)) + err := decoder.Decode(info) + if err != nil { + return fmt.Errorf("cannot parse cquery result '%s': %s", jsonString, err) } + return nil } diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go index 86f128e15..1d30535f0 100644 --- a/bazel/cquery/request_type_test.go +++ b/bazel/cquery/request_type_test.go @@ -3,10 +3,12 @@ package cquery import ( "encoding/json" "reflect" + "strings" "testing" ) func TestGetOutputFilesParseResults(t *testing.T) { + t.Parallel() testCases := []struct { description string input string @@ -29,14 +31,17 @@ func TestGetOutputFilesParseResults(t *testing.T) { }, } for _, tc := range testCases { - actualOutput := GetOutputFiles.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("%q: expected %#v != actual %#v", tc.description, tc.expectedOutput, actualOutput) - } + t.Run(tc.description, func(t *testing.T) { + actualOutput := GetOutputFiles.ParseResult(tc.input) + if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { + t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) + } + }) } } func TestGetPythonBinaryParseResults(t *testing.T) { + t.Parallel() testCases := []struct { description string input string @@ -54,14 +59,17 @@ func TestGetPythonBinaryParseResults(t *testing.T) { }, } for _, tc := range testCases { - actualOutput := GetPythonBinary.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("%q: expected %#v != actual %#v", tc.description, tc.expectedOutput, actualOutput) - } + t.Run(tc.description, func(t *testing.T) { + actualOutput := GetPythonBinary.ParseResult(tc.input) + if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { + t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) + } + }) } } func TestGetCcInfoParseResults(t *testing.T) { + t.Parallel() testCases := []struct { description string inputCcInfo CcInfo @@ -73,24 +81,6 @@ func TestGetCcInfoParseResults(t *testing.T) { expectedOutput: CcInfo{}, }, { - description: "only output", - inputCcInfo: CcInfo{ - OutputFiles: []string{"test", "test3"}, - }, - expectedOutput: CcInfo{ - OutputFiles: []string{"test", "test3"}, - }, - }, - { - description: "only ToC", - inputCcInfo: CcInfo{ - TocFile: "test", - }, - expectedOutput: CcInfo{ - TocFile: "test", - }, - }, - { description: "all items set", inputCcInfo: CcInfo{ OutputFiles: []string{"out1", "out2"}, @@ -119,17 +109,51 @@ func TestGetCcInfoParseResults(t *testing.T) { }, } for _, tc := range testCases { - jsonInput, _ := json.Marshal(tc.inputCcInfo) - actualOutput, err := GetCcInfo.ParseResult(string(jsonInput)) - if err != nil { - t.Errorf("%q:\n test case get error: %q", tc.description, err) - } else if err == nil && !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("%q:\n expected %#v\n!= actual %#v", tc.description, tc.expectedOutput, actualOutput) - } + t.Run(tc.description, func(t *testing.T) { + jsonInput, _ := json.Marshal(tc.inputCcInfo) + actualOutput, err := GetCcInfo.ParseResult(string(jsonInput)) + if err != nil { + t.Errorf("error parsing result: %q", err) + } else if err == nil && !reflect.DeepEqual(tc.expectedOutput, actualOutput) { + t.Errorf("expected %#v\n!= actual %#v", tc.expectedOutput, actualOutput) + } + }) + } +} + +func TestGetCcInfoParseResultsError(t *testing.T) { + t.Parallel() + testCases := []struct { + description string + input string + expectedError string + }{ + { + description: "not json", + input: ``, + expectedError: `cannot parse cquery result '': EOF`, + }, + { + description: "invalid field", + input: `{ + "toc_file": "dir/file.so.toc" +}`, + expectedError: `json: unknown field "toc_file"`, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + _, err := GetCcInfo.ParseResult(tc.input) + if !strings.Contains(err.Error(), tc.expectedError) { + t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) + } + }) } } func TestGetApexInfoParseResults(t *testing.T) { + t.Parallel() testCases := []struct { description string input string @@ -152,6 +176,7 @@ func TestGetApexInfoParseResults(t *testing.T) { "symbols_used_by_apex": "path/to/my.apex_using.txt", "backing_libs":"path/to/backing.txt", "bundle_file": "dir/bundlefile.zip", + "installed_files":"path/to/installed-files.txt", "provides_native_libs":[] }`, expectedOutput: ApexInfo{ @@ -165,18 +190,56 @@ func TestGetApexInfoParseResults(t *testing.T) { SymbolsUsedByApex: "path/to/my.apex_using.txt", BackingLibs: "path/to/backing.txt", BundleFile: "dir/bundlefile.zip", + InstalledFiles: "path/to/installed-files.txt", }, }, } for _, tc := range testCases { - actualOutput := GetApexInfo.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("%q: expected %#v != actual %#v", tc.description, tc.expectedOutput, actualOutput) - } + t.Run(tc.description, func(t *testing.T) { + actualOutput, err := GetApexInfo.ParseResult(tc.input) + if err != nil { + t.Errorf("Unexpected error %q", err) + } + if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { + t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) + } + }) + } +} + +func TestGetApexInfoParseResultsError(t *testing.T) { + t.Parallel() + testCases := []struct { + description string + input string + expectedError string + }{ + { + description: "not json", + input: ``, + expectedError: `cannot parse cquery result '': EOF`, + }, + { + description: "invalid field", + input: `{ + "fake_field": "path/to/file" +}`, + expectedError: `json: unknown field "fake_field"`, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + _, err := GetApexInfo.ParseResult(tc.input) + if !strings.Contains(err.Error(), tc.expectedError) { + t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) + } + }) } } func TestGetCcUnstrippedParseResults(t *testing.T) { + t.Parallel() testCases := []struct { description string input string @@ -197,9 +260,45 @@ func TestGetCcUnstrippedParseResults(t *testing.T) { }, } for _, tc := range testCases { - actualOutput := GetCcUnstrippedInfo.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("%q: expected %#v != actual %#v", tc.description, tc.expectedOutput, actualOutput) - } + t.Run(tc.description, func(t *testing.T) { + actualOutput, err := GetCcUnstrippedInfo.ParseResult(tc.input) + if err != nil { + t.Errorf("Unexpected error %q", err) + } + if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { + t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) + } + }) + } +} + +func TestGetCcUnstrippedParseResultsErrors(t *testing.T) { + t.Parallel() + testCases := []struct { + description string + input string + expectedError string + }{ + { + description: "not json", + input: ``, + expectedError: `cannot parse cquery result '': EOF`, + }, + { + description: "invalid field", + input: `{ + "fake_field": "path/to/file" +}`, + expectedError: `json: unknown field "fake_field"`, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + _, err := GetCcUnstrippedInfo.ParseResult(tc.input) + if !strings.Contains(err.Error(), tc.expectedError) { + t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) + } + }) } } diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 0d6d5b87e..4c86374b5 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3508,3 +3508,38 @@ cc_library_static { }, }) } + +func TestCcLibraryWithTidy(t *testing.T) { + runCcLibraryTestCase(t, Bp2buildTestCase{ + Description: "cc_library uses tidy properties", + ModuleTypeUnderTest: "cc_library", + ModuleTypeUnderTestFactory: cc.LibraryFactory, + Blueprint: ` +cc_library_static { + name: "foo", + srcs: ["foo.cpp"], + tidy: true, + tidy_checks: ["check1", "check2"], + tidy_checks_as_errors: ["check1error", "check2error"], + tidy_disabled_srcs: ["bar.cpp"], + tidy_timeout_srcs: ["baz.cpp"], +}`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("cc_library_static", "foo", AttrNameToString{ + "local_includes": `["."]`, + "srcs": `["foo.cpp"]`, + "tidy": `True`, + "tidy_checks": `[ + "check1", + "check2", + ]`, + "tidy_checks_as_errors": `[ + "check1error", + "check2error", + ]`, + "tidy_disabled_srcs": `["bar.cpp"]`, + "tidy_timeout_srcs": `["baz.cpp"]`, + }), + }, + }) +} diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 8ca13b8af..6eb93bcd5 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -23,6 +23,9 @@ type BazelFile struct { func CreateSoongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []BazelFile { var files []BazelFile + files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. + files = append(files, newFile("android", "constants.bzl", android.BazelCcToolchainVars(cfg))) + files = append(files, newFile("cc_toolchain", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. files = append(files, newFile("cc_toolchain", "constants.bzl", cc_config.BazelCcToolchainVars(cfg))) diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index cfd612815..8de2f83c0 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -88,6 +88,14 @@ func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { expectedFilePaths := []bazelFilepath{ { + dir: "android", + basename: GeneratedBuildFileName, + }, + { + dir: "android", + basename: "constants.bzl", + }, + { dir: "cc_toolchain", basename: GeneratedBuildFileName, }, diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index 74e2dbd09..00056f8ff 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -649,3 +649,46 @@ android_library { }), }}) } + +func TestJavaLibraryKotlinSrcs(t *testing.T) { + runJavaLibraryTestCase(t, Bp2buildTestCase{ + Description: "java_library with kotlin srcs", + Blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java", "b.java", "c.kt"], + bazel_module: { bp2build_available: true }, +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("kt_jvm_library", "java-lib-1", AttrNameToString{ + "srcs": `[ + "a.java", + "b.java", + "c.kt", + ]`, + }), + }, + }) +} + +func TestJavaLibraryKotlinCommonSrcs(t *testing.T) { + runJavaLibraryTestCase(t, Bp2buildTestCase{ + Description: "java_library with kotlin common_srcs", + Blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java", "b.java"], + common_srcs: ["c.kt"], + bazel_module: { bp2build_available: true }, +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("kt_jvm_library", "java-lib-1", AttrNameToString{ + "srcs": `[ + "a.java", + "b.java", + ]`, + "common_srcs": `["c.kt"]`, + }), + }, + }) +} diff --git a/bpf/bpf.go b/bpf/bpf.go index 60a410d7f..a840fa3df 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -18,11 +18,13 @@ import ( "fmt" "io" "path/filepath" + "runtime" "strings" "android/soong/android" "android/soong/bazel" "android/soong/bazel/cquery" + "android/soong/cc" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -31,6 +33,7 @@ import ( func init() { registerBpfBuildComponents(android.InitRegistrationContext) pctx.Import("android/soong/cc/config") + pctx.StaticVariable("relPwd", cc.PwdPrefix()) } var ( @@ -40,7 +43,7 @@ var ( blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, - Command: "$ccCmd --target=bpf -c $cFlags -MD -MF ${out}.d -o $out $in", + Command: "$relPwd $ccCmd --target=bpf -c $cFlags -MD -MF ${out}.d -o $out $in", CommandDeps: []string{"$ccCmd"}, }, "ccCmd", "cFlags") @@ -164,6 +167,9 @@ func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { if proptools.Bool(bpf.properties.Btf) { cflags = append(cflags, "-g") + if runtime.GOOS != "darwin" { + cflags = append(cflags, "-fdebug-prefix-map=/proc/self/cwd=") + } } srcs := android.PathsForModuleSrc(ctx, bpf.properties.Srcs) diff --git a/cc/afdo.go b/cc/afdo.go index fb66bbe52..d36f4afcf 100644 --- a/cc/afdo.go +++ b/cc/afdo.go @@ -36,7 +36,7 @@ const afdoCFlagsFormat = "-funique-internal-linkage-names -fprofile-sample-accur func getAfdoProfileProjects(config android.DeviceConfig) []string { return config.OnceStringSlice(afdoProfileProjectsConfigKey, func() []string { - return append(globalAfdoProfileProjects, config.AfdoAdditionalProfileDirs()...) + return globalAfdoProfileProjects }) } diff --git a/cc/binary.go b/cc/binary.go index d09e74458..c2868e7bb 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -646,7 +646,7 @@ func binaryBp2buildAttrs(ctx android.TopDownMutatorContext, m *Module) binaryAtt sdkAttributes: bp2BuildParseSdkAttributes(m), } - m.convertTidyAttributes(&attrs.tidyAttributes) + m.convertTidyAttributes(ctx, &attrs.tidyAttributes) return attrs } diff --git a/cc/bp2build.go b/cc/bp2build.go index 2f79caec7..6caa85422 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -75,9 +75,11 @@ type tidyAttributes struct { Tidy_flags []string Tidy_checks []string Tidy_checks_as_errors []string + Tidy_disabled_srcs bazel.LabelListAttribute + Tidy_timeout_srcs bazel.LabelListAttribute } -func (m *Module) convertTidyAttributes(moduleAttrs *tidyAttributes) { +func (m *Module) convertTidyAttributes(ctx android.BaseMutatorContext, moduleAttrs *tidyAttributes) { for _, f := range m.features { if tidy, ok := f.(*tidyFeature); ok { moduleAttrs.Tidy = tidy.Properties.Tidy @@ -85,6 +87,18 @@ func (m *Module) convertTidyAttributes(moduleAttrs *tidyAttributes) { moduleAttrs.Tidy_checks = tidy.Properties.Tidy_checks moduleAttrs.Tidy_checks_as_errors = tidy.Properties.Tidy_checks_as_errors } + + } + archVariantProps := m.GetArchVariantProperties(ctx, &BaseCompilerProperties{}) + for axis, configToProps := range archVariantProps { + for config, _props := range configToProps { + if archProps, ok := _props.(*BaseCompilerProperties); ok { + archDisabledSrcs := android.BazelLabelForModuleSrc(ctx, archProps.Tidy_disabled_srcs) + moduleAttrs.Tidy_disabled_srcs.SetSelectValue(axis, config, archDisabledSrcs) + archTimeoutSrcs := android.BazelLabelForModuleSrc(ctx, archProps.Tidy_timeout_srcs) + moduleAttrs.Tidy_timeout_srcs.SetSelectValue(axis, config, archTimeoutSrcs) + } + } } } diff --git a/cc/builder.go b/cc/builder.go index 75e473669..46cea0b0b 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -924,9 +924,9 @@ func unzipRefDump(ctx android.ModuleContext, zippedRefDump android.Path, baseNam } // sourceAbiDiff registers a build statement to compare linked sAbi dump files (.lsdump). -func sourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceDump android.Path, - baseName, exportedHeaderFlags string, diffFlags []string, prevVersion int, - checkAllApis, isLlndk, isNdk, isVndkExt, previousVersionDiff bool) android.OptionalPath { +func sourceAbiDiff(ctx android.ModuleContext, inputDump, referenceDump android.Path, + baseName string, diffFlags []string, prevVersion int, + checkAllApis, isLlndkOrNdk, isVndkExt, previousVersionDiff bool) android.OptionalPath { var outputFile android.ModuleOutPath if previousVersionDiff { @@ -955,7 +955,7 @@ func sourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceD extraFlags = append(extraFlags, "-target-version", "current") } - if isLlndk || isNdk { + if isLlndkOrNdk { extraFlags = append(extraFlags, "-consider-opaque-types-different") } if isVndkExt || previousVersionDiff { diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index b53a09782..981d1eaa6 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -222,6 +222,7 @@ var ( "": "${config.ArmGenericCflags}", "cortex-a7": "${config.ArmCortexA7Cflags}", "cortex-a8": "${config.ArmCortexA8Cflags}", + "cortex-a9": "${config.ArmGenericCflags}", "cortex-a15": "${config.ArmCortexA15Cflags}", "cortex-a53": "${config.ArmCortexA53Cflags}", "cortex-a53.a57": "${config.ArmCortexA53Cflags}", diff --git a/cc/config/global.go b/cc/config/global.go index 9f1878445..e5c0a8e0f 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -292,6 +292,8 @@ var ( llvmNextExtraCommonGlobalCflags = []string{ // New warnings to be fixed after clang-r475365 "-Wno-error=single-bit-bitfield-constant-conversion", // http://b/243965903 + // Skip deprecated flags. + "-Wno-unused-command-line-argument", } IllegalFlags = []string{ diff --git a/cc/config/riscv64_device.go b/cc/config/riscv64_device.go index 825be7fac..67208b261 100644 --- a/cc/config/riscv64_device.go +++ b/cc/config/riscv64_device.go @@ -32,7 +32,6 @@ var ( riscv64Ldflags = []string{ "-Wl,--hash-style=gnu", - "-Wl,-z,separate-code", } riscv64Lldflags = append(riscv64Ldflags, diff --git a/cc/config/tidy.go b/cc/config/tidy.go index f32ebf25b..1180da40a 100644 --- a/cc/config/tidy.go +++ b/cc/config/tidy.go @@ -55,6 +55,21 @@ var ( // http://b/241819232 "-misc-const-correctness", } + + extraArgFlags = []string{ + // We might be using the static analyzer through clang tidy. + // https://bugs.llvm.org/show_bug.cgi?id=32914 + "-D__clang_analyzer__", + + // A recent change in clang-tidy (r328258) enabled destructor inlining, which + // appears to cause a number of false positives. Until that's resolved, this turns + // off the effects of r328258. + // https://bugs.llvm.org/show_bug.cgi?id=37459 + "-Xclang", + "-analyzer-config", + "-Xclang", + "c++-temp-dtor-inlining=false", + } ) func init() { @@ -145,6 +160,8 @@ func init() { return strings.Join(globalNoErrorCheckList, ",") }) + exportedVars.ExportStringListStaticVariable("TidyExtraArgFlags", extraArgFlags) + // To reduce duplicate warnings from the same header files, // header-filter will contain only the module directory and // those specified by DEFAULT_TIDY_HEADER_DIRS. @@ -235,6 +252,10 @@ func TidyGlobalNoErrorChecks() string { return "" } +func TidyExtraArgFlags() []string { + return extraArgFlags +} + func TidyFlagsForSrcFile(srcFile android.Path, flags string) string { // Disable clang-analyzer-* checks globally for generated source files // because some of them are too huge. Local .bp files can add wanted diff --git a/cc/library.go b/cc/library.go index ef10e7cc2..b63993004 100644 --- a/cc/library.go +++ b/cc/library.go @@ -1875,25 +1875,21 @@ func prevDumpRefVersion(ctx ModuleContext) int { } } +func currRefAbiDumpVersion(ctx ModuleContext, isVndk bool) string { + if isVndk { + // Each version of VNDK is independent, so follow the VNDK version which is the codename or PLATFORM_SDK_VERSION. + return ctx.Module().(*Module).VndkVersion() + } else if ctx.Config().PlatformSdkFinal() { + // After sdk finalization, the ABI of the latest API level must be consistent with the source code, + // so choose PLATFORM_SDK_VERSION as the current version. + return ctx.Config().PlatformSdkVersion().String() + } else { + return "current" + } +} + func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objects, fileName string, soFile android.Path) { if library.sabi.shouldCreateSourceAbiDump() { - var version string - var prevVersion int - - if ctx.useVndk() { - // For modules linking against vndk, follow its vndk version - version = ctx.Module().(*Module).VndkVersion() - } else { - // After sdk finalizatoin, the ABI of the latest API level must be consistent with the source code - // so the chosen reference dump is the PLATFORM_SDK_VERSION. - if ctx.Config().PlatformSdkFinal() { - version = ctx.Config().PlatformSdkVersion().String() - } else { - version = "current" - } - prevVersion = prevDumpRefVersion(ctx) - } - exportIncludeDirs := library.flagExporter.exportedIncludes(ctx) var SourceAbiFlags []string for _, dir := range exportIncludeDirs.Strings() { @@ -1910,26 +1906,31 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objec addLsdumpPath(classifySourceAbiDump(ctx) + ":" + library.sAbiOutputFile.String()) + isVndk := ctx.useVndk() && ctx.isVndk() + isNdk := ctx.isNdk(ctx.Config()) + isLlndk := ctx.isImplementationForLLNDKPublic() // If NDK or PLATFORM library, check against previous version ABI. - if !ctx.useVndk() { + if !isVndk { + prevVersion := prevDumpRefVersion(ctx) prevRefAbiDumpFile := getRefAbiDumpFile(ctx, strconv.Itoa(prevVersion), fileName) if prevRefAbiDumpFile != nil { library.prevSAbiDiff = sourceAbiDiff(ctx, library.sAbiOutputFile.Path(), - prevRefAbiDumpFile, fileName, exportedHeaderFlags, + prevRefAbiDumpFile, fileName, library.Properties.Header_abi_checker.Diff_flags, prevVersion, Bool(library.Properties.Header_abi_checker.Check_all_apis), - ctx.IsLlndk(), ctx.isNdk(ctx.Config()), ctx.IsVndkExt(), true) + isLlndk || isNdk, ctx.IsVndkExt(), true) } } - refAbiDumpFile := getRefAbiDumpFile(ctx, version, fileName) + currVersion := currRefAbiDumpVersion(ctx, isVndk) + refAbiDumpFile := getRefAbiDumpFile(ctx, currVersion, fileName) if refAbiDumpFile != nil { library.sAbiDiff = sourceAbiDiff(ctx, library.sAbiOutputFile.Path(), - refAbiDumpFile, fileName, exportedHeaderFlags, + refAbiDumpFile, fileName, library.Properties.Header_abi_checker.Diff_flags, /* unused if not previousVersionDiff */ 0, Bool(library.Properties.Header_abi_checker.Check_all_apis), - ctx.IsLlndk(), ctx.isNdk(ctx.Config()), ctx.IsVndkExt(), false) + isLlndk || isNdk, ctx.IsVndkExt(), false) } } } @@ -2792,7 +2793,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo Runtime_deps: linkerAttrs.runtimeDeps, } - module.convertTidyAttributes(&commonAttrs.tidyAttributes) + module.convertTidyAttributes(ctx, &commonAttrs.tidyAttributes) var attrs interface{} if isStatic { diff --git a/cc/library_stub.go b/cc/library_stub.go index 2fb21a31f..043c03c30 100644 --- a/cc/library_stub.go +++ b/cc/library_stub.go @@ -288,8 +288,8 @@ type ccApiexportProperties struct { } type variantExporterProperties struct { - // Header directory or library to export - Export_headers []string + // Header directory to export + Export_headers []string `android:"arch_variant"` // Export all headers as system include Export_headers_as_system *bool diff --git a/cc/strip.go b/cc/strip.go index 5c32d8b04..c60e13530 100644 --- a/cc/strip.go +++ b/cc/strip.go @@ -56,9 +56,7 @@ func (stripper *Stripper) NeedsStrip(actx android.ModuleContext) bool { forceEnable := Bool(stripper.StripProperties.Strip.All) || Bool(stripper.StripProperties.Strip.Keep_symbols) || Bool(stripper.StripProperties.Strip.Keep_symbols_and_debug_frame) - // create_minidebuginfo doesn't work for riscv64 yet, disable stripping for now - riscv64 := actx.Arch().ArchType == android.Riscv64 - return !forceDisable && (forceEnable || defaultEnable) && !riscv64 + return !forceDisable && (forceEnable || defaultEnable) } // Keep this consistent with //build/bazel/rules/stripped_shared_library.bzl. diff --git a/cc/test.go b/cc/test.go index 92055fa93..536210ba4 100644 --- a/cc/test.go +++ b/cc/test.go @@ -678,7 +678,7 @@ func testBinaryBp2build(ctx android.TopDownMutatorContext, m *Module) { } } - m.convertTidyAttributes(&testBinaryAttrs.tidyAttributes) + m.convertTidyAttributes(ctx, &testBinaryAttrs.tidyAttributes) for _, propIntf := range m.GetProperties() { if testLinkerProps, ok := propIntf.(*TestLinkerProperties); ok { diff --git a/cc/tidy.go b/cc/tidy.go index a3d548bb6..bbcaece24 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -136,19 +136,7 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { flags.TidyFlags = append(flags.TidyFlags, "-extra-arg-before=-fno-caret-diagnostics") } - extraArgFlags := []string{ - // We might be using the static analyzer through clang tidy. - // https://bugs.llvm.org/show_bug.cgi?id=32914 - "-D__clang_analyzer__", - - // A recent change in clang-tidy (r328258) enabled destructor inlining, which - // appears to cause a number of false positives. Until that's resolved, this turns - // off the effects of r328258. - // https://bugs.llvm.org/show_bug.cgi?id=37459 - "-Xclang", "-analyzer-config", "-Xclang", "c++-temp-dtor-inlining=false", - } - - for _, f := range extraArgFlags { + for _, f := range config.TidyExtraArgFlags() { flags.TidyFlags = append(flags.TidyFlags, "-extra-arg-before="+f) } diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 3fed1a172..bc2d5cb07 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -106,24 +106,11 @@ func init() { } func newNameResolver(config android.Config) *android.NameResolver { - namespacePathsToExport := make(map[string]bool) - - for _, namespaceName := range config.ExportedNamespaces() { - namespacePathsToExport[namespaceName] = true - } - - namespacePathsToExport["."] = true // always export the root namespace - - exportFilter := func(namespace *android.Namespace) bool { - return namespacePathsToExport[namespace.Path] - } - - return android.NewNameResolver(exportFilter) + return android.NewNameResolver(config) } func newContext(configuration android.Config) *android.Context { ctx := android.NewContext(configuration) - ctx.Register() ctx.SetNameInterface(newNameResolver(configuration)) ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) return ctx @@ -165,14 +152,12 @@ func newConfig(availableEnv map[string]string) android.Config { // Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a // BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata // for modules that should be handled by Bazel. -func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) { +func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) string { ctx.EventHandler.Begin("mixed_build") defer ctx.EventHandler.End("mixed_build") bazelHook := func() error { - ctx.EventHandler.Begin("bazel") - defer ctx.EventHandler.End("bazel") - return configuration.BazelContext.InvokeBazel(configuration) + return configuration.BazelContext.InvokeBazel(configuration, ctx) } ctx.SetBeforePrepareBuildActionsHook(bazelHook) ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration) @@ -188,6 +173,7 @@ func runMixedModeBuild(configuration android.Config, ctx *android.Context, extra ninjaDeps = append(ninjaDeps, globListFiles...) writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) + return cmdlineArgs.OutFile } // Run the code-generation phase to convert BazelTargetModules to BUILD files. @@ -206,12 +192,11 @@ func runQueryView(queryviewDir, queryviewMarker string, configuration android.Co // Run the code-generation phase to convert API contributions to BUILD files. // Return marker file for the new synthetic workspace -func runApiBp2build(configuration android.Config, extraNinjaDeps []string) string { - // Create a new context and register mutators that are only meaningful to API export - ctx := android.NewContext(configuration) +func runApiBp2build(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) string { ctx.EventHandler.Begin("api_bp2build") defer ctx.EventHandler.End("api_bp2build") - ctx.SetNameInterface(newNameResolver(configuration)) + // Do not allow missing dependencies. + ctx.SetAllowMissingDependencies(false) ctx.RegisterForApiBazelConversion() // Register the Android.bp files in the tree @@ -256,6 +241,11 @@ func runApiBp2build(configuration android.Config, extraNinjaDeps []string) strin // Exclude all src BUILD files excludes = append(excludes, apiBuildFileExcludes()...) + // Android.bp files for api surfaces are mounted to out/, but out/ should not be a + // dep for api_bp2build. + // Otherwise api_bp2build will be run every single time + excludes = append(excludes, configuration.OutDir()) + // Create the symlink forest symlinkDeps := bp2build.PlantSymlinkForest( configuration.IsEnvTrue("BP2BUILD_VERBOSE"), @@ -349,61 +339,77 @@ func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDe // output file of the specific activity. func doChosenActivity(ctx *android.Context, configuration android.Config, extraNinjaDeps []string, metricsDir string) string { if configuration.BuildMode == android.SymlinkForest { - runSymlinkForestCreation(configuration, extraNinjaDeps, metricsDir) - return symlinkForestMarker + return runSymlinkForestCreation(configuration, ctx, extraNinjaDeps, metricsDir) } else if configuration.BuildMode == android.Bp2build { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. - runBp2Build(configuration, extraNinjaDeps, metricsDir) - return bp2buildMarker - } else if configuration.IsMixedBuildsEnabled() { - runMixedModeBuild(configuration, ctx, extraNinjaDeps) + return runBp2Build(configuration, ctx, extraNinjaDeps, metricsDir) } else if configuration.BuildMode == android.ApiBp2build { - return runApiBp2build(configuration, extraNinjaDeps) + outputFile := runApiBp2build(configuration, ctx, extraNinjaDeps) + writeMetrics(configuration, ctx.EventHandler, metricsDir) + return outputFile } else { - var stopBefore bootstrap.StopBefore - if configuration.BuildMode == android.GenerateModuleGraph { - stopBefore = bootstrap.StopBeforeWriteNinja - } else if configuration.BuildMode == android.GenerateQueryView || configuration.BuildMode == android.GenerateDocFile { - stopBefore = bootstrap.StopBeforePrepareBuildActions + ctx.Register() + + var outputFile string + if configuration.IsMixedBuildsEnabled() { + outputFile = runMixedModeBuild(configuration, ctx, extraNinjaDeps) } else { - stopBefore = bootstrap.DoEverything + outputFile = runSoongOnlyBuild(configuration, ctx, extraNinjaDeps) } - ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, stopBefore, ctx.Context, configuration) - ninjaDeps = append(ninjaDeps, extraNinjaDeps...) + writeMetrics(configuration, ctx.EventHandler, metricsDir) - globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) - ninjaDeps = append(ninjaDeps, globListFiles...) + return outputFile + } +} - // Convert the Soong module graph into Bazel BUILD files. - if configuration.BuildMode == android.GenerateQueryView { - queryviewMarkerFile := bazelQueryViewDir + ".marker" - runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx) - writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps) - return queryviewMarkerFile - } else if configuration.BuildMode == android.GenerateModuleGraph { - writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile) - writeDepFile(moduleGraphFile, ctx.EventHandler, ninjaDeps) - return moduleGraphFile - } else if configuration.BuildMode == android.GenerateDocFile { - // TODO: we could make writeDocs() return the list of documentation files - // written and add them to the .d file. Then soong_docs would be re-run - // whenever one is deleted. - if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil { - fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err) - os.Exit(1) - } - writeDepFile(docFile, ctx.EventHandler, ninjaDeps) - return docFile - } else { - // The actual output (build.ninja) was written in the RunBlueprint() call - // above - writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) - } +// runSoongOnlyBuild runs the standard Soong build in a number of different modes. +func runSoongOnlyBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) string { + ctx.EventHandler.Begin("soong_build") + defer ctx.EventHandler.End("soong_build") + + var stopBefore bootstrap.StopBefore + if configuration.BuildMode == android.GenerateModuleGraph { + stopBefore = bootstrap.StopBeforeWriteNinja + } else if configuration.BuildMode == android.GenerateQueryView || configuration.BuildMode == android.GenerateDocFile { + stopBefore = bootstrap.StopBeforePrepareBuildActions + } else { + stopBefore = bootstrap.DoEverything } - return cmdlineArgs.OutFile + ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, stopBefore, ctx.Context, configuration) + ninjaDeps = append(ninjaDeps, extraNinjaDeps...) + + globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) + ninjaDeps = append(ninjaDeps, globListFiles...) + + // Convert the Soong module graph into Bazel BUILD files. + if configuration.BuildMode == android.GenerateQueryView { + queryviewMarkerFile := bazelQueryViewDir + ".marker" + runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx) + writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps) + return queryviewMarkerFile + } else if configuration.BuildMode == android.GenerateModuleGraph { + writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile) + writeDepFile(moduleGraphFile, ctx.EventHandler, ninjaDeps) + return moduleGraphFile + } else if configuration.BuildMode == android.GenerateDocFile { + // TODO: we could make writeDocs() return the list of documentation files + // written and add them to the .d file. Then soong_docs would be re-run + // whenever one is deleted. + if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil { + fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err) + os.Exit(1) + } + writeDepFile(docFile, ctx.EventHandler, ninjaDeps) + return docFile + } else { + // The actual output (build.ninja) was written in the RunBlueprint() call + // above + writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps) + return cmdlineArgs.OutFile + } } // soong_ui dumps the available environment variables to @@ -463,13 +469,9 @@ func main() { logDir := availableEnv["LOG_DIR"] ctx := newContext(configuration) - ctx.EventHandler.Begin("soong_build") finalOutputFile := doChosenActivity(ctx, configuration, extraNinjaDeps, logDir) - ctx.EventHandler.End("soong_build") - writeMetrics(configuration, ctx.EventHandler, logDir) - writeUsedEnvironmentFile(configuration, finalOutputFile) } @@ -614,9 +616,7 @@ func bazelArtifacts() []string { // Ideally, bp2build would write a file that contains instructions to the // symlink tree creation binary. Then the latter would not need to depend on // the very heavy-weight machinery of soong_build . -func runSymlinkForestCreation(configuration android.Config, extraNinjaDeps []string, metricsDir string) { - eventHandler := &metrics.EventHandler{} - +func runSymlinkForestCreation(configuration android.Config, ctx *android.Context, extraNinjaDeps []string, metricsDir string) string { var ninjaDeps []string ninjaDeps = append(ninjaDeps, extraNinjaDeps...) @@ -643,13 +643,13 @@ func runSymlinkForestCreation(configuration android.Config, extraNinjaDeps []str // Such a directory SHOULD be added to `ninjaDeps` so that a child directory // or file created/deleted under it would trigger an update of the symlink // forest. - eventHandler.Do("symlink_forest", func() { + ctx.EventHandler.Do("symlink_forest", func() { symlinkForestDeps := bp2build.PlantSymlinkForest( configuration.IsEnvTrue("BP2BUILD_VERBOSE"), topDir, workspaceRoot, generatedRoot, excludes) ninjaDeps = append(ninjaDeps, symlinkForestDeps...) }) - writeDepFile(symlinkForestMarker, eventHandler, ninjaDeps) + writeDepFile(symlinkForestMarker, ctx.EventHandler, ninjaDeps) touch(shared.JoinPath(topDir, symlinkForestMarker)) codegenMetrics := bp2build.ReadCodegenMetrics(metricsDir) if codegenMetrics == nil { @@ -659,27 +659,24 @@ func runSymlinkForestCreation(configuration android.Config, extraNinjaDeps []str //TODO (usta) we cannot determine if we loaded a stale file, i.e. from an unrelated prior //invocation of codegen. We should simply use a separate .pb file } - writeBp2BuildMetrics(codegenMetrics, eventHandler, metricsDir) + writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir) + + return symlinkForestMarker } // Run Soong in the bp2build mode. This creates a standalone context that registers // an alternate pipeline of mutators and singletons specifically for generating // Bazel BUILD files instead of Ninja files. -func runBp2Build(configuration android.Config, extraNinjaDeps []string, metricsDir string) { +func runBp2Build(configuration android.Config, ctx *android.Context, extraNinjaDeps []string, metricsDir string) string { var codegenMetrics *bp2build.CodegenMetrics - eventHandler := &metrics.EventHandler{} - eventHandler.Do("bp2build", func() { - - // Register an alternate set of singletons and mutators for bazel - // conversion for Bazel conversion. - bp2buildCtx := android.NewContext(configuration) + ctx.EventHandler.Do("bp2build", func() { // Propagate "allow misssing dependencies" bit. This is normally set in - // newContext(), but we create bp2buildCtx without calling that method. - bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) - bp2buildCtx.SetNameInterface(newNameResolver(configuration)) - bp2buildCtx.RegisterForBazelConversion() - bp2buildCtx.SetModuleListFile(cmdlineArgs.ModuleListFile) + // newContext(), but we create ctx without calling that method. + ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) + ctx.SetNameInterface(newNameResolver(configuration)) + ctx.RegisterForBazelConversion() + ctx.SetModuleListFile(cmdlineArgs.ModuleListFile) var ninjaDeps []string ninjaDeps = append(ninjaDeps, extraNinjaDeps...) @@ -687,25 +684,25 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string, metricsD // Run the loading and analysis pipeline to prepare the graph of regular // Modules parsed from Android.bp files, and the BazelTargetModules mapped // from the regular Modules. - eventHandler.Do("bootstrap", func() { + ctx.EventHandler.Do("bootstrap", func() { blueprintArgs := cmdlineArgs - bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, bp2buildCtx.Context, configuration) + bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, ctx.Context, configuration) ninjaDeps = append(ninjaDeps, bootstrapDeps...) }) - globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx, configuration.SoongOutDir(), configuration) + globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, globListFiles...) // Run the code-generation phase to convert BazelTargetModules to BUILD files // and print conversion codegenMetrics to the user. - codegenContext := bp2build.NewCodegenContext(configuration, bp2buildCtx, bp2build.Bp2Build) - eventHandler.Do("codegen", func() { + codegenContext := bp2build.NewCodegenContext(configuration, ctx, bp2build.Bp2Build) + ctx.EventHandler.Do("codegen", func() { codegenMetrics = bp2build.Codegen(codegenContext) }) ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) - writeDepFile(bp2buildMarker, eventHandler, ninjaDeps) + writeDepFile(bp2buildMarker, ctx.EventHandler, ninjaDeps) touch(shared.JoinPath(topDir, bp2buildMarker)) }) @@ -715,7 +712,8 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string, metricsD if configuration.IsEnvTrue("BP2BUILD_VERBOSE") { codegenMetrics.Print() } - writeBp2BuildMetrics(codegenMetrics, eventHandler, metricsDir) + writeBp2BuildMetrics(codegenMetrics, ctx.EventHandler, metricsDir) + return bp2buildMarker } // Write Bp2Build metrics into $LOG_DIR @@ -1,24 +1,21 @@ module android/soong -require google.golang.org/protobuf v0.0.0 - -require github.com/google/blueprint v0.0.0 - -replace google.golang.org/protobuf v0.0.0 => ../../external/golang-protobuf - -replace github.com/google/blueprint v0.0.0 => ../blueprint +require ( + google.golang.org/protobuf v0.0.0 + github.com/google/blueprint v0.0.0 + prebuilts/bazel/common/proto/analysis_v2 v0.0.0 + prebuilts/bazel/common/proto/build v0.0.0 // indirect +) + +replace ( + google.golang.org/protobuf v0.0.0 => ../../external/golang-protobuf + github.com/google/blueprint v0.0.0 => ../blueprint + github.com/google/go-cmp v0.5.5 => ../../external/go-cmp + prebuilts/bazel/common/proto/analysis_v2 => ../../prebuilts/bazel/common/proto/analysis_v2 + prebuilts/bazel/common/proto/build => ../../prebuilts/bazel/common/proto/build +) // Indirect deps from golang-protobuf exclude github.com/golang/protobuf v1.5.0 -replace github.com/google/go-cmp v0.5.5 => ../../external/go-cmp - -require prebuilts/bazel/common/proto/analysis_v2 v0.0.0 - -replace prebuilts/bazel/common/proto/analysis_v2 => ../../prebuilts/bazel/common/proto/analysis_v2 - -require prebuilts/bazel/common/proto/build v0.0.0 // indirect - -replace prebuilts/bazel/common/proto/build => ../../prebuilts/bazel/common/proto/build - -go 1.18 +go 2.0 diff --git a/java/aar.go b/java/aar.go index 6261f2974..0fdde0362 100644 --- a/java/aar.go +++ b/java/aar.go @@ -1046,7 +1046,8 @@ func (a *AARImport) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } func (a *AndroidLibrary) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - commonAttrs, depLabels := a.convertLibraryAttrsBp2Build(ctx) + commonAttrs, bp2buildInfo := a.convertLibraryAttrsBp2Build(ctx) + depLabels := bp2buildInfo.DepLabels deps := depLabels.Deps if !commonAttrs.Srcs.IsEmpty() { diff --git a/java/app.go b/java/app.go index 2a51e10df..e36808fe0 100755 --- a/java/app.go +++ b/java/app.go @@ -1492,7 +1492,8 @@ type bazelAndroidAppAttributes struct { // ConvertWithBp2build is used to convert android_app to Bazel. func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - commonAttrs, depLabels := a.convertLibraryAttrsBp2Build(ctx) + commonAttrs, bp2BuildInfo := a.convertLibraryAttrsBp2Build(ctx) + depLabels := bp2BuildInfo.DepLabels deps := depLabels.Deps deps.Append(depLabels.StaticDeps) diff --git a/java/base.go b/java/base.go index 602e8d882..5d2498169 100644 --- a/java/base.go +++ b/java/base.go @@ -868,7 +868,7 @@ func (j *Module) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.Opt flags = append(flags, genAidlIncludeFlags(ctx, aidlSrcs, includeDirs)) sdkVersion := (j.SdkVersion(ctx)).Kind - defaultTrace := ((sdkVersion == android.SdkSystemServer) || (sdkVersion == android.SdkCore) || (sdkVersion == android.SdkCorePlatform)) + defaultTrace := ((sdkVersion == android.SdkSystemServer) || (sdkVersion == android.SdkCore) || (sdkVersion == android.SdkCorePlatform) || (sdkVersion == android.SdkModule)) if proptools.BoolDefault(j.deviceProperties.Aidl.Generate_traces, defaultTrace) { flags = append(flags, "-t") } diff --git a/java/invalid_implementation_jar.sh b/java/invalid_implementation_jar.sh new file mode 100755 index 000000000..3820058d0 --- /dev/null +++ b/java/invalid_implementation_jar.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copyright 2022 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to detect and report an attempt to access an invalid implementation +# jar. + +MOD=$1 + +cat <<EOF + + $MOD is a java_library that generates a jar file which must not be accessed + from outside the mainline module that provides it. If you are seeing this + message it means that you are incorrectly attempting to use the jar file + from a java_import prebuilt of $MOD. + + This is most likely due to an incorrect dependency on $MOD in an Android.mk + or Android.bp file. Please remove that dependency and replace with + something more appropriate, e.g. a dependency on an API provided by the + module. + + If you do not know where the extraneous dependency was added then you can + run the following command to find a list of all the paths from the target + which you are trying to build to the target which produced this error. + + prebuilts/build-tools/linux-x86/bin/ninja -f out/combined-\${TARGET_PRODUCT}.ninja -t path <target> <invalid-jar> + + Where <target> is the build target you specified on the command line which + produces this error and <invalid-jar> is the rule that failed with this + message. If you are specifying multiple build targets then you will need to + run the above command for every target until you find the cause. + + The command will output one (of the possibly many) dependency paths from + <target> to <invalid-jar>, one file/phony target per line. e.g. it may + output something like this: + + .... + out/soong/.intermediates/acme/broken/android_common/combined/broken.jar + out/soong/.intermediates/prebuilts/module_sdk/art/current/sdk/prebuilt_core-libart/android_common/combined/core-libart.jar + out/soong/.intermediates/prebuilts/module_sdk/art/current/sdk/art-module-sdk_core-libart-error/gen/this-file-will-never-be-created.jar + + The last line is the failing target, the second to last line is a dependency + from the core-libart java_import onto the failing target, the third to last + line is the source of the dependency so you should look in acme/Android.bp + file for the "broken" module. + +EOF + +exit 1 diff --git a/java/java.go b/java/java.go index b6fc6b831..ad46e9880 100644 --- a/java/java.go +++ b/java/java.go @@ -25,6 +25,7 @@ import ( "android/soong/bazel" "android/soong/bazel/cquery" + "android/soong/remoteexec" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -59,6 +60,8 @@ func registerJavaBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("java_device_for_host", DeviceForHostFactory) ctx.RegisterModuleType("java_host_for_device", HostForDeviceFactory) ctx.RegisterModuleType("dex_import", DexImportFactory) + ctx.RegisterModuleType("java_api_library", ApiLibraryFactory) + ctx.RegisterModuleType("java_api_contribution", ApiContributionFactory) // This mutator registers dependencies on dex2oat for modules that should be // dexpreopted. This is done late when the final variants have been @@ -86,11 +89,11 @@ func RegisterJavaSdkMemberTypes() { var ( // Supports adding java header libraries to module_exports and sdk. javaHeaderLibsSdkMemberType = &librarySdkMemberType{ - android.SdkMemberTypeBase{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ PropertyName: "java_header_libs", SupportsSdk: true, }, - func(_ android.SdkMemberContext, j *Library) android.Path { + jarToExportGetter: func(_ android.SdkMemberContext, j *Library) android.Path { headerJars := j.HeaderJars() if len(headerJars) != 1 { panic(fmt.Errorf("there must be only one header jar from %q", j.Name())) @@ -98,8 +101,8 @@ var ( return headerJars[0] }, - sdkSnapshotFilePathForJar, - copyEverythingToSnapshot, + snapshotPathGetter: sdkSnapshotFilePathForJar, + onlyCopyJarToSnapshot: copyEverythingToSnapshot, } // Export implementation classes jar as part of the sdk. @@ -113,12 +116,12 @@ var ( // Supports adding java implementation libraries to module_exports but not sdk. javaLibsSdkMemberType = &librarySdkMemberType{ - android.SdkMemberTypeBase{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ PropertyName: "java_libs", }, - exportImplementationClassesJar, - sdkSnapshotFilePathForJar, - copyEverythingToSnapshot, + jarToExportGetter: exportImplementationClassesJar, + snapshotPathGetter: sdkSnapshotFilePathForJar, + onlyCopyJarToSnapshot: copyEverythingToSnapshot, } snapshotRequiresImplementationJar = func(ctx android.SdkMemberContext) bool { @@ -143,11 +146,11 @@ var ( // necessary. The java_boot_libs property to allow those modules to be exported as part of the // sdk/module_exports without exposing any unnecessary information. javaBootLibsSdkMemberType = &librarySdkMemberType{ - android.SdkMemberTypeBase{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ PropertyName: "java_boot_libs", SupportsSdk: true, }, - func(ctx android.SdkMemberContext, j *Library) android.Path { + jarToExportGetter: func(ctx android.SdkMemberContext, j *Library) android.Path { if snapshotRequiresImplementationJar(ctx) { return exportImplementationClassesJar(ctx, j) } @@ -156,9 +159,9 @@ var ( // jar for use by dexpreopting and boot jars package check. They do not need to provide an // actual implementation jar but the java_import will need a file that exists so just copy an // empty file. Any attempt to use that file as a jar will cause a build error. - return ctx.SnapshotBuilder().EmptyFile() + return nil }, - func(ctx android.SdkMemberContext, osPrefix, name string) string { + snapshotPathGetter: func(ctx android.SdkMemberContext, osPrefix, name string) string { if snapshotRequiresImplementationJar(ctx) { return sdkSnapshotFilePathForJar(ctx, osPrefix, name) } @@ -168,7 +171,7 @@ var ( // TODO(b/175714559): Provide a proper error message in Soong not ninja. return filepath.Join(osPrefix, "java_boot_libs", "snapshot", "jars", "are", "invalid", name+jarFileSuffix) }, - onlyCopyJarToSnapshot, + onlyCopyJarToSnapshot: onlyCopyJarToSnapshot, } // Supports adding java systemserver libraries to module_exports and sdk. @@ -182,27 +185,27 @@ var ( // necessary. The java_systemserver_libs property to allow those modules to be exported as part of // the sdk/module_exports without exposing any unnecessary information. javaSystemserverLibsSdkMemberType = &librarySdkMemberType{ - android.SdkMemberTypeBase{ + SdkMemberTypeBase: android.SdkMemberTypeBase{ PropertyName: "java_systemserver_libs", SupportsSdk: true, // This was only added in Tiramisu. SupportedBuildReleaseSpecification: "Tiramisu+", }, - func(ctx android.SdkMemberContext, j *Library) android.Path { + jarToExportGetter: func(ctx android.SdkMemberContext, j *Library) android.Path { // Java systemserver libs are only provided in the SDK to provide access to their dex // implementation jar for use by dexpreopting. They do not need to provide an actual // implementation jar but the java_import will need a file that exists so just copy an empty // file. Any attempt to use that file as a jar will cause a build error. - return ctx.SnapshotBuilder().EmptyFile() + return nil }, - func(_ android.SdkMemberContext, osPrefix, name string) string { + snapshotPathGetter: func(_ android.SdkMemberContext, osPrefix, name string) string { // Create a special name for the implementation jar to try and provide some useful information // to a developer that attempts to compile against this. // TODO(b/175714559): Provide a proper error message in Soong not ninja. return filepath.Join(osPrefix, "java_systemserver_libs", "snapshot", "jars", "are", "invalid", name+jarFileSuffix) }, - onlyCopyJarToSnapshot, + onlyCopyJarToSnapshot: onlyCopyJarToSnapshot, } // Supports adding java test libraries to module_exports but not sdk. @@ -232,7 +235,7 @@ type JavaInfo struct { ImplementationAndResourcesJars android.Paths // ImplementationJars is a list of jars that contain the implementations of classes in the - //module. + // module. ImplementationJars android.Paths // ResourceJars is a list of jars that contain the resources included in the module. @@ -718,7 +721,8 @@ type librarySdkMemberType struct { android.SdkMemberTypeBase // Function to retrieve the appropriate output jar (implementation or header) from - // the library. + // the library, if this returns nil then it is assumed that the snapshot must not provide access + // to the jar. jarToExportGetter func(ctx android.SdkMemberContext, j *Library) android.Path // Function to compute the snapshot relative path to which the named library's @@ -755,7 +759,11 @@ func (mt *librarySdkMemberType) CreateVariantPropertiesStruct() android.SdkMembe type librarySdkMemberProperties struct { android.SdkMemberPropertiesBase - JarToExport android.Path `android:"arch_variant"` + JarToExport android.Path `android:"arch_variant"` + + // The path to a script to use when the jar is invalid. + InvalidJarScript android.Path + AidlIncludeDirs android.Paths // The list of permitted packages that need to be passed to the prebuilts as they are used to @@ -766,7 +774,15 @@ type librarySdkMemberProperties struct { func (p *librarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { j := variant.(*Library) - p.JarToExport = ctx.MemberType().(*librarySdkMemberType).jarToExportGetter(ctx, j) + memberType := ctx.MemberType().(*librarySdkMemberType) + p.JarToExport = memberType.jarToExportGetter(ctx, j) + + // If no jar was provided for export then disallow access to it completely. + if p.JarToExport == nil { + // Copy the script to prevent access to the jar into the snapshot. + p.InvalidJarScript = android.PathForSource(ctx.SdkModuleContext(), + "build/soong/java/invalid_implementation_jar.sh") + } p.AidlIncludeDirs = j.AidlIncludeDirs() @@ -789,6 +805,21 @@ func (p *librarySdkMemberProperties) AddToPropertySet(ctx android.SdkMemberConte propertySet.AddProperty("jars", []string{snapshotRelativeJavaLibPath}) } + if scriptSrc := p.InvalidJarScript; scriptSrc != nil { + // Copy the script to prevent access to the jar into the snapshot. + scriptDest := filepath.Join("scripts", scriptSrc.Base()) + builder.CopyToSnapshot(scriptSrc, scriptDest) + + // Generate a genrule module that will invoke the script passing in the module name. + genrule := builder.AddInternalModule(p, "genrule", "error") + genRuleName := genrule.Name() + genrule.AddProperty("out", []string{"this-file-will-never-be-created.jar"}) + genrule.AddProperty("tool_files", []string{scriptDest}) + genrule.AddProperty("cmd", fmt.Sprintf("$(location %s) %s", scriptDest, p.Name())) + + propertySet.AddPropertyWithTag("jars", []string{":" + genRuleName}, builder.SdkMemberReferencePropertyTag(true)) + } + if len(p.PermittedPackages) > 0 { propertySet.AddProperty("permitted_packages", p.PermittedPackages) } @@ -1501,6 +1532,182 @@ func BinaryHostFactory() android.Module { return module } +type JavaApiContribution struct { + android.ModuleBase + android.DefaultableModuleBase + + properties struct { + // name of the API surface + Api_surface *string + + // relative path to the API signature text file + Api_file *string `android:"path"` + } +} + +func ApiContributionFactory() android.Module { + module := &JavaApiContribution{} + android.InitAndroidModule(module) + android.InitDefaultableModule(module) + module.AddProperties(&module.properties) + return module +} + +type JavaApiImportInfo struct { + ApiFile android.Path +} + +var JavaApiImportProvider = blueprint.NewProvider(JavaApiImportInfo{}) + +func (ap *JavaApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) { + apiFile := android.PathForModuleSrc(ctx, String(ap.properties.Api_file)) + ctx.SetProvider(JavaApiImportProvider, JavaApiImportInfo{ + ApiFile: apiFile, + }) +} + +type ApiLibrary struct { + android.ModuleBase + android.DefaultableModuleBase + + properties JavaApiLibraryProperties + + stubsSrcJar android.WritablePath + stubsJar android.WritablePath +} + +type JavaApiLibraryProperties struct { + // name of the API surface + Api_surface *string + + // list of API provider modules that consists this API surface + Api_providers []string + + // List of flags to be passed to the javac compiler to generate jar file + Javacflags []string +} + +func ApiLibraryFactory() android.Module { + module := &ApiLibrary{} + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + module.AddProperties(&module.properties) + return module +} + +func (al *ApiLibrary) ApiSurface() *string { + return al.properties.Api_surface +} + +func (al *ApiLibrary) StubsJar() android.Path { + return al.stubsJar +} + +func metalavaStubCmd(ctx android.ModuleContext, rule *android.RuleBuilder, + srcs android.Paths, homeDir android.WritablePath) *android.RuleBuilderCommand { + rule.Command().Text("rm -rf").Flag(homeDir.String()) + rule.Command().Text("mkdir -p").Flag(homeDir.String()) + + cmd := rule.Command() + cmd.FlagWithArg("ANDROID_PREFS_ROOT=", homeDir.String()) + + if metalavaUseRbe(ctx) { + rule.Remoteable(android.RemoteRuleSupports{RBE: true}) + execStrategy := ctx.Config().GetenvWithDefault("RBE_METALAVA_EXEC_STRATEGY", remoteexec.LocalExecStrategy) + labels := map[string]string{"type": "tool", "name": "metalava"} + + pool := ctx.Config().GetenvWithDefault("RBE_METALAVA_POOL", "java16") + rule.Rewrapper(&remoteexec.REParams{ + Labels: labels, + ExecStrategy: execStrategy, + ToolchainInputs: []string{config.JavaCmd(ctx).String()}, + Platform: map[string]string{remoteexec.PoolKey: pool}, + }) + } + + cmd.BuiltTool("metalava").ImplicitTool(ctx.Config().HostJavaToolPath(ctx, "metalava.jar")). + Flag(config.JavacVmFlags). + Flag("-J--add-opens=java.base/java.util=ALL-UNNAMED"). + FlagWithArg("-encoding ", "UTF-8"). + FlagWithInputList("--source-files ", srcs, " ") + + cmd.Flag("--no-banner"). + Flag("--color"). + Flag("--quiet"). + Flag("--format=v2"). + FlagWithArg("--repeat-errors-max ", "10"). + FlagWithArg("--hide ", "UnresolvedImport"). + FlagWithArg("--hide ", "InvalidNullabilityOverride"). + FlagWithArg("--hide ", "ChangedDefault") + + return cmd +} + +func (al *ApiLibrary) stubsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, stubsDir android.OptionalPath) { + if stubsDir.Valid() { + cmd.FlagWithArg("--stubs ", stubsDir.String()) + } +} + +var javaApiProviderTag = dependencyTag{name: "java-api-provider"} + +func (al *ApiLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { + apiProviders := al.properties.Api_providers + for _, apiProviderName := range apiProviders { + ctx.AddDependency(ctx.Module(), javaApiProviderTag, apiProviderName) + } +} + +func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + + rule := android.NewRuleBuilder(pctx, ctx) + + rule.Sbox(android.PathForModuleOut(ctx, "metalava"), + android.PathForModuleOut(ctx, "metalava.sbox.textproto")). + SandboxInputs() + + var stubsDir android.OptionalPath + stubsDir = android.OptionalPathForPath(android.PathForModuleOut(ctx, "metalava", "stubsDir")) + rule.Command().Text("rm -rf").Text(stubsDir.String()) + rule.Command().Text("mkdir -p").Text(stubsDir.String()) + + homeDir := android.PathForModuleOut(ctx, "metalava", "home") + + apiProviders := al.properties.Api_providers + srcFiles := make([]android.Path, len(apiProviders)) + for i, apiProviderName := range apiProviders { + apiProvider := ctx.GetDirectDepWithTag(apiProviderName, javaApiProviderTag) + if apiProvider == nil { + panic(fmt.Errorf("Java API provider module %s not found, called from %s", apiProviderName, al.Name())) + } + provider := ctx.OtherModuleProvider(apiProvider, JavaApiImportProvider).(JavaApiImportInfo) + srcFiles[i] = android.PathForModuleSrc(ctx, provider.ApiFile.String()) + } + + cmd := metalavaStubCmd(ctx, rule, srcFiles, homeDir) + + al.stubsFlags(ctx, cmd, stubsDir) + + al.stubsSrcJar = android.PathForModuleOut(ctx, "metalava", ctx.ModuleName()+"-"+"stubs.srcjar") + rule.Command(). + BuiltTool("soong_zip"). + Flag("-write_if_changed"). + Flag("-jar"). + FlagWithOutput("-o ", al.stubsSrcJar). + FlagWithArg("-C ", stubsDir.String()). + FlagWithArg("-D ", stubsDir.String()) + + rule.Build("metalava", "metalava merged") + + al.stubsJar = android.PathForModuleOut(ctx, ctx.ModuleName(), "android.jar") + + var flags javaBuilderFlags + flags.javacFlags = strings.Join(al.properties.Javacflags, " ") + + TransformJavaToClasses(ctx, al.stubsJar, 0, android.Paths{}, + android.Paths{al.stubsSrcJar}, flags, android.Paths{}) +} + // // Java prebuilts // @@ -1650,7 +1857,7 @@ func (j *Import) DepsMutator(ctx android.BottomUpMutatorContext) { } func (j *Import) commonBuildActions(ctx android.ModuleContext) { - //TODO(b/231322772) these should come from Bazel once available + // TODO(b/231322772) these should come from Bazel once available j.sdkVersion = j.SdkVersion(ctx) j.minSdkVersion = j.MinSdkVersion(ctx) @@ -2253,7 +2460,7 @@ func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorConte resources.Append(android.BazelLabelForModuleSrc(ctx, m.properties.Java_resources)) } - //TODO(b/179889880) handle case where glob includes files outside package + // TODO(b/179889880) handle case where glob includes files outside package resDeps := ResourceDirsToFiles( ctx, m.properties.Java_resource_dirs, @@ -2307,11 +2514,21 @@ type javaAidlLibraryAttributes struct { Deps bazel.LabelListAttribute } -// convertLibraryAttrsBp2Build converts a few shared attributes from java_* modules -// and also separates dependencies into dynamic dependencies and static dependencies. -// Each corresponding Bazel target type, can have a different method for handling -// dynamic vs. static dependencies, and so these are returned to the calling function. -func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) (*javaCommonAttributes, *javaDependencyLabels) { +// bp2BuildJavaInfo has information needed for the conversion of java*_modules +// that is needed bor Bp2Build conversion but that requires different handling +// depending on the module type. +type bp2BuildJavaInfo struct { + // separates dependencies into dynamic dependencies and static dependencies. + DepLabels *javaDependencyLabels + hasKotlinSrcs bool +} + +// convertLibraryAttrsBp2Build returns a javaCommonAttributes struct with +// converted attributes shared across java_* modules and a bp2BuildJavaInfo struct +// which has other non-attribute information needed for bp2build conversion +// that needs different handling depending on the module types, and thus needs +// to be returned to the calling function. +func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) (*javaCommonAttributes, *bp2BuildJavaInfo) { var srcs bazel.LabelListAttribute var deps bazel.LabelList var staticDeps bazel.LabelList @@ -2330,14 +2547,18 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) protoSrcPartition := "proto" logtagSrcPartition := "logtag" aidlSrcPartition := "aidl" + kotlinPartition := "kotlin" srcPartitions := bazel.PartitionLabelListAttribute(ctx, &srcs, bazel.LabelPartitions{ javaSrcPartition: bazel.LabelPartition{Extensions: []string{".java"}, Keep_remainder: true}, logtagSrcPartition: bazel.LabelPartition{Extensions: []string{".logtags", ".logtag"}}, protoSrcPartition: android.ProtoSrcLabelPartition, aidlSrcPartition: android.AidlSrcLabelPartition, + kotlinPartition: bazel.LabelPartition{Extensions: []string{".kt"}}, }) javaSrcs := srcPartitions[javaSrcPartition] + kotlinSrcs := srcPartitions[kotlinPartition] + javaSrcs.Append(kotlinSrcs) if !srcPartitions[logtagSrcPartition].IsEmpty() { logtagsLibName := m.Name() + "_logtags" @@ -2401,7 +2622,7 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) } epEnabled := m.properties.Errorprone.Enabled - //TODO(b/227504307) add configuration that depends on RUN_ERROR_PRONE environment variable + // TODO(b/227504307) add configuration that depends on RUN_ERROR_PRONE environment variable if Bool(epEnabled) { javacopts = append(javacopts, m.properties.Errorprone.Javacflags...) } @@ -2447,18 +2668,25 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) depLabels.Deps = bazel.MakeLabelListAttribute(deps) depLabels.StaticDeps = bazel.MakeLabelListAttribute(staticDeps) - return commonAttrs, depLabels + bp2BuildInfo := &bp2BuildJavaInfo{ + DepLabels: depLabels, + hasKotlinSrcs: !kotlinSrcs.IsEmpty(), + } + + return commonAttrs, bp2BuildInfo } type javaLibraryAttributes struct { *javaCommonAttributes - Deps bazel.LabelListAttribute - Exports bazel.LabelListAttribute - Neverlink bazel.BoolAttribute + Deps bazel.LabelListAttribute + Exports bazel.LabelListAttribute + Neverlink bazel.BoolAttribute + Common_srcs bazel.LabelListAttribute } func javaLibraryBp2Build(ctx android.TopDownMutatorContext, m *Library) { - commonAttrs, depLabels := m.convertLibraryAttrsBp2Build(ctx) + commonAttrs, bp2BuildInfo := m.convertLibraryAttrsBp2Build(ctx) + depLabels := bp2BuildInfo.DepLabels deps := depLabels.Deps if !commonAttrs.Srcs.IsEmpty() { @@ -2473,15 +2701,25 @@ func javaLibraryBp2Build(ctx android.TopDownMutatorContext, m *Library) { ctx.ModuleErrorf("Module has direct dependencies but no sources. Bazel will not allow this.") } + var props bazel.BazelTargetModuleProperties attrs := &javaLibraryAttributes{ javaCommonAttributes: commonAttrs, Deps: deps, Exports: depLabels.StaticDeps, } - props := bazel.BazelTargetModuleProperties{ - Rule_class: "java_library", - Bzl_load_location: "//build/bazel/rules/java:library.bzl", + if !bp2BuildInfo.hasKotlinSrcs && len(m.properties.Common_srcs) == 0 { + props = bazel.BazelTargetModuleProperties{ + Rule_class: "java_library", + Bzl_load_location: "//build/bazel/rules/java:library.bzl", + } + } else { + attrs.Common_srcs = bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrc(ctx, m.properties.Common_srcs)) + + props = bazel.BazelTargetModuleProperties{ + Rule_class: "kt_jvm_library", + Bzl_load_location: "@rules_kotlin//kotlin:jvm_library.bzl", + } } name := m.Name() @@ -2498,7 +2736,8 @@ type javaBinaryHostAttributes struct { // JavaBinaryHostBp2Build is for java_binary_host bp2build. func javaBinaryHostBp2Build(ctx android.TopDownMutatorContext, m *Binary) { - commonAttrs, depLabels := m.convertLibraryAttrsBp2Build(ctx) + commonAttrs, bp2BuildInfo := m.convertLibraryAttrsBp2Build(ctx) + depLabels := bp2BuildInfo.DepLabels deps := depLabels.Deps deps.Append(depLabels.StaticDeps) @@ -2637,7 +2876,7 @@ func (i *Import) ProcessBazelQueryResponse(ctx android.ModuleContext) { HeaderJars: android.PathsIfNonNil(i.combinedClasspathFile), ImplementationAndResourcesJars: android.PathsIfNonNil(i.combinedClasspathFile), ImplementationJars: android.PathsIfNonNil(i.combinedClasspathFile), - //TODO(b/240308299) include AIDL information from Bazel + // TODO(b/240308299) include AIDL information from Bazel }) i.maybeInstall(ctx, jarName, outputFile) diff --git a/java/java_test.go b/java/java_test.go index f06b520d5..3f8cd8e90 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -1806,3 +1806,108 @@ func TestDeviceBinaryWrapperGeneration(t *testing.T) { srcs: ["foo.java"], }`) } + +func TestJavaApiLibraryAndProviderLink(t *testing.T) { + provider_bp_a := ` + java_api_contribution { + name: "foo1", + api_file: "foo1.txt", + } + ` + provider_bp_b := `java_api_contribution { + name: "foo2", + api_file: "foo2.txt", + } + ` + ctx, _ := testJavaWithFS(t, ` + java_api_library { + name: "bar1", + api_surface: "public", + api_providers: ["foo1"], + } + + java_api_library { + name: "bar2", + api_surface: "system", + api_providers: ["foo1", "foo2"], + } + `, + map[string][]byte{ + "a/Android.bp": []byte(provider_bp_a), + "b/Android.bp": []byte(provider_bp_b), + }) + + testcases := []struct { + moduleName string + sourceTextFileDirs []string + }{ + { + moduleName: "bar1", + sourceTextFileDirs: []string{"a/foo1.txt"}, + }, + { + moduleName: "bar2", + sourceTextFileDirs: []string{"a/foo1.txt", "b/foo2.txt"}, + }, + } + for _, c := range testcases { + m := ctx.ModuleForTests(c.moduleName, "android_common") + manifest := m.Output("metalava.sbox.textproto") + sboxProto := android.RuleBuilderSboxProtoForTests(t, manifest) + manifestCommand := sboxProto.Commands[0].GetCommand() + sourceFilesFlag := "--source-files " + strings.Join(c.sourceTextFileDirs, " ") + android.AssertStringDoesContain(t, "source text files not present", manifestCommand, sourceFilesFlag) + } +} + +func TestJavaApiLibraryJarGeneration(t *testing.T) { + provider_bp_a := ` + java_api_contribution { + name: "foo1", + api_file: "foo1.txt", + } + ` + provider_bp_b := `java_api_contribution { + name: "foo2", + api_file: "foo2.txt", + } + ` + ctx, _ := testJavaWithFS(t, ` + java_api_library { + name: "bar1", + api_surface: "public", + api_providers: ["foo1"], + } + + java_api_library { + name: "bar2", + api_surface: "system", + api_providers: ["foo1", "foo2"], + } + `, + map[string][]byte{ + "a/Android.bp": []byte(provider_bp_a), + "b/Android.bp": []byte(provider_bp_b), + }) + + testcases := []struct { + moduleName string + outputJarName string + }{ + { + moduleName: "bar1", + outputJarName: "bar1/android.jar", + }, + { + moduleName: "bar2", + outputJarName: "bar2/android.jar", + }, + } + for _, c := range testcases { + m := ctx.ModuleForTests(c.moduleName, "android_common") + outputs := fmt.Sprint(m.AllOutputs()) + if !strings.Contains(outputs, c.outputJarName) { + t.Errorf("Module output does not contain expected jar %s", c.outputJarName) + } + } +} diff --git a/java/legacy_core_platform_api_usage.go b/java/legacy_core_platform_api_usage.go index 8e224914d..1f374b42e 100644 --- a/java/legacy_core_platform_api_usage.go +++ b/java/legacy_core_platform_api_usage.go @@ -20,155 +20,39 @@ import ( ) var legacyCorePlatformApiModules = []string{ - "AAECarSystemUI", - "AAECarSystemUI-tests", "ArcSettings", - "ahat-test-dump", - "android.car", - "android.test.mock", - "android.test.mock.impl", - "AoapTestDeviceApp", - "AoapTestHostApp", - "api-stubs-docs", - "art_cts_jvmti_test_library", - "art-gtest-jars-MyClassNatives", - "BackupEncryption", - "BackupFrameworksServicesRoboTests", - "backuplib", - "BandwidthEnforcementTest", - "BlockedNumberProvider", - "BluetoothInstrumentationTests", - "BluetoothMidiLib", - "BluetoothMidiService", "BTTestApp", - "CallEnhancement", "CapCtrlInterface", - "CarService", - "CarServiceTest", - "car-service-test-lib", - "car-service-test-static-lib", - "CertInstaller", "com.qti.location.sdk", - "com.qti.media.secureprocessor", - "ConnectivityManagerTest", - "ContactsProvider", - "CorePerfTests", - "core-tests-support", - "cronet_impl_common_java", - "cronet_impl_native_java", - "cronet_impl_platform_java", - "CtsAppExitTestCases", - "CtsContentTestCases", - "CtsLibcoreWycheproofBCTestCases", - "CtsMediaTestCases", - "CtsNetTestCases", - "CtsNetTestCasesLatestSdk", - "CtsSecurityTestCases", - "CtsSuspendAppsTestCases", - "CtsUsageStatsTestCases", - "DeadpoolService", - "DeadpoolServiceBtServices", - "DeviceInfo", - "DiagnosticTools", - "DisplayCutoutEmulationEmu01Overlay", - "DocumentsUIGoogleTests", - "DocumentsUIPerfTests", - "DocumentsUITests", - "DocumentsUIUnitTests", - "DownloadProvider", - "DownloadProviderTests", - "DownloadProviderUi", - "ds-car-docs", // for AAOS API documentation only - "DynamicSystemInstallationService", - "EmergencyInfo-lib", - "EthernetServiceTests", - "ExternalStorageProvider", "face-V1-0-javalib", "FloralClocks", "framework-jobscheduler", "framework-minus-apex", "framework-minus-apex-intdefs", - "FrameworkOverlayG6QU3", "FrameworksCoreTests", - "FrameworksIkeTests", - "FrameworksNetCommonTests", - "FrameworksNetTests", - "FrameworksServicesRoboTests", - "FrameworksServicesTests", - "FrameworksMockingServicesTests", - "FrameworksUtilTests", - "GtsIncrementalInstallTestCases", - "GtsIncrementalInstallTriggerApp", - "GtsInstallerV2TestCases", "HelloOslo", - "hid", - "hidl_test_java_java", - "hwbinder", - "imssettings", "izat.lib.glue", - "KeyChain", - "LocalSettingsLib", - "LocalTransport", - "lockagent", - "mediaframeworktest", "mediatek-ims-base", - "MmsService", "ModemTestMode", "MtkCapCtrl", - "MtpService", - "MultiDisplayProvider", "my.tests.snapdragonsdktest", "NetworkSetting", - "NetworkStackIntegrationTestsLib", - "NetworkStackNextIntegrationTests", - "NetworkStackNextTests", - "NetworkStackTests", - "NetworkStackTestsLib", - "online-gcm-ref-docs", - "online-gts-docs", "PerformanceMode", - "platform_library-docs", - "PowerStatsService", - "PrintSpooler", "pxp-monitor", "QColor", "qcom.fmradio", - "QDCMMobileApp", "Qmmi", "QPerformance", - "remotesimlockmanagerlibrary", - "RollbackTest", "sam", "saminterfacelibrary", "sammanagerlibrary", - "service-blobstore", - "service-connectivity-pre-jarjar", - "service-jobscheduler", "services", - "services.accessibility", - "services.backup", "services.core.unboosted", - "services.devicepolicy", - "services.print", - "services.usage", - "services.usb", "Settings-core", "SettingsGoogle", "SettingsGoogleOverlayCoral", "SettingsGoogleOverlayFlame", "SettingsLib", - "SettingsOverlayG020A", - "SettingsOverlayG020B", - "SettingsOverlayG020C", - "SettingsOverlayG020D", - "SettingsOverlayG020E", - "SettingsOverlayG020E_VN", - "SettingsOverlayG020F", - "SettingsOverlayG020F_VN", - "SettingsOverlayG020G", - "SettingsOverlayG020G_VN", - "SettingsOverlayG020H", - "SettingsOverlayG020H_VN", "SettingsOverlayG020I", "SettingsOverlayG020I_VN", "SettingsOverlayG020J", @@ -177,45 +61,15 @@ var legacyCorePlatformApiModules = []string{ "SettingsOverlayG020P", "SettingsOverlayG020Q", "SettingsOverlayG025H", - "SettingsOverlayG025J", - "SettingsOverlayG025M", - "SettingsOverlayG025N", "SettingsOverlayG5NZ6", - "SettingsProvider", - "SettingsProviderTest", "SettingsRoboTests", - "Shell", - "ShellTests", "SimContact", "SimContacts", "SimSettings", - "sl4a.Common", - "StatementService", - "SystemUI-core", - "SystemUISharedLib", - "SystemUI-tests", "tcmiface", - "Telecom", - "TelecomUnitTests", "telephony-common", - "TelephonyProviderTests", "TeleService", - "testables", - "TetheringTests", - "TetheringTestsLib", - "time_zone_distro_installer", - "time_zone_distro_installer-tests", - "time_zone_distro-tests", - "time_zone_updater", - "TMobilePlanProvider", - "TvProvider", - "uiautomator-stubs-docs", - "uimgbamanagerlibrary", - "UsbHostExternalManagementTestApp", - "UserDictionaryProvider", "UxPerformance", - "WallpaperBackup", - "WallpaperBackupAgentTests", "WfdCommon", } diff --git a/java/plugin.go b/java/plugin.go index 123dbd4c0..731dfda00 100644 --- a/java/plugin.go +++ b/java/plugin.go @@ -66,7 +66,8 @@ type pluginAttributes struct { // ConvertWithBp2build is used to convert android_app to Bazel. func (p *Plugin) ConvertWithBp2build(ctx android.TopDownMutatorContext) { pluginName := p.Name() - commonAttrs, depLabels := p.convertLibraryAttrsBp2Build(ctx) + commonAttrs, bp2BuildInfo := p.convertLibraryAttrsBp2Build(ctx) + depLabels := bp2BuildInfo.DepLabels deps := depLabels.Deps deps.Append(depLabels.StaticDeps) diff --git a/sdk/bootclasspath_fragment_sdk_test.go b/sdk/bootclasspath_fragment_sdk_test.go index 1b64130ce..58c164724 100644 --- a/sdk/bootclasspath_fragment_sdk_test.go +++ b/sdk/bootclasspath_fragment_sdk_test.go @@ -169,7 +169,15 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["com.android.art"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/core1.jar"], + jars: [":mysdk_core1-error"], +} + +genrule { + name: "mysdk_core1-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) core1", } java_import { @@ -177,7 +185,15 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["com.android.art"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/core2.jar"], + jars: [":mysdk_core2-error"], +} + +genrule { + name: "mysdk_core2-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) core2", } `), checkAllCopyRules(` @@ -187,8 +203,7 @@ java_import { .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/signature-patterns.csv -> hiddenapi/signature-patterns.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/filtered-stub-flags.csv -> hiddenapi/filtered-stub-flags.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/filtered-flags.csv -> hiddenapi/filtered-flags.csv -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/core1.jar -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/core2.jar +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh `), snapshotTestPreparer(checkSnapshotWithoutSource, preparerForSnapshot), @@ -357,10 +372,18 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["myapex"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar"], + jars: [":mysdk_mybootlib-error"], permitted_packages: ["mybootlib"], } +genrule { + name: "mysdk_mybootlib-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) mybootlib", +} + java_sdk_library_import { name: "myothersdklibrary", prefer: false, @@ -467,7 +490,7 @@ func TestSnapshotWithBootClasspathFragment_Contents(t *testing.T) { .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/signature-patterns.csv -> hiddenapi/signature-patterns.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/filtered-stub-flags.csv -> hiddenapi/filtered-stub-flags.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/filtered-flags.csv -> hiddenapi/filtered-flags.csv -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh .intermediates/myothersdklibrary.stubs/android_common/javac/myothersdklibrary.stubs.jar -> sdk_library/public/myothersdklibrary-stubs.jar .intermediates/myothersdklibrary.stubs.source/android_common/metalava/myothersdklibrary.stubs.source_api.txt -> sdk_library/public/myothersdklibrary.txt .intermediates/myothersdklibrary.stubs.source/android_common/metalava/myothersdklibrary.stubs.source_removed.txt -> sdk_library/public/myothersdklibrary-removed.txt @@ -487,7 +510,7 @@ func TestSnapshotWithBootClasspathFragment_Contents(t *testing.T) { .intermediates/mybootclasspathfragment/android_common_myapex/modular-hiddenapi/signature-patterns.csv -> hiddenapi/signature-patterns.csv .intermediates/mybootclasspathfragment/android_common_myapex/modular-hiddenapi/filtered-stub-flags.csv -> hiddenapi/filtered-stub-flags.csv .intermediates/mybootclasspathfragment/android_common_myapex/modular-hiddenapi/filtered-flags.csv -> hiddenapi/filtered-flags.csv -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh .intermediates/myothersdklibrary.stubs/android_common/javac/myothersdklibrary.stubs.jar -> sdk_library/public/myothersdklibrary-stubs.jar .intermediates/myothersdklibrary.stubs.source/android_common/metalava/myothersdklibrary.stubs.source_api.txt -> sdk_library/public/myothersdklibrary.txt .intermediates/myothersdklibrary.stubs.source/android_common/metalava/myothersdklibrary.stubs.source_removed.txt -> sdk_library/public/myothersdklibrary-removed.txt @@ -876,10 +899,18 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["myapex"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar"], + jars: [":mysdk_mybootlib-error"], permitted_packages: ["mybootlib"], } +genrule { + name: "mysdk_mybootlib-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) mybootlib", +} + java_sdk_library_import { name: "mynewlibrary", prefer: false, @@ -930,7 +961,7 @@ my-unsupported-packages.txt -> hiddenapi/my-unsupported-packages.txt .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/signature-patterns.csv -> hiddenapi/signature-patterns.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/filtered-stub-flags.csv -> hiddenapi/filtered-stub-flags.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/filtered-flags.csv -> hiddenapi/filtered-flags.csv -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh .intermediates/mynewlibrary.stubs/android_common/javac/mynewlibrary.stubs.jar -> sdk_library/public/mynewlibrary-stubs.jar .intermediates/mynewlibrary.stubs.source/android_common/metalava/mynewlibrary.stubs.source_api.txt -> sdk_library/public/mynewlibrary.txt .intermediates/mynewlibrary.stubs.source/android_common/metalava/mynewlibrary.stubs.source_removed.txt -> sdk_library/public/mynewlibrary-removed.txt diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 51903ce39..c6cb6c220 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -19,11 +19,13 @@ import ( "testing" "android/soong/android" + "android/soong/genrule" "android/soong/java" ) var prepareForSdkTestWithJava = android.GroupFixturePreparers( java.PrepareForTestWithJavaBuildComponents, + genrule.PrepareForTestWithGenRuleBuildComponents, PrepareForTestWithSdkBuildComponents, // Ensure that all source paths are provided. This helps ensure that the snapshot generation is @@ -34,6 +36,7 @@ var prepareForSdkTestWithJava = android.GroupFixturePreparers( // Files needs by most of the tests. android.MockFS{ "Test.java": nil, + "build/soong/java/invalid_implementation_jar.sh": nil, }.AddToFixture(), ) @@ -288,18 +291,26 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/myjavalib.jar"], + jars: [":mysdk_myjavalib-error"], permitted_packages: ["pkg.myjavalib"], } + +genrule { + name: "mysdk_myjavalib-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) myjavalib", +} `), checkAllCopyRules(` -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/myjavalib.jar +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh `), ) } func TestSnapshotWithJavaBootLibrary_UpdatableMedia(t *testing.T) { - runTest := func(t *testing.T, targetBuildRelease, expectedJarPath, expectedCopyRule string) { + runTest := func(t *testing.T, targetBuildRelease, expectedJarPath, expectedGenRule, expectedCopyRule string) { result := android.GroupFixturePreparers( prepareForSdkTestWithJava, android.FixtureMergeEnv(map[string]string{ @@ -334,20 +345,27 @@ java_import { jars: ["%s"], permitted_packages: ["pkg.media"], } -`, expectedJarPath)), +%s`, expectedJarPath, expectedGenRule)), checkAllCopyRules(expectedCopyRule), ) } t.Run("updatable-media in S", func(t *testing.T) { - runTest(t, "S", "java/updatable-media.jar", ` + runTest(t, "S", "java/updatable-media.jar", "", ` .intermediates/updatable-media/android_common/package-check/updatable-media.jar -> java/updatable-media.jar `) }) t.Run("updatable-media in T", func(t *testing.T) { - runTest(t, "Tiramisu", "java_boot_libs/snapshot/jars/are/invalid/updatable-media.jar", ` -.intermediates/mysdk/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/updatable-media.jar + runTest(t, "Tiramisu", ":mysdk_updatable-media-error", ` +genrule { + name: "mysdk_updatable-media-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) updatable-media", +}`, ` +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh `) }) } @@ -389,12 +407,20 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], - jars: ["java_systemserver_libs/snapshot/jars/are/invalid/myjavalib.jar"], + jars: [":myexports_myjavalib-error"], permitted_packages: ["pkg.myjavalib"], } + +genrule { + name: "myexports_myjavalib-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) myjavalib", +} `), checkAllCopyRules(` -.intermediates/myexports/common_os/empty -> java_systemserver_libs/snapshot/jars/are/invalid/myjavalib.jar +build/soong/java/invalid_implementation_jar.sh -> scripts/invalid_implementation_jar.sh `), ) } diff --git a/sdk/systemserverclasspath_fragment_sdk_test.go b/sdk/systemserverclasspath_fragment_sdk_test.go index 1ac405d7d..9540a6b50 100644 --- a/sdk/systemserverclasspath_fragment_sdk_test.go +++ b/sdk/systemserverclasspath_fragment_sdk_test.go @@ -119,10 +119,18 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["myapex"], - jars: ["java_systemserver_libs/snapshot/jars/are/invalid/mylib.jar"], + jars: [":mysdk_mylib-error"], permitted_packages: ["mylib"], } +genrule { + name: "mysdk_mylib-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) mylib", +} + prebuilt_systemserverclasspath_fragment { name: "mysystemserverclasspathfragment", prefer: false, @@ -180,10 +188,18 @@ java_import { prefer: false, visibility: ["//visibility:public"], apex_available: ["myapex"], - jars: ["java_systemserver_libs/snapshot/jars/are/invalid/mylib.jar"], + jars: [":mysdk_mylib-error"], permitted_packages: ["mylib"], } +genrule { + name: "mysdk_mylib-error", + visibility: ["//visibility:private"], + out: ["this-file-will-never-be-created.jar"], + tool_files: ["scripts/invalid_implementation_jar.sh"], + cmd: "$(location scripts/invalid_implementation_jar.sh) mylib", +} + prebuilt_systemserverclasspath_fragment { name: "mysystemserverclasspathfragment", prefer: false, diff --git a/sdk/update.go b/sdk/update.go index 92a13fa7f..6ebbf09d3 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -1049,9 +1049,6 @@ type snapshotBuilder struct { filesToZip android.Paths zipsToMerge android.Paths - // The path to an empty file. - emptyFile android.WritablePath - prebuiltModules map[string]*bpModule prebuiltOrder []*bpModule @@ -1111,19 +1108,6 @@ func (s *snapshotBuilder) UnzipToSnapshot(zipPath android.Path, destDir string) s.zipsToMerge = append(s.zipsToMerge, tmpZipPath) } -func (s *snapshotBuilder) EmptyFile() android.Path { - if s.emptyFile == nil { - ctx := s.ctx - s.emptyFile = android.PathForModuleOut(ctx, "empty") - s.ctx.Build(pctx, android.BuildParams{ - Rule: android.Touch, - Output: s.emptyFile, - }) - } - - return s.emptyFile -} - func (s *snapshotBuilder) AddPrebuiltModule(member android.SdkMember, moduleType string) android.BpModule { name := member.Name() if s.prebuiltModules[name] != nil { @@ -1200,6 +1184,24 @@ func (s *snapshotBuilder) AddPrebuiltModule(member android.SdkMember, moduleType return m } +func (s *snapshotBuilder) AddInternalModule(properties android.SdkMemberProperties, moduleType string, nameSuffix string) android.BpModule { + name := properties.Name() + "-" + nameSuffix + + if s.prebuiltModules[name] != nil { + panic(fmt.Sprintf("Duplicate module detected, module %s has already been added", name)) + } + + m := s.bpFile.newModule(moduleType) + m.AddProperty("name", name) + m.AddProperty("visibility", []string{"//visibility:private"}) + + s.prebuiltModules[name] = m + s.prebuiltOrder = append(s.prebuiltOrder, m) + + s.allMembersByName[name] = struct{}{} + return m +} + func addHostDeviceSupportedProperties(deviceSupported bool, hostSupported bool, bpModule *bpModule) { // If neither device or host is supported then this module does not support either so will not // recognize the properties. @@ -1230,18 +1232,23 @@ func (s *snapshotBuilder) OptionalSdkMemberReferencePropertyTag() android.BpProp // Get a name for sdk snapshot member. If the member is private then generate a snapshot specific // name. As part of the processing this checks to make sure that any required members are part of // the snapshot. -func (s *snapshotBuilder) snapshotSdkMemberName(name string, required bool) string { +func (s *snapshotBuilder) snapshotSdkMemberName(reference string, required bool) string { + prefix := "" + name := strings.TrimPrefix(reference, ":") + if name != reference { + prefix = ":" + } if _, ok := s.allMembersByName[name]; !ok { if required { s.ctx.ModuleErrorf("Required member reference %s is not a member of the sdk", name) } - return name + return reference } if s.isInternalMember(name) { - return s.ctx.ModuleName() + "_" + name + return prefix + s.ctx.ModuleName() + "_" + name } else { - return name + return reference } } @@ -2057,6 +2064,7 @@ func (s *sdk) createMemberSnapshot(ctx *memberContext, member *sdkMember, bpModu variantPropertiesFactory := func() android.SdkMemberProperties { properties := memberType.CreateVariantPropertiesStruct() base := properties.Base() + base.MemberName = member.Name() base.Os_count = osCount return properties } diff --git a/ui/build/Android.bp b/ui/build/Android.bp index cfcf8047a..7a8fca999 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -46,7 +46,6 @@ bootstrap_go_package { "soong-ui-tracer", ], srcs: [ - "bazel.go", "build.go", "cleanbuild.go", "config.go", diff --git a/ui/build/bazel.go b/ui/build/bazel.go deleted file mode 100644 index bd469a4cd..000000000 --- a/ui/build/bazel.go +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package build - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "android/soong/bazel" - "android/soong/shared" - "android/soong/ui/metrics" -) - -func getBazelInfo(ctx Context, config Config, bazelExecutable string, bazelEnv map[string]string, query string) string { - infoCmd := Command(ctx, config, "bazel", bazelExecutable) - - if extraStartupArgs, ok := infoCmd.Environment.Get("BAZEL_STARTUP_ARGS"); ok { - infoCmd.Args = append(infoCmd.Args, strings.Fields(extraStartupArgs)...) - } - - // Obtain the output directory path in the execution root. - infoCmd.Args = append(infoCmd.Args, - "info", - query, - ) - - for k, v := range bazelEnv { - infoCmd.Environment.Set(k, v) - } - - infoCmd.Dir = filepath.Join(config.OutDir(), "..") - - queryResult := strings.TrimSpace(string(infoCmd.OutputOrFatal())) - return queryResult -} - -// Main entry point to construct the Bazel build command line, environment -// variables and post-processing steps (e.g. converge output directories) -func runBazel(ctx Context, config Config) { - ctx.BeginTrace(metrics.RunBazel, "bazel") - defer ctx.EndTrace() - - // "droid" is the default ninja target. - // TODO(b/160568333): stop hardcoding 'droid' to support building any - // Ninja target. - outputGroups := "droid" - if len(config.ninjaArgs) > 0 { - // At this stage, the residue slice of args passed to ninja - // are the ninja targets to build, which can correspond directly - // to ninja_build's output_groups. - outputGroups = strings.Join(config.ninjaArgs, ",") - } - - // Environment variables are the primary mechanism to pass information from - // soong_ui configuration or context to Bazel. - bazelEnv := make(map[string]string) - - // Use *_NINJA variables to pass the root-relative path of the combined, - // kati-generated, soong-generated, and packaging Ninja files to Bazel. - // Bazel reads these from the lunch() repository rule. - bazelEnv["COMBINED_NINJA"] = config.CombinedNinjaFile() - bazelEnv["KATI_NINJA"] = config.KatiBuildNinjaFile() - bazelEnv["PACKAGE_NINJA"] = config.KatiPackageNinjaFile() - bazelEnv["SOONG_NINJA"] = config.SoongNinjaFile() - - // NOTE: When Bazel is used, config.DistDir() is rigged to return a fake distdir under config.OutDir() - // This is to ensure that Bazel can actually write there. See config.go for more details. - bazelEnv["DIST_DIR"] = config.DistDir() - - bazelEnv["SHELL"] = "/bin/bash" - - // `build/bazel/bin/bazel` is the default entry point for executing Bazel in the AOSP - // source tree. - bazelExecutable := filepath.Join("build", "bazel", "bin", "bazel") - cmd := Command(ctx, config, "bazel", bazelExecutable) - - // Append custom startup flags to the Bazel command. Startup flags affect - // the Bazel server itself, and any changes to these flags would incur a - // restart of the server, losing much of the in-memory incrementality. - if extraStartupArgs, ok := cmd.Environment.Get("BAZEL_STARTUP_ARGS"); ok { - cmd.Args = append(cmd.Args, strings.Fields(extraStartupArgs)...) - } - - // Start constructing the `build` command. - actionName := bazel.BazelNinjaExecRunName - cmd.Args = append(cmd.Args, - "build", - // Use output_groups to select the set of outputs to produce from a - // ninja_build target. - "--output_groups="+outputGroups, - // Generate a performance profile - "--profile="+filepath.Join(shared.BazelMetricsFilename(config, actionName)), - "--slim_profile=true", - ) - - if config.UseRBE() { - for _, envVar := range []string{ - // RBE client - "RBE_compare", - "RBE_exec_strategy", - "RBE_invocation_id", - "RBE_log_dir", - "RBE_num_retries_if_mismatched", - "RBE_platform", - "RBE_remote_accept_cache", - "RBE_remote_update_cache", - "RBE_server_address", - // TODO: remove old FLAG_ variables. - "FLAG_compare", - "FLAG_exec_root", - "FLAG_exec_strategy", - "FLAG_invocation_id", - "FLAG_log_dir", - "FLAG_platform", - "FLAG_remote_accept_cache", - "FLAG_remote_update_cache", - "FLAG_server_address", - } { - cmd.Args = append(cmd.Args, - "--action_env="+envVar) - } - - // We need to calculate --RBE_exec_root ourselves - ctx.Println("Getting Bazel execution_root...") - cmd.Args = append(cmd.Args, "--action_env=RBE_exec_root="+getBazelInfo(ctx, config, bazelExecutable, bazelEnv, "execution_root")) - } - - // Ensure that the PATH environment variable value used in the action - // environment is the restricted set computed from soong_ui, and not a - // user-provided one, for hermeticity reasons. - if pathEnvValue, ok := config.environ.Get("PATH"); ok { - cmd.Environment.Set("PATH", pathEnvValue) - cmd.Args = append(cmd.Args, "--action_env=PATH="+pathEnvValue) - } - - // Allow Bazel actions to see the SHELL variable (passed to Bazel above) - cmd.Args = append(cmd.Args, "--action_env=SHELL") - - // Append custom build flags to the Bazel command. Changes to these flags - // may invalidate Bazel's analysis cache. - // These should be appended as the final args, so that they take precedence. - if extraBuildArgs, ok := cmd.Environment.Get("BAZEL_BUILD_ARGS"); ok { - cmd.Args = append(cmd.Args, strings.Fields(extraBuildArgs)...) - } - - // Append the label of the default ninja_build target. - cmd.Args = append(cmd.Args, - "//:"+config.TargetProduct()+"-"+config.TargetBuildVariant(), - ) - - // Execute the command at the root of the directory. - cmd.Dir = filepath.Join(config.OutDir(), "..") - - for k, v := range bazelEnv { - cmd.Environment.Set(k, v) - } - - // Make a human-readable version of the bazelEnv map - bazelEnvStringBuffer := new(bytes.Buffer) - for k, v := range bazelEnv { - fmt.Fprintf(bazelEnvStringBuffer, "%s=%s ", k, v) - } - - // Print the implicit command line - ctx.Println("Bazel implicit command line: " + strings.Join(cmd.Environment.Environ(), " ") + " " + cmd.Cmd.String() + "\n") - - // Print the explicit command line too - ctx.Println("Bazel explicit command line: " + bazelEnvStringBuffer.String() + cmd.Cmd.String() + "\n") - - // Execute the build command. - cmd.RunAndStreamOrFatal() - - // Post-processing steps start here. Once the Bazel build completes, the - // output files are still stored in the execution root, not in $OUT_DIR. - // Ensure that the $OUT_DIR contains the expected set of files by symlinking - // the files from the execution root's output direction into $OUT_DIR. - - ctx.Println("Getting Bazel output_path...") - outputBasePath := getBazelInfo(ctx, config, bazelExecutable, bazelEnv, "output_path") - // TODO: Don't hardcode out/ as the bazel output directory. This is - // currently hardcoded as ninja_build.output_root. - bazelNinjaBuildOutputRoot := filepath.Join(outputBasePath, "..", "out") - - ctx.Println("Populating output directory...") - populateOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".") -} - -// For all files F recursively under rootPath/relativePath, creates symlinks -// such that OutDir/F resolves to rootPath/F via symlinks. -// NOTE: For distdir paths we rename files instead of creating symlinks, so that the distdir is independent. -func populateOutdir(ctx Context, config Config, rootPath string, relativePath string) { - destDir := filepath.Join(rootPath, relativePath) - os.MkdirAll(destDir, 0755) - files, err := ioutil.ReadDir(destDir) - if err != nil { - ctx.Fatal(err) - } - - for _, f := range files { - // The original Bazel file path - destPath := filepath.Join(destDir, f.Name()) - - // The desired Soong file path - srcPath := filepath.Join(config.OutDir(), relativePath, f.Name()) - - destLstatResult, destLstatErr := os.Lstat(destPath) - if destLstatErr != nil { - ctx.Fatalf("Unable to Lstat dest %s: %s", destPath, destLstatErr) - } - - srcLstatResult, srcLstatErr := os.Lstat(srcPath) - - if srcLstatErr == nil { - if srcLstatResult.IsDir() && destLstatResult.IsDir() { - // src and dest are both existing dirs - recurse on the dest dir contents... - populateOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name())) - } else { - // Ignore other pre-existing src files (could be pre-existing files, directories, symlinks, ...) - // This can arise for files which are generated under OutDir outside of soong_build, such as .bootstrap files. - // FIXME: This might cause a problem later e.g. if a symlink in the build graph changes... - } - } else { - if !os.IsNotExist(srcLstatErr) { - ctx.Fatalf("Unable to Lstat src %s: %s", srcPath, srcLstatErr) - } - - if strings.Contains(destDir, config.DistDir()) { - // We need to make a "real" file/dir instead of making a symlink (because the distdir can't have symlinks) - // Rename instead of copy in order to save disk space. - if err := os.Rename(destPath, srcPath); err != nil { - ctx.Fatalf("Unable to rename %s -> %s due to error %s", srcPath, destPath, err) - } - } else { - // src does not exist, so try to create a src -> dest symlink (i.e. a Soong path -> Bazel path symlink) - if err := os.Symlink(destPath, srcPath); err != nil { - ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, err) - } - } - } - } -} diff --git a/ui/build/build.go b/ui/build/build.go index b9bd898b4..d49a7543e 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -90,7 +90,7 @@ func createCombinedBuildNinjaFile(ctx Context, config Config) { } } -// These are bitmasks which can be used to check whether various flags are set e.g. whether to use Bazel. +// These are bitmasks which can be used to check whether various flags are set const ( _ = iota // Whether to run the kati config step. @@ -102,9 +102,7 @@ const ( // Whether to include the kati-generated ninja file in the combined ninja. RunKatiNinja = 1 << iota // Whether to run ninja on the combined ninja. - RunNinja = 1 << iota - // Whether to run bazel on the combined ninja. - RunBazel = 1 << iota + RunNinja = 1 << iota RunBuildTests = 1 << iota RunAll = RunProductConfig | RunSoong | RunKati | RunKatiNinja | RunNinja ) @@ -324,11 +322,6 @@ func Build(ctx Context, config Config) { runNinjaForBuild(ctx, config) } - - // Currently, using Bazel requires Kati and Soong to run first, so check whether to run Bazel last. - if what&RunBazel != 0 { - runBazel(ctx, config) - } } func evaluateWhatToRun(config Config, verboseln func(v ...interface{})) int { diff --git a/ui/build/config.go b/ui/build/config.go index de10112bf..c98601e1b 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -82,6 +82,7 @@ type configImpl struct { skipSoong bool skipNinja bool skipSoongTests bool + searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces // From the product config katiArgs []string @@ -738,6 +739,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { c.bazelDevMode = true } else if arg == "--bazel-mode-staging" { c.bazelStagingMode = true + } else if arg == "--search-api-dir" { + c.searchApiDir = true } else if len(arg) > 0 && arg[0] == '-' { parseArgNum := func(def int) int { if len(arg) > 2 { @@ -896,10 +899,18 @@ func (c *configImpl) BazelOutDir() string { return filepath.Join(c.OutDir(), "bazel") } +func (c *configImpl) bazelOutputBase() string { + return filepath.Join(c.BazelOutDir(), "output") +} + func (c *configImpl) SoongOutDir() string { return filepath.Join(c.OutDir(), "soong") } +func (c *configImpl) ApiSurfacesOutDir() string { + return filepath.Join(c.OutDir(), "api_surfaces") +} + func (c *configImpl) PrebuiltOS() string { switch runtime.GOOS { case "linux": diff --git a/ui/build/finder.go b/ui/build/finder.go index 4d6ad426f..3f628cf7b 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -63,7 +63,7 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { // Set up configuration parameters for the Finder cache. cacheParams := finder.CacheParams{ WorkingDirectory: dir, - RootDirs: []string{"."}, + RootDirs: androidBpSearchDirs(config), FollowSymlinks: config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"), ExcludeDirs: []string{".git", ".repo"}, PruneFiles: pruneFiles, @@ -100,6 +100,15 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { return f } +func androidBpSearchDirs(config Config) []string { + dirs := []string{"."} // always search from root of source tree. + if config.searchApiDir { + // Search in out/api_surfaces + dirs = append(dirs, config.ApiSurfacesOutDir()) + } + return dirs +} + // Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree. func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { matches := []string{} diff --git a/ui/build/sandbox_config.go b/ui/build/sandbox_config.go index 1b4645967..1d32d860a 100644 --- a/ui/build/sandbox_config.go +++ b/ui/build/sandbox_config.go @@ -27,6 +27,15 @@ func (sc *SandboxConfig) SrcDirIsRO() bool { return sc.srcDirIsRO } +// Return the mount flag of the source directory in the nsjail command +func (sc *SandboxConfig) SrcDirMountFlag() string { + ret := "-B" // Read-write + if sc.SrcDirIsRO() { + ret = "-R" // Read-only + } + return ret +} + func (sc *SandboxConfig) SetSrcDirRWAllowlist(allowlist []string) { sc.srcDirRWAllowlist = allowlist } diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go index 5b2046e54..edb3b66d4 100644 --- a/ui/build/sandbox_linux.go +++ b/ui/build/sandbox_linux.go @@ -101,7 +101,7 @@ func (c *Cmd) sandboxSupported() bool { // srcDir is /tmp/.* in integration tests, which is a child dir of /tmp // nsjail throws an error if a child dir is mounted before its parent "-B", "/tmp", - "-B", sandboxConfig.srcDir, + c.config.sandboxConfig.SrcDirMountFlag(), sandboxConfig.srcDir, "-B", sandboxConfig.outDir, } @@ -148,13 +148,6 @@ func (c *Cmd) sandboxSupported() bool { func (c *Cmd) wrapSandbox() { wd, _ := os.Getwd() - var srcDirMountFlag string - if c.config.sandboxConfig.SrcDirIsRO() { - srcDirMountFlag = "-R" - } else { - srcDirMountFlag = "-B" //Read-Write - } - sandboxArgs := []string{ // The executable to run "-x", c.Path, @@ -195,7 +188,7 @@ func (c *Cmd) wrapSandbox() { "-B", "/tmp", // Mount source - srcDirMountFlag, sandboxConfig.srcDir, + c.config.sandboxConfig.SrcDirMountFlag(), sandboxConfig.srcDir, //Mount out dir as read-write "-B", sandboxConfig.outDir, diff --git a/ui/build/soong.go b/ui/build/soong.go index 4aded1705..c0bee4e41 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -302,7 +302,7 @@ func bootstrapBlueprint(ctx Context, config Config) { ) bp2buildWorkspaceInvocation.Inputs = append(bp2buildWorkspaceInvocation.Inputs, - config.Bp2BuildFilesMarkerFile()) + config.Bp2BuildFilesMarkerFile(), filepath.Join(config.FileListDir(), "bazel.list")) jsonModuleGraphInvocation := primaryBuilderInvocation( config, @@ -418,7 +418,7 @@ func runSoong(ctx Context, config Config) { // Bazel's HOME var is set to an output subdirectory which doesn't exist. This // prevents Bazel from file I/O in the actual user HOME directory. soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome"))) - soongBuildEnv.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output")) + soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase()) soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, ".")) soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir()) soongBuildEnv.Set("LOG_DIR", config.LogsDir()) diff --git a/ui/build/test_build.go b/ui/build/test_build.go index 86c85681a..2efc732c2 100644 --- a/ui/build/test_build.go +++ b/ui/build/test_build.go @@ -18,14 +18,44 @@ import ( "bufio" "fmt" "path/filepath" + "regexp" "runtime" "sort" "strings" + "sync" "android/soong/ui/metrics" "android/soong/ui/status" ) +var ( + // bazel output paths are in __main__/bazel-out/<config-specific-path>/bin + bazelOutputPathRegexOnce sync.Once + bazelOutputPathRegexp *regexp.Regexp +) + +func bazelOutputPathPattern(config Config) *regexp.Regexp { + bazelOutputPathRegexOnce.Do(func() { + // Bazel output files are in <Bazel output base>/execroot/__main__/bazel-out/<config>/bin + bazelOutRoot := filepath.Join(regexp.QuoteMeta(config.bazelOutputBase()), "execroot", "__main__", "bazel-out") + bazelOutputPathRegexp = regexp.MustCompile(bazelOutRoot + "/[^/]+/bin") + }) + return bazelOutputPathRegexp +} + +func ignoreBazelPath(config Config, path string) bool { + bazelRoot := filepath.Join(config.bazelOutputBase(), "execroot") + // Don't check bazel output regexp unless it is Bazel path + if strings.HasPrefix(path, bazelRoot) { + bazelOutputRegexp := bazelOutputPathPattern(config) + // if the file is a bazel path that is _not_ a Bazel generated file output, we rely on Bazel to + // ensure the paths to exist. If it _is_ a Bazel output path, we expect that it should be built + // by Ninja. + return !bazelOutputRegexp.MatchString(path) + } + return false +} + // Checks for files in the out directory that have a rule that depends on them but no rule to // create them. This catches a common set of build failures where a rule to generate a file is // deleted (either by deleting a module in an Android.mk file, or by modifying the build system @@ -97,6 +127,10 @@ func testForDanglingRules(ctx Context, config Config) { // full build rules in the primary build.ninja file. continue } + + if ignoreBazelPath(config, line) { + continue + } danglingRules[line] = true } |