diff options
50 files changed, 813 insertions, 599 deletions
diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 1353293f2..4fc6ae368 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -1326,6 +1326,9 @@ var ( // uses glob in $(locations) "libc_musl_sysroot", + + // TODO(b/266459895): depends on libunwindstack + "libutils_test", } MixedBuildsDisabledList = []string{ diff --git a/android/bazel_handler.go b/android/bazel_handler.go index ad21a2e9b..d5886dc34 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -208,9 +208,11 @@ type bazelPaths struct { // and their results after the requests have been made. type mixedBuildBazelContext struct { bazelRunner - paths *bazelPaths - requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel - requestMutex sync.Mutex // requests can be written in parallel + paths *bazelPaths + // cquery requests that have not yet been issued to Bazel. This list is maintained + // in a sorted state, and is guaranteed to have no duplicates. + requests []cqueryKey + requestMutex sync.Mutex // requests can be written in parallel results map[cqueryKey]string // Results of cquery requests after Bazel invocations @@ -321,7 +323,29 @@ func (bazelCtx *mixedBuildBazelContext) QueueBazelRequest(label string, requestT key := makeCqueryKey(label, requestType, cfgKey) bazelCtx.requestMutex.Lock() defer bazelCtx.requestMutex.Unlock() - bazelCtx.requests[key] = true + + // Insert key into requests, maintaining the sort, and only if it's not duplicate. + keyString := key.String() + foundEqual := false + notLessThanKeyString := func(i int) bool { + s := bazelCtx.requests[i].String() + v := strings.Compare(s, keyString) + if v == 0 { + foundEqual = true + } + return v >= 0 + } + targetIndex := sort.Search(len(bazelCtx.requests), notLessThanKeyString) + if foundEqual { + return + } + + if targetIndex == len(bazelCtx.requests) { + bazelCtx.requests = append(bazelCtx.requests, key) + } else { + bazelCtx.requests = append(bazelCtx.requests[:targetIndex+1], bazelCtx.requests[targetIndex:]...) + bazelCtx.requests[targetIndex] = key + } } func (bazelCtx *mixedBuildBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { @@ -512,7 +536,6 @@ func NewBazelContext(c *config) (BazelContext, error) { return &mixedBuildBazelContext{ bazelRunner: &builtinBazelRunner{}, paths: &paths, - requests: make(map[cqueryKey]bool), modulesDefaultToBazel: c.BuildMode == BazelDevMode, bazelEnabledModules: enabledModules, bazelDisabledModules: disabledModules, @@ -759,14 +782,23 @@ config_node(name = "%s", configNodesSection := "" labelsByConfig := map[string][]string{} - for val := range context.requests { + + for _, val := range context.requests { labelString := fmt.Sprintf("\"@%s\"", val.label) configString := getConfigString(val) labelsByConfig[configString] = append(labelsByConfig[configString], labelString) } + // Configs need to be sorted to maintain determinism of the BUILD file. + sortedConfigs := make([]string, 0, len(labelsByConfig)) + for val := range labelsByConfig { + sortedConfigs = append(sortedConfigs, val) + } + sort.Slice(sortedConfigs, func(i, j int) bool { return sortedConfigs[i] < sortedConfigs[j] }) + allLabels := []string{} - for configString, labels := range labelsByConfig { + for _, configString := range sortedConfigs { + labels := labelsByConfig[configString] configTokens := strings.Split(configString, "|") if len(configTokens) != 2 { panic(fmt.Errorf("Unexpected config string format: %s", configString)) @@ -797,7 +829,7 @@ func indent(original string) string { // request type. func (context *mixedBuildBazelContext) cqueryStarlarkFileContents() []byte { requestTypeToCqueryIdEntries := map[cqueryRequest][]string{} - for val := range context.requests { + for _, val := range context.requests { cqueryId := getCqueryId(val) mapEntryString := fmt.Sprintf("%q : True", cqueryId) requestTypeToCqueryIdEntries[val.requestType] = @@ -976,7 +1008,7 @@ func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx *Context) } // Clear requests. - context.requests = map[cqueryKey]bool{} + context.requests = []cqueryKey{} return nil } @@ -993,17 +1025,17 @@ func (context *mixedBuildBazelContext) runCquery(config Config, ctx *Context) er return err } } - if err := os.WriteFile(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil { + if err := writeFileBytesIfChanged(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil { return err } - if err := os.WriteFile(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil { + if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil { return err } - if err := os.WriteFile(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil { + if err := writeFileBytesIfChanged(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil { return err } cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery") - if err := os.WriteFile(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil { + if err := writeFileBytesIfChanged(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil { return err } @@ -1024,7 +1056,7 @@ func (context *mixedBuildBazelContext) runCquery(config Config, ctx *Context) er cqueryResults[splitLine[0]] = splitLine[1] } } - for val := range context.requests { + for _, val := range context.requests { if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok { context.results[val] = cqueryResult } else { @@ -1035,6 +1067,14 @@ func (context *mixedBuildBazelContext) runCquery(config Config, ctx *Context) er return nil } +func writeFileBytesIfChanged(path string, contents []byte, perm os.FileMode) error { + oldContents, err := os.ReadFile(path) + if err != nil || !bytes.Equal(contents, oldContents) { + err = os.WriteFile(path, contents, perm) + } + return nil +} + func (context *mixedBuildBazelContext) runAquery(config Config, ctx *Context) error { if ctx != nil { ctx.EventHandler.Begin("aquery") diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index 013e19c96..d97180205 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -168,6 +168,32 @@ func TestCoverageFlagsAfterInvokeBazel(t *testing.T) { } } +func TestBazelRequestsSorted(t *testing.T) { + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) + + bazelContext.QueueBazelRequest("zzz", cquery.GetOutputFiles, configKey{"arm64_armv8-a", Android}) + bazelContext.QueueBazelRequest("ccc", cquery.GetApexInfo, configKey{"arm64_armv8-a", Android}) + bazelContext.QueueBazelRequest("duplicate", cquery.GetOutputFiles, configKey{"arm64_armv8-a", Android}) + bazelContext.QueueBazelRequest("duplicate", cquery.GetOutputFiles, configKey{"arm64_armv8-a", Android}) + bazelContext.QueueBazelRequest("xxx", cquery.GetOutputFiles, configKey{"arm64_armv8-a", Linux}) + bazelContext.QueueBazelRequest("aaa", cquery.GetOutputFiles, configKey{"arm64_armv8-a", Android}) + bazelContext.QueueBazelRequest("aaa", cquery.GetOutputFiles, configKey{"otherarch", Android}) + bazelContext.QueueBazelRequest("bbb", cquery.GetOutputFiles, configKey{"otherarch", Android}) + + if len(bazelContext.requests) != 7 { + t.Error("Expected 7 request elements, but got", len(bazelContext.requests)) + } + + lastString := "" + for _, val := range bazelContext.requests { + thisString := val.String() + if thisString <= lastString { + t.Errorf("Requests are not ordered correctly. '%s' came before '%s'", lastString, thisString) + } + lastString = thisString + } +} + func verifyExtraFlags(t *testing.T, config Config, expected string) string { bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) @@ -204,7 +230,6 @@ func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) return &mixedBuildBazelContext{ bazelRunner: runner, paths: &p, - requests: map[cqueryKey]bool{}, }, p.soongOutDir } diff --git a/android/config.go b/android/config.go index c30511494..bb3cc97ab 100644 --- a/android/config.go +++ b/android/config.go @@ -397,11 +397,13 @@ product_var_constraints = _product_var_constraints arch_variant_product_var_constraints = _arch_variant_product_var_constraints `, } - err = os.WriteFile(filepath.Join(dir, "product_variables.bzl"), []byte(strings.Join(bzl, "\n")), 0644) + err = pathtools.WriteFileIfChanged(filepath.Join(dir, "product_variables.bzl"), + []byte(strings.Join(bzl, "\n")), 0644) if err != nil { return fmt.Errorf("Could not write .bzl config file %s", err) } - err = os.WriteFile(filepath.Join(dir, "BUILD"), []byte(bazel.GeneratedBazelFileWarning), 0644) + err = pathtools.WriteFileIfChanged(filepath.Join(dir, "BUILD"), + []byte(bazel.GeneratedBazelFileWarning), 0644) if err != nil { return fmt.Errorf("Could not write BUILD config file %s", err) } @@ -725,10 +727,6 @@ func (c *config) IsEnvFalse(key string) bool { return value == "0" || value == "n" || value == "no" || value == "off" || value == "false" } -func (c *config) TargetsJava17() bool { - return c.IsEnvTrue("EXPERIMENTAL_TARGET_JAVA_VERSION_17") -} - // EnvDeps returns the environment variables this build depends on. The first // call to this function blocks future reads from the environment. func (c *config) EnvDeps() map[string]string { diff --git a/android/defaults.go b/android/defaults.go index 7906e946c..925eafcee 100644 --- a/android/defaults.go +++ b/android/defaults.go @@ -55,7 +55,7 @@ func (d *DefaultableModuleBase) SetDefaultableHook(hook DefaultableHook) { d.hook = hook } -func (d *DefaultableModuleBase) callHookIfAvailable(ctx DefaultableHookContext) { +func (d *DefaultableModuleBase) CallHookIfAvailable(ctx DefaultableHookContext) { if d.hook != nil { d.hook(ctx) } @@ -82,7 +82,7 @@ type Defaultable interface { SetDefaultableHook(hook DefaultableHook) // Call the hook if specified. - callHookIfAvailable(context DefaultableHookContext) + CallHookIfAvailable(context DefaultableHookContext) } type DefaultableModule interface { @@ -630,6 +630,6 @@ func defaultsMutator(ctx TopDownMutatorContext) { defaultable.applyDefaults(ctx, defaultsList) } - defaultable.callHookIfAvailable(ctx) + defaultable.CallHookIfAvailable(ctx) } } diff --git a/android/packaging.go b/android/packaging.go index ecd84a2f0..c6b20ea74 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -73,6 +73,10 @@ func (p *PackagingSpec) Partition() string { return p.partition } +func (p *PackagingSpec) SrcPath() Path { + return p.srcPath +} + type PackageModule interface { Module packagingBase() *PackagingBase diff --git a/android/paths.go b/android/paths.go index 0fc39df6c..6c3009f4c 100644 --- a/android/paths.go +++ b/android/paths.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "io/ioutil" "os" "path/filepath" "reflect" @@ -1868,10 +1867,14 @@ func (p InstallPaths) Strings() []string { return ret } -// validateSafePath validates a path that we trust (may contain ninja variables). -// Ensures that each path component does not attempt to leave its component. -func validateSafePath(pathComponents ...string) (string, error) { +// validatePathInternal ensures that a path does not leave its component, and +// optionally doesn't contain Ninja variables. +func validatePathInternal(allowNinjaVariables bool, pathComponents ...string) (string, error) { for _, path := range pathComponents { + if !allowNinjaVariables && strings.Contains(path, "$") { + return "", fmt.Errorf("Path contains invalid character($): %s", path) + } + path := filepath.Clean(path) if path == ".." || strings.HasPrefix(path, "../") || strings.HasPrefix(path, "/") { return "", fmt.Errorf("Path is outside directory: %s", path) @@ -1883,16 +1886,18 @@ func validateSafePath(pathComponents ...string) (string, error) { return filepath.Join(pathComponents...), nil } +// validateSafePath validates a path that we trust (may contain ninja +// variables). Ensures that each path component does not attempt to leave its +// component. Returns a joined version of each path component. +func validateSafePath(pathComponents ...string) (string, error) { + return validatePathInternal(true, pathComponents...) +} + // validatePath validates that a path does not include ninja variables, and that // each path component does not attempt to leave its component. Returns a joined // version of each path component. func validatePath(pathComponents ...string) (string, error) { - for _, path := range pathComponents { - if strings.Contains(path, "$") { - return "", fmt.Errorf("Path contains invalid character($): %s", path) - } - } - return validateSafePath(pathComponents...) + return validatePathInternal(false, pathComponents...) } func PathForPhony(ctx PathContext, phony string) WritablePath { @@ -2093,13 +2098,16 @@ func maybeRelErr(basePath string, targetPath string) (string, bool, error) { // Writes a file to the output directory. Attempting to write directly to the output directory // will fail due to the sandbox of the soong_build process. +// Only writes the file if the file doesn't exist or if it has different contents, to prevent +// updating the timestamp if no changes would be made. (This is better for incremental +// performance.) func WriteFileToOutputDir(path WritablePath, data []byte, perm os.FileMode) error { absPath := absolutePath(path.String()) err := os.MkdirAll(filepath.Dir(absPath), 0777) if err != nil { return err } - return ioutil.WriteFile(absPath, data, perm) + return pathtools.WriteFileIfChanged(absPath, data, perm) } func RemoveAllOutputDir(path WritablePath) error { diff --git a/apex/androidmk.go b/apex/androidmk.go index 7babd45ba..7f0362109 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -23,8 +23,7 @@ import ( "android/soong/android" "android/soong/cc" "android/soong/java" - - "github.com/google/blueprint/proptools" + "android/soong/rust" ) func (a *apexBundle) AndroidMk() android.AndroidMkData { @@ -73,12 +72,15 @@ func (a *apexBundle) fullModuleName(apexBundleName string, fi *apexFile) string return fi.androidMkModuleName + "." + apexBundleName + a.suffix } -func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, apexName, moduleDir string, +func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, moduleDir string, apexAndroidMkData android.AndroidMkData) []string { - // apexBundleName comes from the 'name' property; apexName comes from 'apex_name' property. + // apexBundleName comes from the 'name' property or soong module. + // apexName comes from 'name' property of apex_manifest. // An apex is installed to /system/apex/<apexBundleName> and is activated at /apex/<apexName> // In many cases, the two names are the same, but could be different in general. + // However, symbol files for apex files are installed under /apex/<apexBundleName> to avoid + // conflicts between two apexes with the same apexName. moduleNames := []string{} apexType := a.properties.ApexType @@ -89,25 +91,6 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, apexName, mo return moduleNames } - // b/162366062. Prevent GKI APEXes to emit make rules to avoid conflicts. - if strings.HasPrefix(apexName, "com.android.gki.") && apexType != flattenedApex { - return moduleNames - } - - // b/140136207. When there are overriding APEXes for a VNDK APEX, the symbols file for the overridden - // APEX and the overriding APEX will have the same installation paths at /apex/com.android.vndk.v<ver> - // as their apexName will be the same. To avoid the path conflicts, skip installing the symbol files - // for the overriding VNDK APEXes. - symbolFilesNotNeeded := a.vndkApex && len(a.overridableProperties.Overrides) > 0 - if symbolFilesNotNeeded && apexType != flattenedApex { - return moduleNames - } - - // Avoid creating duplicate build rules for multi-installed APEXes. - if proptools.BoolDefault(a.properties.Multi_install_skip_symbol_files, false) { - return moduleNames - } - seenDataOutPaths := make(map[string]bool) for _, fi := range a.filesInfo { @@ -144,15 +127,15 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, apexName, mo if fi.module != nil && fi.module.Owner() != "" { fmt.Fprintln(w, "LOCAL_MODULE_OWNER :=", fi.module.Owner()) } - // /apex/<apex_name>/{lib|framework|...} - pathWhenActivated := filepath.Join("$(PRODUCT_OUT)", "apex", apexName, fi.installDir) + // /apex/<apexBundleName>/{lib|framework|...} + pathForSymbol := filepath.Join("$(PRODUCT_OUT)", "apex", apexBundleName, fi.installDir) var modulePath string if apexType == flattenedApex { - // /system/apex/<name>/{lib|framework|...} + // /system/apex/<apexBundleName>/{lib|framework|...} modulePath = filepath.Join(a.installDir.String(), apexBundleName, fi.installDir) fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", modulePath) - if a.primaryApexType && !symbolFilesNotNeeded { - fmt.Fprintln(w, "LOCAL_SOONG_SYMBOL_PATH :=", pathWhenActivated) + if a.primaryApexType { + fmt.Fprintln(w, "LOCAL_SOONG_SYMBOL_PATH :=", pathForSymbol) } android.AndroidMkEmitAssignList(w, "LOCAL_MODULE_SYMLINKS", fi.symlinks) newDataPaths := []android.DataPath{} @@ -165,8 +148,8 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, apexName, mo } android.AndroidMkEmitAssignList(w, "LOCAL_TEST_DATA", android.AndroidMkDataPaths(newDataPaths)) } else { - modulePath = pathWhenActivated - fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", pathWhenActivated) + modulePath = pathForSymbol + fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", modulePath) // For non-flattend APEXes, the merged notice file is attached to the APEX itself. // We don't need to have notice file for the individual modules in it. Otherwise, @@ -256,6 +239,10 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexBundleName, apexName, mo if ccMod.CoverageOutputFile().Valid() { fmt.Fprintln(w, "LOCAL_PREBUILT_COVERAGE_ARCHIVE :=", ccMod.CoverageOutputFile().String()) } + } else if rustMod, ok := fi.module.(*rust.Module); ok { + if rustMod.UnstrippedOutputFile() != nil { + fmt.Fprintln(w, "LOCAL_SOONG_UNSTRIPPED_BINARY :=", rustMod.UnstrippedOutputFile().String()) + } } fmt.Fprintln(w, "include $(BUILD_SYSTEM)/soong_cc_rust_prebuilt.mk") default: @@ -320,8 +307,7 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { moduleNames := []string{} apexType := a.properties.ApexType if a.installable() { - apexName := proptools.StringDefault(a.properties.Apex_name, name) - moduleNames = a.androidMkForFiles(w, name, apexName, moduleDir, data) + moduleNames = a.androidMkForFiles(w, name, moduleDir, data) } if apexType == flattenedApex { diff --git a/apex/apex.go b/apex/apex.go index f6fecb5aa..ff38773b8 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -94,10 +94,6 @@ type apexBundleProperties struct { // a default one is automatically generated. AndroidManifest *string `android:"path"` - // Canonical name of this APEX bundle. Used to determine the path to the activated APEX on - // device (/apex/<apex_name>). If unspecified, follows the name property. - Apex_name *string - // Determines the file contexts file for setting the security contexts to files in this APEX // bundle. For platform APEXes, this should points to a file under /system/sepolicy Default: // /system/sepolicy/apex/<module_name>_file_contexts. @@ -149,16 +145,6 @@ type apexBundleProperties struct { // Should be only used in non-system apexes (e.g. vendor: true). Default is false. Use_vndk_as_stable *bool - // Whether this is multi-installed APEX should skip installing symbol files. - // Multi-installed APEXes share the same apex_name and are installed at the same time. - // Default is false. - // - // Should be set to true for all multi-installed APEXes except the singular - // default version within the multi-installed group. - // Only the default version can install symbol files in $(PRODUCT_OUT}/apex, - // or else conflicting build rules may be created. - Multi_install_skip_symbol_files *bool - // The type of APEX to build. Controls what the APEX payload is. Either 'image', 'zip' or // 'both'. When set to image, contents are stored in a filesystem image inside a zip // container. When set to zip, contents are stored in a zip container directly. This type is @@ -1047,7 +1033,7 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { // This is the main part of this mutator. Mark the collected dependencies that they need to // be built for this apexBundle. - apexVariationName := proptools.StringDefault(a.properties.Apex_name, mctx.ModuleName()) // could be com.android.foo + apexVariationName := mctx.ModuleName() // could be com.android.foo a.properties.ApexVariationName = apexVariationName apexInfo := android.ApexInfo{ ApexVariationName: apexVariationName, @@ -1781,6 +1767,18 @@ func apexFileForJavaModuleWithFile(ctx android.BaseModuleContext, module javaMod return af } +func apexFileForJavaModuleProfile(ctx android.BaseModuleContext, module javaModule) *apexFile { + if dexpreopter, ok := module.(java.DexpreopterInterface); ok { + if profilePathOnHost := dexpreopter.ProfilePathOnHost(); profilePathOnHost != nil { + dirInApex := "javalib" + af := newApexFile(ctx, profilePathOnHost, module.BaseModuleName()+"-profile", dirInApex, etc, nil) + af.customStem = module.Stem() + ".jar.prof" + return &af + } + } + return nil +} + // androidApp is an interface to handle all app modules (android_app, android_app_import, etc.) in // the same way. type androidApp interface { @@ -1996,21 +1994,17 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) { a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile, a.compatSymlinks.Paths()...) default: - panic(fmt.Errorf("unexpected apex_type for the ProcessBazelQuery: %v", a.properties.ApexType)) + panic(fmt.Errorf("internal error: unexpected apex_type for the ProcessBazelQueryResponse: %v", a.properties.ApexType)) } - /* - TODO(asmundak): compared to building an APEX with Soong, building it with Bazel does not - return filesInfo and requiredDeps fields (in the Soong build the latter is updated). - Fix this, as these fields are subsequently used in apex/androidmk.go and in apex/builder/go - To find out what Soong build puts there, run: - vctx := visitorContext{handleSpecialLibs: !android.Bool(a.properties.Ignore_system_library_special_case)} - ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool { - return a.depVisitor(&vctx, ctx, child, parent) - }) - vctx.normalizeFileInfo() - */ - + // filesInfo is not set in mixed mode, because all information about the + // apex's contents should completely come from the Starlark providers. + // + // Prevent accidental writes to filesInfo in the earlier parts Soong by + // asserting it to be nil. + if a.filesInfo != nil { + panic(fmt.Errorf("internal error: filesInfo must be nil for an apex handled by Bazel.")) + } } func (a *apexBundle) setCompression(ctx android.ModuleContext) { @@ -2480,6 +2474,9 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, case *java.Library, *java.SdkLibrary: af := apexFileForJavaModule(ctx, child.(javaModule)) vctx.filesInfo = append(vctx.filesInfo, af) + if profileAf := apexFileForJavaModuleProfile(ctx, child.(javaModule)); profileAf != nil { + vctx.filesInfo = append(vctx.filesInfo, *profileAf) + } return true // track transitive dependencies default: ctx.PropertyErrorf("systemserverclasspath_fragments", diff --git a/apex/apex_test.go b/apex/apex_test.go index 31e848e42..eec24b050 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -3484,14 +3484,14 @@ func getFiles(t *testing.T, ctx *android.TestContext, moduleName, variant string return ret } -func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName, variant string, files []string) { +func assertFileListEquals(t *testing.T, expectedFiles []string, actualFiles []fileInApex) { t.Helper() var failed bool var surplus []string filesMatched := make(map[string]bool) - for _, file := range getFiles(t, ctx, moduleName, variant) { + for _, file := range actualFiles { matchFound := false - for _, expected := range files { + for _, expected := range expectedFiles { if file.match(expected) { matchFound = true filesMatched[expected] = true @@ -3509,9 +3509,9 @@ func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName, var failed = true } - if len(files) > len(filesMatched) { + if len(expectedFiles) > len(filesMatched) { var missing []string - for _, expected := range files { + for _, expected := range expectedFiles { if !filesMatched[expected] { missing = append(missing, expected) } @@ -3525,6 +3525,32 @@ func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName, var } } +func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName, variant string, files []string) { + assertFileListEquals(t, files, getFiles(t, ctx, moduleName, variant)) +} + +func ensureExactDeapexedContents(t *testing.T, ctx *android.TestContext, moduleName string, variant string, files []string) { + deapexer := ctx.ModuleForTests(moduleName+".deapexer", variant).Rule("deapexer") + outputs := make([]string, 0, len(deapexer.ImplicitOutputs)+1) + if deapexer.Output != nil { + outputs = append(outputs, deapexer.Output.String()) + } + for _, output := range deapexer.ImplicitOutputs { + outputs = append(outputs, output.String()) + } + actualFiles := make([]fileInApex, 0, len(outputs)) + for _, output := range outputs { + dir := "/deapexer/" + pos := strings.LastIndex(output, dir) + if pos == -1 { + t.Fatal("Unknown deapexer output ", output) + } + path := output[pos+len(dir):] + actualFiles = append(actualFiles, fileInApex{path: path, src: "", isLink: false}) + } + assertFileListEquals(t, files, actualFiles) +} + func TestVndkApexCurrent(t *testing.T) { commonFiles := []string{ "lib/libc++.so", @@ -3806,11 +3832,9 @@ func TestVndkApexNameRule(t *testing.T) { }`+vndkLibrariesTxtFiles("28", "current")) assertApexName := func(expected, moduleName string) { - bundle := ctx.ModuleForTests(moduleName, "android_common_image").Module().(*apexBundle) - actual := proptools.String(bundle.properties.Apex_name) - if !reflect.DeepEqual(actual, expected) { - t.Errorf("Got '%v', expected '%v'", actual, expected) - } + module := ctx.ModuleForTests(moduleName, "android_common_image") + apexManifestRule := module.Rule("apexManifestRule") + ensureContains(t, apexManifestRule.Args["opt"], "-v name "+expected) } assertApexName("com.android.vndk.v29", "com.android.vndk.current") @@ -4107,57 +4131,11 @@ func TestDependenciesInApexManifest(t *testing.T) { ensureListEmpty(t, requireNativeLibs) } -func TestApexName(t *testing.T) { - ctx := testApex(t, ` - apex { - name: "myapex", - key: "myapex.key", - apex_name: "com.android.myapex", - native_shared_libs: ["mylib"], - updatable: false, - } - - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - } - - cc_library { - name: "mylib", - srcs: ["mylib.cpp"], - system_shared_libs: [], - stl: "none", - apex_available: [ - "//apex_available:platform", - "myapex", - ], - } - `) - - module := ctx.ModuleForTests("myapex", "android_common_com.android.myapex_image") - apexManifestRule := module.Rule("apexManifestRule") - ensureContains(t, apexManifestRule.Args["opt"], "-v name com.android.myapex") - apexRule := module.Rule("apexRule") - ensureContains(t, apexRule.Args["opt_flags"], "--do_not_check_keyname") - - apexBundle := module.Module().(*apexBundle) - data := android.AndroidMkDataForTest(t, ctx, apexBundle) - name := apexBundle.BaseModuleName() - prefix := "TARGET_" - var builder strings.Builder - data.Custom(&builder, name, prefix, "", data) - androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n") - ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n") -} - func TestOverrideApexManifestDefaultVersion(t *testing.T) { ctx := testApex(t, ` apex { name: "myapex", key: "myapex.key", - apex_name: "com.android.myapex", native_shared_libs: ["mylib"], updatable: false, } @@ -4182,7 +4160,7 @@ func TestOverrideApexManifestDefaultVersion(t *testing.T) { "OVERRIDE_APEX_MANIFEST_DEFAULT_VERSION": "1234", })) - module := ctx.ModuleForTests("myapex", "android_common_com.android.myapex_image") + module := ctx.ModuleForTests("myapex", "android_common_myapex_image") apexManifestRule := module.Rule("apexManifestRule") ensureContains(t, apexManifestRule.Args["default_version"], "1234") } diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index af4fd9f27..2ddfd0305 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -530,9 +530,8 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { java.FixtureSetBootImageInstallDirOnDevice("art", "apex/com.android.art/javalib"), ).RunTest(t) - ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + ensureExactDeapexedContents(t, result.TestContext, "com.android.art", "android_common", []string{ "etc/boot-image.prof", - "etc/classpaths/bootclasspath.pb", "javalib/arm/boot.art", "javalib/arm/boot.oat", "javalib/arm/boot.vdex", @@ -592,9 +591,8 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), ).RunTest(t) - ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + ensureExactDeapexedContents(t, result.TestContext, "com.android.art", "android_common", []string{ "etc/boot-image.prof", - "etc/classpaths/bootclasspath.pb", "javalib/bar.jar", "javalib/foo.jar", }) diff --git a/apex/bp2build.go b/apex/bp2build.go index d28f5122e..a3dda83b1 100644 --- a/apex/bp2build.go +++ b/apex/bp2build.go @@ -15,16 +15,22 @@ package apex import ( "android/soong/android" + "encoding/json" "strings" ) // This file contains the bp2build integration for the apex package. // Export constants as Starlark using bp2build to Bazel. -func BazelApexToolchainVars() string { +func BazelApexToolchainVars() (string, error) { + marshalled, err := json.Marshal(apexAvailBaseline) + if err != nil { + return "", err + } content := []string{ "# GENERATED BY SOONG. DO NOT EDIT.", "default_manifest_version = " + android.DefaultUpdatableModuleVersion, // constants.go is different in every branch. + "apex_available_baseline = json.decode('''" + string(marshalled) + "''')", } - return strings.Join(content, "\n") + return strings.Join(content, "\n"), nil } diff --git a/apex/builder.go b/apex/builder.go index 4331d3ec2..93ff80d70 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -241,10 +241,11 @@ func (a *apexBundle) buildManifest(ctx android.ModuleContext, provideNativeLibs, provideNativeLibs = android.SortedUniqueStrings(provideNativeLibs) requireNativeLibs = android.SortedUniqueStrings(android.RemoveListFromList(requireNativeLibs, provideNativeLibs)) - // APEX name can be overridden + // VNDK APEX name is determined at runtime, so update "name" in apex_manifest optCommands := []string{} - if a.properties.Apex_name != nil { - optCommands = append(optCommands, "-v name "+*a.properties.Apex_name) + if a.vndkApex { + apexName := vndkApexNamePrefix + a.vndkVersion(ctx.DeviceConfig()) + optCommands = append(optCommands, "-v name "+apexName) } // Collect jniLibs. Notice that a.filesInfo is already sorted @@ -445,7 +446,7 @@ func markManifestTestOnly(ctx android.ModuleContext, androidManifestFile android func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { apexType := a.properties.ApexType suffix := apexType.suffix() - apexName := proptools.StringDefault(a.properties.Apex_name, a.BaseModuleName()) + apexName := a.BaseModuleName() //////////////////////////////////////////////////////////////////////////////////////////// // Step 1: copy built files to appropriate directories under the image directory @@ -454,26 +455,13 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { installSymbolFiles := (!ctx.Config().KatiEnabled() || a.ExportedToMake()) && a.installable() - // b/140136207. When there are overriding APEXes for a VNDK APEX, the symbols file for the overridden - // APEX and the overriding APEX will have the same installation paths at /apex/com.android.vndk.v<ver> - // as their apexName will be the same. To avoid the path conflicts, skip installing the symbol files - // for the overriding VNDK APEXes. - if a.vndkApex && len(a.overridableProperties.Overrides) > 0 { - installSymbolFiles = false - } - - // Avoid creating duplicate build rules for multi-installed APEXes. - if proptools.BoolDefault(a.properties.Multi_install_skip_symbol_files, false) { - installSymbolFiles = false - - } // set of dependency module:location mappings installMapSet := make(map[string]bool) // TODO(jiyong): use the RuleBuilder var copyCommands []string var implicitInputs []android.Path - pathWhenActivated := android.PathForModuleInPartitionInstall(ctx, "apex", apexName) + apexDir := android.PathForModuleInPartitionInstall(ctx, "apex", apexName) for _, fi := range a.filesInfo { destPath := imageDir.Join(ctx, fi.path()).String() // Prepare the destination path @@ -503,12 +491,12 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { fmt.Sprintf("unzip -qDD -d %s %s", destPathDir, fi.module.(*java.AndroidAppSet).PackedAdditionalOutputs().String())) if installSymbolFiles { - installedPath = ctx.InstallFileWithExtraFilesZip(pathWhenActivated.Join(ctx, fi.installDir), + installedPath = ctx.InstallFileWithExtraFilesZip(apexDir.Join(ctx, fi.installDir), fi.stem(), fi.builtFile, fi.module.(*java.AndroidAppSet).PackedAdditionalOutputs()) } } else { if installSymbolFiles { - installedPath = ctx.InstallFile(pathWhenActivated.Join(ctx, fi.installDir), fi.stem(), fi.builtFile) + installedPath = ctx.InstallFile(apexDir.Join(ctx, fi.installDir), fi.stem(), fi.builtFile) } } implicitInputs = append(implicitInputs, fi.builtFile) @@ -522,7 +510,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { symlinkDest := imageDir.Join(ctx, symlinkPath).String() copyCommands = append(copyCommands, "ln -sfn "+filepath.Base(destPath)+" "+symlinkDest) if installSymbolFiles { - installedSymlink := ctx.InstallSymlink(pathWhenActivated.Join(ctx, filepath.Dir(symlinkPath)), filepath.Base(symlinkPath), installedPath) + installedSymlink := ctx.InstallSymlink(apexDir.Join(ctx, filepath.Dir(symlinkPath)), filepath.Base(symlinkPath), installedPath) implicitInputs = append(implicitInputs, installedSymlink) } } @@ -549,8 +537,8 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { } implicitInputs = append(implicitInputs, a.manifestPbOut) if installSymbolFiles { - installedManifest := ctx.InstallFile(pathWhenActivated, "apex_manifest.pb", a.manifestPbOut) - installedKey := ctx.InstallFile(pathWhenActivated, "apex_pubkey", a.publicKeyFile) + installedManifest := ctx.InstallFile(apexDir, "apex_manifest.pb", a.manifestPbOut) + installedKey := ctx.InstallFile(apexDir, "apex_pubkey", a.publicKeyFile) implicitInputs = append(implicitInputs, installedManifest, installedKey) } @@ -706,12 +694,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { optFlags = append(optFlags, "--unsigned_payload") } - if a.properties.Apex_name != nil { - // If apex_name is set, apexer can skip checking if key name matches with - // apex name. Note that apex_manifest is also mended. - optFlags = append(optFlags, "--do_not_check_keyname") - } - if moduleMinSdkVersion == android.SdkVersion_Android10 { implicitInputs = append(implicitInputs, a.manifestJsonOut) optFlags = append(optFlags, "--manifest_json "+a.manifestJsonOut.String()) @@ -1018,7 +1000,7 @@ func (a *apexBundle) getOverrideManifestPackageName(ctx android.ModuleContext) s if a.vndkApex { overrideName, overridden := ctx.DeviceConfig().OverrideManifestPackageNameFor(vndkApexName) if overridden { - return strings.Replace(*a.properties.Apex_name, vndkApexName, overrideName, 1) + return overrideName + ".v" + a.vndkVersion(ctx.DeviceConfig()) } return "" } diff --git a/apex/systemserver_classpath_fragment_test.go b/apex/systemserver_classpath_fragment_test.go index d03766448..c404a2e37 100644 --- a/apex/systemserver_classpath_fragment_test.go +++ b/apex/systemserver_classpath_fragment_test.go @@ -31,7 +31,7 @@ func TestSystemserverclasspathFragmentContents(t *testing.T) { result := android.GroupFixturePreparers( prepareForTestWithSystemserverclasspathFragment, prepareForTestWithMyapex, - dexpreopt.FixtureSetApexSystemServerJars("myapex:foo"), + dexpreopt.FixtureSetApexSystemServerJars("myapex:foo", "myapex:bar"), ).RunTestWithBp(t, ` apex { name: "myapex", @@ -57,10 +57,23 @@ func TestSystemserverclasspathFragmentContents(t *testing.T) { ], } + java_library { + name: "bar", + srcs: ["c.java"], + installable: true, + dex_preopt: { + profile: "bar-art-profile", + }, + apex_available: [ + "myapex", + ], + } + systemserverclasspath_fragment { name: "mysystemserverclasspathfragment", contents: [ "foo", + "bar", ], apex_available: [ "myapex", @@ -71,6 +84,8 @@ func TestSystemserverclasspathFragmentContents(t *testing.T) { ensureExactContents(t, result.TestContext, "myapex", "android_common_myapex_image", []string{ "etc/classpaths/systemserverclasspath.pb", "javalib/foo.jar", + "javalib/bar.jar", + "javalib/bar.jar.prof", }) java.CheckModuleDependencies(t, result.TestContext, "myapex", "android_common_myapex_image", []string{ @@ -236,7 +251,7 @@ func TestSystemserverclasspathFragmentStandaloneContents(t *testing.T) { result := android.GroupFixturePreparers( prepareForTestWithSystemserverclasspathFragment, prepareForTestWithMyapex, - dexpreopt.FixtureSetApexStandaloneSystemServerJars("myapex:foo"), + dexpreopt.FixtureSetApexStandaloneSystemServerJars("myapex:foo", "myapex:bar"), ).RunTestWithBp(t, ` apex { name: "myapex", @@ -262,10 +277,23 @@ func TestSystemserverclasspathFragmentStandaloneContents(t *testing.T) { ], } + java_library { + name: "bar", + srcs: ["c.java"], + dex_preopt: { + profile: "bar-art-profile", + }, + installable: true, + apex_available: [ + "myapex", + ], + } + systemserverclasspath_fragment { name: "mysystemserverclasspathfragment", standalone_contents: [ "foo", + "bar", ], apex_available: [ "myapex", @@ -276,6 +304,8 @@ func TestSystemserverclasspathFragmentStandaloneContents(t *testing.T) { ensureExactContents(t, result.TestContext, "myapex", "android_common_myapex_image", []string{ "etc/classpaths/systemserverclasspath.pb", "javalib/foo.jar", + "javalib/bar.jar", + "javalib/bar.jar.prof", }) } diff --git a/apex/vndk.go b/apex/vndk.go index 80560cf0c..c0be753d1 100644 --- a/apex/vndk.go +++ b/apex/vndk.go @@ -65,10 +65,6 @@ func apexVndkMutator(mctx android.TopDownMutatorContext) { } vndkVersion := ab.vndkVersion(mctx.DeviceConfig()) - - // Ensure VNDK APEX mount point is formatted as com.android.vndk.v### - ab.properties.Apex_name = proptools.StringPtr(vndkApexNamePrefix + vndkVersion) - apiLevel, err := android.ApiLevelFromUser(mctx, vndkVersion) if err != nil { mctx.PropertyErrorf("vndk_version", "%s", err.Error()) diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index 86b9b27e2..062eba88a 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -45,8 +45,12 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { bp2buildFiles := CreateBazelFiles(ctx.Config(), nil, res.buildFileToTargets, ctx.mode) writeFiles(ctx, bp2buildDir, bp2buildFiles) - soongInjectionDir := android.PathForOutput(ctx, bazel.SoongInjectionDirName) - writeFiles(ctx, soongInjectionDir, CreateSoongInjectionDirFiles(ctx, res.metrics)) + injectionFiles, err := CreateSoongInjectionDirFiles(ctx, res.metrics) + if err != nil { + fmt.Printf("%s\n", err.Error()) + os.Exit(1) + } + writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles) return &res.metrics } @@ -55,17 +59,20 @@ func Codegen(ctx *CodegenContext) *CodegenMetrics { // This includes // 1. config value(s) that are hardcoded in Soong // 2. product_config variables -func CreateSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) []BazelFile { +func CreateSoongInjectionDirFiles(ctx *CodegenContext, metrics CodegenMetrics) ([]BazelFile, error) { var ret []BazelFile productConfigFiles, err := CreateProductConfigFiles(ctx) if err != nil { - fmt.Printf("ERROR: %s", err.Error()) - os.Exit(1) + return nil, err } ret = append(ret, productConfigFiles...) - ret = append(ret, soongInjectionFiles(ctx.Config(), metrics)...) - return ret + injectionFiles, err := soongInjectionFiles(ctx.Config(), metrics) + if err != nil { + return nil, err + } + ret = append(ret, injectionFiles...) + return ret, nil } // Get the output directory and create it if it doesn't exist. diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 5b3e19fa2..73df67555 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -22,7 +22,7 @@ type BazelFile struct { } // PRIVATE: Use CreateSoongInjectionDirFiles instead -func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []BazelFile { +func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) ([]BazelFile, error) { var files []BazelFile files = append(files, newFile("android", GeneratedBuildFileName, "")) // Creates a //cc_toolchain package. @@ -36,7 +36,11 @@ func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []BazelFile files = append(files, newFile("java_toolchain", "constants.bzl", java_config.BazelJavaToolchainVars(cfg))) files = append(files, newFile("apex_toolchain", GeneratedBuildFileName, "")) // Creates a //apex_toolchain package. - files = append(files, newFile("apex_toolchain", "constants.bzl", apex.BazelApexToolchainVars())) + apexToolchainVars, err := apex.BazelApexToolchainVars() + if err != nil { + return nil, err + } + files = append(files, newFile("apex_toolchain", "constants.bzl", apexToolchainVars)) files = append(files, newFile("metrics", "converted_modules.txt", strings.Join(metrics.Serialize().ConvertedModules, "\n"))) @@ -52,7 +56,7 @@ func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []BazelFile apiLevelsContent, err := json.Marshal(android.GetApiLevelsMap(cfg)) if err != nil { - panic(err) + return nil, err } files = append(files, newFile("api_levels", GeneratedBuildFileName, `exports_files(["api_levels.json"])`)) files = append(files, newFile("api_levels", "api_levels.json", string(apiLevelsContent))) @@ -64,7 +68,7 @@ func soongInjectionFiles(cfg android.Config, metrics CodegenMetrics) []BazelFile files = append(files, newFile("allowlists", "mixed_build_prod_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelProdMode), "\n")+"\n")) files = append(files, newFile("allowlists", "mixed_build_staging_allowlist.txt", strings.Join(android.GetBazelEnabledModules(android.BazelStagingMode), "\n")+"\n")) - return files + return files, nil } func CreateBazelFiles( diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index 8c240934c..8c1d2ae5b 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -84,8 +84,10 @@ func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) { func TestCreateBazelFiles_Bp2Build_CreatesDefaultFiles(t *testing.T) { testConfig := android.TestConfig("", make(map[string]string), "", make(map[string][]byte)) - files := soongInjectionFiles(testConfig, CreateCodegenMetrics()) - + files, err := soongInjectionFiles(testConfig, CreateCodegenMetrics()) + if err != nil { + t.Error(err) + } expectedFilePaths := []bazelFilepath{ { dir: "android", diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go index 37188f196..aac5e7d2c 100644 --- a/bp2build/symlink_forest.go +++ b/bp2build/symlink_forest.go @@ -15,9 +15,7 @@ package bp2build import ( - "errors" "fmt" - "io/fs" "io/ioutil" "os" "path/filepath" @@ -27,6 +25,7 @@ import ( "sync/atomic" "android/soong/shared" + "github.com/google/blueprint/pathtools" ) // A tree structure that describes what to do at each directory in the created @@ -53,59 +52,6 @@ type symlinkForestContext struct { symlinkCount atomic.Uint64 } -// A simple thread pool to limit concurrency on system calls. -// Necessary because Go spawns a new OS-level thread for each blocking system -// call. This means that if syscalls are too slow and there are too many of -// them, the hard limit on OS-level threads can be exhausted. -type syscallPool struct { - shutdownCh []chan<- struct{} - workCh chan syscall -} - -type syscall struct { - work func() - done chan<- struct{} -} - -func createSyscallPool(count int) *syscallPool { - result := &syscallPool{ - shutdownCh: make([]chan<- struct{}, count), - workCh: make(chan syscall), - } - - for i := 0; i < count; i++ { - shutdownCh := make(chan struct{}) - result.shutdownCh[i] = shutdownCh - go result.worker(shutdownCh) - } - - return result -} - -func (p *syscallPool) do(work func()) { - doneCh := make(chan struct{}) - p.workCh <- syscall{work, doneCh} - <-doneCh -} - -func (p *syscallPool) shutdown() { - for _, ch := range p.shutdownCh { - ch <- struct{}{} // Blocks until the value is received - } -} - -func (p *syscallPool) worker(shutdownCh <-chan struct{}) { - for { - select { - case <-shutdownCh: - return - case work := <-p.workCh: - work.work() - work.done <- struct{}{} - } - } -} - // Ensures that the node for the given path exists in the tree and returns it. func ensureNodeExists(root *instructionsNode, path string) *instructionsNode { if path == "" { @@ -171,25 +117,13 @@ func mergeBuildFiles(output string, srcBuildFile string, generatedBuildFile stri generatedBuildFileContent = packageDefaultVisibilityRegex.ReplaceAll(generatedBuildFileContent, []byte{}) } - outFile, err := os.Create(output) - if err != nil { - return err - } - - _, err = outFile.Write(generatedBuildFileContent) - if err != nil { - return err - } - - if generatedBuildFileContent[len(generatedBuildFileContent)-1] != '\n' { - _, err = outFile.WriteString("\n") - if err != nil { - return err - } + newContents := generatedBuildFileContent + if newContents[len(newContents)-1] != '\n' { + newContents = append(newContents, '\n') } + newContents = append(newContents, srcBuildFileContent...) - _, err = outFile.Write(srcBuildFileContent) - return err + return pathtools.WriteFileIfChanged(output, newContents, 0666) } // Calls readdir() and returns it as a map from the basename of the files in dir @@ -217,12 +151,35 @@ func readdirToMap(dir string) map[string]os.FileInfo { } // Creates a symbolic link at dst pointing to src -func symlinkIntoForest(topdir, dst, src string) { - err := os.Symlink(shared.JoinPath(topdir, src), shared.JoinPath(topdir, dst)) - if err != nil { +func symlinkIntoForest(topdir, dst, src string) uint64 { + srcPath := shared.JoinPath(topdir, src) + dstPath := shared.JoinPath(topdir, dst) + + // Check if a symlink already exists. + if dstInfo, err := os.Lstat(dstPath); err != nil { + if !os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "Failed to lstat '%s': %s", dst, err) + os.Exit(1) + } + } else { + if dstInfo.Mode()&os.ModeSymlink != 0 { + // Assume that the link's target is correct, i.e. no manual tampering. + // E.g. OUT_DIR could have been previously used with a different source tree check-out! + return 0 + } else { + if err := os.RemoveAll(dstPath); err != nil { + fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", dst, err) + os.Exit(1) + } + } + } + + // Create symlink. + if err := os.Symlink(srcPath, dstPath); err != nil { fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err) os.Exit(1) } + return 1 } func isDir(path string, fi os.FileInfo) bool { @@ -253,8 +210,9 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in defer context.wg.Done() if instructions != nil && instructions.excluded { - // This directory is not needed, bail out - return + // Excluded paths are skipped at the level of the non-excluded parent. + fmt.Fprintf(os.Stderr, "may not specify a root-level exclude directory '%s'", srcDir) + os.Exit(1) } // We don't add buildFilesDir here because the bp2build files marker files is @@ -272,6 +230,12 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in renamingBuildFile = true srcDirMap["BUILD.bazel"] = srcDirMap["BUILD"] delete(srcDirMap, "BUILD") + if instructions != nil { + if _, ok := instructions.children["BUILD"]; ok { + instructions.children["BUILD.bazel"] = instructions.children["BUILD"] + delete(instructions.children, "BUILD") + } + } } } } @@ -288,17 +252,41 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in // Tests read the error messages generated, so ensure their order is deterministic sort.Strings(allEntries) - err := os.MkdirAll(shared.JoinPath(context.topdir, forestDir), 0777) - if err != nil { - fmt.Fprintf(os.Stderr, "Cannot mkdir '%s': %s\n", forestDir, err) - os.Exit(1) + fullForestPath := shared.JoinPath(context.topdir, forestDir) + createForestDir := false + if fi, err := os.Lstat(fullForestPath); err != nil { + if os.IsNotExist(err) { + createForestDir = true + } else { + fmt.Fprintf(os.Stderr, "Could not read info for '%s': %s\n", forestDir, err) + } + } else if fi.Mode()&os.ModeDir == 0 { + if err := os.RemoveAll(fullForestPath); err != nil { + fmt.Fprintf(os.Stderr, "Failed to remove '%s': %s", forestDir, err) + os.Exit(1) + } + createForestDir = true + } + if createForestDir { + if err := os.MkdirAll(fullForestPath, 0777); err != nil { + fmt.Fprintf(os.Stderr, "Could not mkdir '%s': %s\n", forestDir, err) + os.Exit(1) + } + context.mkdirCount.Add(1) } - context.mkdirCount.Add(1) + + // Start with a list of items that already exist in the forest, and remove + // each element as it is processed in allEntries. Any remaining items in + // forestMapForDeletion must be removed. (This handles files which were + // removed since the previous forest generation). + forestMapForDeletion := readdirToMap(shared.JoinPath(context.topdir, forestDir)) for _, f := range allEntries { if f[0] == '.' { continue // Ignore dotfiles } + delete(forestMapForDeletion, f) + // todo add deletionCount metric // The full paths of children in the input trees and in the output tree forestChild := shared.JoinPath(forestDir, f) @@ -309,13 +297,9 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in buildFilesChild := shared.JoinPath(buildFilesDir, f) // Descend in the instruction tree if it exists - var instructionsChild *instructionsNode = nil + var instructionsChild *instructionsNode if instructions != nil { - if f == "BUILD.bazel" && renamingBuildFile { - instructionsChild = instructions.children["BUILD"] - } else { - instructionsChild = instructions.children[f] - } + instructionsChild = instructions.children[f] } srcChildEntry, sExists := srcDirMap[f] @@ -323,8 +307,7 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in if instructionsChild != nil && instructionsChild.excluded { if bExists { - symlinkIntoForest(context.topdir, forestChild, buildFilesChild) - context.symlinkCount.Add(1) + context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild)) } continue } @@ -340,8 +323,7 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) } else { // Not in the source tree, symlink BUILD file - symlinkIntoForest(context.topdir, forestChild, buildFilesChild) - context.symlinkCount.Add(1) + context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, buildFilesChild)) } } else if !bExists { if sDir && instructionsChild != nil { @@ -351,8 +333,7 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in go plantSymlinkForestRecursive(context, instructionsChild, forestChild, buildFilesChild, srcChild) } else { // Not in the build file tree, symlink source tree, carry on - symlinkIntoForest(context.topdir, forestChild, srcChild) - context.symlinkCount.Add(1) + context.symlinkCount.Add(symlinkIntoForest(context.topdir, forestChild, srcChild)) } } else if sDir && bDir { // Both are directories. Descend. @@ -365,8 +346,7 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in // The Android.bp file that codegen used to produce `buildFilesChild` is // already a dependency, we can ignore `buildFilesChild`. context.depCh <- srcChild - err = mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose) - if err != nil { + if err := mergeBuildFiles(shared.JoinPath(context.topdir, forestChild), srcBuildFile, generatedBuildFile, context.verbose); err != nil { fmt.Fprintf(os.Stderr, "Error merging %s and %s: %s", srcBuildFile, generatedBuildFile, err) os.Exit(1) @@ -379,51 +359,27 @@ func plantSymlinkForestRecursive(context *symlinkForestContext, instructions *in os.Exit(1) } } -} -func removeParallelRecursive(pool *syscallPool, path string, fi os.FileInfo, wg *sync.WaitGroup) { - defer wg.Done() - - if fi.IsDir() { - children := readdirToMap(path) - childrenWg := &sync.WaitGroup{} - childrenWg.Add(len(children)) - - for child, childFi := range children { - go removeParallelRecursive(pool, shared.JoinPath(path, child), childFi, childrenWg) + // Remove all files in the forest that exist in neither the source + // tree nor the build files tree. (This handles files which were removed + // since the previous forest generation). + for f := range forestMapForDeletion { + var instructionsChild *instructionsNode + if instructions != nil { + instructionsChild = instructions.children[f] } - childrenWg.Wait() - } - - pool.do(func() { - if err := os.Remove(path); err != nil { - fmt.Fprintf(os.Stderr, "Cannot unlink '%s': %s\n", path, err) - os.Exit(1) + if instructionsChild != nil && instructionsChild.excluded { + // This directory may be excluded because bazel writes to it under the + // forest root. Thus this path is intentionally left alone. + continue } - }) -} - -func removeParallel(path string) { - fi, err := os.Lstat(path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return + forestChild := shared.JoinPath(context.topdir, forestDir, f) + if err := os.RemoveAll(forestChild); err != nil { + fmt.Fprintf(os.Stderr, "Failed to remove '%s/%s': %s", forestDir, f, err) + os.Exit(1) } - - fmt.Fprintf(os.Stderr, "Cannot lstat '%s': %s\n", path, err) - os.Exit(1) } - - wg := &sync.WaitGroup{} - wg.Add(1) - - // Random guess as to the best number of syscalls to run in parallel - pool := createSyscallPool(100) - removeParallelRecursive(pool, path, fi, wg) - pool.shutdown() - - wg.Wait() } // PlantSymlinkForest Creates a symlink forest by merging the directory tree at "buildFiles" and @@ -439,8 +395,6 @@ func PlantSymlinkForest(verbose bool, topdir string, forest string, buildFiles s symlinkCount: atomic.Uint64{}, } - removeParallel(shared.JoinPath(topdir, forest)) - instructions := instructionsFromExcludePathList(exclude) go func() { context.wg.Add(1) @@ -1066,6 +1066,31 @@ func (c *Module) CcLibraryInterface() bool { return false } +func (c *Module) IsFuzzModule() bool { + if _, ok := c.compiler.(*fuzzBinary); ok { + return true + } + return false +} + +func (c *Module) FuzzModuleStruct() fuzz.FuzzModule { + return c.FuzzModule +} + +func (c *Module) FuzzPackagedModule() fuzz.FuzzPackagedModule { + if fuzzer, ok := c.compiler.(*fuzzBinary); ok { + return fuzzer.fuzzPackagedModule + } + panic(fmt.Errorf("FuzzPackagedModule called on non-fuzz module: %q", c.BaseModuleName())) +} + +func (c *Module) FuzzSharedLibraries() android.Paths { + if fuzzer, ok := c.compiler.(*fuzzBinary); ok { + return fuzzer.sharedLibraries + } + panic(fmt.Errorf("FuzzSharedLibraries called on non-fuzz module: %q", c.BaseModuleName())) +} + func (c *Module) NonCcVariants() bool { return false } diff --git a/cc/config/global.go b/cc/config/global.go index d557c0b01..454a4dbdf 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -207,10 +207,7 @@ var ( "-Werror=fortify-source", "-Werror=address-of-temporary", - // Bug: http://b/29823425 Disable -Wnull-dereference until the - // new cases detected by this warning in Clang r271374 are - // fixed. - //"-Werror=null-dereference", + "-Werror=null-dereference", "-Werror=return-type", // http://b/72331526 Disable -Wtautological-* until the instances detected by these @@ -279,7 +276,6 @@ var ( // http://b/145211477 "-Wno-pointer-compare", - // http://b/145211022 "-Wno-final-dtor-non-final-class", // http://b/165945989 @@ -293,6 +289,9 @@ var ( // http://b/239661264 "-Wno-deprecated-non-prototype", + + // http://b/191699019 + "-Wno-format-insufficient-args", } llvmNextExtraCommonGlobalCflags = []string{ @@ -432,7 +431,7 @@ func init() { pctx.StaticVariable("ClangBin", "${ClangPath}/bin") pctx.StaticVariableWithEnvOverride("ClangShortVersion", "LLVM_RELEASE_VERSION", ClangDefaultShortVersion) - pctx.StaticVariable("ClangAsanLibDir", "${ClangBase}/linux-x86/${ClangVersion}/lib64/clang/${ClangShortVersion}/lib/linux") + pctx.StaticVariable("ClangAsanLibDir", "${ClangBase}/linux-x86/${ClangVersion}/lib/clang/${ClangShortVersion}/lib/linux") // These are tied to the version of LLVM directly in external/llvm, so they might trail the host prebuilts // being used for the rest of the build process. diff --git a/cc/fuzz.go b/cc/fuzz.go index 7113d87df..7aa8b91c7 100644 --- a/cc/fuzz.go +++ b/cc/fuzz.go @@ -212,7 +212,7 @@ func IsValidSharedDependency(dependency android.Module) bool { return true } -func sharedLibraryInstallLocation( +func SharedLibraryInstallLocation( libraryPath android.Path, isHost bool, fuzzDir string, archString string) string { installLocation := "$(PRODUCT_OUT)/data" if isHost { @@ -224,7 +224,7 @@ func sharedLibraryInstallLocation( } // Get the device-only shared library symbols install directory. -func sharedLibrarySymbolsInstallLocation(libraryPath android.Path, fuzzDir string, archString string) string { +func SharedLibrarySymbolsInstallLocation(libraryPath android.Path, fuzzDir string, archString string) string { return filepath.Join("$(PRODUCT_OUT)/symbols/data/", fuzzDir, archString, "/lib/", libraryPath.Base()) } @@ -237,57 +237,62 @@ func (fuzzBin *fuzzBinary) install(ctx ModuleContext, file android.Path) { installBase, ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) fuzzBin.binaryDecorator.baseInstaller.install(ctx, file) - fuzzBin.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzzBin.fuzzPackagedModule.FuzzProperties.Corpus) + fuzzBin.fuzzPackagedModule = PackageFuzzModule(ctx, fuzzBin.fuzzPackagedModule, pctx) + + // Grab the list of required shared libraries. + fuzzBin.sharedLibraries, _ = CollectAllSharedDependencies(ctx) + + for _, lib := range fuzzBin.sharedLibraries { + fuzzBin.installedSharedDeps = append(fuzzBin.installedSharedDeps, + SharedLibraryInstallLocation( + lib, ctx.Host(), installBase, ctx.Arch().ArchType.String())) + + // Also add the dependency on the shared library symbols dir. + if !ctx.Host() { + fuzzBin.installedSharedDeps = append(fuzzBin.installedSharedDeps, + SharedLibrarySymbolsInstallLocation(lib, installBase, ctx.Arch().ArchType.String())) + } + } +} + +func PackageFuzzModule(ctx android.ModuleContext, fuzzPackagedModule fuzz.FuzzPackagedModule, pctx android.PackageContext) fuzz.FuzzPackagedModule { + fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzzPackagedModule.FuzzProperties.Corpus) builder := android.NewRuleBuilder(pctx, ctx) intermediateDir := android.PathForModuleOut(ctx, "corpus") - for _, entry := range fuzzBin.fuzzPackagedModule.Corpus { + for _, entry := range fuzzPackagedModule.Corpus { builder.Command().Text("cp"). Input(entry). Output(intermediateDir.Join(ctx, entry.Base())) } builder.Build("copy_corpus", "copy corpus") - fuzzBin.fuzzPackagedModule.CorpusIntermediateDir = intermediateDir + fuzzPackagedModule.CorpusIntermediateDir = intermediateDir - fuzzBin.fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzzBin.fuzzPackagedModule.FuzzProperties.Data) + fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzzPackagedModule.FuzzProperties.Data) builder = android.NewRuleBuilder(pctx, ctx) intermediateDir = android.PathForModuleOut(ctx, "data") - for _, entry := range fuzzBin.fuzzPackagedModule.Data { + for _, entry := range fuzzPackagedModule.Data { builder.Command().Text("cp"). Input(entry). Output(intermediateDir.Join(ctx, entry.Rel())) } builder.Build("copy_data", "copy data") - fuzzBin.fuzzPackagedModule.DataIntermediateDir = intermediateDir + fuzzPackagedModule.DataIntermediateDir = intermediateDir - if fuzzBin.fuzzPackagedModule.FuzzProperties.Dictionary != nil { - fuzzBin.fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzzBin.fuzzPackagedModule.FuzzProperties.Dictionary) - if fuzzBin.fuzzPackagedModule.Dictionary.Ext() != ".dict" { + if fuzzPackagedModule.FuzzProperties.Dictionary != nil { + fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzzPackagedModule.FuzzProperties.Dictionary) + if fuzzPackagedModule.Dictionary.Ext() != ".dict" { ctx.PropertyErrorf("dictionary", "Fuzzer dictionary %q does not have '.dict' extension", - fuzzBin.fuzzPackagedModule.Dictionary.String()) + fuzzPackagedModule.Dictionary.String()) } } - if fuzzBin.fuzzPackagedModule.FuzzProperties.Fuzz_config != nil { + if fuzzPackagedModule.FuzzProperties.Fuzz_config != nil { configPath := android.PathForModuleOut(ctx, "config").Join(ctx, "config.json") - android.WriteFileRule(ctx, configPath, fuzzBin.fuzzPackagedModule.FuzzProperties.Fuzz_config.String()) - fuzzBin.fuzzPackagedModule.Config = configPath - } - - // Grab the list of required shared libraries. - fuzzBin.sharedLibraries, _ = CollectAllSharedDependencies(ctx) - - for _, lib := range fuzzBin.sharedLibraries { - fuzzBin.installedSharedDeps = append(fuzzBin.installedSharedDeps, - sharedLibraryInstallLocation( - lib, ctx.Host(), installBase, ctx.Arch().ArchType.String())) - - // Also add the dependency on the shared library symbols dir. - if !ctx.Host() { - fuzzBin.installedSharedDeps = append(fuzzBin.installedSharedDeps, - sharedLibrarySymbolsInstallLocation(lib, installBase, ctx.Arch().ArchType.String())) - } + android.WriteFileRule(ctx, configPath, fuzzPackagedModule.FuzzProperties.Fuzz_config.String()) + fuzzPackagedModule.Config = configPath } + return fuzzPackagedModule } func NewFuzzer(hod android.HostOrDeviceSupported) *Module { @@ -344,7 +349,7 @@ func NewFuzzer(hod android.HostOrDeviceSupported) *Module { // Responsible for generating GNU Make rules that package fuzz targets into // their architecture & target/host specific zip file. -type ccFuzzPackager struct { +type ccRustFuzzPackager struct { fuzz.FuzzPackager fuzzPackagingArchModules string fuzzTargetSharedDepsInstallPairs string @@ -353,7 +358,7 @@ type ccFuzzPackager struct { func fuzzPackagingFactory() android.Singleton { - fuzzPackager := &ccFuzzPackager{ + fuzzPackager := &ccRustFuzzPackager{ fuzzPackagingArchModules: "SOONG_FUZZ_PACKAGING_ARCH_MODULES", fuzzTargetSharedDepsInstallPairs: "FUZZ_TARGET_SHARED_DEPS_INSTALL_PAIRS", allFuzzTargetsName: "ALL_FUZZ_TARGETS", @@ -361,7 +366,7 @@ func fuzzPackagingFactory() android.Singleton { return fuzzPackager } -func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { +func (s *ccRustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { // Map between each architecture + host/device combination, and the files that // need to be packaged (in the tuple of {source file, destination folder in // archive}). @@ -376,19 +381,18 @@ func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { sharedLibraryInstalled := make(map[string]bool) ctx.VisitAllModules(func(module android.Module) { - ccModule, ok := module.(*Module) - if !ok || ccModule.Properties.PreventInstall { + ccModule, ok := module.(LinkableInterface) + if !ok || ccModule.PreventInstall() { return } // Discard non-fuzz targets. - if ok := fuzz.IsValid(ccModule.FuzzModule); !ok { + if ok := fuzz.IsValid(ccModule.FuzzModuleStruct()); !ok { return } sharedLibsInstallDirPrefix := "lib" - fuzzModule, ok := ccModule.compiler.(*fuzzBinary) - if !ok { + if !ccModule.IsFuzzModule() { return } @@ -399,12 +403,12 @@ func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { fpm := fuzz.FuzzPackagedModule{} if ok { - fpm = fuzzModule.fuzzPackagedModule + fpm = ccModule.FuzzPackagedModule() } intermediatePath := "fuzz" - archString := ccModule.Arch().ArchType.String() + archString := ccModule.Target().Arch.ArchType.String() archDir := android.PathForIntermediates(ctx, intermediatePath, hostOrTargetString, archString) archOs := fuzz.ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()} @@ -415,7 +419,7 @@ func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { files = s.PackageArtifacts(ctx, module, fpm, archDir, builder) // Package shared libraries - files = append(files, GetSharedLibsToZip(fuzzModule.sharedLibraries, ccModule, &s.FuzzPackager, archString, sharedLibsInstallDirPrefix, &sharedLibraryInstalled)...) + files = append(files, GetSharedLibsToZip(ccModule.FuzzSharedLibraries(), ccModule, &s.FuzzPackager, archString, sharedLibsInstallDirPrefix, &sharedLibraryInstalled)...) // The executable. files = append(files, fuzz.FileToZip{android.OutputFileForModule(ctx, ccModule, "unstripped"), ""}) @@ -429,7 +433,7 @@ func (s *ccFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { s.CreateFuzzPackage(ctx, archDirs, fuzz.Cc, pctx) } -func (s *ccFuzzPackager) MakeVars(ctx android.MakeVarsContext) { +func (s *ccRustFuzzPackager) MakeVars(ctx android.MakeVarsContext) { packages := s.Packages.Strings() sort.Strings(packages) sort.Strings(s.FuzzPackager.SharedLibInstallStrings) @@ -460,7 +464,7 @@ func GetSharedLibsToZip(sharedLibraries android.Paths, module LinkableInterface, // For each architecture-specific shared library dependency, we need to // install it to the output directory. Setup the install destination here, // which will be used by $(copy-many-files) in the Make backend. - installDestination := sharedLibraryInstallLocation( + installDestination := SharedLibraryInstallLocation( library, module.Host(), fuzzDir, archString) if (*sharedLibraryInstalled)[installDestination] { continue @@ -479,7 +483,7 @@ func GetSharedLibsToZip(sharedLibraries android.Paths, module LinkableInterface, // we want symbolization tools (like `stack`) to be able to find the symbols // in $ANDROID_PRODUCT_OUT/symbols automagically. if !module.Host() { - symbolsInstallDestination := sharedLibrarySymbolsInstallLocation(library, fuzzDir, archString) + symbolsInstallDestination := SharedLibrarySymbolsInstallLocation(library, fuzzDir, archString) symbolsInstallDestination = strings.ReplaceAll(symbolsInstallDestination, "$", "$$") s.SharedLibInstallStrings = append(s.SharedLibInstallStrings, library.String()+":"+symbolsInstallDestination) diff --git a/cc/linkable.go b/cc/linkable.go index 0522fc6fc..9578807f8 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -3,6 +3,7 @@ package cc import ( "android/soong/android" "android/soong/bazel/cquery" + "android/soong/fuzz" "android/soong/snapshot" "github.com/google/blueprint" @@ -120,6 +121,17 @@ type LinkableInterface interface { IsPrebuilt() bool Toc() android.OptionalPath + // IsFuzzModule returns true if this a *_fuzz module. + IsFuzzModule() bool + + // FuzzPackagedModule returns the fuzz.FuzzPackagedModule for this module. + // Expects that IsFuzzModule returns true. + FuzzPackagedModule() fuzz.FuzzPackagedModule + + // FuzzSharedLibraries returns the shared library dependencies for this module. + // Expects that IsFuzzModule returns true. + FuzzSharedLibraries() android.Paths + Device() bool Host() bool @@ -256,6 +268,9 @@ type LinkableInterface interface { // Partition returns the partition string for this module. Partition() string + + // FuzzModule returns the fuzz.FuzzModule associated with the module. + FuzzModuleStruct() fuzz.FuzzModule } var ( diff --git a/cc/lto_test.go b/cc/lto_test.go index fbd91be46..cee5aa3bc 100644 --- a/cc/lto_test.go +++ b/cc/lto_test.go @@ -15,10 +15,11 @@ package cc import ( - "android/soong/android" "strings" "testing" + "android/soong/android" + "github.com/google/blueprint" ) @@ -177,3 +178,34 @@ func TestThinLtoOnlyOnStaticDep(t *testing.T) { t.Errorf("'baz' expected to have flags %q, but got %q", w, libFooCFlags) } } + +func TestLtoDisabledButEnabledForArch(t *testing.T) { + t.Parallel() + bp := ` + cc_library { + name: "libfoo", + srcs: ["foo.c"], + lto: { + never: true, + }, + target: { + android_arm: { + lto: { + never: false, + thin: true, + }, + }, + }, + }` + result := android.GroupFixturePreparers( + prepareForCcTest, + ).RunTestWithBp(t, bp) + + libFooWithLto := result.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("ld") + libFooWithoutLto := result.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Rule("ld") + + android.AssertStringDoesContain(t, "missing flag for LTO in variant that expects it", + libFooWithLto.Args["ldFlags"], "-flto=thin") + android.AssertStringDoesNotContain(t, "got flag for LTO in variant that doesn't expect it", + libFooWithoutLto.Args["ldFlags"], "-flto=thin") +} @@ -38,9 +38,9 @@ func deduplicateStlInput(stl string) string { func getNdkStlFamilyAndLinkType(m LinkableInterface) (string, string) { stl := m.SelectedStl() switch stl { - case "ndk_libc++_shared": + case "ndk_libc++_shared", "libc++": return "libc++", "shared" - case "ndk_libc++_static": + case "ndk_libc++_static", "libc++_static": return "libc++", "static" case "ndk_system": return "system", "shared" @@ -80,7 +80,8 @@ func (stl *stl) begin(ctx BaseModuleContext) { return "" } s = deduplicateStlInput(s) - if ctx.useSdk() && ctx.Device() { + archHasNDKStl := ctx.Arch().ArchType != android.Riscv64 + if ctx.useSdk() && ctx.Device() && archHasNDKStl { switch s { case "", "system": return "ndk_system" diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 29a6f9552..5f27fa783 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -178,7 +178,8 @@ func runApiBp2build(ctx *android.Context, extraNinjaDeps []string) string { ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) // Create soong_injection repository - soongInjectionFiles := bp2build.CreateSoongInjectionDirFiles(codegenContext, bp2build.CreateCodegenMetrics()) + soongInjectionFiles, err := bp2build.CreateSoongInjectionDirFiles(codegenContext, bp2build.CreateCodegenMetrics()) + maybeQuit(err, "") absoluteSoongInjectionDir := shared.JoinPath(topDir, ctx.Config().SoongOutDir(), bazel.SoongInjectionDirName) for _, file := range soongInjectionFiles { // The API targets in api_bp2build workspace do not have any dependency on api_bp2build. @@ -447,6 +448,7 @@ func bazelArtifacts() []string { "bazel-genfiles", "bazel-out", "bazel-testlogs", + "bazel-workspace", "bazel-" + filepath.Base(topDir), } } diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 661bd5d2e..fd718c29f 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -218,12 +218,12 @@ func main() { trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace")) - c.run(buildCtx, config, args) - - defer met.Dump(soongMetricsFile) if !config.SkipMetricsUpload() { defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, bazelProfileFile, bazelMetricsFile, metricsFiles...) } + defer met.Dump(soongMetricsFile) + + c.run(buildCtx, config, args) } diff --git a/compliance/OWNERS b/compliance/OWNERS new file mode 100644 index 000000000..f52e201c1 --- /dev/null +++ b/compliance/OWNERS @@ -0,0 +1,8 @@ +# OSEP Build +bbadour@google.com +kanouche@google.com +napier@google.com + +# Open Source Compliance Tools +rtp@google.com +austinyuan@google.com diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index e3404a541..a590c72a5 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -101,6 +101,10 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC } func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *ModuleConfig) bool { + if ctx.Config().UnbundledBuild() { + return true + } + if contains(global.DisablePreoptModules, module.Name) { return true } diff --git a/java/app.go b/java/app.go index e7e52d471..1731970b3 100755 --- a/java/app.go +++ b/java/app.go @@ -984,8 +984,11 @@ type appTestProperties struct { // The name of the android_app module that the tests will run against. Instrumentation_for *string - // if specified, the instrumentation target package name in the manifest is overwritten by it. + // If specified, the instrumentation target package name in the manifest is overwritten by it. Instrumentation_target_package *string + + // If specified, the mainline module package name in the test config is overwritten by it. + Mainline_package_name *string } type AndroidTest struct { @@ -1063,6 +1066,11 @@ func (a *AndroidTest) FixTestConfig(ctx android.ModuleContext, testConfig androi FlagWithArg("--package-name ", *a.overridableAppProperties.Package_name) } + if a.appTestProperties.Mainline_package_name != nil { + fixNeeded = true + command.FlagWithArg("--mainline-package-name ", *a.appTestProperties.Mainline_package_name) + } + if fixNeeded { rule.Build("fix_test_config", "fix test config") return fixedConfig @@ -1524,7 +1532,6 @@ func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { appAttrs.javaCommonAttributes = commonAttrs appAttrs.bazelAapt = aapt appAttrs.Deps = deps - ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: a.Name()}, appAttrs) } else { ktName := a.Name() + "_kt" commonAttrs.Common_srcs = bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrc(ctx, a.properties.Common_srcs)) @@ -1545,11 +1552,12 @@ func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { appAttrs.bazelAapt = &bazelAapt{Manifest: aapt.Manifest} appAttrs.Deps = bazel.MakeSingleLabelListAttribute(bazel.Label{Label: ":" + ktName}) - ctx.CreateBazelTargetModule( - props, - android.CommonAttributes{Name: a.Name()}, - appAttrs, - ) } + ctx.CreateBazelTargetModule( + props, + android.CommonAttributes{Name: a.Name()}, + appAttrs, + ) + } diff --git a/java/app_test.go b/java/app_test.go index 3fb67c188..c77f29d23 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -2330,12 +2330,14 @@ func TestAndroidTest_FixTestConfig(t *testing.T) { srcs: ["b.java"], package_name: "com.android.bar.test", instrumentation_for: "foo", + mainline_package_name: "com.android.bar", } override_android_test { name: "baz_test", base: "foo_test", package_name: "com.android.baz.test", + mainline_package_name: "com.android.baz", } `) @@ -2354,6 +2356,7 @@ func TestAndroidTest_FixTestConfig(t *testing.T) { expectedFlags: []string{ "--manifest out/soong/.intermediates/bar_test/android_common/manifest_fixer/AndroidManifest.xml", "--package-name com.android.bar.test", + "--mainline-package-name com.android.bar", }, }, { @@ -2363,6 +2366,8 @@ func TestAndroidTest_FixTestConfig(t *testing.T) { "--manifest out/soong/.intermediates/foo_test/android_common_baz_test/manifest_fixer/AndroidManifest.xml", "--package-name com.android.baz.test", "--test-file-name baz_test.apk", + "out/soong/.intermediates/foo_test/android_common_baz_test/test_config_fixer/AndroidTest.xml", + "--mainline-package-name com.android.baz", }, }, } diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 77cbe9cf4..c4b0af441 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -27,6 +27,7 @@ type DexpreopterInterface interface { dexpreoptDisabled(ctx android.BaseModuleContext) bool DexpreoptBuiltInstalledForApex() []dexpreopterInstall AndroidMkEntriesForApex() []android.AndroidMkEntries + ProfilePathOnHost() android.Path } type dexpreopterInstall struct { @@ -103,6 +104,9 @@ type dexpreopter struct { // - Dexpreopt post-processing (using dexpreopt artifacts from a prebuilt system image to incrementally // dexpreopt another partition). configPath android.WritablePath + + // The path to the profile on host. + profilePathOnHost android.Path } type DexpreoptProperties struct { @@ -180,9 +184,8 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) if isApexVariant(ctx) { - // Don't preopt APEX variant module unless the module is an APEX system server jar and we are - // building the entire system image. - if !isApexSystemServerJar || ctx.Config().UnbundledBuild() { + // Don't preopt APEX variant module unless the module is an APEX system server jar. + if !isApexSystemServerJar { return true } } else { @@ -368,21 +371,29 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr installBase := filepath.Base(install.To) arch := filepath.Base(installDir) installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) + isProfile := strings.HasSuffix(installBase, ".prof") + + if isProfile { + d.profilePathOnHost = install.From + } if isApexSystemServerJar { - // APEX variants of java libraries are hidden from Make, so their dexpreopt - // outputs need special handling. Currently, for APEX variants of java - // libraries, only those in the system server classpath are handled here. - // Preopting of boot classpath jars in the ART APEX are handled in - // java/dexpreopt_bootjars.go, and other APEX jars are not preopted. - // The installs will be handled by Make as sub-modules of the java library. - d.builtInstalledForApex = append(d.builtInstalledForApex, dexpreopterInstall{ - name: arch + "-" + installBase, - moduleName: moduleName(ctx), - outputPathOnHost: install.From, - installDirOnDevice: installPath, - installFileOnDevice: installBase, - }) + // Profiles are handled separately because they are installed into the APEX. + if !isProfile { + // APEX variants of java libraries are hidden from Make, so their dexpreopt + // outputs need special handling. Currently, for APEX variants of java + // libraries, only those in the system server classpath are handled here. + // Preopting of boot classpath jars in the ART APEX are handled in + // java/dexpreopt_bootjars.go, and other APEX jars are not preopted. + // The installs will be handled by Make as sub-modules of the java library. + d.builtInstalledForApex = append(d.builtInstalledForApex, dexpreopterInstall{ + name: arch + "-" + installBase, + moduleName: moduleName(ctx), + outputPathOnHost: install.From, + installDirOnDevice: installPath, + installFileOnDevice: installBase, + }) + } } else if !d.preventInstall { ctx.InstallFile(installPath, installBase, install.From) } @@ -404,3 +415,7 @@ func (d *dexpreopter) AndroidMkEntriesForApex() []android.AndroidMkEntries { } return entries } + +func (d *dexpreopter) ProfilePathOnHost() android.Path { + return d.profilePathOnHost +} diff --git a/java/droidstubs.go b/java/droidstubs.go index d9613e536..8a521aabb 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -878,11 +878,13 @@ func (d *Droidstubs) createApiContribution(ctx android.DefaultableHookContext) { Name *string Api_surface *string Api_file *string + Visibility []string }{} props.Name = proptools.StringPtr(d.Name() + ".api.contribution") props.Api_surface = api_surface props.Api_file = api_file + props.Visibility = []string{"//visibility:override", "//visibility:public"} ctx.CreateModule(ApiContributionFactory, &props) } diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go index 6c2293746..7a04d7326 100644 --- a/java/droidstubs_test.go +++ b/java/droidstubs_test.go @@ -370,3 +370,36 @@ func TestDroidStubsApiContributionGeneration(t *testing.T) { ctx.ModuleForTests("foo.api.contribution", "") } + +func TestGeneratedApiContributionVisibilityTest(t *testing.T) { + library_bp := ` + java_api_library { + name: "bar", + api_surface: "public", + api_contributions: ["foo.api.contribution"], + } + ` + ctx, _ := testJavaWithFS(t, ` + droidstubs { + name: "foo", + srcs: ["A/a.java"], + api_surface: "public", + check_api: { + current: { + api_file: "A/current.txt", + removed_api_file: "A/removed.txt", + } + }, + visibility: ["//a"], + } + `, + map[string][]byte{ + "a/a.java": nil, + "a/current.txt": nil, + "a/removed.txt": nil, + "b/Android.bp": []byte(library_bp), + }, + ) + + ctx.ModuleForTests("bar", "android_common") +} diff --git a/java/java.go b/java/java.go index 659f98a7c..874f93576 100644 --- a/java/java.go +++ b/java/java.go @@ -517,14 +517,8 @@ func getJavaVersion(ctx android.ModuleContext, javaVersion string, sdkContext an return normalizeJavaVersion(ctx, javaVersion) } else if ctx.Device() { return defaultJavaLanguageVersion(ctx, sdkContext.SdkVersion(ctx)) - } else if ctx.Config().TargetsJava17() { - // Temporary experimental flag to be able to try and build with - // java version 17 options. The flag, if used, just sets Java - // 17 as the default version, leaving any components that - // target an older version intact. - return JAVA_VERSION_17 } else { - return JAVA_VERSION_11 + return JAVA_VERSION_17 } } diff --git a/java/sdk.go b/java/sdk.go index b0da5afba..10ae3f6e8 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -57,14 +57,10 @@ func defaultJavaLanguageVersion(ctx android.EarlyModuleContext, s android.SdkSpe return JAVA_VERSION_8 } else if sdk.FinalOrFutureInt() <= 31 { return JAVA_VERSION_9 - } else if ctx.Config().TargetsJava17() { - // Temporary experimental flag to be able to try and build with - // java version 17 options. The flag, if used, just sets Java - // 17 as the default version, leaving any components that - // target an older version intact. - return JAVA_VERSION_17 - } else { + } else if sdk.FinalOrFutureInt() <= 32 { return JAVA_VERSION_11 + } else { + return JAVA_VERSION_17 } } diff --git a/java/sdk_library.go b/java/sdk_library.go index b87236596..a2295f4a6 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1749,7 +1749,7 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC } } - mctx.CreateModule(DroidstubsFactory, &props) + mctx.CreateModule(DroidstubsFactory, &props).(*Droidstubs).CallHookIfAvailable(mctx) } func (module *SdkLibrary) compareAgainstLatestApi(apiScope *apiScope) bool { diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 210bfc3a3..1d0c13d4b 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -120,6 +120,7 @@ func TestJavaSdkLibrary(t *testing.T) { result.ModuleForTests(apiScopePublic.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopeSystem.stubsSourceModuleName("foo"), "android_common") result.ModuleForTests(apiScopeTest.stubsSourceModuleName("foo"), "android_common") + result.ModuleForTests(apiScopePublic.stubsSourceModuleName("foo")+".api.contribution", "") result.ModuleForTests("foo"+sdkXmlFileSuffix, "android_common") result.ModuleForTests("foo.api.public.28", "") result.ModuleForTests("foo.api.system.28", "") diff --git a/python/binary.go b/python/binary.go index 75135f345..95c751a82 100644 --- a/python/binary.go +++ b/python/binary.go @@ -125,6 +125,25 @@ func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { launcherPath = provider.IntermPathForModuleOut() } }) + + // TODO: get the list of shared libraries directly from the launcher module somehow + var sharedLibs []string + sharedLibs = append(sharedLibs, "libsqlite") + if ctx.Target().Os.Bionic() { + sharedLibs = append(sharedLibs, "libc", "libdl", "libm") + } + if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { + sharedLibs = append(sharedLibs, "libc_musl") + } + switch p.properties.Actual_version { + case pyVersion2: + sharedLibs = append(sharedLibs, "libc++") + case pyVersion3: + if ctx.Device() { + sharedLibs = append(sharedLibs, "liblog") + } + } + p.androidMkSharedLibs = sharedLibs } srcsZips := make(android.Paths, 0, len(depsSrcsZips)+1) if embeddedLauncher { @@ -136,14 +155,6 @@ func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, p.getHostInterpreterName(ctx, p.properties.Actual_version), main, p.getStem(ctx), srcsZips) - - var sharedLibs []string - // if embedded launcher is enabled, we need to collect the shared library dependencies of the - // launcher - for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { - sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) - } - p.androidMkSharedLibs = sharedLibs } func (p *PythonBinaryModule) AndroidMkEntries() []android.AndroidMkEntries { @@ -176,7 +187,7 @@ func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) { p.PythonLibraryModule.DepsMutator(ctx) if p.isEmbeddedLauncherEnabled() { - p.AddDepsOnPythonLauncherAndStdlib(ctx, pythonLibTag, launcherTag, launcherSharedLibTag, p.autorun(), ctx.Target()) + p.AddDepsOnPythonLauncherAndStdlib(ctx, pythonLibTag, launcherTag, p.autorun(), ctx.Target()) } } diff --git a/python/python.go b/python/python.go index 18e5b68dd..c23be8db8 100644 --- a/python/python.go +++ b/python/python.go @@ -226,20 +226,18 @@ var ( pythonLibTag = dependencyTag{name: "pythonLib"} javaDataTag = dependencyTag{name: "javaData"} // The python interpreter, with soong module name "py3-launcher" or "py3-launcher-autorun". - launcherTag = dependencyTag{name: "launcher"} - launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"} + launcherTag = dependencyTag{name: "launcher"} // The python interpreter built for host so that we can precompile python sources. // This only works because the precompiled sources don't vary by architecture. // The soong module name is "py3-launcher". - hostLauncherTag = dependencyTag{name: "hostLauncher"} - hostlauncherSharedLibTag = dependencyTag{name: "hostlauncherSharedLib"} - hostStdLibTag = dependencyTag{name: "hostStdLib"} - pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) - pyExt = ".py" - protoExt = ".proto" - pyVersion2 = "PY2" - pyVersion3 = "PY3" - internalPath = "internal" + hostLauncherTag = dependencyTag{name: "hostLauncher"} + hostStdLibTag = dependencyTag{name: "hostStdLib"} + pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) + pyExt = ".py" + protoExt = ".proto" + pyVersion2 = "PY2" + pyVersion3 = "PY3" + internalPath = "internal" ) type basePropertiesProvider interface { @@ -323,35 +321,21 @@ func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) { javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}} ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...) - p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, hostlauncherSharedLibTag, false, ctx.Config().BuildOSTarget) + p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, false, ctx.Config().BuildOSTarget) } -// AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib, -// launcher (interpreter), and the launcher's shared libraries. If autorun is true, it will use -// the autorun launcher instead of the regular one. This function acceps a targetForDeps argument -// as the target to use for these dependencies. For embedded launcher python binaries, the launcher -// that will be embedded will be under the same target as the python module itself. But when -// precompiling python code, we need to get the python launcher built for host, even if we're -// compiling the python module for device, so we pass a different target to this function. +// AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib +// and launcher (interpreter). If autorun is true, it will use the autorun launcher instead of the +// regular one. This function accepts a targetForDeps argument as the target to use for these +// dependencies. For embedded launcher python binaries, the launcher that will be embedded will be +// under the same target as the python module itself. But when precompiling python code, we need to +// get the python launcher built for host, even if we're compiling the python module for device, so +// we pass a different target to this function. func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.BottomUpMutatorContext, - stdLibTag, launcherTag, launcherSharedLibTag blueprint.DependencyTag, + stdLibTag, launcherTag blueprint.DependencyTag, autorun bool, targetForDeps android.Target) { var stdLib string var launcherModule string - // Add launcher shared lib dependencies. Ideally, these should be - // derived from the `shared_libs` property of the launcher. TODO: read these from - // the python launcher itself using ctx.OtherModuleProvider() or similar on the result - // of ctx.AddFarVariationDependencies() - launcherSharedLibDeps := []string{ - "libsqlite", - } - // Add launcher-specific dependencies for bionic - if targetForDeps.Os.Bionic() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") - } - if targetForDeps.Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") - } switch p.properties.Actual_version { case pyVersion2: @@ -362,7 +346,6 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto launcherModule = "py2-launcher-autorun" } - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") case pyVersion3: stdLib = "py3-stdlib" @@ -373,9 +356,6 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl { launcherModule += "-static" } - if ctx.Device() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") - } default: panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", p.properties.Actual_version, ctx.ModuleName())) @@ -391,7 +371,6 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib) } ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule) - ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...) } // GenerateAndroidBuildActions performs build actions common to all Python modules @@ -595,22 +574,19 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android. } }) } + var launcherSharedLibs android.Paths + var ldLibraryPath []string ctx.VisitDirectDepsWithTag(hostLauncherTag, func(module android.Module) { if dep, ok := module.(IntermPathProvider); ok { optionalLauncher := dep.IntermPathForModuleOut() if optionalLauncher.Valid() { launcher = optionalLauncher.Path() } - } - }) - var launcherSharedLibs android.Paths - var ldLibraryPath []string - ctx.VisitDirectDepsWithTag(hostlauncherSharedLibTag, func(module android.Module) { - if dep, ok := module.(IntermPathProvider); ok { - optionalPath := dep.IntermPathForModuleOut() - if optionalPath.Valid() { - launcherSharedLibs = append(launcherSharedLibs, optionalPath.Path()) - ldLibraryPath = append(ldLibraryPath, filepath.Dir(optionalPath.Path().String())) + for _, spec := range module.TransitivePackagingSpecs() { + if strings.HasSuffix(spec.SrcPath().String(), ".so") { + launcherSharedLibs = append(launcherSharedLibs, spec.SrcPath()) + ldLibraryPath = append(ldLibraryPath, filepath.Dir(spec.SrcPath().String())) + } } } }) diff --git a/rust/androidmk.go b/rust/androidmk.go index 32c746ed4..20e991967 100644 --- a/rust/androidmk.go +++ b/rust/androidmk.go @@ -205,8 +205,8 @@ func (compiler *baseCompiler) AndroidMk(ctx AndroidMkContext, ret *android.Andro }) } -func (fuzz *fuzzDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { - ctx.SubAndroidMk(entries, fuzz.binaryDecorator) +func (fuzz *fuzzDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkEntries) { + ctx.SubAndroidMk(ret, fuzz.binaryDecorator) var fuzzFiles []string for _, d := range fuzz.fuzzPackagedModule.Corpus { @@ -229,11 +229,14 @@ func (fuzz *fuzzDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *andro filepath.Dir(fuzz.fuzzPackagedModule.Config.String())+":config.json") } - entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, + ret.ExtraEntries = append(ret.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.SetBool("LOCAL_IS_FUZZ_TARGET", true) if len(fuzzFiles) > 0 { entries.AddStrings("LOCAL_TEST_DATA", fuzzFiles...) } + if fuzz.installedSharedDeps != nil { + entries.AddStrings("LOCAL_FUZZ_INSTALLED_SHARED_DEPS", fuzz.installedSharedDeps...) + } }) } diff --git a/rust/bindgen.go b/rust/bindgen.go index e81ec6bf6..878f896b6 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -52,7 +52,7 @@ var ( if ctx.Config().UseHostMusl() { return "musl/lib/" } else { - return "lib64/" + return "lib/" } }) _ = pctx.SourcePathVariable("bindgenClang", diff --git a/rust/config/global.go b/rust/config/global.go index 50ac1f72a..ef428b836 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.65.0.p1" + RustDefaultVersion = "1.66.1" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ diff --git a/rust/config/x86_linux_bionic_host.go b/rust/config/x86_linux_bionic_host.go index b1a2c178e..79c40ce41 100644 --- a/rust/config/x86_linux_bionic_host.go +++ b/rust/config/x86_linux_bionic_host.go @@ -21,7 +21,9 @@ import ( ) var ( - LinuxBionicRustFlags = []string{} + LinuxBionicRustFlags = []string{ + "-C panic=abort", + } LinuxBionicRustLinkFlags = []string{ "-B${cc_config.ClangBin}", "-fuse-ld=lld", diff --git a/rust/fuzz.go b/rust/fuzz.go index 6faf55cf6..d7e7ddfff 100644 --- a/rust/fuzz.go +++ b/rust/fuzz.go @@ -16,8 +16,6 @@ package rust import ( "path/filepath" - "sort" - "strings" "android/soong/android" "android/soong/cc" @@ -27,14 +25,14 @@ import ( func init() { android.RegisterModuleType("rust_fuzz", RustFuzzFactory) - android.RegisterSingletonType("rust_fuzz_packaging", rustFuzzPackagingFactory) } type fuzzDecorator struct { *binaryDecorator - fuzzPackagedModule fuzz.FuzzPackagedModule - sharedLibraries android.Paths + fuzzPackagedModule fuzz.FuzzPackagedModule + sharedLibraries android.Paths + installedSharedDeps []string } var _ compiler = (*fuzzDecorator)(nil) @@ -64,9 +62,14 @@ func (fuzzer *fuzzDecorator) compilerFlags(ctx ModuleContext, flags Flags) Flags flags = fuzzer.binaryDecorator.compilerFlags(ctx, flags) // `../lib` for installed fuzz targets (both host and device), and `./lib` for fuzz target packages. - flags.LinkFlags = append(flags.LinkFlags, `-Wl,-rpath,\$$ORIGIN/../lib`) flags.LinkFlags = append(flags.LinkFlags, `-Wl,-rpath,\$$ORIGIN/lib`) + if ctx.InstallInVendor() { + flags.LinkFlags = append(flags.LinkFlags, `-Wl,-rpath,\$$ORIGIN/../../lib`) + } else { + flags.LinkFlags = append(flags.LinkFlags, `-Wl,-rpath,\$$ORIGIN/../lib`) + + } return flags } @@ -88,10 +91,8 @@ func (fuzzer *fuzzDecorator) compilerProps() []interface{} { } func (fuzzer *fuzzDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { - out := fuzzer.binaryDecorator.compile(ctx, flags, deps) - // Grab the list of required shared libraries. - fuzzer.sharedLibraries, _ = cc.CollectAllSharedDependencies(ctx) + out := fuzzer.binaryDecorator.compile(ctx, flags, deps) return out } @@ -104,83 +105,6 @@ func (fuzzer *fuzzDecorator) autoDep(ctx android.BottomUpMutatorContext) autoDep return rlibAutoDep } -// Responsible for generating GNU Make rules that package fuzz targets into -// their architecture & target/host specific zip file. -type rustFuzzPackager struct { - fuzz.FuzzPackager -} - -func rustFuzzPackagingFactory() android.Singleton { - return &rustFuzzPackager{} -} - -func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { - // Map between each architecture + host/device combination. - archDirs := make(map[fuzz.ArchOs][]fuzz.FileToZip) - - // List of individual fuzz targets. - s.FuzzTargets = make(map[string]bool) - - // Map tracking whether each shared library has an install rule to avoid duplicate install rules from - // multiple fuzzers that depend on the same shared library. - sharedLibraryInstalled := make(map[string]bool) - - ctx.VisitAllModules(func(module android.Module) { - // Discard non-fuzz targets. - rustModule, ok := module.(*Module) - if !ok { - return - } - - if ok := fuzz.IsValid(rustModule.FuzzModule); !ok || rustModule.Properties.PreventInstall { - return - } - - fuzzModule, ok := rustModule.compiler.(*fuzzDecorator) - if !ok { - return - } - - hostOrTargetString := "target" - if rustModule.Host() { - hostOrTargetString = "host" - } - - archString := rustModule.Arch().ArchType.String() - archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) - archOs := fuzz.ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()} - - var files []fuzz.FileToZip - builder := android.NewRuleBuilder(pctx, ctx) - - // Package the artifacts (data, corpus, config and dictionary into a zipfile. - files = s.PackageArtifacts(ctx, module, fuzzModule.fuzzPackagedModule, archDir, builder) - - // The executable. - files = append(files, fuzz.FileToZip{rustModule.UnstrippedOutputFile(), ""}) - - // Package shared libraries - files = append(files, cc.GetSharedLibsToZip(fuzzModule.sharedLibraries, rustModule, &s.FuzzPackager, archString, "lib", &sharedLibraryInstalled)...) - - archDirs[archOs], ok = s.BuildZipFile(ctx, module, fuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs) - if !ok { - return - } - - }) - s.CreateFuzzPackage(ctx, archDirs, fuzz.Rust, pctx) -} - -func (s *rustFuzzPackager) MakeVars(ctx android.MakeVarsContext) { - packages := s.Packages.Strings() - sort.Strings(packages) - - ctx.Strict("SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) - - // Preallocate the slice of fuzz targets to minimize memory allocations. - s.PreallocateSlice(ctx, "ALL_RUST_FUZZ_TARGETS") -} - func (fuzz *fuzzDecorator) install(ctx ModuleContext) { fuzz.binaryDecorator.baseCompiler.dir = filepath.Join( "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) @@ -188,13 +112,22 @@ func (fuzz *fuzzDecorator) install(ctx ModuleContext) { "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) fuzz.binaryDecorator.baseCompiler.install(ctx) - if fuzz.fuzzPackagedModule.FuzzProperties.Corpus != nil { - fuzz.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Corpus) - } - if fuzz.fuzzPackagedModule.FuzzProperties.Data != nil { - fuzz.fuzzPackagedModule.Data = android.PathsForModuleSrc(ctx, fuzz.fuzzPackagedModule.FuzzProperties.Data) - } - if fuzz.fuzzPackagedModule.FuzzProperties.Dictionary != nil { - fuzz.fuzzPackagedModule.Dictionary = android.PathForModuleSrc(ctx, *fuzz.fuzzPackagedModule.FuzzProperties.Dictionary) + fuzz.fuzzPackagedModule = cc.PackageFuzzModule(ctx, fuzz.fuzzPackagedModule, pctx) + + installBase := "fuzz" + + // Grab the list of required shared libraries. + fuzz.sharedLibraries, _ = cc.CollectAllSharedDependencies(ctx) + + for _, lib := range fuzz.sharedLibraries { + fuzz.installedSharedDeps = append(fuzz.installedSharedDeps, + cc.SharedLibraryInstallLocation( + lib, ctx.Host(), installBase, ctx.Arch().ArchType.String())) + + // Also add the dependency on the shared library symbols dir. + if !ctx.Host() { + fuzz.installedSharedDeps = append(fuzz.installedSharedDeps, + cc.SharedLibrarySymbolsInstallLocation(lib, installBase, ctx.Arch().ArchType.String())) + } } } diff --git a/rust/rust.go b/rust/rust.go index 28a300bc6..67e0d7c03 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -208,6 +208,11 @@ func (mod *Module) OutputFiles(tag string) (android.Paths, error) { } return android.Paths{}, nil } + case "unstripped": + if mod.compiler != nil { + return android.PathsIfNonNil(mod.compiler.unstrippedOutputFilePath()), nil + } + return nil, nil default: return nil, fmt.Errorf("unsupported module reference tag %q", tag) } @@ -619,6 +624,31 @@ func (mod *Module) CcLibraryInterface() bool { return false } +func (mod *Module) IsFuzzModule() bool { + if _, ok := mod.compiler.(*fuzzDecorator); ok { + return true + } + return false +} + +func (mod *Module) FuzzModuleStruct() fuzz.FuzzModule { + return mod.FuzzModule +} + +func (mod *Module) FuzzPackagedModule() fuzz.FuzzPackagedModule { + if fuzzer, ok := mod.compiler.(*fuzzDecorator); ok { + return fuzzer.fuzzPackagedModule + } + panic(fmt.Errorf("FuzzPackagedModule called on non-fuzz module: %q", mod.BaseModuleName())) +} + +func (mod *Module) FuzzSharedLibraries() android.Paths { + if fuzzer, ok := mod.compiler.(*fuzzDecorator); ok { + return fuzzer.sharedLibraries + } + panic(fmt.Errorf("FuzzSharedLibraries called on non-fuzz module: %q", mod.BaseModuleName())) +} + func (mod *Module) UnstrippedOutputFile() android.Path { if mod.compiler != nil { return mod.compiler.unstrippedOutputFilePath() diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt index a02c19560..08bd80cab 100644 --- a/scripts/check_boot_jars/package_allowed_list.txt +++ b/scripts/check_boot_jars/package_allowed_list.txt @@ -75,6 +75,7 @@ jdk\.internal\.misc jdk\.internal\.ref jdk\.internal\.reflect jdk\.internal\.util +jdk\.internal\.util\.jar jdk\.internal\.vm\.annotation jdk\.net org\.w3c\.dom diff --git a/scripts/test_config_fixer.py b/scripts/test_config_fixer.py index 3dbc22ec2..07e01a14a 100644 --- a/scripts/test_config_fixer.py +++ b/scripts/test_config_fixer.py @@ -31,6 +31,8 @@ from manifest import write_xml KNOWN_PREPARERS = ['com.android.tradefed.targetprep.TestAppInstallSetup', 'com.android.tradefed.targetprep.suite.SuiteApkInstaller'] +MAINLINE_CONTROLLER = 'com.android.tradefed.testtype.suite.module.MainlineTestModuleController' + def parse_args(): """Parse commandline arguments.""" @@ -41,6 +43,8 @@ def parse_args(): help=('overwrite package fields in the test config')) parser.add_argument('--test-file-name', default='', dest='test_file_name', help=('overwrite test file name in the test config')) + parser.add_argument('--mainline-package-name', default='', dest='mainline_package_name', + help=('overwrite mainline module package name in the test config')) parser.add_argument('input', help='input test config file') parser.add_argument('output', help='output test config file') return parser.parse_args() @@ -72,6 +76,16 @@ def overwrite_test_file_name(test_config_doc, test_file_name): if option.getAttribute('name') == "test-file-name": option.setAttribute('value', test_file_name) +def overwrite_mainline_module_package_name(test_config_doc, mainline_package_name): + + test_config = parse_test_config(test_config_doc) + + for obj in get_children_with_tag(test_config, 'object'): + if obj.getAttribute('class') == MAINLINE_CONTROLLER: + for option in get_children_with_tag(obj, 'option'): + if option.getAttribute('name') == "mainline-module-package-name": + option.setAttribute('value', mainline_package_name) + def main(): """Program entry point.""" try: @@ -88,6 +102,9 @@ def main(): if args.test_file_name: overwrite_test_file_name(doc, args.test_file_name) + if args.mainline_package_name: + overwrite_mainline_module_package_name(doc, args.mainline_package_name) + with open(args.output, 'w') as f: write_xml(f, doc) diff --git a/scripts/test_config_fixer_test.py b/scripts/test_config_fixer_test.py index 39ce5b3c3..699f91eeb 100644 --- a/scripts/test_config_fixer_test.py +++ b/scripts/test_config_fixer_test.py @@ -23,6 +23,8 @@ from xml.dom import minidom import test_config_fixer +from manifest import write_xml + sys.dont_write_bytecode = True @@ -117,5 +119,39 @@ class OverwriteTestFileNameTest(unittest.TestCase): self.assertEqual(expected, output.getvalue()) +class OverwriteMainlineModulePackageNameTest(unittest.TestCase): + """ Unit tests for overwrite_mainline_module_package_name function """ + + test_config = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<configuration description="Runs some tests.">\n' + ' <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup">\n' + ' <option name="test-file-name" value="foo.apk"/>\n' + ' </target_preparer>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="com.android.foo"/>\n' + ' <option name="runtime-hint" value="20s"/>\n' + ' </test>\n' + ' <object type="module_controller" class="com.android.tradefed.testtype.suite.module.MainlineTestModuleController">\n' + ' <option name="enable" value="true"/>\n' + ' <option name="mainline-module-package-name" value="%s"/>\n' + ' </object>\n' + '</configuration>\n') + + def test_testappinstallsetup(self): + doc = minidom.parseString(self.test_config % ("com.android.old.package.name")) + + test_config_fixer.overwrite_mainline_module_package_name(doc, "com.android.new.package.name") + output = io.StringIO() + test_config_fixer.write_xml(output, doc) + + # Only the mainline module package name should be updated. Format the xml + # with minidom first to avoid mismatches due to trivial reformatting. + expected = io.StringIO() + write_xml(expected, minidom.parseString(self.test_config % ("com.android.new.package.name"))) + self.maxDiff = None + self.assertEqual(expected.getvalue(), output.getvalue()) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh index 6477dac6e..878b4a16f 100755 --- a/tests/bp2build_bazel_test.sh +++ b/tests/bp2build_bazel_test.sh @@ -140,7 +140,7 @@ EOF # NOTE: We don't actually use the extra BUILD file for anything here run_bazel build --config=android --config=bp2build --config=ci //foo/... - local the_answer_file="$(find -L bazel-out -name the_answer.txt)" + local -r the_answer_file="$(find -L bazel-out -name the_answer.txt)" if [[ ! -f "${the_answer_file}" ]]; then fail "Expected the_answer.txt to be generated, but was missing" fi @@ -156,6 +156,49 @@ function test_bp2build_generates_all_buildfiles { eval "${_save_trap}" } +function test_bp2build_symlinks_files { + setup + mkdir -p foo + touch foo/BLANK1 + touch foo/BLANK2 + touch foo/F2D + touch foo/BUILD + + run_soong bp2build + + if [[ -e "./out/soong/workspace/foo/BUILD" ]]; then + fail "./out/soong/workspace/foo/BUILD should be omitted" + fi + for file in BLANK1 BLANK2 F2D + do + if [[ ! -L "./out/soong/workspace/foo/$file" ]]; then + fail "./out/soong/workspace/foo/$file should exist" + fi + done + local -r BLANK1_BEFORE=$(stat -c %y "./out/soong/workspace/foo/BLANK1") + + rm foo/BLANK2 + rm foo/F2D + mkdir foo/F2D + touch foo/F2D/BUILD + + run_soong bp2build + + if [[ -e "./out/soong/workspace/foo/BUILD" ]]; then + fail "./out/soong/workspace/foo/BUILD should be omitted" + fi + local -r BLANK1_AFTER=$(stat -c %y "./out/soong/workspace/foo/BLANK1") + if [[ "$BLANK1_AFTER" != "$BLANK1_BEFORE" ]]; then + fail "./out/soong/workspace/foo/BLANK1 should be untouched" + fi + if [[ -e "./out/soong/workspace/foo/BLANK2" ]]; then + fail "./out/soong/workspace/foo/BLANK2 should be removed" + fi + if [[ -L "./out/soong/workspace/foo/F2D" ]] || [[ ! -d "./out/soong/workspace/foo/F2D" ]]; then + fail "./out/soong/workspace/foo/F2D should be a dir" + fi +} + function test_cc_correctness { setup |