diff options
140 files changed, 4169 insertions, 1246 deletions
@@ -188,7 +188,7 @@ a functionally equivalent module. Enter Soong namespaces. #### Namespaces -A presense of the `soong_namespace {..}` in an Android.bp file defines a +The presence of the `soong_namespace {..}` in an Android.bp file defines a **namespace**. For instance, having ``` @@ -550,6 +550,26 @@ logic receives module definitions parsed into Go structures using reflection and produces build rules. The build rules are collected by blueprint and written to a [ninja](http://ninja-build.org) build file. +## Environment Variables Config File + +Soong can optionally load environment variables from a pre-specified +configuration file during startup. These environment variables can be used +to control the behavior of the build. For example, these variables can determine +whether remote-execution should be used for the build or not. + +The `ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR` environment variable specifies the +directory in which the config file should be searched for. The +`ANDROID_BUILD_ENVIRONMENT_CONFIG` variable determines the name of the config +file to be searched for within the config directory. For example, the following +build comand will load `ENV_VAR_1` and `ENV_VAR_2` environment variables from +the `example_config.json` file inside the `build/soong` directory. + +``` +ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR=build/soong \ + ANDROID_BUILD_ENVIRONMENT_CONFIG=example_config \ + build/soong/soong_ui.bash +``` + ## Other documentation * [Best Practices](docs/best_practices.md) diff --git a/android/Android.bp b/android/Android.bp index da369592a..d3540b211 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -16,7 +16,9 @@ bootstrap_go_package { "soong-remoteexec", "soong-response", "soong-shared", + "soong-starlark-format", "soong-ui-metrics_proto", + "golang-protobuf-proto", "golang-protobuf-encoding-prototext", diff --git a/android/arch.go b/android/arch.go index 96a4cbf09..a719cf3f5 100644 --- a/android/arch.go +++ b/android/arch.go @@ -917,7 +917,8 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { for _, archType := range osArchTypeMap[os] { targets = append(targets, GetCompoundTargetField(os, archType)) - // Also add the special "linux_<arch>" and "bionic_<arch>" property structs. + // Also add the special "linux_<arch>", "bionic_<arch>" , "glibc_<arch>", and + // "musl_<arch>" property structs. if os.Linux() { target := "Linux_" + archType.Name if !InList(target, targets) { @@ -930,6 +931,18 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { targets = append(targets, target) } } + if os == Linux { + target := "Glibc_" + archType.Name + if !InList(target, targets) { + targets = append(targets, target) + } + } + if os == LinuxMusl { + target := "Musl_" + archType.Name + if !InList(target, targets) { + targets = append(targets, target) + } + } } } @@ -1379,11 +1392,25 @@ func getArchProperties(ctx BaseMutatorContext, archProperties interface{}, arch result = append(result, osArchProperties) } + if os == Linux { + field := "Glibc_" + archType.Name + userFriendlyField := "target.glibc_" + "_" + archType.Name + if osArchProperties, ok := getChildPropertyStruct(ctx, targetProp, field, userFriendlyField); ok { + result = append(result, osArchProperties) + } + } + if os == LinuxMusl { + field := "Musl_" + archType.Name + userFriendlyField := "target.musl_" + "_" + archType.Name + if osArchProperties, ok := getChildPropertyStruct(ctx, targetProp, field, userFriendlyField); ok { + result = append(result, osArchProperties) + } + // Special case: to ease the transition from glibc to musl, apply linux_glibc // properties (which has historically mean host linux) to musl variants. - field := "Linux_glibc_" + archType.Name - userFriendlyField := "target.linux_glibc_" + archType.Name + field = "Linux_glibc_" + archType.Name + userFriendlyField = "target.linux_glibc_" + archType.Name if osArchProperties, ok := getChildPropertyStruct(ctx, targetProp, field, userFriendlyField); ok { result = append(result, osArchProperties) } diff --git a/android/bazel.go b/android/bazel.go index 528c7b110..7e5736f58 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -217,7 +217,6 @@ var ( "external/bazelbuild-rules_android":/* recursive = */ true, "external/bazel-skylib":/* recursive = */ true, "external/guava":/* recursive = */ true, - "external/error_prone":/* recursive = */ true, "external/jsr305":/* recursive = */ true, "frameworks/ex/common":/* recursive = */ true, @@ -236,6 +235,7 @@ var ( // Configure modules in these directories to enable bp2build_available: true or false by default. bp2buildDefaultConfig = Bp2BuildConfig{ + "art/libartpalette": Bp2BuildDefaultTrueRecursively, "art/libdexfile": Bp2BuildDefaultTrueRecursively, "art/runtime": Bp2BuildDefaultTrueRecursively, "art/tools": Bp2BuildDefaultTrue, @@ -288,14 +288,21 @@ var ( "development/samples/WiFiDirectDemo": Bp2BuildDefaultTrue, "development/sdk": Bp2BuildDefaultTrueRecursively, "external/arm-optimized-routines": Bp2BuildDefaultTrueRecursively, + "external/auto/common": Bp2BuildDefaultTrueRecursively, + "external/auto/service": Bp2BuildDefaultTrueRecursively, "external/boringssl": Bp2BuildDefaultTrueRecursively, "external/bouncycastle": Bp2BuildDefaultTrue, "external/brotli": Bp2BuildDefaultTrue, "external/conscrypt": Bp2BuildDefaultTrue, + "external/error_prone": Bp2BuildDefaultTrueRecursively, "external/fmtlib": Bp2BuildDefaultTrueRecursively, "external/google-benchmark": Bp2BuildDefaultTrueRecursively, "external/googletest": Bp2BuildDefaultTrueRecursively, "external/gwp_asan": Bp2BuildDefaultTrueRecursively, + "external/icu": Bp2BuildDefaultTrueRecursively, + "external/icu/android_icu4j": Bp2BuildDefaultFalse, // java rules incomplete + "external/icu/icu4j": Bp2BuildDefaultFalse, // java rules incomplete + "external/javapoet": Bp2BuildDefaultTrueRecursively, "external/jemalloc_new": Bp2BuildDefaultTrueRecursively, "external/jsoncpp": Bp2BuildDefaultTrueRecursively, "external/libcap": Bp2BuildDefaultTrueRecursively, @@ -341,6 +348,7 @@ var ( "packages/screensavers/Basic": Bp2BuildDefaultTrue, "packages/services/Car/tests/SampleRearViewCamera": Bp2BuildDefaultTrue, "prebuilts/clang/host/linux-x86": Bp2BuildDefaultTrueRecursively, + "prebuilts/tools/common/m2": Bp2BuildDefaultTrue, "system/apex": Bp2BuildDefaultFalse, // TODO(b/207466993): flaky failures "system/apex/proto": Bp2BuildDefaultTrueRecursively, "system/apex/libs": Bp2BuildDefaultTrueRecursively, @@ -430,8 +438,22 @@ var ( "libprotobuf-internal-protos", // b/210751803, we don't handle path property for filegroups "libprotobuf-internal-python-srcs", // b/210751803, we don't handle path property for filegroups "libprotobuf-java-full", // b/210751803, we don't handle path property for filegroups + "host-libprotobuf-java-full", // b/210751803, we don't handle path property for filegroups "libprotobuf-java-util-full", // b/210751803, we don't handle path property for filegroups - "conscrypt", // b/210751803, we don't handle path property for filegroups + + "conscrypt", // b/210751803, we don't handle path property for filegroups + "conscrypt-for-host", // b/210751803, we don't handle path property for filegroups + + "host-libprotobuf-java-lite", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-micro", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-nano", // b/217236083, java_library cannot have deps without srcs + "error_prone_core", // b/217236083, java_library cannot have deps without srcs + "bouncycastle-host", // b/217236083, java_library cannot have deps without srcs + + "apex_manifest_proto_java", // b/215230097, we don't handle .proto files in java_library srcs attribute + + "libc_musl_sysroot_bionic_arch_headers", // b/218405924, depends on soong_zip + "libc_musl_sysroot_bionic_headers", // b/218405924, depends on soong_zip and generates duplicate srcs // python protos "libprotobuf-python", // contains .proto sources @@ -469,13 +491,30 @@ var ( "libdexfiled", // depends on unconverted modules: dexfile_operator_srcs, libartbased, libartpalette // go deps: - "apex-protos", // depends on soong_zip, a go binary - "robolectric_tzdata", // depends on soong_zip, a go binary - "host_bionic_linker_asm", // depends on extract_linker, a go binary. - "host_bionic_linker_script", // depends on extract_linker, a go binary. + "apex-protos", // depends on soong_zip, a go binary + "generated_android_icu4j_src_files", "generated_android_icu4j_test_files", "icu4c_test_data", // depends on unconverted modules: soong_zip + "host_bionic_linker_asm", // depends on extract_linker, a go binary. + "host_bionic_linker_script", // depends on extract_linker, a go binary. + "robolectric-sqlite4java-native", // depends on soong_zip, a go binary + "robolectric_tzdata", // depends on soong_zip, a go binary + + "android_icu4j_srcgen_binary", // Bazel build error: deps not allowed without srcs; move to runtime_deps + "core-icu4j-for-host", // Bazel build error: deps not allowed without srcs; move to runtime_deps // java deps - "bin2c_fastdeployagent", // depends on deployagent, a java binary + "android_icu4j_srcgen", // depends on unconverted modules: currysrc + "bin2c_fastdeployagent", // depends on deployagent, a java binary + "currysrc", // depends on unconverted modules: currysrc_org.eclipse, guavalib, jopt-simple-4.9 + "robolectric-sqlite4java-0.282", // depends on unconverted modules: robolectric-sqlite4java-import, robolectric-sqlite4java-native + "timezone-host", // depends on unconverted modules: art.module.api.annotations + "truth-host-prebuilt", // depends on unconverted modules: truth-prebuilt + "truth-prebuilt", // depends on unconverted modules: asm-7.0, guava + + "generated_android_icu4j_resources", // depends on unconverted modules: android_icu4j_srcgen_binary, soong_zip + "generated_android_icu4j_test_resources", // depends on unconverted modules: android_icu4j_srcgen_binary, soong_zip + + "art-script", // depends on unconverted modules: dalvikvm, dex2oat + "dex2oat-script", // depends on unconverted modules: dex2oat } // Per-module denylist of cc_library modules to only generate the static @@ -683,6 +722,7 @@ func GetMainClassInManifest(c Config, filepath string) (string, error) { if err != nil { return "", err } + defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() diff --git a/android/config.go b/android/config.go index 4472036da..877800a72 100644 --- a/android/config.go +++ b/android/config.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "runtime" "strconv" "strings" @@ -37,6 +38,7 @@ import ( "android/soong/android/soongconfig" "android/soong/bazel" "android/soong/remoteexec" + "android/soong/starlark_fmt" ) // Bool re-exports proptools.Bool for the android package. @@ -273,15 +275,43 @@ func saveToBazelConfigFile(config *productVariables, outDir string) error { return fmt.Errorf("Could not create dir %s: %s", dir, err) } - data, err := json.MarshalIndent(&config, "", " ") + nonArchVariantProductVariables := []string{} + archVariantProductVariables := []string{} + p := variableProperties{} + t := reflect.TypeOf(p.Product_variables) + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + nonArchVariantProductVariables = append(nonArchVariantProductVariables, strings.ToLower(f.Name)) + if proptools.HasTag(f, "android", "arch_variant") { + archVariantProductVariables = append(archVariantProductVariables, strings.ToLower(f.Name)) + } + } + + nonArchVariantProductVariablesJson := starlark_fmt.PrintStringList(nonArchVariantProductVariables, 0) + if err != nil { + return fmt.Errorf("cannot marshal product variable data: %s", err.Error()) + } + + archVariantProductVariablesJson := starlark_fmt.PrintStringList(archVariantProductVariables, 0) + if err != nil { + return fmt.Errorf("cannot marshal arch variant product variable data: %s", err.Error()) + } + + configJson, err := json.MarshalIndent(&config, "", " ") if err != nil { return fmt.Errorf("cannot marshal config data: %s", err.Error()) } bzl := []string{ bazel.GeneratedBazelFileWarning, - fmt.Sprintf(`_product_vars = json.decode("""%s""")`, data), - "product_vars = _product_vars\n", + fmt.Sprintf(`_product_vars = json.decode("""%s""")`, configJson), + fmt.Sprintf(`_product_var_constraints = %s`, nonArchVariantProductVariablesJson), + fmt.Sprintf(`_arch_variant_product_var_constraints = %s`, archVariantProductVariablesJson), + "\n", ` +product_vars = _product_vars +product_var_constraints = _product_var_constraints +arch_variant_product_var_constraints = _arch_variant_product_var_constraints +`, } err = ioutil.WriteFile(filepath.Join(dir, "product_variables.bzl"), []byte(strings.Join(bzl, "\n")), 0644) if err != nil { @@ -1952,3 +1982,8 @@ func (c *config) ApexBootJars() ConfiguredJarList { func (c *config) RBEWrapper() string { return c.GetenvWithDefault("RBE_WRAPPER", remoteexec.DefaultWrapperPath) } + +// UseHostMusl returns true if the host target has been configured to build against musl libc. +func (c *config) UseHostMusl() bool { + return Bool(c.productVariables.HostMusl) +} diff --git a/android/license.go b/android/license.go index 587cb36de..ebee05576 100644 --- a/android/license.go +++ b/android/license.go @@ -63,7 +63,7 @@ func (m *licenseModule) DepsMutator(ctx BottomUpMutatorContext) { func (m *licenseModule) GenerateAndroidBuildActions(ctx ModuleContext) { // license modules have no licenses, but license_kinds must refer to license_kind modules mergeStringProps(&m.base().commonProperties.Effective_licenses, ctx.ModuleName()) - mergePathProps(&m.base().commonProperties.Effective_license_text, PathsForModuleSrc(ctx, m.properties.License_text)...) + namePathProps(&m.base().commonProperties.Effective_license_text, m.properties.Package_name, PathsForModuleSrc(ctx, m.properties.License_text)...) for _, module := range ctx.GetDirectDepsWithTag(licenseKindTag) { if lk, ok := module.(*licenseKindModule); ok { mergeStringProps(&m.base().commonProperties.Effective_license_conditions, lk.properties.Conditions...) diff --git a/android/license_metadata.go b/android/license_metadata.go index 3bc53d6a4..6a5b0dafb 100644 --- a/android/license_metadata.go +++ b/android/license_metadata.go @@ -34,7 +34,7 @@ var ( }, "args") ) -func buildLicenseMetadata(ctx ModuleContext) { +func buildLicenseMetadata(ctx ModuleContext, licenseMetadataFile WritablePath) { base := ctx.Module().base() if !base.Enabled() { @@ -45,9 +45,18 @@ func buildLicenseMetadata(ctx ModuleContext) { return } + var outputFiles Paths + if outputFileProducer, ok := ctx.Module().(OutputFileProducer); ok { + outputFiles, _ = outputFileProducer.OutputFiles("") + outputFiles = PathsIfNonNil(outputFiles...) + } + + isContainer := isContainerFromFileExtensions(base.installFiles, outputFiles) + var allDepMetadataFiles Paths var allDepMetadataArgs []string var allDepOutputFiles Paths + var allDepMetadataDepSets []*PathsDepSet ctx.VisitDirectDepsBlueprint(func(bpdep blueprint.Module) { dep, _ := bpdep.(Module) @@ -61,6 +70,9 @@ func buildLicenseMetadata(ctx ModuleContext) { if ctx.OtherModuleHasProvider(dep, LicenseMetadataProvider) { info := ctx.OtherModuleProvider(dep, LicenseMetadataProvider).(*LicenseMetadataInfo) allDepMetadataFiles = append(allDepMetadataFiles, info.LicenseMetadataPath) + if isContainer || IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) { + allDepMetadataDepSets = append(allDepMetadataDepSets, info.LicenseMetadataDepSet) + } depAnnotations := licenseAnnotationsFromTag(ctx.OtherModuleDependencyTag(dep)) @@ -105,9 +117,16 @@ func buildLicenseMetadata(ctx ModuleContext) { args = append(args, JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(base.commonProperties.Effective_license_text.Strings()), "-n ")) - args = append(args, - JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(allDepMetadataArgs), "-d ")) - orderOnlyDeps = append(orderOnlyDeps, allDepMetadataFiles...) + if isContainer { + transitiveDeps := newPathsDepSet(nil, allDepMetadataDepSets).ToList() + args = append(args, + JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(transitiveDeps.Strings()), "-d ")) + orderOnlyDeps = append(orderOnlyDeps, transitiveDeps...) + } else { + args = append(args, + JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(allDepMetadataArgs), "-d ")) + orderOnlyDeps = append(orderOnlyDeps, allDepMetadataFiles...) + } args = append(args, JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(allDepOutputFiles.Strings()), "-s ")) @@ -117,12 +136,6 @@ func buildLicenseMetadata(ctx ModuleContext) { JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(base.licenseInstallMap), "-m ")) // Built files - var outputFiles Paths - if outputFileProducer, ok := ctx.Module().(OutputFileProducer); ok { - outputFiles, _ = outputFileProducer.OutputFiles("") - outputFiles = PathsIfNonNil(outputFiles...) - } - if len(outputFiles) > 0 { args = append(args, JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(outputFiles.Strings()), "-t ")) @@ -134,13 +147,10 @@ func buildLicenseMetadata(ctx ModuleContext) { args = append(args, JoinWithPrefix(proptools.NinjaAndShellEscapeListIncludingSpaces(base.installFiles.Strings()), "-i ")) - isContainer := isContainerFromFileExtensions(base.installFiles, outputFiles) if isContainer { args = append(args, "--is_container") } - licenseMetadataFile := PathForModuleOut(ctx, "meta_lic") - ctx.Build(pctx, BuildParams{ Rule: licenseMetadataRule, Output: licenseMetadataFile, @@ -152,7 +162,8 @@ func buildLicenseMetadata(ctx ModuleContext) { }) ctx.SetProvider(LicenseMetadataProvider, &LicenseMetadataInfo{ - LicenseMetadataPath: licenseMetadataFile, + LicenseMetadataPath: licenseMetadataFile, + LicenseMetadataDepSet: newPathsDepSet(Paths{licenseMetadataFile}, allDepMetadataDepSets), }) } @@ -179,7 +190,8 @@ var LicenseMetadataProvider = blueprint.NewProvider(&LicenseMetadataInfo{}) // LicenseMetadataInfo stores the license metadata path for a module. type LicenseMetadataInfo struct { - LicenseMetadataPath Path + LicenseMetadataPath Path + LicenseMetadataDepSet *PathsDepSet } // licenseAnnotationsFromTag returns the LicenseAnnotations for a tag (if any) converted into @@ -212,6 +224,9 @@ const ( // LicenseAnnotationSharedDependency should be returned by LicenseAnnotations implementations // of dependency tags when the usage of the dependency is dynamic, for example a shared library // linkage for native modules or as a classpath library for java modules. + // + // Dependency tags that need to always return LicenseAnnotationSharedDependency + // can embed LicenseAnnotationSharedDependencyTag to implement LicenseAnnotations. LicenseAnnotationSharedDependency LicenseAnnotation = "dynamic" // LicenseAnnotationToolchain should be returned by LicenseAnnotations implementations of @@ -222,6 +237,14 @@ const ( LicenseAnnotationToolchain LicenseAnnotation = "toolchain" ) +// LicenseAnnotationSharedDependencyTag can be embedded in a dependency tag to implement +// LicenseAnnotations that always returns LicenseAnnotationSharedDependency. +type LicenseAnnotationSharedDependencyTag struct{} + +func (LicenseAnnotationSharedDependencyTag) LicenseAnnotations() []LicenseAnnotation { + return []LicenseAnnotation{LicenseAnnotationSharedDependency} +} + // LicenseAnnotationToolchainDependencyTag can be embedded in a dependency tag to implement // LicenseAnnotations that always returns LicenseAnnotationToolchain. type LicenseAnnotationToolchainDependencyTag struct{} diff --git a/android/license_sdk_member.go b/android/license_sdk_member.go index 2ce921bb0..b17defea5 100644 --- a/android/license_sdk_member.go +++ b/android/license_sdk_member.go @@ -90,7 +90,10 @@ func (p *licenseSdkMemberProperties) PopulateFromVariant(_ SdkMemberContext, var // Populate the properties from the variant. l := variant.(*licenseModule) p.License_kinds = l.properties.License_kinds - p.License_text = l.base().commonProperties.Effective_license_text + p.License_text = make(Paths, 0, len(l.base().commonProperties.Effective_license_text)) + for _, np := range l.base().commonProperties.Effective_license_text { + p.License_text = append(p.License_text, np.Path) + } } func (p *licenseSdkMemberProperties) AddToPropertySet(ctx SdkMemberContext, propertySet BpPropertySet) { diff --git a/android/licenses.go b/android/licenses.go index e9e271bd2..b51a06b24 100644 --- a/android/licenses.go +++ b/android/licenses.go @@ -213,7 +213,7 @@ func licensesPropertyFlattener(ctx ModuleContext) { m.base().commonProperties.Effective_package_name = l.properties.Package_name } mergeStringProps(&m.base().commonProperties.Effective_licenses, module.base().commonProperties.Effective_licenses...) - mergePathProps(&m.base().commonProperties.Effective_license_text, module.base().commonProperties.Effective_license_text...) + mergeNamedPathProps(&m.base().commonProperties.Effective_license_text, module.base().commonProperties.Effective_license_text...) mergeStringProps(&m.base().commonProperties.Effective_license_kinds, module.base().commonProperties.Effective_license_kinds...) mergeStringProps(&m.base().commonProperties.Effective_license_conditions, module.base().commonProperties.Effective_license_conditions...) } else { @@ -239,10 +239,24 @@ func mergeStringProps(prop *[]string, values ...string) { *prop = SortedUniqueStrings(*prop) } -// Update a property Path array with a distinct union of its values and a list of new values. -func mergePathProps(prop *Paths, values ...Path) { +// Update a property NamedPath array with a distinct union of its values and a list of new values. +func namePathProps(prop *NamedPaths, name *string, values ...Path) { + if name == nil { + for _, value := range values { + *prop = append(*prop, NamedPath{value, ""}) + } + } else { + for _, value := range values { + *prop = append(*prop, NamedPath{value, *name}) + } + } + *prop = SortedUniqueNamedPaths(*prop) +} + +// Update a property NamedPath array with a distinct union of its values and a list of new values. +func mergeNamedPathProps(prop *NamedPaths, values ...NamedPath) { *prop = append(*prop, values...) - *prop = SortedUniquePaths(*prop) + *prop = SortedUniqueNamedPaths(*prop) } // Get the licenses property falling back to the package default. @@ -316,4 +330,7 @@ func init() { func licensesMakeVarsProvider(ctx MakeVarsContext) { ctx.Strict("BUILD_LICENSE_METADATA", ctx.Config().HostToolPath(ctx, "build_license_metadata").String()) + ctx.Strict("HTMLNOTICE", ctx.Config().HostToolPath(ctx, "htmlnotice").String()) + ctx.Strict("XMLNOTICE", ctx.Config().HostToolPath(ctx, "xmlnotice").String()) + ctx.Strict("TEXTNOTICE", ctx.Config().HostToolPath(ctx, "textnotice").String()) } diff --git a/android/licenses_test.go b/android/licenses_test.go index 70160faf2..8a81e1294 100644 --- a/android/licenses_test.go +++ b/android/licenses_test.go @@ -90,9 +90,9 @@ var licensesTests = []struct { "libother": []string{"shownotice"}, }, effectiveNotices: map[string][]string{ - "libexample1": []string{"top/LICENSE", "top/NOTICE"}, - "libnested": []string{"top/LICENSE", "top/NOTICE"}, - "libother": []string{"top/LICENSE", "top/NOTICE"}, + "libexample1": []string{"top/LICENSE:topDog", "top/NOTICE:topDog"}, + "libnested": []string{"top/LICENSE:topDog", "top/NOTICE:topDog"}, + "libother": []string{"top/LICENSE:topDog", "top/NOTICE:topDog"}, }, }, diff --git a/android/module.go b/android/module.go index 3c8c7770f..03d3f80bc 100644 --- a/android/module.go +++ b/android/module.go @@ -16,11 +16,13 @@ package android import ( "fmt" + "net/url" "os" "path" "path/filepath" "reflect" "regexp" + "sort" "strings" "text/scanner" @@ -267,7 +269,7 @@ type BaseModuleContext interface { // // The Modules passed to the visit function should not be retained outside of the visit function, they may be // invalidated by future mutators. - WalkDeps(visit func(Module, Module) bool) + WalkDeps(visit func(child, parent Module) bool) // WalkDepsBlueprint calls visit for each transitive dependency, traversing the dependency // tree in top down order. visit may be called multiple times for the same (child, parent) @@ -616,6 +618,53 @@ type Dist struct { Tag *string `android:"arch_variant"` } +// NamedPath associates a path with a name. e.g. a license text path with a package name +type NamedPath struct { + Path Path + Name string +} + +// String returns an escaped string representing the `NamedPath`. +func (p NamedPath) String() string { + if len(p.Name) > 0 { + return p.Path.String() + ":" + url.QueryEscape(p.Name) + } + return p.Path.String() +} + +// NamedPaths describes a list of paths each associated with a name. +type NamedPaths []NamedPath + +// Strings returns a list of escaped strings representing each `NamedPath` in the list. +func (l NamedPaths) Strings() []string { + result := make([]string, 0, len(l)) + for _, p := range l { + result = append(result, p.String()) + } + return result +} + +// SortedUniqueNamedPaths modifies `l` in place to return the sorted unique subset. +func SortedUniqueNamedPaths(l NamedPaths) NamedPaths { + if len(l) == 0 { + return l + } + sort.Slice(l, func(i, j int) bool { + return l[i].String() < l[j].String() + }) + k := 0 + for i := 1; i < len(l); i++ { + if l[i].String() == l[k].String() { + continue + } + k++ + if k < i { + l[k] = l[i] + } + } + return l[:k+1] +} + type nameProperties struct { // The name of the module. Must be unique across all modules. Name *string @@ -684,7 +733,7 @@ type commonProperties struct { // Override of module name when reporting licenses Effective_package_name *string `blueprint:"mutated"` // Notice files - Effective_license_text Paths `blueprint:"mutated"` + Effective_license_text NamedPaths `blueprint:"mutated"` // License names Effective_license_kinds []string `blueprint:"mutated"` // License conditions @@ -1165,6 +1214,11 @@ func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *topDownMutator productConfigEnabledLabels, nil, }) + moduleSupportsDevice := mod.commonProperties.HostOrDeviceSupported&deviceSupported == deviceSupported + if mod.commonProperties.HostOrDeviceSupported != NeitherHostNorDeviceSupported && !moduleSupportsDevice { + enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(false)) + } + platformEnabledAttribute, err := enabledProperty.ToLabelListAttribute( bazel.LabelList{[]bazel.Label{bazel.Label{Label: "@platforms//:incompatible"}}, nil}, bazel.LabelList{[]bazel.Label{}, nil}) @@ -1321,6 +1375,9 @@ type ModuleBase struct { // set of dependency module:location mappings used to populate the license metadata for // apex containers. licenseInstallMap []string + + // The path to the generated license metadata file for the module. + licenseMetadataFile WritablePath } // A struct containing all relevant information about a Bazel target converted via bp2build. @@ -1793,7 +1850,11 @@ func (m *ModuleBase) ExportedToMake() bool { } func (m *ModuleBase) EffectiveLicenseFiles() Paths { - return m.commonProperties.Effective_license_text + result := make(Paths, 0, len(m.commonProperties.Effective_license_text)) + for _, p := range m.commonProperties.Effective_license_text { + result = append(result, p.Path) + } + return result } // computeInstallDeps finds the installed paths of all dependencies that have a dependency @@ -2076,6 +2137,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) variables: make(map[string]string), } + m.licenseMetadataFile = PathForModuleOut(ctx, "meta_lic") + dependencyInstallFiles, dependencyPackagingSpecs := m.computeInstallDeps(ctx) // set m.installFilesDepSet to only the transitive dependencies to be used as the dependencies // of installed files of this module. It will be replaced by a depset including the installed @@ -2207,7 +2270,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.installFilesDepSet = newInstallPathsDepSet(m.installFiles, dependencyInstallFiles) m.packagingSpecsDepSet = newPackagingSpecsDepSet(m.packagingSpecs, dependencyPackagingSpecs) - buildLicenseMetadata(ctx) + buildLicenseMetadata(ctx, m.licenseMetadataFile) m.buildParams = ctx.buildParams m.ruleParams = ctx.ruleParams @@ -2620,7 +2683,7 @@ func (b *baseModuleContext) validateAndroidModule(module blueprint.Module, tag b } if aModule == nil { - b.ModuleErrorf("module %q not an android module", b.OtherModuleName(module)) + b.ModuleErrorf("module %q (%#v) not an android module", b.OtherModuleName(module), tag) return nil } @@ -2742,8 +2805,8 @@ func (b *baseModuleContext) VisitDirectDeps(visit func(Module)) { func (b *baseModuleContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { b.bp.VisitDirectDeps(func(module blueprint.Module) { - if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { - if b.bp.OtherModuleDependencyTag(aModule) == tag { + if b.bp.OtherModuleDependencyTag(module) == tag { + if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { visit(aModule) } } diff --git a/android/module_test.go b/android/module_test.go index c35e66ed6..a1bab6d01 100644 --- a/android/module_test.go +++ b/android/module_test.go @@ -816,3 +816,120 @@ test { }) } } + +func TestSortedUniqueNamedPaths(t *testing.T) { + type np struct { + path, name string + } + makePaths := func(l []np) NamedPaths { + result := make(NamedPaths, 0, len(l)) + for _, p := range l { + result = append(result, NamedPath{PathForTesting(p.path), p.name}) + } + return result + } + + tests := []struct { + name string + in []np + expectedOut []np + }{ + { + name: "empty", + in: []np{}, + expectedOut: []np{}, + }, + { + name: "all_same", + in: []np{ + {"a.txt", "A"}, + {"a.txt", "A"}, + {"a.txt", "A"}, + {"a.txt", "A"}, + {"a.txt", "A"}, + }, + expectedOut: []np{ + {"a.txt", "A"}, + }, + }, + { + name: "same_path_different_names", + in: []np{ + {"a.txt", "C"}, + {"a.txt", "A"}, + {"a.txt", "D"}, + {"a.txt", "B"}, + {"a.txt", "E"}, + }, + expectedOut: []np{ + {"a.txt", "A"}, + {"a.txt", "B"}, + {"a.txt", "C"}, + {"a.txt", "D"}, + {"a.txt", "E"}, + }, + }, + { + name: "different_paths_same_name", + in: []np{ + {"b/b.txt", "A"}, + {"a/a.txt", "A"}, + {"a/txt", "A"}, + {"b", "A"}, + {"a/b/d", "A"}, + }, + expectedOut: []np{ + {"a/a.txt", "A"}, + {"a/b/d", "A"}, + {"a/txt", "A"}, + {"b/b.txt", "A"}, + {"b", "A"}, + }, + }, + { + name: "all_different", + in: []np{ + {"b/b.txt", "A"}, + {"a/a.txt", "B"}, + {"a/txt", "D"}, + {"b", "C"}, + {"a/b/d", "E"}, + }, + expectedOut: []np{ + {"a/a.txt", "B"}, + {"a/b/d", "E"}, + {"a/txt", "D"}, + {"b/b.txt", "A"}, + {"b", "C"}, + }, + }, + { + name: "some_different", + in: []np{ + {"b/b.txt", "A"}, + {"a/a.txt", "B"}, + {"a/txt", "D"}, + {"a/b/d", "E"}, + {"b", "C"}, + {"a/a.txt", "B"}, + {"a/b/d", "E"}, + }, + expectedOut: []np{ + {"a/a.txt", "B"}, + {"a/b/d", "E"}, + {"a/txt", "D"}, + {"b/b.txt", "A"}, + {"b", "C"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := SortedUniqueNamedPaths(makePaths(tt.in)) + expected := makePaths(tt.expectedOut) + t.Logf("actual: %v", actual) + t.Logf("expected: %v", expected) + AssertDeepEquals(t, "SortedUniqueNamedPaths ", expected, actual) + }) + } +} diff --git a/android/mutator.go b/android/mutator.go index fa6f2be8d..739e4ee6d 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -323,13 +323,13 @@ type BottomUpMutatorContext interface { // AddVariationDependencies adds deps as dependencies of the current module, but uses the variations // argument to select which variant of the dependency to use. It returns a slice of modules for // each dependency (some entries may be nil). A variant of the dependency must exist that matches - // the all of the non-local variations of the current module, plus the variations argument. + // all the non-local variations of the current module, plus the variations argument. // // If the mutator is parallel (see MutatorHandle.Parallel), this method will pause until the // new dependencies have had the current mutator called on them. If the mutator is not // parallel this method does not affect the ordering of the current mutator pass, but will // be ordered correctly for all future mutator passes. - AddVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string) []blueprint.Module + AddVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module // AddFarVariationDependencies adds deps as dependencies of the current module, but uses the // variations argument to select which variant of the dependency to use. It returns a slice of diff --git a/android/namespace.go b/android/namespace.go index 4f727e14e..fc7bc290f 100644 --- a/android/namespace.go +++ b/android/namespace.go @@ -74,7 +74,7 @@ func (s *sortedNamespaces) index(namespace *Namespace) int { // A NameResolver implements blueprint.NameInterface, and implements the logic to // find a module from namespaces based on a query string. -// A query string can be a module name or can be be "//namespace_path:module_path" +// A query string can be a module name or can be "//namespace_path:module_path" type NameResolver struct { rootNamespace *Namespace diff --git a/android/notices.go b/android/notices.go index d8cfaf2a1..194a734d3 100644 --- a/android/notices.go +++ b/android/notices.go @@ -16,6 +16,7 @@ package android import ( "path/filepath" + "strings" "github.com/google/blueprint" ) @@ -101,55 +102,15 @@ func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilena } } -// BuildNotices merges the supplied NOTICE files into a single file that lists notices -// for every key in noticeMap (which would normally be installed files). -func BuildNotices(ctx ModuleContext, noticeMap map[string]Paths) NoticeOutputs { - // TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass. - // - // generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules - // about input NOTICE file paths. - // 1. Their relative paths to the src root become their NOTICE index titles. We want to use - // on-device paths as titles, and so output the merged NOTICE file the corresponding location. - // 2. They must end with .txt extension. Otherwise, they're ignored. - - mergeTool := PathForSource(ctx, "build/soong/scripts/mergenotice.py") - generateNoticeTool := PathForSource(ctx, "build/soong/scripts/generate-notice-files.py") - - outputDir := PathForModuleOut(ctx, "notices") - builder := NewRuleBuilder(pctx, ctx). - Sbox(outputDir, PathForModuleOut(ctx, "notices.sbox.textproto")) - for _, installPath := range SortedStringKeys(noticeMap) { - noticePath := outputDir.Join(ctx, installPath+".txt") - // It would be nice if sbox created directories for temporaries, but until then - // this is simple enough. - builder.Command(). - Text("(cd").OutputDir().Text("&&"). - Text("mkdir -p").Text(filepath.Dir(installPath)).Text(")") - builder.Temporary(noticePath) - builder.Command(). - Tool(mergeTool). - Flag("--output").Output(noticePath). - Inputs(noticeMap[installPath]) - } - - // Transform the merged NOTICE file into a gzipped HTML file. - txtOutput := outputDir.Join(ctx, "NOTICE.txt") - htmlOutput := outputDir.Join(ctx, "NOTICE.html") - htmlGzOutput := outputDir.Join(ctx, "NOTICE.html.gz") - title := "\"Notices for " + ctx.ModuleName() + "\"" - builder.Command().Tool(generateNoticeTool). - FlagWithOutput("--text-output ", txtOutput). - FlagWithOutput("--html-output ", htmlOutput). - FlagWithArg("-t ", title). - Flag("-s").OutputDir() - builder.Command().BuiltTool("minigzip"). - FlagWithInput("-c ", htmlOutput). - FlagWithOutput("> ", htmlGzOutput) - builder.Build("build_notices", "generate notice output") - - return NoticeOutputs{ - TxtOutput: OptionalPathForPath(txtOutput), - HtmlOutput: OptionalPathForPath(htmlOutput), - HtmlGzOutput: OptionalPathForPath(htmlGzOutput), - } +// BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's +// generated license metadata file. +func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { + depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", ".")) + rule := NewRuleBuilder(pctx, ctx) + rule.Command(). + BuiltTool("textnotice"). + FlagWithOutput("-o ", outputFile). + FlagWithDepFile("-d ", depsFile). + Input(ctx.Module().base().licenseMetadataFile) + rule.Build("container_notice", "container notice file") } diff --git a/android/paths.go b/android/paths.go index 70e427b2a..4c69de706 100644 --- a/android/paths.go +++ b/android/paths.go @@ -2149,3 +2149,23 @@ func IsThirdPartyPath(path string) bool { } return false } + +// PathsDepSet is a thin type-safe wrapper around the generic depSet. It always uses +// topological order. +type PathsDepSet struct { + depSet +} + +// newPathsDepSet returns an immutable PathsDepSet with the given direct and +// transitive contents. +func newPathsDepSet(direct Paths, transitive []*PathsDepSet) *PathsDepSet { + return &PathsDepSet{*newDepSet(TOPOLOGICAL, direct, transitive)} +} + +// ToList returns the PathsDepSet flattened to a list in topological order. +func (d *PathsDepSet) ToList() Paths { + if d == nil { + return nil + } + return d.depSet.ToList().(Paths) +} diff --git a/android/register.go b/android/register.go index 1ac44402d..10e14e04d 100644 --- a/android/register.go +++ b/android/register.go @@ -31,7 +31,7 @@ import ( type sortableComponent interface { // componentName returns the name of the component. // - // Uniquely identifies the components within the set of components used at runtimr and during + // Uniquely identifies the components within the set of components used at runtime and during // tests. componentName() string diff --git a/android/rule_builder.go b/android/rule_builder.go index 1c6b1c086..098c1fcc4 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -470,7 +470,7 @@ var _ BuilderContext = SingletonContext(nil) func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderCommand { return r.Command(). - BuiltTool("dep_fixer"). + builtToolWithoutDeps("dep_fixer"). Inputs(depFiles.Paths()) } @@ -638,7 +638,7 @@ func (r *RuleBuilder) Build(name string, desc string) { } sboxCmd.Text("rm -rf").Output(r.outDir) sboxCmd.Text("&&") - sboxCmd.BuiltTool("sbox"). + sboxCmd.builtToolWithoutDeps("sbox"). Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(r.ctx).String())). Flag("--manifest").Input(r.sboxManifestPath) @@ -1040,6 +1040,19 @@ func (c *RuleBuilderCommand) ImplicitTools(paths Paths) *RuleBuilderCommand { // It is equivalent to: // cmd.Tool(ctx.Config().HostToolPath(ctx, tool)) func (c *RuleBuilderCommand) BuiltTool(tool string) *RuleBuilderCommand { + if c.rule.ctx.Config().UseHostMusl() { + // If the host is using musl, assume that the tool was built against musl libc and include + // libc_musl.so in the sandbox. + // TODO(ccross): if we supported adding new dependencies during GenerateAndroidBuildActions + // this could be a dependency + TransitivePackagingSpecs. + c.ImplicitTool(c.rule.ctx.Config().HostJNIToolPath(c.rule.ctx, "libc_musl")) + } + return c.builtToolWithoutDeps(tool) +} + +// builtToolWithoutDeps is similar to BuiltTool, but doesn't add any dependencies. It is used +// internally by RuleBuilder for helper tools that are known to be compiled statically. +func (c *RuleBuilderCommand) builtToolWithoutDeps(tool string) *RuleBuilderCommand { return c.Tool(c.rule.ctx.Config().HostToolPath(c.rule.ctx, tool)) } diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 91bbce68e..bd7364582 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -378,6 +378,7 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s ctx.PropertyErrorf("from", "failed to open %q: %s", from, err) return (map[string]blueprint.ModuleFactory)(nil) } + defer r.Close() mtDef, errs := soongconfig.Parse(r, from) if ctx.Config().runningAsBp2Build { diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go index acb9d180e..ceb8e45a5 100644 --- a/android/soong_config_modules_test.go +++ b/android/soong_config_modules_test.go @@ -386,6 +386,46 @@ func TestNonExistentPropertyInSoongConfigModule(t *testing.T) { })).RunTest(t) } +func TestDuplicateStringValueInSoongConfigStringVariable(t *testing.T) { + bp := ` + soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b", "soc_c", "soc_a"], + } + + soong_config_module_type { + name: "acme_test", + module_type: "test", + config_namespace: "acme", + variables: ["board"], + properties: ["cflags", "srcs", "defaults"], + } + ` + + fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer { + return FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.VendorVars = vars + }) + } + + GroupFixturePreparers( + fixtureForVendorVars(map[string]map[string]string{"acme": {"feature1": "1"}}), + PrepareForTestWithDefaults, + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterModuleType("soong_config_module_type_import", SoongConfigModuleTypeImportFactory) + ctx.RegisterModuleType("soong_config_module_type", SoongConfigModuleTypeFactory) + ctx.RegisterModuleType("soong_config_string_variable", SoongConfigStringVariableDummyFactory) + ctx.RegisterModuleType("soong_config_bool_variable", SoongConfigBoolVariableDummyFactory) + ctx.RegisterModuleType("test_defaults", soongConfigTestDefaultsModuleFactory) + ctx.RegisterModuleType("test", soongConfigTestModuleFactory) + }), + FixtureWithRootAndroidBp(bp), + ).ExtendWithErrorHandler(FixtureExpectsAllErrorsToMatchAPattern([]string{ + // TODO(b/171232169): improve the error message for non-existent properties + `Android.bp: soong_config_string_variable: values property error: duplicate value: "soc_a"`, + })).RunTest(t) +} + func testConfigWithVendorVars(buildDir, bp string, fs map[string][]byte, vendorVars map[string]map[string]string) Config { config := TestConfig(buildDir, nil, bp, fs) diff --git a/android/soongconfig/Android.bp b/android/soongconfig/Android.bp index 9bf334480..8fe1ff1eb 100644 --- a/android/soongconfig/Android.bp +++ b/android/soongconfig/Android.bp @@ -10,6 +10,7 @@ bootstrap_go_package { "blueprint-parser", "blueprint-proptools", "soong-bazel", + "soong-starlark-format", ], srcs: [ "config.go", diff --git a/android/soongconfig/modules.go b/android/soongconfig/modules.go index 09a505722..212b752d6 100644 --- a/android/soongconfig/modules.go +++ b/android/soongconfig/modules.go @@ -25,6 +25,8 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/parser" "github.com/google/blueprint/proptools" + + "android/soong/starlark_fmt" ) const conditionsDefault = "conditions_default" @@ -177,10 +179,14 @@ func processStringVariableDef(v *SoongConfigDefinition, def *parser.Module) (err return []error{fmt.Errorf("values property must be set")} } + vals := make(map[string]bool, len(stringProps.Values)) for _, name := range stringProps.Values { if err := checkVariableName(name); err != nil { return []error{fmt.Errorf("soong_config_string_variable: values property error %s", err)} + } else if _, ok := vals[name]; ok { + return []error{fmt.Errorf("soong_config_string_variable: values property error: duplicate value: %q", name)} } + vals[name] = true } v.variables[base.variable] = &stringVariable{ @@ -235,7 +241,12 @@ type SoongConfigDefinition struct { // string vars, bool vars and value vars created by every // soong_config_module_type in this build. type Bp2BuildSoongConfigDefinitions struct { - StringVars map[string]map[string]bool + // varCache contains a cache of string variables namespace + property + // The same variable may be used in multiple module types (for example, if need support + // for cc_default and java_default), only need to process once + varCache map[string]bool + + StringVars map[string][]string BoolVars map[string]bool ValueVars map[string]bool } @@ -253,7 +264,7 @@ func (defs *Bp2BuildSoongConfigDefinitions) AddVars(mtDef SoongConfigDefinition) defer bp2buildSoongConfigVarsLock.Unlock() if defs.StringVars == nil { - defs.StringVars = make(map[string]map[string]bool) + defs.StringVars = make(map[string][]string) } if defs.BoolVars == nil { defs.BoolVars = make(map[string]bool) @@ -261,15 +272,24 @@ func (defs *Bp2BuildSoongConfigDefinitions) AddVars(mtDef SoongConfigDefinition) if defs.ValueVars == nil { defs.ValueVars = make(map[string]bool) } + if defs.varCache == nil { + defs.varCache = make(map[string]bool) + } for _, moduleType := range mtDef.ModuleTypes { for _, v := range moduleType.Variables { key := strings.Join([]string{moduleType.ConfigNamespace, v.variableProperty()}, "__") + + // The same variable may be used in multiple module types (for example, if need support + // for cc_default and java_default), only need to process once + if _, keyInCache := defs.varCache[key]; keyInCache { + continue + } else { + defs.varCache[key] = true + } + if strVar, ok := v.(*stringVariable); ok { - if _, ok := defs.StringVars[key]; !ok { - defs.StringVars[key] = make(map[string]bool, 0) - } for _, value := range strVar.values { - defs.StringVars[key][value] = true + defs.StringVars[key] = append(defs.StringVars[key], value) } } else if _, ok := v.(*boolVariable); ok { defs.BoolVars[key] = true @@ -302,29 +322,16 @@ func sortedStringKeys(m interface{}) []string { // String emits the Soong config variable definitions as Starlark dictionaries. func (defs Bp2BuildSoongConfigDefinitions) String() string { ret := "" - ret += "soong_config_bool_variables = {\n" - for _, boolVar := range sortedStringKeys(defs.BoolVars) { - ret += fmt.Sprintf(" \"%s\": True,\n", boolVar) - } - ret += "}\n" - ret += "\n" + ret += "soong_config_bool_variables = " + ret += starlark_fmt.PrintBoolDict(defs.BoolVars, 0) + ret += "\n\n" - ret += "soong_config_value_variables = {\n" - for _, valueVar := range sortedStringKeys(defs.ValueVars) { - ret += fmt.Sprintf(" \"%s\": True,\n", valueVar) - } - ret += "}\n" - ret += "\n" + ret += "soong_config_value_variables = " + ret += starlark_fmt.PrintBoolDict(defs.ValueVars, 0) + ret += "\n\n" - ret += "soong_config_string_variables = {\n" - for _, stringVar := range sortedStringKeys(defs.StringVars) { - ret += fmt.Sprintf(" \"%s\": [\n", stringVar) - for _, choice := range sortedStringKeys(defs.StringVars[stringVar]) { - ret += fmt.Sprintf(" \"%s\",\n", choice) - } - ret += fmt.Sprintf(" ],\n") - } - ret += "}" + ret += "soong_config_string_variables = " + ret += starlark_fmt.PrintStringListDict(defs.StringVars, 0) return ret } diff --git a/android/soongconfig/modules_test.go b/android/soongconfig/modules_test.go index b14f8b43b..a7800e8ef 100644 --- a/android/soongconfig/modules_test.go +++ b/android/soongconfig/modules_test.go @@ -367,19 +367,19 @@ func Test_PropertiesToApply(t *testing.T) { func Test_Bp2BuildSoongConfigDefinitions(t *testing.T) { testCases := []struct { + desc string defs Bp2BuildSoongConfigDefinitions expected string }{ { + desc: "all empty", defs: Bp2BuildSoongConfigDefinitions{}, - expected: `soong_config_bool_variables = { -} + expected: `soong_config_bool_variables = {} -soong_config_value_variables = { -} +soong_config_value_variables = {} -soong_config_string_variables = { -}`}, { +soong_config_string_variables = {}`}, { + desc: "only bool", defs: Bp2BuildSoongConfigDefinitions{ BoolVars: map[string]bool{ "bool_var": true, @@ -389,39 +389,35 @@ soong_config_string_variables = { "bool_var": True, } -soong_config_value_variables = { -} +soong_config_value_variables = {} -soong_config_string_variables = { -}`}, { +soong_config_string_variables = {}`}, { + desc: "only value vars", defs: Bp2BuildSoongConfigDefinitions{ ValueVars: map[string]bool{ "value_var": true, }, }, - expected: `soong_config_bool_variables = { -} + expected: `soong_config_bool_variables = {} soong_config_value_variables = { "value_var": True, } -soong_config_string_variables = { -}`}, { +soong_config_string_variables = {}`}, { + desc: "only string vars", defs: Bp2BuildSoongConfigDefinitions{ - StringVars: map[string]map[string]bool{ - "string_var": map[string]bool{ - "choice1": true, - "choice2": true, - "choice3": true, + StringVars: map[string][]string{ + "string_var": []string{ + "choice1", + "choice2", + "choice3", }, }, }, - expected: `soong_config_bool_variables = { -} + expected: `soong_config_bool_variables = {} -soong_config_value_variables = { -} +soong_config_value_variables = {} soong_config_string_variables = { "string_var": [ @@ -430,6 +426,7 @@ soong_config_string_variables = { "choice3", ], }`}, { + desc: "all vars", defs: Bp2BuildSoongConfigDefinitions{ BoolVars: map[string]bool{ "bool_var_one": true, @@ -438,15 +435,15 @@ soong_config_string_variables = { "value_var_one": true, "value_var_two": true, }, - StringVars: map[string]map[string]bool{ - "string_var_one": map[string]bool{ - "choice1": true, - "choice2": true, - "choice3": true, + StringVars: map[string][]string{ + "string_var_one": []string{ + "choice1", + "choice2", + "choice3", }, - "string_var_two": map[string]bool{ - "foo": true, - "bar": true, + "string_var_two": []string{ + "foo", + "bar", }, }, }, @@ -466,15 +463,17 @@ soong_config_string_variables = { "choice3", ], "string_var_two": [ - "bar", "foo", + "bar", ], }`}, } for _, test := range testCases { - actual := test.defs.String() - if actual != test.expected { - t.Errorf("Expected:\n%s\nbut got:\n%s", test.expected, actual) - } + t.Run(test.desc, func(t *testing.T) { + actual := test.defs.String() + if actual != test.expected { + t.Errorf("Expected:\n%s\nbut got:\n%s", test.expected, actual) + } + }) } } diff --git a/android/testing.go b/android/testing.go index 8daf6b783..39864e103 100644 --- a/android/testing.go +++ b/android/testing.go @@ -781,19 +781,21 @@ func (b baseTestingComponent) buildParamsFromRule(rule string) TestingBuildParam return p } -func (b baseTestingComponent) maybeBuildParamsFromDescription(desc string) TestingBuildParams { +func (b baseTestingComponent) maybeBuildParamsFromDescription(desc string) (TestingBuildParams, []string) { + var searchedDescriptions []string for _, p := range b.provider.BuildParamsForTests() { + searchedDescriptions = append(searchedDescriptions, p.Description) if strings.Contains(p.Description, desc) { - return b.newTestingBuildParams(p) + return b.newTestingBuildParams(p), searchedDescriptions } } - return TestingBuildParams{} + return TestingBuildParams{}, searchedDescriptions } func (b baseTestingComponent) buildParamsFromDescription(desc string) TestingBuildParams { - p := b.maybeBuildParamsFromDescription(desc) + p, searchedDescriptions := b.maybeBuildParamsFromDescription(desc) if p.Rule == nil { - panic(fmt.Errorf("couldn't find description %q", desc)) + panic(fmt.Errorf("couldn't find description %q\nall descriptions:\n%s", desc, strings.Join(searchedDescriptions, "\n"))) } return p } @@ -860,7 +862,8 @@ func (b baseTestingComponent) Rule(rule string) TestingBuildParams { // MaybeDescription finds a call to ctx.Build with BuildParams.Description set to a the given string. Returns an empty // BuildParams if no rule is found. func (b baseTestingComponent) MaybeDescription(desc string) TestingBuildParams { - return b.maybeBuildParamsFromDescription(desc) + p, _ := b.maybeBuildParamsFromDescription(desc) + return p } // Description finds a call to ctx.Build with BuildParams.Description set to a the given string. Panics if no rule is diff --git a/android/variable.go b/android/variable.go index 40dd2d82c..ff77fefe6 100644 --- a/android/variable.go +++ b/android/variable.go @@ -129,7 +129,7 @@ type variableProperties struct { Exclude_srcs []string } - // eng is true for -eng builds, and can be used to turn on additionaly heavyweight debugging + // eng is true for -eng builds, and can be used to turn on additional heavyweight debugging // features. Eng struct { Cflags []string diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index d64eb7a92..f050a2e87 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -122,17 +122,10 @@ func (s *sdkRepoHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { s.CopySpecsToDir(ctx, builder, packageSpecs, dir) - // Collect licenses to write into NOTICE.txt - noticeMap := map[string]android.Paths{} - for path, pkgSpec := range packageSpecs { - licenseFiles := pkgSpec.EffectiveLicenseFiles() - if len(licenseFiles) > 0 { - noticeMap[path] = pkgSpec.EffectiveLicenseFiles() - } - } - notices := android.BuildNotices(ctx, noticeMap) + noticeFile := android.PathForModuleOut(ctx, "NOTICES.txt") + android.BuildNoticeTextOutputFromLicenseMetadata(ctx, noticeFile) builder.Command().Text("cp"). - Input(notices.TxtOutput.Path()). + Input(noticeFile). Text(filepath.Join(dir.String(), "NOTICE.txt")) // Handle `merge_zips` by extracting their contents into our tmpdir diff --git a/androidmk/androidmk/android.go b/androidmk/androidmk/android.go index ae526887a..295b0e50e 100644 --- a/androidmk/androidmk/android.go +++ b/androidmk/androidmk/android.go @@ -68,6 +68,8 @@ var rewriteProperties = map[string](func(variableAssignmentContext) error){ "LOCAL_MODULE_PATH": prebuiltModulePath, "LOCAL_REPLACE_PREBUILT_APK_INSTALLED": prebuiltPreprocessed, + "LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG": invert("auto_gen_config"), + // composite functions "LOCAL_MODULE_TAGS": includeVariableIf(bpVariable{"tags", bpparser.ListType}, not(valueDumpEquals("optional"))), @@ -191,6 +193,7 @@ func init() { // will be removed later by byfix // TODO: does this property matter in the license module? "LOCAL_LICENSE_CONDITIONS": "android_license_conditions", + "LOCAL_GENERATED_SOURCES": "generated_sources", }) addStandardProperties(bpparser.BoolType, diff --git a/androidmk/androidmk/androidmk_test.go b/androidmk/androidmk/androidmk_test.go index 3c3619769..e8b6f78cf 100644 --- a/androidmk/androidmk/androidmk_test.go +++ b/androidmk/androidmk/androidmk_test.go @@ -699,7 +699,7 @@ include $(call all-makefiles-under,$(LOCAL_PATH)) expected: ` android_library { srcs: ["test.java"], - resource_dirs: ["res"], + jacoco: { include_filter: ["foo.*"], }, @@ -1458,7 +1458,7 @@ include $(BUILD_RRO_PACKAGE) runtime_resource_overlay { name: "foo", product_specific: true, - resource_dirs: ["res"], + sdk_version: "current", theme: "FooTheme", @@ -1604,6 +1604,77 @@ cc_prebuilt_library_shared { } `, }, + { + desc: "Drop default resource and asset dirs from bp", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_ASSET_DIR := $(LOCAL_PATH)/assets +LOCAL_RESOURCE_DIR := $(LOCAL_PATH)/res +include $(BUILD_PACKAGE) +`, + expected: ` +android_app { + name: "foo", + +} +`, + }, + { + desc: "LOCAL_GENERATED_SOURCES", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_SRC_FILES := src1, src2, src3 +LOCAL_GENERATED_SOURCES := gen_src1, gen_src2, gen_src3 +include $(BUILD_PACKAGE) + `, + expected: ` +android_app { + name: "foo", + srcs: [ + "src1,", + "src2,", + "src3", + ], + generated_sources: [ + "gen_src1,", + "gen_src2,", + "gen_src3", + ], +} +`, + }, + { + desc: "LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG is true", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG := true +include $(BUILD_PACKAGE) + `, + expected: ` +android_app { + name: "foo", + auto_gen_config: false, +} +`, + }, + { + desc: "LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG is false", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG := false +include $(BUILD_PACKAGE) + `, + expected: ` +android_app { + name: "foo", + auto_gen_config: true, +} +`, + }, } func TestEndToEnd(t *testing.T) { diff --git a/apex/apex.go b/apex/apex.go index a6c63de35..d12a7865b 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -888,9 +888,18 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { // APEX, but shared across APEXes via the VNDK APEX. useVndk := a.SocSpecific() || a.DeviceSpecific() || (a.ProductSpecific() && mctx.Config().EnforceProductPartitionInterface()) excludeVndkLibs := useVndk && proptools.Bool(a.properties.Use_vndk_as_stable) - if !useVndk && proptools.Bool(a.properties.Use_vndk_as_stable) { - mctx.PropertyErrorf("use_vndk_as_stable", "not supported for system/system_ext APEXes") - return + if proptools.Bool(a.properties.Use_vndk_as_stable) { + if !useVndk { + mctx.PropertyErrorf("use_vndk_as_stable", "not supported for system/system_ext APEXes") + } + mctx.VisitDirectDepsWithTag(sharedLibTag, func(dep android.Module) { + if c, ok := dep.(*cc.Module); ok && c.IsVndk() { + mctx.PropertyErrorf("use_vndk_as_stable", "Trying to include a VNDK library(%s) while use_vndk_as_stable is true.", dep.Name()) + } + }) + if mctx.Failed() { + return + } } continueApexDepsWalk := func(child, parent android.Module) bool { @@ -3289,6 +3298,7 @@ type bazelApexBundleAttributes struct { Prebuilts bazel.LabelListAttribute Native_shared_libs_32 bazel.LabelListAttribute Native_shared_libs_64 bazel.LabelListAttribute + Compressible bazel.BoolAttribute } type convertedNativeSharedLibs struct { @@ -3366,6 +3376,11 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { installableAttribute.Value = a.properties.Installable } + var compressibleAttribute bazel.BoolAttribute + if a.overridableProperties.Compressible != nil { + compressibleAttribute.Value = a.overridableProperties.Compressible + } + attrs := &bazelApexBundleAttributes{ Manifest: manifestLabelAttribute, Android_manifest: androidManifestLabelAttribute, @@ -3379,6 +3394,7 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { Native_shared_libs_64: nativeSharedLibs.Native_shared_libs_64, Binaries: binariesLabelListAttribute, Prebuilts: prebuiltsLabelListAttribute, + Compressible: compressibleAttribute, } props := bazel.BazelTargetModuleProperties{ diff --git a/apex/apex_test.go b/apex/apex_test.go index c546fa13c..6d77b0623 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -2713,7 +2713,33 @@ func TestVendorApex(t *testing.T) { ensureListNotContains(t, requireNativeLibs, ":vndk") } +func TestVendorApex_use_vndk_as_stable_TryingToIncludeVNDKLib(t *testing.T) { + testApexError(t, `Trying to include a VNDK library`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libc++"], // libc++ is a VNDK lib + vendor: true, + use_vndk_as_stable: true, + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + }`) +} + func TestVendorApex_use_vndk_as_stable(t *testing.T) { + // myapex myapex2 + // | | + // mybin ------. mybin2 + // \ \ / | + // (stable) .---\--------` | + // \ / \ | + // \ / \ / + // libvndk libvendor + // (vndk) ctx := testApex(t, ` apex { name: "myapex", @@ -2744,28 +2770,95 @@ func TestVendorApex_use_vndk_as_stable(t *testing.T) { cc_library { name: "libvendor", vendor: true, + stl: "none", + } + apex { + name: "myapex2", + key: "myapex.key", + binaries: ["mybin2"], + vendor: true, + use_vndk_as_stable: false, + updatable: false, + } + cc_binary { + name: "mybin2", + vendor: true, + shared_libs: ["libvndk", "libvendor"], } `) vendorVariant := "android_vendor.29_arm64_armv8-a" - ldRule := ctx.ModuleForTests("mybin", vendorVariant+"_apex10000").Rule("ld") - libs := names(ldRule.Args["libFlags"]) - // VNDK libs(libvndk/libc++) as they are - ensureListContains(t, libs, "out/soong/.intermediates/libvndk/"+vendorVariant+"_shared/libvndk.so") - ensureListContains(t, libs, "out/soong/.intermediates/"+cc.DefaultCcCommonTestModulesDir+"libc++/"+vendorVariant+"_shared/libc++.so") - // non-stable Vendor libs as APEX variants - ensureListContains(t, libs, "out/soong/.intermediates/libvendor/"+vendorVariant+"_shared_apex10000/libvendor.so") - - // VNDK libs are not included when use_vndk_as_stable: true - ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ - "bin/mybin", - "lib64/libvendor.so", - }) - - apexManifestRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexManifestRule") - requireNativeLibs := names(apexManifestRule.Args["requireNativeLibs"]) - ensureListContains(t, requireNativeLibs, ":vndk") + for _, tc := range []struct { + name string + apexName string + moduleName string + moduleVariant string + libs []string + contents []string + requireVndkNamespace bool + }{ + { + name: "use_vndk_as_stable", + apexName: "myapex", + moduleName: "mybin", + moduleVariant: vendorVariant + "_apex10000", + libs: []string{ + // should link with vendor variants of VNDK libs(libvndk/libc++) + "out/soong/.intermediates/libvndk/" + vendorVariant + "_shared/libvndk.so", + "out/soong/.intermediates/" + cc.DefaultCcCommonTestModulesDir + "libc++/" + vendorVariant + "_shared/libc++.so", + // unstable Vendor libs as APEX variant + "out/soong/.intermediates/libvendor/" + vendorVariant + "_shared_apex10000/libvendor.so", + }, + contents: []string{ + "bin/mybin", + "lib64/libvendor.so", + // VNDK libs (libvndk/libc++) are not included + }, + requireVndkNamespace: true, + }, + { + name: "!use_vndk_as_stable", + apexName: "myapex2", + moduleName: "mybin2", + moduleVariant: vendorVariant + "_myapex2", + libs: []string{ + // should link with "unique" APEX(myapex2) variant of VNDK libs(libvndk/libc++) + "out/soong/.intermediates/libvndk/" + vendorVariant + "_shared_myapex2/libvndk.so", + "out/soong/.intermediates/" + cc.DefaultCcCommonTestModulesDir + "libc++/" + vendorVariant + "_shared_myapex2/libc++.so", + // unstable vendor libs have "merged" APEX variants + "out/soong/.intermediates/libvendor/" + vendorVariant + "_shared_apex10000/libvendor.so", + }, + contents: []string{ + "bin/mybin2", + "lib64/libvendor.so", + // VNDK libs are included as well + "lib64/libvndk.so", + "lib64/libc++.so", + }, + requireVndkNamespace: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + // Check linked libs + ldRule := ctx.ModuleForTests(tc.moduleName, tc.moduleVariant).Rule("ld") + libs := names(ldRule.Args["libFlags"]) + for _, lib := range tc.libs { + ensureListContains(t, libs, lib) + } + // Check apex contents + ensureExactContents(t, ctx, tc.apexName, "android_common_"+tc.apexName+"_image", tc.contents) + + // Check "requireNativeLibs" + apexManifestRule := ctx.ModuleForTests(tc.apexName, "android_common_"+tc.apexName+"_image").Rule("apexManifestRule") + requireNativeLibs := names(apexManifestRule.Args["requireNativeLibs"]) + if tc.requireVndkNamespace { + ensureListContains(t, requireNativeLibs, ":vndk") + } else { + ensureListNotContains(t, requireNativeLibs, ":vndk") + } + }) + } } func TestProductVariant(t *testing.T) { @@ -2796,7 +2889,7 @@ func TestProductVariant(t *testing.T) { ) cflags := strings.Fields( - ctx.ModuleForTests("foo", "android_product.29_arm64_armv8-a_apex10000").Rule("cc").Args["cFlags"]) + ctx.ModuleForTests("foo", "android_product.29_arm64_armv8-a_myapex").Rule("cc").Args["cFlags"]) ensureListContains(t, cflags, "-D__ANDROID_VNDK__") ensureListContains(t, cflags, "-D__ANDROID_APEX__") ensureListContains(t, cflags, "-D__ANDROID_PRODUCT__") diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 8f44fc537..ce6b7f730 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -553,12 +553,66 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { `prebuilt_com.android.art`, }) + // The boot images are installed in the APEX by Soong, so there shouldn't be any dexpreopt-related Make modules. + ensureDoesNotContainRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "mybootclasspathfragment-dexpreopt-arm64-boot.art", + "mybootclasspathfragment-dexpreopt-arm64-boot.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot.art", + "mybootclasspathfragment-dexpreopt-arm-boot.oat", + "mybootclasspathfragment-dexpreopt-arm-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", + }) + // Make sure that the prebuilt bootclasspath_fragment copies its dex files to the predefined // locations for the art image. module := result.ModuleForTests("prebuilt_mybootclasspathfragment", "android_common_com.android.art") checkCopiesToPredefinedLocationForArt(t, result.Config, module, "bar", "foo") }) + t.Run("boot image files from preferred prebuilt no boot image in apex", func(t *testing.T) { + result := android.GroupFixturePreparers( + commonPreparer, + + // Configure some libraries in the art bootclasspath_fragment that match the source + // bootclasspath_fragment's contents property. + java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), + addSource("foo", "bar"), + + // Make sure that a preferred prebuilt with consistent contents doesn't affect the apex. + addPrebuilt(true, "foo", "bar"), + + java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), + ).RunTest(t) + + ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "etc/boot-image.prof", + "etc/classpaths/bootclasspath.pb", + "javalib/bar.jar", + "javalib/foo.jar", + }) + + ensureContainsRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "mybootclasspathfragment-dexpreopt-arm64-boot.art", + "mybootclasspathfragment-dexpreopt-arm64-boot.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot.art", + "mybootclasspathfragment-dexpreopt-arm-boot.oat", + "mybootclasspathfragment-dexpreopt-arm-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", + }) + }) + t.Run("source with inconsistency between config and contents", func(t *testing.T) { android.GroupFixturePreparers( commonPreparer, @@ -631,6 +685,7 @@ func TestBootclasspathFragmentInPrebuiltArtApex(t *testing.T) { // Configure some libraries in the art bootclasspath_fragment. java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), + java.FixtureSetBootImageInstallDirOnDevice("art", "apex/com.android.art/javalib"), ) bp := ` diff --git a/apex/builder.go b/apex/builder.go index a66e1e0fc..fc4bf8a22 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -397,6 +397,12 @@ func (a *apexBundle) buildBundleConfig(ctx android.ModuleContext) android.Output return output.OutputPath } +func markManifestTestOnly(ctx android.ModuleContext, androidManifestFile android.Path) android.Path { + return java.ManifestFixer(ctx, androidManifestFile, java.ManifestFixerParams{ + TestOnly: true, + }) +} + // buildUnflattendApex creates build rules to build an APEX using apexer. func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { apexType := a.properties.ApexType @@ -595,8 +601,15 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { if a.properties.AndroidManifest != nil { androidManifestFile := android.PathForModuleSrc(ctx, proptools.String(a.properties.AndroidManifest)) + + if a.testApex { + androidManifestFile = markManifestTestOnly(ctx, androidManifestFile) + } + implicitInputs = append(implicitInputs, androidManifestFile) optFlags = append(optFlags, "--android_manifest "+androidManifestFile.String()) + } else if a.testApex { + optFlags = append(optFlags, "--test_only") } // Determine target/min sdk version from the context diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 02d807516..158c8046f 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -65,7 +65,8 @@ type prebuiltCommon struct { // Installed locations of symlinks for backward compatibility. compatSymlinks android.InstallPaths - hostRequired []string + hostRequired []string + requiredModuleNames []string } type sanitizedPrebuilt interface { @@ -195,9 +196,19 @@ func (p *prebuiltCommon) initApexFilesForAndroidMk(ctx android.ModuleContext) { } p.apexFilesForAndroidMk = append(p.apexFilesForAndroidMk, af) } - } else if tag == exportedBootclasspathFragmentTag || - tag == exportedSystemserverclasspathFragmentTag { - // Visit the children of the bootclasspath_fragment and systemserver_fragment. + } else if tag == exportedBootclasspathFragmentTag { + bcpfModule, ok := child.(*java.PrebuiltBootclasspathFragmentModule) + if !ok { + ctx.PropertyErrorf("exported_bootclasspath_fragments", "%q is not a prebuilt_bootclasspath_fragment module", name) + return false + } + for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { + p.requiredModuleNames = append(p.requiredModuleNames, makeModuleName) + } + // Visit the children of the bootclasspath_fragment. + return true + } else if tag == exportedSystemserverclasspathFragmentTag { + // Visit the children of the systemserver_fragment. return true } @@ -211,6 +222,7 @@ func (p *prebuiltCommon) addRequiredModules(entries *android.AndroidMkEntries) { entries.AddStrings("LOCAL_TARGET_REQUIRED_MODULES", fi.targetRequiredModuleNames...) entries.AddStrings("LOCAL_HOST_REQUIRED_MODULES", fi.hostRequiredModuleNames...) } + entries.AddStrings("LOCAL_REQUIRED_MODULES", p.requiredModuleNames...) } func (p *prebuiltCommon) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 4dc0e4cd1..5d00b0b29 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -114,7 +114,7 @@ staticLibraries = [] rootStaticArchives = [] linker_inputs = cc_info.linking_context.linker_inputs.to_list() -static_info_tag = "//build/bazel/rules:cc_library_static.bzl%CcStaticLibraryInfo" +static_info_tag = "//build/bazel/rules/cc:cc_library_static.bzl%CcStaticLibraryInfo" if static_info_tag in providers(target): static_info = providers(target)[static_info_tag] ccObjectFiles = [f.path for f in static_info.objects] @@ -149,7 +149,7 @@ else: rootSharedLibraries.append(path) toc_file = "" -toc_file_tag = "//build/bazel/rules:generate_toc.bzl%CcTocInfo" +toc_file_tag = "//build/bazel/rules/cc:generate_toc.bzl%CcTocInfo" if toc_file_tag in providers(target): toc_file = providers(target)[toc_file_tag].toc.path else: diff --git a/bp2build/Android.bp b/bp2build/Android.bp index 4bcfa6110..b904c3533 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -28,6 +28,7 @@ bootstrap_go_package { "soong-genrule", "soong-python", "soong-sh", + "soong-starlark-format", "soong-ui-metrics", ], testSrcs: [ diff --git a/bp2build/apex_conversion_test.go b/bp2build/apex_conversion_test.go index 4b141c954..90571893c 100644 --- a/bp2build/apex_conversion_test.go +++ b/bp2build/apex_conversion_test.go @@ -104,6 +104,7 @@ apex { certificate: "com.android.apogee.certificate", updatable: false, installable: false, + compressible: false, native_shared_libs: [ "native_shared_lib_1", "native_shared_lib_2", @@ -150,7 +151,8 @@ apex { ":pretend_prebuilt_1", ":pretend_prebuilt_2", ]`, - "updatable": "False", + "updatable": "False", + "compressible": "False", }), }}) } diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index b3bec65da..1d3b10550 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -27,6 +27,7 @@ import ( "android/soong/android" "android/soong/bazel" + "android/soong/starlark_fmt" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -559,48 +560,27 @@ func prettyPrint(propertyValue reflect.Value, indent int, emitZeroValues bool) ( return "", nil } - var ret string switch propertyValue.Kind() { case reflect.String: - ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String())) + return fmt.Sprintf("\"%v\"", escapeString(propertyValue.String())), nil case reflect.Bool: - ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface())) + return starlark_fmt.PrintBool(propertyValue.Bool()), nil case reflect.Int, reflect.Uint, reflect.Int64: - ret = fmt.Sprintf("%v", propertyValue.Interface()) + return fmt.Sprintf("%v", propertyValue.Interface()), nil case reflect.Ptr: return prettyPrint(propertyValue.Elem(), indent, emitZeroValues) case reflect.Slice: - if propertyValue.Len() == 0 { - return "[]", nil - } - - if propertyValue.Len() == 1 { - // Single-line list for list with only 1 element - ret += "[" - indexedValue, err := prettyPrint(propertyValue.Index(0), indent, emitZeroValues) + elements := make([]string, 0, propertyValue.Len()) + for i := 0; i < propertyValue.Len(); i++ { + val, err := prettyPrint(propertyValue.Index(i), indent, emitZeroValues) if err != nil { return "", err } - ret += indexedValue - ret += "]" - } else { - // otherwise, use a multiline list. - ret += "[\n" - for i := 0; i < propertyValue.Len(); i++ { - indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1, emitZeroValues) - if err != nil { - return "", err - } - - if indexedValue != "" { - ret += makeIndent(indent + 1) - ret += indexedValue - ret += ",\n" - } + if val != "" { + elements = append(elements, val) } - ret += makeIndent(indent) - ret += "]" } + return starlark_fmt.PrintList(elements, indent, "%s"), nil case reflect.Struct: // Special cases where the bp2build sends additional information to the codegenerator @@ -611,18 +591,12 @@ func prettyPrint(propertyValue reflect.Value, indent int, emitZeroValues bool) ( return fmt.Sprintf("%q", label.Label), nil } - ret = "{\n" // Sort and print the struct props by the key. structProps := extractStructProperties(propertyValue, indent) if len(structProps) == 0 { return "", nil } - for _, k := range android.SortedStringKeys(structProps) { - ret += makeIndent(indent + 1) - ret += fmt.Sprintf("%q: %s,\n", k, structProps[k]) - } - ret += makeIndent(indent) - ret += "}" + return starlark_fmt.PrintDict(structProps, indent), nil case reflect.Interface: // TODO(b/164227191): implement pretty print for interfaces. // Interfaces are used for for arch, multilib and target properties. @@ -631,7 +605,6 @@ func prettyPrint(propertyValue reflect.Value, indent int, emitZeroValues bool) ( return "", fmt.Errorf( "unexpected kind for property struct field: %s", propertyValue.Kind()) } - return ret, nil } // Converts a reflected property struct value into a map of property names and property values, @@ -736,13 +709,6 @@ func escapeString(s string) string { return strings.ReplaceAll(s, "\"", "\\\"") } -func makeIndent(indent int) string { - if indent < 0 { - panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent)) - } - return strings.Repeat(" ", indent) -} - func targetNameWithVariant(c bpToBuildContext, logicModule blueprint.Module) string { name := "" if c.ModuleSubDir(logicModule) != "" { diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go index a156480d2..f9adc78f8 100644 --- a/bp2build/cc_binary_conversion_test.go +++ b/bp2build/cc_binary_conversion_test.go @@ -84,13 +84,13 @@ func runCcHostBinaryTestCase(t *testing.T, tc ccBinaryBp2buildTestCase) { t.Helper() testCase := tc for i, tar := range testCase.targets { - if tar.typ != "cc_binary" { - continue - } - tar.attrs["target_compatible_with"] = `select({ + switch tar.typ { + case "cc_binary", "proto_library", "cc_lite_proto_library": + tar.attrs["target_compatible_with"] = `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], })` + } testCase.targets[i] = tar } moduleTypeUnderTest := "cc_binary_host" diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index ee1978320..3bf022126 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -1287,6 +1287,7 @@ func makeCcLibraryTargets(name string, attrs attrNameToString) []string { "strip": true, "stubs_symbol_file": true, "stubs_versions": true, + "inject_bssl_hash": true, } sharedAttrs := attrNameToString{} staticAttrs := attrNameToString{} @@ -1822,6 +1823,33 @@ cc_library { ) } +func TestLibcryptoHashInjection(t *testing.T) { + runCcLibraryTestCase(t, bp2buildTestCase{ + description: "cc_library - libcrypto hash injection", + moduleTypeUnderTest: "cc_library", + moduleTypeUnderTestFactory: cc.LibraryFactory, + filesystem: map[string]string{}, + blueprint: soongCcLibraryPreamble + ` +cc_library { + name: "libcrypto", + target: { + android: { + inject_bssl_hash: true, + }, + }, + include_build_directory: false, +} +`, + expectedBazelTargets: makeCcLibraryTargets("libcrypto", attrNameToString{ + "inject_bssl_hash": `select({ + "//build/bazel/platforms/os:android": True, + "//conditions:default": None, + })`, + }), + }, + ) +} + func TestCcLibraryCppStdWithGnuExtensions_ConvertsToFeatureAttr(t *testing.T) { type testCase struct { cpp_std string @@ -2409,3 +2437,18 @@ cc_library { }, ) } + +func TestCcLibraryEscapeLdflags(t *testing.T) { + runCcLibraryTestCase(t, bp2buildTestCase{ + moduleTypeUnderTest: "cc_library", + moduleTypeUnderTestFactory: cc.LibraryFactory, + blueprint: soongCcProtoPreamble + `cc_library { + name: "foo", + ldflags: ["-Wl,--rpath,${ORIGIN}"], + include_build_directory: false, +}`, + expectedBazelTargets: makeCcLibraryTargets("foo", attrNameToString{ + "linkopts": `["-Wl,--rpath,$${ORIGIN}"]`, + }), + }) +} diff --git a/bp2build/configurability.go b/bp2build/configurability.go index dfbb265d2..d37a52394 100644 --- a/bp2build/configurability.go +++ b/bp2build/configurability.go @@ -6,6 +6,7 @@ import ( "android/soong/android" "android/soong/bazel" + "android/soong/starlark_fmt" ) // Configurability support for bp2build. @@ -250,10 +251,10 @@ func prettyPrintSelectMap(selectMap map[string]reflect.Value, defaultValue *stri } else if defaultValue != nil { // Print an explicit empty list (the default value) even if the value is // empty, to avoid errors about not finding a configuration that matches. - ret += fmt.Sprintf("%s\"%s\": %s,\n", makeIndent(indent+1), bazel.ConditionsDefaultSelectKey, *defaultValue) + ret += fmt.Sprintf("%s\"%s\": %s,\n", starlark_fmt.Indention(indent+1), bazel.ConditionsDefaultSelectKey, *defaultValue) } - ret += makeIndent(indent) + ret += starlark_fmt.Indention(indent) ret += "})" return ret, nil @@ -262,7 +263,7 @@ func prettyPrintSelectMap(selectMap map[string]reflect.Value, defaultValue *stri // prettyPrintSelectEntry converts a reflect.Value into an entry in a select map // with a provided key. func prettyPrintSelectEntry(value reflect.Value, key string, indent int, emitZeroValues bool) (string, error) { - s := makeIndent(indent + 1) + s := starlark_fmt.Indention(indent + 1) v, err := prettyPrint(value, indent+1, emitZeroValues) if err != nil { return "", err diff --git a/bp2build/genrule_conversion_test.go b/bp2build/genrule_conversion_test.go index 0666da778..9244b9922 100644 --- a/bp2build/genrule_conversion_test.go +++ b/bp2build/genrule_conversion_test.go @@ -97,13 +97,22 @@ func TestGenruleCliVariableReplacement(t *testing.T) { }` for _, tc := range testCases { + moduleAttrs := attrNameToString{ + "cmd": fmt.Sprintf(`"$(location :foo.tool) --genDir=%s arg $(SRCS) $(OUTS)"`, tc.genDir), + "outs": `["foo.out"]`, + "srcs": `["foo.in"]`, + "tools": `[":foo.tool"]`, + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ - "cmd": fmt.Sprintf(`"$(location :foo.tool) --genDir=%s arg $(SRCS) $(OUTS)"`, tc.genDir), - "outs": `["foo.out"]`, - "srcs": `["foo.in"]`, - "tools": `[":foo.tool"]`, - }), + makeBazelTarget("genrule", "foo", moduleAttrs), } t.Run(tc.moduleType, func(t *testing.T) { @@ -158,25 +167,36 @@ func TestGenruleLocationsLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := - []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ - "cmd": `"$(locations :foo.tools) -s $(OUTS) $(SRCS)"`, - "outs": `["foo.out"]`, - "srcs": `["foo.in"]`, - "tools": `[":foo.tools"]`, - }), - makeBazelTarget("genrule", "foo.tools", attrNameToString{ - "cmd": `"cp $(SRCS) $(OUTS)"`, - "outs": `[ + for _, tc := range testCases { + fooAttrs := attrNameToString{ + "cmd": `"$(locations :foo.tools) -s $(OUTS) $(SRCS)"`, + "outs": `["foo.out"]`, + "srcs": `["foo.in"]`, + "tools": `[":foo.tools"]`, + } + fooToolsAttrs := attrNameToString{ + "cmd": `"cp $(SRCS) $(OUTS)"`, + "outs": `[ "foo_tool.out", "foo_tool2.out", ]`, - "srcs": `["foo_tool.in"]`, - }), + "srcs": `["foo_tool.in"]`, + } + + if tc.moduleType == "java_genrule_host" { + compatibilityAttrs := `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + fooAttrs["target_compatible_with"] = compatibilityAttrs + fooToolsAttrs["target_compatible_with"] = compatibilityAttrs + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", fooAttrs), + makeBazelTarget("genrule", "foo.tools", fooToolsAttrs), } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -221,16 +241,25 @@ func TestGenruleLocationsAbsoluteLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(locations //other:foo.tool) -s $(OUTS) $(SRCS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, "tools": `["//other:foo.tool"]`, - }), - } + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -276,16 +305,25 @@ func TestGenruleSrcsLocationsAbsoluteLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(locations //other:foo.tool) -s $(OUTS) $(location //other:other.tool)"`, "outs": `["foo.out"]`, "srcs": `["//other:other.tool"]`, "tools": `["//other:foo.tool"]`, - }), - } + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -331,8 +369,8 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(location //other:foo.tool) -s $(OUTS) $(SRCS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, @@ -340,9 +378,19 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { "//other:foo.tool", "//other:other.tool", ]`, - })} + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -388,8 +436,8 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(locations //other:foo.tool) -s $(OUTS) $(SRCS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, @@ -397,9 +445,19 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { "//other:foo.tool", "//other:other.tool", ]`, - })} + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -444,14 +502,24 @@ func TestGenruleWithoutToolsOrToolFiles(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"cp $(SRCS) $(OUTS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, - })} + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go index c683b25f5..65136d9f1 100644 --- a/bp2build/java_binary_host_conversion_test.go +++ b/bp2build/java_binary_host_conversion_test.go @@ -59,6 +59,10 @@ func TestJavaBinaryHost(t *testing.T) { "deps": `["//other:jni-lib-1"]`, "jvm_flags": `["-Djava.library.path=$${RUNPATH}other"]`, "javacopts": `["-Xdoclint:all/protected"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) diff --git a/bp2build/java_import_conversion_test.go b/bp2build/java_import_conversion_test.go new file mode 100644 index 000000000..2f7211ccc --- /dev/null +++ b/bp2build/java_import_conversion_test.go @@ -0,0 +1,52 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "android/soong/java" + + "testing" +) + +func runJavaImportTestCase(t *testing.T, tc bp2buildTestCase) { + t.Helper() + runBp2BuildTestCase(t, registerJavaImportModuleTypes, tc) +} + +func registerJavaImportModuleTypes(ctx android.RegistrationContext) { +} + +func TestMinimalJavaImport(t *testing.T) { + runJavaImportTestCase(t, bp2buildTestCase{ + description: "Java import - simple example", + moduleTypeUnderTest: "java_import", + moduleTypeUnderTestFactory: java.ImportFactory, + filesystem: map[string]string{ + "import.jar": "", + }, + blueprint: ` +java_import { + name: "example_import", + jars: ["import.jar"], + bazel_module: { bp2build_available: true }, +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("java_import", "example_import", attrNameToString{ + "jars": `["import.jar"]`, + }), + }}) +} diff --git a/bp2build/java_library_host_conversion_test.go b/bp2build/java_library_host_conversion_test.go index 6ac82dbd2..73abdd2ea 100644 --- a/bp2build/java_library_host_conversion_test.go +++ b/bp2build/java_library_host_conversion_test.go @@ -48,9 +48,17 @@ java_library_host { makeBazelTarget("java_library", "java-lib-host-1", attrNameToString{ "srcs": `["a.java"]`, "deps": `[":java-lib-host-2"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), makeBazelTarget("java_library", "java-lib-host-2", attrNameToString{ "srcs": `["c.java"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) diff --git a/bp2build/java_plugin_conversion_test.go b/bp2build/java_plugin_conversion_test.go new file mode 100644 index 000000000..ff13bb054 --- /dev/null +++ b/bp2build/java_plugin_conversion_test.go @@ -0,0 +1,72 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "testing" + + "android/soong/android" + "android/soong/java" +) + +func runJavaPluginTestCase(t *testing.T, tc bp2buildTestCase) { + t.Helper() + (&tc).moduleTypeUnderTest = "java_plugin" + (&tc).moduleTypeUnderTestFactory = java.PluginFactory + runBp2BuildTestCase(t, func(ctx android.RegistrationContext) { + ctx.RegisterModuleType("java_library", java.LibraryFactory) + }, tc) +} + +func TestJavaPlugin(t *testing.T) { + runJavaPluginTestCase(t, bp2buildTestCase{ + description: "java_plugin with srcs, libs, static_libs", + blueprint: `java_plugin { + name: "java-plug-1", + srcs: ["a.java", "b.java"], + libs: ["java-lib-1"], + static_libs: ["java-lib-2"], + bazel_module: { bp2build_available: true }, +} + +java_library { + name: "java-lib-1", + srcs: ["b.java"], + bazel_module: { bp2build_available: false }, +} + +java_library { + name: "java-lib-2", + srcs: ["c.java"], + bazel_module: { bp2build_available: false }, +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_plugin", "java-plug-1", attrNameToString{ + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, + "deps": `[ + ":java-lib-1", + ":java-lib-2", + ]`, + "srcs": `[ + "a.java", + "b.java", + ]`, + }), + }, + }) +} diff --git a/bp2build/python_binary_conversion_test.go b/bp2build/python_binary_conversion_test.go index 40c8ba1e9..dfa11d1be 100644 --- a/bp2build/python_binary_conversion_test.go +++ b/bp2build/python_binary_conversion_test.go @@ -51,6 +51,10 @@ func TestPythonBinaryHostSimple(t *testing.T) { "b/c.py", "b/d.py", ]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) @@ -80,6 +84,10 @@ func TestPythonBinaryHostPy2(t *testing.T) { makeBazelTarget("py_binary", "foo", attrNameToString{ "python_version": `"PY2"`, "srcs": `["a.py"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) @@ -109,6 +117,10 @@ func TestPythonBinaryHostPy3(t *testing.T) { // python_version is PY3 by default. makeBazelTarget("py_binary", "foo", attrNameToString{ "srcs": `["a.py"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) @@ -141,6 +153,10 @@ func TestPythonBinaryHostArchVariance(t *testing.T) { "//build/bazel/platforms/arch:x86": ["x86.py"], "//conditions:default": [], })`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) diff --git a/bp2build/python_library_conversion_test.go b/bp2build/python_library_conversion_test.go index 6b261052c..356d52e8c 100644 --- a/bp2build/python_library_conversion_test.go +++ b/bp2build/python_library_conversion_test.go @@ -11,19 +11,51 @@ import ( // TODO(alexmarquez): Should be lifted into a generic Bp2Build file type PythonLibBp2Build func(ctx android.TopDownMutatorContext) -func runPythonLibraryTestCase(t *testing.T, tc bp2buildTestCase) { +type pythonLibBp2BuildTestCase struct { + description string + filesystem map[string]string + blueprint string + expectedBazelTargets []testBazelTarget +} + +func convertPythonLibTestCaseToBp2build_Host(tc pythonLibBp2BuildTestCase) bp2buildTestCase { + for i := range tc.expectedBazelTargets { + tc.expectedBazelTargets[i].attrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + return convertPythonLibTestCaseToBp2build(tc) +} + +func convertPythonLibTestCaseToBp2build(tc pythonLibBp2BuildTestCase) bp2buildTestCase { + var bp2BuildTargets []string + for _, t := range tc.expectedBazelTargets { + bp2BuildTargets = append(bp2BuildTargets, makeBazelTarget(t.typ, t.name, t.attrs)) + } + return bp2buildTestCase{ + description: tc.description, + filesystem: tc.filesystem, + blueprint: tc.blueprint, + expectedBazelTargets: bp2BuildTargets, + } +} + +func runPythonLibraryTestCase(t *testing.T, tc pythonLibBp2BuildTestCase) { t.Helper() - testCase := tc + testCase := convertPythonLibTestCaseToBp2build(tc) testCase.description = fmt.Sprintf(testCase.description, "python_library") testCase.blueprint = fmt.Sprintf(testCase.blueprint, "python_library") testCase.moduleTypeUnderTest = "python_library" testCase.moduleTypeUnderTestFactory = python.PythonLibraryFactory + runBp2BuildTestCaseSimple(t, testCase) } -func runPythonLibraryHostTestCase(t *testing.T, tc bp2buildTestCase) { +func runPythonLibraryHostTestCase(t *testing.T, tc pythonLibBp2BuildTestCase) { t.Helper() - testCase := tc + testCase := convertPythonLibTestCaseToBp2build_Host(tc) testCase.description = fmt.Sprintf(testCase.description, "python_library_host") testCase.blueprint = fmt.Sprintf(testCase.blueprint, "python_library_host") testCase.moduleTypeUnderTest = "python_library_host" @@ -34,14 +66,14 @@ func runPythonLibraryHostTestCase(t *testing.T, tc bp2buildTestCase) { testCase) } -func runPythonLibraryTestCases(t *testing.T, tc bp2buildTestCase) { +func runPythonLibraryTestCases(t *testing.T, tc pythonLibBp2BuildTestCase) { t.Helper() runPythonLibraryTestCase(t, tc) runPythonLibraryHostTestCase(t, tc) } func TestSimplePythonLib(t *testing.T) { - testCases := []bp2buildTestCase{ + testCases := []pythonLibBp2BuildTestCase{ { description: "simple %s converts to a native py_library", filesystem: map[string]string{ @@ -64,17 +96,21 @@ func TestSimplePythonLib(t *testing.T) { srcs: ["b/e.py"], bazel_module: { bp2build_available: false }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "data": `["files/data.txt"]`, - "deps": `[":bar"]`, - "srcs": `[ + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "data": `["files/data.txt"]`, + "deps": `[":bar"]`, + "srcs": `[ "a.py", "b/c.py", "b/d.py", ]`, - "srcs_version": `"PY3"`, - }), + "srcs_version": `"PY3"`, + }, + }, }, }, { @@ -93,11 +129,15 @@ func TestSimplePythonLib(t *testing.T) { bazel_module: { bp2build_available: true }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `["a.py"]`, - "srcs_version": `"PY2"`, - }), + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + "srcs_version": `"PY2"`, + }, + }, }, }, { @@ -116,11 +156,15 @@ func TestSimplePythonLib(t *testing.T) { bazel_module: { bp2build_available: true }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `["a.py"]`, - "srcs_version": `"PY3"`, - }), + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + "srcs_version": `"PY3"`, + }, + }, }, }, { @@ -139,11 +183,15 @@ func TestSimplePythonLib(t *testing.T) { bazel_module: { bp2build_available: true }, }`, - expectedBazelTargets: []string{ - // srcs_version is PY2ANDPY3 by default. - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `["a.py"]`, - }), + expectedBazelTargets: []testBazelTarget{ + { + // srcs_version is PY2ANDPY3 by default. + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + }, + }, }, }, } @@ -156,7 +204,7 @@ func TestSimplePythonLib(t *testing.T) { } func TestPythonArchVariance(t *testing.T) { - runPythonLibraryTestCases(t, bp2buildTestCase{ + runPythonLibraryTestCases(t, pythonLibBp2BuildTestCase{ description: "test %s arch variants", filesystem: map[string]string{ "dir/arm.py": "", @@ -173,15 +221,19 @@ func TestPythonArchVariance(t *testing.T) { }, }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `select({ + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `select({ "//build/bazel/platforms/arch:arm": ["arm.py"], "//build/bazel/platforms/arch:x86": ["x86.py"], "//conditions:default": [], })`, - "srcs_version": `"PY3"`, - }), + "srcs_version": `"PY3"`, + }, + }, }, }) } diff --git a/bpf/bpf.go b/bpf/bpf.go index 187b4db61..14b2d84c5 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -20,9 +20,9 @@ import ( "strings" "android/soong/android" - _ "android/soong/cc/config" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) func init() { @@ -41,6 +41,14 @@ var ( CommandDeps: []string{"$ccCmd"}, }, "ccCmd", "cFlags") + + stripRule = pctx.AndroidStaticRule("stripRule", + blueprint.RuleParams{ + Command: `$stripCmd --strip-unneeded --remove-section=.rel.BTF ` + + `--remove-section=.rel.BTF.ext --remove-section=.BTF.ext $in -o $out`, + CommandDeps: []string{"$stripCmd"}, + }, + "stripCmd") ) func registerBpfBuildComponents(ctx android.RegistrationContext) { @@ -64,6 +72,11 @@ type BpfProperties struct { Cflags []string Include_dirs []string Sub_dir string + // If set to true, generate BTF debug info for maps & programs + Btf *bool + Vendor *bool + + VendorInternal bool `blueprint:"mutated"` } type bpf struct { @@ -74,6 +87,41 @@ type bpf struct { objs android.Paths } +var _ android.ImageInterface = (*bpf)(nil) + +func (bpf *bpf) ImageMutatorBegin(ctx android.BaseModuleContext) {} + +func (bpf *bpf) CoreVariantNeeded(ctx android.BaseModuleContext) bool { + return !proptools.Bool(bpf.properties.Vendor) +} + +func (bpf *bpf) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + +func (bpf *bpf) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + +func (bpf *bpf) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + +func (bpf *bpf) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + +func (bpf *bpf) ExtraImageVariations(ctx android.BaseModuleContext) []string { + if proptools.Bool(bpf.properties.Vendor) { + return []string{"vendor"} + } + return nil +} + +func (bpf *bpf) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { + bpf.properties.VendorInternal = variation == "vendor" +} + func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { cflags := []string{ "-nostdlibinc", @@ -99,10 +147,14 @@ func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { cflags = append(cflags, bpf.properties.Cflags...) + if proptools.Bool(bpf.properties.Btf) { + cflags = append(cflags, "-g") + } + srcs := android.PathsForModuleSrc(ctx, bpf.properties.Srcs) for _, src := range srcs { - obj := android.ObjPathWithExt(ctx, "", src, "o") + obj := android.ObjPathWithExt(ctx, "unstripped", src, "o") ctx.Build(pctx, android.BuildParams{ Rule: ccRule, @@ -114,7 +166,21 @@ func (bpf *bpf) GenerateAndroidBuildActions(ctx android.ModuleContext) { }, }) - bpf.objs = append(bpf.objs, obj.WithoutRel()) + if proptools.Bool(bpf.properties.Btf) { + objStripped := android.ObjPathWithExt(ctx, "", src, "o") + ctx.Build(pctx, android.BuildParams{ + Rule: stripRule, + Input: obj, + Output: objStripped, + Args: map[string]string{ + "stripCmd": "${config.ClangBin}/llvm-strip", + }, + }) + bpf.objs = append(bpf.objs, objStripped.WithoutRel()) + } else { + bpf.objs = append(bpf.objs, obj.WithoutRel()) + } + } } @@ -125,7 +191,12 @@ func (bpf *bpf) AndroidMk() android.AndroidMkData { fmt.Fprintln(w) fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) fmt.Fprintln(w) - localModulePath := "LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf" + var localModulePath string + if bpf.properties.VendorInternal { + localModulePath = "LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_ETC)/bpf" + } else { + localModulePath = "LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf" + } if len(bpf.properties.Sub_dir) > 0 { localModulePath += "/" + bpf.properties.Sub_dir } diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go index a9a0b1f1e..4f7d88ce6 100644 --- a/bpfix/bpfix/bpfix.go +++ b/bpfix/bpfix/bpfix.go @@ -158,6 +158,10 @@ var fixSteps = []FixStep{ Name: "formatFlagProperties", Fix: runPatchListMod(formatFlagProperties), }, + { + Name: "removeResourcesAndAssetsIfDefault", + Fix: removeResourceAndAssetsIfDefault, + }, } // for fix that only need to run once @@ -886,6 +890,24 @@ func removeSoongConfigBoolVariable(f *Fixer) error { return nil } +func removeResourceAndAssetsIfDefault(f *Fixer) error { + for _, def := range f.tree.Defs { + mod, ok := def.(*parser.Module) + if !ok { + continue + } + resourceDirList, resourceDirFound := getLiteralListPropertyValue(mod, "resource_dirs") + if resourceDirFound && len(resourceDirList) == 1 && resourceDirList[0] == "res" { + removeProperty(mod, "resource_dirs") + } + assetDirList, assetDirFound := getLiteralListPropertyValue(mod, "asset_dirs") + if assetDirFound && len(assetDirList) == 1 && assetDirList[0] == "assets" { + removeProperty(mod, "asset_dirs") + } + } + return nil +} + // Converts the default source list property, 'srcs', to a single source property with a given name. // "LOCAL_MODULE" reference is also resolved during the conversion process. func convertToSingleSource(mod *parser.Module, srcPropertyName string) { diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index 1941837bc..17b3c2444 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -19,11 +19,10 @@ package bpfix import ( "bytes" "fmt" + "reflect" "strings" "testing" - "reflect" - "github.com/google/blueprint/parser" "github.com/google/blueprint/pathtools" ) @@ -2063,3 +2062,124 @@ func TestHaveSameLicense(t *testing.T) { }) } } + +func TestRemoveResourceAndAssetsIfDefault(t *testing.T) { + tests := []struct { + name string + in string + out string + }{ + { + name: "resource_dirs default", + in: ` + android_app { + name: "foo", + resource_dirs: ["res"], + } + `, + out: ` + android_app { + name: "foo", + + } + `, + }, + { + name: "resource_dirs not default", + in: ` + android_app { + name: "foo", + resource_dirs: ["reso"], + } + `, + out: ` + android_app { + name: "foo", + resource_dirs: ["reso"], + } + `, + }, + { + name: "resource_dirs includes not default", + in: ` + android_app { + name: "foo", + resource_dirs: ["res", "reso"], + } + `, + out: ` + android_app { + name: "foo", + resource_dirs: ["res", "reso"], + } + `, + }, { + name: "asset_dirs default", + in: ` + android_app { + name: "foo", + asset_dirs: ["assets"], + } + `, + out: ` + android_app { + name: "foo", + + } + `, + }, + { + name: "asset_dirs not default", + in: ` + android_app { + name: "foo", + asset_dirs: ["assety"], + } + `, + out: ` + android_app { + name: "foo", + asset_dirs: ["assety"], + } + `, + }, + { + name: "asset_dirs includes not default", + in: ` + android_app { + name: "foo", + asset_dirs: ["assets", "assety"], + } + `, + out: ` + android_app { + name: "foo", + asset_dirs: ["assets", "assety"], + } + `, + }, + { + name: "resource_dirs and asset_dirs both default", + in: ` + android_app { + name: "foo", + asset_dirs: ["assets"], + resource_dirs: ["res"], + } + `, + out: ` + android_app { + name: "foo", + + } + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runPassOnce(t, test.in, test.out, func(fixer *Fixer) error { + return removeResourceAndAssetsIfDefault(fixer) + }) + }) + } +} diff --git a/cc/Android.bp b/cc/Android.bp index cf4563070..9103a48b4 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -88,6 +88,7 @@ bootstrap_go_package { "stub_library.go", ], testSrcs: [ + "afdo_test.go", "cc_test.go", "compiler_test.go", "gen_test.go", @@ -1,4 +1,4 @@ per-file ndk_*.go = danalbert@google.com per-file tidy.go = srhines@google.com, chh@google.com -per-file afdo.go,lto.go,pgo.go = srhines@google.com, pirama@google.com, yikong@google.com - +per-file afdo.go,afdo_test.go,lto.go,pgo.go = srhines@google.com, pirama@google.com, yikong@google.com +per-file coverage.go = pirama@google.com, srhines@google.com, allenhair@google.com diff --git a/cc/afdo_test.go b/cc/afdo_test.go new file mode 100644 index 000000000..551546424 --- /dev/null +++ b/cc/afdo_test.go @@ -0,0 +1,70 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "testing" + + "android/soong/android" + "github.com/google/blueprint" +) + +func TestAfdoDeps(t *testing.T) { + bp := ` + cc_library { + name: "libTest", + srcs: ["foo.c"], + static_libs: ["libFoo"], + afdo: true, + } + + cc_library { + name: "libFoo", + static_libs: ["libBar"], + } + + cc_library { + name: "libBar", + } + ` + prepareForAfdoTest := android.FixtureAddTextFile("toolchain/pgo-profiles/sampling/libTest.afdo", "TEST") + + result := android.GroupFixturePreparers( + prepareForCcTest, + prepareForAfdoTest, + ).RunTestWithBp(t, bp) + + libTest := result.ModuleForTests("libTest", "android_arm64_armv8-a_shared").Module() + libFoo := result.ModuleForTests("libFoo", "android_arm64_armv8-a_static_afdo-libTest").Module() + libBar := result.ModuleForTests("libBar", "android_arm64_armv8-a_static_afdo-libTest").Module() + + hasDep := func(m android.Module, wantDep android.Module) bool { + var found bool + result.VisitDirectDeps(m, func(dep blueprint.Module) { + if dep == wantDep { + found = true + } + }) + return found + } + + if !hasDep(libTest, libFoo) { + t.Errorf("libTest missing dependency on afdo variant of libFoo") + } + + if !hasDep(libFoo, libBar) { + t.Errorf("libTest missing dependency on afdo variant of libBar") + } +} diff --git a/cc/androidmk.go b/cc/androidmk.go index b56d689d6..715490572 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -108,6 +108,9 @@ func (c *Module) AndroidMkEntries() []android.AndroidMkEntries { if len(c.Properties.AndroidMkHeaderLibs) > 0 { entries.AddStrings("LOCAL_HEADER_LIBRARIES", c.Properties.AndroidMkHeaderLibs...) } + if len(c.Properties.AndroidMkRuntimeLibs) > 0 { + entries.AddStrings("LOCAL_RUNTIME_LIBRARIES", c.Properties.AndroidMkRuntimeLibs...) + } entries.SetString("LOCAL_SOONG_LINK_TYPE", c.makeLinkType) if c.UseVndk() { entries.SetBool("LOCAL_USE_VNDK", true) diff --git a/cc/binary.go b/cc/binary.go index ee3de3f5a..54fd339c2 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -76,7 +76,6 @@ func BinaryFactory() android.Module { // cc_binary_host produces a binary that is runnable on a host. func BinaryHostFactory() android.Module { module, _ := newBinary(android.HostSupported, true) - module.bazelable = true return module.Init() } @@ -606,19 +605,12 @@ func binaryBp2build(ctx android.TopDownMutatorContext, m *Module, typ string) { Features: baseAttrs.features, } - var enabledProperty bazel.BoolAttribute - if typ == "cc_binary_host" { - falseVal := false - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, android.Android.Name, &falseVal) - } - - ctx.CreateBazelTargetModuleWithRestrictions(bazel.BazelTargetModuleProperties{ + ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ Rule_class: "cc_binary", - Bzl_load_location: "//build/bazel/rules:cc_binary.bzl", + Bzl_load_location: "//build/bazel/rules/cc:cc_binary.bzl", }, android.CommonAttributes{Name: m.Name()}, - attrs, - enabledProperty) + attrs) } // binaryAttributes contains Bazel attributes corresponding to a cc binary diff --git a/cc/bp2build.go b/cc/bp2build.go index c5eab0683..42fc0e494 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -644,7 +644,7 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion var linkerFlags []string if len(props.Ldflags) > 0 { - linkerFlags = append(linkerFlags, props.Ldflags...) + linkerFlags = append(linkerFlags, proptools.NinjaEscapeList(props.Ldflags)...) // binaries remove static flag if -shared is in the linker flags if isBinary && android.InList("-shared", linkerFlags) { axisFeatures = append(axisFeatures, "-static_flag") @@ -844,10 +844,8 @@ func bp2BuildParseExportedIncludesHelper(ctx android.BazelConversionPathContext, func bazelLabelForStaticModule(ctx android.BazelConversionPathContext, m blueprint.Module) string { label := android.BazelModuleLabel(ctx, m) - if aModule, ok := m.(android.Module); ok { - if ctx.OtherModuleType(aModule) == "cc_library" && !android.GenerateCcLibraryStaticOnly(m.Name()) { - label += "_bp2build_cc_library_static" - } + if ccModule, ok := m.(*Module); ok && ccModule.typ() == fullLibrary && !android.GenerateCcLibraryStaticOnly(m.Name()) { + label += "_bp2build_cc_library_static" } return label } diff --git a/cc/builder.go b/cc/builder.go index 512f83885..ee3ea2dd1 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -203,25 +203,34 @@ var ( "clangBin", "format") // Rule for invoking clang-tidy (a clang-based linter). + clangTidyDep, clangTidyDepRE = pctx.RemoteStaticRules("clangTidyDep", + blueprint.RuleParams{ + Depfile: "$out", + Deps: blueprint.DepsGCC, + Command: "${config.CcWrapper}$ccCmd $cFlags -E -o /dev/null $in " + + "-MQ $tidyFile -MD -MF $out", + CommandDeps: []string{"$ccCmd"}, + }, + &remoteexec.REParams{ + Labels: map[string]string{"type": "lint", "tool": "clang-tidy", "lang": "cpp"}, + ExecStrategy: "${config.REClangTidyExecStrategy}", + Inputs: []string{"$in"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REClangTidyPool}"}, + }, []string{"ccCmd", "cFlags", "tidyFile"}, []string{}) + clangTidy, clangTidyRE = pctx.RemoteStaticRules("clangTidy", blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, - // Pick bash because some machines with old /bin/sh cannot handle arrays. - // All $cFlags and $tidyFlags should have single quotes escaped. - // Assume no single quotes in other parameters like $in, $out, $ccCmd. - Command: "/bin/bash -c 'SRCF=$in; TIDYF=$out; CLANGFLAGS=($cFlags); " + - "rm -f $$TIDYF $${TIDYF}.d && " + - "${config.CcWrapper}$ccCmd \"$${CLANGFLAGS[@]}\" -E -o /dev/null $$SRCF " + - "-MQ $$TIDYF -MD -MF $${TIDYF}.d && " + - "$tidyVars $reTemplate${config.ClangBin}/clang-tidy $tidyFlags $$SRCF " + - "-- \"$${CLANGFLAGS[@]}\" && touch $$TIDYF'", - CommandDeps: []string{"${config.ClangBin}/clang-tidy", "$ccCmd"}, + Command: "cp ${out}.dep ${out}.d && " + + "$tidyVars$reTemplate${config.ClangBin}/clang-tidy $tidyFlags $in -- $cFlags && " + + "touch $out", + CommandDeps: []string{"${config.ClangBin}/clang-tidy"}, }, &remoteexec.REParams{ Labels: map[string]string{"type": "lint", "tool": "clang-tidy", "lang": "cpp"}, ExecStrategy: "${config.REClangTidyExecStrategy}", - Inputs: []string{"$in"}, + Inputs: []string{"$in", "${out}.dep"}, EnvironmentVariables: []string{"TIDY_TIMEOUT"}, // Although clang-tidy has an option to "fix" source files, that feature is hardly useable // under parallel compilation and RBE. So we assume no OutputFiles here. @@ -230,7 +239,7 @@ var ( // (1) New timestamps trigger clang and clang-tidy compilations again. // (2) Changing source files caused concurrent clang or clang-tidy jobs to crash. Platform: map[string]string{remoteexec.PoolKey: "${config.REClangTidyPool}"}, - }, []string{"ccCmd", "cFlags", "tidyFlags", "tidyVars"}, []string{}) + }, []string{"cFlags", "tidyFlags", "tidyVars"}, []string{}) _ = pctx.SourcePathVariable("yasmCmd", "prebuilts/misc/${config.HostPrebuiltTag}/yasm/yasm") @@ -449,12 +458,6 @@ func (a Objects) Append(b Objects) Objects { } } -func escapeSingleQuotes(s string) string { - // Replace single quotes to work when embedded in a single quoted string for bash. - // Relying on string concatenation of bash to get A'B from quoted 'A'\''B'. - return strings.Replace(s, `'`, `'\''`, -1) -} - // Generate rules for compiling multiple .c, .cpp, or .S files to individual .o files func transformSourceToObj(ctx ModuleContext, subdir string, srcFiles, noTidySrcs android.Paths, flags builderFlags, pathDeps android.Paths, cFlagsDeps android.Paths) Objects { @@ -470,7 +473,7 @@ func transformSourceToObj(ctx ModuleContext, subdir string, srcFiles, noTidySrcs } tidyTimeout := ctx.Config().Getenv("TIDY_TIMEOUT") if len(tidyTimeout) > 0 { - tidyVars += "TIDY_TIMEOUT=" + tidyTimeout + tidyVars += "TIDY_TIMEOUT=" + tidyTimeout + " " } } var coverageFiles android.Paths @@ -674,24 +677,44 @@ func transformSourceToObj(ctx ModuleContext, subdir string, srcFiles, noTidySrcs // Even with tidy, some src file could be skipped by noTidySrcsMap. if tidy && !noTidySrcsMap[srcFile] { tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy") + tidyDepFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy.dep") tidyFiles = append(tidyFiles, tidyFile) + ruleDep := clangTidyDep rule := clangTidy if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CLANG_TIDY") { + ruleDep = clangTidyDepRE rule = clangTidyRE } + sharedCFlags := shareFlags("cFlags", moduleFlags) + srcRelPath := srcFile.Rel() + + // Add the .tidy.d rule + ctx.Build(pctx, android.BuildParams{ + Rule: ruleDep, + Description: "clang-tidy-dep " + srcRelPath, + Output: tidyDepFile, + Input: srcFile, + Implicits: cFlagsDeps, + OrderOnly: pathDeps, + Args: map[string]string{ + "ccCmd": ccCmd, + "cFlags": sharedCFlags, + "tidyFile": tidyFile.String(), + }, + }) + // Add the .tidy rule with order only dependency on the .tidy.d file ctx.Build(pctx, android.BuildParams{ Rule: rule, - Description: "clang-tidy " + srcFile.Rel(), + Description: "clang-tidy " + srcRelPath, Output: tidyFile, Input: srcFile, Implicits: cFlagsDeps, - OrderOnly: pathDeps, + OrderOnly: append(android.Paths{}, tidyDepFile), Args: map[string]string{ - "ccCmd": ccCmd, - "cFlags": shareFlags("cFlags", escapeSingleQuotes(moduleToolingFlags)), - "tidyFlags": shareFlags("tidyFlags", escapeSingleQuotes(config.TidyFlagsForSrcFile(srcFile, flags.tidyFlags))), + "cFlags": sharedCFlags, + "tidyFlags": shareFlags("tidyFlags", config.TidyFlagsForSrcFile(srcFile, flags.tidyFlags)), "tidyVars": tidyVars, // short and not shared }, }) @@ -744,7 +767,7 @@ func transformObjToStaticLib(ctx android.ModuleContext, arCmd := "${config.ClangBin}/llvm-ar" arFlags := "" if !ctx.Darwin() { - arFlags += " -format=gnu" + arFlags += " --format=gnu" } if len(wholeStaticLibs) == 0 { @@ -1751,7 +1751,7 @@ func (c *Module) setSubnameProperty(actx android.ModuleContext) { // Returns true if Bazel was successfully used for the analysis of this module. func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { var bazelModuleLabel string - if actx.ModuleType() == "cc_library" && c.static() { + if c.typ() == fullLibrary && c.static() { // cc_library is a special case in bp2build; two targets are generated -- one for each // of the shared and static variants. The shared variant keeps the module name, but the // static variant uses a different suffixed name. @@ -1759,6 +1759,7 @@ func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { } else { bazelModuleLabel = c.GetBazelLabel(actx, c) } + bazelActionsUsed := false // Mixed builds mode is disabled for modules outside of device OS. // TODO(b/200841190): Support non-device OS in mixed builds. @@ -3464,19 +3465,33 @@ func (c *Module) AlwaysRequiresPlatformApexVariant() bool { return c.IsStubs() || c.Target().NativeBridge == android.NativeBridgeEnabled } +// Overrides android.ApexModuleBase.UniqueApexVariations +func (c *Module) UniqueApexVariations() bool { + // When a vendor APEX needs a VNDK lib in it (use_vndk_as_stable: false), it should be a unique + // APEX variation. Otherwise, another vendor APEX with use_vndk_as_stable:true may use a wrong + // variation of the VNDK lib because APEX variations are merged/grouped. + return c.UseVndk() && c.IsVndk() +} + var _ snapshot.RelativeInstallPath = (*Module)(nil) -// ConvertWithBp2build converts Module to Bazel for bp2build. -func (c *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - prebuilt := c.IsPrebuilt() +type moduleType int + +const ( + unknownType moduleType = iota + binary + object + fullLibrary + staticLibrary + sharedLibrary + headerLibrary +) + +func (c *Module) typ() moduleType { if c.Binary() { - if !prebuilt { - binaryBp2build(ctx, c, ctx.ModuleType()) - } + return binary } else if c.Object() { - if !prebuilt { - objectBp2Build(ctx, c) - } + return object } else if c.CcLibrary() { static := false shared := false @@ -3487,25 +3502,48 @@ func (c *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { static = library.MutatedProperties.BuildStatic shared = library.MutatedProperties.BuildShared } - if static && shared { - if !prebuilt { - libraryBp2Build(ctx, c) - } + return fullLibrary } else if !static && !shared { - libraryHeadersBp2Build(ctx, c) + return headerLibrary } else if static { - if prebuilt { - prebuiltLibraryStaticBp2Build(ctx, c) - } else { - sharedOrStaticLibraryBp2Build(ctx, c, true) - } - } else if shared { - if prebuilt { - prebuiltLibrarySharedBp2Build(ctx, c) - } else { - sharedOrStaticLibraryBp2Build(ctx, c, false) - } + return staticLibrary + } + return sharedLibrary + } + return unknownType +} + +// ConvertWithBp2build converts Module to Bazel for bp2build. +func (c *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + prebuilt := c.IsPrebuilt() + switch c.typ() { + case binary: + if !prebuilt { + binaryBp2build(ctx, c, ctx.ModuleType()) + } + case object: + if !prebuilt { + objectBp2Build(ctx, c) + } + case fullLibrary: + if !prebuilt { + libraryBp2Build(ctx, c) + } + case headerLibrary: + libraryHeadersBp2Build(ctx, c) + case staticLibrary: + + if prebuilt { + prebuiltLibraryStaticBp2Build(ctx, c) + } else { + sharedOrStaticLibraryBp2Build(ctx, c, true) + } + case sharedLibrary: + if prebuilt { + prebuiltLibrarySharedBp2Build(ctx, c) + } else { + sharedOrStaticLibraryBp2Build(ctx, c, false) } } } diff --git a/cc/config/Android.bp b/cc/config/Android.bp index 7b7ee2849..e1b06057b 100644 --- a/cc/config/Android.bp +++ b/cc/config/Android.bp @@ -8,6 +8,7 @@ bootstrap_go_package { deps: [ "soong-android", "soong-remoteexec", + "soong-starlark-format", ], srcs: [ "bp2build.go", diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index 2d6bcb89d..979c82525 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -33,7 +33,9 @@ var ( }, "armv8-a-branchprot": []string{ "-march=armv8-a", - "-mbranch-protection=standard", + // Disable BTI until drm vendors stop using OS libraries as sources + // of gadgets (https://issuetracker.google.com/216395195). + "-mbranch-protection=pac-ret", }, "armv8-2a": []string{ "-march=armv8.2-a", diff --git a/cc/config/arm64_linux_host.go b/cc/config/arm64_linux_host.go index 853d818f1..5c7f926f8 100644 --- a/cc/config/arm64_linux_host.go +++ b/cc/config/arm64_linux_host.go @@ -41,7 +41,6 @@ var ( "-Wl,-z,relro", "-Wl,-z,now", "-Wl,--build-id=md5", - "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", "-Wl,--hash-style=gnu", "-Wl,--no-undefined-version", diff --git a/cc/config/bp2build.go b/cc/config/bp2build.go index 982b43648..eca516107 100644 --- a/cc/config/bp2build.go +++ b/cc/config/bp2build.go @@ -22,14 +22,11 @@ import ( "strings" "android/soong/android" + "android/soong/starlark_fmt" "github.com/google/blueprint" ) -const ( - bazelIndent = 4 -) - type bazelVarExporter interface { asBazel(android.Config, exportedStringVariables, exportedStringListVariables, exportedConfigDependingVariables) []bazelConstant } @@ -73,21 +70,6 @@ func (m exportedStringVariables) Set(k string, v string) { m[k] = v } -func bazelIndention(level int) string { - return strings.Repeat(" ", level*bazelIndent) -} - -func printBazelList(items []string, indentLevel int) string { - list := make([]string, 0, len(items)+2) - list = append(list, "[") - innerIndent := bazelIndention(indentLevel + 1) - for _, item := range items { - list = append(list, fmt.Sprintf(`%s"%s",`, innerIndent, item)) - } - list = append(list, bazelIndention(indentLevel)+"]") - return strings.Join(list, "\n") -} - func (m exportedStringVariables) asBazel(config android.Config, stringVars exportedStringVariables, stringListVars exportedStringListVariables, cfgDepVars exportedConfigDependingVariables) []bazelConstant { ret := make([]bazelConstant, 0, len(m)) @@ -139,7 +121,7 @@ func (m exportedStringListVariables) asBazel(config android.Config, // out through a constants struct later. ret = append(ret, bazelConstant{ variableName: k, - internalDefinition: printBazelList(expandedVars, 0), + internalDefinition: starlark_fmt.PrintStringList(expandedVars, 0), }) } return ret @@ -173,17 +155,6 @@ func (m exportedStringListDictVariables) Set(k string, v map[string][]string) { m[k] = v } -func printBazelStringListDict(dict map[string][]string) string { - bazelDict := make([]string, 0, len(dict)+2) - bazelDict = append(bazelDict, "{") - for k, v := range dict { - bazelDict = append(bazelDict, - fmt.Sprintf(`%s"%s": %s,`, bazelIndention(1), k, printBazelList(v, 1))) - } - bazelDict = append(bazelDict, "}") - return strings.Join(bazelDict, "\n") -} - // Since dictionaries are not supported in Ninja, we do not expand variables for dictionaries func (m exportedStringListDictVariables) asBazel(_ android.Config, _ exportedStringVariables, _ exportedStringListVariables, _ exportedConfigDependingVariables) []bazelConstant { @@ -191,7 +162,7 @@ func (m exportedStringListDictVariables) asBazel(_ android.Config, _ exportedStr for k, dict := range m { ret = append(ret, bazelConstant{ variableName: k, - internalDefinition: printBazelStringListDict(dict), + internalDefinition: starlark_fmt.PrintStringListDict(dict, 0), }) } return ret @@ -223,7 +194,7 @@ func bazelToolchainVars(config android.Config, vars ...bazelVarExporter) string definitions = append(definitions, fmt.Sprintf("_%s = %s", b.variableName, b.internalDefinition)) constants = append(constants, - fmt.Sprintf("%[1]s%[2]s = _%[2]s,", bazelIndention(1), b.variableName)) + fmt.Sprintf("%[1]s%[2]s = _%[2]s,", starlark_fmt.Indention(1), b.variableName)) } // Build the exported constants struct. diff --git a/cc/config/bp2build_test.go b/cc/config/bp2build_test.go index 3118df1f8..4cbf0c6f3 100644 --- a/cc/config/bp2build_test.go +++ b/cc/config/bp2build_test.go @@ -211,15 +211,11 @@ constants = struct( expectedOut: `# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT. _a = { - "b1": [ - "b2", - ], + "b1": ["b2"], } _c = { - "d1": [ - "d2", - ], + "d1": ["d2"], } constants = struct( @@ -246,27 +242,19 @@ constants = struct( expectedOut: `# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT. _a = { - "a1": [ - "a2", - ], + "a1": ["a2"], } _b = "b-val" -_c = [ - "c-val", -] +_c = ["c-val"] _d = "d-val" -_e = [ - "e-val", -] +_e = ["e-val"] _f = { - "f1": [ - "f2", - ], + "f1": ["f2"], } constants = struct( diff --git a/cc/config/global.go b/cc/config/global.go index 5acc7f52c..48a8b4805 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -95,6 +95,9 @@ var ( // Nested and array designated initialization is nice to have. "-Wno-c99-designator", + // Many old files still have GNU designator syntax. + "-Wno-gnu-designator", + // Warnings from clang-12 "-Wno-gnu-folding-constant", @@ -139,7 +142,6 @@ var ( "-Wl,-z,relro", "-Wl,-z,now", "-Wl,--build-id=md5", - "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", "-Wl,--no-undefined-version", // TODO: Eventually we should link against a libunwind.a with hidden symbols, and then these diff --git a/cc/config/tidy.go b/cc/config/tidy.go index fdc246cdf..ba1043b0d 100644 --- a/cc/config/tidy.go +++ b/cc/config/tidy.go @@ -62,8 +62,9 @@ func init() { }, ",") // clang-analyzer-* checks are too slow to be in the default for WITH_TIDY=1. // nightly builds add CLANG_ANALYZER_CHECKS=1 to run those checks. + // The insecureAPI.DeprecatedOrUnsafeBufferHandling warning does not apply to Android. if ctx.Config().IsEnvTrue("CLANG_ANALYZER_CHECKS") { - checks += ",clang-analyzer-*" + checks += ",clang-analyzer-*,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling" } return checks }) diff --git a/cc/config/x86_64_device.go b/cc/config/x86_64_device.go index 00f07ff26..d789cde5c 100644 --- a/cc/config/x86_64_device.go +++ b/cc/config/x86_64_device.go @@ -15,6 +15,7 @@ package config import ( + "fmt" "strings" "android/soong/android" @@ -190,6 +191,11 @@ func (toolchainX86_64) LibclangRuntimeLibraryArch() string { } func x86_64ToolchainFactory(arch android.Arch) Toolchain { + // Error now rather than having a confusing Ninja error + if _, ok := x86_64ArchVariantCflags[arch.ArchVariant]; !ok { + panic(fmt.Sprintf("Unknown x86_64 architecture version: %q", arch.ArchVariant)) + } + toolchainCflags := []string{ "${config.X86_64ToolchainCflags}", "${config.X86_64" + arch.ArchVariant + "VariantCflags}", diff --git a/cc/config/x86_device.go b/cc/config/x86_device.go index 29f059303..e32e1bde7 100644 --- a/cc/config/x86_device.go +++ b/cc/config/x86_device.go @@ -15,6 +15,7 @@ package config import ( + "fmt" "strings" "android/soong/android" @@ -186,6 +187,11 @@ func (toolchainX86) LibclangRuntimeLibraryArch() string { } func x86ToolchainFactory(arch android.Arch) Toolchain { + // Error now rather than having a confusing Ninja error + if _, ok := x86ArchVariantCflags[arch.ArchVariant]; !ok { + panic(fmt.Sprintf("Unknown x86 architecture version: %q", arch.ArchVariant)) + } + toolchainCflags := []string{ "${config.X86ToolchainCflags}", "${config.X86" + arch.ArchVariant + "VariantCflags}", diff --git a/cc/config/x86_linux_bionic_host.go b/cc/config/x86_linux_bionic_host.go index 4b7ba6a85..976cc25be 100644 --- a/cc/config/x86_linux_bionic_host.go +++ b/cc/config/x86_linux_bionic_host.go @@ -47,7 +47,6 @@ var ( "-Wl,-z,relro", "-Wl,-z,now", "-Wl,--build-id=md5", - "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", "-Wl,--hash-style=gnu", "-Wl,--no-undefined-version", diff --git a/cc/config/x86_linux_host.go b/cc/config/x86_linux_host.go index 43333fada..60f03c2c9 100644 --- a/cc/config/x86_linux_host.go +++ b/cc/config/x86_linux_host.go @@ -194,10 +194,6 @@ func (t *toolchainLinux) IncludeFlags() string { return "" } -func (t *toolchainLinuxX86) ClangTriple() string { - return "i686-linux-gnu" -} - func (t *toolchainLinuxX86) Cflags() string { return "${config.LinuxCflags} ${config.LinuxX86Cflags}" } @@ -206,10 +202,6 @@ func (t *toolchainLinuxX86) Cppflags() string { return "" } -func (t *toolchainLinuxX8664) ClangTriple() string { - return "x86_64-linux-gnu" -} - func (t *toolchainLinuxX8664) Cflags() string { return "${config.LinuxCflags} ${config.LinuxX8664Cflags}" } @@ -283,6 +275,10 @@ type toolchainLinuxGlibcX8664 struct { toolchainGlibc } +func (t *toolchainLinuxGlibcX86) ClangTriple() string { + return "i686-linux-gnu" +} + func (t *toolchainLinuxGlibcX86) Cflags() string { return t.toolchainLinuxX86.Cflags() + " " + t.toolchainGlibc.Cflags() } @@ -295,6 +291,10 @@ func (t *toolchainLinuxGlibcX86) Lldflags() string { return t.toolchainLinuxX86.Lldflags() + " " + t.toolchainGlibc.Lldflags() } +func (t *toolchainLinuxGlibcX8664) ClangTriple() string { + return "x86_64-linux-gnu" +} + func (t *toolchainLinuxGlibcX8664) Cflags() string { return t.toolchainLinuxX8664.Cflags() + " " + t.toolchainGlibc.Cflags() } @@ -356,6 +356,10 @@ type toolchainLinuxMuslX8664 struct { toolchainMusl } +func (t *toolchainLinuxMuslX86) ClangTriple() string { + return "i686-linux-musl" +} + func (t *toolchainLinuxMuslX86) Cflags() string { return t.toolchainLinuxX86.Cflags() + " " + t.toolchainMusl.Cflags() } @@ -368,6 +372,10 @@ func (t *toolchainLinuxMuslX86) Lldflags() string { return t.toolchainLinuxX86.Lldflags() + " " + t.toolchainMusl.Lldflags() } +func (t *toolchainLinuxMuslX8664) ClangTriple() string { + return "x86_64-linux-musl" +} + func (t *toolchainLinuxMuslX8664) Cflags() string { return t.toolchainLinuxX8664.Cflags() + " " + t.toolchainMusl.Cflags() } diff --git a/cc/coverage.go b/cc/coverage.go index 8dd2db19d..f2b5425e6 100644 --- a/cc/coverage.go +++ b/cc/coverage.go @@ -98,6 +98,9 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags } else if clangCoverage { flags.Local.CommonFlags = append(flags.Local.CommonFlags, profileInstrFlag, "-fcoverage-mapping", "-Wno-pass-failed", "-D__ANDROID_CLANG_COVERAGE__") + // Override -Wframe-larger-than. We can expect frame size increase after + // coverage instrumentation. + flags.Local.CFlags = append(flags.Local.CFlags, "-Wno-frame-larger-than=") } } diff --git a/cc/library.go b/cc/library.go index cefbf6c44..708aa1045 100644 --- a/cc/library.go +++ b/cc/library.go @@ -32,7 +32,7 @@ import ( "github.com/google/blueprint/pathtools" ) -// LibraryProperties is a collection of properties shared by cc library rules. +// LibraryProperties is a collection of properties shared by cc library rules/cc. type LibraryProperties struct { // local file name to pass to the linker as -unexported_symbols_list Unexported_symbols_list *string `android:"path,arch_variant"` @@ -386,13 +386,28 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { Stubs_versions: compilerAttrs.stubsVersions, } + for axis, configToProps := range m.GetArchVariantProperties(ctx, &LibraryProperties{}) { + for config, props := range configToProps { + if props, ok := props.(*LibraryProperties); ok { + if props.Inject_bssl_hash != nil { + // This is an edge case applies only to libcrypto + if m.Name() == "libcrypto" { + sharedTargetAttrs.Inject_bssl_hash.SetSelectValue(axis, config, props.Inject_bssl_hash) + } else { + ctx.PropertyErrorf("inject_bssl_hash", "only applies to libcrypto") + } + } + } + } + } + staticProps := bazel.BazelTargetModuleProperties{ Rule_class: "cc_library_static", - Bzl_load_location: "//build/bazel/rules:cc_library_static.bzl", + Bzl_load_location: "//build/bazel/rules/cc:cc_library_static.bzl", } sharedProps := bazel.BazelTargetModuleProperties{ Rule_class: "cc_library_shared", - Bzl_load_location: "//build/bazel/rules:cc_library_shared.bzl", + Bzl_load_location: "//build/bazel/rules/cc:cc_library_shared.bzl", } ctx.CreateBazelTargetModuleWithRestrictions(staticProps, @@ -757,9 +772,10 @@ func GlobHeadersForSnapshot(ctx android.ModuleContext, paths android.Paths) andr if dir == "external/eigen" { // Only these two directories contains exported headers. for _, subdir := range []string{"Eigen", "unsupported/Eigen"} { - glob, err := ctx.GlobWithDeps("external/eigen/"+subdir+"/**/*", nil) + globDir := "external/eigen/" + subdir + "/**/*" + glob, err := ctx.GlobWithDeps(globDir, nil) if err != nil { - ctx.ModuleErrorf("glob failed: %#v", err) + ctx.ModuleErrorf("glob of %q failed: %s", globDir, err) return nil } for _, header := range glob { @@ -775,9 +791,10 @@ func GlobHeadersForSnapshot(ctx android.ModuleContext, paths android.Paths) andr } continue } - glob, err := ctx.GlobWithDeps(dir+"/**/*", nil) + globDir := dir + "/**/*" + glob, err := ctx.GlobWithDeps(globDir, nil) if err != nil { - ctx.ModuleErrorf("glob failed: %#v", err) + ctx.ModuleErrorf("glob of %q failed: %s", globDir, err) return nil } isLibcxx := strings.HasPrefix(dir, "external/libcxx/include") @@ -2535,7 +2552,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo } props := bazel.BazelTargetModuleProperties{ Rule_class: modType, - Bzl_load_location: fmt.Sprintf("//build/bazel/rules:%s.bzl", modType), + Bzl_load_location: fmt.Sprintf("//build/bazel/rules/cc:%s.bzl", modType), } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs) @@ -2600,4 +2617,5 @@ type bazelCcLibrarySharedAttributes struct { Stubs_symbol_file *string Stubs_versions bazel.StringListAttribute + Inject_bssl_hash bazel.BoolAttribute } diff --git a/cc/library_headers.go b/cc/library_headers.go index 064e2b807..5d38fba03 100644 --- a/cc/library_headers.go +++ b/cc/library_headers.go @@ -136,7 +136,7 @@ func libraryHeadersBp2Build(ctx android.TopDownMutatorContext, module *Module) { props := bazel.BazelTargetModuleProperties{ Rule_class: "cc_library_headers", - Bzl_load_location: "//build/bazel/rules:cc_library_headers.bzl", + Bzl_load_location: "//build/bazel/rules/cc:cc_library_headers.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs) diff --git a/cc/object.go b/cc/object.go index 24f6ed455..fdd0b113c 100644 --- a/cc/object.go +++ b/cc/object.go @@ -195,7 +195,7 @@ func objectBp2Build(ctx android.TopDownMutatorContext, m *Module) { props := bazel.BazelTargetModuleProperties{ Rule_class: "cc_object", - Bzl_load_location: "//build/bazel/rules:cc_object.bzl", + Bzl_load_location: "//build/bazel/rules/cc:cc_object.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs) @@ -208,6 +208,10 @@ func (props *PgoProperties) isPGO(ctx BaseModuleContext) bool { ctx.ModuleErrorf("Instrumentation PGO specification is missing benchmark property") } + if isSampling { + ctx.ModuleErrorf("Sampling PGO is deprecated, use AFDO instead") + } + if isSampling && isInstrumentation { ctx.PropertyErrorf("pgo", "Exactly one of \"instrumentation\" and \"sampling\" properties must be set") } diff --git a/cc/prebuilt.go b/cc/prebuilt.go index c928ed960..339a16d9d 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -339,7 +339,7 @@ func prebuiltLibraryStaticBp2Build(ctx android.TopDownMutatorContext, module *Mo props := bazel.BazelTargetModuleProperties{ Rule_class: "prebuilt_library_static", - Bzl_load_location: "//build/bazel/rules:prebuilt_library_static.bzl", + Bzl_load_location: "//build/bazel/rules/cc:prebuilt_library_static.bzl", } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) @@ -359,7 +359,7 @@ func prebuiltLibrarySharedBp2Build(ctx android.TopDownMutatorContext, module *Mo props := bazel.BazelTargetModuleProperties{ Rule_class: "prebuilt_library_shared", - Bzl_load_location: "//build/bazel/rules:prebuilt_library_shared.bzl", + Bzl_load_location: "//build/bazel/rules/cc:prebuilt_library_shared.bzl", } name := android.RemoveOptionalPrebuiltPrefix(module.Name()) diff --git a/cc/proto.go b/cc/proto.go index f3410bc2b..3cf1453c8 100644 --- a/cc/proto.go +++ b/cc/proto.go @@ -210,7 +210,7 @@ func bp2buildProto(ctx android.Bp2buildMutatorContext, m *Module, protoSrcs baze ctx.CreateBazelTargetModule( bazel.BazelTargetModuleProperties{ Rule_class: rule_class, - Bzl_load_location: "//build/bazel/rules:cc_proto.bzl", + Bzl_load_location: "//build/bazel/rules/cc:cc_proto.bzl", }, android.CommonAttributes{Name: name}, &protoAttrs) diff --git a/cc/sanitize.go b/cc/sanitize.go index a054c9148..b8e1468e4 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -42,7 +42,6 @@ var ( "-fno-omit-frame-pointer", "-Wno-frame-larger-than=", "-fsanitize-hwaddress-abi=platform", - "-mllvm", "-hwasan-use-after-scope=1", } // ThinLTO performs codegen during link time, thus these flags need to @@ -706,19 +705,22 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { if len(sanitize.Properties.Sanitizers) > 0 { sanitizeArg := "-fsanitize=" + strings.Join(sanitize.Properties.Sanitizers, ",") - flags.Local.CFlags = append(flags.Local.CFlags, sanitizeArg) flags.Local.AsFlags = append(flags.Local.AsFlags, sanitizeArg) - if ctx.Host() { + flags.Local.LdFlags = append(flags.Local.LdFlags, sanitizeArg) + + if ctx.toolchain().Bionic() { + // Bionic sanitizer runtimes have already been added as dependencies so that + // the right variant of the runtime will be used (with the "-android" + // suffix), so don't let clang the runtime library. + flags.Local.LdFlags = append(flags.Local.LdFlags, "-fno-sanitize-link-runtime") + } else { // Host sanitizers only link symbols in the final executable, so // there will always be undefined symbols in intermediate libraries. _, flags.Global.LdFlags = removeFromList("-Wl,--no-undefined", flags.Global.LdFlags) - flags.Local.LdFlags = append(flags.Local.LdFlags, sanitizeArg) - // non-Bionic toolchain prebuilts are missing UBSan's vptr and function sanitizers - if !ctx.toolchain().Bionic() { - flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=vptr,function") - } + // non-Bionic toolchain prebuilts are missing UBSan's vptr and function san + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=vptr,function") } if enableMinimalRuntime(sanitize) { diff --git a/cc/tidy.go b/cc/tidy.go index 97418fe17..750e9de1e 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -102,6 +102,12 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { } flags.TidyFlags = append(flags.TidyFlags, headerFilter) } + // Work around RBE bug in parsing clang-tidy flags, replace "--flag" with "-flag". + // Some C/C++ modules added local tidy flags like --header-filter= and --extra-arg-before=. + doubleDash := regexp.MustCompile("^('?)--(.*)$") + for i, s := range flags.TidyFlags { + flags.TidyFlags[i] = doubleDash.ReplaceAllString(s, "$1-$2") + } // If clang-tidy is not enabled globally, add the -quiet flag. if !ctx.Config().ClangTidy() { @@ -156,6 +162,9 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { // Too many existing functions trigger this rule, and fixing it requires large code // refactoring. The cost of maintaining this tidy rule outweighs the benefit it brings. tidyChecks = tidyChecks + ",-bugprone-easily-swappable-parameters" + // http://b/216364337 - TODO: Follow-up after compiler update to + // disable or fix individual instances. + tidyChecks = tidyChecks + ",-cert-err33-c" flags.TidyFlags = append(flags.TidyFlags, tidyChecks) if ctx.Config().IsEnvTrue("WITH_TIDY") { diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index f07eafabe..b3a6ee07d 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -48,6 +48,7 @@ var ( delvePath string moduleGraphFile string + moduleActionsFile string docFile string bazelQueryViewDir string bp2buildMarker string @@ -76,6 +77,7 @@ func init() { // Flags representing various modes soong_build can run in flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output") + flag.StringVar(&moduleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules") flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") @@ -176,15 +178,17 @@ func writeMetrics(configuration android.Config) { } } -func writeJsonModuleGraph(ctx *android.Context, path string) { - f, err := os.Create(shared.JoinPath(topDir, path)) - if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) +func writeJsonModuleGraphAndActions(ctx *android.Context, graphPath string, actionsPath string) { + graphFile, graphErr := os.Create(shared.JoinPath(topDir, graphPath)) + actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, actionsPath)) + if graphErr != nil || actionsErr != nil { + fmt.Fprintf(os.Stderr, "Graph err: %s, actions err: %s", graphErr, actionsErr) os.Exit(1) } - defer f.Close() - ctx.Context.PrintJSONGraph(f) + defer graphFile.Close() + defer actionsFile.Close() + ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile) } func writeBuildGlobsNinjaFile(srcDir, buildDir string, globs func() pathtools.MultipleGlobResults, config interface{}) []string { @@ -254,7 +258,7 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str writeDepFile(queryviewMarkerFile, ninjaDeps) return queryviewMarkerFile } else if generateModuleGraphFile { - writeJsonModuleGraph(ctx, moduleGraphFile) + writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile) writeDepFile(moduleGraphFile, ninjaDeps) return moduleGraphFile } else if generateDocFile { diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index d8cb47a44..a0cfbea5d 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "encoding/json" "flag" "fmt" "io/ioutil" @@ -24,6 +23,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "time" "android/soong/shared" @@ -36,11 +36,6 @@ import ( "android/soong/ui/tracer" ) -const ( - configDir = "vendor/google/tools/soong_config" - jsonSuffix = "json" -) - // A command represents an operation to be executed in the soong build // system. type command struct { @@ -67,7 +62,7 @@ type command struct { } // list of supported commands (flags) supported by soong ui -var commands []command = []command{ +var commands = []command{ { flag: "--make-mode", description: "build the modules by the target name (i.e. soong_docs)", @@ -117,43 +112,6 @@ func inList(s string, list []string) bool { return indexList(s, list) != -1 } -func loadEnvConfig(config build.Config) error { - bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") - if bc == "" { - return nil - } - configDirs := []string{ - os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), - config.OutDir(), - configDir, - } - var cfgFile string - for _, dir := range configDirs { - cfgFile = filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) - if _, err := os.Stat(cfgFile); err == nil { - break - } - } - - envVarsJSON, err := ioutil.ReadFile(cfgFile) - if err != nil { - fmt.Fprintf(os.Stderr, "\033[33mWARNING:\033[0m failed to open config file %s: %s\n", cfgFile, err.Error()) - return nil - } - - var envVars map[string]map[string]string - if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { - return fmt.Errorf("env vars config file: %s did not parse correctly: %s", cfgFile, err.Error()) - } - for k, v := range envVars["env"] { - if os.Getenv(k) != "" { - continue - } - config.Environment().Set(k, v) - } - return nil -} - // Main execution of soong_ui. The command format is as follows: // // soong_ui <command> [<arg 1> <arg 2> ... <arg n>] @@ -218,11 +176,6 @@ func main() { config := c.config(buildCtx, args...) - if err := loadEnvConfig(config); err != nil { - fmt.Fprintf(os.Stderr, "failed to parse env config files: %v", err) - os.Exit(1) - } - build.SetupOutDir(buildCtx, config) if config.UseBazel() && config.Dist() { @@ -252,6 +205,8 @@ func main() { buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v", config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) + setMaxFiles(buildCtx) + { // The order of the function calls is important. The last defer function call // is the first one that is executed to save the rbe metrics to a protobuf @@ -571,7 +526,11 @@ func getCommand(args []string) (*command, []string, error) { } // command not found - return nil, nil, fmt.Errorf("Command not found: %q", args) + flags := make([]string, len(commands)) + for i, c := range commands { + flags[i] = c.flag + } + return nil, nil, fmt.Errorf("Command not found: %q\nDid you mean one of these: %q", args, flags) } // For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary. @@ -648,3 +607,24 @@ func populateExternalDistDirHelper(ctx build.Context, config build.Config, inter } } } + +func setMaxFiles(ctx build.Context) { + var limits syscall.Rlimit + + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits) + if err != nil { + ctx.Println("Failed to get file limit:", err) + return + } + + ctx.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max) + if limits.Cur == limits.Max { + return + } + + limits.Cur = limits.Max + err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits) + if err != nil { + ctx.Println("Failed to increase file limit:", err) + } +} diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 658e8e2dc..36513b64b 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -210,6 +210,34 @@ type ClassLoaderContext struct { Subcontexts []*ClassLoaderContext } +// excludeLibs excludes the libraries from this ClassLoaderContext. +// +// This treats the supplied context as being immutable (as it may come from a dependency). So, it +// implements copy-on-exclusion logic. That means that if any of the excluded libraries are used +// within this context then this will return a deep copy of this without those libraries. +// +// If this ClassLoaderContext matches one of the libraries to exclude then this returns (nil, true) +// to indicate that this context should be excluded from the containing list. +// +// If any of this ClassLoaderContext's Subcontexts reference the excluded libraries then this +// returns a pointer to a copy of this without the excluded libraries and true to indicate that this +// was copied. +// +// Otherwise, this returns a pointer to this and false to indicate that this was not copied. +func (c *ClassLoaderContext) excludeLibs(excludedLibs []string) (*ClassLoaderContext, bool) { + if android.InList(c.Name, excludedLibs) { + return nil, true + } + + if excludedList, modified := excludeLibsFromCLCList(c.Subcontexts, excludedLibs); modified { + clcCopy := *c + clcCopy.Subcontexts = excludedList + return &clcCopy, true + } + + return c, false +} + // ClassLoaderContextMap is a map from SDK version to CLC. There is a special entry with key // AnySdkVersion that stores unconditional CLC that is added regardless of the target SDK version. // @@ -408,6 +436,67 @@ func (clcMap ClassLoaderContextMap) Dump() string { return string(bytes) } +// excludeLibsFromCLCList excludes the libraries from the ClassLoaderContext in this list. +// +// This treats the supplied list as being immutable (as it may come from a dependency). So, it +// implements copy-on-exclusion logic. That means that if any of the excluded libraries are used +// within the contexts in the list then this will return a deep copy of the list without those +// libraries. +// +// If any of the ClassLoaderContext in the list reference the excluded libraries then this returns a +// copy of this list without the excluded libraries and true to indicate that this was copied. +// +// Otherwise, this returns the list and false to indicate that this was not copied. +func excludeLibsFromCLCList(clcList []*ClassLoaderContext, excludedLibs []string) ([]*ClassLoaderContext, bool) { + modifiedList := false + copiedList := make([]*ClassLoaderContext, 0, len(clcList)) + for _, clc := range clcList { + resultClc, modifiedClc := clc.excludeLibs(excludedLibs) + if resultClc != nil { + copiedList = append(copiedList, resultClc) + } + modifiedList = modifiedList || modifiedClc + } + + if modifiedList { + return copiedList, true + } else { + return clcList, false + } +} + +// ExcludeLibs excludes the libraries from the ClassLoaderContextMap. +// +// If the list o libraries is empty then this returns the ClassLoaderContextMap. +// +// This treats the ClassLoaderContextMap as being immutable (as it may come from a dependency). So, +// it implements copy-on-exclusion logic. That means that if any of the excluded libraries are used +// within the contexts in the map then this will return a deep copy of the map without those +// libraries. +// +// Otherwise, this returns the map unchanged. +func (clcMap ClassLoaderContextMap) ExcludeLibs(excludedLibs []string) ClassLoaderContextMap { + if len(excludedLibs) == 0 { + return clcMap + } + + excludedClcMap := make(ClassLoaderContextMap) + modifiedMap := false + for sdkVersion, clcList := range clcMap { + excludedList, modifiedList := excludeLibsFromCLCList(clcList, excludedLibs) + if len(excludedList) != 0 { + excludedClcMap[sdkVersion] = excludedList + } + modifiedMap = modifiedMap || modifiedList + } + + if modifiedMap { + return excludedClcMap + } else { + return clcMap + } +} + // Now that the full unconditional context is known, reconstruct conditional context. // Apply filters for individual libraries, mirroring what the PackageManager does when it // constructs class loader context on device. diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index 4a3d390c2..5d3a9d943 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -284,6 +284,111 @@ func TestCLCSdkVersionOrder(t *testing.T) { }) } +func TestCLCMExcludeLibs(t *testing.T) { + ctx := testContext() + const optional = false + const implicit = true + + excludeLibs := func(t *testing.T, m ClassLoaderContextMap, excluded_libs ...string) ClassLoaderContextMap { + // Dump the CLCM before creating a new copy that excludes a specific set of libraries. + before := m.Dump() + + // Create a new CLCM that excludes some libraries. + c := m.ExcludeLibs(excluded_libs) + + // Make sure that the original CLCM was not changed. + after := m.Dump() + android.AssertStringEquals(t, "input CLCM modified", before, after) + + return c + } + + t.Run("exclude nothing", func(t *testing.T) { + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + + a := excludeLibs(t, m) + + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Implicit": true, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [] + } + ] +}`, a.Dump()) + }) + + t.Run("one item from list", func(t *testing.T) { + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + + a := excludeLibs(t, m, "a") + + expected := `{ + "28": [ + { + "Name": "b", + "Optional": false, + "Implicit": true, + "Host": "out/soong/b.jar", + "Device": "/system/b.jar", + "Subcontexts": [] + } + ] +}` + android.AssertStringEquals(t, "output CLCM ", expected, a.Dump()) + }) + + t.Run("all items from a list", func(t *testing.T) { + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + + a := excludeLibs(t, m, "a", "b") + + android.AssertStringEquals(t, "output CLCM ", `{}`, a.Dump()) + }) + + t.Run("items from a subcontext", func(t *testing.T) { + s := make(ClassLoaderContextMap) + s.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + s.AddContext(ctx, AnySdkVersion, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), s) + + a := excludeLibs(t, m, "b") + + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Implicit": true, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [ + { + "Name": "c", + "Optional": false, + "Implicit": true, + "Host": "out/soong/c.jar", + "Device": "/system/c.jar", + "Subcontexts": [] + } + ] + } + ] +}`, a.Dump()) + }) +} + func checkError(t *testing.T, have error, want string) { if have == nil { t.Errorf("\nwant error: '%s'\nhave: none", want) diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 1f99a96c1..153b0256f 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -408,6 +408,7 @@ func dex2oatModuleName(config android.Config) string { type dex2oatDependencyTag struct { blueprint.BaseDependencyTag + android.LicenseAnnotationToolchainDependencyTag } func (d dex2oatDependencyTag) ExcludeFromVisibilityEnforcement() { diff --git a/docs/OWNERS b/docs/OWNERS index d1433171d..2d7127147 100644 --- a/docs/OWNERS +++ b/docs/OWNERS @@ -1 +1,3 @@ per-file map_files.md = danalbert@google.com, enh@google.com, jiyong@google.com +per-file native_code_coverage.md = pirama@google.com, srhines@google.com, allenhair@google.com +per-file tidy.md = chh@google.com, srhines@google.com diff --git a/docs/native_code_coverage.md b/docs/native_code_coverage.md new file mode 100644 index 000000000..fabbfa013 --- /dev/null +++ b/docs/native_code_coverage.md @@ -0,0 +1,241 @@ +## Native Code Coverage for Android + +## Scope + +These instructions are for Android developers to collect and inspect code +coverage for C++ and Rust code on the Android platform. + +## Building with Native Code Coverage Instrumentation + +Identify the paths where native code-coverage instrumentation should be enabled +and set up the following environment variables. + +``` + export CLANG_COVERAGE=true + export NATIVE_COVERAGE_PATHS="<paths-to-instrument-for-coverage>" +``` + +`NATIVE_COVERAGE_PATHS` should be a list of paths. Any Android.bp module defined +under these paths is instrumented for code-coverage. E.g: + +``` +export NATIVE_COVERAGE_PATHS="external/libcxx system/core/adb" +``` + +### Additional Notes + +- Native Code coverage is not supported for host modules or `Android.mk` + modules. +- `NATIVE_COVERAGE_PATHS="*"` enables coverage instrumentation for all paths. +- Set `native_coverage: false` blueprint property to always disable code + coverage instrumentation for a module. This is useful if this module has + issues when building or running with coverage. +- `NATIVE_COVERAGE_EXCLUDE_PATHS` can be set to exclude subdirs under + `NATIVE_COVERAGE_PATHS` from coverage instrumentation. E.g. + `NATIVE_COVERAGE_PATHS=frameworks/native + NATIVE_COVERAGE_PATHS=frameworks/native/vulkan` will instrument all native + code under `frameworks/native` except`frameworks/native/vulkan`. + +## Running Tests + +### Collecting Profiles + +When an instrumented program is run, the profiles are stored to the path and +name specified in the `LLVM_PROFILE_FILE` environment variable. On Android +coverage builds it is set to `/data/misc/trace/clang-%p-%20m.profraw`. + +* `%`p is replaced by the pid of the process +* `%m` by the hash of the library/binary +* The `20` in`%20m` creates a pool of 20 profraw files and "online" profile + merging is used to merge coverage to profiles onto this pool. + +Reference:`LLVM_PROFILE_FILE` can include additional specifiers as described +[here](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program). + +For this and following steps, use the `acov-llvm.py` script: +`$ANDROID_BUILD_TOP/development/scripts/acov-llvm.py`. + +There may be profiles in `/data/misc/trace` collected before the test is run. +Clear this data before running the test. + +``` + # Clear any coverage that's already written to /data/misc/trace + # and reset coverage for all daemons. + <host>$ acov-llvm.py clean-device + + # Run the test. The exact command depends on the nature of the test. + <device>$ /data/local/tmp/$MY_TEST +``` + +For tests that exercise a daemon/service running in another process, write out +the coverage for those processes as well. + +``` + # Flush coverage of all daemons/processes running on the device. + <host>$ acov-llvm.py flush + + # Flush coverage for a particular daemon, say adbd. + <host>$ acov-llvm.py flush adbd +``` + +## Viewing Coverage Data (acov-llvm.py) + +To post-process and view coverage information we use the `acov-llvm.py report` +command. It invokes two LLVM utilities `llvm-profdata` and `llvm-cov`. An +advanced user can manually invoke these utilities for fine-grained control. This +is discussed [below](#viewing-coverage-data-manual). + +To generate coverage report need the following parameters. These are dependent +on the test/module: + +1. One or more binaries and shared libraries from which coverage was collected. + E.g.: + + 1. ART mainline module contains a few libraries such as `libart.so`, + `libart-compiler.so`. + 2. Bionic tests exercise code in `libc.so` and `libm.so`. + + We need the *unstripped* copies of these binaries. Source information + included in the debuginfo is used to process the coverage data. + +2. One or more source directories under `$ANDROID_BUILD_TOP` for which coverage + needs to be reported. + +Invoke the report subcommand of acov-llvm.py to produce a html coverage summary: + +``` + $ acov-llvm.py report \ + -s <one-or-more-source-paths-in-$ANDROID_BUILD_TOP \ + -b <one-or-more-(unstripped)-binaries-in-$OUT> +``` + +E.g.: + +``` + $ acov-llvm.py report \ + -s bionic \ + -b \ + $OUT/symbols/apex/com.android.runtime/lib/bionic/libc.so \ + $OUT/symbols/apex/com.android.runtime/lib/bionic/libm.so +``` + +The script will produce a report in a temporary directory under +`$ANDROID_BUILD_TOP`. It'll produce a log as below: + +``` + generating coverage report in covreport-xxxxxx +``` + +A html report would be generated under `covreport-xxxxxx/html`. + +## Viewing Coverage Data (manual) + +`acov-llvm.py report` does a few operations under the hood which we can also +manually invoke for flexibility. + +### Post-processing Coverage Files + +Fetch coverage files from the device and post-process them to a `.profdata` file +as follows: + +``` + # Fetch the coverage data from the device. + <host>$ cd coverage_data + <host>$ adb pull /data/misc/trace/ $TRACE_DIR_HOST + + # Convert from .profraw format to the .profdata format. + <host>$ llvm-profdata merge --output=$MY_TEST.profdata \ + $TRACE_DIR_HOST/clang-*.profraw +``` + +For added specificity, restrict the above command to just the <PID>s of the +daemon or test processes of interest. + +``` + <host>$ llvm-profdata merge --output=$MY_TEST.profdata \ + $MY_TEST.profraw \ + trace/clang-<pid1>.profraw trace/clang-<pid2>.profraw ... +``` + +### Generating Coverage report + +Documentation on Clang source-instrumentation-based coverage is available +[here](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#creating-coverage-reports). +The `llvm-cov` utility is used to show coverage from a `.profdata` file. The +documentation for commonly used `llvm-cov` command-line arguments is available +[here](https://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-report). (Try +`llvm-cov show --help` for a complete list). + +#### `show` subcommand + +The `show` command displays the function and line coverage for each source file +in the binary. + +``` + <host>$ llvm-cov show \ + --show-region-summary=false + --format=html --output-dir=coverage-html \ + --instr-profile=$MY_TEST.profdata \ + $MY_BIN \ +``` + +* In the above command, `$MY_BIN` should be the unstripped binary (i.e. with + debuginfo) since `llvm-cov` reads some debuginfo to process the coverage + data. + + E.g.: + + ~~~ + ``` + <host>$ llvm-cov report \ + --instr-profile=adbd.profdata \ + $LOCATION_OF_UNSTRIPPED_ADBD/adbd \ + --show-region-summary=false + ``` + ~~~ + +* The `-ignore-filename-regex=<regex>` option can be used to ignore files that + are not of interest. E.g: `-ignore-filename-regex="external/*"` + +* Use the `--object=<BIN>` argument to specify additional binaries and shared + libraries whose coverage is included in this profdata. See the earlier + [section](#viewing-coverage-data-acov-llvm-py) for examples where more than + one binary may need to be used. + + E.g., the following command is used for `bionic-unit-tests`, which tests + both `libc.so` and `libm.so`: + + ~~~ + ``` + <host>$ llvm-cov report \ + --instr-profile=bionic.profdata \ + $OUT/.../libc.so \ + --object=$OUT/.../libm.so + ``` + ~~~ + +* `llvm-cov` also takes positional SOURCES argument to consider/display only + particular paths of interest. E.g: + + ~~~ + ``` + <host>$ llvm-cov report \ + --instr-profile=adbd.profdata \ + $LOCATION_OF_ADBD/adbd \ + --show-region-summary=false \ + /proc/self/cwd/system/core/adb + ``` + ~~~ + +Note that the paths for the sources need to be prepended with +'`/proc/self/cwd/`'. This is because Android C/C++ compilations run with +`PWD=/proc/self/cwd` and consequently the source names are recorded with that +prefix. Alternatively, the +[`--path-equivalence`](https://llvm.org/docs/CommandGuide/llvm-cov.html#cmdoption-llvm-cov-show-path-equivalence) +option to `llvm-cov` can be used. + +#### `report` subcommand + +The [`report`](https://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-report) +subcommand summarizes the percentage of covered lines to the console. It takes +options similar to the `show` subcommand. diff --git a/docs/tidy.md b/docs/tidy.md new file mode 100644 index 000000000..3140198c3 --- /dev/null +++ b/docs/tidy.md @@ -0,0 +1,244 @@ +# Clang-Tidy Rules and Checks + +Android C/C++ source files can be checked by clang-tidy for issues like +coding style, error-prone/performance patterns, and static flow analysis. +See the official +[clang-tidy document](https://clang.llvm.org/extra/clang-tidy) +and list of +[clang-tidy checks](https://clang.llvm.org/extra/clang-tidy/checks/list.html). + +## Global defaults for Android builds + +The simplest way to enable clang-tidy checks is +to set environment variable `WITH_TIDY`. +``` +$ WITH_TIDY=1 make +``` + +This will turn on the global default to run clang-tidy for every required +C/C++ source file compilation. The global default clang-tidy checks +do not include time-consuming static analyzer checks. To enable those +checks, set the `CLANG_ANALYZER_CHECKS` variable. +``` +$ WITH_TIDY=1 CLANG_ANALYZER_CHECKS=1 make +``` + +The default global clang-tidy checks and flags are defined in +[build/soong/cc/config/tidy.go](https://android.googlesource.com/platform/build/soong/+/refs/heads/master/cc/config/tidy.go). + + +## Module clang-tidy properties + +The global default can be overwritten by module properties in Android.bp. + +### `tidy` and `tidy_checks` + +For example, in +[system/bpf/Android.bp](https://android.googlesource.com/platform/system/bpf/+/refs/heads/master/Android.bp), +clang-tidy is enabled explicitly and with a different check list: +``` +cc_defaults { + name: "bpf_defaults", + // snipped + tidy: true, + tidy_checks: [ + "android-*", + "cert-*", + "-cert-err34-c", + "clang-analyzer-security*", + // Disabling due to many unavoidable warnings from POSIX API usage. + "-google-runtime-int", + ], +} +``` +That means in normal builds, even without `WITH_TIDY=1`, +the modules that use `bpf_defaults` will run clang-tidy +over C/C++ source files with the given `tidy_checks`. +Note that `clang-analyzer-security*` is included in `tidy_checks` +but not all `clang-analyzer-*` checks. Check `cert-err34-c` is +disabled, although `cert-*` is selected. + +Some modules might want to disable clang-tidy even when +environment variable `WITH_TIDY=1` is set. +Examples can be found in +[system/netd/tests/Android.bp](https://android.googlesource.com/platform/system/netd/+/refs/heads/master/tests/Android.bp) +``` +cc_test { + name: "netd_integration_test", + // snipped + defaults: ["netd_defaults"], + tidy: false, // cuts test build time by almost 1 minute +``` +and in +[bionic/tests/Android.bp](https://android.googlesource.com/platform/bionic/+/refs/heads/master/tests/Android.bp). +``` +cc_test_library { + name: "fortify_disabled_for_tidy", + // snipped + srcs: ["clang_fortify_tests.cpp"], + tidy: false, +} +``` + +### `tidy_checks_as_errors` + +The global tidy checks are enabled as warnings. +If a C/C++ module wants to be free of certain clang-tidy warnings, +it can chose those checks to be treated as errors. +For example +[system/core/libsysutils/Android.bp](https://android.googlesource.com/platform/system/core/+/refs/heads/master/libsysutils/Android.bp) +has enabled clang-tidy explicitly, selected its own tidy checks, +and set three groups of tidy checks as errors: +``` +cc_library { + name: "libsysutils", + // snipped + tidy: true, + tidy_checks: [ + "-*", + "cert-*", + "clang-analyzer-security*", + "android-*", + ], + tidy_checks_as_errors: [ + "cert-*", + "clang-analyzer-security*", + "android-*", + ], + // snipped +} +``` + +### `tidy_flags` and `tidy_disabled_srcs` + +Extra clang-tidy flags can be passed with the `tidy_flags` property. + +Some Android modules use the `tidy_flags` to pass "-warnings-as-errors=" +to clang-tidy. This usage should now be replaced with the +`tidy_checks_as_errors` property. + +Some other tidy flags examples are `-format-style=` and `-header-filter=` +For example, in +[art/odrefresh/Android.bp](https://android.googlesource.com/platform/art/+/refs/heads/master/odrefresh/Android.bp), +we found +``` +cc_defaults { + name: "odrefresh-defaults", + srcs: [ + "odrefresh.cc", + "odr_common.cc", + "odr_compilation_log.cc", + "odr_fs_utils.cc", + "odr_metrics.cc", + "odr_metrics_record.cc", + ], + // snipped + generated_sources: [ + "apex-info-list-tinyxml", + "art-apex-cache-info", + "art-odrefresh-operator-srcs", + ], + // snipped + tidy: true, + tidy_disabled_srcs: [":art-apex-cache-info"], + tidy_flags: [ + "-format-style=file", + "-header-filter=(art/odrefresh/|system/apex/)", + ], +} +``` +That means all modules with the `odrefresh-defaults` will +have clang-tidy enabled, but not for generated source +files in `art-apex-cache-info`. +The clang-tidy is called with extra flags to specify the +format-style and header-filter. + +Note that the globally set default for header-filter is to +include only the module directory. So, the default clang-tidy +warnings for `art/odrefresh` modules will include source files +under that directory. Now `odrefresh-defaults` is interested +in seeing warnings from both `art/odrefresh/` and `system/apex/` +and it redefines `-header-filter` in its `tidy_flags`. + + +## Phony tidy-* targets + +### The tidy-*directory* targets + +Setting `WITH_TIDY=1` is easy to enable clang-tidy globally for any build. +However, it adds extra compilation time. + +For developers focusing on just one directory, they only want to compile +their files with clang-tidy and wish to build other Android components as +fast as possible. Changing the `WITH_TIDY=1` variable setting is also expensive +since the build.ninja file will be regenerated due to any such variable change. + +To manually select only some directories or modules to compile with clang-tidy, +do not set the `WITH_TIDY=1` variable, but use the special `tidy-<directory>` +phony target. For example, a person working on `system/libbase` can build +Android quickly with +``` +unset WITH_TIDY # Optional, not if you haven't set WITH_TIDY +make droid tidy-system-libbase +``` + +For any directory `d1/d2/d3`, a phony target tidy-d1-d2-d3 is generated +if there is any C/C++ source file under `d1/d2/d3`. + +Note that with `make droid tidy-system-libbase`, some C/C++ files +that are not needed by the `droid` target will be passed to clang-tidy +if they are under `system/libbase`. This is like a `checkbuild` +under `system/libbase` to include all modules, but only C/C++ +files of those modules are compiled with clang-tidy. + +### The tidy-soong target + +A special `tidy-soong` target is defined to include all C/C++ +source files in *all* directories. This phony target is sometimes +used to test if all source files compile with a new clang-tidy release. + +### The tidy-*_subset targets + +A *subset* of each tidy-* phony target is defined to reduce test time. +Since any Android module, a C/C++ library or binary, can be built +for many different *variants*, one C/C++ source file is usually +compiled multiple times with different compilation flags. +Many of such *variant* flags have little or no effect on clang-tidy +checks. To reduce clang-tidy check time, a *subset* target like +`tidy-soong_subset` or `tidy-system-libbase_subset` is generated +to include only a subset, the first variant, of each module in +the directory. + +Hence, for C/C++ source code quality, instead of a long +"make checkbuild", we can use "make tidy-soong_subset". + + +## Limit clang-tidy runtime + +Some Android modules have large files that take a long time to compile +with clang-tidy, with or without the clang-analyzer checks. +To limit clang-tidy time, an environment variable can be set as +```base +WITH_TIDY=1 TIDY_TIMEOUT=90 make +``` +This 90-second limit is actually the default time limit +in several Android continuous builds where `WITH_TIDY=1` and +`CLANG_ANALYZER_CHECKS=1` are set. + +## Capabilities for Android.bp and Android.mk + +Some of the previously mentioned features are defined only +for modules in Android.bp files, not for Android.mk modules yet. + +* The global `WITH_TIDY=1` variable will enable clang-tidy for all C/C++ + modules in Android.bp or Android.mk files. + +* The global `TIDY_TIMEOUT` variable is recognized by Android prebuilt + clang-tidy, so it should work for any clang-tidy invocation. + +* The clang-tidy module level properties are defined for Android.bp modules. + For Android.mk modules, old `LOCAL_TIDY`, `LOCAL_TIDY_CHECKS`, + `LOCAL_TIDY_FLAGS` work similarly, but it would be better to convert + those modules to use Android.bp files. + +* The `tidy-*` phony targets are only generated for Android.bp modules. diff --git a/example_config.json b/example_config.json new file mode 100644 index 000000000..7489840a9 --- /dev/null +++ b/example_config.json @@ -0,0 +1,6 @@ +{ + "env": { + "ENV_VAR_1": "Value1", + "ENV_VAR_2": "Value2" + } +} diff --git a/finder/finder.go b/finder/finder.go index 5413fa6bb..b4834b16b 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -847,6 +847,7 @@ func (f *Finder) startFromExternalCache() (err error) { if err != nil { return errors.New("No data to load from database\n") } + defer reader.Close() bufferedReader := bufio.NewReader(reader) if !f.validateCacheHeader(bufferedReader) { return errors.New("Cache header does not match") diff --git a/finder/fs/test.go b/finder/fs/test.go index cb2140e75..ed981fd0e 100644 --- a/finder/fs/test.go +++ b/finder/fs/test.go @@ -74,6 +74,7 @@ func Read(t *testing.T, path string, filesystem *MockFs) string { if err != nil { t.Fatalf(err.Error()) } + defer reader.Close() bytes, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err.Error()) diff --git a/java/OWNERS b/java/OWNERS index 52427120d..5b71b1e35 100644 --- a/java/OWNERS +++ b/java/OWNERS @@ -1 +1,4 @@ per-file dexpreopt*.go = ngeoffray@google.com,calin@google.com,skvadrik@google.com + +# For metalava team to disable lint checks in platform +per-file droidstubs.go = aurimas@google.com,emberrose@google.com,sjgilbert@google.com
\ No newline at end of file diff --git a/java/aar.go b/java/aar.go index aabbec6a3..8e1025361 100644 --- a/java/aar.go +++ b/java/aar.go @@ -267,18 +267,29 @@ var extractAssetsRule = pctx.AndroidStaticRule("extractAssets", }) func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkContext, - classLoaderContexts dexpreopt.ClassLoaderContextMap, extraLinkFlags ...string) { + classLoaderContexts dexpreopt.ClassLoaderContextMap, excludedLibs []string, + extraLinkFlags ...string) { transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assetPackages, libDeps, libFlags := aaptLibs(ctx, sdkContext, classLoaderContexts) + // Exclude any libraries from the supplied list. + classLoaderContexts = classLoaderContexts.ExcludeLibs(excludedLibs) + // App manifest file manifestFile := proptools.StringDefault(a.aaptProperties.Manifest, "AndroidManifest.xml") manifestSrcPath := android.PathForModuleSrc(ctx, manifestFile) - manifestPath := manifestFixer(ctx, manifestSrcPath, sdkContext, classLoaderContexts, - a.isLibrary, a.useEmbeddedNativeLibs, a.usesNonSdkApis, a.useEmbeddedDex, a.hasNoCode, - a.LoggingParent) + manifestPath := ManifestFixer(ctx, manifestSrcPath, ManifestFixerParams{ + SdkContext: sdkContext, + ClassLoaderContexts: classLoaderContexts, + IsLibrary: a.isLibrary, + UseEmbeddedNativeLibs: a.useEmbeddedNativeLibs, + UsesNonSdkApis: a.usesNonSdkApis, + UseEmbeddedDex: a.useEmbeddedDex, + HasNoCode: a.hasNoCode, + LoggingParent: a.LoggingParent, + }) // Add additional manifest files to transitive manifests. additionalManifests := android.PathsForModuleSrc(ctx, a.aaptProperties.Additional_manifests) @@ -520,7 +531,7 @@ func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.aapt.isLibrary = true a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) - a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts) + a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts, nil) a.hideApexVariantFromMake = !ctx.Provider(android.ApexInfoProvider).(android.ApexInfo).IsForPlatform() diff --git a/java/android_manifest.go b/java/android_manifest.go index f29d8ad1a..7772b7090 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -28,13 +28,10 @@ import ( var manifestFixerRule = pctx.AndroidStaticRule("manifestFixer", blueprint.RuleParams{ Command: `${config.ManifestFixerCmd} ` + - `--minSdkVersion ${minSdkVersion} ` + - `--targetSdkVersion ${targetSdkVersion} ` + - `--raise-min-sdk-version ` + `$args $in $out`, CommandDeps: []string{"${config.ManifestFixerCmd}"}, }, - "minSdkVersion", "targetSdkVersion", "args") + "args") var manifestMergerRule = pctx.AndroidStaticRule("manifestMerger", blueprint.RuleParams{ @@ -58,84 +55,109 @@ func targetSdkVersionForManifestFixer(ctx android.ModuleContext, sdkContext andr return targetSdkVersion } -// Uses manifest_fixer.py to inject minSdkVersion, etc. into an AndroidManifest.xml -func manifestFixer(ctx android.ModuleContext, manifest android.Path, sdkContext android.SdkContext, - classLoaderContexts dexpreopt.ClassLoaderContextMap, isLibrary, useEmbeddedNativeLibs, usesNonSdkApis, - useEmbeddedDex, hasNoCode bool, loggingParent string) android.Path { +type ManifestFixerParams struct { + SdkContext android.SdkContext + ClassLoaderContexts dexpreopt.ClassLoaderContextMap + IsLibrary bool + UseEmbeddedNativeLibs bool + UsesNonSdkApis bool + UseEmbeddedDex bool + HasNoCode bool + TestOnly bool + LoggingParent string +} +// Uses manifest_fixer.py to inject minSdkVersion, etc. into an AndroidManifest.xml +func ManifestFixer(ctx android.ModuleContext, manifest android.Path, + params ManifestFixerParams) android.Path { var args []string - if isLibrary { + + if params.IsLibrary { args = append(args, "--library") - } else { - minSdkVersion, err := sdkContext.MinSdkVersion(ctx).EffectiveVersion(ctx) + } else if params.SdkContext != nil { + minSdkVersion, err := params.SdkContext.MinSdkVersion(ctx).EffectiveVersion(ctx) if err != nil { ctx.ModuleErrorf("invalid minSdkVersion: %s", err) } if minSdkVersion.FinalOrFutureInt() >= 23 { - args = append(args, fmt.Sprintf("--extract-native-libs=%v", !useEmbeddedNativeLibs)) - } else if useEmbeddedNativeLibs { + args = append(args, fmt.Sprintf("--extract-native-libs=%v", !params.UseEmbeddedNativeLibs)) + } else if params.UseEmbeddedNativeLibs { ctx.ModuleErrorf("module attempted to store uncompressed native libraries, but minSdkVersion=%d doesn't support it", minSdkVersion) } } - if usesNonSdkApis { + if params.UsesNonSdkApis { args = append(args, "--uses-non-sdk-api") } - if useEmbeddedDex { + if params.UseEmbeddedDex { args = append(args, "--use-embedded-dex") } - // manifest_fixer should add only the implicit SDK libraries inferred by Soong, not those added - // explicitly via `uses_libs`/`optional_uses_libs`. - requiredUsesLibs, optionalUsesLibs := classLoaderContexts.ImplicitUsesLibs() - for _, usesLib := range requiredUsesLibs { - args = append(args, "--uses-library", usesLib) - } - for _, usesLib := range optionalUsesLibs { - args = append(args, "--optional-uses-library", usesLib) + if params.ClassLoaderContexts != nil { + // manifest_fixer should add only the implicit SDK libraries inferred by Soong, not those added + // explicitly via `uses_libs`/`optional_uses_libs`. + requiredUsesLibs, optionalUsesLibs := params.ClassLoaderContexts.ImplicitUsesLibs() + + for _, usesLib := range requiredUsesLibs { + args = append(args, "--uses-library", usesLib) + } + for _, usesLib := range optionalUsesLibs { + args = append(args, "--optional-uses-library", usesLib) + } } - if hasNoCode { + if params.HasNoCode { args = append(args, "--has-no-code") } - if loggingParent != "" { - args = append(args, "--logging-parent", loggingParent) + if params.TestOnly { + args = append(args, "--test-only") } - var deps android.Paths - targetSdkVersion := targetSdkVersionForManifestFixer(ctx, sdkContext) - if UseApiFingerprint(ctx) && ctx.ModuleName() != "framework-res" { - targetSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", ApiFingerprintPath(ctx).String()) - deps = append(deps, ApiFingerprintPath(ctx)) + if params.LoggingParent != "" { + args = append(args, "--logging-parent", params.LoggingParent) } + var deps android.Paths + var argsMapper = make(map[string]string) - minSdkVersion, err := sdkContext.MinSdkVersion(ctx).EffectiveVersionString(ctx) - if err != nil { - ctx.ModuleErrorf("invalid minSdkVersion: %s", err) - } - if UseApiFingerprint(ctx) && ctx.ModuleName() != "framework-res" { - minSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", ApiFingerprintPath(ctx).String()) - deps = append(deps, ApiFingerprintPath(ctx)) + if params.SdkContext != nil { + targetSdkVersion := targetSdkVersionForManifestFixer(ctx, params.SdkContext) + args = append(args, "--targetSdkVersion ", targetSdkVersion) + + if UseApiFingerprint(ctx) && ctx.ModuleName() != "framework-res" { + targetSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", ApiFingerprintPath(ctx).String()) + deps = append(deps, ApiFingerprintPath(ctx)) + } + + minSdkVersion, err := params.SdkContext.MinSdkVersion(ctx).EffectiveVersionString(ctx) + if err != nil { + ctx.ModuleErrorf("invalid minSdkVersion: %s", err) + } + + if UseApiFingerprint(ctx) && ctx.ModuleName() != "framework-res" { + minSdkVersion = ctx.Config().PlatformSdkCodename() + fmt.Sprintf(".$$(cat %s)", ApiFingerprintPath(ctx).String()) + deps = append(deps, ApiFingerprintPath(ctx)) + } + + if err != nil { + ctx.ModuleErrorf("invalid minSdkVersion: %s", err) + } + args = append(args, "--minSdkVersion ", minSdkVersion) + args = append(args, "--raise-min-sdk-version") } fixedManifest := android.PathForModuleOut(ctx, "manifest_fixer", "AndroidManifest.xml") - if err != nil { - ctx.ModuleErrorf("invalid minSdkVersion: %s", err) - } + argsMapper["args"] = strings.Join(args, " ") + ctx.Build(pctx, android.BuildParams{ Rule: manifestFixerRule, Description: "fix manifest", Input: manifest, Implicits: deps, Output: fixedManifest, - Args: map[string]string{ - "minSdkVersion": minSdkVersion, - "targetSdkVersion": targetSdkVersion, - "args": strings.Join(args, " "), - }, + Args: argsMapper, }) return fixedManifest.WithoutRel() diff --git a/java/app.go b/java/app.go index 7ae73f797..e4432ff4b 100755 --- a/java/app.go +++ b/java/app.go @@ -425,7 +425,8 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { a.aapt.splitNames = a.appProperties.Package_splits a.aapt.LoggingParent = String(a.overridableAppProperties.Logging_parent) - a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts, aaptLinkFlags...) + a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts, + a.usesLibraryProperties.Exclude_uses_libs, aaptLinkFlags...) // apps manifests are handled by aapt, don't let Module see them a.properties.Manifest = nil @@ -1211,6 +1212,23 @@ type UsesLibraryProperties struct { // libraries, because SDK ones are automatically picked up by Soong. The <uses-library> name // normally is the same as the module name, but there are exceptions. Provides_uses_lib *string + + // A list of shared library names to exclude from the classpath of the APK. Adding a library here + // will prevent it from being used when precompiling the APK and prevent it from being implicitly + // added to the APK's manifest's <uses-library> elements. + // + // Care must be taken when using this as it could result in runtime errors if the APK actually + // uses classes provided by the library and which are not provided in any other way. + // + // This is primarily intended for use by various CTS tests that check the runtime handling of the + // android.test.base shared library (and related libraries) but which depend on some common + // libraries that depend on the android.test.base library. Without this those tests will end up + // with a <uses-library android:name="android.test.base"/> in their manifest which would either + // render the tests worthless (as they would be testing the wrong behavior), or would break the + // test altogether by providing access to classes that the tests were not expecting. Those tests + // provide the android.test.base statically and use jarjar to rename them so they do not collide + // with the classes provided by the android.test.base library. + Exclude_uses_libs []string } // usesLibrary provides properties and helper functions for AndroidApp and AndroidAppImport to verify that the @@ -1432,17 +1450,15 @@ func androidAppCertificateBp2Build(ctx android.TopDownMutatorContext, module *An } type bazelAndroidAppAttributes struct { - Srcs bazel.LabelListAttribute + *javaLibraryAttributes Manifest bazel.Label Custom_package *string Resource_files bazel.LabelListAttribute - Deps bazel.LabelListAttribute } // ConvertWithBp2build is used to convert android_app to Bazel. func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - //TODO(b/209577426): Support multiple arch variants - srcs := bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrcExcludes(ctx, a.properties.Srcs, a.properties.Exclude_srcs)) + libAttrs := a.convertLibraryAttrsBp2Build(ctx) manifest := proptools.StringDefault(a.aaptProperties.Manifest, "AndroidManifest.xml") @@ -1454,15 +1470,12 @@ func (a *AndroidApp) ConvertWithBp2build(ctx android.TopDownMutatorContext) { resourceFiles.Includes = append(resourceFiles.Includes, files...) } - deps := bazel.MakeLabelListAttribute(android.BazelLabelForModuleDeps(ctx, a.properties.Static_libs)) - attrs := &bazelAndroidAppAttributes{ - Srcs: srcs, - Manifest: android.BazelLabelForModuleSrcSingle(ctx, manifest), + libAttrs, + android.BazelLabelForModuleSrcSingle(ctx, manifest), // TODO(b/209576404): handle package name override by product variable PRODUCT_MANIFEST_PACKAGE_NAME_OVERRIDES - Custom_package: a.overridableAppProperties.Package_name, - Resource_files: bazel.MakeLabelListAttribute(resourceFiles), - Deps: deps, + a.overridableAppProperties.Package_name, + bazel.MakeLabelListAttribute(resourceFiles), } props := bazel.BazelTargetModuleProperties{Rule_class: "android_binary", Bzl_load_location: "@rules_android//rules:rules.bzl"} diff --git a/java/app_test.go b/java/app_test.go index 2322ef44e..16bbec158 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -2512,7 +2512,7 @@ func TestUsesLibraries(t *testing.T) { `--uses-library qux ` + `--uses-library quuz ` + `--uses-library runtime-library` - android.AssertStringEquals(t, "manifest_fixer args", expectManifestFixerArgs, actualManifestFixerArgs) + android.AssertStringDoesContain(t, "manifest_fixer args", actualManifestFixerArgs, expectManifestFixerArgs) // Test that all libraries are verified (library order matters). verifyCmd := app.Rule("verify_uses_libraries").RuleParams.Command @@ -3055,7 +3055,7 @@ func TestTargetSdkVersionManifestFixer(t *testing.T) { result := fixture.RunTestWithBp(t, bp) foo := result.ModuleForTests("foo", "android_common") - manifestFixerArgs := foo.Output("manifest_fixer/AndroidManifest.xml").Args - android.AssertStringEquals(t, testCase.name, testCase.targetSdkVersionExpected, manifestFixerArgs["targetSdkVersion"]) + manifestFixerArgs := foo.Output("manifest_fixer/AndroidManifest.xml").Args["args"] + android.AssertStringDoesContain(t, testCase.name, manifestFixerArgs, "--targetSdkVersion "+testCase.targetSdkVersionExpected) } } diff --git a/java/base.go b/java/base.go index a3eb8de21..42d7733ca 100644 --- a/java/base.go +++ b/java/base.go @@ -1969,7 +1969,7 @@ var _ ModuleWithStem = (*Module)(nil) func (j *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { switch ctx.ModuleType() { - case "java_library", "java_library_host": + case "java_library", "java_library_host", "java_library_static": if lib, ok := ctx.Module().(*Library); ok { javaLibraryBp2Build(ctx, lib) } @@ -1978,5 +1978,4 @@ func (j *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { javaBinaryHostBp2Build(ctx, binary) } } - } diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index fee51d72b..5fe409e25 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -669,6 +669,8 @@ func (b *BootclasspathFragmentModule) configuredJars(ctx android.ModuleContext) // This is an exception to support end-to-end test for SdkExtensions, until such support exists. if android.InList("test_framework-sdkextensions", possibleUpdatableModules) { jars = jars.Append("com.android.sdkext", "test_framework-sdkextensions") + } else if android.InList("AddNewActivity", possibleUpdatableModules) { + jars = jars.Append("test_com.android.cts.frameworkresapkplits", "AddNewActivity") } else if android.InList("test_framework-apexd", possibleUpdatableModules) { jars = jars.Append("com.android.apex.test_package", "test_framework-apexd") } else if global.ApexBootJars.Len() != 0 && !android.IsModuleInVersionedSdk(ctx.Module()) { @@ -929,13 +931,13 @@ type bootclasspathFragmentSdkMemberProperties struct { All_flags_path android.OptionalPath `supported_build_releases:"S"` // The path to the generated signature-patterns.csv file. - Signature_patterns_path android.OptionalPath `supported_build_releases:"T+"` + Signature_patterns_path android.OptionalPath `supported_build_releases:"Tiramisu+"` // The path to the generated filtered-stub-flags.csv file. - Filtered_stub_flags_path android.OptionalPath `supported_build_releases:"T+"` + Filtered_stub_flags_path android.OptionalPath `supported_build_releases:"Tiramisu+"` // The path to the generated filtered-flags.csv file. - Filtered_flags_path android.OptionalPath `supported_build_releases:"T+"` + Filtered_flags_path android.OptionalPath `supported_build_releases:"Tiramisu+"` } func (b *bootclasspathFragmentSdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { @@ -1071,7 +1073,7 @@ type prebuiltBootclasspathFragmentProperties struct { // At the moment this is basically just a bootclasspath_fragment module that can be used as a // prebuilt. Eventually as more functionality is migrated into the bootclasspath_fragment module // type from the various singletons then this will diverge. -type prebuiltBootclasspathFragmentModule struct { +type PrebuiltBootclasspathFragmentModule struct { BootclasspathFragmentModule prebuilt android.Prebuilt @@ -1079,16 +1081,16 @@ type prebuiltBootclasspathFragmentModule struct { prebuiltProperties prebuiltBootclasspathFragmentProperties } -func (module *prebuiltBootclasspathFragmentModule) Prebuilt() *android.Prebuilt { +func (module *PrebuiltBootclasspathFragmentModule) Prebuilt() *android.Prebuilt { return &module.prebuilt } -func (module *prebuiltBootclasspathFragmentModule) Name() string { +func (module *PrebuiltBootclasspathFragmentModule) Name() string { return module.prebuilt.Name(module.ModuleBase.Name()) } // produceHiddenAPIOutput returns a path to the prebuilt all-flags.csv or nil if none is specified. -func (module *prebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput { +func (module *PrebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput { pathForOptionalSrc := func(src *string, defaultPath android.Path) android.Path { if src == nil { return defaultPath @@ -1129,7 +1131,7 @@ func (module *prebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx an } // produceBootImageFiles extracts the boot image files from the APEX if available. -func (module *prebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageFilesByArch { +func (module *PrebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageFilesByArch { if !shouldCopyBootFilesToPredefinedLocations(ctx, imageConfig) { return nil } @@ -1139,37 +1141,53 @@ func (module *prebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx and return nil // An error has been reported by FindDeapexerProviderForModule. } - files := bootImageFilesByArch{} - for _, variant := range imageConfig.apexVariants() { - arch := variant.target.Arch.ArchType - for _, toPath := range variant.imagesDeps { - apexRelativePath := apexRootRelativePathToBootImageFile(arch, toPath.Base()) - // Get the path to the file that the deapexer extracted from the prebuilt apex file. - fromPath := di.PrebuiltExportPath(apexRelativePath) - - // Return the toPath as the calling code expects the paths in the returned map to be the - // paths predefined in the bootImageConfig. - files[arch] = append(files[arch], toPath) - - // Copy the file to the predefined location. - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: fromPath, - Output: toPath, - }) - } + profile := (android.WritablePath)(nil) + if imageConfig.profileInstallPathInApex != "" { + profile = di.PrebuiltExportPath(imageConfig.profileInstallPathInApex) } - // Build the boot image files for the host variants. These are built from the dex files provided - // by the contents of this module as prebuilt versions of the host boot image files are not - // available, i.e. there is no host specific prebuilt apex containing them. This has to be built - // without a profile as the prebuilt modules do not provide a profile. - buildBootImageVariantsForBuildOs(ctx, imageConfig, nil) + // Build the boot image files for the host variants. These are always built from the dex files + // provided by the contents of this module as prebuilt versions of the host boot image files are + // not available, i.e. there is no host specific prebuilt apex containing them. This has to be + // built without a profile as the prebuilt modules do not provide a profile. + buildBootImageVariantsForBuildOs(ctx, imageConfig, profile) - return files + if imageConfig.shouldInstallInApex() { + // If the boot image files for the android variants are in the prebuilt apex, we must use those + // rather than building new ones because those boot image files are going to be used on device. + files := bootImageFilesByArch{} + for _, variant := range imageConfig.apexVariants() { + arch := variant.target.Arch.ArchType + for _, toPath := range variant.imagesDeps { + apexRelativePath := apexRootRelativePathToBootImageFile(arch, toPath.Base()) + // Get the path to the file that the deapexer extracted from the prebuilt apex file. + fromPath := di.PrebuiltExportPath(apexRelativePath) + + // Return the toPath as the calling code expects the paths in the returned map to be the + // paths predefined in the bootImageConfig. + files[arch] = append(files[arch], toPath) + + // Copy the file to the predefined location. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: fromPath, + Output: toPath, + }) + } + } + return files + } else { + if profile == nil { + ctx.ModuleErrorf("Unable to produce boot image files: neither boot image files nor profiles exists in the prebuilt apex") + return nil + } + // Build boot image files for the android variants from the dex files provided by the contents + // of this module. + return buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) + } } -var _ commonBootclasspathFragment = (*prebuiltBootclasspathFragmentModule)(nil) +var _ commonBootclasspathFragment = (*PrebuiltBootclasspathFragmentModule)(nil) // createBootImageTag creates the tag to uniquely identify the boot image file among all of the // files that a module requires from the prebuilt .apex file. @@ -1183,16 +1201,22 @@ func createBootImageTag(arch android.ArchType, baseName string) string { // // If there is no image config associated with this fragment then it returns nil. Otherwise, it // returns the files that are listed in the image config. -func (module *prebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex(ctx android.BaseModuleContext) []string { +func (module *PrebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex(ctx android.BaseModuleContext) []string { imageConfig := module.getImageConfig(ctx) if imageConfig != nil { - // Add the boot image files, e.g. .art, .oat and .vdex files. files := []string{} - for _, variant := range imageConfig.apexVariants() { - arch := variant.target.Arch.ArchType - for _, path := range variant.imagesDeps.Paths() { - base := path.Base() - files = append(files, apexRootRelativePathToBootImageFile(arch, base)) + if imageConfig.profileInstallPathInApex != "" { + // Add the boot image profile. + files = append(files, imageConfig.profileInstallPathInApex) + } + if imageConfig.shouldInstallInApex() { + // Add the boot image files, e.g. .art, .oat and .vdex files. + for _, variant := range imageConfig.apexVariants() { + arch := variant.target.Arch.ArchType + for _, path := range variant.imagesDeps.Paths() { + base := path.Base() + files = append(files, apexRootRelativePathToBootImageFile(arch, base)) + } } } return files @@ -1204,10 +1228,10 @@ func apexRootRelativePathToBootImageFile(arch android.ArchType, base string) str return filepath.Join("javalib", arch.String(), base) } -var _ android.RequiredFilesFromPrebuiltApex = (*prebuiltBootclasspathFragmentModule)(nil) +var _ android.RequiredFilesFromPrebuiltApex = (*PrebuiltBootclasspathFragmentModule)(nil) func prebuiltBootclasspathFragmentFactory() android.Module { - m := &prebuiltBootclasspathFragmentModule{} + m := &PrebuiltBootclasspathFragmentModule{} m.AddProperties(&m.properties, &m.prebuiltProperties) // This doesn't actually have any prebuilt files of its own so pass a placeholder for the srcs // array. diff --git a/java/config/config.go b/java/config/config.go index 30c6f91aa..ea2f93449 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -60,7 +60,7 @@ var ( const ( JavaVmFlags = `-XX:OnError="cat hs_err_pid%p.log" -XX:CICompilerCount=6 -XX:+UseDynamicNumberOfGCThreads` - JavacVmFlags = `-J-XX:OnError="cat hs_err_pid%p.log" -J-XX:CICompilerCount=6 -J-XX:+UseDynamicNumberOfGCThreads` + JavacVmFlags = `-J-XX:OnError="cat hs_err_pid%p.log" -J-XX:CICompilerCount=6 -J-XX:+UseDynamicNumberOfGCThreads -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1` ) func init() { diff --git a/java/droidstubs.go b/java/droidstubs.go index 5a84e051c..f9dcfd668 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -19,7 +19,6 @@ import ( "path/filepath" "strings" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" "android/soong/android" @@ -335,7 +334,11 @@ func (d *Droidstubs) annotationsFlags(ctx android.ModuleContext, cmd *android.Ru // TODO(tnorbye): find owners to fix these warnings when annotation was enabled. cmd.FlagWithArg("--hide ", "HiddenTypedefConstant"). FlagWithArg("--hide ", "SuperfluousPrefix"). - FlagWithArg("--hide ", "AnnotationExtraction") + FlagWithArg("--hide ", "AnnotationExtraction"). + // (b/217545629) + FlagWithArg("--hide ", "ChangedThrows"). + // (b/217552813) + FlagWithArg("--hide ", "ChangedAbstract") } } @@ -807,8 +810,7 @@ type PrebuiltStubsSources struct { properties PrebuiltStubsSourcesProperties - stubsSrcJar android.Path - jsonDataActions []blueprint.JSONDataAction + stubsSrcJar android.Path } func (p *PrebuiltStubsSources) OutputFiles(tag string) (android.Paths, error) { @@ -824,13 +826,6 @@ func (d *PrebuiltStubsSources) StubsSrcJar() android.Path { return d.stubsSrcJar } -// AddJSONData is a temporary solution for droidstubs module to put action -// related data into the module json graph. -func (p *PrebuiltStubsSources) AddJSONData(d *map[string]interface{}) { - p.ModuleBase.AddJSONData(d) - (*d)["Actions"] = blueprint.FormatJSONDataActions(p.jsonDataActions) -} - func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleContext) { if len(p.properties.Srcs) != 1 { ctx.PropertyErrorf("srcs", "must only specify one directory path or srcjar, contains %d paths", len(p.properties.Srcs)) @@ -838,12 +833,9 @@ func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleCon } src := p.properties.Srcs[0] - var jsonDataAction blueprint.JSONDataAction if filepath.Ext(src) == ".srcjar" { // This is a srcjar. We can use it directly. p.stubsSrcJar = android.PathForModuleSrc(ctx, src) - jsonDataAction.Inputs = []string{src} - jsonDataAction.Outputs = []string{src} } else { outPath := android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") @@ -867,10 +859,7 @@ func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleCon rule.Restat() rule.Build("zip src", "Create srcjar from prebuilt source") p.stubsSrcJar = outPath - jsonDataAction.Inputs = srcPaths.Strings() - jsonDataAction.Outputs = []string{outPath.String()} } - p.jsonDataActions = []blueprint.JSONDataAction{jsonDataAction} } func (p *PrebuiltStubsSources) Prebuilt() *android.Prebuilt { diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go index 82ebba7b1..10d99f3a5 100644 --- a/java/droidstubs_test.go +++ b/java/droidstubs_test.go @@ -21,8 +21,6 @@ import ( "strings" "testing" - "github.com/google/blueprint" - "android/soong/android" ) @@ -234,27 +232,6 @@ func TestDroidstubsWithSystemModules(t *testing.T) { checkSystemModulesUseByDroidstubs(t, ctx, "stubs-prebuilt-system-modules", "prebuilt-jar.jar") } -func TestAddJSONData(t *testing.T) { - prebuiltStubsSources := PrebuiltStubsSources{} - prebuiltStubsSources.jsonDataActions = []blueprint.JSONDataAction{ - blueprint.JSONDataAction{ - Inputs: []string{}, - Outputs: []string{}, - }, - } - jsonData := map[string]interface{}{} - prebuiltStubsSources.AddJSONData(&jsonData) - expectedOut := []map[string]interface{}{ - map[string]interface{}{ - "Inputs": []string{}, - "Outputs": []string{}, - }, - } - if !reflect.DeepEqual(jsonData["Actions"], expectedOut) { - t.Errorf("The JSON action data %#v isn't as expected %#v.", jsonData["Actions"], expectedOut) - } -} - func checkSystemModulesUseByDroidstubs(t *testing.T, ctx *android.TestContext, moduleName string, systemJar string) { metalavaRule := ctx.ModuleForTests(moduleName, "android_common").Rule("metalava") var systemJars []string diff --git a/java/hiddenapi.go b/java/hiddenapi.go index 7c8be1e6e..3af5f1c7b 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -297,6 +297,7 @@ func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Pa type hiddenApiAnnotationsDependencyTag struct { blueprint.BaseDependencyTag + android.LicenseAnnotationSharedDependencyTag } // Tag used to mark dependencies on java_library instances that contains Java source files whose diff --git a/java/java.go b/java/java.go index e3e972153..e55f04599 100644 --- a/java/java.go +++ b/java/java.go @@ -274,6 +274,9 @@ type dependencyTag struct { // True if the dependency is relinked at runtime. runtimeLinked bool + + // True if the dependency is a toolchain, for example an annotation processor. + toolchain bool } // installDependencyTag is a dependency tag that is annotated to cause the installed files of the @@ -287,6 +290,8 @@ type installDependencyTag struct { func (d dependencyTag) LicenseAnnotations() []android.LicenseAnnotation { if d.runtimeLinked { return []android.LicenseAnnotation{android.LicenseAnnotationSharedDependency} + } else if d.toolchain { + return []android.LicenseAnnotation{android.LicenseAnnotationToolchain} } return nil } @@ -325,22 +330,23 @@ func IsJniDepTag(depTag blueprint.DependencyTag) bool { var ( dataNativeBinsTag = dependencyTag{name: "dataNativeBins"} + dataDeviceBinsTag = dependencyTag{name: "dataDeviceBins"} staticLibTag = dependencyTag{name: "staticlib"} libTag = dependencyTag{name: "javalib", runtimeLinked: true} java9LibTag = dependencyTag{name: "java9lib", runtimeLinked: true} - pluginTag = dependencyTag{name: "plugin"} - errorpronePluginTag = dependencyTag{name: "errorprone-plugin"} - exportedPluginTag = dependencyTag{name: "exported-plugin"} + pluginTag = dependencyTag{name: "plugin", toolchain: true} + errorpronePluginTag = dependencyTag{name: "errorprone-plugin", toolchain: true} + exportedPluginTag = dependencyTag{name: "exported-plugin", toolchain: true} bootClasspathTag = dependencyTag{name: "bootclasspath", runtimeLinked: true} systemModulesTag = dependencyTag{name: "system modules", runtimeLinked: true} frameworkResTag = dependencyTag{name: "framework-res"} kotlinStdlibTag = dependencyTag{name: "kotlin-stdlib", runtimeLinked: true} kotlinAnnotationsTag = dependencyTag{name: "kotlin-annotations", runtimeLinked: true} - kotlinPluginTag = dependencyTag{name: "kotlin-plugin"} + kotlinPluginTag = dependencyTag{name: "kotlin-plugin", toolchain: true} proguardRaiseTag = dependencyTag{name: "proguard-raise"} certificateTag = dependencyTag{name: "certificate"} instrumentationForTag = dependencyTag{name: "instrumentation_for"} - extraLintCheckTag = dependencyTag{name: "extra-lint-check"} + extraLintCheckTag = dependencyTag{name: "extra-lint-check", toolchain: true} jniLibTag = dependencyTag{name: "jnilib", runtimeLinked: true} syspropPublicStubDepTag = dependencyTag{name: "sysprop public stub"} jniInstallTag = installDependencyTag{name: "jni install"} @@ -831,6 +837,9 @@ type testProperties struct { type hostTestProperties struct { // list of native binary modules that should be installed alongside the test Data_native_bins []string `android:"arch_variant"` + + // list of device binary modules that should be installed alongside the test + Data_device_bins []string `android:"arch_variant"` } type testHelperLibraryProperties struct { @@ -904,6 +913,11 @@ func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { } } + if len(j.testHostProperties.Data_device_bins) > 0 { + deviceVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() + ctx.AddFarVariationDependencies(deviceVariations, dataDeviceBinsTag, j.testHostProperties.Data_device_bins...) + } + if len(j.testProperties.Jni_libs) > 0 { for _, target := range ctx.MultiTargets() { sharedLibVariations := append(target.Variations(), blueprint.Variation{Mutator: "link", Variation: "shared"}) @@ -918,14 +932,35 @@ func (j *TestHost) AddExtraResource(p android.Path) { j.extraResources = append(j.extraResources, p) } +func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { + var configs []tradefed.Config + if len(j.testHostProperties.Data_device_bins) > 0 { + // add Tradefed configuration to push device bins to device for testing + remoteDir := filepath.Join("/data/local/tests/unrestricted/", j.Name()) + options := []tradefed.Option{{Name: "cleanup", Value: "true"}} + for _, bin := range j.testHostProperties.Data_device_bins { + fullPath := filepath.Join(remoteDir, bin) + options = append(options, tradefed.Option{Name: "push-file", Key: bin, Value: fullPath}) + } + configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.PushFilePreparer", options}) + } + + j.Test.generateAndroidBuildActionsWithConfig(ctx, configs) +} + func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { + j.generateAndroidBuildActionsWithConfig(ctx, nil) +} + +func (j *Test) generateAndroidBuildActionsWithConfig(ctx android.ModuleContext, configs []tradefed.Config) { if j.testProperties.Test_options.Unit_test == nil && ctx.Host() { // TODO(b/): Clean temporary heuristic to avoid unexpected onboarding. defaultUnitTest := !inList("tradefed", j.properties.Libs) && !inList("cts", j.testProperties.Test_suites) j.testProperties.Test_options.Unit_test = proptools.BoolPtr(defaultUnitTest) } + j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.testProperties.Test_config, j.testProperties.Test_config_template, - j.testProperties.Test_suites, j.testProperties.Auto_gen_config, j.testProperties.Test_options.Unit_test) + j.testProperties.Test_suites, configs, j.testProperties.Auto_gen_config, j.testProperties.Test_options.Unit_test) j.data = android.PathsForModuleSrc(ctx, j.testProperties.Data) @@ -935,6 +970,10 @@ func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.data = append(j.data, android.OutputFileForModule(ctx, dep, "")) }) + ctx.VisitDirectDepsWithTag(dataDeviceBinsTag, func(dep android.Module) { + j.data = append(j.data, android.OutputFileForModule(ctx, dep, "")) + }) + ctx.VisitDirectDepsWithTag(jniLibTag, func(dep android.Module) { sharedLibInfo := ctx.OtherModuleProvider(dep, cc.SharedLibraryInfoProvider).(cc.SharedLibraryInfo) if sharedLibInfo.SharedLibrary != nil { @@ -967,7 +1006,7 @@ func (j *TestHelperLibrary) GenerateAndroidBuildActions(ctx android.ModuleContex func (j *JavaTestImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.prebuiltTestProperties.Test_config, nil, - j.prebuiltTestProperties.Test_suites, nil, nil) + j.prebuiltTestProperties.Test_suites, nil, nil, nil) j.Import.GenerateAndroidBuildActions(ctx) } @@ -1294,6 +1333,7 @@ type Import struct { android.ModuleBase android.DefaultableModuleBase android.ApexModuleBase + android.BazelModuleBase prebuilt android.Prebuilt android.SdkBase @@ -1649,6 +1689,7 @@ func ImportFactory() android.Module { android.InitPrebuiltModule(module, &module.properties.Jars) android.InitApexModule(module) android.InitSdkAwareModule(module) + android.InitBazelModule(module) InitJavaModule(module, android.HostAndDeviceSupported) return module } @@ -1969,7 +2010,8 @@ type javaLibraryAttributes struct { Javacopts bazel.StringListAttribute } -func javaLibraryBp2Build(ctx android.TopDownMutatorContext, m *Library) { +func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) *javaLibraryAttributes { + //TODO(b/209577426): Support multiple arch variants srcs := bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs)) attrs := &javaLibraryAttributes{ Srcs: srcs, @@ -1979,9 +2021,21 @@ func javaLibraryBp2Build(ctx android.TopDownMutatorContext, m *Library) { attrs.Javacopts = bazel.MakeStringListAttribute(m.properties.Javacflags) } + var deps bazel.LabelList if m.properties.Libs != nil { - attrs.Deps = bazel.MakeLabelListAttribute(android.BazelLabelForModuleDeps(ctx, m.properties.Libs)) + deps.Append(android.BazelLabelForModuleDeps(ctx, m.properties.Libs)) + } + if m.properties.Static_libs != nil { + //TODO(b/217236083) handle static libs similarly to Soong + deps.Append(android.BazelLabelForModuleDeps(ctx, m.properties.Static_libs)) } + attrs.Deps = bazel.MakeLabelListAttribute(deps) + + return attrs +} + +func javaLibraryBp2Build(ctx android.TopDownMutatorContext, m *Library) { + attrs := m.convertLibraryAttrsBp2Build(ctx) props := bazel.BazelTargetModuleProperties{ Rule_class: "java_library", @@ -2067,3 +2121,21 @@ func javaBinaryHostBp2Build(ctx android.TopDownMutatorContext, m *Binary) { // Create the BazelTargetModule. ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs) } + +type bazelJavaImportAttributes struct { + Jars bazel.LabelListAttribute +} + +// java_import bp2Build converter. +func (i *Import) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + //TODO(b/209577426): Support multiple arch variants + jars := bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrcExcludes(ctx, i.properties.Jars, []string(nil))) + + attrs := &bazelJavaImportAttributes{ + Jars: jars, + } + props := bazel.BazelTargetModuleProperties{Rule_class: "java_import"} + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: android.RemoveOptionalPrebuiltPrefix(i.Name())}, attrs) + +} diff --git a/java/java_test.go b/java/java_test.go index 3a51981e6..21c76b6c1 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -1460,3 +1460,64 @@ func TestErrorproneEnabledOnlyByEnvironmentVariable(t *testing.T) { t.Errorf("expected errorprone to contain %q, got %q", expectedSubstring, javac.Args["javacFlags"]) } } + +func TestDataDeviceBinsBuildsDeviceBinary(t *testing.T) { + bp := ` + java_test_host { + name: "foo", + srcs: ["test.java"], + data_device_bins: ["bar"], + } + + cc_binary { + name: "bar", + } + ` + + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + ).RunTestWithBp(t, bp) + + buildOS := ctx.Config.BuildOS.String() + fooVariant := ctx.ModuleForTests("foo", buildOS+"_common") + barVariant := ctx.ModuleForTests("bar", "android_arm64_armv8-a") + fooMod := fooVariant.Module().(*TestHost) + + relocated := barVariant.Output("bar") + expectedInput := "out/soong/.intermediates/bar/android_arm64_armv8-a/unstripped/bar" + android.AssertPathRelativeToTopEquals(t, "relocation input", expectedInput, relocated.Input) + + entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, fooMod)[0] + expectedData := []string{ + "out/soong/.intermediates/bar/android_arm64_armv8-a/bar:bar", + } + actualData := entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"] + android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", ctx.Config, expectedData, actualData) +} + +func TestDataDeviceBinsAutogenTradefedConfig(t *testing.T) { + bp := ` + java_test_host { + name: "foo", + srcs: ["test.java"], + data_device_bins: ["bar"], + } + + cc_binary { + name: "bar", + } + ` + + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + ).RunTestWithBp(t, bp) + + buildOS := ctx.Config.BuildOS.String() + fooModule := ctx.ModuleForTests("foo", buildOS+"_common") + expectedAutogenConfig := `<option name="push-file" key="bar" value="/data/local/tests/unrestricted/foo/bar" />` + + autogen := fooModule.Rule("autogen") + if !strings.Contains(autogen.Args["extraConfigs"], expectedAutogenConfig) { + t.Errorf("foo extraConfigs %v does not contain %q", autogen.Args["extraConfigs"], expectedAutogenConfig) + } +} diff --git a/java/plugin.go b/java/plugin.go index 297ac2cb8..4b174b930 100644 --- a/java/plugin.go +++ b/java/plugin.go @@ -14,7 +14,10 @@ package java -import "android/soong/android" +import ( + "android/soong/android" + "android/soong/bazel" +) func init() { registerJavaPluginBuildComponents(android.InitRegistrationContext) @@ -24,7 +27,6 @@ func registerJavaPluginBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("java_plugin", PluginFactory) } -// A java_plugin module describes a host java library that will be used by javac as an annotation processor. func PluginFactory() android.Module { module := &Plugin{} @@ -32,9 +34,13 @@ func PluginFactory() android.Module { module.AddProperties(&module.pluginProperties) InitJavaModule(module, android.HostSupported) + + android.InitBazelModule(module) + return module } +// Plugin describes a java_plugin module, a host java library that will be used by javac as an annotation processor. type Plugin struct { Library @@ -50,3 +56,29 @@ type PluginProperties struct { // parallelism and cause more recompilation for modules that depend on modules that use this plugin. Generates_api *bool } + +type pluginAttributes struct { + *javaLibraryAttributes + Processor_class *string + Target_compatible_with bazel.LabelListAttribute +} + +// ConvertWithBp2build is used to convert android_app to Bazel. +func (p *Plugin) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + libAttrs := p.convertLibraryAttrsBp2Build(ctx) + attrs := &pluginAttributes{ + libAttrs, + nil, + bazel.LabelListAttribute{}, + } + + if p.pluginProperties.Processor_class != nil { + attrs.Processor_class = p.pluginProperties.Processor_class + } + + props := bazel.BazelTargetModuleProperties{ + Rule_class: "java_plugin", + } + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: p.Name()}, attrs) +} diff --git a/java/proto.go b/java/proto.go index ab913d868..8d2380322 100644 --- a/java/proto.go +++ b/java/proto.go @@ -73,13 +73,15 @@ func genProto(ctx android.ModuleContext, protoFiles android.Paths, flags android } func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) { + const unspecifiedProtobufPluginType = "" if String(p.Proto.Plugin) == "" { switch String(p.Proto.Type) { + case "stream": // does not require additional dependencies case "micro": ctx.AddVariationDependencies(nil, staticLibTag, "libprotobuf-java-micro") case "nano": ctx.AddVariationDependencies(nil, staticLibTag, "libprotobuf-java-nano") - case "lite", "": + case "lite", unspecifiedProtobufPluginType: ctx.AddVariationDependencies(nil, staticLibTag, "libprotobuf-java-lite") case "full": if ctx.Host() || ctx.BazelConversionMode() { diff --git a/java/proto_test.go b/java/proto_test.go new file mode 100644 index 000000000..d1cb71448 --- /dev/null +++ b/java/proto_test.go @@ -0,0 +1,53 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "strings" + "testing" + + "android/soong/android" +) + +const protoModules = ` +java_library_static { + name: "libprotobuf-java-lite", +} +` + +func TestProtoStream(t *testing.T) { + bp := ` + java_library { + name: "java-stream-protos", + proto: { + type: "stream", + }, + srcs: [ + "a.proto", + "b.proto", + ], + } + ` + + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + ).RunTestWithBp(t, protoModules+bp) + + proto0 := ctx.ModuleForTests("java-stream-protos", "android_common").Output("proto/proto0.srcjar") + + if cmd := proto0.RuleParams.Command; !strings.Contains(cmd, "--javastream_out=") { + t.Errorf("expected '--javastream_out' in %q", cmd) + } +} diff --git a/java/rro.go b/java/rro.go index 0b4d0916a..be84afffc 100644 --- a/java/rro.go +++ b/java/rro.go @@ -139,7 +139,7 @@ func (r *RuntimeResourceOverlay) GenerateAndroidBuildActions(ctx android.ModuleC aaptLinkFlags = append(aaptLinkFlags, "--rename-overlay-target-package "+*r.overridableProperties.Target_package_name) } - r.aapt.buildActions(ctx, r, nil, aaptLinkFlags...) + r.aapt.buildActions(ctx, r, nil, nil, aaptLinkFlags...) // Sign the built package _, certificates := collectAppDeps(ctx, r, false, false) diff --git a/java/sdk_library.go b/java/sdk_library.go index 57ab2686f..e794a48c1 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -21,7 +21,6 @@ import ( "reflect" "regexp" "sort" - "strconv" "strings" "sync" @@ -2551,8 +2550,14 @@ func formattedOptionalSdkLevelAttribute(ctx android.ModuleContext, attrName stri ctx.PropertyErrorf(strings.ReplaceAll(attrName, "-", "_"), err.Error()) return "" } - intStr := strconv.Itoa(apiLevel.FinalOrPreviewInt()) - return formattedOptionalAttribute(attrName, &intStr) + if apiLevel.IsCurrent() { + // passing "current" would always mean a future release, never the current (or the current in + // progress) which means some conditions would never be triggered. + ctx.PropertyErrorf(strings.ReplaceAll(attrName, "-", "_"), + `"current" is not an allowed value for this attribute`) + return "" + } + return formattedOptionalAttribute(attrName, value) } // formats an attribute for the xml permissions file if the value is not null @@ -2755,7 +2760,7 @@ type sdkLibrarySdkMemberProperties struct { android.SdkMemberPropertiesBase // Scope to per scope properties. - Scopes map[*apiScope]scopeProperties + Scopes map[*apiScope]*scopeProperties // The Java stubs source files. Stub_srcs []string @@ -2808,14 +2813,14 @@ type scopeProperties struct { StubsSrcJar android.Path CurrentApiFile android.Path RemovedApiFile android.Path - AnnotationsZip android.Path + AnnotationsZip android.Path `supported_build_releases:"Tiramisu+"` SdkVersion string } func (s *sdkLibrarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { sdk := variant.(*SdkLibrary) - s.Scopes = make(map[*apiScope]scopeProperties) + s.Scopes = make(map[*apiScope]*scopeProperties) for _, apiScope := range allApiScopes { paths := sdk.findScopePaths(apiScope) if paths == nil { @@ -2838,7 +2843,7 @@ func (s *sdkLibrarySdkMemberProperties) PopulateFromVariant(ctx android.SdkMembe if paths.annotationsZip.Valid() { properties.AnnotationsZip = paths.annotationsZip.Path() } - s.Scopes[apiScope] = properties + s.Scopes[apiScope] = &properties } } diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index e0e5b5697..3500c84d2 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -182,7 +182,7 @@ func TestJavaSdkLibrary_UpdatableLibrary(t *testing.T) { "30": {"foo", "fooUpdatable", "fooUpdatableErr"}, }), android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - variables.Platform_version_active_codenames = []string{"Tiramisu", "U", "V", "W"} + variables.Platform_version_active_codenames = []string{"Tiramisu", "U", "V", "W", "X"} }), ).RunTestWithBp(t, ` @@ -193,7 +193,7 @@ func TestJavaSdkLibrary_UpdatableLibrary(t *testing.T) { on_bootclasspath_since: "U", on_bootclasspath_before: "V", min_device_sdk: "W", - max_device_sdk: "current", + max_device_sdk: "X", min_sdk_version: "S", } java_sdk_library { @@ -202,12 +202,13 @@ func TestJavaSdkLibrary_UpdatableLibrary(t *testing.T) { api_packages: ["foo"], } `) + // test that updatability attributes are passed on correctly fooUpdatable := result.ModuleForTests("fooUpdatable.xml", "android_common").Rule("java_sdk_xml") - android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `on-bootclasspath-since=\"9001\"`) - android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `on-bootclasspath-before=\"9002\"`) - android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `min-device-sdk=\"9003\"`) - android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `max-device-sdk=\"10000\"`) + android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `on-bootclasspath-since=\"U\"`) + android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `on-bootclasspath-before=\"V\"`) + android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `min-device-sdk=\"W\"`) + android.AssertStringDoesContain(t, "fooUpdatable.xml java_sdk_xml command", fooUpdatable.RuleParams.Command, `max-device-sdk=\"X\"`) // double check that updatability attributes are not written if they don't exist in the bp file // the permissions file for the foo library defined above @@ -230,7 +231,7 @@ func TestJavaSdkLibrary_UpdatableLibrary_Validation_ValidVersion(t *testing.T) { `on_bootclasspath_since: "aaa" could not be parsed as an integer and is not a recognized codename`, `on_bootclasspath_before: "bbc" could not be parsed as an integer and is not a recognized codename`, `min_device_sdk: "ccc" could not be parsed as an integer and is not a recognized codename`, - `max_device_sdk: "ddd" could not be parsed as an integer and is not a recognized codename`, + `max_device_sdk: "current" is not an allowed value for this attribute`, })).RunTestWithBp(t, ` java_sdk_library { @@ -240,7 +241,7 @@ func TestJavaSdkLibrary_UpdatableLibrary_Validation_ValidVersion(t *testing.T) { on_bootclasspath_since: "aaa", on_bootclasspath_before: "bbc", min_device_sdk: "ccc", - max_device_sdk: "ddd", + max_device_sdk: "current", } `) } diff --git a/licenses/Android.bp b/licenses/Android.bp index 5b764dcd4..8db001f44 100644 --- a/licenses/Android.bp +++ b/licenses/Android.bp @@ -18,6 +18,11 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], } +filegroup { + name: "Apache-2.0-License-Text", + srcs: ["LICENSE"], +} + license { name: "Android-Apache-2.0", package_name: "Android", diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index d9b4e86b7..e84eacd86 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -40,7 +40,6 @@ import ( ) var ( - rootDir = flag.String("root", ".", "the value of // for load paths") // TODO(asmundak): remove this option once there is a consensus on suffix suffix = flag.String("suffix", ".rbc", "generated files' suffix") dryRun = flag.Bool("dry_run", false, "dry run") @@ -57,6 +56,7 @@ var ( cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file") traceCalls = flag.Bool("trace_calls", false, "trace function calls") inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables") + makefileList = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)") ) func init() { @@ -70,7 +70,6 @@ func init() { quit("cannot alias unknown flag " + target) } flagAlias("suffix", "s") - flagAlias("root", "d") flagAlias("dry_run", "n") flagAlias("convert_dependents", "r") flagAlias("error_stat", "e") @@ -79,7 +78,7 @@ func init() { var backupSuffix string var tracedVariables []string var errorLogger = errorSink{data: make(map[string]datum)} -var makefileFinder = &LinuxMakefileFinder{} +var makefileFinder mk2rbc.MakefileFinder func main() { flag.Usage = func() { @@ -90,6 +89,10 @@ func main() { } flag.Parse() + if _, err := os.Stat("build/soong/mk2rbc"); err != nil { + quit("Must be run from the root of the android tree. (build/soong/mk2rbc does not exist)") + } + // Delouse if *suffix == ".mk" { quit("cannot use .mk as generated file suffix") @@ -133,6 +136,16 @@ func main() { pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } + + if *makefileList != "" { + makefileFinder = &FileListMakefileFinder{ + cachedMakefiles: nil, + filePath: *makefileList, + } + } else { + makefileFinder = &FindCommandMakefileFinder{} + } + // Find out global variables getConfigVariables() getSoongVariables() @@ -217,17 +230,16 @@ func buildProductConfigMap() map[string]string { const androidProductsMk = "AndroidProducts.mk" // Build the list of AndroidProducts.mk files: it's // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk - targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk) + targetAndroidProductsFile := filepath.Join("build", "make", "target", "product", androidProductsMk) if _, err := os.Stat(targetAndroidProductsFile); err != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n", - targetAndroidProductsFile, err, *rootDir) + fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) } productConfigMap := make(map[string]string) if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) } for _, t := range []string{"device", "vendor"} { - _ = filepath.WalkDir(filepath.Join(*rootDir, t), + _ = filepath.WalkDir(t, func(path string, d os.DirEntry, err error) error { if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk { return nil @@ -243,10 +255,9 @@ func buildProductConfigMap() map[string]string { } func getConfigVariables() { - path := filepath.Join(*rootDir, "build", "make", "core", "product.mk") + path := filepath.Join("build", "make", "core", "product.mk") if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil { - quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)", - err, *rootDir)) + quit(err) } } @@ -259,11 +270,11 @@ func (s fileNameScope) Get(name string) string { if name != "BUILD_SYSTEM" { return fmt.Sprintf("$(%s)", name) } - return filepath.Join(*rootDir, "build", "make", "core") + return filepath.Join("build", "make", "core") } func getSoongVariables() { - path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk") + path := filepath.Join("build", "make", "core", "soong_config.mk") err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables) if err != nil { quit(err) @@ -314,12 +325,11 @@ func convertOne(mkFile string) (ok bool) { mk2starRequest := mk2rbc.Request{ MkFile: mkFile, Reader: nil, - RootDir: *rootDir, OutputDir: *outputTop, OutputSuffix: *suffix, TracedVariables: tracedVariables, TraceCalls: *traceCalls, - SourceFS: os.DirFS(*rootDir), + SourceFS: os.DirFS("."), MakefileFinder: makefileFinder, ErrorLogger: errorLogger, } @@ -519,17 +529,17 @@ func stringsWithFreq(items []string, topN int) (string, int) { return res, len(sorted) } -type LinuxMakefileFinder struct { +// FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that +// runs the unix find command to find all the makefiles in the source tree. +type FindCommandMakefileFinder struct { cachedRoot string cachedMakefiles []string } -func (l *LinuxMakefileFinder) Find(root string) []string { +func (l *FindCommandMakefileFinder) Find(root string) []string { if l.cachedMakefiles != nil && l.cachedRoot == root { return l.cachedMakefiles } - l.cachedRoot = root - l.cachedMakefiles = make([]string, 0) // Return all *.mk files but not in hidden directories. @@ -548,9 +558,60 @@ func (l *LinuxMakefileFinder) Find(root string) []string { panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err)) } scanner := bufio.NewScanner(stdout) + result := make([]string, 0) for scanner.Scan() { - l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./")) + result = append(result, strings.TrimPrefix(scanner.Text(), "./")) } stdout.Close() + err = scanner.Err() + if err != nil { + panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err)) + } + l.cachedRoot = root + l.cachedMakefiles = result + return l.cachedMakefiles +} + +// FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that +// reads a file containing the list of makefiles in the android source tree. +// This file is generated by soong's finder, so that it can be computed while +// soong is already walking the source tree looking for other files. If the root +// to find makefiles under is not the root of the android source tree, it will +// fall back to using FindCommandMakefileFinder. +type FileListMakefileFinder struct { + FindCommandMakefileFinder + cachedMakefiles []string + filePath string +} + +func (l *FileListMakefileFinder) Find(root string) []string { + root, err1 := filepath.Abs(root) + wd, err2 := os.Getwd() + if root != wd || err1 != nil || err2 != nil { + return l.FindCommandMakefileFinder.Find(root) + } + if l.cachedMakefiles != nil { + return l.cachedMakefiles + } + + file, err := os.Open(l.filePath) + if err != nil { + panic(fmt.Errorf("Cannot read makefile list: %s\n", err)) + } + defer file.Close() + + result := make([]string, 0) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 { + result = append(result, line) + } + } + + if err = scanner.Err(); err != nil { + panic(fmt.Errorf("Cannot read makefile list: %s\n", err)) + } + l.cachedMakefiles = result return l.cachedMakefiles } diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go index 3f355ac84..07f7ca189 100644 --- a/mk2rbc/expr.go +++ b/mk2rbc/expr.go @@ -378,32 +378,6 @@ func (eq *eqExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) st } } -// variableDefinedExpr corresponds to Make's ifdef VAR -type variableDefinedExpr struct { - v variable -} - -func (v *variableDefinedExpr) emit(gctx *generationContext) { - if v.v != nil { - v.v.emitDefined(gctx) - return - } - gctx.writef("%s(%q)", cfnWarning, "TODO(VAR)") -} - -func (_ *variableDefinedExpr) typ() starlarkType { - return starlarkTypeBool -} - -func (v *variableDefinedExpr) emitListVarCopy(gctx *generationContext) { - v.emit(gctx) -} - -func (v *variableDefinedExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { - // TODO: VariableDefinedExpr isn't really an expression? - return v -} - type listExpr struct { items []starlarkExpr } diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index e317cadff..cb50a5014 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -142,7 +142,6 @@ var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") type Request struct { MkFile string // file to convert Reader io.Reader // if set, read input from this stream instead - RootDir string // root directory path used to resolve included files OutputSuffix string // generated Starlark files suffix OutputDir string // if set, root of the output hierarchy ErrorLogger ErrorLogger @@ -370,10 +369,6 @@ func init() { } } -type nodeReceiver interface { - newNode(node starlarkNode) -} - // Information about the generated Starlark script. type StarlarkScript struct { mkFile string @@ -382,17 +377,12 @@ type StarlarkScript struct { nodes []starlarkNode inherited []*moduleInfo hasErrors bool - topDir string traceCalls bool // print enter/exit each init function sourceFS fs.FS makefileFinder MakefileFinder nodeLocator func(pos mkparser.Pos) int } -func (ss *StarlarkScript) newNode(node starlarkNode) { - ss.nodes = append(ss.nodes, node) -} - // varAssignmentScope points to the last assignment for each variable // in the current block. It is used during the parsing to chain // the assignments to a variable together. @@ -415,8 +405,6 @@ type parseContext struct { tracedVariables map[string]bool // variables to be traced in the generated script variables map[string]variable varAssignments *varAssignmentScope - receiver nodeReceiver // receptacle for the generated starlarkNode's - receiverStack []nodeReceiver outputDir string dependentModules map[string]*moduleInfo soongNamespaces map[string]map[string]bool @@ -424,11 +412,10 @@ type parseContext struct { } func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { - topdir, _ := filepath.Split(filepath.Join(ss.topDir, "foo")) predefined := []struct{ name, value string }{ {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")}, {"LOCAL_PATH", filepath.Dir(ss.mkFile)}, - {"TOPDIR", topdir}, + {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk // TODO(asmundak): maybe read it from build/make/core/envsetup.mk? {"TARGET_COPY_OUT_SYSTEM", "system"}, {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"}, @@ -503,20 +490,6 @@ func (ctx *parseContext) popVarAssignments() { ctx.varAssignments = ctx.varAssignments.outer } -func (ctx *parseContext) pushReceiver(rcv nodeReceiver) { - ctx.receiverStack = append(ctx.receiverStack, ctx.receiver) - ctx.receiver = rcv -} - -func (ctx *parseContext) popReceiver() { - last := len(ctx.receiverStack) - 1 - if last < 0 { - panic(fmt.Errorf("popReceiver: receiver stack empty")) - } - ctx.receiver = ctx.receiverStack[last] - ctx.receiverStack = ctx.receiverStack[0:last] -} - func (ctx *parseContext) hasNodes() bool { return ctx.currentNodeIndex < len(ctx.nodes) } @@ -537,11 +510,10 @@ func (ctx *parseContext) backNode() { ctx.currentNodeIndex-- } -func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { +func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode { // Handle only simple variables if !a.Name.Const() { - ctx.errorf(a, "Only simple variables are handled") - return + return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")} } name := a.Name.Strings[0] // The `override` directive @@ -549,18 +521,16 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { // is parsed as an assignment to a variable named `override FOO`. // There are very few places where `override` is used, just flag it. if strings.HasPrefix(name, "override ") { - ctx.errorf(a, "cannot handle override directive") + return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")} } // Soong configuration if strings.HasPrefix(name, soongNsPrefix) { - ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a) - return + return ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a) } lhs := ctx.addVariable(name) if lhs == nil { - ctx.errorf(a, "unknown variable %s", name) - return + return []starlarkNode{ctx.newBadNode(a, "unknown variable %s", name)} } _, isTraced := ctx.tracedVariables[name] asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced, location: ctx.errorLocation(a)} @@ -568,8 +538,7 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { // Try to divine variable type from the RHS asgn.value = ctx.parseMakeString(a, a.Value) if xBad, ok := asgn.value.(*badExpr); ok { - ctx.wrapBadExpr(xBad) - return + return []starlarkNode{&exprNode{xBad}} } inferred_type := asgn.value.typ() if inferred_type != starlarkTypeUnknown { @@ -577,9 +546,9 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { } } if lhs.valueType() == starlarkTypeList { - xConcat := ctx.buildConcatExpr(a) - if xConcat == nil { - return + xConcat, xBad := ctx.buildConcatExpr(a) + if xBad != nil { + return []starlarkNode{&exprNode{expr: xBad}} } switch len(xConcat.items) { case 0: @@ -592,8 +561,7 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { } else { asgn.value = ctx.parseMakeString(a, a.Value) if xBad, ok := asgn.value.(*badExpr); ok { - ctx.wrapBadExpr(xBad) - return + return []starlarkNode{&exprNode{expr: xBad}} } } @@ -614,14 +582,13 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { panic(fmt.Errorf("unexpected assignment type %s", a.Type)) } - ctx.receiver.newNode(asgn) + return []starlarkNode{asgn} } -func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Assignment) { +func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Assignment) []starlarkNode { val := ctx.parseMakeString(asgn, asgn.Value) if xBad, ok := val.(*badExpr); ok { - ctx.wrapBadExpr(xBad) - return + return []starlarkNode{&exprNode{expr: xBad}} } // Unfortunately, Soong namespaces can be set up by directly setting corresponding Make @@ -634,17 +601,18 @@ func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Ass // $(call add_soong_config_namespace,foo) s, ok := maybeString(val) if !ok { - ctx.errorf(asgn, "cannot handle variables in SOONG_CONFIG_NAMESPACES assignment, please use add_soong_config_namespace instead") - return + return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_NAMESPACES assignment, please use add_soong_config_namespace instead")} } + result := make([]starlarkNode, 0) for _, ns := range strings.Fields(s) { ctx.addSoongNamespace(ns) - ctx.receiver.newNode(&exprNode{&callExpr{ + result = append(result, &exprNode{&callExpr{ name: baseName + ".soong_config_namespace", args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{ns}}, returnType: starlarkTypeVoid, }}) } + return result } else { // Upon seeing // SOONG_CONFIG_x_y = v @@ -664,45 +632,41 @@ func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Ass continue } if namespaceName != "" { - ctx.errorf(asgn, "ambiguous soong namespace (may be either `%s` or `%s`)", namespaceName, name[0:pos]) - return + return []starlarkNode{ctx.newBadNode(asgn, "ambiguous soong namespace (may be either `%s` or `%s`)", namespaceName, name[0:pos])} } namespaceName = name[0:pos] varName = name[pos+1:] } if namespaceName == "" { - ctx.errorf(asgn, "cannot figure out Soong namespace, please use add_soong_config_var_value macro instead") - return + return []starlarkNode{ctx.newBadNode(asgn, "cannot figure out Soong namespace, please use add_soong_config_var_value macro instead")} } if varName == "" { // Remember variables in this namespace s, ok := maybeString(val) if !ok { - ctx.errorf(asgn, "cannot handle variables in SOONG_CONFIG_ assignment, please use add_soong_config_var_value instead") - return + return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_ assignment, please use add_soong_config_var_value instead")} } ctx.updateSoongNamespace(asgn.Type != "+=", namespaceName, strings.Fields(s)) - return + return []starlarkNode{} } // Finally, handle assignment to a namespace variable if !ctx.hasNamespaceVar(namespaceName, varName) { - ctx.errorf(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName) - return + return []starlarkNode{ctx.newBadNode(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName)} } fname := baseName + "." + soongConfigAssign if asgn.Type == "+=" { fname = baseName + "." + soongConfigAppend } - ctx.receiver.newNode(&exprNode{&callExpr{ + return []starlarkNode{&exprNode{&callExpr{ name: fname, args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{namespaceName}, &stringLiteralExpr{varName}, val}, returnType: starlarkTypeVoid, - }}) + }}} } } -func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr { +func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) (*concatExpr, *badExpr) { xConcat := &concatExpr{} var xItemList *listExpr addToItemList := func(x ...starlarkExpr) { @@ -724,8 +688,7 @@ func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr { // expressions return individual elements. switch x := ctx.parseMakeString(a, item).(type) { case *badExpr: - ctx.wrapBadExpr(x) - return nil + return nil, x case *stringLiteralExpr: addToItemList(maybeConvertToStringList(x).(*listExpr).items...) default: @@ -749,7 +712,7 @@ func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) *concatExpr { if xItemList != nil { xConcat.items = append(xConcat.items, xItemList) } - return xConcat + return xConcat, nil } func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo { @@ -779,7 +742,7 @@ func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleI } func (ctx *parseContext) handleSubConfig( - v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule)) { + v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule) starlarkNode) []starlarkNode { // In a simple case, the name of a module to inherit/include is known statically. if path, ok := maybeString(pathExpr); ok { @@ -788,18 +751,19 @@ func (ctx *parseContext) handleSubConfig( moduleShouldExist := loadAlways && ctx.ifNestLevel == 0 if strings.Contains(path, "*") { if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil { + result := make([]starlarkNode, 0) for _, p := range paths { mi := ctx.newDependentModule(p, !moduleShouldExist) - processModule(inheritedStaticModule{mi, loadAlways}) + result = append(result, processModule(inheritedStaticModule{mi, loadAlways})) } + return result } else { - ctx.errorf(v, "cannot glob wildcard argument") + return []starlarkNode{ctx.newBadNode(v, "cannot glob wildcard argument")} } } else { mi := ctx.newDependentModule(path, !moduleShouldExist) - processModule(inheritedStaticModule{mi, loadAlways}) + return []starlarkNode{processModule(inheritedStaticModule{mi, loadAlways})} } - return } // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the @@ -819,8 +783,7 @@ func (ctx *parseContext) handleSubConfig( var matchingPaths []string varPath, ok := pathExpr.(*interpolateExpr) if !ok { - ctx.errorf(v, "inherit-product/include argument is too complex") - return + return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} } pathPattern := []string{varPath.chunks[0]} @@ -829,11 +792,7 @@ func (ctx *parseContext) handleSubConfig( pathPattern = append(pathPattern, chunk) } } - if pathPattern[0] == "" { - if len(ctx.includeTops) == 0 { - ctx.errorf(v, "inherit-product/include statements must not be prefixed with a variable, or must include a #RBC# include_top comment beforehand giving a root directory to search.") - return - } + if pathPattern[0] == "" && len(ctx.includeTops) > 0 { // If pattern starts from the top. restrict it to the directories where // we know inherit-product uses dynamically calculated path. for _, p := range ctx.includeTops { @@ -846,21 +805,26 @@ func (ctx *parseContext) handleSubConfig( // Safeguard against $(call inherit-product,$(PRODUCT_PATH)) const maxMatchingFiles = 150 if len(matchingPaths) > maxMatchingFiles { - ctx.errorf(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles) - return + return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)} } - res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways} - for _, p := range matchingPaths { - // A product configuration files discovered dynamically may attempt to inherit - // from another one which does not exist in this source tree. Prevent load errors - // by always loading the dynamic files as optional. - res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true)) + if len(matchingPaths) == 1 { + res := inheritedStaticModule{ctx.newDependentModule(matchingPaths[0], loadAlways && ctx.ifNestLevel == 0), loadAlways} + return []starlarkNode{processModule(res)} + } else { + needsWarning := pathPattern[0] == "" && len(ctx.includeTops) == 0 + res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} + for _, p := range matchingPaths { + // A product configuration files discovered dynamically may attempt to inherit + // from another one which does not exist in this source tree. Prevent load errors + // by always loading the dynamic files as optional. + res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true)) + } + return []starlarkNode{processModule(res)} } - processModule(res) } func (ctx *parseContext) findMatchingPaths(pattern []string) []string { - files := ctx.script.makefileFinder.Find(ctx.script.topDir) + files := ctx.script.makefileFinder.Find(".") if len(pattern) == 0 { return files } @@ -883,25 +847,25 @@ func (ctx *parseContext) findMatchingPaths(pattern []string) []string { return res } -func (ctx *parseContext) handleInheritModule(v mkparser.Node, args *mkparser.MakeString, loadAlways bool) { +func (ctx *parseContext) handleInheritModule(v mkparser.Node, args *mkparser.MakeString, loadAlways bool) []starlarkNode { args.TrimLeftSpaces() args.TrimRightSpaces() pathExpr := ctx.parseMakeString(v, args) if _, ok := pathExpr.(*badExpr); ok { - ctx.errorf(v, "Unable to parse argument to inherit") + return []starlarkNode{ctx.newBadNode(v, "Unable to parse argument to inherit")} } - ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) { - ctx.receiver.newNode(&inheritNode{im, loadAlways}) + return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode { + return &inheritNode{im, loadAlways} }) } -func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) { - ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) { - ctx.receiver.newNode(&includeNode{im, loadAlways}) +func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) []starlarkNode { + return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode { + return &includeNode{im, loadAlways} }) } -func (ctx *parseContext) handleVariable(v *mkparser.Variable) { +func (ctx *parseContext) handleVariable(v *mkparser.Variable) []starlarkNode { // Handle: // $(call inherit-product,...) // $(call inherit-product-if-exists,...) @@ -916,67 +880,57 @@ func (ctx *parseContext) handleVariable(v *mkparser.Variable) { if strings.HasPrefix(v.Name.Dump(), "call inherit-product,") { args := v.Name.Clone() args.ReplaceLiteral("call inherit-product,", "") - ctx.handleInheritModule(v, args, true) - return + return ctx.handleInheritModule(v, args, true) } if strings.HasPrefix(v.Name.Dump(), "call inherit-product-if-exists,") { args := v.Name.Clone() args.ReplaceLiteral("call inherit-product-if-exists,", "") - ctx.handleInheritModule(v, args, false) - return - } - expr := ctx.parseReference(v, v.Name) - switch x := expr.(type) { - case *callExpr: - ctx.receiver.newNode(&exprNode{expr}) - case *badExpr: - ctx.wrapBadExpr(x) - default: - ctx.errorf(v, "cannot handle %s", v.Dump()) + return ctx.handleInheritModule(v, args, false) } + return []starlarkNode{&exprNode{expr: ctx.parseReference(v, v.Name)}} } -func (ctx *parseContext) handleDefine(directive *mkparser.Directive) { +func (ctx *parseContext) maybeHandleDefine(directive *mkparser.Directive) starlarkNode { macro_name := strings.Fields(directive.Args.Strings[0])[0] // Ignore the macros that we handle _, ignored := ignoredDefines[macro_name] _, known := knownFunctions[macro_name] if !ignored && !known { - ctx.errorf(directive, "define is not supported: %s", macro_name) + return ctx.newBadNode(directive, "define is not supported: %s", macro_name) } + return nil } -func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) { - ssSwitch := &switchNode{} - ctx.pushReceiver(ssSwitch) - for ctx.processBranch(ifDirective); ctx.hasNodes() && ctx.fatalError == nil; { +func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) starlarkNode { + ssSwitch := &switchNode{ + ssCases: []*switchCase{ctx.processBranch(ifDirective)}, + } + for ctx.hasNodes() && ctx.fatalError == nil { node := ctx.getNode() switch x := node.(type) { case *mkparser.Directive: switch x.Name { case "else", "elifdef", "elifndef", "elifeq", "elifneq": - ctx.processBranch(x) + ssSwitch.ssCases = append(ssSwitch.ssCases, ctx.processBranch(x)) case "endif": - ctx.popReceiver() - ctx.receiver.newNode(ssSwitch) - return + return ssSwitch default: - ctx.errorf(node, "unexpected directive %s", x.Name) + return ctx.newBadNode(node, "unexpected directive %s", x.Name) } default: - ctx.errorf(ifDirective, "unexpected statement") + return ctx.newBadNode(ifDirective, "unexpected statement") } } if ctx.fatalError == nil { ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump()) } - ctx.popReceiver() + return ctx.newBadNode(ifDirective, "no matching endif for %s", ifDirective.Dump()) } // processBranch processes a single branch (if/elseif/else) until the next directive // on the same level. -func (ctx *parseContext) processBranch(check *mkparser.Directive) { - block := switchCase{gate: ctx.parseCondition(check)} +func (ctx *parseContext) processBranch(check *mkparser.Directive) *switchCase { + block := &switchCase{gate: ctx.parseCondition(check)} defer func() { ctx.popVarAssignments() ctx.ifNestLevel-- @@ -985,37 +939,29 @@ func (ctx *parseContext) processBranch(check *mkparser.Directive) { ctx.pushVarAssignments() ctx.ifNestLevel++ - ctx.pushReceiver(&block) for ctx.hasNodes() { node := ctx.getNode() if d, ok := node.(*mkparser.Directive); ok { switch d.Name { case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif": - ctx.popReceiver() - ctx.receiver.newNode(&block) ctx.backNode() - return + return block } } - ctx.handleSimpleStatement(node) + block.nodes = append(block.nodes, ctx.handleSimpleStatement(node)...) } ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump()) - ctx.popReceiver() -} - -func (ctx *parseContext) newIfDefinedNode(check *mkparser.Directive) (starlarkExpr, bool) { - if !check.Args.Const() { - return ctx.newBadExpr(check, "ifdef variable ref too complex: %s", check.Args.Dump()), false - } - v := ctx.addVariable(check.Args.Strings[0]) - return &variableDefinedExpr{v}, true + return block } func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode { switch check.Name { case "ifdef", "ifndef", "elifdef", "elifndef": - v, ok := ctx.newIfDefinedNode(check) - if ok && strings.HasSuffix(check.Name, "ndef") { + if !check.Args.Const() { + return ctx.newBadNode(check, "ifdef variable ref too complex: %s", check.Args.Dump()) + } + v := NewVariableRefExpr(ctx.addVariable(check.Args.Strings[0]), false) + if strings.HasSuffix(check.Name, "ndef") { v = ¬Expr{v} } return &ifNode{ @@ -1035,12 +981,16 @@ func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode } func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr { - message := fmt.Sprintf(text, args...) if ctx.errorLogger != nil { ctx.errorLogger.NewError(ctx.errorLocation(node), node, text, args...) } ctx.script.hasErrors = true - return &badExpr{errorLocation: ctx.errorLocation(node), message: message} + return &badExpr{errorLocation: ctx.errorLocation(node), message: fmt.Sprintf(text, args...)} +} + +// records that the given node failed to be converted and includes an explanatory message +func (ctx *parseContext) newBadNode(failedNode mkparser.Node, message string, args ...interface{}) starlarkNode { + return &exprNode{ctx.newBadExpr(failedNode, message, args...)} } func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { @@ -1142,7 +1092,7 @@ func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { // Given an if statement's directive and the left/right starlarkExprs, // check if the starlarkExprs are one of a few hardcoded special cases -// that can be converted to a simpler equalify expression than simply comparing +// that can be converted to a simpler equality expression than simply comparing // the two. func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, left starlarkExpr, right starlarkExpr) (starlarkExpr, bool) { @@ -1171,8 +1121,8 @@ func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, } switch call.name { - case baseName + ".filter", baseName + ".filter-out": - return ctx.parseCompareFilterFuncResult(directive, call, value, isEq), true + case baseName + ".filter": + return ctx.parseCompareFilterFuncResult(directive, call, value, isEq) case baseName + ".expand_wildcard": return ctx.parseCompareWildcardFuncResult(directive, call, value, !isEq), true case baseName + ".findstring": @@ -1184,68 +1134,39 @@ func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, } func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive, - filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { + filterFuncCall *callExpr, xValue starlarkExpr, negate bool) (starlarkExpr, bool) { // We handle: // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...] // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...] - // * ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"] - // TODO(Asmundak): check the last case works for filter-out, too. + if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" { + return nil, false + } xPattern := filterFuncCall.args[0] xText := filterFuncCall.args[1] var xInList *stringLiteralExpr var expr starlarkExpr var ok bool - switch x := xValue.(type) { - case *stringLiteralExpr: - if x.literal != "" { - return ctx.newBadExpr(cond, "filter comparison to non-empty value: %s", xValue) - } - // Either pattern or text should be const, and the - // non-const one should be varRefExpr - if xInList, ok = xPattern.(*stringLiteralExpr); ok && !strings.ContainsRune(xInList.literal, '%') && xText.typ() == starlarkTypeList { - expr = xText - } else if xInList, ok = xText.(*stringLiteralExpr); ok { - expr = xPattern - } else { - expr = &callExpr{ - object: nil, - name: filterFuncCall.name, - args: filterFuncCall.args, - returnType: starlarkTypeBool, - } - if negate { - expr = ¬Expr{expr: expr} - } - return expr - } - case *variableRefExpr: - if v, ok := xPattern.(*variableRefExpr); ok { - if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() { - // ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate, - // it's the opposite to what is done when comparing to empty. - expr = xPattern - negate = !negate - } - } + if xInList, ok = xPattern.(*stringLiteralExpr); ok && !strings.ContainsRune(xInList.literal, '%') && xText.typ() == starlarkTypeList { + expr = xText + } else if xInList, ok = xText.(*stringLiteralExpr); ok { + expr = xPattern + } else { + return nil, false } - if expr != nil && xInList != nil { - slExpr := newStringListExpr(strings.Fields(xInList.literal)) - // Generate simpler code for the common cases: - if expr.typ() == starlarkTypeList { - if len(slExpr.items) == 1 { - // Checking that a string belongs to list - return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]} - } else { - // TODO(asmundak): - panic("TBD") - } - } else if len(slExpr.items) == 1 { - return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate} + slExpr := newStringListExpr(strings.Fields(xInList.literal)) + // Generate simpler code for the common cases: + if expr.typ() == starlarkTypeList { + if len(slExpr.items) == 1 { + // Checking that a string belongs to list + return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}, true } else { - return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr} + return nil, false } + } else if len(slExpr.items) == 1 { + return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}, true + } else { + return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}, true } - return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump()) } func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive, @@ -1733,28 +1654,34 @@ func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeSt // Handles the statements whose treatment is the same in all contexts: comment, // assignment, variable (which is a macro call in reality) and all constructs that // do not handle in any context ('define directive and any unrecognized stuff). -func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) { +func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNode { + var result []starlarkNode switch x := node.(type) { case *mkparser.Comment: - ctx.maybeHandleAnnotation(x) - ctx.insertComment("#" + x.Comment) + if n, handled := ctx.maybeHandleAnnotation(x); handled && n != nil { + result = []starlarkNode{n} + } else if !handled { + result = []starlarkNode{&commentNode{strings.TrimSpace("#" + x.Comment)}} + } case *mkparser.Assignment: - ctx.handleAssignment(x) + result = ctx.handleAssignment(x) case *mkparser.Variable: - ctx.handleVariable(x) + result = ctx.handleVariable(x) case *mkparser.Directive: switch x.Name { case "define": - ctx.handleDefine(x) + if res := ctx.maybeHandleDefine(x); res != nil { + result = []starlarkNode{res} + } case "include", "-include": - ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') + result = ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') case "ifeq", "ifneq", "ifdef", "ifndef": - ctx.handleIfBlock(x) + result = []starlarkNode{ctx.handleIfBlock(x)} default: - ctx.errorf(x, "unexpected directive %s", x.Name) + result = []starlarkNode{ctx.newBadNode(x, "unexpected directive %s", x.Name)} } default: - ctx.errorf(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#")) + result = []starlarkNode{ctx.newBadNode(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#"))} } // Clear the includeTops after each non-comment statement @@ -1763,12 +1690,17 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) { if _, wasComment := node.(*mkparser.Comment); !wasComment && len(ctx.includeTops) > 0 { ctx.includeTops = []string{} } + + if result == nil { + result = []starlarkNode{} + } + return result } // Processes annotation. An annotation is a comment that starts with #RBC# and provides // a conversion hint -- say, where to look for the dynamically calculated inherit/include -// paths. -func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) { +// paths. Returns true if the comment was a successfully-handled annotation. +func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) (starlarkNode, bool) { maybeTrim := func(s, prefix string) (string, bool) { if strings.HasPrefix(s, prefix) { return strings.TrimSpace(strings.TrimPrefix(s, prefix)), true @@ -1777,44 +1709,20 @@ func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) { } annotation, ok := maybeTrim(cnode.Comment, annotationCommentPrefix) if !ok { - return + return nil, false } if p, ok := maybeTrim(annotation, "include_top"); ok { // Don't allow duplicate include tops, because then we will generate // invalid starlark code. (duplicate keys in the _entry dictionary) for _, top := range ctx.includeTops { if top == p { - return + return nil, true } } ctx.includeTops = append(ctx.includeTops, p) - return - } - ctx.errorf(cnode, "unsupported annotation %s", cnode.Comment) - -} - -func (ctx *parseContext) insertComment(s string) { - ctx.receiver.newNode(&commentNode{strings.TrimSpace(s)}) -} - -func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) { - for _, line := range strings.Split(failedNode.Dump(), "\n") { - ctx.insertComment("# " + line) + return nil, true } -} - -// records that the given node failed to be converted and includes an explanatory message -func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) { - if ctx.errorLogger != nil { - ctx.errorLogger.NewError(ctx.errorLocation(failedNode), failedNode, message, args...) - } - ctx.receiver.newNode(&exprNode{ctx.newBadExpr(failedNode, message, args...)}) - ctx.script.hasErrors = true -} - -func (ctx *parseContext) wrapBadExpr(xBad *badExpr) { - ctx.receiver.newNode(&exprNode{xBad}) + return ctx.newBadNode(cnode, "unsupported annotation %s", cnode.Comment), true } func (ctx *parseContext) loadedModulePath(path string) string { @@ -1924,11 +1832,11 @@ func Convert(req Request) (*StarlarkScript, error) { starScript := &StarlarkScript{ moduleName: moduleNameForFile(req.MkFile), mkFile: req.MkFile, - topDir: req.RootDir, traceCalls: req.TraceCalls, sourceFS: req.SourceFS, makefileFinder: req.MakefileFinder, nodeLocator: func(pos mkparser.Pos) int { return parser.Unpack(pos).Line }, + nodes: make([]starlarkNode, 0), } ctx := newParseContext(starScript, nodes) ctx.outputSuffix = req.OutputSuffix @@ -1940,9 +1848,8 @@ func Convert(req Request) (*StarlarkScript, error) { ctx.tracedVariables[v] = true } } - ctx.pushReceiver(starScript) for ctx.hasNodes() && ctx.fatalError == nil { - ctx.handleSimpleStatement(ctx.getNode()) + starScript.nodes = append(starScript.nodes, ctx.handleSimpleStatement(ctx.getNode())...) } if ctx.fatalError != nil { return nil, ctx.fatalError diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index d62882d45..447f658ee 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -131,7 +131,7 @@ load(":part1.star|init", _part1_init = "init") def init(g, handle): cfg = rblf.cfg(handle) rblf.inherit(handle, "part", _part_init) - if g.get("PRODUCT_NAME") != None: + if cfg.get("PRODUCT_NAME", ""): if not _part1_init: rblf.mkerror("product.mk", "Cannot find %s" % (":part1.star")) rblf.inherit(handle, "part1", _part1_init) @@ -174,7 +174,7 @@ load(":part1.star|init", _part1_init = "init") def init(g, handle): cfg = rblf.cfg(handle) _part_init(g, handle) - if g.get("PRODUCT_NAME") != None: + if cfg.get("PRODUCT_NAME", ""): if not _part1_init: rblf.mkerror("product.mk", "Cannot find %s" % (":part1.star")) _part1_init(g, handle) @@ -231,7 +231,7 @@ endif def init(g, handle): cfg = rblf.cfg(handle) - if g.get("PRODUCT_NAME") != None: + if cfg.get("PRODUCT_NAME", ""): cfg["PRODUCT_NAME"] = "gizmo" else: pass @@ -275,7 +275,7 @@ endif def init(g, handle): cfg = rblf.cfg(handle) - if g.get("PRODUCT_NAME") != None: + if cfg.get("PRODUCT_NAME", ""): # Comment pass else: @@ -296,7 +296,7 @@ endif def init(g, handle): cfg = rblf.cfg(handle) - if not g.get("PRODUCT_NAME") != None: + if not cfg.get("PRODUCT_NAME", ""): cfg["PRODUCT_NAME"] = "gizmo1" else: cfg["PRODUCT_NAME"] = "gizmo2" @@ -315,9 +315,9 @@ endif def init(g, handle): cfg = rblf.cfg(handle) - if g.get("PRODUCT_NAME") != None: + if cfg.get("PRODUCT_NAME", ""): cfg["PRODUCT_NAME"] = "gizmo" - elif not g.get("PRODUCT_PACKAGES") != None: + elif not cfg.get("PRODUCT_PACKAGES", []): # Comment pass `, @@ -389,6 +389,10 @@ ifneq (,$(filter plaf,$(PLATFORM_LIST))) endif ifeq ($(TARGET_BUILD_VARIANT), $(filter $(TARGET_BUILD_VARIANT), userdebug eng)) endif +ifneq (, $(filter $(TARGET_BUILD_VARIANT), userdebug eng)) +endif +ifneq (,$(filter userdebug eng, $(TARGET_BUILD_VARIANT))) +endif ifneq (,$(filter true, $(v1)$(v2))) endif ifeq (,$(filter barbet coral%,$(TARGET_PRODUCT))) @@ -407,8 +411,12 @@ def init(g, handle): pass if "plaf" in g.get("PLATFORM_LIST", []): pass + if g["TARGET_BUILD_VARIANT"] == " ".join(rblf.filter(g["TARGET_BUILD_VARIANT"], "userdebug eng")): + pass if g["TARGET_BUILD_VARIANT"] in ["userdebug", "eng"]: pass + if rblf.filter("userdebug eng", g["TARGET_BUILD_VARIANT"]): + pass if rblf.filter("true", "%s%s" % (_v1, _v2)): pass if not rblf.filter("barbet coral%", g["TARGET_PRODUCT"]): @@ -509,11 +517,11 @@ endif def init(g, handle): cfg = rblf.cfg(handle) - if g.get("PRODUCT_NAME") != None: + if cfg.get("PRODUCT_NAME", ""): cfg["PRODUCT_PACKAGES"] = ["pack-if0"] - if g.get("PRODUCT_MODEL") != None: + if cfg.get("PRODUCT_MODEL", ""): cfg["PRODUCT_PACKAGES"] = ["pack-if-if"] - elif g.get("PRODUCT_NAME") != None: + elif cfg.get("PRODUCT_NAME", ""): cfg["PRODUCT_PACKAGES"] = ["pack-if-elif"] else: cfg["PRODUCT_PACKAGES"] = ["pack-if-else"] @@ -1048,8 +1056,8 @@ def init(g, handle): cfg = rblf.cfg(handle) g["MY_PATH"] = "foo" _entry = { - "vendor/foo1/cfg.mk": ("_cfg", _cfg_init), - "vendor/bar/baz/cfg.mk": ("_cfg1", _cfg1_init), + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + "vendor/bar/baz/cfg.mk": ("vendor/bar/baz/cfg", _cfg1_init), }.get("vendor/%s/cfg.mk" % g["MY_PATH"]) (_varmod, _varmod_init) = _entry if _entry else (None, None) if not _varmod_init: @@ -1071,14 +1079,7 @@ load("//vendor/foo1:cfg.star|init", _cfg_init = "init") def init(g, handle): cfg = rblf.cfg(handle) g["MY_PATH"] = "foo" - #RBC# include_top vendor/foo1 - _entry = { - "vendor/foo1/cfg.mk": ("_cfg", _cfg_init), - }.get("%s/cfg.mk" % g["MY_PATH"]) - (_varmod, _varmod_init) = _entry if _entry else (None, None) - if not _varmod_init: - rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"])) - rblf.inherit(handle, _varmod, _varmod_init) + rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init) `, }, { @@ -1089,6 +1090,7 @@ MY_PATH:=foo #RBC# include_top vendor/foo1 $(call inherit-product,$(MY_PATH)/cfg.mk) #RBC# include_top vendor/foo1 +#RBC# include_top vendor/foo1 $(call inherit-product,$(MY_PATH)/cfg.mk) `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -1097,26 +1099,12 @@ load("//vendor/foo1:cfg.star|init", _cfg_init = "init") def init(g, handle): cfg = rblf.cfg(handle) g["MY_PATH"] = "foo" - #RBC# include_top vendor/foo1 - _entry = { - "vendor/foo1/cfg.mk": ("_cfg", _cfg_init), - }.get("%s/cfg.mk" % g["MY_PATH"]) - (_varmod, _varmod_init) = _entry if _entry else (None, None) - if not _varmod_init: - rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"])) - rblf.inherit(handle, _varmod, _varmod_init) - #RBC# include_top vendor/foo1 - _entry = { - "vendor/foo1/cfg.mk": ("_cfg", _cfg_init), - }.get("%s/cfg.mk" % g["MY_PATH"]) - (_varmod, _varmod_init) = _entry if _entry else (None, None) - if not _varmod_init: - rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"])) - rblf.inherit(handle, _varmod, _varmod_init) + rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init) + rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init) `, }, { - desc: "Dynamic inherit path that lacks necessary hint", + desc: "Dynamic inherit path that lacks hint", mkname: "product.mk", in: ` #RBC# include_top foo @@ -1130,29 +1118,24 @@ $(call inherit-product,$(MY_VAR)/font.mk) $(call inherit-product,$(MY_VAR)/font.mk) `, - expected: `#RBC# include_top foo -load("//build/make/core:product_config.rbc", "rblf") + expected: `load("//build/make/core:product_config.rbc", "rblf") load("//foo:font.star|init", _font_init = "init") +load("//bar:font.star|init", _font1_init = "init") def init(g, handle): cfg = rblf.cfg(handle) - _entry = { - "foo/font.mk": ("_font", _font_init), - }.get("%s/font.mk" % g.get("MY_VAR", "")) - (_varmod, _varmod_init) = _entry if _entry else (None, None) - if not _varmod_init: - rblf.mkerror("product.mk", "Cannot find %s" % ("%s/font.mk" % g.get("MY_VAR", ""))) - rblf.inherit(handle, _varmod, _varmod_init) - #RBC# include_top foo + rblf.inherit(handle, "foo/font", _font_init) # There's some space and even this comment between the include_top and the inherit-product + rblf.inherit(handle, "foo/font", _font_init) + rblf.mkwarning("product.mk:11", "Please avoid starting an include path with a variable. See https://source.android.com/setup/build/bazel/product_config/issues/includes for details.") _entry = { - "foo/font.mk": ("_font", _font_init), + "foo/font.mk": ("foo/font", _font_init), + "bar/font.mk": ("bar/font", _font1_init), }.get("%s/font.mk" % g.get("MY_VAR", "")) (_varmod, _varmod_init) = _entry if _entry else (None, None) if not _varmod_init: rblf.mkerror("product.mk", "Cannot find %s" % ("%s/font.mk" % g.get("MY_VAR", ""))) rblf.inherit(handle, _varmod, _varmod_init) - rblf.mk2rbc_error("product.mk:11", "inherit-product/include statements must not be prefixed with a variable, or must include a #RBC# include_top comment beforehand giving a root directory to search.") `, }, { @@ -1178,7 +1161,6 @@ override FOO:=`, def init(g, handle): cfg = rblf.cfg(handle) rblf.mk2rbc_error("product.mk:2", "cannot handle override directive") - g["override FOO"] = "" `, }, { @@ -1413,7 +1395,6 @@ func TestGood(t *testing.T) { ss, err := Convert(Request{ MkFile: test.mkname, Reader: bytes.NewBufferString(test.in), - RootDir: ".", OutputSuffix: ".star", SourceFS: fs, MakefileFinder: &testMakefileFinder{fs: fs}, diff --git a/mk2rbc/node.go b/mk2rbc/node.go index 333a8da17..5d98d7bc1 100644 --- a/mk2rbc/node.go +++ b/mk2rbc/node.go @@ -54,6 +54,10 @@ func (im moduleInfo) entryName() string { return im.moduleLocalName + "_init" } +func (mi moduleInfo) name() string { + return fmt.Sprintf("%q", MakePath2ModuleName(mi.originalPath)) +} + type inheritedModule interface { name() string entryName() string @@ -67,10 +71,6 @@ type inheritedStaticModule struct { loadAlways bool } -func (im inheritedStaticModule) name() string { - return fmt.Sprintf("%q", MakePath2ModuleName(im.originalPath)) -} - func (im inheritedStaticModule) emitSelect(_ *generationContext) { } @@ -86,6 +86,8 @@ type inheritedDynamicModule struct { path interpolateExpr candidateModules []*moduleInfo loadAlways bool + location ErrorLocation + needsWarning bool } func (i inheritedDynamicModule) name() string { @@ -97,12 +99,16 @@ func (i inheritedDynamicModule) entryName() string { } func (i inheritedDynamicModule) emitSelect(gctx *generationContext) { + if i.needsWarning { + gctx.newLine() + gctx.writef("%s.mkwarning(%q, %q)", baseName, i.location, "Please avoid starting an include path with a variable. See https://source.android.com/setup/build/bazel/product_config/issues/includes for details.") + } gctx.newLine() gctx.writef("_entry = {") gctx.indentLevel++ for _, mi := range i.candidateModules { gctx.newLine() - gctx.writef(`"%s": (%q, %s),`, mi.originalPath, mi.moduleLocalName, mi.entryName()) + gctx.writef(`"%s": (%s, %s),`, mi.originalPath, mi.name(), mi.entryName()) } gctx.indentLevel-- gctx.newLine() @@ -249,29 +255,17 @@ type switchCase struct { nodes []starlarkNode } -func (cb *switchCase) newNode(node starlarkNode) { - cb.nodes = append(cb.nodes, node) -} - func (cb *switchCase) emit(gctx *generationContext) { cb.gate.emit(gctx) gctx.indentLevel++ hasStatements := false - emitNode := func(node starlarkNode) { + for _, node := range cb.nodes { if _, ok := node.(*commentNode); !ok { hasStatements = true } node.emit(gctx) } - if len(cb.nodes) > 0 { - emitNode(cb.nodes[0]) - for _, node := range cb.nodes[1:] { - emitNode(node) - } - if !hasStatements { - gctx.emitPass() - } - } else { + if !hasStatements { gctx.emitPass() } gctx.indentLevel-- @@ -282,22 +276,8 @@ type switchNode struct { ssCases []*switchCase } -func (ssw *switchNode) newNode(node starlarkNode) { - switch br := node.(type) { - case *switchCase: - ssw.ssCases = append(ssw.ssCases, br) - default: - panic(fmt.Errorf("expected switchCase node, got %t", br)) - } -} - func (ssw *switchNode) emit(gctx *generationContext) { - if len(ssw.ssCases) == 0 { - gctx.emitPass() - } else { - ssw.ssCases[0].emit(gctx) - for _, ssCase := range ssw.ssCases[1:] { - ssCase.emit(gctx) - } + for _, ssCase := range ssw.ssCases { + ssCase.emit(gctx) } } diff --git a/rust/binary.go b/rust/binary.go index db91ccb9a..0dc320e5f 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -92,14 +92,23 @@ func (binary *binaryDecorator) compilerFlags(ctx ModuleContext, flags Flags) Fla func (binary *binaryDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { deps = binary.baseCompiler.compilerDeps(ctx, deps) + static := Bool(binary.Properties.Static_executable) if ctx.toolchain().Bionic() { - deps = bionicDeps(ctx, deps, Bool(binary.Properties.Static_executable)) - if Bool(binary.Properties.Static_executable) { - deps.CrtBegin = "crtbegin_static" + deps = bionicDeps(ctx, deps, static) + if static { + deps.CrtBegin = []string{"crtbegin_static"} + } else { + deps.CrtBegin = []string{"crtbegin_dynamic"} + } + deps.CrtEnd = []string{"crtend_android"} + } else if ctx.Os() == android.LinuxMusl { + deps = muslDeps(ctx, deps, static) + if static { + deps.CrtBegin = []string{"libc_musl_crtbegin_static"} } else { - deps.CrtBegin = "crtbegin_dynamic" + deps.CrtBegin = []string{"libc_musl_crtbegin_dynamic", "musl_linker_script"} } - deps.CrtEnd = "crtend_android" + deps.CrtEnd = []string{"libc_musl_crtend"} } return deps diff --git a/rust/bindgen.go b/rust/bindgen.go index ef5702bcd..f4c337d69 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -288,6 +288,8 @@ func (b *bindgenDecorator) SourceProviderDeps(ctx DepsContext, deps Deps) Deps { deps = b.BaseSourceProvider.SourceProviderDeps(ctx, deps) if ctx.toolchain().Bionic() { deps = bionicDeps(ctx, deps, false) + } else if ctx.Os() == android.LinuxMusl { + deps = muslDeps(ctx, deps, false) } deps.SharedLibs = append(deps.SharedLibs, b.ClangProperties.Shared_libs...) diff --git a/rust/builder.go b/rust/builder.go index e66a6f071..00035b9b7 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -247,9 +247,8 @@ func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, fl implicits = append(implicits, deps.srcProviderFiles...) implicits = append(implicits, deps.AfdoProfiles...) - if deps.CrtBegin.Valid() { - implicits = append(implicits, deps.CrtBegin.Path(), deps.CrtEnd.Path()) - } + implicits = append(implicits, deps.CrtBegin...) + implicits = append(implicits, deps.CrtEnd...) if len(deps.SrcDeps) > 0 { moduleGenDir := ctx.RustModule().compiler.CargoOutDir() @@ -319,8 +318,8 @@ func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, fl "rustcFlags": strings.Join(rustcFlags, " "), "linkFlags": strings.Join(linkFlags, " "), "libFlags": strings.Join(libFlags, " "), - "crtBegin": deps.CrtBegin.String(), - "crtEnd": deps.CrtEnd.String(), + "crtBegin": strings.Join(deps.CrtBegin.Strings(), " "), + "crtEnd": strings.Join(deps.CrtEnd.Strings(), " "), "envVars": strings.Join(envVars, " "), }, }) diff --git a/rust/compiler.go b/rust/compiler.go index 3040e5d9e..c5d40f4dc 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -364,7 +364,7 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { if !Bool(compiler.Properties.No_stdlibs) { for _, stdlib := range config.Stdlibs { // If we're building for the build host, use the prebuilt stdlibs - if ctx.Target().Os == ctx.Config().BuildOS { + if ctx.Target().Os == android.Linux || ctx.Target().Os == android.Darwin { stdlib = "prebuilt_" + stdlib } deps.Stdlibs = append(deps.Stdlibs, stdlib) @@ -394,6 +394,20 @@ func bionicDeps(ctx DepsContext, deps Deps, static bool) Deps { return deps } +func muslDeps(ctx DepsContext, deps Deps, static bool) Deps { + muslLibs := []string{"libc_musl"} + if static { + deps.StaticLibs = append(deps.StaticLibs, muslLibs...) + } else { + deps.SharedLibs = append(deps.SharedLibs, muslLibs...) + } + if libRuntimeBuiltins := config.BuiltinsRuntimeLibrary(ctx.toolchain()); libRuntimeBuiltins != "" { + deps.StaticLibs = append(deps.StaticLibs, libRuntimeBuiltins) + } + + return deps +} + func (compiler *baseCompiler) crateName() string { return compiler.Properties.Crate_name } diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go index f318507ae..14fcb028f 100644 --- a/rust/config/allowed_list.go +++ b/rust/config/allowed_list.go @@ -19,6 +19,7 @@ var ( "frameworks/native/libs/binder/rust", "frameworks/proto_logging/stats", "hardware/interfaces/security", + "hardware/interfaces/uwb", "packages/modules/Bluetooth", "packages/modules/DnsResolver", "packages/modules/Uwb", @@ -36,6 +37,7 @@ var ( "system/tools/aidl", "tools/security/fuzzing/example_rust_fuzzer", "tools/security/fuzzing/orphans", + "tools/security/remote_provisioning/cert_validator", "tools/vendor", "vendor/", } diff --git a/rust/config/x86_linux_host.go b/rust/config/x86_linux_host.go index c10afd86f..760834929 100644 --- a/rust/config/x86_linux_host.go +++ b/rust/config/x86_linux_host.go @@ -22,12 +22,29 @@ import ( var ( LinuxRustFlags = []string{} + LinuxMuslRustFlags = []string{ + // disable rustc's builtin fallbacks for crt objects + "-C link_self_contained=no", + // force rustc to use a dynamic musl libc + "-C target-feature=-crt-static", + "-Z link-native-libraries=no", + } LinuxRustLinkFlags = []string{ "-B${cc_config.ClangBin}", "-fuse-ld=lld", "-Wl,--undefined-version", + } + LinuxRustGlibcLinkFlags = []string{ "--sysroot ${cc_config.LinuxGccRoot}/sysroot", } + LinuxRustMuslLinkFlags = []string{ + "--sysroot /dev/null", + "-nodefaultlibs", + "-nostdlib", + "-Wl,--no-dynamic-linker", + // for unwind + "-lgcc", "-lgcc_eh", + } linuxX86Rustflags = []string{} linuxX86Linkflags = []string{} linuxX8664Rustflags = []string{} @@ -35,15 +52,17 @@ var ( ) func init() { - registerToolchainFactory(android.Linux, android.X86_64, linuxX8664ToolchainFactory) - registerToolchainFactory(android.Linux, android.X86, linuxX86ToolchainFactory) + registerToolchainFactory(android.Linux, android.X86_64, linuxGlibcX8664ToolchainFactory) + registerToolchainFactory(android.Linux, android.X86, linuxGlibcX86ToolchainFactory) - // TODO: musl rust support - registerToolchainFactory(android.LinuxMusl, android.X86_64, linuxX8664ToolchainFactory) - registerToolchainFactory(android.LinuxMusl, android.X86, linuxX86ToolchainFactory) + registerToolchainFactory(android.LinuxMusl, android.X86_64, linuxMuslX8664ToolchainFactory) + registerToolchainFactory(android.LinuxMusl, android.X86, linuxMuslX86ToolchainFactory) pctx.StaticVariable("LinuxToolchainRustFlags", strings.Join(LinuxRustFlags, " ")) + pctx.StaticVariable("LinuxMuslToolchainRustFlags", strings.Join(LinuxMuslRustFlags, " ")) pctx.StaticVariable("LinuxToolchainLinkFlags", strings.Join(LinuxRustLinkFlags, " ")) + pctx.StaticVariable("LinuxGlibcToolchainLinkFlags", strings.Join(LinuxRustGlibcLinkFlags, " ")) + pctx.StaticVariable("LinuxMuslToolchainLinkFlags", strings.Join(LinuxRustMuslLinkFlags, " ")) pctx.StaticVariable("LinuxToolchainX86RustFlags", strings.Join(linuxX86Rustflags, " ")) pctx.StaticVariable("LinuxToolchainX86LinkFlags", strings.Join(linuxX86Linkflags, " ")) pctx.StaticVariable("LinuxToolchainX8664RustFlags", strings.Join(linuxX8664Rustflags, " ")) @@ -51,19 +70,9 @@ func init() { } -type toolchainLinux struct { - toolchainRustFlags string - toolchainLinkFlags string -} - -type toolchainLinuxX86 struct { - toolchain32Bit - toolchainLinux -} - +// Base 64-bit linux rust toolchain type toolchainLinuxX8664 struct { toolchain64Bit - toolchainLinux } func (toolchainLinuxX8664) Supported() bool { @@ -78,10 +87,6 @@ func (t *toolchainLinuxX8664) Name() string { return "x86_64" } -func (t *toolchainLinuxX8664) RustTriple() string { - return "x86_64-unknown-linux-gnu" -} - func (t *toolchainLinuxX8664) ToolchainLinkFlags() string { // Prepend the lld flags from cc_config so we stay in sync with cc return "${cc_config.LinuxLldflags} ${cc_config.LinuxX8664Lldflags} " + @@ -92,8 +97,49 @@ func (t *toolchainLinuxX8664) ToolchainRustFlags() string { return "${config.LinuxToolchainRustFlags} ${config.LinuxToolchainX8664RustFlags}" } -func linuxX8664ToolchainFactory(arch android.Arch) Toolchain { - return toolchainLinuxX8664Singleton +// Specialization of the 64-bit linux rust toolchain for glibc. Adds the gnu rust triple and +// sysroot linker flags. +type toolchainLinuxGlibcX8664 struct { + toolchainLinuxX8664 +} + +func (t *toolchainLinuxX8664) RustTriple() string { + return "x86_64-unknown-linux-gnu" +} + +func (t *toolchainLinuxGlibcX8664) ToolchainLinkFlags() string { + return t.toolchainLinuxX8664.ToolchainLinkFlags() + " " + "${config.LinuxGlibcToolchainLinkFlags}" +} + +func linuxGlibcX8664ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxGlibcX8664Singleton +} + +// Specialization of the 64-bit linux rust toolchain for musl. Adds the musl rust triple and +// linker flags to avoid using the host sysroot. +type toolchainLinuxMuslX8664 struct { + toolchainLinuxX8664 +} + +func (t *toolchainLinuxMuslX8664) RustTriple() string { + return "x86_64-unknown-linux-musl" +} + +func (t *toolchainLinuxMuslX8664) ToolchainLinkFlags() string { + return t.toolchainLinuxX8664.ToolchainLinkFlags() + " " + "${config.LinuxMuslToolchainLinkFlags}" +} + +func (t *toolchainLinuxMuslX8664) ToolchainRustFlags() string { + return t.toolchainLinuxX8664.ToolchainRustFlags() + " " + "${config.LinuxMuslToolchainRustFlags}" +} + +func linuxMuslX8664ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxMuslX8664Singleton +} + +// Base 32-bit linux rust toolchain +type toolchainLinuxX86 struct { + toolchain32Bit } func (toolchainLinuxX86) Supported() bool { @@ -116,10 +162,6 @@ func (toolchainLinuxX8664) LibclangRuntimeLibraryArch() string { return "x86_64" } -func (t *toolchainLinuxX86) RustTriple() string { - return "i686-unknown-linux-gnu" -} - func (t *toolchainLinuxX86) ToolchainLinkFlags() string { // Prepend the lld flags from cc_config so we stay in sync with cc return "${cc_config.LinuxLldflags} ${cc_config.LinuxX86Lldflags} " + @@ -130,9 +172,47 @@ func (t *toolchainLinuxX86) ToolchainRustFlags() string { return "${config.LinuxToolchainRustFlags} ${config.LinuxToolchainX86RustFlags}" } -func linuxX86ToolchainFactory(arch android.Arch) Toolchain { - return toolchainLinuxX86Singleton +// Specialization of the 32-bit linux rust toolchain for glibc. Adds the gnu rust triple and +// sysroot linker flags. +type toolchainLinuxGlibcX86 struct { + toolchainLinuxX86 +} + +func (t *toolchainLinuxGlibcX86) RustTriple() string { + return "i686-unknown-linux-gnu" +} + +func (t *toolchainLinuxGlibcX86) ToolchainLinkFlags() string { + return t.toolchainLinuxX86.ToolchainLinkFlags() + " " + "${config.LinuxGlibcToolchainLinkFlags}" +} + +func linuxGlibcX86ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxGlibcX86Singleton +} + +// Specialization of the 32-bit linux rust toolchain for musl. Adds the musl rust triple and +// linker flags to avoid using the host sysroot. +type toolchainLinuxMuslX86 struct { + toolchainLinuxX86 +} + +func (t *toolchainLinuxMuslX86) RustTriple() string { + return "i686-unknown-linux-musl" +} + +func (t *toolchainLinuxMuslX86) ToolchainLinkFlags() string { + return t.toolchainLinuxX86.ToolchainLinkFlags() + " " + "${config.LinuxMuslToolchainLinkFlags}" +} + +func (t *toolchainLinuxMuslX86) ToolchainRustFlags() string { + return t.toolchainLinuxX86.ToolchainRustFlags() + " " + "${config.LinuxMuslToolchainRustFlags}" +} + +func linuxMuslX86ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxMuslX86Singleton } -var toolchainLinuxX8664Singleton Toolchain = &toolchainLinuxX8664{} -var toolchainLinuxX86Singleton Toolchain = &toolchainLinuxX86{} +var toolchainLinuxGlibcX8664Singleton Toolchain = &toolchainLinuxGlibcX8664{} +var toolchainLinuxGlibcX86Singleton Toolchain = &toolchainLinuxGlibcX86{} +var toolchainLinuxMuslX8664Singleton Toolchain = &toolchainLinuxMuslX8664{} +var toolchainLinuxMuslX86Singleton Toolchain = &toolchainLinuxMuslX86{} diff --git a/rust/coverage.go b/rust/coverage.go index 8fdfa2342..050b811c7 100644 --- a/rust/coverage.go +++ b/rust/coverage.go @@ -57,18 +57,7 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags flags.RustFlags = append(flags.RustFlags, "-Z instrument-coverage", "-g") flags.LinkFlags = append(flags.LinkFlags, - profileInstrFlag, "-g", coverage.OutputFile().Path().String(), "-Wl,--wrap,open", - // Upstream LLVM change 6d2d3bd0a6 made - // -z,start-stop-gc the default. It drops metadata - // sections like __llvm_prf_data unless they are marked - // SHF_GNU_RETAIN. https://reviews.llvm.org/D97448 - // marks generated sections, including __llvm_prf_data - // as SHF_GNU_RETAIN. However this change is not in - // the Rust toolchain. Since we link Rust libs with - // new lld, we should use nostart-stop-gc until the - // Rust toolchain updates past D97448. - "-Wl,-z,nostart-stop-gc", - ) + profileInstrFlag, "-g", coverage.OutputFile().Path().String(), "-Wl,--wrap,open") deps.StaticLibs = append(deps.StaticLibs, coverage.OutputFile().Path()) } diff --git a/rust/library.go b/rust/library.go index bb2e83fdb..62eaefd68 100644 --- a/rust/library.go +++ b/rust/library.go @@ -426,10 +426,16 @@ func (library *libraryDecorator) compilerProps() []interface{} { func (library *libraryDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { deps = library.baseCompiler.compilerDeps(ctx, deps) - if ctx.toolchain().Bionic() && (library.dylib() || library.shared()) { - deps = bionicDeps(ctx, deps, false) - deps.CrtBegin = "crtbegin_so" - deps.CrtEnd = "crtend_so" + if library.dylib() || library.shared() { + if ctx.toolchain().Bionic() { + deps = bionicDeps(ctx, deps, false) + deps.CrtBegin = []string{"crtbegin_so"} + deps.CrtEnd = []string{"crtend_so"} + } else if ctx.Os() == android.LinuxMusl { + deps = muslDeps(ctx, deps, false) + deps.CrtBegin = []string{"libc_musl_crtbegin_so"} + deps.CrtEnd = []string{"libc_musl_crtend_so"} + } } return deps @@ -767,9 +773,10 @@ func (l *libraryDecorator) collectHeadersForSnapshot(ctx android.ModuleContext, // Glob together the headers from the modules include_dirs property for _, path := range android.CopyOfPaths(l.includeDirs) { dir := path.String() - glob, err := ctx.GlobWithDeps(dir+"/**/*", nil) + globDir := dir + "/**/*" + glob, err := ctx.GlobWithDeps(globDir, nil) if err != nil { - ctx.ModuleErrorf("glob failed: %#v", err) + ctx.ModuleErrorf("glob of %q failed: %s", globDir, err) return } diff --git a/rust/rust.go b/rust/rust.go index 0f7b76823..018d1dd27 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -394,7 +394,7 @@ type Deps struct { DataLibs []string DataBins []string - CrtBegin, CrtEnd string + CrtBegin, CrtEnd []string } type PathDeps struct { @@ -421,8 +421,8 @@ type PathDeps struct { depGeneratedHeaders android.Paths depSystemIncludePaths android.Paths - CrtBegin android.OptionalPath - CrtEnd android.OptionalPath + CrtBegin android.Paths + CrtEnd android.Paths // Paths to generated source files SrcDeps android.Paths @@ -1224,9 +1224,9 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { depPaths.depSystemIncludePaths = append(depPaths.depSystemIncludePaths, exportedInfo.SystemIncludeDirs...) depPaths.depGeneratedHeaders = append(depPaths.depGeneratedHeaders, exportedInfo.GeneratedHeaders...) case depTag == cc.CrtBeginDepTag: - depPaths.CrtBegin = linkObject + depPaths.CrtBegin = append(depPaths.CrtBegin, linkObject.Path()) case depTag == cc.CrtEndDepTag: - depPaths.CrtEnd = linkObject + depPaths.CrtEnd = append(depPaths.CrtEnd, linkObject.Path()) } // Make sure these dependencies are propagated @@ -1234,6 +1234,13 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { lib.exportLinkDirs(linkPath) lib.exportLinkObjects(linkObject.String()) } + } else { + switch { + case depTag == cc.CrtBeginDepTag: + depPaths.CrtBegin = append(depPaths.CrtBegin, android.OutputFileForModule(ctx, dep, "")) + case depTag == cc.CrtEndDepTag: + depPaths.CrtEnd = append(depPaths.CrtEnd, android.OutputFileForModule(ctx, dep, "")) + } } if srcDep, ok := dep.(android.SourceFileProducer); ok { @@ -1432,13 +1439,13 @@ func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { actx.AddVariationDependencies(nil, cc.HeaderDepTag(), deps.HeaderLibs...) crtVariations := cc.GetCrtVariations(ctx, mod) - if deps.CrtBegin != "" { + for _, crt := range deps.CrtBegin { actx.AddVariationDependencies(crtVariations, cc.CrtBeginDepTag, - cc.RewriteSnapshotLib(deps.CrtBegin, cc.GetSnapshot(mod, &snapshotInfo, actx).Objects)) + cc.RewriteSnapshotLib(crt, cc.GetSnapshot(mod, &snapshotInfo, actx).Objects)) } - if deps.CrtEnd != "" { + for _, crt := range deps.CrtEnd { actx.AddVariationDependencies(crtVariations, cc.CrtEndDepTag, - cc.RewriteSnapshotLib(deps.CrtEnd, cc.GetSnapshot(mod, &snapshotInfo, actx).Objects)) + cc.RewriteSnapshotLib(crt, cc.GetSnapshot(mod, &snapshotInfo, actx).Objects)) } if mod.sourceProvider != nil { diff --git a/scripts/manifest_fixer.py b/scripts/manifest_fixer.py index d80a617a1..2d3103b43 100755 --- a/scripts/manifest_fixer.py +++ b/scripts/manifest_fixer.py @@ -65,6 +65,9 @@ def parse_args(): parser.add_argument('--has-no-code', dest='has_no_code', action='store_true', help=('adds hasCode="false" attribute to application. Ignored if application elem ' 'already has a hasCode attribute.')) + parser.add_argument('--test-only', dest='test_only', action='store_true', + help=('adds testOnly="true" attribute to application. Assign true value if application elem ' + 'already has a testOnly attribute.')) parser.add_argument('input', help='input AndroidManifest.xml file') parser.add_argument('output', help='output AndroidManifest.xml file') return parser.parse_args() @@ -318,6 +321,26 @@ def set_has_code_to_false(doc): attr.value = 'false' application.setAttributeNode(attr) +def set_test_only_flag_to_true(doc): + manifest = parse_manifest(doc) + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + application = doc.createElement('application') + indent = get_indent(manifest.firstChild, 1) + first = manifest.firstChild + manifest.insertBefore(doc.createTextNode(indent), first) + manifest.insertBefore(application, first) + + attr = application.getAttributeNodeNS(android_ns, 'testOnly') + if attr is not None: + # Do nothing If the application already has a testOnly attribute. + return + attr = doc.createAttributeNS(android_ns, 'android:testOnly') + attr.value = 'true' + application.setAttributeNode(attr) def main(): """Program entry point.""" @@ -349,6 +372,9 @@ def main(): if args.has_no_code: set_has_code_to_false(doc) + if args.test_only: + set_test_only_flag_to_true(doc) + if args.extract_native_libs is not None: add_extract_native_libs(doc, args.extract_native_libs) diff --git a/scripts/manifest_fixer_test.py b/scripts/manifest_fixer_test.py index f6fcaafe5..199b279cd 100755 --- a/scripts/manifest_fixer_test.py +++ b/scripts/manifest_fixer_test.py @@ -521,12 +521,55 @@ class AddNoCodeApplicationTest(unittest.TestCase): self.assert_xml_equal(output, manifest_input) def test_has_application_has_code_true(self): - """ Do nothing if there's already an application elemeent even if its + """ Do nothing if there's already an application element even if its hasCode attribute is true. """ manifest_input = self.manifest_tmpl % ' <application android:hasCode="true"/>\n' output = self.run_test(manifest_input) self.assert_xml_equal(output, manifest_input) +class AddTestOnlyApplicationTest(unittest.TestCase): + """Unit tests for set_test_only_flag_to_true function.""" + + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + + def run_test(self, input_manifest): + doc = minidom.parseString(input_manifest) + manifest_fixer.set_test_only_flag_to_true(doc) + output = io.StringIO() + manifest_fixer.write_xml(output, doc) + return output.getvalue() + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + '%s' + '</manifest>\n') + + def test_no_application(self): + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % ' <application android:testOnly="true"/>\n' + output = self.run_test(manifest_input) + self.assert_xml_equal(output, expected) + + def test_has_application_no_test_only(self): + manifest_input = self.manifest_tmpl % ' <application/>\n' + expected = self.manifest_tmpl % ' <application android:testOnly="true"/>\n' + output = self.run_test(manifest_input) + self.assert_xml_equal(output, expected) + + def test_has_application_test_only_true(self): + """ If there's already an application element.""" + manifest_input = self.manifest_tmpl % ' <application android:testOnly="true"/>\n' + output = self.run_test(manifest_input) + self.assert_xml_equal(output, manifest_input) + + def test_has_application_test_only_false(self): + """ If there's already an application element with the testOnly attribute as false.""" + manifest_input = self.manifest_tmpl % ' <application android:testOnly="false"/>\n' + output = self.run_test(manifest_input) + self.assert_xml_equal(output, manifest_input) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/scripts/rbc-run b/scripts/rbc-run index 7243421ff..b8a6c0c73 100755 --- a/scripts/rbc-run +++ b/scripts/rbc-run @@ -9,9 +9,10 @@ declare -r output_root="${OUT_DIR:-out}" declare -r runner="${output_root}/soong/rbcrun" declare -r converter="${output_root}/soong/mk2rbc" declare -r launcher="${output_root}/rbc/launcher.rbc" +declare -r makefile_list="${output_root}/.module_paths/configuration.list" declare -r makefile="$1" declare -r input_variables="$2" shift 2 -"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" "${makefile}" +"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" --makefile_list="${makefile_list}" "${makefile}" "${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}" diff --git a/scripts/rustfmt.toml b/scripts/rustfmt.toml index 617d42585..cefaa42a3 100644 --- a/scripts/rustfmt.toml +++ b/scripts/rustfmt.toml @@ -1,5 +1,5 @@ # Android Format Style -edition = "2018" +edition = "2021" use_small_heuristics = "Max" newline_style = "Unix" diff --git a/sdk/build_release.go b/sdk/build_release.go index a3f089973..4c2277e85 100644 --- a/sdk/build_release.go +++ b/sdk/build_release.go @@ -85,7 +85,7 @@ var ( // Add the build releases from oldest to newest. buildReleaseS = initBuildRelease("S") - buildReleaseT = initBuildRelease("T") + buildReleaseT = initBuildRelease("Tiramisu") ) // initBuildRelease creates a new build release with the specified name. @@ -230,51 +230,108 @@ func (p *propertyPruner) gatherFields(structType reflect.Type, containingStructA return container.Field(fieldIndex) } - zeroValue := reflect.Zero(field.Type) - fieldPruner := func(container reflect.Value) { - if containingStructAccessor != nil { - // This is an embedded structure so first access the field for the embedded - // structure. - container = containingStructAccessor(container) - } + fieldType := field.Type + if selector(name, field) { + zeroValue := reflect.Zero(fieldType) + fieldPruner := func(container reflect.Value) { + if containingStructAccessor != nil { + // This is an embedded structure so first access the field for the embedded + // structure. + container = containingStructAccessor(container) + } - // Skip through interface and pointer values to find the structure. - container = getStructValue(container) + // Skip through interface and pointer values to find the structure. + container = getStructValue(container) - defer func() { - if r := recover(); r != nil { - panic(fmt.Errorf("%s for fieldIndex %d of field %s of container %#v", r, fieldIndex, name, container.Interface())) - } - }() + defer func() { + if r := recover(); r != nil { + panic(fmt.Errorf("%s\n\tfor field (index %d, name %s)", r, fieldIndex, name)) + } + }() - // Set the field. - container.Field(fieldIndex).Set(zeroValue) - } + // Set the field. + container.Field(fieldIndex).Set(zeroValue) + } - if selector(name, field) { property := prunerProperty{ name, fieldPruner, } p.properties = append(p.properties, property) - } else if field.Type.Kind() == reflect.Struct { - // Gather fields from the nested or embedded structure. - var subNamePrefix string - if field.Anonymous { - subNamePrefix = namePrefix - } else { - subNamePrefix = name + "." + } else { + switch fieldType.Kind() { + case reflect.Struct: + // Gather fields from the nested or embedded structure. + var subNamePrefix string + if field.Anonymous { + subNamePrefix = namePrefix + } else { + subNamePrefix = name + "." + } + p.gatherFields(fieldType, fieldGetter, subNamePrefix, selector) + + case reflect.Map: + // Get the type of the values stored in the map. + valueType := fieldType.Elem() + // Skip over * types. + if valueType.Kind() == reflect.Ptr { + valueType = valueType.Elem() + } + if valueType.Kind() == reflect.Struct { + // If this is not referenced by a pointer then it is an error as it is impossible to + // modify a struct that is stored directly as a value in a map. + if fieldType.Elem().Kind() != reflect.Ptr { + panic(fmt.Errorf("Cannot prune struct %s stored by value in map %s, map values must"+ + " be pointers to structs", + fieldType.Elem(), name)) + } + + // Create a new pruner for the values of the map. + valuePruner := newPropertyPrunerForStructType(valueType, selector) + + // Create a new fieldPruner that will iterate over all the items in the map and call the + // pruner on them. + fieldPruner := func(container reflect.Value) { + mapValue := fieldGetter(container) + + for _, keyValue := range mapValue.MapKeys() { + itemValue := mapValue.MapIndex(keyValue) + + defer func() { + if r := recover(); r != nil { + panic(fmt.Errorf("%s\n\tfor key %q", r, keyValue)) + } + }() + + valuePruner.pruneProperties(itemValue.Interface()) + } + } + + // Add the map field pruner to the list of property pruners. + property := prunerProperty{ + name + "[*]", + fieldPruner, + } + p.properties = append(p.properties, property) + } } - p.gatherFields(field.Type, fieldGetter, subNamePrefix, selector) } } } -// pruneProperties will prune (set to zero value) any properties in the supplied struct. +// pruneProperties will prune (set to zero value) any properties in the struct referenced by the +// supplied struct pointer. // // The struct must be of the same type as was originally passed to newPropertyPruner to create this // propertyPruner. func (p *propertyPruner) pruneProperties(propertiesStruct interface{}) { + + defer func() { + if r := recover(); r != nil { + panic(fmt.Errorf("%s\n\tof container %#v", r, propertiesStruct)) + } + }() + structValue := reflect.ValueOf(propertiesStruct) for _, property := range p.properties { property.prunerFunc(structValue) @@ -292,6 +349,13 @@ type fieldSelectorFunc func(name string, field reflect.StructField) bool // of properties. func newPropertyPruner(propertiesStruct interface{}, selector fieldSelectorFunc) *propertyPruner { structType := getStructValue(reflect.ValueOf(propertiesStruct)).Type() + return newPropertyPrunerForStructType(structType, selector) +} + +// newPropertyPruner creates a new property pruner for the supplied properties struct type. +// +// The returned pruner can be used on any properties structure of the supplied type. +func newPropertyPrunerForStructType(structType reflect.Type, selector fieldSelectorFunc) *propertyPruner { pruner := &propertyPruner{} pruner.gatherFields(structType, nil, "", selector) return pruner diff --git a/sdk/build_release_test.go b/sdk/build_release_test.go index dff276d0c..6f1ef9e30 100644 --- a/sdk/build_release_test.go +++ b/sdk/build_release_test.go @@ -15,6 +15,7 @@ package sdk import ( + "encoding/json" "fmt" "testing" @@ -59,7 +60,7 @@ func TestParseBuildReleaseSet(t *testing.T) { t.Run("closed range", func(t *testing.T) { set, err := parseBuildReleaseSet("S-F1") android.AssertDeepEquals(t, "errors", nil, err) - android.AssertStringEquals(t, "set", "[S,T,F1]", set.String()) + android.AssertStringEquals(t, "set", "[S,Tiramisu,F1]", set.String()) }) invalidAReleaseMessage := `unknown release "A", expected one of ` + allBuildReleaseSet.String() t.Run("invalid release", func(t *testing.T) { @@ -78,7 +79,7 @@ func TestParseBuildReleaseSet(t *testing.T) { android.AssertStringDoesContain(t, "errors", fmt.Sprint(err), invalidAReleaseMessage) }) t.Run("invalid release in closed range end", func(t *testing.T) { - set, err := parseBuildReleaseSet("T-A") + set, err := parseBuildReleaseSet("Tiramisu-A") android.AssertDeepEquals(t, "set", (*buildReleaseSet)(nil), set) android.AssertStringDoesContain(t, "errors", fmt.Sprint(err), invalidAReleaseMessage) }) @@ -125,61 +126,102 @@ func TestPropertyPrunerByBuildRelease(t *testing.T) { F1_only string `supported_build_releases:"F1"` } + type mapped struct { + Default string + T_only string `supported_build_releases:"Tiramisu"` + } + type testBuildReleasePruner struct { Default string - S_and_T_only string `supported_build_releases:"S-T"` - T_later string `supported_build_releases:"T+"` + S_and_T_only string `supported_build_releases:"S-Tiramisu"` + T_later string `supported_build_releases:"Tiramisu+"` Nested nested + Mapped map[string]*mapped + } + + inputFactory := func() testBuildReleasePruner { + return testBuildReleasePruner{ + Default: "Default", + S_and_T_only: "S_and_T_only", + T_later: "T_later", + Nested: nested{ + F1_only: "F1_only", + }, + Mapped: map[string]*mapped{ + "one": { + Default: "one-default", + T_only: "one-t-only", + }, + "two": { + Default: "two-default", + T_only: "two-t-only", + }, + }, + } + } + + marshal := func(t interface{}) string { + bytes, err := json.MarshalIndent(t, "", " ") + if err != nil { + panic(err) + } + return string(bytes) } - input := testBuildReleasePruner{ - Default: "Default", - S_and_T_only: "S_and_T_only", - T_later: "T_later", - Nested: nested{ - F1_only: "F1_only", - }, + assertJsonEquals := func(t *testing.T, expected, actual interface{}) { + t.Helper() + expectedJson := marshal(expected) + actualJson := marshal(actual) + if actualJson != expectedJson { + t.Errorf("test struct: expected:\n%s\n got:\n%s", expectedJson, actualJson) + } } t.Run("target S", func(t *testing.T) { - testStruct := input + testStruct := inputFactory() pruner := newPropertyPrunerByBuildRelease(&testStruct, buildReleaseS) pruner.pruneProperties(&testStruct) - expected := input + expected := inputFactory() expected.T_later = "" expected.Nested.F1_only = "" - android.AssertDeepEquals(t, "test struct", expected, testStruct) + expected.Mapped["one"].T_only = "" + expected.Mapped["two"].T_only = "" + assertJsonEquals(t, expected, testStruct) }) t.Run("target T", func(t *testing.T) { - testStruct := input + testStruct := inputFactory() pruner := newPropertyPrunerByBuildRelease(&testStruct, buildReleaseT) pruner.pruneProperties(&testStruct) - expected := input + expected := inputFactory() expected.Nested.F1_only = "" - android.AssertDeepEquals(t, "test struct", expected, testStruct) + assertJsonEquals(t, expected, testStruct) }) t.Run("target F1", func(t *testing.T) { - testStruct := input + testStruct := inputFactory() pruner := newPropertyPrunerByBuildRelease(&testStruct, buildReleaseFuture1) pruner.pruneProperties(&testStruct) - expected := input + expected := inputFactory() expected.S_and_T_only = "" - android.AssertDeepEquals(t, "test struct", expected, testStruct) + expected.Mapped["one"].T_only = "" + expected.Mapped["two"].T_only = "" + assertJsonEquals(t, expected, testStruct) }) t.Run("target F2", func(t *testing.T) { - testStruct := input + testStruct := inputFactory() pruner := newPropertyPrunerByBuildRelease(&testStruct, buildReleaseFuture2) pruner.pruneProperties(&testStruct) - expected := input + expected := inputFactory() expected.S_and_T_only = "" expected.Nested.F1_only = "" - android.AssertDeepEquals(t, "test struct", expected, testStruct) + expected.Mapped["one"].T_only = "" + expected.Mapped["two"].T_only = "" + assertJsonEquals(t, expected, testStruct) }) } diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 0d9b4a063..f0d3b35d7 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -1319,6 +1319,58 @@ java_sdk_library_import { ) } +func TestSnapshotWithJavaSdkLibrary_AnnotationsZip_PreT(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForSdkTestWithJavaSdkLibrary, + android.FixtureMergeEnv(map[string]string{ + "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": "S", + }), + ).RunTestWithBp(t, ` + sdk { + name: "mysdk", + java_sdk_libs: ["myjavalib"], + } + + java_sdk_library { + name: "myjavalib", + srcs: ["Test.java"], + sdk_version: "current", + shared_library: false, + annotations_enabled: true, + public: { + enabled: true, + }, + } + `) + + CheckSnapshot(t, result, "mysdk", "", + checkUnversionedAndroidBpContents(` +// This is auto-generated. DO NOT EDIT. + +java_sdk_library_import { + name: "myjavalib", + prefer: false, + visibility: ["//visibility:public"], + apex_available: ["//apex_available:platform"], + shared_library: false, + public: { + jars: ["sdk_library/public/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], + current_api: "sdk_library/public/myjavalib.txt", + removed_api: "sdk_library/public/myjavalib-removed.txt", + sdk_version: "current", + }, +} + `), + checkAllCopyRules(` +.intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt +.intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt + `), + checkMergeZips(".intermediates/mysdk/common_os/tmp/sdk_library/public/myjavalib_stub_sources.zip"), + ) +} + func TestSnapshotWithJavaSdkLibrary_CompileDex(t *testing.T) { result := android.GroupFixturePreparers(prepareForSdkTestWithJavaSdkLibrary).RunTestWithBp(t, ` sdk { diff --git a/sh/sh_binary_test.go b/sh/sh_binary_test.go index 28b6fb93a..7fe1d8538 100644 --- a/sh/sh_binary_test.go +++ b/sh/sh_binary_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "testing" "android/soong/android" @@ -215,3 +216,40 @@ func TestShTestHost_dataDeviceModules(t *testing.T) { actualData := entries.EntryMap["LOCAL_TEST_DATA"] android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", config, expectedData, actualData) } + +func TestShTestHost_dataDeviceModulesAutogenTradefedConfig(t *testing.T) { + ctx, config := testShBinary(t, ` + sh_test_host { + name: "foo", + src: "test.sh", + data_device_bins: ["bar"], + data_device_libs: ["libbar"], + } + + cc_binary { + name: "bar", + shared_libs: ["libbar"], + no_libcrt: true, + nocrt: true, + system_shared_libs: [], + stl: "none", + } + + cc_library { + name: "libbar", + no_libcrt: true, + nocrt: true, + system_shared_libs: [], + stl: "none", + } + `) + + buildOS := config.BuildOS.String() + fooModule := ctx.ModuleForTests("foo", buildOS+"_x86_64") + + expectedBinAutogenConfig := `<option name="push-file" key="bar" value="/data/local/tests/unrestricted/foo/bar" />` + autogen := fooModule.Rule("autogen") + if !strings.Contains(autogen.Args["extraConfigs"], expectedBinAutogenConfig) { + t.Errorf("foo extraConfings %v does not contain %q", autogen.Args["extraConfigs"], expectedBinAutogenConfig) + } +} diff --git a/snapshot/host_snapshot.go b/snapshot/host_snapshot.go index 252cef816..09a382e6e 100644 --- a/snapshot/host_snapshot.go +++ b/snapshot/host_snapshot.go @@ -58,7 +58,7 @@ type hostSnapshot struct { android.ModuleBase android.PackagingBase - zipFile android.OptionalPath + outputFile android.OutputPath installDir android.InstallPath } @@ -141,7 +141,7 @@ func (f *hostSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Create a zip file for the binaries, and a zip of the meta data, then merge zips depsZipFile := android.PathForModuleOut(ctx, f.Name()+"_deps.zip").OutputPath modsZipFile := android.PathForModuleOut(ctx, f.Name()+"_mods.zip").OutputPath - outputFile := android.PathForModuleOut(ctx, f.installFileName()).OutputPath + f.outputFile = android.PathForModuleOut(ctx, f.installFileName()).OutputPath f.installDir = android.PathForModuleInstall(ctx) @@ -158,26 +158,21 @@ func (f *hostSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { builder.Command(). BuiltTool("merge_zips"). - Output(outputFile). + Output(f.outputFile). Input(metaZipFile). Input(modsZipFile) builder.Build("manifest", fmt.Sprintf("Adding manifest %s", f.installFileName())) - zip := ctx.InstallFile(f.installDir, f.installFileName(), outputFile) - f.zipFile = android.OptionalPathForPath(zip) + ctx.InstallFile(f.installDir, f.installFileName(), f.outputFile) } // Implements android.AndroidMkEntriesProvider func (f *hostSnapshot) AndroidMkEntries() []android.AndroidMkEntries { - if !f.zipFile.Valid() { - return []android.AndroidMkEntries{} - } - return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "ETC", - OutputFile: f.zipFile, - DistFiles: android.MakeDefaultDistFiles(f.zipFile.Path()), + OutputFile: android.OptionalPathForPath(f.outputFile), + DistFiles: android.MakeDefaultDistFiles(f.outputFile), ExtraEntries: []android.AndroidMkExtraEntriesFunc{ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.SetString("LOCAL_MODULE_PATH", f.installDir.String()) diff --git a/starlark_fmt/Android.bp b/starlark_fmt/Android.bp new file mode 100644 index 000000000..8d80ccdca --- /dev/null +++ b/starlark_fmt/Android.bp @@ -0,0 +1,28 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-starlark-format", + pkgPath: "android/soong/starlark_fmt", + srcs: [ + "format.go", + ], + testSrcs: [ + "format_test.go", + ], +} diff --git a/starlark_fmt/format.go b/starlark_fmt/format.go new file mode 100644 index 000000000..23eee59b3 --- /dev/null +++ b/starlark_fmt/format.go @@ -0,0 +1,96 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark_fmt + +import ( + "fmt" + "sort" + "strings" +) + +const ( + indent = 4 +) + +// Indention returns an indent string of the specified level. +func Indention(level int) string { + if level < 0 { + panic(fmt.Errorf("indent level cannot be less than 0, but got %d", level)) + } + return strings.Repeat(" ", level*indent) +} + +// PrintBool returns a Starlark compatible bool string. +func PrintBool(item bool) string { + return strings.Title(fmt.Sprintf("%t", item)) +} + +// PrintsStringList returns a Starlark-compatible string of a list of Strings/Labels. +func PrintStringList(items []string, indentLevel int) string { + return PrintList(items, indentLevel, `"%s"`) +} + +// PrintList returns a Starlark-compatible string of list formmated as requested. +func PrintList(items []string, indentLevel int, formatString string) string { + if len(items) == 0 { + return "[]" + } else if len(items) == 1 { + return fmt.Sprintf("["+formatString+"]", items[0]) + } + list := make([]string, 0, len(items)+2) + list = append(list, "[") + innerIndent := Indention(indentLevel + 1) + for _, item := range items { + list = append(list, fmt.Sprintf(`%s`+formatString+`,`, innerIndent, item)) + } + list = append(list, Indention(indentLevel)+"]") + return strings.Join(list, "\n") +} + +// PrintStringListDict returns a Starlark-compatible string formatted as dictionary with +// string keys and list of string values. +func PrintStringListDict(dict map[string][]string, indentLevel int) string { + formattedValueDict := make(map[string]string, len(dict)) + for k, v := range dict { + formattedValueDict[k] = PrintStringList(v, indentLevel+1) + } + return PrintDict(formattedValueDict, indentLevel) +} + +// PrintBoolDict returns a starlark-compatible string containing a dictionary with string keys and +// values printed with no additional formatting. +func PrintBoolDict(dict map[string]bool, indentLevel int) string { + formattedValueDict := make(map[string]string, len(dict)) + for k, v := range dict { + formattedValueDict[k] = PrintBool(v) + } + return PrintDict(formattedValueDict, indentLevel) +} + +// PrintDict returns a starlark-compatible string containing a dictionary with string keys and +// values printed with no additional formatting. +func PrintDict(dict map[string]string, indentLevel int) string { + if len(dict) == 0 { + return "{}" + } + items := make([]string, 0, len(dict)) + for k, v := range dict { + items = append(items, fmt.Sprintf(`%s"%s": %s,`, Indention(indentLevel+1), k, v)) + } + sort.Strings(items) + return fmt.Sprintf(`{ +%s +%s}`, strings.Join(items, "\n"), Indention(indentLevel)) +} diff --git a/starlark_fmt/format_test.go b/starlark_fmt/format_test.go new file mode 100644 index 000000000..90f78ef7a --- /dev/null +++ b/starlark_fmt/format_test.go @@ -0,0 +1,169 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark_fmt + +import ( + "testing" +) + +func TestPrintEmptyStringList(t *testing.T) { + in := []string{} + indentLevel := 0 + out := PrintStringList(in, indentLevel) + expectedOut := "[]" + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintSingleElementStringList(t *testing.T) { + in := []string{"a"} + indentLevel := 0 + out := PrintStringList(in, indentLevel) + expectedOut := `["a"]` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintMultiElementStringList(t *testing.T) { + in := []string{"a", "b"} + indentLevel := 0 + out := PrintStringList(in, indentLevel) + expectedOut := `[ + "a", + "b", +]` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintEmptyList(t *testing.T) { + in := []string{} + indentLevel := 0 + out := PrintList(in, indentLevel, "%s") + expectedOut := "[]" + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintSingleElementList(t *testing.T) { + in := []string{"1"} + indentLevel := 0 + out := PrintList(in, indentLevel, "%s") + expectedOut := `[1]` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintMultiElementList(t *testing.T) { + in := []string{"1", "2"} + indentLevel := 0 + out := PrintList(in, indentLevel, "%s") + expectedOut := `[ + 1, + 2, +]` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestListWithNonZeroIndent(t *testing.T) { + in := []string{"1", "2"} + indentLevel := 1 + out := PrintList(in, indentLevel, "%s") + expectedOut := `[ + 1, + 2, + ]` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestStringListDictEmpty(t *testing.T) { + in := map[string][]string{} + indentLevel := 0 + out := PrintStringListDict(in, indentLevel) + expectedOut := `{}` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestStringListDict(t *testing.T) { + in := map[string][]string{ + "key1": []string{}, + "key2": []string{"a"}, + "key3": []string{"1", "2"}, + } + indentLevel := 0 + out := PrintStringListDict(in, indentLevel) + expectedOut := `{ + "key1": [], + "key2": ["a"], + "key3": [ + "1", + "2", + ], +}` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintDict(t *testing.T) { + in := map[string]string{ + "key1": `""`, + "key2": `"a"`, + "key3": `[ + 1, + 2, + ]`, + } + indentLevel := 0 + out := PrintDict(in, indentLevel) + expectedOut := `{ + "key1": "", + "key2": "a", + "key3": [ + 1, + 2, + ], +}` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} + +func TestPrintDictWithIndent(t *testing.T) { + in := map[string]string{ + "key1": `""`, + "key2": `"a"`, + } + indentLevel := 1 + out := PrintDict(in, indentLevel) + expectedOut := `{ + "key1": "", + "key2": "a", + }` + if out != expectedOut { + t.Errorf("Expected %q, got %q", expectedOut, out) + } +} diff --git a/tradefed/autogen.go b/tradefed/autogen.go index da5582973..c2429ab7a 100644 --- a/tradefed/autogen.go +++ b/tradefed/autogen.go @@ -188,20 +188,20 @@ func AutoGenNativeBenchmarkTestConfig(ctx android.ModuleContext, testConfigProp } func AutoGenJavaTestConfig(ctx android.ModuleContext, testConfigProp *string, testConfigTemplateProp *string, - testSuites []string, autoGenConfig *bool, unitTest *bool) android.Path { + testSuites []string, config []Config, autoGenConfig *bool, unitTest *bool) android.Path { path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) if autogenPath != nil { templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) if templatePath.Valid() { - autogenTemplate(ctx, autogenPath, templatePath.String(), nil, "") + autogenTemplate(ctx, autogenPath, templatePath.String(), config, "") } else { if ctx.Device() { - autogenTemplate(ctx, autogenPath, "${JavaTestConfigTemplate}", nil, "") + autogenTemplate(ctx, autogenPath, "${JavaTestConfigTemplate}", config, "") } else { if Bool(unitTest) { - autogenTemplate(ctx, autogenPath, "${JavaHostUnitTestConfigTemplate}", nil, "") + autogenTemplate(ctx, autogenPath, "${JavaHostUnitTestConfigTemplate}", config, "") } else { - autogenTemplate(ctx, autogenPath, "${JavaHostTestConfigTemplate}", nil, "") + autogenTemplate(ctx, autogenPath, "${JavaHostTestConfigTemplate}", config, "") } } } diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go index a3a1aaf68..1c80cff31 100644 --- a/ui/build/cleanbuild.go +++ b/ui/build/cleanbuild.go @@ -171,6 +171,7 @@ func installClean(ctx Context, config Config) { productOut("recovery"), productOut("root"), productOut("system"), + productOut("system_dlkm"), productOut("system_other"), productOut("vendor"), productOut("vendor_dlkm"), diff --git a/ui/build/config.go b/ui/build/config.go index b6d0d2747..1dd948ce5 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -15,8 +15,12 @@ package build import ( + "context" + "encoding/json" "fmt" + "io/ioutil" "os" + "os/exec" "path/filepath" "runtime" "strconv" @@ -30,6 +34,14 @@ import ( smpb "android/soong/ui/metrics/metrics_proto" ) +const ( + envConfigDir = "vendor/google/tools/soong_config" + jsonSuffix = "json" + + configFetcher = "vendor/google/tools/soong/expconfigfetcher" + envConfigFetchTimeout = 10 * time.Second +) + type Config struct{ *configImpl } type configImpl struct { @@ -128,6 +140,85 @@ func checkTopDir(ctx Context) { } } +// fetchEnvConfig optionally fetches environment config from an +// experiments system to control Soong features dynamically. +func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error { + s, err := os.Stat(configFetcher) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if s.Mode()&0111 == 0 { + return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode()) + } + + configExists := false + outConfigFilePath := filepath.Join(config.OutDir(), envConfigName + jsonSuffix) + if _, err := os.Stat(outConfigFilePath); err == nil { + configExists = true + } + + tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout) + defer cancel() + cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir()) + if err := cmd.Start(); err != nil { + return err + } + + // If a config file already exists, return immediately and run the config file + // fetch in the background. Otherwise, wait for the config file to be fetched. + if configExists { + go cmd.Wait() + return nil + } + if err := cmd.Wait(); err != nil { + return err + } + return nil +} + +func loadEnvConfig(ctx Context, config *configImpl) error { + bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") + if bc == "" { + return nil + } + + if err := fetchEnvConfig(ctx, config, bc); err != nil { + fmt.Fprintf(os.Stderr, "Failed to fetch config file: %v", err) + } + + configDirs := []string{ + config.OutDir(), + os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), + envConfigDir, + } + for _, dir := range configDirs { + cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) + envVarsJSON, err := ioutil.ReadFile(cfgFile) + if err != nil { + continue + } + ctx.Verbosef("Loading config file %v\n", cfgFile) + var envVars map[string]map[string]string + if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { + fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error()) + continue + } + for k, v := range envVars["env"] { + if os.Getenv(k) != "" { + continue + } + config.environ.Set(k, v) + } + ctx.Verbosef("Finished loading config file %v\n", cfgFile) + break + } + + return nil +} + func NewConfig(ctx Context, args ...string) Config { ret := &configImpl{ environ: OsEnvironment(), @@ -157,6 +248,12 @@ func NewConfig(ctx Context, args ...string) Config { ret.environ.Set("OUT_DIR", outDir) } + // loadEnvConfig needs to know what the OUT_DIR is, so it should + // be called after we determine the appropriate out directory. + if err := loadEnvConfig(ctx, ret); err != nil { + ctx.Fatalln("Failed to parse env config files: %v", err) + } + if distDir, ok := ret.environ.Get("DIST_DIR"); ok { ret.distDir = filepath.Clean(distDir) } else { @@ -818,6 +915,10 @@ func (c *configImpl) ModuleGraphFile() string { return shared.JoinPath(c.SoongOutDir(), "module-graph.json") } +func (c *configImpl) ModuleActionsFile() string { + return shared.JoinPath(c.SoongOutDir(), "module-actions.json") +} + func (c *configImpl) TempDir() string { return shared.TempDirForOutDir(c.SoongOutDir()) } @@ -988,7 +1089,7 @@ func (c *configImpl) StartGoma() bool { } func (c *configImpl) UseRBE() bool { - if v, ok := c.environ.Get("USE_RBE"); ok { + if v, ok := c.Environment().Get("USE_RBE"); ok { v = strings.TrimSpace(v) if v != "" && v != "false" { return true diff --git a/ui/build/finder.go b/ui/build/finder.go index 8f74969fb..68efe2150 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -87,8 +87,8 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { // Bazel top-level file to mark a directory as a Bazel workspace. "WORKSPACE", }, - // Bazel Starlark configuration files. - IncludeSuffixes: []string{".bzl"}, + // Bazel Starlark configuration files and all .mk files for product/board configuration. + IncludeSuffixes: []string{".bzl", ".mk"}, } dumpDir := config.FileListDir() f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard), @@ -110,6 +110,19 @@ func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []s return entries.DirNames, matches } +func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { + matches := []string{} + for _, foundName := range entries.FileNames { + if foundName != "Android.mk" && + foundName != "AndroidProducts.mk" && + foundName != "CleanSpec.mk" && + strings.HasSuffix(foundName, ".mk") { + matches = append(matches, foundName) + } + } + return entries.DirNames, matches +} + // FindSources searches for source files known to <f> and writes them to the filesystem for // use later. func FindSources(ctx Context, config Config, f *finder.Finder) { @@ -172,6 +185,13 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { ctx.Fatalf("Could not find modules: %v", err) } + // Recursively look for all product/board config files. + configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles) + err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list")) + if err != nil { + ctx.Fatalf("Could not export product/board configuration list: %v", err) + } + if config.Dist() { f.WaitForDbDump() // Dist the files.db plain text database. diff --git a/ui/build/soong.go b/ui/build/soong.go index 81337620d..117a2a573 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -284,6 +284,7 @@ func bootstrapBlueprint(ctx Context, config Config) { config.ModuleGraphFile(), []string{ "--module_graph_file", config.ModuleGraphFile(), + "--module_actions_file", config.ModuleActionsFile(), }, fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()), ) |