diff options
77 files changed, 2964 insertions, 2300 deletions
@@ -33,8 +33,9 @@ cc_binary { Every module must have a `name` property, and the value must be unique across all Android.bp files. -For a list of valid module types and their properties see -[$OUT_DIR/soong/docs/soong_build.html](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html). +The list of valid module types and their properties can be generated by calling +`m soong_docs`. It will be written to `$OUT_DIR/soong/docs/soong_build.html`. +This list for the current version of Soong can be found [here](https://ci.android.com/builds/latest/branches/aosp-build-tools/targets/linux/view/soong_build.html). ### File lists diff --git a/android/Android.bp b/android/Android.bp index 1bccd7bfb..f3a385025 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -66,7 +66,6 @@ bootstrap_go_package { "prebuilt.go", "prebuilt_build_tool.go", "proto.go", - "queryview.go", "register.go", "rule_builder.go", "sandbox.go", @@ -81,7 +80,6 @@ bootstrap_go_package { "util.go", "variable.go", "visibility.go", - "writedocs.go", ], testSrcs: [ "android_test.go", diff --git a/android/bazel.go b/android/bazel.go index bebb61b09..373e292f2 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -120,6 +120,10 @@ const ( // allows modules to opt-out. Bp2BuildDefaultTrueRecursively BazelConversionConfigEntry = iota + 1 + // all modules in this package (not recursively) default to bp2build_available: true. + // allows modules to opt-out. + Bp2BuildDefaultTrue + // all modules in this package (not recursively) default to bp2build_available: false. // allows modules to opt-in. Bp2BuildDefaultFalse @@ -141,6 +145,7 @@ var ( "build/bazel/bazel_skylib":/* recursive = */ true, "build/bazel/rules":/* recursive = */ true, "build/bazel/rules_cc":/* recursive = */ true, + "build/bazel/scripts":/* recursive = */ true, "build/bazel/tests":/* recursive = */ true, "build/bazel/platforms":/* recursive = */ true, "build/bazel/product_variables":/* recursive = */ true, @@ -164,7 +169,9 @@ var ( "build/bazel/examples/apex/minimal": Bp2BuildDefaultTrueRecursively, "development/sdk": Bp2BuildDefaultTrueRecursively, "external/gwp_asan": Bp2BuildDefaultTrueRecursively, + "external/brotli": Bp2BuildDefaultTrue, "system/core/libcutils": Bp2BuildDefaultTrueRecursively, + "system/core/libprocessgroup": Bp2BuildDefaultTrue, "system/core/property_service/libpropertyinfoparser": Bp2BuildDefaultTrueRecursively, "system/libbase": Bp2BuildDefaultTrueRecursively, "system/logging/liblog": Bp2BuildDefaultTrueRecursively, @@ -173,7 +180,9 @@ var ( "external/arm-optimized-routines": Bp2BuildDefaultTrueRecursively, "external/fmtlib": Bp2BuildDefaultTrueRecursively, "external/jemalloc_new": Bp2BuildDefaultTrueRecursively, + "external/libcxx": Bp2BuildDefaultTrueRecursively, "external/libcxxabi": Bp2BuildDefaultTrueRecursively, + "external/libcap": Bp2BuildDefaultTrueRecursively, "external/scudo": Bp2BuildDefaultTrueRecursively, "prebuilts/clang/host/linux-x86": Bp2BuildDefaultTrueRecursively, } @@ -217,6 +226,16 @@ var ( "gwp_asan_crash_handler", // cc_library, ld.lld: error: undefined symbol: memset + //system/core/libprocessgroup/... + "libprocessgroup", // depends on //system/core/libprocessgroup/cgrouprc:libcgrouprc + + //external/brotli/... + "brotli-fuzzer-corpus", // "declared output 'external/brotli/c/fuzz/73231c6592f195ffd41100b8706d1138ff6893b9' was not created by genrule" + + // //external/libcap/... + "libcap", // http://b/198595332, depends on _makenames, a cc_binary + "cap_names.h", // http://b/198596102, depends on _makenames, a cc_binary + // Tests. Handle later. "libbionic_tests_headers_posix", // http://b/186024507, cc_library_static, sched.h, time.h not found "libjemalloc5_integrationtest", @@ -237,8 +256,12 @@ var ( // Per-module denylist to opt modules out of mixed builds. Such modules will // still be generated via bp2build. mixedBuildsDisabledList = []string{ - "libc++abi", // http://b/195970501, cc_library_static, duplicate symbols because it propagates libc objects. - "libc++demangle", // http://b/195970501, cc_library_static, duplicate symbols because it propagates libc objects. + "libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy + "libc++fs", // http://b/198403271, Missing symbols/members in the global namespace when referenced from headers in //external/libcxx/includes + "libc++_experimental", // http://b/198403271, Missing symbols/members in the global namespace when referenced from headers in //external/libcxx/includes + "libc++_static", // http://b/198403271, Missing symbols/members in the global namespace when referenced from headers in //external/libcxx/includes + "libc++abi", // http://b/195970501, cc_library_static, duplicate symbols because it propagates libc objects. + "libc++demangle", // http://b/195970501, cc_library_static, duplicate symbols because it propagates libc objects. } // Used for quicker lookups @@ -340,11 +363,10 @@ func (b *BazelModuleBase) ConvertWithBp2build(ctx BazelConversionPathContext) bo func bp2buildDefaultTrueRecursively(packagePath string, config Bp2BuildConfig) bool { ret := false - // Return exact matches in the config. - if config[packagePath] == Bp2BuildDefaultTrueRecursively { + // Check if the package path has an exact match in the config. + if config[packagePath] == Bp2BuildDefaultTrue || config[packagePath] == Bp2BuildDefaultTrueRecursively { return true - } - if config[packagePath] == Bp2BuildDefaultFalse { + } else if config[packagePath] == Bp2BuildDefaultFalse { return false } diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 9c922dda4..50b79fa15 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -27,9 +27,9 @@ import ( "sync" "android/soong/bazel/cquery" + "android/soong/shared" "android/soong/bazel" - "android/soong/shared" ) type cqueryRequest interface { @@ -492,6 +492,12 @@ config_node(name = "%s", ) ` + commonArchFilegroupString := ` +filegroup(name = "common", + srcs = [%s], +) +` + configNodesSection := "" labelsByArch := map[string][]string{} @@ -501,14 +507,22 @@ config_node(name = "%s", labelsByArch[archString] = append(labelsByArch[archString], labelString) } - configNodeLabels := []string{} + allLabels := []string{} for archString, labels := range labelsByArch { - configNodeLabels = append(configNodeLabels, fmt.Sprintf("\":%s\"", archString)) - labelsString := strings.Join(labels, ",\n ") - configNodesSection += fmt.Sprintf(configNodeFormatString, archString, archString, labelsString) + if archString == "common" { + // arch-less labels (e.g. filegroups) don't need a config_node + allLabels = append(allLabels, "\":common\"") + labelsString := strings.Join(labels, ",\n ") + configNodesSection += fmt.Sprintf(commonArchFilegroupString, labelsString) + } else { + // Create a config_node, and add the config_node's label to allLabels + allLabels = append(allLabels, fmt.Sprintf("\":%s\"", archString)) + labelsString := strings.Join(labels, ",\n ") + configNodesSection += fmt.Sprintf(configNodeFormatString, archString, archString, labelsString) + } } - return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(configNodeLabels, ",\n "))) + return []byte(fmt.Sprintf(formatString, configNodesSection, strings.Join(allLabels, ",\n "))) } func indent(original string) string { @@ -573,6 +587,12 @@ def %s(target): %s def get_arch(target): + # TODO(b/199363072): filegroups and file targets aren't associated with any + # specific platform architecture in mixed builds. This is consistent with how + # Soong treats filegroups, but it may not be the case with manually-written + # filegroup BUILD targets. + if target.kind in ["filegroup", ""]: + return "common" buildoptions = build_options(target) platforms = build_options(target)["//command_line_option:platforms"] if len(platforms) != 1: @@ -671,11 +691,12 @@ func (context *bazelContext) InvokeBazel() error { if err != nil { return err } + buildrootLabel := "@soong_injection//mixed_builds:buildroot" cqueryOutput, cqueryErr, err = context.issueBazelCommand( context.paths, bazel.CqueryBuildRootRunName, - bazelCommand{"cquery", fmt.Sprintf("kind(rule, deps(%s))", buildrootLabel)}, + bazelCommand{"cquery", fmt.Sprintf("deps(%s)", buildrootLabel)}, "--output=starlark", "--starlark:file="+absolutePath(cqueryFileRelpath)) err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index 557faea21..cdf1a6316 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -11,7 +11,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) { label := "//foo:bar" arch := Arm64 bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ - bazelCommand{command: "cquery", expression: "kind(rule, deps(@soong_injection//mixed_builds:buildroot))"}: `//foo:bar|arm64>>out/foo/bar.txt`, + bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: `//foo:bar|arm64>>out/foo/bar.txt`, }) g, ok := bazelContext.GetOutputFiles(label, arch) if ok { diff --git a/android/config.go b/android/config.go index d3db68fd2..0767e7b51 100644 --- a/android/config.go +++ b/android/config.go @@ -72,13 +72,25 @@ func (c Config) SoongOutDir() string { } func (c Config) OutDir() string { - return c.soongOutDir + return c.outDir +} + +func (c Config) RunGoTests() bool { + return c.runGoTests } func (c Config) DebugCompilation() bool { return false // Never compile Go code in the main build for debugging } +func (c Config) Subninjas() []string { + return []string{} +} + +func (c Config) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation { + return []bootstrap.PrimaryBuilderInvocation{} +} + // A DeviceConfig object represents the configuration for a particular device // being built. For now there will only be one of these, but in the future there // may be multiple devices being built. @@ -122,9 +134,12 @@ type config struct { deviceConfig *deviceConfig - soongOutDir string // the path of the build output directory + outDir string // The output directory (usually out/) + soongOutDir string moduleListFile string // the path to the file which lists blueprint files to parse. + runGoTests bool + env map[string]string envLock sync.Mutex envDeps map[string]string @@ -137,8 +152,6 @@ type config struct { captureBuild bool // true for tests, saves build parameters for each module ignoreEnvironment bool // true for tests, returns empty from all Getenv calls - stopBefore bootstrap.StopBefore - fs pathtools.FileSystem mockBpList string @@ -283,9 +296,10 @@ func saveToBazelConfigFile(config *productVariables, outDir string) error { // NullConfig returns a mostly empty Config for use by standalone tools like dexpreopt_gen that // use the android package. -func NullConfig(soongOutDir string) Config { +func NullConfig(outDir, soongOutDir string) Config { return Config{ config: &config{ + outDir: outDir, soongOutDir: soongOutDir, fs: pathtools.OsFs, }, @@ -319,6 +333,9 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string ShippingApiLevel: stringPtr("30"), }, + outDir: buildDir, + // soongOutDir is inconsistent with production (it should be buildDir + "/soong") + // but a lot of tests assume this :( soongOutDir: buildDir, captureBuild: true, env: envCopy, @@ -397,7 +414,7 @@ func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[st // multiple runs in the same program execution is carried over (such as Bazel // context or environment deps). func ConfigForAdditionalRun(c Config) (Config, error) { - newConfig, err := NewConfig(c.soongOutDir, c.moduleListFile, c.env) + newConfig, err := NewConfig(c.moduleListFile, c.runGoTests, c.outDir, c.soongOutDir, c.env) if err != nil { return Config{}, err } @@ -408,14 +425,16 @@ func ConfigForAdditionalRun(c Config) (Config, error) { // NewConfig creates a new Config object. The srcDir argument specifies the path // to the root source directory. It also loads the config file, if found. -func NewConfig(soongOutDir string, moduleListFile string, availableEnv map[string]string) (Config, error) { +func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) { // Make a config with default options. config := &config{ ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName), env: availableEnv, + outDir: outDir, soongOutDir: soongOutDir, + runGoTests: runGoTests, multilibConflicts: make(map[ArchType]bool), moduleListFile: moduleListFile, @@ -525,7 +544,7 @@ func (c *config) mockFileSystem(bp string, fs map[string][]byte) { pathsToParse := []string{} for candidate := range mockFS { base := filepath.Base(candidate) - if base == "Blueprints" || base == "Android.bp" { + if base == "Android.bp" { pathsToParse = append(pathsToParse, candidate) } } @@ -538,29 +557,16 @@ func (c *config) mockFileSystem(bp string, fs map[string][]byte) { c.mockBpList = blueprint.MockModuleListFile } -func (c *config) StopBefore() bootstrap.StopBefore { - return c.stopBefore -} - -// SetStopBefore configures soong_build to exit earlier at a specific point. -func (c *config) SetStopBefore(stopBefore bootstrap.StopBefore) { - c.stopBefore = stopBefore -} - func (c *config) SetAllowMissingDependencies() { c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true) } -var _ bootstrap.ConfigStopBefore = (*config)(nil) - // BlueprintToolLocation returns the directory containing build system tools // from Blueprint, like soong_zip and merge_zips. -func (c *config) BlueprintToolLocation() string { +func (c *config) HostToolDir() string { return filepath.Join(c.soongOutDir, "host", c.PrebuiltOS(), "bin") } -var _ bootstrap.ConfigBlueprintToolLocation = (*config)(nil) - func (c *config) HostToolPath(ctx PathContext, tool string) Path { return PathForOutput(ctx, "host", c.PrebuiltOS(), "bin", tool) } diff --git a/android/filegroup.go b/android/filegroup.go index 54d01d39f..4db165f87 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -42,6 +42,27 @@ func FilegroupBp2Build(ctx TopDownMutatorContext) { srcs := bazel.MakeLabelListAttribute( BazelLabelForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)) + + // For Bazel compatibility, don't generate the filegroup if there is only 1 + // source file, and that the source file is named the same as the module + // itself. In Bazel, eponymous filegroups like this would be an error. + // + // Instead, dependents on this single-file filegroup can just depend + // on the file target, instead of rule target, directly. + // + // You may ask: what if a filegroup has multiple files, and one of them + // shares the name? The answer: we haven't seen that in the wild, and + // should lock Soong itself down to prevent the behavior. For now, + // we raise an error if bp2build sees this problem. + for _, f := range srcs.Value.Includes { + if f.Label == fg.Name() { + if len(srcs.Value.Includes) > 1 { + ctx.ModuleErrorf("filegroup '%s' cannot contain a file with the same name", fg.Name()) + } + return + } + } + attrs := &bazelFilegroupAttributes{ Srcs: srcs, } @@ -97,7 +118,7 @@ func (fg *fileGroup) GenerateBazelBuildActions(ctx ModuleContext) bool { } bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), ctx.Arch().ArchType) + filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), Common) if !ok { return false } diff --git a/android/license_kind_test.go b/android/license_kind_test.go index 1f09568db..7a909a6f9 100644 --- a/android/license_kind_test.go +++ b/android/license_kind_test.go @@ -14,38 +14,38 @@ var licenseKindTests = []struct { { name: "license_kind must not accept licenses property", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_license", licenses: ["other_license"], }`), }, expectedErrors: []string{ - `top/Blueprints:4:14: unrecognized property "licenses"`, + `top/Android.bp:4:14: unrecognized property "licenses"`, }, }, { name: "bad license_kind", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_notice", conditions: ["notice"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_license { name: "other_notice", license_kinds: ["notice"], }`), }, expectedErrors: []string{ - `other/Blueprints:2:5: "other_notice" depends on undefined module "notice"`, + `other/Android.bp:2:5: "other_notice" depends on undefined module "notice"`, }, }, { name: "good license kind", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_by_exception_only", conditions: ["by_exception_only"], @@ -55,7 +55,7 @@ var licenseKindTests = []struct { name: "top_proprietary", license_kinds: ["top_by_exception_only"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_license { name: "other_proprietary", license_kinds: ["top_proprietary"], @@ -65,7 +65,7 @@ var licenseKindTests = []struct { { name: "multiple license kinds", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_notice", conditions: ["notice"], @@ -85,7 +85,7 @@ var licenseKindTests = []struct { name: "top_proprietary", license_kinds: ["top_by_exception_only"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_license { name: "other_rule", license_kinds: ["top_by_exception_only"], diff --git a/android/license_test.go b/android/license_test.go index 26b33c367..7222cd741 100644 --- a/android/license_test.go +++ b/android/license_test.go @@ -27,7 +27,7 @@ var licenseTests = []struct { { name: "license must not accept licenses property", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license { name: "top_license", visibility: ["//visibility:private"], @@ -35,13 +35,13 @@ var licenseTests = []struct { }`), }, expectedErrors: []string{ - `top/Blueprints:5:14: unrecognized property "licenses"`, + `top/Android.bp:5:14: unrecognized property "licenses"`, }, }, { name: "private license", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_notice", conditions: ["notice"], @@ -53,27 +53,27 @@ var licenseTests = []struct { license_kinds: ["top_notice"], visibility: ["//visibility:private"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` rule { name: "arule", licenses: ["top_allowed_as_notice"], }`), - "yetmore/Blueprints": []byte(` + "yetmore/Android.bp": []byte(` package { default_applicable_licenses: ["top_allowed_as_notice"], }`), }, expectedErrors: []string{ - `other/Blueprints:2:5: module "arule": depends on //top:top_allowed_as_notice ` + + `other/Android.bp:2:5: module "arule": depends on //top:top_allowed_as_notice ` + `which is not visible to this module`, - `yetmore/Blueprints:2:5: module "//yetmore": depends on //top:top_allowed_as_notice ` + + `yetmore/Android.bp:2:5: module "//yetmore": depends on //top:top_allowed_as_notice ` + `which is not visible to this module`, }, }, { name: "must reference license_kind module", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` rule { name: "top_by_exception_only", } @@ -85,14 +85,14 @@ var licenseTests = []struct { }`), }, expectedErrors: []string{ - `top/Blueprints:6:5: module "top_proprietary": license_kinds property ` + + `top/Android.bp:6:5: module "top_proprietary": license_kinds property ` + `"top_by_exception_only" is not a license_kind module`, }, }, { name: "license_kind module must exist", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license { name: "top_notice_allowed", license_kinds: ["top_notice"], @@ -100,13 +100,13 @@ var licenseTests = []struct { }`), }, expectedErrors: []string{ - `top/Blueprints:2:5: "top_notice_allowed" depends on undefined module "top_notice"`, + `top/Android.bp:2:5: "top_notice_allowed" depends on undefined module "top_notice"`, }, }, { name: "public license", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_by_exception_only", conditions: ["by_exception_only"], @@ -118,12 +118,12 @@ var licenseTests = []struct { license_kinds: ["top_by_exception_only"], visibility: ["//visibility:public"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` rule { name: "arule", licenses: ["top_proprietary"], }`), - "yetmore/Blueprints": []byte(` + "yetmore/Android.bp": []byte(` package { default_applicable_licenses: ["top_proprietary"], }`), @@ -132,7 +132,7 @@ var licenseTests = []struct { { name: "multiple licenses", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_applicable_licenses: ["top_proprietary"], } @@ -162,12 +162,12 @@ var licenseTests = []struct { name: "myrule", licenses: ["top_allowed_as_notice", "top_proprietary"] }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` rule { name: "arule", licenses: ["top_proprietary"], }`), - "yetmore/Blueprints": []byte(` + "yetmore/Android.bp": []byte(` package { default_applicable_licenses: ["top_proprietary"], }`), diff --git a/android/licenses.go b/android/licenses.go index 464ba49b4..d54f8f4d0 100644 --- a/android/licenses.go +++ b/android/licenses.go @@ -293,7 +293,7 @@ func exemptFromRequiredApplicableLicensesProperty(module Module) bool { case "*android.soongConfigModuleTypeModule": // creates aliases for modules with licenses case "*android.soongConfigModuleTypeImport": // creates aliases for modules with licenses case "*android.soongConfigStringVariableDummyModule": // used for creating aliases - case "*android.SoongConfigBoolVariableDummyModule": // used for creating aliases + case "*android.soongConfigBoolVariableDummyModule": // used for creating aliases default: return false } diff --git a/android/licenses_test.go b/android/licenses_test.go index 85033100f..d05b0a35f 100644 --- a/android/licenses_test.go +++ b/android/licenses_test.go @@ -20,7 +20,7 @@ var licensesTests = []struct { { name: "invalid module type without licenses property", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_bad_module { name: "libexample", }`), @@ -30,7 +30,7 @@ var licensesTests = []struct { { name: "license must exist", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", licenses: ["notice"], @@ -41,7 +41,7 @@ var licensesTests = []struct { { name: "all good", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "notice", conditions: ["shownotice"], @@ -58,12 +58,12 @@ var licensesTests = []struct { name: "libexample1", licenses: ["top_Apache2"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", licenses: ["top_Apache2"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", licenses: ["top_Apache2"], @@ -101,7 +101,7 @@ var licensesTests = []struct { // Check that licenses is the union of the defaults modules. name: "defaults union, basic", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license_kind { name: "top_notice", conditions: ["notice"], @@ -125,7 +125,7 @@ var licensesTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` license_kind { name: "nested_notice", conditions: ["notice"], @@ -140,7 +140,7 @@ var licensesTests = []struct { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -174,7 +174,7 @@ var licensesTests = []struct { { name: "defaults union, multiple defaults", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` license { name: "top", } @@ -194,7 +194,7 @@ var licensesTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` license { name: "top_nested", license_text: ["LICENSE.txt"], @@ -203,7 +203,7 @@ var licensesTests = []struct { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` license { name: "other", } @@ -211,7 +211,7 @@ var licensesTests = []struct { name: "libother", deps: ["libexample"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -251,7 +251,7 @@ var licensesTests = []struct { { name: "defaults_licenses invalid", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "top_defaults", licenses: ["notice"], @@ -262,7 +262,7 @@ var licensesTests = []struct { { name: "defaults_licenses overrides package default", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_applicable_licenses: ["by_exception_only"], } @@ -298,7 +298,7 @@ var licensesTests = []struct { { name: "package default_applicable_licenses must exist", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_applicable_licenses: ["notice"], }`), @@ -309,7 +309,7 @@ var licensesTests = []struct { // This test relies on the default licenses being legacy_public. name: "package default_applicable_licenses property used when no licenses specified", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_applicable_licenses: ["top_notice"], } @@ -320,7 +320,7 @@ var licensesTests = []struct { mock_library { name: "libexample", }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -338,7 +338,7 @@ var licensesTests = []struct { { name: "package default_applicable_licenses not inherited to subpackages", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_applicable_licenses: ["top_notice"], } @@ -348,7 +348,7 @@ var licensesTests = []struct { mock_library { name: "libexample", }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` package { default_applicable_licenses: ["outsider"], } @@ -356,11 +356,11 @@ var licensesTests = []struct { mock_library { name: "libnested", }`), - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` mock_library { name: "libother", }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` license { name: "outsider", } @@ -385,7 +385,7 @@ var licensesTests = []struct { { name: "verify that prebuilt dependencies are included", fs: map[string][]byte{ - "prebuilts/Blueprints": []byte(` + "prebuilts/Android.bp": []byte(` license { name: "prebuilt" } @@ -394,7 +394,7 @@ var licensesTests = []struct { licenses: ["prebuilt"], }`), "top/sources/source_file": nil, - "top/sources/Blueprints": []byte(` + "top/sources/Android.bp": []byte(` license { name: "top_sources" } @@ -403,7 +403,7 @@ var licensesTests = []struct { licenses: ["top_sources"], }`), "top/other/source_file": nil, - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` source { name: "other", deps: [":module"], @@ -419,7 +419,7 @@ var licensesTests = []struct { { name: "verify that prebuilt dependencies are ignored for licenses reasons (preferred)", fs: map[string][]byte{ - "prebuilts/Blueprints": []byte(` + "prebuilts/Android.bp": []byte(` license { name: "prebuilt" } @@ -429,7 +429,7 @@ var licensesTests = []struct { prefer: true, }`), "top/sources/source_file": nil, - "top/sources/Blueprints": []byte(` + "top/sources/Android.bp": []byte(` license { name: "top_sources" } @@ -438,7 +438,7 @@ var licensesTests = []struct { licenses: ["top_sources"], }`), "top/other/source_file": nil, - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` source { name: "other", deps: [":module"], diff --git a/android/module.go b/android/module.go index cc0341808..dd6a25a14 100644 --- a/android/module.go +++ b/android/module.go @@ -405,6 +405,7 @@ type ModuleContext interface { PackageFile(installPath InstallPath, name string, srcPath Path) PackagingSpec CheckbuildFile(srcPath Path) + TidyFile(srcPath Path) InstallInData() bool InstallInTestcases() bool @@ -521,62 +522,6 @@ type Module interface { TransitivePackagingSpecs() []PackagingSpec } -// BazelTargetModule is a lightweight wrapper interface around Module for -// bp2build conversion purposes. -// -// In bp2build's bootstrap.Main execution, Soong runs an alternate pipeline of -// mutators that creates BazelTargetModules from regular Module objects, -// performing the mapping from Soong properties to Bazel rule attributes in the -// process. This process may optionally create additional BazelTargetModules, -// resulting in a 1:many mapping. -// -// bp2build.Codegen is then responsible for visiting all modules in the graph, -// filtering for BazelTargetModules, and code-generating BUILD targets from -// them. -type BazelTargetModule interface { - Module - - bazelTargetModuleProperties() *bazel.BazelTargetModuleProperties - SetBazelTargetModuleProperties(props bazel.BazelTargetModuleProperties) - - RuleClass() string - BzlLoadLocation() string -} - -// InitBazelTargetModule is a wrapper function that decorates BazelTargetModule -// with property structs containing metadata for bp2build conversion. -func InitBazelTargetModule(module BazelTargetModule) { - module.AddProperties(module.bazelTargetModuleProperties()) - InitAndroidModule(module) -} - -// BazelTargetModuleBase contains the property structs with metadata for -// bp2build conversion. -type BazelTargetModuleBase struct { - ModuleBase - Properties bazel.BazelTargetModuleProperties -} - -// bazelTargetModuleProperties getter. -func (btmb *BazelTargetModuleBase) bazelTargetModuleProperties() *bazel.BazelTargetModuleProperties { - return &btmb.Properties -} - -// SetBazelTargetModuleProperties setter for BazelTargetModuleProperties -func (btmb *BazelTargetModuleBase) SetBazelTargetModuleProperties(props bazel.BazelTargetModuleProperties) { - btmb.Properties = props -} - -// RuleClass returns the rule class for this Bazel target -func (b *BazelTargetModuleBase) RuleClass() string { - return b.bazelTargetModuleProperties().Rule_class -} - -// BzlLoadLocation returns the rule class for this Bazel target -func (b *BazelTargetModuleBase) BzlLoadLocation() string { - return b.bazelTargetModuleProperties().Bzl_load_location -} - // Qualified id for a module type qualifiedModuleName struct { // The package (i.e. directory) in which the module is defined, without trailing / @@ -987,12 +932,12 @@ const ( DeviceSupported = deviceSupported | deviceDefault // By default, _only_ device variant is built. Device variant can be disabled with `device_supported: false` - // Host and HostCross are disabled by default and can be enabled with `host_supported: true` + // Host and HostCross are disabled by default and can be enabled with `host_supported: true` HostAndDeviceSupported = hostSupported | hostCrossSupported | deviceSupported | deviceDefault // Host, HostCross, and Device are built by default. - // Building Device can be disabled with `device_supported: false` - // Building Host and HostCross can be disabled with `host_supported: false` + // Building Device can be disabled with `device_supported: false` + // Building Host and HostCross can be disabled with `host_supported: false` HostAndDeviceDefault = hostSupported | hostCrossSupported | hostDefault | deviceSupported | deviceDefault @@ -1190,6 +1135,7 @@ type ModuleBase struct { installFiles InstallPaths installFilesDepSet *installPathsDepSet checkbuildFiles Paths + tidyFiles Paths packagingSpecs []PackagingSpec packagingSpecsDepSet *packagingSpecsDepSet noticeFiles Paths @@ -1202,6 +1148,7 @@ type ModuleBase struct { // Only set on the final variant of each module installTarget WritablePath checkbuildTarget WritablePath + tidyTarget WritablePath blueprintDir string hooks hooks @@ -1727,10 +1674,12 @@ func (m *ModuleBase) VintfFragments() Paths { func (m *ModuleBase) generateModuleTarget(ctx ModuleContext) { var allInstalledFiles InstallPaths var allCheckbuildFiles Paths + var allTidyFiles Paths ctx.VisitAllModuleVariants(func(module Module) { a := module.base() allInstalledFiles = append(allInstalledFiles, a.installFiles...) allCheckbuildFiles = append(allCheckbuildFiles, a.checkbuildFiles...) + allTidyFiles = append(allTidyFiles, a.tidyFiles...) }) var deps Paths @@ -1754,6 +1703,13 @@ func (m *ModuleBase) generateModuleTarget(ctx ModuleContext) { deps = append(deps, m.checkbuildTarget) } + if len(allTidyFiles) > 0 { + name := namespacePrefix + ctx.ModuleName() + "-tidy" + ctx.Phony(name, allTidyFiles...) + m.tidyTarget = PathForPhony(ctx, name) + deps = append(deps, m.tidyTarget) + } + if len(deps) > 0 { suffix := "" if ctx.Config().KatiEnabled() { @@ -1962,6 +1918,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.installFiles = append(m.installFiles, ctx.installFiles...) m.checkbuildFiles = append(m.checkbuildFiles, ctx.checkbuildFiles...) + m.tidyFiles = append(m.tidyFiles, ctx.tidyFiles...) m.packagingSpecs = append(m.packagingSpecs, ctx.packagingSpecs...) for k, v := range ctx.phonies { m.phonies[k] = append(m.phonies[k], v...) @@ -2160,6 +2117,7 @@ type moduleContext struct { packagingSpecs []PackagingSpec installFiles InstallPaths checkbuildFiles Paths + tidyFiles Paths module Module phonies map[string]Paths @@ -2892,6 +2850,10 @@ func (m *moduleContext) CheckbuildFile(srcPath Path) { m.checkbuildFiles = append(m.checkbuildFiles, srcPath) } +func (m *moduleContext) TidyFile(srcPath Path) { + m.tidyFiles = append(m.tidyFiles, srcPath) +} + func (m *moduleContext) blueprintModuleContext() blueprint.ModuleContext { return m.bp } @@ -3150,19 +3112,49 @@ func parentDir(dir string) string { type buildTargetSingleton struct{} +func addAncestors(ctx SingletonContext, dirMap map[string]Paths, mmName func(string) string) []string { + // Ensure ancestor directories are in dirMap + // Make directories build their direct subdirectories + dirs := SortedStringKeys(dirMap) + for _, dir := range dirs { + dir := parentDir(dir) + for dir != "." && dir != "/" { + if _, exists := dirMap[dir]; exists { + break + } + dirMap[dir] = nil + dir = parentDir(dir) + } + } + dirs = SortedStringKeys(dirMap) + for _, dir := range dirs { + p := parentDir(dir) + if p != "." && p != "/" { + dirMap[p] = append(dirMap[p], PathForPhony(ctx, mmName(dir))) + } + } + return SortedStringKeys(dirMap) +} + func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { var checkbuildDeps Paths + var tidyDeps Paths mmTarget := func(dir string) string { return "MODULES-IN-" + strings.Replace(filepath.Clean(dir), "/", "-", -1) } + mmTidyTarget := func(dir string) string { + return "tidy-" + strings.Replace(filepath.Clean(dir), "/", "-", -1) + } modulesInDir := make(map[string]Paths) + tidyModulesInDir := make(map[string]Paths) ctx.VisitAllModules(func(module Module) { blueprintDir := module.base().blueprintDir installTarget := module.base().installTarget checkbuildTarget := module.base().checkbuildTarget + tidyTarget := module.base().tidyTarget if checkbuildTarget != nil { checkbuildDeps = append(checkbuildDeps, checkbuildTarget) @@ -3172,6 +3164,16 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { if installTarget != nil { modulesInDir[blueprintDir] = append(modulesInDir[blueprintDir], installTarget) } + + if tidyTarget != nil { + tidyDeps = append(tidyDeps, tidyTarget) + // tidyTarget is in modulesInDir so it will be built with "mm". + modulesInDir[blueprintDir] = append(modulesInDir[blueprintDir], tidyTarget) + // tidyModulesInDir contains tidyTarget but not checkbuildTarget + // or installTarget, so tidy targets in a directory can be built + // without other checkbuild or install targets. + tidyModulesInDir[blueprintDir] = append(tidyModulesInDir[blueprintDir], tidyTarget) + } }) suffix := "" @@ -3182,32 +3184,25 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx SingletonContext) { // Create a top-level checkbuild target that depends on all modules ctx.Phony("checkbuild"+suffix, checkbuildDeps...) - // Make will generate the MODULES-IN-* targets - if ctx.Config().KatiEnabled() { - return - } + // Create a top-level tidy target that depends on all modules + ctx.Phony("tidy"+suffix, tidyDeps...) + + dirs := addAncestors(ctx, tidyModulesInDir, mmTidyTarget) - // Ensure ancestor directories are in modulesInDir - dirs := SortedStringKeys(modulesInDir) + // Kati does not generate tidy-* phony targets yet. + // Create a tidy-<directory> target that depends on all subdirectories + // and modules in the directory. for _, dir := range dirs { - dir := parentDir(dir) - for dir != "." && dir != "/" { - if _, exists := modulesInDir[dir]; exists { - break - } - modulesInDir[dir] = nil - dir = parentDir(dir) - } + ctx.Phony(mmTidyTarget(dir), tidyModulesInDir[dir]...) } - // Make directories build their direct subdirectories - for _, dir := range dirs { - p := parentDir(dir) - if p != "." && p != "/" { - modulesInDir[p] = append(modulesInDir[p], PathForPhony(ctx, mmTarget(dir))) - } + // Make will generate the MODULES-IN-* targets + if ctx.Config().KatiEnabled() { + return } + dirs = addAncestors(ctx, modulesInDir, mmTarget) + // Create a MODULES-IN-<directory> target that depends on all modules in a directory, and // depends on the MODULES-IN-* targets of all of its subdirectories that contain Android.bp // files. diff --git a/android/override_module.go b/android/override_module.go index e72cb787a..51e74d489 100644 --- a/android/override_module.go +++ b/android/override_module.go @@ -295,7 +295,7 @@ func performOverrideMutator(ctx BottomUpMutatorContext) { } func overridableModuleDepsMutator(ctx BottomUpMutatorContext) { - if b, ok := ctx.Module().(OverridableModule); ok { + if b, ok := ctx.Module().(OverridableModule); ok && b.Enabled() { b.OverridablePropertiesDepsMutator(ctx) } } diff --git a/android/package_test.go b/android/package_test.go index 3bd30cc93..7ea10a4da 100644 --- a/android/package_test.go +++ b/android/package_test.go @@ -13,7 +13,7 @@ var packageTests = []struct { { name: "package must not accept visibility and name properties", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { name: "package", visibility: ["//visibility:private"], @@ -21,21 +21,21 @@ var packageTests = []struct { }`), }, expectedErrors: []string{ - `top/Blueprints:5:14: unrecognized property "licenses"`, - `top/Blueprints:3:10: unrecognized property "name"`, - `top/Blueprints:4:16: unrecognized property "visibility"`, + `top/Android.bp:5:14: unrecognized property "licenses"`, + `top/Android.bp:3:10: unrecognized property "name"`, + `top/Android.bp:4:16: unrecognized property "visibility"`, }, }, { name: "multiple packages in separate directories", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` package { }`), - "other/nested/Blueprints": []byte(` + "other/nested/Android.bp": []byte(` package { }`), }, @@ -43,7 +43,7 @@ var packageTests = []struct { { name: "package must not be specified more than once per package", fs: map[string][]byte{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:private"], default_applicable_licenses: ["license"], diff --git a/android/queryview.go b/android/queryview.go deleted file mode 100644 index 224652eba..000000000 --- a/android/queryview.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "os" - "strings" - - "github.com/google/blueprint" -) - -// The Bazel QueryView singleton is responsible for generating the Ninja actions -// for calling the soong_build primary builder in the main build.ninja file. -func init() { - RegisterSingletonType("bazel_queryview", BazelQueryViewSingleton) -} - -// BazelQueryViewSingleton is the singleton responsible for registering the -// soong_build build statement that will convert the Soong module graph after -// applying *all* mutators, enabing the feature to query the final state of the -// Soong graph. This mode is meant for querying the build graph state, and not meant -// for generating BUILD files to be checked in. -func BazelQueryViewSingleton() Singleton { - return &bazelQueryViewSingleton{} -} - -// BazelConverterSingleton is the singleton responsible for registering the soong_build -// build statement that will convert the Soong module graph by applying an alternate -// pipeline of mutators, with the goal of reaching semantic equivalence between the original -// Blueprint and final BUILD files. Using this mode, the goal is to be able to -// build with these BUILD files directly in the source tree. -func BazelConverterSingleton() Singleton { - return &bazelConverterSingleton{} -} - -type bazelQueryViewSingleton struct{} -type bazelConverterSingleton struct{} - -func generateBuildActionsForBazelConversion(ctx SingletonContext, converterMode bool) { - name := "queryview" - descriptionTemplate := "[EXPERIMENTAL, PRE-PRODUCTION] Creating the Bazel QueryView workspace with %s at $outDir" - - // Create a build and rule statement, using the Bazel QueryView's WORKSPACE - // file as the output file marker. - var deps Paths - moduleListFilePath := pathForBuildToolDep(ctx, ctx.Config().moduleListFile) - deps = append(deps, moduleListFilePath) - deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName)) - - bazelQueryViewDirectory := PathForOutput(ctx, name) - bazelQueryViewWorkspaceFile := bazelQueryViewDirectory.Join(ctx, "WORKSPACE") - primaryBuilder := primaryBuilderPath(ctx) - bazelQueryView := ctx.Rule(pctx, "bazelQueryView", - blueprint.RuleParams{ - Command: fmt.Sprintf( - `rm -rf "${outDir}/"* && `+ - `mkdir -p "${outDir}" && `+ - `echo WORKSPACE: $$(cat "%s") > "${outDir}/.queryview-depfile.d" && `+ - `BUILDER="%s" && `+ - `echo BUILDER=$$BUILDER && `+ - `cd "$$(dirname "$$BUILDER")" && `+ - `echo PWD=$$PWD && `+ - `ABSBUILDER="$$PWD/$$(basename "$$BUILDER")" && `+ - `echo ABSBUILDER=$$ABSBUILDER && `+ - `cd / && `+ - `env -i "$$ABSBUILDER" --bazel_queryview_dir "${outDir}" "%s"`, - moduleListFilePath.String(), // Use the contents of Android.bp.list as the depfile. - primaryBuilder.String(), - strings.Join(os.Args[1:], "\" \""), - ), - CommandDeps: []string{primaryBuilder.String()}, - Description: fmt.Sprintf( - descriptionTemplate, - primaryBuilder.Base()), - Deps: blueprint.DepsGCC, - Depfile: "${outDir}/.queryview-depfile.d", - }, - "outDir") - - ctx.Build(pctx, BuildParams{ - Rule: bazelQueryView, - Output: bazelQueryViewWorkspaceFile, - Inputs: deps, - Args: map[string]string{ - "outDir": bazelQueryViewDirectory.String(), - }, - }) - - // Add a phony target for generating the workspace - ctx.Phony(name, bazelQueryViewWorkspaceFile) -} - -func (c *bazelQueryViewSingleton) GenerateBuildActions(ctx SingletonContext) { - generateBuildActionsForBazelConversion(ctx, false) -} - -func (c *bazelConverterSingleton) GenerateBuildActions(ctx SingletonContext) { - generateBuildActionsForBazelConversion(ctx, true) -} diff --git a/android/sdk.go b/android/sdk.go index cf434b01e..b8f76c180 100644 --- a/android/sdk.go +++ b/android/sdk.go @@ -401,26 +401,26 @@ type SdkMemberTypeDependencyTag interface { ExportMember() bool } -var _ SdkMemberTypeDependencyTag = (*sdkMemberDependencyTag)(nil) -var _ ReplaceSourceWithPrebuilt = (*sdkMemberDependencyTag)(nil) +var _ SdkMemberTypeDependencyTag = (*sdkMemberTypeDependencyTag)(nil) +var _ ReplaceSourceWithPrebuilt = (*sdkMemberTypeDependencyTag)(nil) -type sdkMemberDependencyTag struct { +type sdkMemberTypeDependencyTag struct { blueprint.BaseDependencyTag memberType SdkMemberType export bool } -func (t *sdkMemberDependencyTag) SdkMemberType(_ Module) SdkMemberType { +func (t *sdkMemberTypeDependencyTag) SdkMemberType(_ Module) SdkMemberType { return t.memberType } -func (t *sdkMemberDependencyTag) ExportMember() bool { +func (t *sdkMemberTypeDependencyTag) ExportMember() bool { return t.export } // Prevent dependencies from the sdk/module_exports onto their members from being // replaced with a preferred prebuilt. -func (t *sdkMemberDependencyTag) ReplaceSourceWithPrebuilt() bool { +func (t *sdkMemberTypeDependencyTag) ReplaceSourceWithPrebuilt() bool { return false } @@ -428,7 +428,7 @@ func (t *sdkMemberDependencyTag) ReplaceSourceWithPrebuilt() bool { // dependencies added by the tag to be added to the sdk as the specified SdkMemberType and exported // (or not) as specified by the export parameter. func DependencyTagForSdkMemberType(memberType SdkMemberType, export bool) SdkMemberTypeDependencyTag { - return &sdkMemberDependencyTag{memberType: memberType, export: export} + return &sdkMemberTypeDependencyTag{memberType: memberType, export: export} } // Interface that must be implemented for every type that can be a member of an @@ -610,8 +610,10 @@ func (r *SdkMemberTypesRegistry) UniqueOnceKey() OnceKey { return NewCustomOnceKey(r) } -// The set of registered SdkMemberTypes, one for sdk module and one for module_exports. +// The set of registered SdkMemberTypes for module_exports modules. var ModuleExportsMemberTypes = &SdkMemberTypesRegistry{} + +// The set of registered SdkMemberTypes for sdk modules. var SdkMemberTypes = &SdkMemberTypesRegistry{} // Register an SdkMemberType object to allow them to be used in the sdk and sdk_snapshot module diff --git a/android/testing.go b/android/testing.go index e25e5c5f5..bd2faa291 100644 --- a/android/testing.go +++ b/android/testing.go @@ -509,12 +509,11 @@ func (ctx *TestContext) ModuleForTests(name, variant string) TestingModule { allVariants = append(allVariants, ctx.ModuleSubDir(m)) } }) - sort.Strings(allModuleNames) sort.Strings(allVariants) if len(allVariants) == 0 { panic(fmt.Errorf("failed to find module %q. All modules:\n %s", - name, strings.Join(allModuleNames, "\n "))) + name, strings.Join(SortedUniqueStrings(allModuleNames), "\n "))) } else { panic(fmt.Errorf("failed to find module %q variant %q. All variants:\n %s", name, variant, strings.Join(allVariants, "\n "))) diff --git a/android/variable.go b/android/variable.go index 9d7c1e624..a1af5279e 100644 --- a/android/variable.go +++ b/android/variable.go @@ -46,6 +46,10 @@ type variableProperties struct { Java_resource_dirs []string } + Platform_sdk_extension_version struct { + Cmd *string + } + // unbundled_build is a catch-all property to annotate modules that don't build in one or // more unbundled branches, usually due to dependencies missing from the manifest. Unbundled_build struct { @@ -172,6 +176,7 @@ type productVariables struct { Platform_sdk_codename *string `json:",omitempty"` Platform_sdk_version_or_codename *string `json:",omitempty"` Platform_sdk_final *bool `json:",omitempty"` + Platform_sdk_extension_version *int `json:",omitempty"` Platform_version_active_codenames []string `json:",omitempty"` Platform_vndk_version *string `json:",omitempty"` Platform_systemsdk_versions []string `json:",omitempty"` diff --git a/android/visibility_test.go b/android/visibility_test.go index ffd79093e..714c92a71 100644 --- a/android/visibility_test.go +++ b/android/visibility_test.go @@ -16,7 +16,7 @@ var visibilityTests = []struct { { name: "invalid visibility: empty list", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: [], @@ -27,7 +27,7 @@ var visibilityTests = []struct { { name: "invalid visibility: empty rule", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: [""], @@ -38,7 +38,7 @@ var visibilityTests = []struct { { name: "invalid visibility: unqualified", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["target"], @@ -49,7 +49,7 @@ var visibilityTests = []struct { { name: "invalid visibility: empty namespace", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//"], @@ -60,7 +60,7 @@ var visibilityTests = []struct { { name: "invalid visibility: empty module", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: [":"], @@ -71,7 +71,7 @@ var visibilityTests = []struct { { name: "invalid visibility: empty namespace and module", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//:"], @@ -82,7 +82,7 @@ var visibilityTests = []struct { { name: "//visibility:unknown", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//visibility:unknown"], @@ -93,7 +93,7 @@ var visibilityTests = []struct { { name: "//visibility:xxx mixed", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//visibility:public", "//namespace"], @@ -114,7 +114,7 @@ var visibilityTests = []struct { { name: "//visibility:legacy_public", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//visibility:legacy_public"], @@ -130,7 +130,7 @@ var visibilityTests = []struct { // the current directory, a nested directory and a directory in a separate tree. name: "//visibility:public", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//visibility:public"], @@ -140,12 +140,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -157,7 +157,7 @@ var visibilityTests = []struct { // directory only. name: "//visibility:private", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//visibility:private"], @@ -167,12 +167,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -189,7 +189,7 @@ var visibilityTests = []struct { // Verify that :__pkg__ allows the module to be referenced from the current directory only. name: ":__pkg__", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: [":__pkg__"], @@ -199,12 +199,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -222,7 +222,7 @@ var visibilityTests = []struct { // the top/nested directory only, not a subdirectory of top/nested and not peak directory. name: "//top/nested", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//top/nested"], @@ -232,17 +232,17 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "top/nested/again/Blueprints": []byte(` + "top/nested/again/Android.bp": []byte(` mock_library { name: "libnestedagain", deps: ["libexample"], }`), - "peak/Blueprints": []byte(` + "peak/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -260,7 +260,7 @@ var visibilityTests = []struct { // and sub directories but nowhere else. name: ":__subpackages__", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: [":__subpackages__"], @@ -270,12 +270,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "peak/other/Blueprints": []byte(` + "peak/other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -291,7 +291,7 @@ var visibilityTests = []struct { // directory and sub directories but nowhere else. name: "//top/nested:__subpackages__", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//top/nested:__subpackages__", "//other"], @@ -301,12 +301,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -322,7 +322,7 @@ var visibilityTests = []struct { // the current directory, top/nested and peak and all its subpackages. name: `["//top/nested", "//peak:__subpackages__"]`, fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//top/nested", "//peak:__subpackages__"], @@ -332,12 +332,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "peak/other/Blueprints": []byte(` + "peak/other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -348,7 +348,7 @@ var visibilityTests = []struct { // Verify that //vendor... cannot be used outside vendor apart from //vendor:__subpackages__ name: `//vendor`, fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//vendor:__subpackages__"], @@ -358,13 +358,13 @@ var visibilityTests = []struct { name: "libsamepackage", visibility: ["//vendor/apps/AcmeSettings"], }`), - "vendor/Blueprints": []byte(` + "vendor/Android.bp": []byte(` mock_library { name: "libvendorexample", deps: ["libexample"], visibility: ["//vendor/nested"], }`), - "vendor/nested/Blueprints": []byte(` + "vendor/nested/Android.bp": []byte(` mock_library { name: "libvendornested", deps: ["libexample", "libvendorexample"], @@ -382,7 +382,7 @@ var visibilityTests = []struct { // Check that visibility is the union of the defaults modules. name: "defaults union, basic", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//other"], @@ -396,17 +396,17 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -420,7 +420,7 @@ var visibilityTests = []struct { { name: "defaults union, multiple defaults", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults_1", visibility: ["//other"], @@ -437,17 +437,17 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -461,7 +461,7 @@ var visibilityTests = []struct { { name: "//visibility:public mixed with other in defaults", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:public", "//namespace"], @@ -479,7 +479,7 @@ var visibilityTests = []struct { { name: "//visibility:public overriding defaults", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//namespace"], @@ -489,7 +489,7 @@ var visibilityTests = []struct { visibility: ["//visibility:public"], defaults: ["libexample_defaults"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -502,7 +502,7 @@ var visibilityTests = []struct { { name: "//visibility:public mixed with other from different defaults 1", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults_1", visibility: ["//namespace"], @@ -515,7 +515,7 @@ var visibilityTests = []struct { name: "libexample", defaults: ["libexample_defaults_1", "libexample_defaults_2"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -525,7 +525,7 @@ var visibilityTests = []struct { { name: "//visibility:public mixed with other from different defaults 2", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults_1", visibility: ["//visibility:public"], @@ -538,7 +538,7 @@ var visibilityTests = []struct { name: "libexample", defaults: ["libexample_defaults_1", "libexample_defaults_2"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -548,7 +548,7 @@ var visibilityTests = []struct { { name: "//visibility:private in defaults", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:private"], @@ -561,12 +561,12 @@ var visibilityTests = []struct { name: "libsamepackage", deps: ["libexample"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -582,7 +582,7 @@ var visibilityTests = []struct { { name: "//visibility:private mixed with other in defaults", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:private", "//namespace"], @@ -600,7 +600,7 @@ var visibilityTests = []struct { { name: "//visibility:private overriding defaults", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//namespace"], @@ -619,7 +619,7 @@ var visibilityTests = []struct { { name: "//visibility:private in defaults overridden", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:private"], @@ -638,7 +638,7 @@ var visibilityTests = []struct { { name: "//visibility:private override //visibility:public", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:public"], @@ -656,7 +656,7 @@ var visibilityTests = []struct { { name: "//visibility:public override //visibility:private", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:private"], @@ -674,7 +674,7 @@ var visibilityTests = []struct { { name: "//visibility:override must be first in the list", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_library { name: "libexample", visibility: ["//other", "//visibility:override", "//namespace"], @@ -687,7 +687,7 @@ var visibilityTests = []struct { { name: "//visibility:override discards //visibility:private", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:private"], @@ -698,7 +698,7 @@ var visibilityTests = []struct { visibility: ["//visibility:override", "//other"], defaults: ["libexample_defaults"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], @@ -708,7 +708,7 @@ var visibilityTests = []struct { { name: "//visibility:override discards //visibility:public", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:public"], @@ -719,12 +719,12 @@ var visibilityTests = []struct { visibility: ["//visibility:override", "//other"], defaults: ["libexample_defaults"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], }`), - "namespace/Blueprints": []byte(` + "namespace/Android.bp": []byte(` mock_library { name: "libnamespace", deps: ["libexample"], @@ -737,7 +737,7 @@ var visibilityTests = []struct { { name: "//visibility:override discards defaults supplied rules", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//namespace"], @@ -748,12 +748,12 @@ var visibilityTests = []struct { visibility: ["//visibility:override", "//other"], defaults: ["libexample_defaults"], }`), - "other/Blueprints": []byte(` + "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], }`), - "namespace/Blueprints": []byte(` + "namespace/Android.bp": []byte(` mock_library { name: "libnamespace", deps: ["libexample"], @@ -766,7 +766,7 @@ var visibilityTests = []struct { { name: "//visibility:override can override //visibility:public with //visibility:private", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:public"], @@ -776,7 +776,7 @@ var visibilityTests = []struct { visibility: ["//visibility:override", "//visibility:private"], defaults: ["libexample_defaults"], }`), - "namespace/Blueprints": []byte(` + "namespace/Android.bp": []byte(` mock_library { name: "libnamespace", deps: ["libexample"], @@ -789,7 +789,7 @@ var visibilityTests = []struct { { name: "//visibility:override can override //visibility:private with //visibility:public", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults", visibility: ["//visibility:private"], @@ -799,7 +799,7 @@ var visibilityTests = []struct { visibility: ["//visibility:override", "//visibility:public"], defaults: ["libexample_defaults"], }`), - "namespace/Blueprints": []byte(` + "namespace/Android.bp": []byte(` mock_library { name: "libnamespace", deps: ["libexample"], @@ -809,7 +809,7 @@ var visibilityTests = []struct { { name: "//visibility:private mixed with itself", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "libexample_defaults_1", visibility: ["//visibility:private"], @@ -823,7 +823,7 @@ var visibilityTests = []struct { visibility: ["//visibility:private"], defaults: ["libexample_defaults_1", "libexample_defaults_2"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -839,7 +839,7 @@ var visibilityTests = []struct { { name: "defaults_visibility invalid", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_defaults { name: "top_defaults", defaults_visibility: ["//visibility:invalid"], @@ -852,7 +852,7 @@ var visibilityTests = []struct { { name: "defaults_visibility overrides package default", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:private"], } @@ -860,7 +860,7 @@ var visibilityTests = []struct { name: "top_defaults", defaults_visibility: ["//visibility:public"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", defaults: ["top_defaults"], @@ -872,7 +872,7 @@ var visibilityTests = []struct { { name: "package default_visibility property is checked", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:invalid"], }`), @@ -883,7 +883,7 @@ var visibilityTests = []struct { // This test relies on the default visibility being legacy_public. name: "package default_visibility property used when no visibility specified", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:private"], } @@ -891,7 +891,7 @@ var visibilityTests = []struct { mock_library { name: "libexample", }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -905,7 +905,7 @@ var visibilityTests = []struct { { name: "package default_visibility public does not override visibility private", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:public"], } @@ -914,7 +914,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//visibility:private"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -928,7 +928,7 @@ var visibilityTests = []struct { { name: "package default_visibility private does not override visibility public", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:private"], } @@ -937,7 +937,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//visibility:public"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -947,7 +947,7 @@ var visibilityTests = []struct { { name: "package default_visibility :__subpackages__", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: [":__subpackages__"], } @@ -955,12 +955,12 @@ var visibilityTests = []struct { mock_library { name: "libexample", }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample"], @@ -974,7 +974,7 @@ var visibilityTests = []struct { { name: "package default_visibility inherited to subpackages", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//outsider"], } @@ -983,12 +983,12 @@ var visibilityTests = []struct { name: "libexample", visibility: [":__subpackages__"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libexample", "libnested"], @@ -1002,11 +1002,11 @@ var visibilityTests = []struct { { name: "package default_visibility inherited to subpackages", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` package { default_visibility: ["//visibility:private"], }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` package { default_visibility: ["//outsider"], } @@ -1014,11 +1014,11 @@ var visibilityTests = []struct { mock_library { name: "libnested", }`), - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` mock_library { name: "libother", }`), - "outsider/Blueprints": []byte(` + "outsider/Android.bp": []byte(` mock_library { name: "liboutsider", deps: ["libother", "libnested"], @@ -1032,19 +1032,19 @@ var visibilityTests = []struct { { name: "verify that prebuilt dependencies are ignored for visibility reasons (not preferred)", fs: MockFS{ - "prebuilts/Blueprints": []byte(` + "prebuilts/Android.bp": []byte(` prebuilt { name: "module", visibility: ["//top/other"], }`), "top/sources/source_file": nil, - "top/sources/Blueprints": []byte(` + "top/sources/Android.bp": []byte(` source { name: "module", visibility: ["//top/other"], }`), "top/other/source_file": nil, - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` source { name: "other", deps: [":module"], @@ -1054,20 +1054,20 @@ var visibilityTests = []struct { { name: "verify that prebuilt dependencies are ignored for visibility reasons (preferred)", fs: MockFS{ - "prebuilts/Blueprints": []byte(` + "prebuilts/Android.bp": []byte(` prebuilt { name: "module", visibility: ["//top/other"], prefer: true, }`), "top/sources/source_file": nil, - "top/sources/Blueprints": []byte(` + "top/sources/Android.bp": []byte(` source { name: "module", visibility: ["//top/other"], }`), "top/other/source_file": nil, - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` source { name: "other", deps: [":module"], @@ -1077,7 +1077,7 @@ var visibilityTests = []struct { { name: "ensure visibility properties are checked for correctness", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_parent { name: "parent", visibility: ["//top/nested"], @@ -1094,7 +1094,7 @@ var visibilityTests = []struct { { name: "invalid visibility added to child detected during gather phase", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_parent { name: "parent", visibility: ["//top/nested"], @@ -1116,7 +1116,7 @@ var visibilityTests = []struct { { name: "automatic visibility inheritance enabled", fs: MockFS{ - "top/Blueprints": []byte(` + "top/Android.bp": []byte(` mock_parent { name: "parent", visibility: ["//top/nested"], @@ -1125,12 +1125,12 @@ var visibilityTests = []struct { visibility: ["//top/other"], }, }`), - "top/nested/Blueprints": []byte(` + "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libchild"], }`), - "top/other/Blueprints": []byte(` + "top/other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libchild"], diff --git a/android/writedocs.go b/android/writedocs.go deleted file mode 100644 index c380a3d84..000000000 --- a/android/writedocs.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package android - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/google/blueprint" -) - -func init() { - RegisterSingletonType("writedocs", DocsSingleton) -} - -func DocsSingleton() Singleton { - return &docsSingleton{} -} - -type docsSingleton struct{} - -func primaryBuilderPath(ctx SingletonContext) Path { - soongOutDir := absolutePath(ctx.Config().SoongOutDir()) - binary := absolutePath(os.Args[0]) - primaryBuilder, err := filepath.Rel(soongOutDir, binary) - if err != nil { - ctx.Errorf("path to primary builder %q is not in build dir %q (%q)", - os.Args[0], ctx.Config().SoongOutDir(), err) - } - - return PathForOutput(ctx, primaryBuilder) -} - -func (c *docsSingleton) GenerateBuildActions(ctx SingletonContext) { - var deps Paths - deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().moduleListFile)) - deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName)) - - // The dexpreopt configuration may not exist, but if it does, it's a dependency - // of soong_build. - dexpreoptConfigPath := ctx.Config().DexpreoptGlobalConfigPath(ctx) - if dexpreoptConfigPath.Valid() { - deps = append(deps, dexpreoptConfigPath.Path()) - } - - // Generate build system docs for the primary builder. Generating docs reads the source - // files used to build the primary builder, but that dependency will be picked up through - // the dependency on the primary builder itself. There are no dependencies on the - // Blueprints files, as any relevant changes to the Blueprints files would have caused - // a rebuild of the primary builder. - docsFile := PathForOutput(ctx, "docs", "soong_build.html") - primaryBuilder := primaryBuilderPath(ctx) - soongDocs := ctx.Rule(pctx, "soongDocs", - blueprint.RuleParams{ - Command: fmt.Sprintf("rm -f ${outDir}/* && %s --soong_docs %s %s", - primaryBuilder.String(), - docsFile.String(), - "\""+strings.Join(os.Args[1:], "\" \"")+"\""), - CommandDeps: []string{primaryBuilder.String()}, - Description: fmt.Sprintf("%s docs $out", primaryBuilder.Base()), - }, - "outDir") - - ctx.Build(pctx, BuildParams{ - Rule: soongDocs, - Output: docsFile, - Inputs: deps, - Args: map[string]string{ - "outDir": PathForOutput(ctx, "docs").String(), - }, - }) - - // Add a phony target for building the documentation - ctx.Phony("soong_docs", docsFile) -} diff --git a/androidmk/androidmk/android.go b/androidmk/androidmk/android.go index 08616a90e..80801b290 100644 --- a/androidmk/androidmk/android.go +++ b/androidmk/androidmk/android.go @@ -201,6 +201,7 @@ func init() { "LOCAL_VENDOR_MODULE": "vendor", "LOCAL_ODM_MODULE": "device_specific", "LOCAL_PRODUCT_MODULE": "product_specific", + "LOCAL_PRODUCT_SERVICES_MODULE": "product_specific", "LOCAL_SYSTEM_EXT_MODULE": "system_ext_specific", "LOCAL_EXPORT_PACKAGE_RESOURCES": "export_package_resources", "LOCAL_PRIVILEGED_MODULE": "privileged", diff --git a/apex/androidmk.go b/apex/androidmk.go index 2f1090421..94b81163d 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -382,7 +382,7 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", a.installDir.ToMakePath().String()) stemSuffix := apexType.suffix() if a.isCompressed { - stemSuffix = ".capex" + stemSuffix = imageCapexSuffix } fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+stemSuffix) fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !a.installable()) diff --git a/apex/apex.go b/apex/apex.go index 6bafcae52..e3edc681d 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -111,9 +111,6 @@ type apexBundleProperties struct { // List of java libraries that are embedded inside this APEX bundle. Java_libs []string - // List of prebuilt files that are embedded inside this APEX bundle. - Prebuilts []string - // List of platform_compat_config files that are embedded inside this APEX bundle. Compat_configs []string @@ -291,6 +288,9 @@ type overridableProperties struct { // List of APKs that are embedded inside this APEX. Apps []string + // List of prebuilt files that are embedded inside this APEX bundle. + Prebuilts []string + // List of runtime resource overlays (RROs) that are embedded inside this APEX. Rros []string @@ -684,7 +684,6 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { // each target os/architectures, appropriate dependencies are selected by their // target.<os>.multilib.<type> groups and are added as (direct) dependencies. targets := ctx.MultiTargets() - config := ctx.DeviceConfig() imageVariation := a.getImageVariation(ctx) a.combineProperties(ctx) @@ -758,23 +757,6 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { } } - if prebuilts := a.properties.Prebuilts; len(prebuilts) > 0 { - // For prebuilt_etc, use the first variant (64 on 64/32bit device, 32 on 32bit device) - // regardless of the TARGET_PREFER_* setting. See b/144532908 - archForPrebuiltEtc := config.Arches()[0] - for _, arch := range config.Arches() { - // Prefer 64-bit arch if there is any - if arch.ArchType.Multilib == "lib64" { - archForPrebuiltEtc = arch - break - } - } - ctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "os", Variation: ctx.Os().String()}, - {Mutator: "arch", Variation: archForPrebuiltEtc.String()}, - }, prebuiltTag, prebuilts...) - } - // Common-arch dependencies come next commonVariation := ctx.Config().AndroidCommonTarget.Variations() ctx.AddFarVariationDependencies(commonVariation, bcpfTag, a.properties.Bootclasspath_fragments...) @@ -814,6 +796,25 @@ func (a *apexBundle) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato ctx.AddFarVariationDependencies(commonVariation, androidAppTag, a.overridableProperties.Apps...) ctx.AddFarVariationDependencies(commonVariation, bpfTag, a.overridableProperties.Bpfs...) ctx.AddFarVariationDependencies(commonVariation, rroTag, a.overridableProperties.Rros...) + if prebuilts := a.overridableProperties.Prebuilts; len(prebuilts) > 0 { + // For prebuilt_etc, use the first variant (64 on 64/32bit device, 32 on 32bit device) + // regardless of the TARGET_PREFER_* setting. See b/144532908 + arches := ctx.DeviceConfig().Arches() + if len(arches) != 0 { + archForPrebuiltEtc := arches[0] + for _, arch := range arches { + // Prefer 64-bit arch if there is any + if arch.ArchType.Multilib == "lib64" { + archForPrebuiltEtc = arch + break + } + } + ctx.AddFarVariationDependencies([]blueprint.Variation{ + {Mutator: "os", Variation: ctx.Os().String()}, + {Mutator: "arch", Variation: archForPrebuiltEtc.String()}, + }, prebuiltTag, prebuilts...) + } + } // Dependencies for signing if String(a.overridableProperties.Key) == "" { @@ -1151,9 +1152,10 @@ const ( const ( // File extensions of an APEX for different packaging methods - imageApexSuffix = ".apex" - zipApexSuffix = ".zipapex" - flattenedSuffix = ".flattened" + imageApexSuffix = ".apex" + imageCapexSuffix = ".capex" + zipApexSuffix = ".zipapex" + flattenedSuffix = ".flattened" // variant names each of which is for a packaging method imageApexType = "image" @@ -3282,7 +3284,7 @@ func apexBundleBp2BuildInternal(ctx android.TopDownMutatorContext, module *apexB nativeSharedLibsLabelList := android.BazelLabelForModuleDeps(ctx, nativeSharedLibs) nativeSharedLibsLabelListAttribute := bazel.MakeLabelListAttribute(nativeSharedLibsLabelList) - prebuilts := module.properties.Prebuilts + prebuilts := module.overridableProperties.Prebuilts prebuiltsLabelList := android.BazelLabelForModuleDeps(ctx, prebuilts) prebuiltsLabelListAttribute := bazel.MakeLabelListAttribute(prebuiltsLabelList) diff --git a/apex/apex_test.go b/apex/apex_test.go index daaa5cb46..6027f9b46 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -4640,6 +4640,35 @@ func TestPrebuiltFilenameOverride(t *testing.T) { } } +func TestApexSetFilenameOverride(t *testing.T) { + testApex(t, ` + apex_set { + name: "com.company.android.myapex", + apex_name: "com.android.myapex", + set: "company-myapex.apks", + filename: "com.company.android.myapex.apex" + } + `).ModuleForTests("com.company.android.myapex", "android_common_com.android.myapex") + + testApex(t, ` + apex_set { + name: "com.company.android.myapex", + apex_name: "com.android.myapex", + set: "company-myapex.apks", + filename: "com.company.android.myapex.capex" + } + `).ModuleForTests("com.company.android.myapex", "android_common_com.android.myapex") + + testApexError(t, `filename should end in .apex or .capex for apex_set`, ` + apex_set { + name: "com.company.android.myapex", + apex_name: "com.android.myapex", + set: "company-myapex.apks", + filename: "some-random-suffix" + } + `) +} + func TestPrebuiltOverrides(t *testing.T) { ctx := testApex(t, ` prebuilt_apex { @@ -6080,6 +6109,7 @@ func TestOverrideApex(t *testing.T) { key: "myapex.key", apps: ["app"], bpfs: ["bpf"], + prebuilts: ["myetc"], overrides: ["oldapex"], updatable: false, } @@ -6089,6 +6119,7 @@ func TestOverrideApex(t *testing.T) { base: "myapex", apps: ["override_app"], bpfs: ["override_bpf"], + prebuilts: ["override_myetc"], overrides: ["unknownapex"], logging_parent: "com.foo.bar", package_name: "test.overridden.package", @@ -6137,6 +6168,16 @@ func TestOverrideApex(t *testing.T) { name: "override_bpf", srcs: ["override_bpf.c"], } + + prebuilt_etc { + name: "myetc", + src: "myprebuilt", + } + + prebuilt_etc { + name: "override_myetc", + src: "override_myprebuilt", + } `, withManifestPackageNameOverrides([]string{"myapex:com.android.myapex"})) originalVariant := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(android.OverridableModule) @@ -6158,6 +6199,9 @@ func TestOverrideApex(t *testing.T) { ensureNotContains(t, copyCmds, "image.apex/etc/bpf/bpf.o") ensureContains(t, copyCmds, "image.apex/etc/bpf/override_bpf.o") + ensureNotContains(t, copyCmds, "image.apex/etc/myetc") + ensureContains(t, copyCmds, "image.apex/etc/override_myetc") + apexBundle := module.Module().(*apexBundle) name := apexBundle.Name() if name != "override_myapex" { diff --git a/apex/builder.go b/apex/builder.go index 5baa5c0cc..3177ee075 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -786,7 +786,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { if apexType == imageApex && (compressionEnabled || a.testOnlyShouldForceCompression()) { a.isCompressed = true - unsignedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+".capex.unsigned") + unsignedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+imageCapexSuffix+".unsigned") compressRule := android.NewRuleBuilder(pctx, ctx) compressRule.Command(). @@ -800,7 +800,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { FlagWithOutput("--output ", unsignedCompressedOutputFile) compressRule.Build("compressRule", "Generate unsigned compressed APEX file") - signedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+".capex") + signedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+imageCapexSuffix) if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_SIGNAPK") { args["outCommaList"] = signedCompressedOutputFile.String() } diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 1bb0fb582..c4794dc89 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -924,8 +924,8 @@ func (a *ApexSet) ApexInfoMutator(mctx android.TopDownMutatorContext) { func (a *ApexSet) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.installFilename = a.InstallFilename() - if !strings.HasSuffix(a.installFilename, imageApexSuffix) { - ctx.ModuleErrorf("filename should end in %s for apex_set", imageApexSuffix) + if !strings.HasSuffix(a.installFilename, imageApexSuffix) && !strings.HasSuffix(a.installFilename, imageCapexSuffix) { + ctx.ModuleErrorf("filename should end in %s or %s for apex_set", imageApexSuffix, imageCapexSuffix) } inputApex := android.OptionalPathForModuleSrc(ctx, a.prebuiltCommonProperties.Selected_apex).Path() diff --git a/bp2build/Android.bp b/bp2build/Android.bp index b1ccc963c..5ee04f989 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -39,6 +39,8 @@ bootstrap_go_package { "cc_library_static_conversion_test.go", "cc_object_conversion_test.go", "conversion_test.go", + "filegroup_conversion_test.go", + "genrule_conversion_test.go", "performance_test.go", "prebuilt_etc_conversion_test.go", "python_binary_conversion_test.go", diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 0d9106c96..ecea6b2d5 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -16,7 +16,6 @@ package bp2build import ( "android/soong/android" - "android/soong/genrule" "strings" "testing" ) @@ -218,13 +217,9 @@ func TestGenerateSoongModuleTargets(t *testing.T) { } func TestGenerateBazelTargetModules(t *testing.T) { - testCases := []struct { - name string - bp string - expectedBazelTargets []string - }{ + testCases := []bp2buildTestCase{ { - bp: `custom { + blueprint: `custom { name: "foo", string_list_prop: ["a", "b"], string_prop: "a", @@ -241,7 +236,7 @@ func TestGenerateBazelTargetModules(t *testing.T) { }, }, { - bp: `custom { + blueprint: `custom { name: "control_characters", string_list_prop: ["\t", "\n"], string_prop: "a\t\n\r", @@ -258,7 +253,7 @@ func TestGenerateBazelTargetModules(t *testing.T) { }, }, { - bp: `custom { + blueprint: `custom { name: "has_dep", arch_paths: [":dep"], bazel_module: { bp2build_available: true }, @@ -280,7 +275,7 @@ custom { }, }, { - bp: `custom { + blueprint: `custom { name: "arch_paths", arch: { x86: { @@ -299,7 +294,7 @@ custom { }, }, { - bp: `custom { + blueprint: `custom { name: "has_dep", arch: { x86: { @@ -331,17 +326,17 @@ custom { dir := "." for _, testCase := range testCases { - config := android.TestConfig(buildDir, nil, testCase.bp, nil) + config := android.TestConfig(buildDir, nil, testCase.blueprint, nil) ctx := android.NewTestContext(config) registerCustomModuleForBp2buildConversion(ctx) _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) - if errored(t, "", errs) { + if errored(t, testCase, errs) { continue } _, errs = ctx.ResolveDependencies(config) - if errored(t, "", errs) { + if errored(t, testCase, errs) { continue } @@ -533,38 +528,13 @@ load("//build/bazel/rules:rules.bzl", "my_library")`, } func TestModuleTypeBp2Build(t *testing.T) { - otherGenruleBp := map[string]string{ - "other/Android.bp": `genrule { - name: "foo.tool", - out: ["foo_tool.out"], - srcs: ["foo_tool.in"], - cmd: "cp $(in) $(out)", -} -genrule { - name: "other.tool", - out: ["other_tool.out"], - srcs: ["other_tool.in"], - cmd: "cp $(in) $(out)", -}`, - } - - testCases := []struct { - description string - moduleTypeUnderTest string - moduleTypeUnderTestFactory android.ModuleFactory - moduleTypeUnderTestBp2BuildMutator func(android.TopDownMutatorContext) - preArchMutators []android.RegisterMutatorFunc - bp string - expectedBazelTargets []string - fs map[string]string - dir string - }{ + testCases := []bp2buildTestCase{ { description: "filegroup with does not specify srcs", moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", bazel_module: { bp2build_available: true }, }`, @@ -579,7 +549,7 @@ genrule { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", srcs: [], bazel_module: { bp2build_available: true }, @@ -595,7 +565,7 @@ genrule { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", srcs: ["a", "b"], bazel_module: { bp2build_available: true }, @@ -614,7 +584,7 @@ genrule { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", srcs: ["a", "b"], exclude_srcs: ["a"], @@ -631,7 +601,7 @@ genrule { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "foo", srcs: ["**/*.txt"], bazel_module: { bp2build_available: true }, @@ -645,7 +615,7 @@ genrule { ], )`, }, - fs: map[string]string{ + filesystem: map[string]string{ "other/a.txt": "", "other/b.txt": "", "other/subdir/a.txt": "", @@ -657,7 +627,7 @@ genrule { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "foo", srcs: ["a.txt"], bazel_module: { bp2build_available: true }, @@ -672,7 +642,7 @@ genrule { ], )`, }, - fs: map[string]string{ + filesystem: map[string]string{ "other/Android.bp": `filegroup { name: "fg_foo", srcs: ["**/*.txt"], @@ -689,7 +659,7 @@ genrule { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "foobar", srcs: [ ":foo", @@ -705,207 +675,13 @@ genrule { ], )`, }, - fs: map[string]string{ + filesystem: map[string]string{ "other/Android.bp": `filegroup { name: "foo", srcs: ["a", "b"], }`, }, }, - { - description: "genrule with command line variable replacements", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo.tool", - out: ["foo_tool.out"], - srcs: ["foo_tool.in"], - cmd: "cp $(in) $(out)", - bazel_module: { bp2build_available: true }, -} - -genrule { - name: "foo", - out: ["foo.out"], - srcs: ["foo.in"], - tools: [":foo.tool"], - cmd: "$(location :foo.tool) --genDir=$(genDir) arg $(in) $(out)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{ - `genrule( - name = "foo", - cmd = "$(location :foo.tool) --genDir=$(GENDIR) arg $(SRCS) $(OUTS)", - outs = ["foo.out"], - srcs = ["foo.in"], - tools = [":foo.tool"], -)`, - `genrule( - name = "foo.tool", - cmd = "cp $(SRCS) $(OUTS)", - outs = ["foo_tool.out"], - srcs = ["foo_tool.in"], -)`, - }, - }, - { - description: "genrule using $(locations :label)", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo.tools", - out: ["foo_tool.out", "foo_tool2.out"], - srcs: ["foo_tool.in"], - cmd: "cp $(in) $(out)", - bazel_module: { bp2build_available: true }, -} - -genrule { - name: "foo", - out: ["foo.out"], - srcs: ["foo.in"], - tools: [":foo.tools"], - cmd: "$(locations :foo.tools) -s $(out) $(in)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{`genrule( - name = "foo", - cmd = "$(locations :foo.tools) -s $(OUTS) $(SRCS)", - outs = ["foo.out"], - srcs = ["foo.in"], - tools = [":foo.tools"], -)`, - `genrule( - name = "foo.tools", - cmd = "cp $(SRCS) $(OUTS)", - outs = [ - "foo_tool.out", - "foo_tool2.out", - ], - srcs = ["foo_tool.in"], -)`, - }, - }, - { - description: "genrule using $(locations //absolute:label)", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo", - out: ["foo.out"], - srcs: ["foo.in"], - tool_files: [":foo.tool"], - cmd: "$(locations :foo.tool) -s $(out) $(in)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{`genrule( - name = "foo", - cmd = "$(locations //other:foo.tool) -s $(OUTS) $(SRCS)", - outs = ["foo.out"], - srcs = ["foo.in"], - tools = ["//other:foo.tool"], -)`, - }, - fs: otherGenruleBp, - }, - { - description: "genrule srcs using $(locations //absolute:label)", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo", - out: ["foo.out"], - srcs: [":other.tool"], - tool_files: [":foo.tool"], - cmd: "$(locations :foo.tool) -s $(out) $(location :other.tool)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{`genrule( - name = "foo", - cmd = "$(locations //other:foo.tool) -s $(OUTS) $(location //other:other.tool)", - outs = ["foo.out"], - srcs = ["//other:other.tool"], - tools = ["//other:foo.tool"], -)`, - }, - fs: otherGenruleBp, - }, - { - description: "genrule using $(location) label should substitute first tool label automatically", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo", - out: ["foo.out"], - srcs: ["foo.in"], - tool_files: [":foo.tool", ":other.tool"], - cmd: "$(location) -s $(out) $(in)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{`genrule( - name = "foo", - cmd = "$(location //other:foo.tool) -s $(OUTS) $(SRCS)", - outs = ["foo.out"], - srcs = ["foo.in"], - tools = [ - "//other:foo.tool", - "//other:other.tool", - ], -)`, - }, - fs: otherGenruleBp, - }, - { - description: "genrule using $(locations) label should substitute first tool label automatically", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo", - out: ["foo.out"], - srcs: ["foo.in"], - tools: [":foo.tool", ":other.tool"], - cmd: "$(locations) -s $(out) $(in)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{`genrule( - name = "foo", - cmd = "$(locations //other:foo.tool) -s $(OUTS) $(SRCS)", - outs = ["foo.out"], - srcs = ["foo.in"], - tools = [ - "//other:foo.tool", - "//other:other.tool", - ], -)`, - }, - fs: otherGenruleBp, - }, - { - description: "genrule without tools or tool_files can convert successfully", - moduleTypeUnderTest: "genrule", - moduleTypeUnderTestFactory: genrule.GenRuleFactory, - moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, - bp: `genrule { - name: "foo", - out: ["foo.out"], - srcs: ["foo.in"], - cmd: "cp $(in) $(out)", - bazel_module: { bp2build_available: true }, -}`, - expectedBazelTargets: []string{`genrule( - name = "foo", - cmd = "cp $(SRCS) $(OUTS)", - outs = ["foo.out"], - srcs = ["foo.in"], -)`, - }, - }, } dir := "." @@ -914,24 +690,24 @@ genrule { toParse := []string{ "Android.bp", } - for f, content := range testCase.fs { + for f, content := range testCase.filesystem { if strings.HasSuffix(f, "Android.bp") { toParse = append(toParse, f) } fs[f] = []byte(content) } - config := android.TestConfig(buildDir, nil, testCase.bp, fs) + config := android.TestConfig(buildDir, nil, testCase.blueprint, fs) ctx := android.NewTestContext(config) ctx.RegisterModuleType(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestFactory) ctx.RegisterBp2BuildMutator(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestBp2BuildMutator) ctx.RegisterForBazelConversion() _, errs := ctx.ParseFileList(dir, toParse) - if errored(t, testCase.description, errs) { + if errored(t, testCase, errs) { continue } _, errs = ctx.ResolveDependencies(config) - if errored(t, testCase.description, errs) { + if errored(t, testCase, errs) { continue } @@ -961,199 +737,6 @@ genrule { type bp2buildMutator = func(android.TopDownMutatorContext) -func TestBp2BuildInlinesDefaults(t *testing.T) { - testCases := []struct { - moduleTypesUnderTest map[string]android.ModuleFactory - bp2buildMutatorsUnderTest map[string]bp2buildMutator - bp string - expectedBazelTarget string - description string - }{ - { - moduleTypesUnderTest: map[string]android.ModuleFactory{ - "genrule": genrule.GenRuleFactory, - "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, - }, - bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ - "genrule": genrule.GenruleBp2Build, - }, - bp: `genrule_defaults { - name: "gen_defaults", - cmd: "do-something $(in) $(out)", -} -genrule { - name: "gen", - out: ["out"], - srcs: ["in1"], - defaults: ["gen_defaults"], - bazel_module: { bp2build_available: true }, -} -`, - expectedBazelTarget: `genrule( - name = "gen", - cmd = "do-something $(SRCS) $(OUTS)", - outs = ["out"], - srcs = ["in1"], -)`, - description: "genrule applies properties from a genrule_defaults dependency if not specified", - }, - { - moduleTypesUnderTest: map[string]android.ModuleFactory{ - "genrule": genrule.GenRuleFactory, - "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, - }, - bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ - "genrule": genrule.GenruleBp2Build, - }, - bp: `genrule_defaults { - name: "gen_defaults", - out: ["out-from-defaults"], - srcs: ["in-from-defaults"], - cmd: "cmd-from-defaults", -} -genrule { - name: "gen", - out: ["out"], - srcs: ["in1"], - defaults: ["gen_defaults"], - cmd: "do-something $(in) $(out)", - bazel_module: { bp2build_available: true }, -} -`, - expectedBazelTarget: `genrule( - name = "gen", - cmd = "do-something $(SRCS) $(OUTS)", - outs = [ - "out-from-defaults", - "out", - ], - srcs = [ - "in-from-defaults", - "in1", - ], -)`, - description: "genrule does merges properties from a genrule_defaults dependency, latest-first", - }, - { - moduleTypesUnderTest: map[string]android.ModuleFactory{ - "genrule": genrule.GenRuleFactory, - "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, - }, - bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ - "genrule": genrule.GenruleBp2Build, - }, - bp: `genrule_defaults { - name: "gen_defaults1", - cmd: "cp $(in) $(out)", -} - -genrule_defaults { - name: "gen_defaults2", - srcs: ["in1"], -} - -genrule { - name: "gen", - out: ["out"], - defaults: ["gen_defaults1", "gen_defaults2"], - bazel_module: { bp2build_available: true }, -} -`, - expectedBazelTarget: `genrule( - name = "gen", - cmd = "cp $(SRCS) $(OUTS)", - outs = ["out"], - srcs = ["in1"], -)`, - description: "genrule applies properties from list of genrule_defaults", - }, - { - moduleTypesUnderTest: map[string]android.ModuleFactory{ - "genrule": genrule.GenRuleFactory, - "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, - }, - bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ - "genrule": genrule.GenruleBp2Build, - }, - bp: `genrule_defaults { - name: "gen_defaults1", - defaults: ["gen_defaults2"], - cmd: "cmd1 $(in) $(out)", // overrides gen_defaults2's cmd property value. -} - -genrule_defaults { - name: "gen_defaults2", - defaults: ["gen_defaults3"], - cmd: "cmd2 $(in) $(out)", - out: ["out-from-2"], - srcs: ["in1"], -} - -genrule_defaults { - name: "gen_defaults3", - out: ["out-from-3"], - srcs: ["srcs-from-3"], -} - -genrule { - name: "gen", - out: ["out"], - defaults: ["gen_defaults1"], - bazel_module: { bp2build_available: true }, -} -`, - expectedBazelTarget: `genrule( - name = "gen", - cmd = "cmd1 $(SRCS) $(OUTS)", - outs = [ - "out-from-3", - "out-from-2", - "out", - ], - srcs = [ - "srcs-from-3", - "in1", - ], -)`, - description: "genrule applies properties from genrule_defaults transitively", - }, - } - - dir := "." - for _, testCase := range testCases { - config := android.TestConfig(buildDir, nil, testCase.bp, nil) - ctx := android.NewTestContext(config) - for m, factory := range testCase.moduleTypesUnderTest { - ctx.RegisterModuleType(m, factory) - } - for mutator, f := range testCase.bp2buildMutatorsUnderTest { - ctx.RegisterBp2BuildMutator(mutator, f) - } - ctx.RegisterForBazelConversion() - - _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) - android.FailIfErrored(t, errs) - _, errs = ctx.ResolveDependencies(config) - android.FailIfErrored(t, errs) - - codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build) - bazelTargets := generateBazelTargetsForDir(codegenCtx, dir) - if actualCount := len(bazelTargets); actualCount != 1 { - t.Fatalf("%s: Expected 1 bazel target, got %d", testCase.description, actualCount) - } - - actualBazelTarget := bazelTargets[0] - if actualBazelTarget.content != testCase.expectedBazelTarget { - t.Errorf( - "%s: Expected generated Bazel target to be '%s', got '%s'", - testCase.description, - testCase.expectedBazelTarget, - actualBazelTarget.content, - ) - } - } -} - func TestAllowlistingBp2buildTargetsExplicitly(t *testing.T) { testCases := []struct { moduleTypeUnderTest string @@ -1353,30 +936,20 @@ filegroup { name: "opt-out-h", bazel_module: { bp2build_available: false } } } func TestCombineBuildFilesBp2buildTargets(t *testing.T) { - testCases := []struct { - description string - moduleTypeUnderTest string - moduleTypeUnderTestFactory android.ModuleFactory - moduleTypeUnderTestBp2BuildMutator func(android.TopDownMutatorContext) - preArchMutators []android.RegisterMutatorFunc - bp string - expectedBazelTargets []string - fs map[string]string - dir string - }{ + testCases := []bp2buildTestCase{ { description: "filegroup bazel_module.label", moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", bazel_module: { label: "//other:fg_foo" }, }`, expectedBazelTargets: []string{ `// BUILD file`, }, - fs: map[string]string{ + filesystem: map[string]string{ "other/BUILD.bazel": `// BUILD file`, }, }, @@ -1385,7 +958,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", bazel_module: { label: "//other:fg_foo" }, } @@ -1397,7 +970,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { expectedBazelTargets: []string{ `// BUILD file`, }, - fs: map[string]string{ + filesystem: map[string]string{ "other/BUILD.bazel": `// BUILD file`, }, }, @@ -1407,8 +980,8 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, dir: "other", - bp: ``, - fs: map[string]string{ + blueprint: ``, + filesystem: map[string]string{ "other/Android.bp": `filegroup { name: "fg_foo", bazel_module: { @@ -1434,7 +1007,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", bazel_module: { label: "//other:fg_foo", @@ -1453,7 +1026,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { )`, `// BUILD file`, }, - fs: map[string]string{ + filesystem: map[string]string{ "other/BUILD.bazel": `// BUILD file`, }, }, @@ -1466,24 +1039,24 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { toParse := []string{ "Android.bp", } - for f, content := range testCase.fs { + for f, content := range testCase.filesystem { if strings.HasSuffix(f, "Android.bp") { toParse = append(toParse, f) } fs[f] = []byte(content) } - config := android.TestConfig(buildDir, nil, testCase.bp, fs) + config := android.TestConfig(buildDir, nil, testCase.blueprint, fs) ctx := android.NewTestContext(config) ctx.RegisterModuleType(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestFactory) ctx.RegisterBp2BuildMutator(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestBp2BuildMutator) ctx.RegisterForBazelConversion() _, errs := ctx.ParseFileList(dir, toParse) - if errored(t, testCase.description, errs) { + if errored(t, testCase, errs) { return } _, errs = ctx.ResolveDependencies(config) - if errored(t, testCase.description, errs) { + if errored(t, testCase, errs) { return } @@ -1517,22 +1090,13 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { } func TestGlobExcludeSrcs(t *testing.T) { - testCases := []struct { - description string - moduleTypeUnderTest string - moduleTypeUnderTestFactory android.ModuleFactory - moduleTypeUnderTestBp2BuildMutator func(android.TopDownMutatorContext) - bp string - expectedBazelTargets []string - fs map[string]string - dir string - }{ + testCases := []bp2buildTestCase{ { description: "filegroup top level exclude_srcs", moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: `filegroup { + blueprint: `filegroup { name: "fg_foo", srcs: ["**/*.txt"], exclude_srcs: ["c.txt"], @@ -1548,7 +1112,7 @@ func TestGlobExcludeSrcs(t *testing.T) { ], )`, }, - fs: map[string]string{ + filesystem: map[string]string{ "a.txt": "", "b.txt": "", "c.txt": "", @@ -1562,9 +1126,9 @@ func TestGlobExcludeSrcs(t *testing.T) { moduleTypeUnderTest: "filegroup", moduleTypeUnderTestFactory: android.FileGroupFactory, moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, - bp: "", + blueprint: "", dir: "dir", - fs: map[string]string{ + filesystem: map[string]string{ "dir/Android.bp": `filegroup { name: "fg_foo", srcs: ["**/*.txt"], @@ -1596,24 +1160,24 @@ func TestGlobExcludeSrcs(t *testing.T) { toParse := []string{ "Android.bp", } - for f, content := range testCase.fs { + for f, content := range testCase.filesystem { if strings.HasSuffix(f, "Android.bp") { toParse = append(toParse, f) } fs[f] = []byte(content) } - config := android.TestConfig(buildDir, nil, testCase.bp, fs) + config := android.TestConfig(buildDir, nil, testCase.blueprint, fs) ctx := android.NewTestContext(config) ctx.RegisterModuleType(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestFactory) ctx.RegisterBp2BuildMutator(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestBp2BuildMutator) ctx.RegisterForBazelConversion() _, errs := ctx.ParseFileList(dir, toParse) - if errored(t, testCase.description, errs) { + if errored(t, testCase, errs) { continue } _, errs = ctx.ResolveDependencies(config) - if errored(t, testCase.description, errs) { + if errored(t, testCase, errs) { continue } diff --git a/bp2build/filegroup_conversion_test.go b/bp2build/filegroup_conversion_test.go new file mode 100644 index 000000000..ad9923610 --- /dev/null +++ b/bp2build/filegroup_conversion_test.go @@ -0,0 +1,62 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "fmt" + + "testing" +) + +func runFilegroupTestCase(t *testing.T, tc bp2buildTestCase) { + t.Helper() + runBp2BuildTestCase(t, registerFilegroupModuleTypes, tc) +} + +func registerFilegroupModuleTypes(ctx android.RegistrationContext) {} + +func TestFilegroupSameNameAsFile_OneFile(t *testing.T) { + runFilegroupTestCase(t, bp2buildTestCase{ + description: "filegroup - same name as file, with one file", + moduleTypeUnderTest: "filegroup", + moduleTypeUnderTestFactory: android.FileGroupFactory, + moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, + filesystem: map[string]string{}, + blueprint: ` +filegroup { + name: "foo", + srcs: ["foo"], +} +`, + expectedBazelTargets: []string{}}) +} + +func TestFilegroupSameNameAsFile_MultipleFiles(t *testing.T) { + runFilegroupTestCase(t, bp2buildTestCase{ + description: "filegroup - same name as file, with multiple files", + moduleTypeUnderTest: "filegroup", + moduleTypeUnderTestFactory: android.FileGroupFactory, + moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, + filesystem: map[string]string{}, + blueprint: ` +filegroup { + name: "foo", + srcs: ["foo", "bar"], +} +`, + expectedErr: fmt.Errorf("filegroup 'foo' cannot contain a file with the same name"), + }) +} diff --git a/bp2build/genrule_conversion_test.go b/bp2build/genrule_conversion_test.go new file mode 100644 index 000000000..a9911808e --- /dev/null +++ b/bp2build/genrule_conversion_test.go @@ -0,0 +1,479 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "android/soong/genrule" + "strings" + "testing" +) + +func TestGenruleBp2Build(t *testing.T) { + otherGenruleBp := map[string]string{ + "other/Android.bp": `genrule { + name: "foo.tool", + out: ["foo_tool.out"], + srcs: ["foo_tool.in"], + cmd: "cp $(in) $(out)", +} +genrule { + name: "other.tool", + out: ["other_tool.out"], + srcs: ["other_tool.in"], + cmd: "cp $(in) $(out)", +}`, + } + + testCases := []bp2buildTestCase{ + { + description: "genrule with command line variable replacements", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo.tool", + out: ["foo_tool.out"], + srcs: ["foo_tool.in"], + cmd: "cp $(in) $(out)", + bazel_module: { bp2build_available: true }, +} + +genrule { + name: "foo", + out: ["foo.out"], + srcs: ["foo.in"], + tools: [":foo.tool"], + cmd: "$(location :foo.tool) --genDir=$(genDir) arg $(in) $(out)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + `genrule( + name = "foo", + cmd = "$(location :foo.tool) --genDir=$(GENDIR) arg $(SRCS) $(OUTS)", + outs = ["foo.out"], + srcs = ["foo.in"], + tools = [":foo.tool"], +)`, + `genrule( + name = "foo.tool", + cmd = "cp $(SRCS) $(OUTS)", + outs = ["foo_tool.out"], + srcs = ["foo_tool.in"], +)`, + }, + }, + { + description: "genrule using $(locations :label)", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo.tools", + out: ["foo_tool.out", "foo_tool2.out"], + srcs: ["foo_tool.in"], + cmd: "cp $(in) $(out)", + bazel_module: { bp2build_available: true }, +} + +genrule { + name: "foo", + out: ["foo.out"], + srcs: ["foo.in"], + tools: [":foo.tools"], + cmd: "$(locations :foo.tools) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`genrule( + name = "foo", + cmd = "$(locations :foo.tools) -s $(OUTS) $(SRCS)", + outs = ["foo.out"], + srcs = ["foo.in"], + tools = [":foo.tools"], +)`, + `genrule( + name = "foo.tools", + cmd = "cp $(SRCS) $(OUTS)", + outs = [ + "foo_tool.out", + "foo_tool2.out", + ], + srcs = ["foo_tool.in"], +)`, + }, + }, + { + description: "genrule using $(locations //absolute:label)", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo", + out: ["foo.out"], + srcs: ["foo.in"], + tool_files: [":foo.tool"], + cmd: "$(locations :foo.tool) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`genrule( + name = "foo", + cmd = "$(locations //other:foo.tool) -s $(OUTS) $(SRCS)", + outs = ["foo.out"], + srcs = ["foo.in"], + tools = ["//other:foo.tool"], +)`, + }, + filesystem: otherGenruleBp, + }, + { + description: "genrule srcs using $(locations //absolute:label)", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo", + out: ["foo.out"], + srcs: [":other.tool"], + tool_files: [":foo.tool"], + cmd: "$(locations :foo.tool) -s $(out) $(location :other.tool)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`genrule( + name = "foo", + cmd = "$(locations //other:foo.tool) -s $(OUTS) $(location //other:other.tool)", + outs = ["foo.out"], + srcs = ["//other:other.tool"], + tools = ["//other:foo.tool"], +)`, + }, + filesystem: otherGenruleBp, + }, + { + description: "genrule using $(location) label should substitute first tool label automatically", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo", + out: ["foo.out"], + srcs: ["foo.in"], + tool_files: [":foo.tool", ":other.tool"], + cmd: "$(location) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`genrule( + name = "foo", + cmd = "$(location //other:foo.tool) -s $(OUTS) $(SRCS)", + outs = ["foo.out"], + srcs = ["foo.in"], + tools = [ + "//other:foo.tool", + "//other:other.tool", + ], +)`, + }, + filesystem: otherGenruleBp, + }, + { + description: "genrule using $(locations) label should substitute first tool label automatically", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo", + out: ["foo.out"], + srcs: ["foo.in"], + tools: [":foo.tool", ":other.tool"], + cmd: "$(locations) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`genrule( + name = "foo", + cmd = "$(locations //other:foo.tool) -s $(OUTS) $(SRCS)", + outs = ["foo.out"], + srcs = ["foo.in"], + tools = [ + "//other:foo.tool", + "//other:other.tool", + ], +)`, + }, + filesystem: otherGenruleBp, + }, + { + description: "genrule without tools or tool_files can convert successfully", + moduleTypeUnderTest: "genrule", + moduleTypeUnderTestFactory: genrule.GenRuleFactory, + moduleTypeUnderTestBp2BuildMutator: genrule.GenruleBp2Build, + blueprint: `genrule { + name: "foo", + out: ["foo.out"], + srcs: ["foo.in"], + cmd: "cp $(in) $(out)", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`genrule( + name = "foo", + cmd = "cp $(SRCS) $(OUTS)", + outs = ["foo.out"], + srcs = ["foo.in"], +)`, + }, + }, + } + + dir := "." + for _, testCase := range testCases { + fs := make(map[string][]byte) + toParse := []string{ + "Android.bp", + } + for f, content := range testCase.filesystem { + if strings.HasSuffix(f, "Android.bp") { + toParse = append(toParse, f) + } + fs[f] = []byte(content) + } + config := android.TestConfig(buildDir, nil, testCase.blueprint, fs) + ctx := android.NewTestContext(config) + ctx.RegisterModuleType(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestFactory) + ctx.RegisterBp2BuildMutator(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestBp2BuildMutator) + ctx.RegisterForBazelConversion() + + _, errs := ctx.ParseFileList(dir, toParse) + if errored(t, testCase, errs) { + continue + } + _, errs = ctx.ResolveDependencies(config) + if errored(t, testCase, errs) { + continue + } + + checkDir := dir + if testCase.dir != "" { + checkDir = testCase.dir + } + + codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build) + bazelTargets := generateBazelTargetsForDir(codegenCtx, checkDir) + if actualCount, expectedCount := len(bazelTargets), len(testCase.expectedBazelTargets); actualCount != expectedCount { + t.Errorf("%s: Expected %d bazel target, got %d", testCase.description, expectedCount, actualCount) + } else { + for i, target := range bazelTargets { + if w, g := testCase.expectedBazelTargets[i], target.content; w != g { + t.Errorf( + "%s: Expected generated Bazel target to be '%s', got '%s'", + testCase.description, + w, + g, + ) + } + } + } + } +} + +func TestBp2BuildInlinesDefaults(t *testing.T) { + testCases := []struct { + moduleTypesUnderTest map[string]android.ModuleFactory + bp2buildMutatorsUnderTest map[string]bp2buildMutator + bp string + expectedBazelTarget string + description string + }{ + { + moduleTypesUnderTest: map[string]android.ModuleFactory{ + "genrule": genrule.GenRuleFactory, + "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, + }, + bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ + "genrule": genrule.GenruleBp2Build, + }, + bp: `genrule_defaults { + name: "gen_defaults", + cmd: "do-something $(in) $(out)", +} +genrule { + name: "gen", + out: ["out"], + srcs: ["in1"], + defaults: ["gen_defaults"], + bazel_module: { bp2build_available: true }, +} +`, + expectedBazelTarget: `genrule( + name = "gen", + cmd = "do-something $(SRCS) $(OUTS)", + outs = ["out"], + srcs = ["in1"], +)`, + description: "genrule applies properties from a genrule_defaults dependency if not specified", + }, + { + moduleTypesUnderTest: map[string]android.ModuleFactory{ + "genrule": genrule.GenRuleFactory, + "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, + }, + bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ + "genrule": genrule.GenruleBp2Build, + }, + bp: `genrule_defaults { + name: "gen_defaults", + out: ["out-from-defaults"], + srcs: ["in-from-defaults"], + cmd: "cmd-from-defaults", +} +genrule { + name: "gen", + out: ["out"], + srcs: ["in1"], + defaults: ["gen_defaults"], + cmd: "do-something $(in) $(out)", + bazel_module: { bp2build_available: true }, +} +`, + expectedBazelTarget: `genrule( + name = "gen", + cmd = "do-something $(SRCS) $(OUTS)", + outs = [ + "out-from-defaults", + "out", + ], + srcs = [ + "in-from-defaults", + "in1", + ], +)`, + description: "genrule does merges properties from a genrule_defaults dependency, latest-first", + }, + { + moduleTypesUnderTest: map[string]android.ModuleFactory{ + "genrule": genrule.GenRuleFactory, + "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, + }, + bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ + "genrule": genrule.GenruleBp2Build, + }, + bp: `genrule_defaults { + name: "gen_defaults1", + cmd: "cp $(in) $(out)", +} + +genrule_defaults { + name: "gen_defaults2", + srcs: ["in1"], +} + +genrule { + name: "gen", + out: ["out"], + defaults: ["gen_defaults1", "gen_defaults2"], + bazel_module: { bp2build_available: true }, +} +`, + expectedBazelTarget: `genrule( + name = "gen", + cmd = "cp $(SRCS) $(OUTS)", + outs = ["out"], + srcs = ["in1"], +)`, + description: "genrule applies properties from list of genrule_defaults", + }, + { + moduleTypesUnderTest: map[string]android.ModuleFactory{ + "genrule": genrule.GenRuleFactory, + "genrule_defaults": func() android.Module { return genrule.DefaultsFactory() }, + }, + bp2buildMutatorsUnderTest: map[string]bp2buildMutator{ + "genrule": genrule.GenruleBp2Build, + }, + bp: `genrule_defaults { + name: "gen_defaults1", + defaults: ["gen_defaults2"], + cmd: "cmd1 $(in) $(out)", // overrides gen_defaults2's cmd property value. +} + +genrule_defaults { + name: "gen_defaults2", + defaults: ["gen_defaults3"], + cmd: "cmd2 $(in) $(out)", + out: ["out-from-2"], + srcs: ["in1"], +} + +genrule_defaults { + name: "gen_defaults3", + out: ["out-from-3"], + srcs: ["srcs-from-3"], +} + +genrule { + name: "gen", + out: ["out"], + defaults: ["gen_defaults1"], + bazel_module: { bp2build_available: true }, +} +`, + expectedBazelTarget: `genrule( + name = "gen", + cmd = "cmd1 $(SRCS) $(OUTS)", + outs = [ + "out-from-3", + "out-from-2", + "out", + ], + srcs = [ + "srcs-from-3", + "in1", + ], +)`, + description: "genrule applies properties from genrule_defaults transitively", + }, + } + + dir := "." + for _, testCase := range testCases { + config := android.TestConfig(buildDir, nil, testCase.bp, nil) + ctx := android.NewTestContext(config) + for m, factory := range testCase.moduleTypesUnderTest { + ctx.RegisterModuleType(m, factory) + } + for mutator, f := range testCase.bp2buildMutatorsUnderTest { + ctx.RegisterBp2BuildMutator(mutator, f) + } + ctx.RegisterForBazelConversion() + + _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.ResolveDependencies(config) + android.FailIfErrored(t, errs) + + codegenCtx := NewCodegenContext(config, *ctx.Context, Bp2Build) + bazelTargets := generateBazelTargetsForDir(codegenCtx, dir) + if actualCount := len(bazelTargets); actualCount != 1 { + t.Fatalf("%s: Expected 1 bazel target, got %d", testCase.description, actualCount) + } + + actualBazelTarget := bazelTargets[0] + if actualBazelTarget.content != testCase.expectedBazelTarget { + t.Errorf( + "%s: Expected generated Bazel target to be '%s', got '%s'", + testCase.description, + testCase.expectedBazelTarget, + actualBazelTarget.content, + ) + } + } +} diff --git a/bp2build/testing.go b/bp2build/testing.go index a549a9369..3ebe63d5a 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -36,14 +36,35 @@ var ( buildDir string ) -func errored(t *testing.T, desc string, errs []error) bool { +func checkError(t *testing.T, errs []error, expectedErr error) bool { t.Helper() + + // expectedErr is not nil, find it in the list of errors + if len(errs) != 1 { + t.Errorf("Expected only 1 error, got %d: %q", len(errs), errs) + } + if errs[0].Error() == expectedErr.Error() { + return true + } + + return false +} + +func errored(t *testing.T, tc bp2buildTestCase, errs []error) bool { + t.Helper() + if tc.expectedErr != nil { + // Rely on checkErrors, as this test case is expected to have an error. + return false + } + if len(errs) > 0 { for _, err := range errs { - t.Errorf("%s: %s", desc, err) + t.Errorf("%s: %s", tc.description, err) } return true } + + // All good, continue execution. return false } @@ -61,6 +82,7 @@ type bp2buildTestCase struct { expectedBazelTargets []string filesystem map[string]string dir string + expectedErr error } func runBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.RegistrationContext), tc bp2buildTestCase) { @@ -85,12 +107,17 @@ func runBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.Regi ctx.RegisterBp2BuildMutator(tc.moduleTypeUnderTest, tc.moduleTypeUnderTestBp2BuildMutator) ctx.RegisterForBazelConversion() - _, errs := ctx.ParseFileList(dir, toParse) - if errored(t, tc.description, errs) { + _, parseErrs := ctx.ParseFileList(dir, toParse) + if errored(t, tc, parseErrs) { + return + } + _, resolveDepsErrs := ctx.ResolveDependencies(config) + if errored(t, tc, resolveDepsErrs) { return } - _, errs = ctx.ResolveDependencies(config) - if errored(t, tc.description, errs) { + + errs := append(parseErrs, resolveDepsErrs...) + if tc.expectedErr != nil && checkError(t, errs, tc.expectedErr) { return } @@ -225,11 +252,6 @@ type customBazelModuleAttributes struct { Arch_paths bazel.LabelListAttribute } -type customBazelModule struct { - android.BazelTargetModuleBase - customBazelModuleAttributes -} - func customBp2BuildMutator(ctx android.TopDownMutatorContext) { if m, ok := ctx.Module().(*customModule); ok { if !m.ConvertWithBp2build(ctx) { diff --git a/bpfix/cmd_lib/bpfix.go b/bpfix/cmd_lib/bpfix.go index f90f65be6..1106d4af7 100644 --- a/bpfix/cmd_lib/bpfix.go +++ b/bpfix/cmd_lib/bpfix.go @@ -114,7 +114,7 @@ func processFile(filename string, in io.Reader, out io.Writer, fixRequest bpfix. func makeFileVisitor(fixRequest bpfix.FixRequest) func(string, os.FileInfo, error) error { return func(path string, f os.FileInfo, err error) error { - if err == nil && (f.Name() == "Blueprints" || f.Name() == "Android.bp") { + if err == nil && f.Name() == "Android.bp" { err = openAndProcess(path, os.Stdout, fixRequest) } if err != nil { diff --git a/cc/builder.go b/cc/builder.go index d0527cb44..6a4d94040 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -198,19 +198,22 @@ var ( // Rule for invoking clang-tidy (a clang-based linter). clangTidy, clangTidyRE = pctx.RemoteStaticRules("clangTidy", blueprint.RuleParams{ - Command: "rm -f $out && $reTemplate${config.ClangBin}/clang-tidy $tidyFlags $in -- $cFlags && touch $out", + Command: "rm -f $out && $tidyVars $reTemplate${config.ClangBin}/clang-tidy $tidyFlags $in -- $cFlags && touch $out", CommandDeps: []string{"${config.ClangBin}/clang-tidy"}, }, &remoteexec.REParams{ - Labels: map[string]string{"type": "lint", "tool": "clang-tidy", "lang": "cpp"}, - ExecStrategy: "${config.REClangTidyExecStrategy}", - Inputs: []string{"$in"}, - // OutputFile here is $in for remote-execution since its possible that - // clang-tidy modifies the given input file itself and $out refers to the - // ".tidy" file generated for ninja-dependency reasons. - OutputFiles: []string{"$in"}, - Platform: map[string]string{remoteexec.PoolKey: "${config.REClangTidyPool}"}, - }, []string{"cFlags", "tidyFlags"}, []string{}) + Labels: map[string]string{"type": "lint", "tool": "clang-tidy", "lang": "cpp"}, + ExecStrategy: "${config.REClangTidyExecStrategy}", + Inputs: []string{"$in"}, + EnvironmentVariables: []string{"TIDY_TIMEOUT"}, + // Although clang-tidy has an option to "fix" source files, that feature is hardly useable + // under parallel compilation and RBE. So we assume no OutputFiles here. + // The clang-tidy fix option is best run locally in single thread. + // Copying source file back to local caused two problems: + // (1) New timestamps trigger clang and clang-tidy compilations again. + // (2) Changing source files caused concurrent clang or clang-tidy jobs to crash. + Platform: map[string]string{remoteexec.PoolKey: "${config.REClangTidyPool}"}, + }, []string{"cFlags", "tidyFlags", "tidyVars"}, []string{}) _ = pctx.SourcePathVariable("yasmCmd", "prebuilts/misc/${config.HostPrebuiltTag}/yasm/yasm") @@ -440,8 +443,13 @@ func transformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and // Source files are one-to-one with tidy, coverage, or kythe files, if enabled. objFiles := make(android.Paths, len(srcFiles)) var tidyFiles android.Paths + var tidyVars string if flags.tidy { tidyFiles = make(android.Paths, 0, len(srcFiles)) + tidyTimeout := ctx.Config().Getenv("TIDY_TIMEOUT") + if len(tidyTimeout) > 0 { + tidyVars += "TIDY_TIMEOUT=" + tidyTimeout + } } var coverageFiles android.Paths if flags.gcovCoverage { @@ -631,6 +639,7 @@ func transformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and rule = clangTidyRE } + ctx.TidyFile(tidyFile) ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "clang-tidy " + srcFile.Rel(), @@ -644,6 +653,7 @@ func transformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and Args: map[string]string{ "cFlags": moduleToolingFlags, "tidyFlags": config.TidyFlagsForSrcFile(srcFile, flags.tidyFlags), + "tidyVars": tidyVars, }, }) } diff --git a/cc/config/OWNERS b/cc/config/OWNERS index 701db92d7..580f215c7 100644 --- a/cc/config/OWNERS +++ b/cc/config/OWNERS @@ -1,3 +1,3 @@ per-file vndk.go = smoreland@google.com, victoryang@google.com -per-file clang.go,global.go = srhines@google.com, chh@google.com, pirama@google.com, yikong@google.com +per-file clang.go,global.go,tidy.go = srhines@google.com, chh@google.com, pirama@google.com, yikong@google.com diff --git a/cc/config/global.go b/cc/config/global.go index 977334522..5f41f9e06 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -230,6 +230,9 @@ var ( "-Wno-string-concatenation", // http://b/175068488 // New warnings to be fixed after clang-r428724 "-Wno-align-mismatch", // http://b/193679946 + // New warnings to be fixed after clang-r433403 + "-Wno-error=unused-but-set-variable", // http://b/197240255 + "-Wno-error=unused-but-set-parameter", // http://b/197240255 } // Extra cflags for external third-party projects to disable warnings that @@ -255,6 +258,9 @@ var ( // http://b/165945989 "-Wno-psabi", + + // http://b/199369603 + "-Wno-null-pointer-subtraction", } IllegalFlags = []string{ @@ -268,8 +274,8 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r428724" - ClangDefaultShortVersion = "13.0.1" + ClangDefaultVersion = "clang-r433403" + ClangDefaultShortVersion = "13.0.2" // Directories with warnings from Android.bp files. WarningAllowedProjects = []string{ diff --git a/cc/config/tidy.go b/cc/config/tidy.go index cf1350381..86825029e 100644 --- a/cc/config/tidy.go +++ b/cc/config/tidy.go @@ -115,6 +115,7 @@ var DefaultLocalTidyChecks = []PathBasedTidyCheck{ {"external/", tidyExternalVendor}, {"external/google", tidyDefault}, {"external/webrtc", tidyDefault}, + {"external/googletest/", tidyExternalVendor}, {"frameworks/compile/mclinker/", tidyExternalVendor}, {"hardware/qcom", tidyExternalVendor}, {"vendor/", tidyExternalVendor}, @@ -133,6 +134,7 @@ func reverseTidyChecks(in []PathBasedTidyCheck) []PathBasedTidyCheck { } func TidyChecksForDir(dir string) string { + dir = dir + "/" for _, pathCheck := range reversedDefaultLocalTidyChecks { if strings.HasPrefix(dir, pathCheck.PathPrefix) { return pathCheck.Checks diff --git a/cc/config/vndk.go b/cc/config/vndk.go index 24e8fa4e8..8c678a129 100644 --- a/cc/config/vndk.go +++ b/cc/config/vndk.go @@ -53,6 +53,8 @@ var VndkMustUseVendorVariantList = []string{ "android.hardware.power.stats-V1-ndk_platform", "android.hardware.power.stats-ndk_platform", "android.hardware.power.stats-unstable-ndk_platform", + "android.hardware.radio-V1-ndk", + "android.hardware.radio-V1-ndk_platform", "android.hardware.rebootescrow-ndk_platform", "android.hardware.security.keymint-V1-ndk", "android.hardware.security.keymint-V1-ndk_platform", @@ -74,6 +76,8 @@ var VndkMustUseVendorVariantList = []string{ "android.hardware.weaver-ndk_platform", "android.hardware.weaver-unstable-ndk_platform", "android.system.keystore2-V1-ndk", + "android.hardware.wifi.hostapd-V1-ndk", + "android.hardware.wifi.hostapd-V1-ndk_platform", "android.system.keystore2-V1-ndk_platform", "android.system.keystore2-ndk_platform", "android.system.keystore2-unstable-ndk_platform", diff --git a/cc/library.go b/cc/library.go index 196116bd6..92d9771dd 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2341,18 +2341,6 @@ type bazelCcLibraryStaticAttributes struct { Static staticOrSharedAttributes } -type bazelCcLibraryStatic struct { - android.BazelTargetModuleBase - bazelCcLibraryStaticAttributes -} - -func BazelCcLibraryStaticFactory() android.Module { - module := &bazelCcLibraryStatic{} - module.AddProperties(&module.bazelCcLibraryStaticAttributes) - android.InitBazelTargetModule(module) - return module -} - func ccLibraryStaticBp2BuildInternal(ctx android.TopDownMutatorContext, module *Module) { compilerAttrs := bp2BuildParseCompilerProps(ctx, module) linkerAttrs := bp2BuildParseLinkerProps(ctx, module) @@ -2423,9 +2411,3 @@ func CcLibraryStaticBp2Build(ctx android.TopDownMutatorContext) { ccLibraryStaticBp2BuildInternal(ctx, module) } - -func (m *bazelCcLibraryStatic) Name() string { - return m.BaseModuleName() -} - -func (m *bazelCcLibraryStatic) GenerateAndroidBuildActions(ctx android.ModuleContext) {} diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 51cdddfb0..704b03afb 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -45,11 +45,17 @@ var ( abidw = pctx.AndroidStaticRule("abidw", blueprint.RuleParams{ Command: "$abidw --type-id-style hash --no-corpus-path " + - "--no-show-locs --no-comp-dir-path -w $symbolList $in | " + - "$abitidy --all -o $out", - CommandDeps: []string{"$abitidy", "$abidw"}, + "--no-show-locs --no-comp-dir-path -w $symbolList " + + "$in --out-file $out", + CommandDeps: []string{"$abidw"}, }, "symbolList") + abitidy = pctx.AndroidStaticRule("abitidy", + blueprint.RuleParams{ + Command: "$abitidy --all -i $in -o $out", + CommandDeps: []string{"$abitidy"}, + }) + abidiff = pctx.AndroidStaticRule("abidiff", blueprint.RuleParams{ // Need to create *some* output for ninja. We don't want to use tee @@ -313,19 +319,28 @@ func canDiffAbi() bool { func (this *stubDecorator) dumpAbi(ctx ModuleContext, symbolList android.Path) { implementationLibrary := this.findImplementationLibrary(ctx) - this.abiDumpPath = getNdkAbiDumpInstallBase(ctx).Join(ctx, + abiRawPath := getNdkAbiDumpInstallBase(ctx).Join(ctx, this.apiLevel.String(), ctx.Arch().ArchType.String(), - this.libraryName(ctx), "abi.xml") + this.libraryName(ctx), "abi.raw.xml") ctx.Build(pctx, android.BuildParams{ Rule: abidw, Description: fmt.Sprintf("abidw %s", implementationLibrary), - Output: this.abiDumpPath, Input: implementationLibrary, + Output: abiRawPath, Implicit: symbolList, Args: map[string]string{ "symbolList": symbolList.String(), }, }) + this.abiDumpPath = getNdkAbiDumpInstallBase(ctx).Join(ctx, + this.apiLevel.String(), ctx.Arch().ArchType.String(), + this.libraryName(ctx), "abi.xml") + ctx.Build(pctx, android.BuildParams{ + Rule: abitidy, + Description: fmt.Sprintf("abitidy %s", implementationLibrary), + Input: abiRawPath, + Output: this.abiDumpPath, + }) } func findNextApiLevel(ctx ModuleContext, apiLevel android.ApiLevel) *android.ApiLevel { diff --git a/cc/test_data_test.go b/cc/test_data_test.go index 426dfc57c..a62116677 100644 --- a/cc/test_data_test.go +++ b/cc/test_data_test.go @@ -127,7 +127,7 @@ func TestDataTests(t *testing.T) { ctx.RegisterModuleType("test", newTest) ctx.Register() - _, errs := ctx.ParseBlueprintsFiles("Blueprints") + _, errs := ctx.ParseBlueprintsFiles("Android.bp") android.FailIfErrored(t, errs) _, errs = ctx.PrepareBuildActions(config) android.FailIfErrored(t, errs) diff --git a/cc/tidy.go b/cc/tidy.go index fefa7f0f1..53ff1564b 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -148,6 +148,9 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { tidyChecks = tidyChecks + ",-bugprone-branch-clone" // http://b/193716442 tidyChecks = tidyChecks + ",-bugprone-implicit-widening-of-multiplication-result" + // Too many existing functions trigger this rule, and fixing it requires large code + // refactoring. The cost of maintaining this tidy rule outweighs the benefit it brings. + tidyChecks = tidyChecks + ",-bugprone-easily-swappable-parameters" flags.TidyFlags = append(flags.TidyFlags, tidyChecks) if ctx.Config().IsEnvTrue("WITH_TIDY") { diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 284638772..fa63b465d 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -20,6 +20,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "log" "os" "os/exec" @@ -76,6 +77,36 @@ func (m *multipleStringArg) Set(s string) error { return nil } +const errorLeadingLines = 20 +const errorTrailingLines = 20 + +func errMsgFromLog(filename string) string { + if filename == "" { + return "" + } + + data, err := ioutil.ReadFile(filename) + if err != nil { + return "" + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) > errorLeadingLines+errorTrailingLines+1 { + lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...", + len(lines)-errorLeadingLines-errorTrailingLines) + + lines = append(lines[:errorLeadingLines+1], + lines[len(lines)-errorTrailingLines:]...) + } + var buf strings.Builder + for _, line := range lines { + buf.WriteString("> ") + buf.WriteString(line) + buf.WriteString("\n") + } + return buf.String() +} + // TODO(b/70370883): This tool uses a lot of open files -- over the default // soft limit of 1024 on some systems. So bump up to the hard limit until I fix // the algorithm. @@ -170,6 +201,23 @@ func ensureEmptyFileExists(file string, log logger.Logger) { } } +func outDirBase() string { + outDirBase := os.Getenv("OUT_DIR") + if outDirBase == "" { + return "out" + } else { + return outDirBase + } +} + +func distDir(outDir string) string { + if distDir := os.Getenv("DIST_DIR"); distDir != "" { + return filepath.Clean(distDir) + } else { + return filepath.Join(outDir, "dist") + } +} + func main() { stdio := terminal.StdioImpl{} @@ -177,6 +225,12 @@ func main() { log := logger.New(output) defer log.Cleanup() + for _, v := range os.Environ() { + log.Println("Environment: " + v) + } + + log.Printf("Argv: %v\n", os.Args) + flag.Parse() _, cancel := context.WithCancel(context.Background()) @@ -208,13 +262,7 @@ func main() { if !*incremental { name += "-" + time.Now().Format("20060102150405") } - - outDirBase := os.Getenv("OUT_DIR") - if outDirBase == "" { - outDirBase = "out" - } - - outputDir = filepath.Join(outDirBase, name) + outputDir = filepath.Join(outDirBase(), name) } log.Println("Output directory:", outputDir) @@ -231,11 +279,13 @@ func main() { var configLogsDir string if *alternateResultDir { - configLogsDir = filepath.Join(outputDir, "dist/logs") + configLogsDir = filepath.Join(distDir(outDirBase()), "logs") } else { configLogsDir = outputDir } + log.Println("Logs dir: " + configLogsDir) + os.MkdirAll(configLogsDir, 0777) log.SetOutput(filepath.Join(configLogsDir, "soong.log")) trace.SetOutput(filepath.Join(configLogsDir, "build.trace")) @@ -348,10 +398,11 @@ func main() { FileArgs: []zip.FileArg{ {GlobDir: logsDir, SourcePrefixToStrip: logsDir}, }, - OutputFilePath: filepath.Join(outputDir, "dist/logs.zip"), + OutputFilePath: filepath.Join(distDir(outDirBase()), "logs.zip"), NumParallelJobs: runtime.NumCPU(), CompressionLevel: 5, } + log.Printf("Logs zip: %v\n", args.OutputFilePath) if err := zip.Zip(args); err != nil { log.Fatalf("Error zipping logs: %v", err) } @@ -424,10 +475,6 @@ func runSoongUiForProduct(mpctx *mpContext, product string) { args = append(args, "--soong-only") } - if *alternateResultDir { - args = append(args, "dist") - } - cmd := exec.Command(mpctx.SoongUi, args...) cmd.Stdout = consoleLogWriter cmd.Stderr = consoleLogWriter @@ -439,6 +486,11 @@ func runSoongUiForProduct(mpctx *mpContext, product string) { "TARGET_BUILD_APPS=", "TARGET_BUILD_UNBUNDLED=") + if *alternateResultDir { + cmd.Env = append(cmd.Env, + "DIST_DIR="+filepath.Join(distDir(outDirBase()), "products/"+product)) + } + action := &status.Action{ Description: product, Outputs: []string{product}, @@ -459,9 +511,17 @@ func runSoongUiForProduct(mpctx *mpContext, product string) { } } } + var errOutput string + if err == nil { + errOutput = "" + } else { + errOutput = errMsgFromLog(consoleLogPath) + } + mpctx.Status.FinishAction(status.ActionResult{ Action: action, Error: err, + Output: errOutput, }) } diff --git a/cmd/soong_build/Android.bp b/cmd/soong_build/Android.bp index 9f0922449..703a8759a 100644 --- a/cmd/soong_build/Android.bp +++ b/cmd/soong_build/Android.bp @@ -16,7 +16,7 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], } -bootstrap_go_binary { +blueprint_go_binary { name: "soong_build", deps: [ "blueprint", diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 16a4e1a8a..09a223473 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -36,9 +36,12 @@ import ( var ( topDir string outDir string + soongOutDir string availableEnvFile string usedEnvFile string + runGoTests bool + globFile string globListDir string delveListen string @@ -55,13 +58,12 @@ var ( func init() { // Flags that make sense in every mode flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree") - flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)") + flag.StringVar(&soongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)") flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables") flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables") flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output") flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files") - flag.StringVar(&cmdlineArgs.SoongOutDir, "b", ".", "the build output directory") - flag.StringVar(&cmdlineArgs.OutDir, "n", "", "the ninja builddir directory") + flag.StringVar(&outDir, "out", "", "the ninja builddir directory") flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse") // Debug flags @@ -82,8 +84,7 @@ func init() { // Flags that probably shouldn't be flags of soong_build but we haven't found // the time to remove them yet - flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap") - flag.BoolVar(&cmdlineArgs.UseValidations, "use-validations", false, "use validations to depend on go tests") + flag.BoolVar(&runGoTests, "t", false, "build and run go tests during bootstrap") } func newNameResolver(config android.Config) *android.NameResolver { @@ -102,19 +103,16 @@ func newNameResolver(config android.Config) *android.NameResolver { return android.NewNameResolver(exportFilter) } -func newContext(configuration android.Config, prepareBuildActions bool) *android.Context { +func newContext(configuration android.Config) *android.Context { ctx := android.NewContext(configuration) ctx.Register() - if !prepareBuildActions { - configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) - } ctx.SetNameInterface(newNameResolver(configuration)) ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) return ctx } -func newConfig(outDir string, availableEnv map[string]string) android.Config { - configuration, err := android.NewConfig(outDir, cmdlineArgs.ModuleListFile, availableEnv) +func newConfig(availableEnv map[string]string) android.Config { + configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, runGoTests, outDir, soongOutDir, availableEnv) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -128,11 +126,7 @@ func newConfig(outDir string, availableEnv map[string]string) android.Config { // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite // the incorrect results from the first pass, and file I/O is expensive. func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { - var firstArgs, secondArgs bootstrap.Args - - firstArgs = cmdlineArgs - configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja) - bootstrap.RunBlueprint(firstArgs, firstCtx.Context, configuration) + bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) // Invoke bazel commands and save results for second pass. if err := configuration.BazelContext.InvokeBazel(); err != nil { @@ -145,35 +139,26 @@ func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } - secondCtx := newContext(secondConfig, true) - secondArgs = cmdlineArgs - ninjaDeps := bootstrap.RunBlueprint(secondArgs, secondCtx.Context, secondConfig) + secondCtx := newContext(secondConfig) + ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) globListFiles := writeBuildGlobsNinjaFile(secondCtx.SrcDir(), configuration.SoongOutDir(), secondCtx.Globs, configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - writeDepFile(secondArgs.OutFile, ninjaDeps) + writeDepFile(cmdlineArgs.OutFile, ninjaDeps) } // Run the code-generation phase to convert BazelTargetModules to BUILD files. -func runQueryView(configuration android.Config, ctx *android.Context) { +func runQueryView(queryviewDir, queryviewMarker string, configuration android.Config, ctx *android.Context) { codegenContext := bp2build.NewCodegenContext(configuration, *ctx, bp2build.QueryView) - absoluteQueryViewDir := shared.JoinPath(topDir, bazelQueryViewDir) + absoluteQueryViewDir := shared.JoinPath(topDir, queryviewDir) if err := createBazelQueryView(codegenContext, absoluteQueryViewDir); err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } -} -func runSoongDocs(configuration android.Config) { - ctx := newContext(configuration, false) - soongDocsArgs := cmdlineArgs - bootstrap.RunBlueprint(soongDocsArgs, ctx.Context, configuration) - if err := writeDocs(ctx, configuration, docFile); err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) - } + touch(shared.JoinPath(topDir, queryviewMarker)) } func writeMetrics(configuration android.Config) { @@ -220,24 +205,33 @@ func writeDepFile(outputFile string, ninjaDeps []string) { // or the actual Soong build for the build.ninja file. Returns the top level // output file of the specific activity. func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string { - bazelConversionRequested := bp2buildMarker != "" mixedModeBuild := configuration.BazelContext.BazelEnabled() + generateBazelWorkspace := bp2buildMarker != "" generateQueryView := bazelQueryViewDir != "" + generateModuleGraphFile := moduleGraphFile != "" + generateDocFile := docFile != "" blueprintArgs := cmdlineArgs - prepareBuildActions := !generateQueryView && moduleGraphFile == "" - if bazelConversionRequested { + + var stopBefore bootstrap.StopBefore + if !generateModuleGraphFile && !generateQueryView && !generateDocFile { + stopBefore = bootstrap.DoEverything + } else { + stopBefore = bootstrap.StopBeforePrepareBuildActions + } + + if generateBazelWorkspace { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. runBp2Build(configuration, extraNinjaDeps) return bp2buildMarker } - ctx := newContext(configuration, prepareBuildActions) + ctx := newContext(configuration) if mixedModeBuild { runMixedModeBuild(configuration, ctx, extraNinjaDeps) } else { - ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, ctx.Context, configuration) + ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, stopBefore, ctx.Context, configuration) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) globListFiles := writeBuildGlobsNinjaFile(ctx.SrcDir(), configuration.SoongOutDir(), ctx.Globs, configuration) @@ -245,12 +239,24 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str // Convert the Soong module graph into Bazel BUILD files. if generateQueryView { - runQueryView(configuration, ctx) - return cmdlineArgs.OutFile // TODO: This is a lie - } else if moduleGraphFile != "" { + queryviewMarkerFile := bazelQueryViewDir + ".marker" + runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx) + writeDepFile(queryviewMarkerFile, ninjaDeps) + return queryviewMarkerFile + } else if generateModuleGraphFile { writeJsonModuleGraph(ctx, moduleGraphFile) writeDepFile(moduleGraphFile, ninjaDeps) return moduleGraphFile + } else if generateDocFile { + // TODO: we could make writeDocs() return the list of documentation files + // written and add them to the .d file. Then soong_docs would be re-run + // whenever one is deleted. + if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil { + fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err) + os.Exit(1) + } + writeDepFile(docFile, ninjaDeps) + return docFile } else { // The actual output (build.ninja) was written in the RunBlueprint() call // above @@ -298,7 +304,7 @@ func main() { availableEnv := parseAvailableEnv() - configuration := newConfig(outDir, availableEnv) + configuration := newConfig(availableEnv) extraNinjaDeps := []string{ configuration.ProductVariablesFileName, usedEnvFile, @@ -314,16 +320,6 @@ func main() { extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve")) } - if docFile != "" { - // We don't write an used variables file when generating documentation - // because that is done from within the actual builds as a Ninja action and - // thus it would overwrite the actual used variables file so this is - // special-cased. - // TODO: Fix this by not passing --used_env to the soong_docs invocation - runSoongDocs(configuration) - return - } - finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) writeUsedEnvironmentFile(configuration, finalOutputFile) } @@ -476,14 +472,11 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { extraNinjaDeps = append(extraNinjaDeps, modulePaths...) - // No need to generate Ninja build rules/statements from Modules and Singletons. - configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) - // Run the loading and analysis pipeline to prepare the graph of regular // Modules parsed from Android.bp files, and the BazelTargetModules mapped // from the regular Modules. blueprintArgs := cmdlineArgs - ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bp2buildCtx.Context, configuration) + ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, bp2buildCtx.Context, configuration) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx.SrcDir(), configuration.SoongOutDir(), bp2buildCtx.Globs, configuration) @@ -505,8 +498,8 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { "bazel-" + filepath.Base(topDir), } - if cmdlineArgs.OutDir[0] != '/' { - excludes = append(excludes, cmdlineArgs.OutDir) + if outDir[0] != '/' { + excludes = append(excludes, outDir) } existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) diff --git a/cmd/soong_build/queryview.go b/cmd/soong_build/queryview.go index a8602deb7..98e27c62b 100644 --- a/cmd/soong_build/queryview.go +++ b/cmd/soong_build/queryview.go @@ -15,14 +15,16 @@ package main import ( - "android/soong/android" - "android/soong/bp2build" "io/ioutil" "os" "path/filepath" + + "android/soong/android" + "android/soong/bp2build" ) func createBazelQueryView(ctx *bp2build.CodegenContext, bazelQueryViewDir string) error { + os.RemoveAll(bazelQueryViewDir) ruleShims := bp2build.CreateRuleShims(android.ModuleTypeFactories()) // Ignore metrics reporting and compat layers for queryview, since queryview diff --git a/cmd/soong_build/writedocs.go b/cmd/soong_build/writedocs.go index b7c260c6a..8d8f37f4b 100644 --- a/cmd/soong_build/writedocs.go +++ b/cmd/soong_build/writedocs.go @@ -15,13 +15,14 @@ package main import ( - "android/soong/android" "bytes" "html/template" "io/ioutil" "path/filepath" "sort" + "android/soong/android" + "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/bootstrap/bpdoc" ) @@ -95,13 +96,13 @@ func moduleTypeDocsToTemplates(moduleTypeList []*bpdoc.ModuleType) []moduleTypeT return result } -func getPackages(ctx *android.Context, config interface{}) ([]*bpdoc.Package, error) { +func getPackages(ctx *android.Context) ([]*bpdoc.Package, error) { moduleTypeFactories := android.ModuleTypeFactoriesForDocs() - return bootstrap.ModuleTypeDocs(ctx.Context, config, moduleTypeFactories) + return bootstrap.ModuleTypeDocs(ctx.Context, moduleTypeFactories) } -func writeDocs(ctx *android.Context, config interface{}, filename string) error { - packages, err := getPackages(ctx, config) +func writeDocs(ctx *android.Context, filename string) error { + packages, err := getPackages(ctx) if err != nil { return err } diff --git a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go index 7dbe74c27..ba05d945d 100644 --- a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go +++ b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go @@ -87,7 +87,9 @@ func main() { usage("--module configuration file is required") } - ctx := &builderContext{android.NullConfig(*outDir)} + // NOTE: duplicating --out_dir here is incorrect (one should be the another + // plus "/soong" but doing so apparently breaks dexpreopt + ctx := &builderContext{android.NullConfig(*outDir, *outDir)} globalSoongConfigData, err := ioutil.ReadFile(*globalSoongConfigPath) if err != nil { diff --git a/java/app_import.go b/java/app_import.go index b5a608412..3e5f972a4 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -204,9 +204,9 @@ func (a *AndroidAppImport) shouldUncompressDex(ctx android.ModuleContext) bool { return false } - // Uncompress dex in APKs of privileged apps - if ctx.Config().UncompressPrivAppDex() && a.Privileged() { - return true + // Uncompress dex in APKs of priv-apps if and only if DONT_UNCOMPRESS_PRIV_APPS_DEXS is false. + if a.Privileged() { + return ctx.Config().UncompressPrivAppDex() } return shouldUncompressDex(ctx, &a.dexpreopter) diff --git a/java/app_import_test.go b/java/app_import_test.go index 024a3df51..efa52c178 100644 --- a/java/app_import_test.go +++ b/java/app_import_test.go @@ -15,6 +15,7 @@ package java import ( + "fmt" "reflect" "regexp" "strings" @@ -656,3 +657,74 @@ func TestAndroidTestImport_Preprocessed(t *testing.T) { } } } + +func TestAndroidTestImport_UncompressDex(t *testing.T) { + testCases := []struct { + name string + bp string + }{ + { + name: "normal", + bp: ` + android_app_import { + name: "foo", + presigned: true, + apk: "prebuilts/apk/app.apk", + } + `, + }, + { + name: "privileged", + bp: ` + android_app_import { + name: "foo", + presigned: true, + privileged: true, + apk: "prebuilts/apk/app.apk", + } + `, + }, + } + + test := func(t *testing.T, bp string, unbundled bool, dontUncompressPrivAppDexs bool) { + t.Helper() + + result := android.GroupFixturePreparers( + prepareForJavaTest, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + if unbundled { + variables.Unbundled_build = proptools.BoolPtr(true) + } + variables.UncompressPrivAppDex = proptools.BoolPtr(!dontUncompressPrivAppDexs) + }), + ).RunTestWithBp(t, bp) + + foo := result.ModuleForTests("foo", "android_common") + actual := foo.MaybeRule("uncompress-dex").Rule != nil + + expect := !unbundled + if strings.Contains(bp, "privileged: true") { + if dontUncompressPrivAppDexs { + expect = false + } else { + // TODO(b/194504107): shouldn't priv-apps be always uncompressed unless + // DONT_UNCOMPRESS_PRIV_APPS_DEXS is true (regardless of unbundling)? + // expect = true + } + } + + android.AssertBoolEquals(t, "uncompress dex", expect, actual) + } + + for _, unbundled := range []bool{false, true} { + for _, dontUncompressPrivAppDexs := range []bool{false, true} { + for _, tt := range testCases { + name := fmt.Sprintf("%s,unbundled:%t,dontUncompressPrivAppDexs:%t", + tt.name, unbundled, dontUncompressPrivAppDexs) + t.Run(name, func(t *testing.T) { + test(t, tt.bp, unbundled, dontUncompressPrivAppDexs) + }) + } + } + } +} diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index 72525c4cd..209e82bbd 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -202,8 +202,7 @@ func quit(s interface{}) { func buildProductConfigMap() map[string]string { const androidProductsMk = "AndroidProducts.mk" // Build the list of AndroidProducts.mk files: it's - // build/make/target/product/AndroidProducts.mk plus - // device/**/AndroidProducts.mk + // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk) if _, err := os.Stat(targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n", @@ -213,17 +212,19 @@ func buildProductConfigMap() map[string]string { if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) } - _ = filepath.Walk(filepath.Join(*rootDir, "device"), - func(path string, info os.FileInfo, err error) error { - if info.IsDir() || filepath.Base(path) != androidProductsMk { + for _, t := range []string{"device", "vendor"} { + _ = filepath.WalkDir(filepath.Join(*rootDir, t), + func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk { + return nil + } + if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil { + fmt.Fprintf(os.Stderr, "%s: %s\n", path, err) + // Keep going, we want to find all such errors in a single run + } return nil - } - if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n", path, err) - // Keep going, we want to find all such errors in a single run - } - return nil - }) + }) + } return productConfigMap } diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index 46212ee8b..ca7fe6f23 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -414,7 +414,7 @@ endif def init(g, handle): cfg = rblf.cfg(handle) - if rblf.filter(g.get("PRODUCT_LIST", ""), g["TARGET_PRODUCT"]): + if rblf.filter(g.get("PRODUCT_LIST", []), g["TARGET_PRODUCT"]): pass `, }, diff --git a/mk2rbc/variable.go b/mk2rbc/variable.go index 88d63c96e..4bb9ed52d 100644 --- a/mk2rbc/variable.go +++ b/mk2rbc/variable.go @@ -299,6 +299,10 @@ func (ctx *parseContext) addVariable(name string) variable { vt = vi.valueType } } + if strings.HasSuffix(name, "_LIST") && vt == starlarkTypeUnknown { + // Heuristics: Variables with "_LIST" suffix are lists + vt = starlarkTypeList + } v = &otherGlobalVariable{baseVariable{nam: name, typ: vt}} } ctx.variables[name] = v diff --git a/rust/config/x86_linux_host.go b/rust/config/x86_linux_host.go index 0aa534f87..c10afd86f 100644 --- a/rust/config/x86_linux_host.go +++ b/rust/config/x86_linux_host.go @@ -26,6 +26,7 @@ var ( "-B${cc_config.ClangBin}", "-fuse-ld=lld", "-Wl,--undefined-version", + "--sysroot ${cc_config.LinuxGccRoot}/sysroot", } linuxX86Rustflags = []string{} linuxX86Linkflags = []string{} diff --git a/rust/coverage.go b/rust/coverage.go index 050b811c7..8fdfa2342 100644 --- a/rust/coverage.go +++ b/rust/coverage.go @@ -57,7 +57,18 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags flags.RustFlags = append(flags.RustFlags, "-Z instrument-coverage", "-g") flags.LinkFlags = append(flags.LinkFlags, - profileInstrFlag, "-g", coverage.OutputFile().Path().String(), "-Wl,--wrap,open") + profileInstrFlag, "-g", coverage.OutputFile().Path().String(), "-Wl,--wrap,open", + // Upstream LLVM change 6d2d3bd0a6 made + // -z,start-stop-gc the default. It drops metadata + // sections like __llvm_prf_data unless they are marked + // SHF_GNU_RETAIN. https://reviews.llvm.org/D97448 + // marks generated sections, including __llvm_prf_data + // as SHF_GNU_RETAIN. However this change is not in + // the Rust toolchain. Since we link Rust libs with + // new lld, we should use nostart-stop-gc until the + // Rust toolchain updates past D97448. + "-Wl,-z,nostart-stop-gc", + ) deps.StaticLibs = append(deps.StaticLibs, coverage.OutputFile().Path()) } diff --git a/scripts/check_boot_jars/check_boot_jars.py b/scripts/check_boot_jars/check_boot_jars.py index c2712116b..b711f9dcc 100755 --- a/scripts/check_boot_jars/check_boot_jars.py +++ b/scripts/check_boot_jars/check_boot_jars.py @@ -1,101 +1,102 @@ #!/usr/bin/env python +"""Check boot jars. +Usage: check_boot_jars.py <dexdump_path> <package_allow_list_file> <jar1> \ +<jar2> ... """ -Check boot jars. - -Usage: check_boot_jars.py <dexdump_path> <package_allow_list_file> <jar1> <jar2> ... -""" +from __future__ import print_function import logging -import os.path import re import subprocess import sys import xml.etree.ElementTree - # The compiled allow list RE. allow_list_re = None def LoadAllowList(filename): - """ Load and compile allow list regular expressions from filename. - """ - lines = [] - with open(filename, 'r') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - lines.append(line) - combined_re = r'^(%s)$' % '|'.join(lines) - global allow_list_re - try: - allow_list_re = re.compile(combined_re) - except re.error: - logging.exception( - 'Cannot compile package allow list regular expression: %r', - combined_re) - allow_list_re = None - return False - return True + """ Load and compile allow list regular expressions from filename.""" + lines = [] + with open(filename, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + lines.append(line) + combined_re = r'^(%s)$' % '|'.join(lines) + global allow_list_re #pylint: disable=global-statement + try: + allow_list_re = re.compile(combined_re) + except re.error: + logging.exception( + 'Cannot compile package allow list regular expression: %r', + combined_re) + allow_list_re = None + return False + return True def CheckDexJar(dexdump_path, allow_list_path, jar): - """Check a dex jar file. - """ - # Use dexdump to generate the XML representation of the dex jar file. - p = subprocess.Popen(args='%s -l xml %s' % (dexdump_path, jar), - stdout=subprocess.PIPE, shell=True) - stdout, _ = p.communicate() - if p.returncode != 0: - return False - - packages = 0 - try: - # TODO(b/172063475) - improve performance - root = xml.etree.ElementTree.fromstring(stdout) - except xml.etree.ElementTree.ParseError as e: - print >> sys.stderr, 'Error processing jar %s - %s' % (jar, e) - print >> sys.stderr, stdout - return False - for package_elt in root.iterfind('package'): - packages += 1 - package_name = package_elt.get('name') - if not package_name or not allow_list_re.match(package_name): - # Report the name of a class in the package as it is easier to navigate to - # the source of a concrete class than to a package which is often required - # to investigate this failure. - class_name = package_elt[0].get('name') - if package_name != "": - class_name = package_name + "." + class_name - print >> sys.stderr, ('Error: %s contains class file %s, whose package name "%s" is empty or' - ' not in the allow list %s of packages allowed on the bootclasspath.' - % (jar, class_name, package_name, allow_list_path)) - return False - if packages == 0: - print >> sys.stderr, ('Error: %s does not contain any packages.' % jar) - return False - return True + """Check a dex jar file.""" + # Use dexdump to generate the XML representation of the dex jar file. + p = subprocess.Popen( + args='%s -l xml %s' % (dexdump_path, jar), + stdout=subprocess.PIPE, + shell=True) + stdout, _ = p.communicate() + if p.returncode != 0: + return False + packages = 0 + try: + # TODO(b/172063475) - improve performance + root = xml.etree.ElementTree.fromstring(stdout) + except xml.etree.ElementTree.ParseError as e: + print('Error processing jar %s - %s' % (jar, e), file=sys.stderr) + print(stdout, file=sys.stderr) + return False + for package_elt in root.iterfind('package'): + packages += 1 + package_name = package_elt.get('name') + if not package_name or not allow_list_re.match(package_name): + # Report the name of a class in the package as it is easier to + # navigate to the source of a concrete class than to a package + # which is often required to investigate this failure. + class_name = package_elt[0].get('name') + if package_name: + class_name = package_name + '.' + class_name + print(( + 'Error: %s contains class file %s, whose package name "%s" is ' + 'empty or not in the allow list %s of packages allowed on the ' + 'bootclasspath.' + % (jar, class_name, package_name, allow_list_path)), + file=sys.stderr) + return False + if packages == 0: + print(('Error: %s does not contain any packages.' % jar), + file=sys.stderr) + return False + return True def main(argv): - if len(argv) < 3: - print __doc__ - return 1 - dexdump_path = argv[0] - allow_list_path = argv[1] + if len(argv) < 3: + print(__doc__) + return 1 + dexdump_path = argv[0] + allow_list_path = argv[1] - if not LoadAllowList(allow_list_path): - return 1 + if not LoadAllowList(allow_list_path): + return 1 - passed = True - for jar in argv[2:]: - if not CheckDexJar(dexdump_path, allow_list_path, jar): - passed = False - if not passed: - return 1 + passed = True + for jar in argv[2:]: + if not CheckDexJar(dexdump_path, allow_list_path, jar): + passed = False + if not passed: + return 1 - return 0 + return 0 if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt index 18ab427b5..942f26a5c 100644 --- a/scripts/check_boot_jars/package_allowed_list.txt +++ b/scripts/check_boot_jars/package_allowed_list.txt @@ -69,6 +69,7 @@ javax\.xml\.transform\.sax javax\.xml\.transform\.stream javax\.xml\.validation javax\.xml\.xpath +jdk\.internal\.math jdk\.internal\.util jdk\.internal\.vm\.annotation jdk\.net diff --git a/scripts/get_clang_version.py b/scripts/get_clang_version.py index 17bc88bbd..64d922a20 100755 --- a/scripts/get_clang_version.py +++ b/scripts/get_clang_version.py @@ -21,8 +21,12 @@ import sys ANDROID_BUILD_TOP = os.environ.get("ANDROID_BUILD_TOP", ".") +LLVM_PREBUILTS_VERSION = os.environ.get("LLVM_PREBUILTS_VERSION") def get_clang_prebuilts_version(global_go): + if LLVM_PREBUILTS_VERSION: + return LLVM_PREBUILTS_VERSION + # TODO(b/187231324): Get clang version from the json file once it is no longer # hard-coded in global.go if global_go is None: diff --git a/scripts/hiddenapi/generate_hiddenapi_lists.py b/scripts/hiddenapi/generate_hiddenapi_lists.py index 5ab93d185..35e0948f0 100755 --- a/scripts/hiddenapi/generate_hiddenapi_lists.py +++ b/scripts/hiddenapi/generate_hiddenapi_lists.py @@ -16,8 +16,6 @@ """Generate API lists for non-SDK API enforcement.""" import argparse from collections import defaultdict, namedtuple -import functools -import os import re import sys @@ -60,15 +58,15 @@ ALL_FLAGS_SET = set(ALL_FLAGS) # For example, the max-target-P list is checked in as it was in P, # but signatures have changes since then. The flag instructs this # script to skip any entries which do not exist any more. -FLAG_IGNORE_CONFLICTS = "ignore-conflicts" +FLAG_IGNORE_CONFLICTS = 'ignore-conflicts' # Option specified after one of FLAGS_API_LIST to express that all # apis within a given set of packages should be assign the given flag. -FLAG_PACKAGES = "packages" +FLAG_PACKAGES = 'packages' # Option specified after one of FLAGS_API_LIST to indicate an extra # tag that should be added to the matching APIs. -FLAG_TAG = "tag" +FLAG_TAG = 'tag' # Regex patterns of fields/methods used in serialization. These are # considered public API despite being hidden. @@ -84,24 +82,30 @@ SERIALIZATION_PATTERNS = [ # Single regex used to match serialization API. It combines all the # SERIALIZATION_PATTERNS into a single regular expression. -SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + r')$') +SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + + r')$') # Predicates to be used with filter_apis. -HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags) +HAS_NO_API_LIST_ASSIGNED = \ + lambda api,flags: not FLAGS_API_LIST_SET.intersection(flags) + IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api) class StoreOrderedOptions(argparse.Action): - """An argparse action that stores a number of option arguments in the order that - they were specified. + """An argparse action that stores a number of option arguments in the order + + that they were specified. """ - def __call__(self, parser, args, values, option_string = None): + + def __call__(self, parser, args, values, option_string=None): items = getattr(args, self.dest, None) if items is None: items = [] items.append([option_string.lstrip('-'), values]) setattr(args, self.dest, items) + def get_args(): """Parses command line arguments. @@ -110,22 +114,43 @@ def get_args(): """ parser = argparse.ArgumentParser() parser.add_argument('--output', required=True) - parser.add_argument('--csv', nargs='*', default=[], metavar='CSV_FILE', + parser.add_argument( + '--csv', + nargs='*', + default=[], + metavar='CSV_FILE', help='CSV files to be merged into output') for flag in ALL_FLAGS: - parser.add_argument('--' + flag, dest='ordered_flags', metavar='TXT_FILE', - action=StoreOrderedOptions, help='lists of entries with flag "' + flag + '"') - parser.add_argument('--' + FLAG_IGNORE_CONFLICTS, dest='ordered_flags', nargs=0, - action=StoreOrderedOptions, help='Indicates that only known and otherwise unassigned ' - 'entries should be assign the given flag. Must follow a list of entries and applies ' - 'to the preceding such list.') - parser.add_argument('--' + FLAG_PACKAGES, dest='ordered_flags', nargs=0, - action=StoreOrderedOptions, help='Indicates that the previous list of entries ' - 'is a list of packages. All members in those packages will be given the flag. ' - 'Must follow a list of entries and applies to the preceding such list.') - parser.add_argument('--' + FLAG_TAG, dest='ordered_flags', nargs=1, - action=StoreOrderedOptions, help='Adds an extra tag to the previous list of entries. ' + parser.add_argument( + '--' + flag, + dest='ordered_flags', + metavar='TXT_FILE', + action=StoreOrderedOptions, + help='lists of entries with flag "' + flag + '"') + parser.add_argument( + '--' + FLAG_IGNORE_CONFLICTS, + dest='ordered_flags', + nargs=0, + action=StoreOrderedOptions, + help='Indicates that only known and otherwise unassigned ' + 'entries should be assign the given flag. Must follow a list of ' + 'entries and applies to the preceding such list.') + parser.add_argument( + '--' + FLAG_PACKAGES, + dest='ordered_flags', + nargs=0, + action=StoreOrderedOptions, + help='Indicates that the previous list of entries ' + 'is a list of packages. All members in those packages will be given ' + 'the flag. Must follow a list of entries and applies to the preceding ' + 'such list.') + parser.add_argument( + '--' + FLAG_TAG, + dest='ordered_flags', + nargs=1, + action=StoreOrderedOptions, + help='Adds an extra tag to the previous list of entries. ' 'Must follow a list of entries and applies to the preceding such list.') return parser.parse_args() @@ -143,9 +168,9 @@ def read_lines(filename): Lines of the file as a list of string. """ with open(filename, 'r') as f: - lines = f.readlines(); - lines = filter(lambda line: not line.startswith('#'), lines) - lines = map(lambda line: line.strip(), lines) + lines = f.readlines() + lines = [line for line in lines if not line.startswith('#')] + lines = [line.strip() for line in lines] return set(lines) @@ -156,7 +181,7 @@ def write_lines(filename, lines): filename (string): Path to the file to be writing into. lines (list): List of strings to write into the file. """ - lines = map(lambda line: line + '\n', lines) + lines = [line + '\n' for line in lines] with open(filename, 'w') as f: f.writelines(lines) @@ -170,17 +195,19 @@ def extract_package(signature): Returns: The package name of the class containing the field/method. """ - full_class_name = signature.split(";->")[0] + full_class_name = signature.split(';->')[0] # Example: Landroid/hardware/radio/V1_2/IRadio$Proxy - if (full_class_name[0] != "L"): - raise ValueError("Expected to start with 'L': %s" % full_class_name) + if full_class_name[0] != 'L': + raise ValueError("Expected to start with 'L': %s" + % full_class_name) full_class_name = full_class_name[1:] # If full_class_name doesn't contain '/', then package_name will be ''. - package_name = full_class_name.rpartition("/")[0] + package_name = full_class_name.rpartition('/')[0] return package_name.replace('/', '.') class FlagsDict: + def __init__(self): self._dict_keyset = set() self._dict = defaultdict(set) @@ -188,37 +215,43 @@ class FlagsDict: def _check_entries_set(self, keys_subset, source): assert isinstance(keys_subset, set) assert keys_subset.issubset(self._dict_keyset), ( - "Error: {} specifies signatures not present in code:\n" - "{}" - "Please visit go/hiddenapi for more information.").format( - source, "".join(map(lambda x: " " + str(x) + "\n", keys_subset - self._dict_keyset))) + 'Error: {} specifies signatures not present in code:\n' + '{}' + 'Please visit go/hiddenapi for more information.').format( + source, ''.join( + [' ' + str(x) + '\n' for x in + keys_subset - self._dict_keyset])) def _check_flags_set(self, flags_subset, source): assert isinstance(flags_subset, set) assert flags_subset.issubset(ALL_FLAGS_SET), ( - "Error processing: {}\n" - "The following flags were not recognized: \n" - "{}\n" - "Please visit go/hiddenapi for more information.").format( - source, "\n".join(flags_subset - ALL_FLAGS_SET)) + 'Error processing: {}\n' + 'The following flags were not recognized: \n' + '{}\n' + 'Please visit go/hiddenapi for more information.').format( + source, '\n'.join(flags_subset - ALL_FLAGS_SET)) def filter_apis(self, filter_fn): """Returns APIs which match a given predicate. - This is a helper function which allows to filter on both signatures (keys) and - flags (values). The built-in filter() invokes the lambda only with dict's keys. + This is a helper function which allows to filter on both signatures + (keys) and + flags (values). The built-in filter() invokes the lambda only with + dict's keys. Args: - filter_fn : Function which takes two arguments (signature/flags) and returns a boolean. + filter_fn : Function which takes two arguments (signature/flags) and + returns a boolean. Returns: A set of APIs which match the predicate. """ - return set(filter(lambda x: filter_fn(x, self._dict[x]), self._dict_keyset)) + return {x for x in self._dict_keyset if filter_fn(x, self._dict[x])} def get_valid_subset_of_unassigned_apis(self, api_subset): - """Sanitizes a key set input to only include keys which exist in the dictionary - and have not been assigned any API list flags. + """Sanitizes a key set input to only include keys which exist in the + + dictionary and have not been assigned any API list flags. Args: entries_subset (set/list): Key set to be sanitized. @@ -227,7 +260,8 @@ class FlagsDict: Sanitized key set. """ assert isinstance(api_subset, set) - return api_subset.intersection(self.filter_apis(HAS_NO_API_LIST_ASSIGNED)) + return api_subset.intersection( + self.filter_apis(HAS_NO_API_LIST_ASSIGNED)) def generate_csv(self): """Constructs CSV entries from a dictionary. @@ -235,15 +269,16 @@ class FlagsDict: Old versions of flags are used to generate the file. Returns: - List of lines comprising a CSV file. See "parse_and_merge_csv" for format description. + List of lines comprising a CSV file. See "parse_and_merge_csv" for + format description. """ lines = [] for api in self._dict: - flags = sorted(self._dict[api]) - lines.append(",".join([api] + flags)) + flags = sorted(self._dict[api]) + lines.append(','.join([api] + flags)) return sorted(lines) - def parse_and_merge_csv(self, csv_lines, source = "<unknown>"): + def parse_and_merge_csv(self, csv_lines, source='<unknown>'): """Parses CSV entries and merges them into a given dictionary. The expected CSV format is: @@ -251,21 +286,20 @@ class FlagsDict: Args: csv_lines (list of strings): Lines read from a CSV file. - source (string): Origin of `csv_lines`. Will be printed in error messages. - - Throws: - AssertionError if parsed flags are invalid. + source (string): Origin of `csv_lines`. Will be printed in error + messages. + Throws: AssertionError if parsed flags are invalid. """ # Split CSV lines into arrays of values. - csv_values = [ line.split(',') for line in csv_lines ] + csv_values = [line.split(',') for line in csv_lines] # Update the full set of API signatures. - self._dict_keyset.update([ csv[0] for csv in csv_values ]) + self._dict_keyset.update([csv[0] for csv in csv_values]) # Check that all flags are known. csv_flags = set() for csv in csv_values: - csv_flags.update(csv[1:]) + csv_flags.update(csv[1:]) self._check_flags_set(csv_flags, source) # Iterate over all CSV lines, find entry in dict and append flags to it. @@ -275,47 +309,53 @@ class FlagsDict: flags.append(FLAG_SDK) self._dict[csv[0]].update(flags) - def assign_flag(self, flag, apis, source="<unknown>", tag = None): + def assign_flag(self, flag, apis, source='<unknown>', tag=None): """Assigns a flag to given subset of entries. Args: flag (string): One of ALL_FLAGS. apis (set): Subset of APIs to receive the flag. - source (string): Origin of `entries_subset`. Will be printed in error messages. - - Throws: - AssertionError if parsed API signatures of flags are invalid. + source (string): Origin of `entries_subset`. Will be printed in + error messages. + Throws: AssertionError if parsed API signatures of flags are invalid. """ # Check that all APIs exist in the dict. self._check_entries_set(apis, source) # Check that the flag is known. - self._check_flags_set(set([ flag ]), source) + self._check_flags_set(set([flag]), source) - # Iterate over the API subset, find each entry in dict and assign the flag to it. + # Iterate over the API subset, find each entry in dict and assign the + # flag to it. for api in apis: self._dict[api].add(flag) if tag: self._dict[api].add(tag) -FlagFile = namedtuple('FlagFile', ('flag', 'file', 'ignore_conflicts', 'packages', 'tag')) +FlagFile = namedtuple('FlagFile', + ('flag', 'file', 'ignore_conflicts', 'packages', 'tag')) + def parse_ordered_flags(ordered_flags): r = [] - currentflag, file, ignore_conflicts, packages, tag = None, None, False, False, None + currentflag, file, ignore_conflicts, packages, tag = None, None, False, \ + False, None for flag_value in ordered_flags: flag, value = flag_value[0], flag_value[1] if flag in ALL_FLAGS_SET: if currentflag: - r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag)) + r.append( + FlagFile(currentflag, file, ignore_conflicts, packages, + tag)) ignore_conflicts, packages, tag = False, False, None currentflag = flag file = value else: if currentflag is None: - raise argparse.ArgumentError('--%s is only allowed after one of %s' % ( - flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET]))) + raise argparse.ArgumentError( #pylint: disable=no-value-for-parameter + '--%s is only allowed after one of %s' % + (flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET]))) if flag == FLAG_IGNORE_CONFLICTS: ignore_conflicts = True elif flag == FLAG_PACKAGES: @@ -323,13 +363,12 @@ def parse_ordered_flags(ordered_flags): elif flag == FLAG_TAG: tag = value[0] - if currentflag: r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag)) return r -def main(argv): +def main(argv): #pylint: disable=unused-argument # Parse arguments. args = vars(get_args()) flagfiles = parse_ordered_flags(args['ordered_flags'] or []) @@ -342,7 +381,7 @@ def main(argv): # contain the full set of APIs. Subsequent additions from text files # will be able to detect invalid entries, and/or filter all as-yet # unassigned entries. - for filename in args["csv"]: + for filename in args['csv']: flags.parse_and_merge_csv(read_lines(filename), filename) # Combine inputs which do not require any particular order. @@ -352,24 +391,28 @@ def main(argv): # (2) Merge text files with a known flag into the dictionary. for info in flagfiles: if (not info.ignore_conflicts) and (not info.packages): - flags.assign_flag(info.flag, read_lines(info.file), info.file, info.tag) + flags.assign_flag(info.flag, read_lines(info.file), info.file, + info.tag) # Merge text files where conflicts should be ignored. # This will only assign the given flag if: # (a) the entry exists, and # (b) it has not been assigned any other flag. - # Because of (b), this must run after all strict assignments have been performed. + # Because of (b), this must run after all strict assignments have been + # performed. for info in flagfiles: if info.ignore_conflicts: - valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(info.file)) - flags.assign_flag(info.flag, valid_entries, filename, info.tag) + valid_entries = flags.get_valid_subset_of_unassigned_apis( + read_lines(info.file)) + flags.assign_flag(info.flag, valid_entries, filename, info.tag) #pylint: disable=undefined-loop-variable - # All members in the specified packages will be assigned the appropriate flag. + # All members in the specified packages will be assigned the appropriate + # flag. for info in flagfiles: if info.packages: packages_needing_list = set(read_lines(info.file)) - should_add_signature_to_list = lambda sig,lists: extract_package( - sig) in packages_needing_list and not lists + should_add_signature_to_list = lambda sig, lists: extract_package( + sig) in packages_needing_list and not lists #pylint: disable=cell-var-from-loop valid_entries = flags.filter_apis(should_add_signature_to_list) flags.assign_flag(info.flag, valid_entries, info.file, info.tag) @@ -377,7 +420,8 @@ def main(argv): flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED)) # Write output. - write_lines(args["output"], flags.generate_csv()) + write_lines(args['output'], flags.generate_csv()) + -if __name__ == "__main__": +if __name__ == '__main__': main(sys.argv) diff --git a/scripts/hiddenapi/generate_hiddenapi_lists_test.py b/scripts/hiddenapi/generate_hiddenapi_lists_test.py index b81424b7a..204de9723 100755 --- a/scripts/hiddenapi/generate_hiddenapi_lists_test.py +++ b/scripts/hiddenapi/generate_hiddenapi_lists_test.py @@ -15,34 +15,39 @@ # limitations under the License. """Unit tests for Hidden API list generation.""" import unittest -from generate_hiddenapi_lists import * +from generate_hiddenapi_lists import * # pylint: disable=wildcard-import,unused-wildcard-import -class TestHiddenapiListGeneration(unittest.TestCase): +class TestHiddenapiListGeneration(unittest.TestCase): def test_filter_apis(self): # Initialize flags so that A and B are put on the allow list and # C, D, E are left unassigned. Try filtering for the unassigned ones. flags = FlagsDict() - flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B,' + FLAG_SDK, - 'C', 'D', 'E']) + flags.parse_and_merge_csv( + ['A,' + FLAG_SDK, 'B,' + FLAG_SDK, 'C', 'D', 'E'] + ) filter_set = flags.filter_apis(lambda api, flags: not flags) self.assertTrue(isinstance(filter_set, set)) - self.assertEqual(filter_set, set([ 'C', 'D', 'E' ])) + self.assertEqual(filter_set, set(['C', 'D', 'E'])) def test_get_valid_subset_of_unassigned_keys(self): # Create flags where only A is unassigned. flags = FlagsDict() flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B', 'C']) flags.assign_flag(FLAG_UNSUPPORTED, set(['C'])) - self.assertEqual(flags.generate_csv(), - [ 'A,' + FLAG_SDK, 'B', 'C,' + FLAG_UNSUPPORTED ]) + self.assertEqual( + flags.generate_csv(), + ['A,' + FLAG_SDK, 'B', 'C,' + FLAG_UNSUPPORTED], + ) # Check three things: # (1) B is selected as valid unassigned # (2) A is not selected because it is assigned to the allow list # (3) D is not selected because it is not a valid key self.assertEqual( - flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ])) + flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), + set(['B']), + ) def test_parse_and_merge_csv(self): flags = FlagsDict() @@ -51,41 +56,48 @@ class TestHiddenapiListGeneration(unittest.TestCase): self.assertEqual(flags.generate_csv(), []) # Test new additions. - flags.parse_and_merge_csv([ - 'A,' + FLAG_UNSUPPORTED, - 'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O, - 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, - 'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, - 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, - ]) - self.assertEqual(flags.generate_csv(), [ - 'A,' + FLAG_UNSUPPORTED, - 'B,' + FLAG_BLOCKED + "," + FLAG_MAX_TARGET_O, - 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, - 'D,' + FLAG_TEST_API + ',' + FLAG_UNSUPPORTED, - 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, - ]) + flags.parse_and_merge_csv( + [ + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, + ] + ) + self.assertEqual( + flags.generate_csv(), + [ + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + "," + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_TEST_API + ',' + FLAG_UNSUPPORTED, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, + ], + ) # Test unknown flag. with self.assertRaises(AssertionError): - flags.parse_and_merge_csv([ 'Z,foo' ]) + flags.parse_and_merge_csv(['Z,foo']) def test_assign_flag(self): flags = FlagsDict() flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B']) # Test new additions. - flags.assign_flag(FLAG_UNSUPPORTED, set([ 'A', 'B' ])) - self.assertEqual(flags.generate_csv(), - [ 'A,' + FLAG_SDK + "," + FLAG_UNSUPPORTED, 'B,' + FLAG_UNSUPPORTED ]) + flags.assign_flag(FLAG_UNSUPPORTED, set(['A', 'B'])) + self.assertEqual( + flags.generate_csv(), + ['A,' + FLAG_SDK + "," + FLAG_UNSUPPORTED, 'B,' + FLAG_UNSUPPORTED], + ) # Test invalid API signature. with self.assertRaises(AssertionError): - flags.assign_flag(FLAG_SDK, set([ 'C' ])) + flags.assign_flag(FLAG_SDK, set(['C'])) # Test invalid flag. with self.assertRaises(AssertionError): - flags.assign_flag('foo', set([ 'A' ])) + flags.assign_flag('foo', set(['A'])) def test_extract_package(self): signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;' @@ -100,5 +112,6 @@ class TestHiddenapiListGeneration(unittest.TestCase): expected_package = 'com.foo_bar.baz' self.assertEqual(extract_package(signature), expected_package) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/scripts/hiddenapi/merge_csv.py b/scripts/hiddenapi/merge_csv.py index a65326c51..c17ec25f1 100755 --- a/scripts/hiddenapi/merge_csv.py +++ b/scripts/hiddenapi/merge_csv.py @@ -13,8 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Merge multiple CSV files, possibly with different columns. +"""Merge multiple CSV files, possibly with different columns. """ import argparse @@ -26,34 +25,52 @@ import operator from zipfile import ZipFile -args_parser = argparse.ArgumentParser(description='Merge given CSV files into a single one.') -args_parser.add_argument('--header', help='Comma separated field names; ' - 'if missing determines the header from input files.') -args_parser.add_argument('--zip_input', help='Treat files as ZIP archives containing CSV files to merge.', - action="store_true") -args_parser.add_argument('--key_field', help='The name of the field by which the rows should be sorted. ' - 'Must be in the field names. ' - 'Will be the first field in the output. ' - 'All input files must be sorted by that field.') -args_parser.add_argument('--output', help='Output file for merged CSV.', - default='-', type=argparse.FileType('w')) +args_parser = argparse.ArgumentParser( + description='Merge given CSV files into a single one.' +) +args_parser.add_argument( + '--header', + help='Comma separated field names; ' + 'if missing determines the header from input files.', +) +args_parser.add_argument( + '--zip_input', + help='Treat files as ZIP archives containing CSV files to merge.', + action="store_true", +) +args_parser.add_argument( + '--key_field', + help='The name of the field by which the rows should be sorted. ' + 'Must be in the field names. ' + 'Will be the first field in the output. ' + 'All input files must be sorted by that field.', +) +args_parser.add_argument( + '--output', + help='Output file for merged CSV.', + default='-', + type=argparse.FileType('w'), +) args_parser.add_argument('files', nargs=argparse.REMAINDER) args = args_parser.parse_args() -def dict_reader(input): - return csv.DictReader(input, delimiter=',', quotechar='|') +def dict_reader(csvfile): + return csv.DictReader(csvfile, delimiter=',', quotechar='|') + csv_readers = [] -if not(args.zip_input): +if not args.zip_input: for file in args.files: csv_readers.append(dict_reader(open(file, 'r'))) else: for file in args.files: - with ZipFile(file) as zip: - for entry in zip.namelist(): + with ZipFile(file) as zipfile: + for entry in zipfile.namelist(): if entry.endswith('.uau'): - csv_readers.append(dict_reader(io.TextIOWrapper(zip.open(entry, 'r')))) + csv_readers.append( + dict_reader(io.TextIOWrapper(zipfile.open(entry, 'r'))) + ) if args.header: fieldnames = args.header.split(',') @@ -73,8 +90,8 @@ if len(csv_readers) > 0: keyField = args.key_field if keyField: assert keyField in fieldnames, ( - "--key_field {} not found, must be one of {}\n").format( - keyField, ",".join(fieldnames)) + "--key_field {} not found, must be one of {}\n" + ).format(keyField, ",".join(fieldnames)) # Make the key field the first field in the output keyFieldIndex = fieldnames.index(args.key_field) fieldnames.insert(0, fieldnames.pop(keyFieldIndex)) @@ -83,11 +100,17 @@ if len(csv_readers) > 0: all_rows = heapq.merge(*csv_readers, key=operator.itemgetter(keyField)) # Write all rows from the input files to the output: -writer = csv.DictWriter(args.output, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL, - dialect='unix', fieldnames=fieldnames) +writer = csv.DictWriter( + args.output, + delimiter=',', + quotechar='|', + quoting=csv.QUOTE_MINIMAL, + dialect='unix', + fieldnames=fieldnames, +) writer.writeheader() # Read all the rows from the input and write them to the output in the correct # order: for row in all_rows: - writer.writerow(row) + writer.writerow(row) diff --git a/scripts/hiddenapi/signature_patterns.py b/scripts/hiddenapi/signature_patterns.py index a7c5bb4f3..0acb2a004 100755 --- a/scripts/hiddenapi/signature_patterns.py +++ b/scripts/hiddenapi/signature_patterns.py @@ -13,22 +13,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Generate a set of signature patterns from the modular flags generated by a +"""Generate a set of signature patterns from the modular flags generated by a bootclasspath_fragment that can be used to select a subset of monolithic flags against which the modular flags can be compared. """ import argparse import csv +import sys + +def dict_reader(csvfile): + return csv.DictReader( + csvfile, delimiter=',', quotechar='|', fieldnames=['signature'] + ) -def dict_reader(input): - return csv.DictReader(input, delimiter=',', quotechar='|', fieldnames=['signature']) def produce_patterns_from_file(file): with open(file, 'r') as f: return produce_patterns_from_stream(f) + def produce_patterns_from_stream(stream): # Read in all the signatures into a list and remove member names. patterns = set() @@ -38,18 +42,26 @@ def produce_patterns_from_stream(stream): # Remove the class specific member signature pieces = text.split(";->") qualifiedClassName = pieces[0] - # Remove inner class names as they cannot be separated from the containing outer class. + # Remove inner class names as they cannot be separated + # from the containing outer class. pieces = qualifiedClassName.split("$", maxsplit=1) pattern = pieces[0] patterns.add(pattern) - patterns = list(patterns) + patterns = list(patterns) #pylint: disable=redefined-variable-type patterns.sort() return patterns + def main(args): - args_parser = argparse.ArgumentParser(description='Generate a set of signature patterns that select a subset of monolithic hidden API files.') - args_parser.add_argument('--flags', help='The stub flags file which contains an entry for every dex member') + args_parser = argparse.ArgumentParser( + description='Generate a set of signature patterns ' + 'that select a subset of monolithic hidden API files.' + ) + args_parser.add_argument( + '--flags', + help='The stub flags file which contains an entry for every dex member', + ) args_parser.add_argument('--output', help='Generated signature prefixes') args = args_parser.parse_args(args) @@ -62,5 +74,6 @@ def main(args): outputFile.write(pattern) outputFile.write("\n") + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/scripts/hiddenapi/signature_patterns_test.py b/scripts/hiddenapi/signature_patterns_test.py index 0431f4501..3babe5420 100755 --- a/scripts/hiddenapi/signature_patterns_test.py +++ b/scripts/hiddenapi/signature_patterns_test.py @@ -18,21 +18,25 @@ import io import unittest -from signature_patterns import * +from signature_patterns import * #pylint: disable=unused-wildcard-import,wildcard-import -class TestGeneratedPatterns(unittest.TestCase): - def produce_patterns_from_string(self, csv): - with io.StringIO(csv) as f: +class TestGeneratedPatterns(unittest.TestCase): + def produce_patterns_from_string(self, csvdata): + with io.StringIO(csvdata) as f: return produce_patterns_from_stream(f) def test_generate(self): - patterns = self.produce_patterns_from_string(''' + #pylint: disable=line-too-long + patterns = self.produce_patterns_from_string( + ''' Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;,public-api Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api Ljava/lang/Object;->toString()Ljava/lang/String;,blocked -''') +''' + ) + #pylint: enable=line-too-long expected = [ "java/lang/Character", "java/lang/Object", @@ -40,5 +44,6 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked ] self.assertEqual(expected, patterns) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/scripts/hiddenapi/verify_overlaps.py b/scripts/hiddenapi/verify_overlaps.py index 6432bf1a4..4cd7e63ac 100755 --- a/scripts/hiddenapi/verify_overlaps.py +++ b/scripts/hiddenapi/verify_overlaps.py @@ -13,8 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Verify that one set of hidden API flags is a subset of another. +"""Verify that one set of hidden API flags is a subset of another. """ import argparse @@ -22,9 +21,9 @@ import csv import sys from itertools import chain +#pylint: disable=line-too-long class InteriorNode: - """ - An interior node in a trie. + """An interior node in a trie. Each interior node has a dict that maps from an element of a signature to either another interior node or a leaf. Each interior node represents either @@ -52,19 +51,21 @@ class InteriorNode: Attributes: nodes: a dict from an element of the signature to the Node/Leaf - containing the next element/value. + containing the next element/value. """ + #pylint: enable=line-too-long + def __init__(self): self.nodes = {} + #pylint: disable=line-too-long def signatureToElements(self, signature): - """ - Split a signature or a prefix into a number of elements: + """Split a signature or a prefix into a number of elements: 1. The packages (excluding the leading L preceding the first package). 2. The class names, from outermost to innermost. 3. The member signature. - - e.g. Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; + e.g. + Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; will be broken down into these elements: 1. package:java 2. package:lang @@ -88,19 +89,21 @@ class InteriorNode: elements = parts[0].split("/") packages = elements[0:-1] className = elements[-1] - if className == "*" or className == "**": + if className in ("*" , "**"): #pylint: disable=no-else-return # Cannot specify a wildcard and target a specific member if len(member) != 0: - raise Exception("Invalid signature %s: contains wildcard %s and member signature %s" - % (signature, className, member[0])) + raise Exception( + "Invalid signature %s: contains wildcard %s and member " \ + "signature %s" + % (signature, className, member[0])) wildcard = [className] # Assemble the parts into a single list, adding prefixes to identify # the different parts. # 0 - package:java # 1 - package:lang # 2 - * - return list(chain(map(lambda x : "package:" + x, packages), - wildcard)) + return list( + chain(["package:" + x for x in packages], wildcard)) else: # Split the class name into outer / inner classes # 0 - Character @@ -113,13 +116,16 @@ class InteriorNode: # 2 - class:Character # 3 - class:UnicodeScript # 4 - member:of(I)Ljava/lang/Character$UnicodeScript; - return list(chain(map(lambda x : "package:" + x, packages), - map(lambda x : "class:" + x, classes), - map(lambda x : "member:" + x, member))) + return list( + chain( + ["package:" + x for x in packages], + ["class:" + x for x in classes], + ["member:" + x for x in member])) + #pylint: enable=line-too-long def add(self, signature, value): - """ - Associate the value with the specific signature. + """Associate the value with the specific signature. + :param signature: the member signature :param value: the value to associated with the signature :return: n/a @@ -132,21 +138,22 @@ class InteriorNode: if element in node.nodes: node = node.nodes[element] else: - next = InteriorNode() - node.nodes[element] = next - node = next + next_node = InteriorNode() + node.nodes[element] = next_node + node = next_node # Add a Leaf containing the value and associate it with the member # signature within the class. lastElement = elements[-1] if not lastElement.startswith("member:"): - raise Exception("Invalid signature: %s, does not identify a specific member" % signature) + raise Exception( + "Invalid signature: %s, does not identify a specific member" % + signature) if lastElement in node.nodes: raise Exception("Duplicate signature: %s" % signature) node.nodes[lastElement] = Leaf(value) def getMatchingRows(self, pattern): - """ - Get the values (plural) associated with the pattern. + """Get the values (plural) associated with the pattern. e.g. If the pattern is a full signature then this will return a list containing the value associated with that signature. @@ -175,13 +182,13 @@ class InteriorNode: elements = self.signatureToElements(pattern) node = self # Include all values from this node and all its children. - selector = lambda x : True + selector = lambda x: True lastElement = elements[-1] - if lastElement == "*" or lastElement == "**": + if lastElement in ("*", "**"): elements = elements[:-1] if lastElement == "*": # Do not include values from sub-packages. - selector = lambda x : not x.startswith("package:") + selector = lambda x: not x.startswith("package:") for element in elements: if element in node.nodes: node = node.nodes[element] @@ -190,19 +197,18 @@ class InteriorNode: return chain.from_iterable(node.values(selector)) def values(self, selector): - """ - :param selector: a function that can be applied to a key in the nodes + """:param selector: a function that can be applied to a key in the nodes attribute to determine whether to return its values. - :return: A list of iterables of all the values associated with this - node and its children. + + :return: A list of iterables of all the values associated with + this node and its children. """ values = [] self.appendValues(values, selector) return values def appendValues(self, values, selector): - """ - Append the values associated with this node and its children to the + """Append the values associated with this node and its children to the list. For each item (key, child) in nodes the child node's values are returned @@ -216,105 +222,116 @@ class InteriorNode: """ for key, node in self.nodes.items(): if selector(key): - node.appendValues(values, lambda x : True) + node.appendValues(values, lambda x: True) + class Leaf: - """ - A leaf of the trie + """A leaf of the trie Attributes: value: the value associated with this leaf. """ + def __init__(self, value): self.value = value - def values(self, selector): - """ - :return: A list of a list of the value associated with this node. + def values(self, selector): #pylint: disable=unused-argument + """:return: A list of a list of the value associated with this node. """ return [[self.value]] - def appendValues(self, values, selector): - """ - Appends a list of the value associated with this node to the list. + def appendValues(self, values, selector): #pylint: disable=unused-argument + """Appends a list of the value associated with this node to the list. + :param values: a list of a iterables of values. """ values.append([self.value]) -def dict_reader(input): - return csv.DictReader(input, delimiter=',', quotechar='|', fieldnames=['signature']) + +def dict_reader(csvfile): + return csv.DictReader( + csvfile, delimiter=",", quotechar="|", fieldnames=["signature"]) + def read_flag_trie_from_file(file): - with open(file, 'r') as stream: + with open(file, "r") as stream: return read_flag_trie_from_stream(stream) + def read_flag_trie_from_stream(stream): trie = InteriorNode() reader = dict_reader(stream) for row in reader: - signature = row['signature'] + signature = row["signature"] trie.add(signature, row) return trie -def extract_subset_from_monolithic_flags_as_dict_from_file(monolithicTrie, patternsFile): - """ - Extract a subset of flags from the dict containing all the monolithic flags. + +def extract_subset_from_monolithic_flags_as_dict_from_file( + monolithicTrie, patternsFile): + """Extract a subset of flags from the dict containing all the monolithic + flags. :param monolithicFlagsDict: the dict containing all the monolithic flags. :param patternsFile: a file containing a list of signature patterns that define the subset. :return: the dict from signature to row. """ - with open(patternsFile, 'r') as stream: - return extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicTrie, stream) + with open(patternsFile, "r") as stream: + return extract_subset_from_monolithic_flags_as_dict_from_stream( + monolithicTrie, stream) -def extract_subset_from_monolithic_flags_as_dict_from_stream(monolithicTrie, stream): - """ - Extract a subset of flags from the trie containing all the monolithic flags. + +def extract_subset_from_monolithic_flags_as_dict_from_stream( + monolithicTrie, stream): + """Extract a subset of flags from the trie containing all the monolithic + flags. :param monolithicTrie: the trie containing all the monolithic flags. :param stream: a stream containing a list of signature patterns that define the subset. :return: the dict from signature to row. """ - dict = {} + dict_signature_to_row = {} for pattern in stream: pattern = pattern.rstrip() rows = monolithicTrie.getMatchingRows(pattern) for row in rows: - signature = row['signature'] - dict[signature] = row - return dict + signature = row["signature"] + dict_signature_to_row[signature] = row + return dict_signature_to_row + def read_signature_csv_from_stream_as_dict(stream): - """ - Read the csv contents from the stream into a dict. The first column is assumed to be the - signature and used as the key. The whole row is stored as the value. + """Read the csv contents from the stream into a dict. The first column is + assumed to be the signature and used as the key. + The whole row is stored as the value. :param stream: the csv contents to read :return: the dict from signature to row. """ - dict = {} + dict_signature_to_row = {} reader = dict_reader(stream) for row in reader: - signature = row['signature'] - dict[signature] = row - return dict + signature = row["signature"] + dict_signature_to_row[signature] = row + return dict_signature_to_row + def read_signature_csv_from_file_as_dict(csvFile): - """ - Read the csvFile into a dict. The first column is assumed to be the - signature and used as the key. The whole row is stored as the value. + """Read the csvFile into a dict. The first column is assumed to be the + signature and used as the key. + The whole row is stored as the value. :param csvFile: the csv file to read :return: the dict from signature to row. """ - with open(csvFile, 'r') as f: + with open(csvFile, "r") as f: return read_signature_csv_from_stream_as_dict(f) + def compare_signature_flags(monolithicFlagsDict, modularFlagsDict): - """ - Compare the signature flags between the two dicts. + """Compare the signature flags between the two dicts. :param monolithicFlagsDict: the dict containing the subset of the monolithic flags that should be equal to the modular flags. @@ -327,7 +344,8 @@ def compare_signature_flags(monolithicFlagsDict, modularFlagsDict): mismatchingSignatures = [] # Create a sorted set of all the signatures from both the monolithic and # modular dicts. - allSignatures = sorted(set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys()))) + allSignatures = sorted( + set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys()))) for signature in allSignatures: monolithicRow = monolithicFlagsDict.get(signature, {}) monolithicFlags = monolithicRow.get(None, []) @@ -337,13 +355,21 @@ def compare_signature_flags(monolithicFlagsDict, modularFlagsDict): else: modularFlags = ["blocked"] if monolithicFlags != modularFlags: - mismatchingSignatures.append((signature, modularFlags, monolithicFlags)) + mismatchingSignatures.append( + (signature, modularFlags, monolithicFlags)) return mismatchingSignatures + def main(argv): - args_parser = argparse.ArgumentParser(description='Verify that sets of hidden API flags are each a subset of the monolithic flag file.') - args_parser.add_argument('monolithicFlags', help='The monolithic flag file') - args_parser.add_argument('modularFlags', nargs=argparse.REMAINDER, help='Flags produced by individual bootclasspath_fragment modules') + args_parser = argparse.ArgumentParser( + description="Verify that sets of hidden API flags are each a subset of " + "the monolithic flag file." + ) + args_parser.add_argument("monolithicFlags", help="The monolithic flag file") + args_parser.add_argument( + "modularFlags", + nargs=argparse.REMAINDER, + help="Flags produced by individual bootclasspath_fragment modules") args = args_parser.parse_args(argv[1:]) # Read in all the flags into the trie @@ -358,9 +384,13 @@ def main(argv): parts = modularPair.split(":") modularFlagsPath = parts[0] modularPatternsPath = parts[1] - modularFlagsDict = read_signature_csv_from_file_as_dict(modularFlagsPath) - monolithicFlagsSubsetDict = extract_subset_from_monolithic_flags_as_dict_from_file(monolithicTrie, modularPatternsPath) - mismatchingSignatures = compare_signature_flags(monolithicFlagsSubsetDict, modularFlagsDict) + modularFlagsDict = read_signature_csv_from_file_as_dict( + modularFlagsPath) + monolithicFlagsSubsetDict = \ + extract_subset_from_monolithic_flags_as_dict_from_file( + monolithicTrie, modularPatternsPath) + mismatchingSignatures = compare_signature_flags( + monolithicFlagsSubsetDict, modularFlagsDict) if mismatchingSignatures: failed = True print("ERROR: Hidden API flags are inconsistent:") @@ -369,11 +399,12 @@ def main(argv): for mismatch in mismatchingSignatures: signature = mismatch[0] print() - print("< " + ",".join([signature]+ mismatch[1])) - print("> " + ",".join([signature]+ mismatch[2])) + print("< " + ",".join([signature] + mismatch[1])) + print("> " + ",".join([signature] + mismatch[2])) if failed: sys.exit(1) + if __name__ == "__main__": main(sys.argv) diff --git a/scripts/hiddenapi/verify_overlaps_test.py b/scripts/hiddenapi/verify_overlaps_test.py index 00c0611d6..22a1cdf6d 100755 --- a/scripts/hiddenapi/verify_overlaps_test.py +++ b/scripts/hiddenapi/verify_overlaps_test.py @@ -13,12 +13,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Unit tests for verify_overlaps_test.py.""" import io import unittest -from verify_overlaps import * +from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import + class TestSignatureToElements(unittest.TestCase): @@ -34,8 +34,10 @@ class TestSignatureToElements(unittest.TestCase): 'class:1', 'member:<init>()V', ] - self.assertEqual(expected, self.signatureToElements( - "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V")) + self.assertEqual( + expected, + self.signatureToElements( + 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V')) def test_signatureToElements_2(self): expected = [ @@ -44,8 +46,9 @@ class TestSignatureToElements(unittest.TestCase): 'class:Object', 'member:hashCode()I', ] - self.assertEqual(expected, self.signatureToElements( - "Ljava/lang/Object;->hashCode()I")) + self.assertEqual( + expected, + self.signatureToElements('Ljava/lang/Object;->hashCode()I')) def test_signatureToElements_3(self): expected = [ @@ -56,39 +59,46 @@ class TestSignatureToElements(unittest.TestCase): 'class:ExternalSyntheticLambda0', 'member:<init>(Ljava/lang/CharSequence;)V', ] - self.assertEqual(expected, self.signatureToElements( - "Ljava/lang/CharSequence$$ExternalSyntheticLambda0;" - "-><init>(Ljava/lang/CharSequence;)V")) + self.assertEqual( + expected, + self.signatureToElements( + 'Ljava/lang/CharSequence$$ExternalSyntheticLambda0;' + '-><init>(Ljava/lang/CharSequence;)V')) +#pylint: disable=line-too-long class TestDetectOverlaps(unittest.TestCase): - def read_flag_trie_from_string(self, csv): - with io.StringIO(csv) as f: + def read_flag_trie_from_string(self, csvdata): + with io.StringIO(csvdata) as f: return read_flag_trie_from_stream(f) - def read_signature_csv_from_string_as_dict(self, csv): - with io.StringIO(csv) as f: + def read_signature_csv_from_string_as_dict(self, csvdata): + with io.StringIO(csvdata) as f: return read_signature_csv_from_stream_as_dict(f) - def extract_subset_from_monolithic_flags_as_dict_from_string(self, monolithic, patterns): + def extract_subset_from_monolithic_flags_as_dict_from_string( + self, monolithic, patterns): with io.StringIO(patterns) as f: - return extract_subset_from_monolithic_flags_as_dict_from_stream(monolithic, f) + return extract_subset_from_monolithic_flags_as_dict_from_stream( + monolithic, f) - extractInput = ''' + extractInput = """ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api Ljava/lang/Object;->toString()Ljava/lang/String;,blocked Ljava/util/zip/ZipFile;-><clinit>()V,blocked Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;,blocked Ljava/lang/Character;->serialVersionUID:J,sdk Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked -''' +""" def test_extract_subset_signature(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'Ljava/lang/Object;->hashCode()I' - subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) + subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) expected = { 'Ljava/lang/Object;->hashCode()I': { None: ['public-api', 'system-api', 'test-api'], @@ -98,11 +108,13 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked self.assertEqual(expected, subset) def test_extract_subset_class(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'java/lang/Object' - subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) + subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) expected = { 'Ljava/lang/Object;->hashCode()I': { None: ['public-api', 'system-api', 'test-api'], @@ -116,16 +128,20 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked self.assertEqual(expected, subset) def test_extract_subset_outer_class(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'java/lang/Character' - subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) + subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': + { + None: ['blocked'], + 'signature': + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', + }, 'Ljava/lang/Character;->serialVersionUID:J': { None: ['sdk'], 'signature': 'Ljava/lang/Character;->serialVersionUID:J', @@ -134,30 +150,38 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked self.assertEqual(expected, subset) def test_extract_subset_nested_class(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'java/lang/Character$UnicodeScript' - subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) + subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': + { + None: ['blocked'], + 'signature': + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', + }, } self.assertEqual(expected, subset) def test_extract_subset_package(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'java/lang/*' - subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) + subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': + { + None: ['blocked'], + 'signature': + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', + }, 'Ljava/lang/Character;->serialVersionUID:J': { None: ['sdk'], 'signature': 'Ljava/lang/Character;->serialVersionUID:J', @@ -178,16 +202,20 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked self.assertEqual(expected, subset) def test_extract_subset_recursive_package(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'java/**' - subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) + subset = self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) expected = { - 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': { - None: ['blocked'], - 'signature': 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', - }, + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;': + { + None: ['blocked'], + 'signature': + 'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;', + }, 'Ljava/lang/Character;->serialVersionUID:J': { None: ['sdk'], 'signature': 'Ljava/lang/Character;->serialVersionUID:J', @@ -212,47 +240,53 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked self.assertEqual(expected, subset) def test_extract_subset_invalid_pattern_wildcard_and_member(self): - monolithic = self.read_flag_trie_from_string(TestDetectOverlaps.extractInput) + monolithic = self.read_flag_trie_from_string( + TestDetectOverlaps.extractInput) patterns = 'Ljava/lang/*;->hashCode()I' with self.assertRaises(Exception) as context: - self.extract_subset_from_monolithic_flags_as_dict_from_string(monolithic, patterns) - self.assertTrue("contains wildcard * and member signature hashCode()I" in str(context.exception)) + self.extract_subset_from_monolithic_flags_as_dict_from_string( + monolithic, patterns) + self.assertTrue('contains wildcard * and member signature hashCode()I' + in str(context.exception)) def test_read_trie_duplicate(self): with self.assertRaises(Exception) as context: - self.read_flag_trie_from_string(''' + self.read_flag_trie_from_string(""" Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api Ljava/lang/Object;->hashCode()I,blocked -''') - self.assertTrue("Duplicate signature: Ljava/lang/Object;->hashCode()I" in str(context.exception)) +""") + self.assertTrue('Duplicate signature: Ljava/lang/Object;->hashCode()I' + in str(context.exception)) def test_read_trie_missing_member(self): with self.assertRaises(Exception) as context: - self.read_flag_trie_from_string(''' + self.read_flag_trie_from_string(""" Ljava/lang/Object,public-api,system-api,test-api -''') - self.assertTrue("Invalid signature: Ljava/lang/Object, does not identify a specific member" in str(context.exception)) +""") + self.assertTrue( + 'Invalid signature: Ljava/lang/Object, does not identify a specific member' + in str(context.exception)) def test_match(self): - monolithic = self.read_signature_csv_from_string_as_dict(''' + monolithic = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api -''') - modular = self.read_signature_csv_from_string_as_dict(''' +""") + modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api -''') +""") mismatches = compare_signature_flags(monolithic, modular) expected = [] self.assertEqual(expected, mismatches) def test_mismatch_overlapping_flags(self): - monolithic = self.read_signature_csv_from_string_as_dict(''' + monolithic = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api -''') - modular = self.read_signature_csv_from_string_as_dict(''' +""") + modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api -''') +""") mismatches = compare_signature_flags(monolithic, modular) expected = [ ( @@ -263,14 +297,13 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api ] self.assertEqual(expected, mismatches) - def test_mismatch_monolithic_blocked(self): - monolithic = self.read_signature_csv_from_string_as_dict(''' + monolithic = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,blocked -''') - modular = self.read_signature_csv_from_string_as_dict(''' +""") + modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api -''') +""") mismatches = compare_signature_flags(monolithic, modular) expected = [ ( @@ -282,12 +315,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api self.assertEqual(expected, mismatches) def test_mismatch_modular_blocked(self): - monolithic = self.read_signature_csv_from_string_as_dict(''' + monolithic = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api -''') - modular = self.read_signature_csv_from_string_as_dict(''' +""") + modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,blocked -''') +""") mismatches = compare_signature_flags(monolithic, modular) expected = [ ( @@ -300,9 +333,9 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked def test_match_treat_missing_from_modular_as_blocked(self): monolithic = self.read_signature_csv_from_string_as_dict('') - modular = self.read_signature_csv_from_string_as_dict(''' + modular = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api -''') +""") mismatches = compare_signature_flags(monolithic, modular) expected = [ ( @@ -314,9 +347,9 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api self.assertEqual(expected, mismatches) def test_mismatch_treat_missing_from_modular_as_blocked(self): - monolithic = self.read_signature_csv_from_string_as_dict(''' + monolithic = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api -''') +""") modular = {} mismatches = compare_signature_flags(monolithic, modular) expected = [ @@ -329,13 +362,14 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api self.assertEqual(expected, mismatches) def test_blocked_missing_from_modular(self): - monolithic = self.read_signature_csv_from_string_as_dict(''' + monolithic = self.read_signature_csv_from_string_as_dict(""" Ljava/lang/Object;->hashCode()I,blocked -''') +""") modular = {} mismatches = compare_signature_flags(monolithic, modular) expected = [] self.assertEqual(expected, mismatches) +#pylint: enable=line-too-long if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/scripts/manifest_check.py b/scripts/manifest_check.py index 4ef4399ca..71fe358ff 100755 --- a/scripts/manifest_check.py +++ b/scripts/manifest_check.py @@ -25,7 +25,6 @@ import subprocess import sys from xml.dom import minidom - from manifest import android_ns from manifest import get_children_with_tag from manifest import parse_manifest @@ -33,49 +32,61 @@ from manifest import write_xml class ManifestMismatchError(Exception): - pass + pass def parse_args(): - """Parse commandline arguments.""" - - parser = argparse.ArgumentParser() - parser.add_argument('--uses-library', dest='uses_libraries', - action='append', - help='specify uses-library entries known to the build system') - parser.add_argument('--optional-uses-library', - dest='optional_uses_libraries', - action='append', - help='specify uses-library entries known to the build system with required:false') - parser.add_argument('--enforce-uses-libraries', - dest='enforce_uses_libraries', - action='store_true', - help='check the uses-library entries known to the build system against the manifest') - parser.add_argument('--enforce-uses-libraries-relax', - dest='enforce_uses_libraries_relax', - action='store_true', - help='do not fail immediately, just save the error message to file') - parser.add_argument('--enforce-uses-libraries-status', - dest='enforce_uses_libraries_status', - help='output file to store check status (error message)') - parser.add_argument('--extract-target-sdk-version', - dest='extract_target_sdk_version', - action='store_true', - help='print the targetSdkVersion from the manifest') - parser.add_argument('--dexpreopt-config', - dest='dexpreopt_configs', - action='append', - help='a paths to a dexpreopt.config of some library') - parser.add_argument('--aapt', - dest='aapt', - help='path to aapt executable') - parser.add_argument('--output', '-o', dest='output', help='output AndroidManifest.xml file') - parser.add_argument('input', help='input AndroidManifest.xml file') - return parser.parse_args() + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument( + '--uses-library', + dest='uses_libraries', + action='append', + help='specify uses-library entries known to the build system') + parser.add_argument( + '--optional-uses-library', + dest='optional_uses_libraries', + action='append', + help='specify uses-library entries known to the build system with ' + 'required:false' + ) + parser.add_argument( + '--enforce-uses-libraries', + dest='enforce_uses_libraries', + action='store_true', + help='check the uses-library entries known to the build system against ' + 'the manifest' + ) + parser.add_argument( + '--enforce-uses-libraries-relax', + dest='enforce_uses_libraries_relax', + action='store_true', + help='do not fail immediately, just save the error message to file') + parser.add_argument( + '--enforce-uses-libraries-status', + dest='enforce_uses_libraries_status', + help='output file to store check status (error message)') + parser.add_argument( + '--extract-target-sdk-version', + dest='extract_target_sdk_version', + action='store_true', + help='print the targetSdkVersion from the manifest') + parser.add_argument( + '--dexpreopt-config', + dest='dexpreopt_configs', + action='append', + help='a paths to a dexpreopt.config of some library') + parser.add_argument('--aapt', dest='aapt', help='path to aapt executable') + parser.add_argument( + '--output', '-o', dest='output', help='output AndroidManifest.xml file') + parser.add_argument('input', help='input AndroidManifest.xml file') + return parser.parse_args() def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): - """Verify that the <uses-library> tags in the manifest match those provided + """Verify that the <uses-library> tags in the manifest match those provided + by the build system. Args: @@ -84,274 +95,294 @@ def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): optional: optional libs known to the build system relax: if true, suppress error on mismatch and just write it to file is_apk: if the manifest comes from an APK or an XML file - """ - if is_apk: - manifest_required, manifest_optional, tags = extract_uses_libs_apk(manifest) - else: - manifest_required, manifest_optional, tags = extract_uses_libs_xml(manifest) - - # Trim namespace component. Normally Soong does that automatically when it - # handles module names specified in Android.bp properties. However not all - # <uses-library> entries in the manifest correspond to real modules: some of - # the optional libraries may be missing at build time. Therefor this script - # accepts raw module names as spelled in Android.bp/Amdroid.mk and trims the - # optional namespace part manually. - required = trim_namespace_parts(required) - optional = trim_namespace_parts(optional) - - if manifest_required == required and manifest_optional == optional: - return None - - errmsg = ''.join([ - 'mismatch in the <uses-library> tags between the build system and the ' - 'manifest:\n', - '\t- required libraries in build system: [%s]\n' % ', '.join(required), - '\t vs. in the manifest: [%s]\n' % ', '.join(manifest_required), - '\t- optional libraries in build system: [%s]\n' % ', '.join(optional), - '\t vs. in the manifest: [%s]\n' % ', '.join(manifest_optional), - '\t- tags in the manifest (%s):\n' % path, - '\t\t%s\n' % '\t\t'.join(tags), - 'note: the following options are available:\n', - '\t- to temporarily disable the check on command line, rebuild with ', - 'RELAX_USES_LIBRARY_CHECK=true (this will set compiler filter "verify" ', - 'and disable AOT-compilation in dexpreopt)\n', - '\t- to temporarily disable the check for the whole product, set ', - 'PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true in the product makefiles\n', - '\t- to fix the check, make build system properties coherent with the ' - 'manifest\n', - '\t- see build/make/Changes.md for details\n']) - - if not relax: - raise ManifestMismatchError(errmsg) - - return errmsg - - -MODULE_NAMESPACE = re.compile("^//[^:]+:") - -def trim_namespace_parts(modules): - """Trim the namespace part of each module, if present. Leave only the name.""" - - trimmed = [] - for module in modules: - trimmed.append(MODULE_NAMESPACE.sub('', module)) - return trimmed - - -def extract_uses_libs_apk(badging): - """Extract <uses-library> tags from the manifest of an APK.""" - - pattern = re.compile("^uses-library(-not-required)?:'(.*)'$", re.MULTILINE) - - required = [] - optional = [] - lines = [] - for match in re.finditer(pattern, badging): - lines.append(match.group(0)) - libname = match.group(2) - if match.group(1) == None: - required.append(libname) + """ + if is_apk: + manifest_required, manifest_optional, tags = extract_uses_libs_apk( + manifest) else: - optional.append(libname) - - required = first_unique_elements(required) - optional = first_unique_elements(optional) - tags = first_unique_elements(lines) - return required, optional, tags + manifest_required, manifest_optional, tags = extract_uses_libs_xml( + manifest) + + # Trim namespace component. Normally Soong does that automatically when it + # handles module names specified in Android.bp properties. However not all + # <uses-library> entries in the manifest correspond to real modules: some of + # the optional libraries may be missing at build time. Therefor this script + # accepts raw module names as spelled in Android.bp/Amdroid.mk and trims the + # optional namespace part manually. + required = trim_namespace_parts(required) + optional = trim_namespace_parts(optional) + + if manifest_required == required and manifest_optional == optional: + return None + + #pylint: disable=line-too-long + errmsg = ''.join([ + 'mismatch in the <uses-library> tags between the build system and the ' + 'manifest:\n', + '\t- required libraries in build system: [%s]\n' % ', '.join(required), + '\t vs. in the manifest: [%s]\n' % + ', '.join(manifest_required), + '\t- optional libraries in build system: [%s]\n' % ', '.join(optional), + '\t vs. in the manifest: [%s]\n' % + ', '.join(manifest_optional), + '\t- tags in the manifest (%s):\n' % path, + '\t\t%s\n' % '\t\t'.join(tags), + 'note: the following options are available:\n', + '\t- to temporarily disable the check on command line, rebuild with ', + 'RELAX_USES_LIBRARY_CHECK=true (this will set compiler filter "verify" ', + 'and disable AOT-compilation in dexpreopt)\n', + '\t- to temporarily disable the check for the whole product, set ', + 'PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true in the product makefiles\n', + '\t- to fix the check, make build system properties coherent with the ' + 'manifest\n', '\t- see build/make/Changes.md for details\n' + ]) + #pylint: enable=line-too-long + + if not relax: + raise ManifestMismatchError(errmsg) + + return errmsg + + +MODULE_NAMESPACE = re.compile('^//[^:]+:') -def extract_uses_libs_xml(xml): - """Extract <uses-library> tags from the manifest.""" - - manifest = parse_manifest(xml) - elems = get_children_with_tag(manifest, 'application') - application = elems[0] if len(elems) == 1 else None - if len(elems) > 1: - raise RuntimeError('found multiple <application> tags') - elif not elems: - if uses_libraries or optional_uses_libraries: - raise ManifestMismatchError('no <application> tag found') - return +def trim_namespace_parts(modules): + """Trim the namespace part of each module, if present. - libs = get_children_with_tag(application, 'uses-library') + Leave only the name. + """ - required = [uses_library_name(x) for x in libs if uses_library_required(x)] - optional = [uses_library_name(x) for x in libs if not uses_library_required(x)] + trimmed = [] + for module in modules: + trimmed.append(MODULE_NAMESPACE.sub('', module)) + return trimmed - # render <uses-library> tags as XML for a pretty error message - tags = [] - for lib in libs: - tags.append(lib.toprettyxml()) - required = first_unique_elements(required) - optional = first_unique_elements(optional) - tags = first_unique_elements(tags) - return required, optional, tags +def extract_uses_libs_apk(badging): + """Extract <uses-library> tags from the manifest of an APK.""" + + pattern = re.compile("^uses-library(-not-required)?:'(.*)'$", re.MULTILINE) + + required = [] + optional = [] + lines = [] + for match in re.finditer(pattern, badging): + lines.append(match.group(0)) + libname = match.group(2) + if match.group(1) is None: + required.append(libname) + else: + optional.append(libname) + + required = first_unique_elements(required) + optional = first_unique_elements(optional) + tags = first_unique_elements(lines) + return required, optional, tags + + +def extract_uses_libs_xml(xml): #pylint: disable=inconsistent-return-statements + """Extract <uses-library> tags from the manifest.""" + + manifest = parse_manifest(xml) + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: #pylint: disable=no-else-raise + raise RuntimeError('found multiple <application> tags') + elif not elems: + if uses_libraries or optional_uses_libraries: #pylint: disable=undefined-variable + raise ManifestMismatchError('no <application> tag found') + return + + libs = get_children_with_tag(application, 'uses-library') + + required = [uses_library_name(x) for x in libs if uses_library_required(x)] + optional = [ + uses_library_name(x) for x in libs if not uses_library_required(x) + ] + + # render <uses-library> tags as XML for a pretty error message + tags = [] + for lib in libs: + tags.append(lib.toprettyxml()) + + required = first_unique_elements(required) + optional = first_unique_elements(optional) + tags = first_unique_elements(tags) + return required, optional, tags def first_unique_elements(l): - result = [] - [result.append(x) for x in l if x not in result] - return result + result = [] + for x in l: + if x not in result: + result.append(x) + return result def uses_library_name(lib): - """Extract the name attribute of a uses-library tag. + """Extract the name attribute of a uses-library tag. Args: lib: a <uses-library> tag. - """ - name = lib.getAttributeNodeNS(android_ns, 'name') - return name.value if name is not None else "" + """ + name = lib.getAttributeNodeNS(android_ns, 'name') + return name.value if name is not None else '' def uses_library_required(lib): - """Extract the required attribute of a uses-library tag. + """Extract the required attribute of a uses-library tag. Args: lib: a <uses-library> tag. - """ - required = lib.getAttributeNodeNS(android_ns, 'required') - return (required.value == 'true') if required is not None else True + """ + required = lib.getAttributeNodeNS(android_ns, 'required') + return (required.value == 'true') if required is not None else True -def extract_target_sdk_version(manifest, is_apk = False): - """Returns the targetSdkVersion from the manifest. +def extract_target_sdk_version(manifest, is_apk=False): + """Returns the targetSdkVersion from the manifest. Args: manifest: manifest (either parsed XML or aapt dump of APK) is_apk: if the manifest comes from an APK or an XML file - """ - if is_apk: - return extract_target_sdk_version_apk(manifest) - else: - return extract_target_sdk_version_xml(manifest) + """ + if is_apk: #pylint: disable=no-else-return + return extract_target_sdk_version_apk(manifest) + else: + return extract_target_sdk_version_xml(manifest) def extract_target_sdk_version_apk(badging): - """Extract targetSdkVersion tags from the manifest of an APK.""" + """Extract targetSdkVersion tags from the manifest of an APK.""" - pattern = re.compile("^targetSdkVersion?:'(.*)'$", re.MULTILINE) + pattern = re.compile("^targetSdkVersion?:'(.*)'$", re.MULTILINE) - for match in re.finditer(pattern, badging): - return match.group(1) + for match in re.finditer(pattern, badging): + return match.group(1) - raise RuntimeError('cannot find targetSdkVersion in the manifest') + raise RuntimeError('cannot find targetSdkVersion in the manifest') def extract_target_sdk_version_xml(xml): - """Extract targetSdkVersion tags from the manifest.""" + """Extract targetSdkVersion tags from the manifest.""" - manifest = parse_manifest(xml) + manifest = parse_manifest(xml) - # Get or insert the uses-sdk element - uses_sdk = get_children_with_tag(manifest, 'uses-sdk') - if len(uses_sdk) > 1: - raise RuntimeError('found multiple uses-sdk elements') - elif len(uses_sdk) == 0: - raise RuntimeError('missing uses-sdk element') + # Get or insert the uses-sdk element + uses_sdk = get_children_with_tag(manifest, 'uses-sdk') + if len(uses_sdk) > 1: #pylint: disable=no-else-raise + raise RuntimeError('found multiple uses-sdk elements') + elif len(uses_sdk) == 0: + raise RuntimeError('missing uses-sdk element') - uses_sdk = uses_sdk[0] + uses_sdk = uses_sdk[0] - min_attr = uses_sdk.getAttributeNodeNS(android_ns, 'minSdkVersion') - if min_attr is None: - raise RuntimeError('minSdkVersion is not specified') + min_attr = uses_sdk.getAttributeNodeNS(android_ns, 'minSdkVersion') + if min_attr is None: + raise RuntimeError('minSdkVersion is not specified') - target_attr = uses_sdk.getAttributeNodeNS(android_ns, 'targetSdkVersion') - if target_attr is None: - target_attr = min_attr + target_attr = uses_sdk.getAttributeNodeNS(android_ns, 'targetSdkVersion') + if target_attr is None: + target_attr = min_attr - return target_attr.value + return target_attr.value def load_dexpreopt_configs(configs): - """Load dexpreopt.config files and map module names to library names.""" - module_to_libname = {} + """Load dexpreopt.config files and map module names to library names.""" + module_to_libname = {} - if configs is None: - configs = [] + if configs is None: + configs = [] - for config in configs: - with open(config, 'r') as f: - contents = json.load(f) - module_to_libname[contents['Name']] = contents['ProvidesUsesLibrary'] + for config in configs: + with open(config, 'r') as f: + contents = json.load(f) + module_to_libname[contents['Name']] = contents['ProvidesUsesLibrary'] - return module_to_libname + return module_to_libname def translate_libnames(modules, module_to_libname): - """Translate module names into library names using the mapping.""" - if modules is None: - modules = [] + """Translate module names into library names using the mapping.""" + if modules is None: + modules = [] - libnames = [] - for name in modules: - if name in module_to_libname: - name = module_to_libname[name] - libnames.append(name) + libnames = [] + for name in modules: + if name in module_to_libname: + name = module_to_libname[name] + libnames.append(name) - return libnames + return libnames def main(): - """Program entry point.""" - try: - args = parse_args() + """Program entry point.""" + try: + args = parse_args() + + # The input can be either an XML manifest or an APK, they are parsed and + # processed in different ways. + is_apk = args.input.endswith('.apk') + if is_apk: + aapt = args.aapt if args.aapt is not None else 'aapt' + manifest = subprocess.check_output( + [aapt, 'dump', 'badging', args.input]) + else: + manifest = minidom.parse(args.input) + + if args.enforce_uses_libraries: + # Load dexpreopt.config files and build a mapping from module + # names to library names. This is necessary because build system + # addresses libraries by their module name (`uses_libs`, + # `optional_uses_libs`, `LOCAL_USES_LIBRARIES`, + # `LOCAL_OPTIONAL_LIBRARY_NAMES` all contain module names), while + # the manifest addresses libraries by their name. + mod_to_lib = load_dexpreopt_configs(args.dexpreopt_configs) + required = translate_libnames(args.uses_libraries, mod_to_lib) + optional = translate_libnames(args.optional_uses_libraries, + mod_to_lib) + + # Check if the <uses-library> lists in the build system agree with + # those in the manifest. Raise an exception on mismatch, unless the + # script was passed a special parameter to suppress exceptions. + errmsg = enforce_uses_libraries(manifest, required, optional, + args.enforce_uses_libraries_relax, + is_apk, args.input) + + # Create a status file that is empty on success, or contains an + # error message on failure. When exceptions are suppressed, + # dexpreopt command command will check file size to determine if + # the check has failed. + if args.enforce_uses_libraries_status: + with open(args.enforce_uses_libraries_status, 'w') as f: + if not errmsg is not None: + f.write('%s\n' % errmsg) + + if args.extract_target_sdk_version: + try: + print(extract_target_sdk_version(manifest, is_apk)) + except: #pylint: disable=bare-except + # Failed; don't crash, return "any" SDK version. This will + # result in dexpreopt not adding any compatibility libraries. + print(10000) + + if args.output: + # XML output is supposed to be written only when this script is + # invoked with XML input manifest, not with an APK. + if is_apk: + raise RuntimeError('cannot save APK manifest as XML') + + with open(args.output, 'wb') as f: + write_xml(f, manifest) + + # pylint: disable=broad-except + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) - # The input can be either an XML manifest or an APK, they are parsed and - # processed in different ways. - is_apk = args.input.endswith('.apk') - if is_apk: - aapt = args.aapt if args.aapt != None else "aapt" - manifest = subprocess.check_output([aapt, "dump", "badging", args.input]) - else: - manifest = minidom.parse(args.input) - - if args.enforce_uses_libraries: - # Load dexpreopt.config files and build a mapping from module names to - # library names. This is necessary because build system addresses - # libraries by their module name (`uses_libs`, `optional_uses_libs`, - # `LOCAL_USES_LIBRARIES`, `LOCAL_OPTIONAL_LIBRARY_NAMES` all contain - # module names), while the manifest addresses libraries by their name. - mod_to_lib = load_dexpreopt_configs(args.dexpreopt_configs) - required = translate_libnames(args.uses_libraries, mod_to_lib) - optional = translate_libnames(args.optional_uses_libraries, mod_to_lib) - - # Check if the <uses-library> lists in the build system agree with those - # in the manifest. Raise an exception on mismatch, unless the script was - # passed a special parameter to suppress exceptions. - errmsg = enforce_uses_libraries(manifest, required, optional, - args.enforce_uses_libraries_relax, is_apk, args.input) - - # Create a status file that is empty on success, or contains an error - # message on failure. When exceptions are suppressed, dexpreopt command - # command will check file size to determine if the check has failed. - if args.enforce_uses_libraries_status: - with open(args.enforce_uses_libraries_status, 'w') as f: - if not errmsg == None: - f.write("%s\n" % errmsg) - - if args.extract_target_sdk_version: - try: - print(extract_target_sdk_version(manifest, is_apk)) - except: - # Failed; don't crash, return "any" SDK version. This will result in - # dexpreopt not adding any compatibility libraries. - print(10000) - - if args.output: - # XML output is supposed to be written only when this script is invoked - # with XML input manifest, not with an APK. - if is_apk: - raise RuntimeError('cannot save APK manifest as XML') - - with open(args.output, 'wb') as f: - write_xml(f, manifest) - - # pylint: disable=broad-except - except Exception as err: - print('error: ' + str(err), file=sys.stderr) - sys.exit(-1) if __name__ == '__main__': - main() + main() diff --git a/scripts/manifest_check_test.py b/scripts/manifest_check_test.py index e3e8ac468..3be7a30bf 100755 --- a/scripts/manifest_check_test.py +++ b/scripts/manifest_check_test.py @@ -26,202 +26,235 @@ sys.dont_write_bytecode = True def uses_library_xml(name, attr=''): - return '<uses-library android:name="%s"%s />' % (name, attr) + return '<uses-library android:name="%s"%s />' % (name, attr) def required_xml(value): - return ' android:required="%s"' % ('true' if value else 'false') + return ' android:required="%s"' % ('true' if value else 'false') def uses_library_apk(name, sfx=''): - return "uses-library%s:'%s'" % (sfx, name) + return "uses-library%s:'%s'" % (sfx, name) def required_apk(value): - return '' if value else '-not-required' + return '' if value else '-not-required' class EnforceUsesLibrariesTest(unittest.TestCase): - """Unit tests for add_extract_native_libs function.""" - - def run_test(self, xml, apk, uses_libraries=[], optional_uses_libraries=[]): - doc = minidom.parseString(xml) - try: - relax = False - manifest_check.enforce_uses_libraries(doc, uses_libraries, - optional_uses_libraries, relax, False, 'path/to/X/AndroidManifest.xml') - manifest_check.enforce_uses_libraries(apk, uses_libraries, - optional_uses_libraries, relax, True, 'path/to/X/X.apk') - return True - except manifest_check.ManifestMismatchError: - return False - - xml_tmpl = ( - '<?xml version="1.0" encoding="utf-8"?>\n' - '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' - ' <application>\n' - ' %s\n' - ' </application>\n' - '</manifest>\n') - - apk_tmpl = ( - "package: name='com.google.android.something' versionCode='100'\n" - "sdkVersion:'29'\n" - "targetSdkVersion:'29'\n" - "uses-permission: name='android.permission.ACCESS_NETWORK_STATE'\n" - "%s\n" - "densities: '160' '240' '320' '480' '640' '65534") - - def test_uses_library(self): - xml = self.xml_tmpl % (uses_library_xml('foo')) - apk = self.apk_tmpl % (uses_library_apk('foo')) - matches = self.run_test(xml, apk, uses_libraries=['foo']) - self.assertTrue(matches) - - def test_uses_library_required(self): - xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(True))) - apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(True))) - matches = self.run_test(xml, apk, uses_libraries=['foo']) - self.assertTrue(matches) - - def test_optional_uses_library(self): - xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(False))) - apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(False))) - matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) - self.assertTrue(matches) - - def test_expected_uses_library(self): - xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(False))) - apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(False))) - matches = self.run_test(xml, apk, uses_libraries=['foo']) - self.assertFalse(matches) - - def test_expected_optional_uses_library(self): - xml = self.xml_tmpl % (uses_library_xml('foo')) - apk = self.apk_tmpl % (uses_library_apk('foo')) - matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) - self.assertFalse(matches) - - def test_missing_uses_library(self): - xml = self.xml_tmpl % ('') - apk = self.apk_tmpl % ('') - matches = self.run_test(xml, apk, uses_libraries=['foo']) - self.assertFalse(matches) - - def test_missing_optional_uses_library(self): - xml = self.xml_tmpl % ('') - apk = self.apk_tmpl % ('') - matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) - self.assertFalse(matches) - - def test_extra_uses_library(self): - xml = self.xml_tmpl % (uses_library_xml('foo')) - apk = self.apk_tmpl % (uses_library_xml('foo')) - matches = self.run_test(xml, apk) - self.assertFalse(matches) - - def test_extra_optional_uses_library(self): - xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(False))) - apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(False))) - matches = self.run_test(xml, apk) - self.assertFalse(matches) - - def test_multiple_uses_library(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo'), - uses_library_xml('bar')])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo'), - uses_library_apk('bar')])) - matches = self.run_test(xml, apk, uses_libraries=['foo', 'bar']) - self.assertTrue(matches) - - def test_multiple_optional_uses_library(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo', required_xml(False)), - uses_library_xml('bar', required_xml(False))])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo', required_apk(False)), - uses_library_apk('bar', required_apk(False))])) - matches = self.run_test(xml, apk, optional_uses_libraries=['foo', 'bar']) - self.assertTrue(matches) - - def test_order_uses_library(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo'), - uses_library_xml('bar')])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo'), - uses_library_apk('bar')])) - matches = self.run_test(xml, apk, uses_libraries=['bar', 'foo']) - self.assertFalse(matches) - - def test_order_optional_uses_library(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo', required_xml(False)), - uses_library_xml('bar', required_xml(False))])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo', required_apk(False)), - uses_library_apk('bar', required_apk(False))])) - matches = self.run_test(xml, apk, optional_uses_libraries=['bar', 'foo']) - self.assertFalse(matches) - - def test_duplicate_uses_library(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo'), - uses_library_xml('foo')])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo'), - uses_library_apk('foo')])) - matches = self.run_test(xml, apk, uses_libraries=['foo']) - self.assertTrue(matches) - - def test_duplicate_optional_uses_library(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo', required_xml(False)), - uses_library_xml('foo', required_xml(False))])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo', required_apk(False)), - uses_library_apk('foo', required_apk(False))])) - matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) - self.assertTrue(matches) - - def test_mixed(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo'), - uses_library_xml('bar', required_xml(False))])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo'), - uses_library_apk('bar', required_apk(False))])) - matches = self.run_test(xml, apk, uses_libraries=['foo'], - optional_uses_libraries=['bar']) - self.assertTrue(matches) - - def test_mixed_with_namespace(self): - xml = self.xml_tmpl % ('\n'.join([uses_library_xml('foo'), - uses_library_xml('bar', required_xml(False))])) - apk = self.apk_tmpl % ('\n'.join([uses_library_apk('foo'), - uses_library_apk('bar', required_apk(False))])) - matches = self.run_test(xml, apk, uses_libraries=['//x/y/z:foo'], - optional_uses_libraries=['//x/y/z:bar']) - self.assertTrue(matches) + """Unit tests for add_extract_native_libs function.""" + + def run_test(self, xml, apk, uses_libraries=[], optional_uses_libraries=[]): #pylint: disable=dangerous-default-value + doc = minidom.parseString(xml) + try: + relax = False + manifest_check.enforce_uses_libraries( + doc, uses_libraries, optional_uses_libraries, relax, False, + 'path/to/X/AndroidManifest.xml') + manifest_check.enforce_uses_libraries(apk, uses_libraries, + optional_uses_libraries, + relax, True, + 'path/to/X/X.apk') + return True + except manifest_check.ManifestMismatchError: + return False + + xml_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n<manifest ' + 'xmlns:android="http://schemas.android.com/apk/res/android">\n ' + '<application>\n %s\n </application>\n</manifest>\n') + + apk_tmpl = ( + "package: name='com.google.android.something' versionCode='100'\n" + "sdkVersion:'29'\n" + "targetSdkVersion:'29'\n" + "uses-permission: name='android.permission.ACCESS_NETWORK_STATE'\n" + '%s\n' + "densities: '160' '240' '320' '480' '640' '65534") + + def test_uses_library(self): + xml = self.xml_tmpl % (uses_library_xml('foo')) + apk = self.apk_tmpl % (uses_library_apk('foo')) + matches = self.run_test(xml, apk, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_uses_library_required(self): + xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(True))) + apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(True))) + matches = self.run_test(xml, apk, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_optional_uses_library(self): + xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(False))) + apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(False))) + matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) + self.assertTrue(matches) + + def test_expected_uses_library(self): + xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(False))) + apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(False))) + matches = self.run_test(xml, apk, uses_libraries=['foo']) + self.assertFalse(matches) + + def test_expected_optional_uses_library(self): + xml = self.xml_tmpl % (uses_library_xml('foo')) + apk = self.apk_tmpl % (uses_library_apk('foo')) + matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) + self.assertFalse(matches) + + def test_missing_uses_library(self): + xml = self.xml_tmpl % ('') + apk = self.apk_tmpl % ('') + matches = self.run_test(xml, apk, uses_libraries=['foo']) + self.assertFalse(matches) + + def test_missing_optional_uses_library(self): + xml = self.xml_tmpl % ('') + apk = self.apk_tmpl % ('') + matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) + self.assertFalse(matches) + + def test_extra_uses_library(self): + xml = self.xml_tmpl % (uses_library_xml('foo')) + apk = self.apk_tmpl % (uses_library_xml('foo')) + matches = self.run_test(xml, apk) + self.assertFalse(matches) + + def test_extra_optional_uses_library(self): + xml = self.xml_tmpl % (uses_library_xml('foo', required_xml(False))) + apk = self.apk_tmpl % (uses_library_apk('foo', required_apk(False))) + matches = self.run_test(xml, apk) + self.assertFalse(matches) + + def test_multiple_uses_library(self): + xml = self.xml_tmpl % ('\n'.join( + [uses_library_xml('foo'), + uses_library_xml('bar')])) + apk = self.apk_tmpl % ('\n'.join( + [uses_library_apk('foo'), + uses_library_apk('bar')])) + matches = self.run_test(xml, apk, uses_libraries=['foo', 'bar']) + self.assertTrue(matches) + + def test_multiple_optional_uses_library(self): + xml = self.xml_tmpl % ('\n'.join([ + uses_library_xml('foo', required_xml(False)), + uses_library_xml('bar', required_xml(False)) + ])) + apk = self.apk_tmpl % ('\n'.join([ + uses_library_apk('foo', required_apk(False)), + uses_library_apk('bar', required_apk(False)) + ])) + matches = self.run_test( + xml, apk, optional_uses_libraries=['foo', 'bar']) + self.assertTrue(matches) + + def test_order_uses_library(self): + xml = self.xml_tmpl % ('\n'.join( + [uses_library_xml('foo'), + uses_library_xml('bar')])) + apk = self.apk_tmpl % ('\n'.join( + [uses_library_apk('foo'), + uses_library_apk('bar')])) + matches = self.run_test(xml, apk, uses_libraries=['bar', 'foo']) + self.assertFalse(matches) + + def test_order_optional_uses_library(self): + xml = self.xml_tmpl % ('\n'.join([ + uses_library_xml('foo', required_xml(False)), + uses_library_xml('bar', required_xml(False)) + ])) + apk = self.apk_tmpl % ('\n'.join([ + uses_library_apk('foo', required_apk(False)), + uses_library_apk('bar', required_apk(False)) + ])) + matches = self.run_test( + xml, apk, optional_uses_libraries=['bar', 'foo']) + self.assertFalse(matches) + + def test_duplicate_uses_library(self): + xml = self.xml_tmpl % ('\n'.join( + [uses_library_xml('foo'), + uses_library_xml('foo')])) + apk = self.apk_tmpl % ('\n'.join( + [uses_library_apk('foo'), + uses_library_apk('foo')])) + matches = self.run_test(xml, apk, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_duplicate_optional_uses_library(self): + xml = self.xml_tmpl % ('\n'.join([ + uses_library_xml('foo', required_xml(False)), + uses_library_xml('foo', required_xml(False)) + ])) + apk = self.apk_tmpl % ('\n'.join([ + uses_library_apk('foo', required_apk(False)), + uses_library_apk('foo', required_apk(False)) + ])) + matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) + self.assertTrue(matches) + + def test_mixed(self): + xml = self.xml_tmpl % ('\n'.join([ + uses_library_xml('foo'), + uses_library_xml('bar', required_xml(False)) + ])) + apk = self.apk_tmpl % ('\n'.join([ + uses_library_apk('foo'), + uses_library_apk('bar', required_apk(False)) + ])) + matches = self.run_test( + xml, apk, uses_libraries=['foo'], optional_uses_libraries=['bar']) + self.assertTrue(matches) + + def test_mixed_with_namespace(self): + xml = self.xml_tmpl % ('\n'.join([ + uses_library_xml('foo'), + uses_library_xml('bar', required_xml(False)) + ])) + apk = self.apk_tmpl % ('\n'.join([ + uses_library_apk('foo'), + uses_library_apk('bar', required_apk(False)) + ])) + matches = self.run_test( + xml, + apk, + uses_libraries=['//x/y/z:foo'], + optional_uses_libraries=['//x/y/z:bar']) + self.assertTrue(matches) class ExtractTargetSdkVersionTest(unittest.TestCase): - def run_test(self, xml, apk, version): - doc = minidom.parseString(xml) - v = manifest_check.extract_target_sdk_version(doc, is_apk=False) - self.assertEqual(v, version) - v = manifest_check.extract_target_sdk_version(apk, is_apk=True) - self.assertEqual(v, version) - - xml_tmpl = ( - '<?xml version="1.0" encoding="utf-8"?>\n' - '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' - ' <uses-sdk android:minSdkVersion="28" android:targetSdkVersion="%s" />\n' - '</manifest>\n') - - apk_tmpl = ( - "package: name='com.google.android.something' versionCode='100'\n" - "sdkVersion:'28'\n" - "targetSdkVersion:'%s'\n" - "uses-permission: name='android.permission.ACCESS_NETWORK_STATE'\n") - - def test_targert_sdk_version_28(self): - xml = self.xml_tmpl % "28" - apk = self.apk_tmpl % "28" - self.run_test(xml, apk, "28") - - def test_targert_sdk_version_29(self): - xml = self.xml_tmpl % "29" - apk = self.apk_tmpl % "29" - self.run_test(xml, apk, "29") + + def run_test(self, xml, apk, version): + doc = minidom.parseString(xml) + v = manifest_check.extract_target_sdk_version(doc, is_apk=False) + self.assertEqual(v, version) + v = manifest_check.extract_target_sdk_version(apk, is_apk=True) + self.assertEqual(v, version) + + xml_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n<manifest ' + 'xmlns:android="http://schemas.android.com/apk/res/android">\n ' + '<uses-sdk android:minSdkVersion="28" android:targetSdkVersion="%s" ' + '/>\n</manifest>\n') + + apk_tmpl = ( + "package: name='com.google.android.something' versionCode='100'\n" + "sdkVersion:'28'\n" + "targetSdkVersion:'%s'\n" + "uses-permission: name='android.permission.ACCESS_NETWORK_STATE'\n") + + def test_targert_sdk_version_28(self): + xml = self.xml_tmpl % '28' + apk = self.apk_tmpl % '28' + self.run_test(xml, apk, '28') + + def test_targert_sdk_version_29(self): + xml = self.xml_tmpl % '29' + apk = self.apk_tmpl % '29' + self.run_test(xml, apk, '29') + if __name__ == '__main__': - unittest.main(verbosity=2) + unittest.main(verbosity=2) diff --git a/sdk/Android.bp b/sdk/Android.bp index 368c03aec..0c9bf27fd 100644 --- a/sdk/Android.bp +++ b/sdk/Android.bp @@ -16,6 +16,7 @@ bootstrap_go_package { srcs: [ "bp.go", "exports.go", + "member_type.go", "sdk.go", "update.go", ], diff --git a/sdk/member_type.go b/sdk/member_type.go new file mode 100644 index 000000000..ee27c8687 --- /dev/null +++ b/sdk/member_type.go @@ -0,0 +1,164 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "reflect" + + "android/soong/android" + "github.com/google/blueprint/proptools" +) + +// Contains information about the sdk properties that list sdk members by type, e.g. +// Java_header_libs. +type sdkMemberTypeListProperty struct { + // getter for the list of member names + getter func(properties interface{}) []string + + // setter for the list of member names + setter func(properties interface{}, list []string) + + // the type of member referenced in the list + memberType android.SdkMemberType + + // the dependency tag used for items in this list that can be used to determine the memberType + // for a resolved dependency. + dependencyTag android.SdkMemberTypeDependencyTag +} + +func (p *sdkMemberTypeListProperty) propertyName() string { + return p.memberType.SdkPropertyName() +} + +// Cache of dynamically generated dynamicSdkMemberTypes objects. The key is the pointer +// to a slice of SdkMemberType instances held in android.SdkMemberTypes. +var dynamicSdkMemberTypesMap android.OncePer + +// A dynamically generated set of member list properties and associated structure type. +type dynamicSdkMemberTypes struct { + // The dynamically generated structure type. + // + // Contains one []string exported field for each android.SdkMemberTypes. The name of the field + // is the exported form of the value returned by SdkMemberType.SdkPropertyName(). + propertiesStructType reflect.Type + + // Information about each of the member type specific list properties. + memberTypeListProperties []*sdkMemberTypeListProperty + + memberTypeToProperty map[android.SdkMemberType]*sdkMemberTypeListProperty +} + +func (d *dynamicSdkMemberTypes) createMemberTypeListProperties() interface{} { + return reflect.New(d.propertiesStructType).Interface() +} + +func getDynamicSdkMemberTypes(registry *android.SdkMemberTypesRegistry) *dynamicSdkMemberTypes { + + // Get a key that uniquely identifies the registry contents. + key := registry.UniqueOnceKey() + + // Get the registered types. + registeredTypes := registry.RegisteredTypes() + + // Get the cached value, creating new instance if necessary. + return dynamicSdkMemberTypesMap.Once(key, func() interface{} { + return createDynamicSdkMemberTypes(registeredTypes) + }).(*dynamicSdkMemberTypes) +} + +// Create the dynamicSdkMemberTypes from the list of registered member types. +// +// A struct is created which contains one exported field per member type corresponding to +// the SdkMemberType.SdkPropertyName() value. +// +// A list of sdkMemberTypeListProperty instances is created, one per member type that provides: +// * a reference to the member type. +// * a getter for the corresponding field in the properties struct. +// * a dependency tag that identifies the member type of a resolved dependency. +// +func createDynamicSdkMemberTypes(sdkMemberTypes []android.SdkMemberType) *dynamicSdkMemberTypes { + + var listProperties []*sdkMemberTypeListProperty + memberTypeToProperty := map[android.SdkMemberType]*sdkMemberTypeListProperty{} + var fields []reflect.StructField + + // Iterate over the member types creating StructField and sdkMemberTypeListProperty objects. + nextFieldIndex := 0 + for _, memberType := range sdkMemberTypes { + + p := memberType.SdkPropertyName() + + var getter func(properties interface{}) []string + var setter func(properties interface{}, list []string) + if memberType.RequiresBpProperty() { + // Create a dynamic exported field for the member type's property. + fields = append(fields, reflect.StructField{ + Name: proptools.FieldNameForProperty(p), + Type: reflect.TypeOf([]string{}), + Tag: `android:"arch_variant"`, + }) + + // Copy the field index for use in the getter func as using the loop variable directly will + // cause all funcs to use the last value. + fieldIndex := nextFieldIndex + nextFieldIndex += 1 + + getter = func(properties interface{}) []string { + // The properties is expected to be of the following form (where + // <Module_types> is the name of an SdkMemberType.SdkPropertyName(). + // properties *struct {<Module_types> []string, ....} + // + // Although it accesses the field by index the following reflection code is equivalent to: + // *properties.<Module_types> + // + list := reflect.ValueOf(properties).Elem().Field(fieldIndex).Interface().([]string) + return list + } + + setter = func(properties interface{}, list []string) { + // The properties is expected to be of the following form (where + // <Module_types> is the name of an SdkMemberType.SdkPropertyName(). + // properties *struct {<Module_types> []string, ....} + // + // Although it accesses the field by index the following reflection code is equivalent to: + // *properties.<Module_types> = list + // + reflect.ValueOf(properties).Elem().Field(fieldIndex).Set(reflect.ValueOf(list)) + } + } + + // Create an sdkMemberTypeListProperty for the member type. + memberListProperty := &sdkMemberTypeListProperty{ + getter: getter, + setter: setter, + memberType: memberType, + + // Dependencies added directly from member properties are always exported. + dependencyTag: android.DependencyTagForSdkMemberType(memberType, true), + } + + memberTypeToProperty[memberType] = memberListProperty + listProperties = append(listProperties, memberListProperty) + } + + // Create a dynamic struct from the collated fields. + propertiesStructType := reflect.StructOf(fields) + + return &dynamicSdkMemberTypes{ + memberTypeListProperties: listProperties, + memberTypeToProperty: memberTypeToProperty, + propertiesStructType: propertiesStructType, + } +} diff --git a/sdk/sdk.go b/sdk/sdk.go index a972f31da..6dea752b3 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -17,7 +17,6 @@ package sdk import ( "fmt" "io" - "reflect" "strconv" "github.com/google/blueprint" @@ -50,7 +49,7 @@ type sdk struct { // The dynamically generated information about the registered SdkMemberType dynamicSdkMemberTypes *dynamicSdkMemberTypes - // The dynamically created instance of the properties struct containing the sdk member + // The dynamically created instance of the properties struct containing the sdk member type // list properties, e.g. java_libs. dynamicMemberTypeListProperties interface{} @@ -95,148 +94,6 @@ type sdkProperties struct { Prebuilt_visibility []string } -// Contains information about the sdk properties that list sdk members, e.g. -// Java_header_libs. -type sdkMemberListProperty struct { - // getter for the list of member names - getter func(properties interface{}) []string - - // setter for the list of member names - setter func(properties interface{}, list []string) - - // the type of member referenced in the list - memberType android.SdkMemberType - - // the dependency tag used for items in this list that can be used to determine the memberType - // for a resolved dependency. - dependencyTag android.SdkMemberTypeDependencyTag -} - -func (p *sdkMemberListProperty) propertyName() string { - return p.memberType.SdkPropertyName() -} - -// Cache of dynamically generated dynamicSdkMemberTypes objects. The key is the pointer -// to a slice of SdkMemberType instances held in android.SdkMemberTypes. -var dynamicSdkMemberTypesMap android.OncePer - -// A dynamically generated set of member list properties and associated structure type. -type dynamicSdkMemberTypes struct { - // The dynamically generated structure type. - // - // Contains one []string exported field for each android.SdkMemberTypes. The name of the field - // is the exported form of the value returned by SdkMemberType.SdkPropertyName(). - propertiesStructType reflect.Type - - // Information about each of the member type specific list properties. - memberListProperties []*sdkMemberListProperty - - memberTypeToProperty map[android.SdkMemberType]*sdkMemberListProperty -} - -func (d *dynamicSdkMemberTypes) createMemberListProperties() interface{} { - return reflect.New(d.propertiesStructType).Interface() -} - -func getDynamicSdkMemberTypes(registry *android.SdkMemberTypesRegistry) *dynamicSdkMemberTypes { - - // Get a key that uniquely identifies the registry contents. - key := registry.UniqueOnceKey() - - // Get the registered types. - registeredTypes := registry.RegisteredTypes() - - // Get the cached value, creating new instance if necessary. - return dynamicSdkMemberTypesMap.Once(key, func() interface{} { - return createDynamicSdkMemberTypes(registeredTypes) - }).(*dynamicSdkMemberTypes) -} - -// Create the dynamicSdkMemberTypes from the list of registered member types. -// -// A struct is created which contains one exported field per member type corresponding to -// the SdkMemberType.SdkPropertyName() value. -// -// A list of sdkMemberListProperty instances is created, one per member type that provides: -// * a reference to the member type. -// * a getter for the corresponding field in the properties struct. -// * a dependency tag that identifies the member type of a resolved dependency. -// -func createDynamicSdkMemberTypes(sdkMemberTypes []android.SdkMemberType) *dynamicSdkMemberTypes { - - var listProperties []*sdkMemberListProperty - memberTypeToProperty := map[android.SdkMemberType]*sdkMemberListProperty{} - var fields []reflect.StructField - - // Iterate over the member types creating StructField and sdkMemberListProperty objects. - nextFieldIndex := 0 - for _, memberType := range sdkMemberTypes { - - p := memberType.SdkPropertyName() - - var getter func(properties interface{}) []string - var setter func(properties interface{}, list []string) - if memberType.RequiresBpProperty() { - // Create a dynamic exported field for the member type's property. - fields = append(fields, reflect.StructField{ - Name: proptools.FieldNameForProperty(p), - Type: reflect.TypeOf([]string{}), - Tag: `android:"arch_variant"`, - }) - - // Copy the field index for use in the getter func as using the loop variable directly will - // cause all funcs to use the last value. - fieldIndex := nextFieldIndex - nextFieldIndex += 1 - - getter = func(properties interface{}) []string { - // The properties is expected to be of the following form (where - // <Module_types> is the name of an SdkMemberType.SdkPropertyName(). - // properties *struct {<Module_types> []string, ....} - // - // Although it accesses the field by index the following reflection code is equivalent to: - // *properties.<Module_types> - // - list := reflect.ValueOf(properties).Elem().Field(fieldIndex).Interface().([]string) - return list - } - - setter = func(properties interface{}, list []string) { - // The properties is expected to be of the following form (where - // <Module_types> is the name of an SdkMemberType.SdkPropertyName(). - // properties *struct {<Module_types> []string, ....} - // - // Although it accesses the field by index the following reflection code is equivalent to: - // *properties.<Module_types> = list - // - reflect.ValueOf(properties).Elem().Field(fieldIndex).Set(reflect.ValueOf(list)) - } - } - - // Create an sdkMemberListProperty for the member type. - memberListProperty := &sdkMemberListProperty{ - getter: getter, - setter: setter, - memberType: memberType, - - // Dependencies added directly from member properties are always exported. - dependencyTag: android.DependencyTagForSdkMemberType(memberType, true), - } - - memberTypeToProperty[memberType] = memberListProperty - listProperties = append(listProperties, memberListProperty) - } - - // Create a dynamic struct from the collated fields. - propertiesStructType := reflect.StructOf(fields) - - return &dynamicSdkMemberTypes{ - memberListProperties: listProperties, - memberTypeToProperty: memberTypeToProperty, - propertiesStructType: propertiesStructType, - } -} - // sdk defines an SDK which is a logical group of modules (e.g. native libs, headers, java libs, etc.) // which Mainline modules like APEX can choose to build with. func SdkModuleFactory() android.Module { @@ -247,16 +104,16 @@ func newSdkModule(moduleExports bool) *sdk { s := &sdk{} s.properties.Module_exports = moduleExports // Get the dynamic sdk member type data for the currently registered sdk member types. - var registry *android.SdkMemberTypesRegistry + var typeRegistry *android.SdkMemberTypesRegistry if moduleExports { - registry = android.ModuleExportsMemberTypes + typeRegistry = android.ModuleExportsMemberTypes } else { - registry = android.SdkMemberTypes + typeRegistry = android.SdkMemberTypes } - s.dynamicSdkMemberTypes = getDynamicSdkMemberTypes(registry) + s.dynamicSdkMemberTypes = getDynamicSdkMemberTypes(typeRegistry) // Create an instance of the dynamically created struct that contains all the // properties for the member type specific list properties. - s.dynamicMemberTypeListProperties = s.dynamicSdkMemberTypes.createMemberListProperties() + s.dynamicMemberTypeListProperties = s.dynamicSdkMemberTypes.createMemberTypeListProperties() s.AddProperties(&s.properties, s.dynamicMemberTypeListProperties) // Make sure that the prebuilt visibility property is verified for errors. @@ -280,11 +137,11 @@ func SnapshotModuleFactory() android.Module { return s } -func (s *sdk) memberListProperties() []*sdkMemberListProperty { - return s.dynamicSdkMemberTypes.memberListProperties +func (s *sdk) memberTypeListProperties() []*sdkMemberTypeListProperty { + return s.dynamicSdkMemberTypes.memberTypeListProperties } -func (s *sdk) memberListProperty(memberType android.SdkMemberType) *sdkMemberListProperty { +func (s *sdk) memberTypeListProperty(memberType android.SdkMemberType) *sdkMemberTypeListProperty { return s.dynamicSdkMemberTypes.memberTypeToProperty[memberType] } @@ -424,7 +281,7 @@ func memberMutator(mctx android.BottomUpMutatorContext) { // Add dependencies from enabled and non CommonOS variants to the sdk member variants. if s.Enabled() && !s.IsCommonOSVariant() { ctx := s.newDependencyContext(mctx) - for _, memberListProperty := range s.memberListProperties() { + for _, memberListProperty := range s.memberTypeListProperties() { if memberListProperty.getter == nil { continue } diff --git a/sdk/update.go b/sdk/update.go index 1cd8f135a..96a6e6913 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -251,7 +251,7 @@ func (s *sdk) groupMemberVariantsByMemberThenType(ctx android.ModuleContext, mem } var members []*sdkMember - for _, memberListProperty := range s.memberListProperties() { + for _, memberListProperty := range s.memberTypeListProperties() { membersOfType := byType[memberListProperty.memberType] members = append(members, membersOfType...) } @@ -667,7 +667,7 @@ func (s *sdk) collateSnapshotModuleInfo(ctx android.BaseModuleContext, sdkVarian staticProperties := &snapshotModuleStaticProperties{ Compile_multilib: sdkVariant.multilibUsages.String(), } - dynamicProperties := s.dynamicSdkMemberTypes.createMemberListProperties() + dynamicProperties := s.dynamicSdkMemberTypes.createMemberTypeListProperties() combinedProperties := &combinedSnapshotModuleProperties{ sdkVariant: sdkVariant, @@ -687,7 +687,7 @@ func (s *sdk) collateSnapshotModuleInfo(ctx android.BaseModuleContext, sdkVarian } combined := sdkVariantToCombinedProperties[memberVariantDep.sdkVariant] - memberListProperty := s.memberListProperty(memberVariantDep.memberType) + memberListProperty := s.memberTypeListProperty(memberVariantDep.memberType) memberName := ctx.OtherModuleName(memberVariantDep.variant) if memberListProperty.getter == nil { @@ -717,7 +717,7 @@ func (s *sdk) optimizeSnapshotModuleProperties(ctx android.ModuleContext, list [ } // Extract the common members, removing them from the original properties. - commonDynamicProperties := s.dynamicSdkMemberTypes.createMemberListProperties() + commonDynamicProperties := s.dynamicSdkMemberTypes.createMemberTypeListProperties() extractor := newCommonValueExtractor(commonDynamicProperties) extractCommonProperties(ctx, extractor, commonDynamicProperties, propertyContainers) @@ -750,7 +750,7 @@ func (s *sdk) addSnapshotPropertiesToPropertySet(builder *snapshotBuilder, prope } dynamicMemberTypeListProperties := combined.dynamicProperties - for _, memberListProperty := range s.memberListProperties() { + for _, memberListProperty := range s.memberTypeListProperties() { if memberListProperty.getter == nil { continue } diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 610e42771..a22adc5aa 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -144,7 +144,7 @@ EOF run_soong local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja) - local glob_deps_file=out/soong/.bootstrap/globs/0.d + local glob_deps_file=out/soong/globs/build/0.d if [ -e "$glob_deps_file" ]; then fail "Glob deps file unexpectedly written on first build" @@ -472,17 +472,35 @@ EOF fi } -function test_null_build_after_docs { +function test_soong_docs_smoke() { setup + + run_soong soong_docs + + [[ -e "out/soong/docs/soong_build.html" ]] || fail "Documentation for main page not created" + [[ -e "out/soong/docs/cc.html" ]] || fail "Documentation for C++ modules not created" +} + +function test_null_build_after_soong_docs() { + setup + run_soong - local mtime1=$(stat -c "%y" out/soong/build.ninja) + local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja) + + run_soong soong_docs + local docs_mtime1=$(stat -c "%y" out/soong/docs/soong_build.html) + + run_soong soong_docs + local docs_mtime2=$(stat -c "%y" out/soong/docs/soong_build.html) - prebuilts/build-tools/linux-x86/bin/ninja -f out/combined.ninja soong_docs + if [[ "$docs_mtime1" != "$docs_mtime2" ]]; then + fail "Output Ninja file changed on null build" + fi run_soong - local mtime2=$(stat -c "%y" out/soong/build.ninja) + local ninja_mtime2=$(stat -c "%y" out/soong/build.ninja) - if [[ "$mtime1" != "$mtime2" ]]; then + if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then fail "Output Ninja file changed on null build" fi } @@ -522,7 +540,7 @@ EOF function test_bp2build_smoke { setup - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build [[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created" [[ -e out/soong/workspace ]] || fail "Bazel workspace not created" } @@ -531,7 +549,7 @@ function test_bp2build_generates_marker_file { setup create_mock_bazel - run_bp2build + run_soong bp2build if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then fail "Marker file was not generated" @@ -551,7 +569,7 @@ filegroup { } EOF - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build [[ -e out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not created" [[ -L out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not symlinked" @@ -565,7 +583,7 @@ filegroup { } EOF - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build [[ -e out/soong/bp2build/b/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not created" [[ -L out/soong/workspace/b/${GENERATED_BUILD_FILE_NAME} ]] || fail "a/${GENERATED_BUILD_FILE_NAME} not symlinked" } @@ -573,10 +591,10 @@ EOF function test_bp2build_null_build { setup - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) if [[ "$mtime1" != "$mtime2" ]]; then @@ -597,18 +615,35 @@ filegroup { } EOF - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build grep -q a1.txt "out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME}" || fail "a1.txt not in ${GENERATED_BUILD_FILE_NAME} file" touch a/a2.txt - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build grep -q a2.txt "out/soong/bp2build/a/${GENERATED_BUILD_FILE_NAME}" || fail "a2.txt not in ${GENERATED_BUILD_FILE_NAME} file" } +function test_multiple_soong_build_modes() { + setup + run_soong json-module-graph bp2build nothing + if [[ ! -f "out/soong/.bootstrap/bp2build_workspace_marker" ]]; then + fail "bp2build marker file was not generated" + fi + + + if [[ ! -f "out/soong/module-graph.json" ]]; then + fail "JSON file was not created" + fi + + if [[ ! -f "out/soong/build.ninja" ]]; then + fail "Main build.ninja file was not created" + fi +} + function test_dump_json_module_graph() { setup - GENERATE_JSON_MODULE_GRAPH=1 run_soong - if [[ ! -r "out/soong//module-graph.json" ]]; then + run_soong json-module-graph + if [[ ! -r "out/soong/module-graph.json" ]]; then fail "JSON file was not created" fi } @@ -619,7 +654,7 @@ function test_json_module_graph_back_and_forth_null_build() { run_soong local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja) - GENERATE_JSON_MODULE_GRAPH=1 run_soong + run_soong json-module-graph local json_mtime1=$(stat -c "%y" out/soong/module-graph.json) run_soong @@ -628,7 +663,7 @@ function test_json_module_graph_back_and_forth_null_build() { fail "Output Ninja file changed after writing JSON module graph" fi - GENERATE_JSON_MODULE_GRAPH=1 run_soong + run_soong json-module-graph local json_mtime2=$(stat -c "%y" out/soong/module-graph.json) if [[ "$json_mtime1" != "$json_mtime2" ]]; then fail "JSON module graph file changed after writing Ninja file" @@ -651,7 +686,7 @@ filegroup { } EOF - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build [[ -e out/soong/workspace ]] || fail "Bazel workspace not created" [[ -d out/soong/workspace/a/b ]] || fail "module directory not a directory" [[ -L "out/soong/workspace/a/b/${GENERATED_BUILD_FILE_NAME}" ]] || fail "${GENERATED_BUILD_FILE_NAME} file not symlinked" @@ -675,10 +710,10 @@ filegroup { } EOF - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build touch a/a2.txt # No reference in the .bp file needed - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build [[ -L out/soong/workspace/a/a2.txt ]] || fail "a/a2.txt not symlinked" } @@ -696,7 +731,7 @@ filegroup { } EOF - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build [[ -L "out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME}" ]] || fail "${GENERATED_BUILD_FILE_NAME} file not symlinked" [[ "$(readlink -f out/soong/workspace/a/${GENERATED_BUILD_FILE_NAME})" =~ "bp2build/a/${GENERATED_BUILD_FILE_NAME}"$ ]] \ || fail "${GENERATED_BUILD_FILE_NAME} files symlinked to the wrong place" @@ -725,7 +760,7 @@ filegroup { } EOF - if GENERATE_BAZEL_FILES=1 run_soong >& "$MOCK_TOP/errors"; then + if run_soong bp2build >& "$MOCK_TOP/errors"; then fail "Build should have failed" fi @@ -739,7 +774,7 @@ function test_bp2build_back_and_forth_null_build { run_soong local output_mtime1=$(stat -c "%y" out/soong/build.ninja) - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build local output_mtime2=$(stat -c "%y" out/soong/build.ninja) if [[ "$output_mtime1" != "$output_mtime2" ]]; then fail "Output Ninja file changed when switching to bp2build" @@ -757,7 +792,7 @@ function test_bp2build_back_and_forth_null_build { fail "bp2build marker file changed when switching to regular build from bp2build" fi - GENERATE_BAZEL_FILES=1 run_soong + run_soong bp2build local output_mtime4=$(stat -c "%y" out/soong/build.ninja) local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime4" ]]; then @@ -768,9 +803,32 @@ function test_bp2build_back_and_forth_null_build { fi } +function test_queryview_smoke() { + setup + + run_soong queryview + [[ -e out/soong/queryview/WORKSPACE ]] || fail "queryview WORKSPACE file not created" + +} + +function test_queryview_null_build() { + setup + + run_soong queryview + local output_mtime1=$(stat -c "%y" out/soong/queryview.marker) + + run_soong queryview + local output_mtime2=$(stat -c "%y" out/soong/queryview.marker) + + if [[ "$output_mtime1" != "$output_mtime2" ]]; then + fail "Queryview marker file changed on null build" + fi +} + test_smoke test_null_build -test_null_build_after_docs +test_soong_docs_smoke +test_null_build_after_soong_docs test_soong_build_rebuilt_if_blueprint_changes test_glob_noop_incremental test_add_file_to_glob @@ -780,9 +838,12 @@ test_delete_android_bp test_add_file_to_soong_build test_glob_during_bootstrapping test_soong_build_rerun_iff_environment_changes +test_multiple_soong_build_modes test_dump_json_module_graph test_json_module_graph_back_and_forth_null_build test_write_to_source_tree +test_queryview_smoke +test_queryview_null_build test_bp2build_smoke test_bp2build_generates_marker_file test_bp2build_null_build diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh index 9bd85a414..379eb6548 100755 --- a/tests/bp2build_bazel_test.sh +++ b/tests/bp2build_bazel_test.sh @@ -10,10 +10,10 @@ readonly GENERATED_BUILD_FILE_NAME="BUILD.bazel" function test_bp2build_null_build() { setup - run_bp2build + run_soong bp2build local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) - run_bp2build + run_soong bp2build local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime2" ]]; then @@ -35,10 +35,10 @@ filegroup { EOF touch foo/bar/a.txt foo/bar/b.txt - run_bp2build + run_soong bp2build local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) - run_bp2build + run_soong bp2build local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime2" ]]; then @@ -80,7 +80,7 @@ genrule { } EOF - run_bp2build + run_soong bp2build if [[ ! -f "./out/soong/workspace/foo/convertible_soong_module/${GENERATED_BUILD_FILE_NAME}" ]]; then fail "./out/soong/workspace/foo/convertible_soong_module/${GENERATED_BUILD_FILE_NAME} was not generated" diff --git a/tests/lib.sh b/tests/lib.sh index 813a9dd7d..e77782066 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -124,10 +124,6 @@ run_bazel() { tools/bazel "$@" } -run_bp2build() { - GENERATE_BAZEL_FILES=true build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests nothing -} - run_ninja() { build/soong/soong_ui.bash --make-mode --skip-make --skip-soong-tests "$@" } diff --git a/ui/build/build.go b/ui/build/build.go index d869bf0bc..2e44aaac6 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -248,6 +248,16 @@ func Build(ctx Context, config Config) { what = what &^ RunNinja } + if !config.SoongBuildInvocationNeeded() { + // This means that the output of soong_build is not needed and thus it would + // run unnecessarily. In addition, if this code wasn't there invocations + // with only special-cased target names like "m bp2build" would result in + // passing Ninja the empty target list and it would then build the default + // targets which is not what the user asked for. + what = what &^ RunNinja + what = what &^ RunKati + } + if config.StartGoma() { startGoma(ctx, config) } @@ -278,16 +288,6 @@ func Build(ctx Context, config Config) { if what&RunSoong != 0 { runSoong(ctx, config) - - if config.bazelBuildMode() == generateBuildFiles { - // Return early, if we're using Soong as solely the generator of BUILD files. - return - } - - if config.bazelBuildMode() == generateJsonModuleGraph { - // Return early, if we're using Soong as solely the generator of the JSON module graph - return - } } if what&RunKati != 0 { diff --git a/ui/build/config.go b/ui/build/config.go index 6a05f4fc4..35dacf2b8 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -33,7 +33,8 @@ import ( type Config struct{ *configImpl } type configImpl struct { - // From the environment + // Some targets that are implemented in soong_build + // (bp2build, json-module-graph) are not here and have their own bits below. arguments []string goma bool environ *Environment @@ -41,17 +42,21 @@ type configImpl struct { buildDateTime string // From the arguments - parallel int - keepGoing int - verbose bool - checkbuild bool - dist bool - skipConfig bool - skipKati bool - skipKatiNinja bool - skipSoong bool - skipNinja bool - skipSoongTests bool + parallel int + keepGoing int + verbose bool + checkbuild bool + dist bool + jsonModuleGraph bool + bp2build bool + queryview bool + soongDocs bool + skipConfig bool + skipKati bool + skipKatiNinja bool + skipSoong bool + skipNinja bool + skipSoongTests bool // From the product config katiArgs []string @@ -106,12 +111,6 @@ const ( // Don't use bazel at all. noBazel bazelBuildMode = iota - // Only generate build files (in a subdirectory of the out directory) and exit. - generateBuildFiles - - // Only generate the Soong json module graph for use with jq, and exit. - generateJsonModuleGraph - // Generate synthetic build files and incorporate these files into a build which // partially uses Bazel. Build metadata may come from Android.bp or BUILD files. mixedBuild @@ -639,6 +638,14 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { c.environ.Set(k, v) } else if arg == "dist" { c.dist = true + } else if arg == "json-module-graph" { + c.jsonModuleGraph = true + } else if arg == "bp2build" { + c.bp2build = true + } else if arg == "queryview" { + c.queryview = true + } else if arg == "soong_docs" { + c.soongDocs = true } else { if arg == "checkbuild" { c.checkbuild = true @@ -705,6 +712,26 @@ func (c *configImpl) Arguments() []string { return c.arguments } +func (c *configImpl) SoongBuildInvocationNeeded() bool { + if c.Dist() { + return true + } + + if len(c.Arguments()) > 0 { + // Explicit targets requested that are not special targets like b2pbuild + // or the JSON module graph + return true + } + + if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() { + // Command line was empty, the default Ninja target is built + return true + } + + // build.ninja doesn't need to be generated + return false +} + func (c *configImpl) OutDir() string { if outDir, ok := c.environ.Get("OUT_DIR"); ok { return outDir @@ -739,6 +766,28 @@ func (c *configImpl) SoongOutDir() string { return filepath.Join(c.OutDir(), "soong") } +func (c *configImpl) PrebuiltOS() string { + switch runtime.GOOS { + case "linux": + return "linux-x86" + case "darwin": + return "darwin-x86" + default: + panic("Unknown GOOS") + } +} +func (c *configImpl) HostToolDir() string { + return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") +} + +func (c *configImpl) NamedGlobFile(name string) string { + return shared.JoinPath(c.SoongOutDir(), ".bootstrap/build-globs."+name+".ninja") +} + +func (c *configImpl) UsedEnvFile(tag string) string { + return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) +} + func (c *configImpl) MainNinjaFile() string { return shared.JoinPath(c.SoongOutDir(), "build.ninja") } @@ -747,6 +796,14 @@ func (c *configImpl) Bp2BuildMarkerFile() string { return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker") } +func (c *configImpl) SoongDocsHtml() string { + return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") +} + +func (c *configImpl) QueryviewMarkerFile() string { + return shared.JoinPath(c.SoongOutDir(), "queryview.marker") +} + func (c *configImpl) ModuleGraphFile() string { return shared.JoinPath(c.SoongOutDir(), "module-graph.json") } @@ -776,6 +833,22 @@ func (c *configImpl) Dist() bool { return c.dist } +func (c *configImpl) JsonModuleGraph() bool { + return c.jsonModuleGraph +} + +func (c *configImpl) Bp2Build() bool { + return c.bp2build +} + +func (c *configImpl) Queryview() bool { + return c.queryview +} + +func (c *configImpl) SoongDocs() bool { + return c.soongDocs +} + func (c *configImpl) IsVerbose() bool { return c.verbose } @@ -921,10 +994,6 @@ func (c *configImpl) UseBazel() bool { func (c *configImpl) bazelBuildMode() bazelBuildMode { if c.Environment().IsEnvTrue("USE_BAZEL_ANALYSIS") { return mixedBuild - } else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") { - return generateBuildFiles - } else if c.Environment().IsEnvTrue("GENERATE_JSON_MODULE_GRAPH") { - return generateJsonModuleGraph } else { return noBazel } diff --git a/ui/build/finder.go b/ui/build/finder.go index 09d53cc1e..8f74969fb 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -15,15 +15,16 @@ package build import ( - "android/soong/finder" - "android/soong/finder/fs" - "android/soong/ui/logger" "bytes" "io/ioutil" "os" "path/filepath" "strings" + "android/soong/finder" + "android/soong/finder/fs" + "android/soong/ui/logger" + "android/soong/ui/metrics" ) @@ -72,8 +73,6 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { "AndroidProducts.mk", // General Soong build definitions, using the Blueprint syntax. "Android.bp", - // build/blueprint build definitions, using the Blueprint syntax. - "Blueprints", // Bazel build definitions. "BUILD.bazel", // Bazel build definitions. @@ -165,8 +164,6 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { // Recursively look for all Android.bp files androidBps := f.FindNamedAt(".", "Android.bp") - // The files are named "Blueprints" only in the build/blueprint directory. - androidBps = append(androidBps, f.FindNamedAt("build/blueprint", "Blueprints")...) if len(androidBps) == 0 { ctx.Fatalf("No Android.bp found") } diff --git a/ui/build/soong.go b/ui/build/soong.go index 726b5418b..617d29311 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -37,6 +37,12 @@ import ( const ( availableEnvFile = "soong.environment.available" usedEnvFile = "soong.environment.used" + + soongBuildTag = "build" + bp2buildTag = "bp2build" + jsonModuleGraphTag = "modulegraph" + queryviewTag = "queryview" + soongDocsTag = "soong_docs" ) func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error { @@ -71,9 +77,17 @@ func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string // A tiny struct used to tell Blueprint that it's in bootstrap mode. It would // probably be nicer to use a flag in bootstrap.Args instead. type BlueprintConfig struct { - soongOutDir string - outDir string - debugCompilation bool + toolDir string + soongOutDir string + outDir string + runGoTests bool + debugCompilation bool + subninjas []string + primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation +} + +func (c BlueprintConfig) HostToolDir() string { + return c.toolDir } func (c BlueprintConfig) SoongOutDir() string { @@ -84,14 +98,26 @@ func (c BlueprintConfig) OutDir() string { return c.outDir } +func (c BlueprintConfig) RunGoTests() bool { + return c.runGoTests +} + func (c BlueprintConfig) DebugCompilation() bool { return c.debugCompilation } -func environmentArgs(config Config, suffix string) []string { +func (c BlueprintConfig) Subninjas() []string { + return c.subninjas +} + +func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation { + return c.primaryBuilderInvocations +} + +func environmentArgs(config Config, tag string) []string { return []string{ "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile), - "--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix), + "--used_env", config.UsedEnvFile(tag), } } @@ -109,111 +135,134 @@ func writeEmptyGlobFile(ctx Context, path string) { } } -func bootstrapBlueprint(ctx Context, config Config) { - ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap") - defer ctx.EndTrace() - - var args bootstrap.Args +func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation { + commonArgs := make([]string, 0, 0) - bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja") - bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja") - moduleGraphGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.modulegraph.ninja") - - // The glob .ninja files are subninja'd. However, they are generated during - // the build itself so we write an empty file so that the subninja doesn't - // fail on clean builds - writeEmptyGlobFile(ctx, bootstrapGlobFile) - writeEmptyGlobFile(ctx, bp2buildGlobFile) - bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") - - args.RunGoTests = !config.skipSoongTests - args.UseValidations = true // Use validations to depend on tests - args.SoongOutDir = config.SoongOutDir() - args.OutDir = config.OutDir() - args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") - args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") - // The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d) - // Building soong_build does not require a glob file - // Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary - args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile} - args.EmptyNinjaFile = config.EmptyNinjaFile() - - args.DelveListen = os.Getenv("SOONG_DELVE") - if args.DelveListen != "" { - args.DelvePath = shared.ResolveDelveBinary() + if !config.skipSoongTests { + commonArgs = append(commonArgs, "-t") } - commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile()) - mainSoongBuildInputs := []string{"Android.bp"} + commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list")) - if config.bazelBuildMode() == mixedBuild { - mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile()) + if os.Getenv("SOONG_DELVE") != "" { + commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE")) + commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary()) } - soongBuildArgs := []string{ - "--globListDir", "globs", - "--globFile", bootstrapGlobFile, - } + allArgs := make([]string, 0, 0) + allArgs = append(allArgs, specificArgs...) + allArgs = append(allArgs, + "--globListDir", name, + "--globFile", config.NamedGlobFile(name)) - soongBuildArgs = append(soongBuildArgs, commonArgs...) - soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...) - soongBuildArgs = append(soongBuildArgs, "Android.bp") + allArgs = append(allArgs, commonArgs...) + allArgs = append(allArgs, environmentArgs(config, name)...) + allArgs = append(allArgs, "Android.bp") - mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: mainSoongBuildInputs, - Outputs: []string{config.MainNinjaFile()}, - Args: soongBuildArgs, + return bootstrap.PrimaryBuilderInvocation{ + Inputs: []string{"Android.bp"}, + Outputs: []string{output}, + Args: allArgs, } +} - bp2buildArgs := []string{ - "--bp2build_marker", config.Bp2BuildMarkerFile(), - "--globListDir", "globs.bp2build", - "--globFile", bp2buildGlobFile, +func bootstrapBlueprint(ctx Context, config Config) { + ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap") + defer ctx.EndTrace() + + mainSoongBuildExtraArgs := []string{"-o", config.MainNinjaFile()} + if config.EmptyNinjaFile() { + mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file") } - bp2buildArgs = append(bp2buildArgs, commonArgs...) - bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...) - bp2buildArgs = append(bp2buildArgs, "Android.bp") + mainSoongBuildInvocation := primaryBuilderInvocation( + config, + soongBuildTag, + config.MainNinjaFile(), + mainSoongBuildExtraArgs) - bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: []string{"Android.bp"}, - Outputs: []string{config.Bp2BuildMarkerFile()}, - Args: bp2buildArgs, + if config.bazelBuildMode() == mixedBuild { + // Mixed builds call Bazel from soong_build and they therefore need the + // Bazel workspace to be available. Make that so by adding a dependency on + // the bp2build marker file to the action that invokes soong_build . + mainSoongBuildInvocation.Inputs = append(mainSoongBuildInvocation.Inputs, + config.Bp2BuildMarkerFile()) } - moduleGraphArgs := []string{ - "--module_graph_file", config.ModuleGraphFile(), - "--globListDir", "globs.modulegraph", - "--globFile", moduleGraphGlobFile, + bp2buildInvocation := primaryBuilderInvocation( + config, + bp2buildTag, + config.Bp2BuildMarkerFile(), + []string{ + "--bp2build_marker", config.Bp2BuildMarkerFile(), + }) + + jsonModuleGraphInvocation := primaryBuilderInvocation( + config, + jsonModuleGraphTag, + config.ModuleGraphFile(), + []string{ + "--module_graph_file", config.ModuleGraphFile(), + }) + + queryviewInvocation := primaryBuilderInvocation( + config, + queryviewTag, + config.QueryviewMarkerFile(), + []string{ + "--bazel_queryview_dir", filepath.Join(config.SoongOutDir(), "queryview"), + }) + + soongDocsInvocation := primaryBuilderInvocation( + config, + soongDocsTag, + config.SoongDocsHtml(), + []string{ + "--soong_docs", config.SoongDocsHtml(), + }) + + globFiles := []string{ + config.NamedGlobFile(soongBuildTag), + config.NamedGlobFile(bp2buildTag), + config.NamedGlobFile(jsonModuleGraphTag), + config.NamedGlobFile(queryviewTag), + config.NamedGlobFile(soongDocsTag), } - moduleGraphArgs = append(moduleGraphArgs, commonArgs...) - moduleGraphArgs = append(moduleGraphArgs, environmentArgs(config, ".modulegraph")...) - moduleGraphArgs = append(moduleGraphArgs, "Android.bp") - - moduleGraphInvocation := bootstrap.PrimaryBuilderInvocation{ - Inputs: []string{"Android.bp"}, - Outputs: []string{config.ModuleGraphFile()}, - Args: moduleGraphArgs, + // The glob .ninja files are subninja'd. However, they are generated during + // the build itself so we write an empty file if the file does not exist yet + // so that the subninja doesn't fail on clean builds + for _, globFile := range globFiles { + writeEmptyGlobFile(ctx, globFile) } - args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{ - bp2buildInvocation, - mainSoongBuildInvocation, - moduleGraphInvocation, - } + var blueprintArgs bootstrap.Args + + blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") + blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") + blueprintArgs.EmptyNinjaFile = false blueprintCtx := blueprint.NewContext() blueprintCtx.SetIgnoreUnknownModuleTypes(true) blueprintConfig := BlueprintConfig{ - soongOutDir: config.SoongOutDir(), - outDir: config.OutDir(), + soongOutDir: config.SoongOutDir(), + toolDir: config.HostToolDir(), + outDir: config.OutDir(), + runGoTests: !config.skipSoongTests, + // If we want to debug soong_build, we need to compile it for debugging debugCompilation: os.Getenv("SOONG_DELVE") != "", + subninjas: globFiles, + primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{ + mainSoongBuildInvocation, + bp2buildInvocation, + jsonModuleGraphInvocation, + queryviewInvocation, + soongDocsInvocation}, } - args.EmptyNinjaFile = false - bootstrapDeps := bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig) - err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps) + bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig) + bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") + err := deptools.WriteDepFile(bootstrapDepFile, blueprintArgs.OutFile, bootstrapDeps) if err != nil { ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err) } @@ -224,6 +273,7 @@ func checkEnvironmentFile(currentEnv *Environment, envFile string) { v, _ := currentEnv.Get(k) return v } + if stale, _ := shared.StaleEnvFile(envFile, getenv); stale { os.Remove(envFile) } @@ -245,7 +295,7 @@ func runSoong(ctx Context, config Config) { } buildMode := config.bazelBuildMode() - integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles) + integratedBp2Build := buildMode == mixedBuild // This is done unconditionally, but does not take a measurable amount of time bootstrapBlueprint(ctx, config) @@ -273,16 +323,26 @@ func runSoong(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "environment check") defer ctx.EndTrace() - soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile) - checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile) + checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongBuildTag)) + + if integratedBp2Build || config.Bp2Build() { + checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(bp2buildTag)) + } + + if config.JsonModuleGraph() { + checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag)) + } + + if config.Queryview() { + checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(queryviewTag)) + } - if integratedBp2Build { - bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build") - checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile) + if config.SoongDocs() { + checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongDocsTag)) } }() - runMicrofactory(ctx, config, ".bootstrap/bpglob", "github.com/google/blueprint/bootstrap/bpglob", + runMicrofactory(ctx, config, filepath.Join(config.HostToolDir(), "bpglob"), "github.com/google/blueprint/bootstrap/bpglob", map[string]string{"github.com/google/blueprint": "build/blueprint"}) ninja := func(name, ninjaFile string, targets ...string) { @@ -321,18 +381,30 @@ func runSoong(ctx Context, config Config) { cmd.RunAndStreamOrFatal() } - var target string + targets := make([]string, 0, 0) + + if config.JsonModuleGraph() { + targets = append(targets, config.ModuleGraphFile()) + } + + if config.Bp2Build() { + targets = append(targets, config.Bp2BuildMarkerFile()) + } + + if config.Queryview() { + targets = append(targets, config.QueryviewMarkerFile()) + } + + if config.SoongDocs() { + targets = append(targets, config.SoongDocsHtml()) + } - if config.bazelBuildMode() == generateBuildFiles { - target = config.Bp2BuildMarkerFile() - } else if config.bazelBuildMode() == generateJsonModuleGraph { - target = config.ModuleGraphFile() - } else { + if config.SoongBuildInvocationNeeded() { // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build(). - target = config.MainNinjaFile() + targets = append(targets, config.MainNinjaFile()) } - ninja("bootstrap", ".bootstrap/build.ninja", target) + ninja("bootstrap", ".bootstrap/build.ninja", targets...) var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics if shouldCollectBuildSoongMetrics(config) { @@ -374,7 +446,7 @@ func runMicrofactory(ctx Context, config Config, relExePath string, pkg string, func shouldCollectBuildSoongMetrics(config Config) bool { // Do not collect metrics protobuf if the soong_build binary ran as the // bp2build converter or the JSON graph dump. - return config.bazelBuildMode() != generateBuildFiles && config.bazelBuildMode() != generateJsonModuleGraph + return config.SoongBuildInvocationNeeded() } func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics { |