diff options
34 files changed, 793 insertions, 196 deletions
diff --git a/android/bazel.go b/android/bazel.go index 8e2e3508e..fee1de221 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -235,6 +235,7 @@ var ( // Configure modules in these directories to enable bp2build_available: true or false by default. bp2buildDefaultConfig = Bp2BuildConfig{ + "art/libartpalette": Bp2BuildDefaultTrueRecursively, "art/libdexfile": Bp2BuildDefaultTrueRecursively, "art/runtime": Bp2BuildDefaultTrueRecursively, "art/tools": Bp2BuildDefaultTrue, @@ -296,6 +297,9 @@ var ( "external/google-benchmark": Bp2BuildDefaultTrueRecursively, "external/googletest": Bp2BuildDefaultTrueRecursively, "external/gwp_asan": Bp2BuildDefaultTrueRecursively, + "external/icu": Bp2BuildDefaultTrueRecursively, + "external/icu/android_icu4j": Bp2BuildDefaultFalse, // java rules incomplete + "external/icu/icu4j": Bp2BuildDefaultFalse, // java rules incomplete "external/jemalloc_new": Bp2BuildDefaultTrueRecursively, "external/jsoncpp": Bp2BuildDefaultTrueRecursively, "external/libcap": Bp2BuildDefaultTrueRecursively, @@ -481,14 +485,30 @@ var ( "libdexfiled", // depends on unconverted modules: dexfile_operator_srcs, libartbased, libartpalette // go deps: - "apex-protos", // depends on soong_zip, a go binary - "robolectric_tzdata", // depends on soong_zip, a go binary - "robolectric-sqlite4java-native", // depends on soong_zip, a go binary + "apex-protos", // depends on soong_zip, a go binary + "generated_android_icu4j_src_files", "generated_android_icu4j_test_files", "icu4c_test_data", // depends on unconverted modules: soong_zip "host_bionic_linker_asm", // depends on extract_linker, a go binary. "host_bionic_linker_script", // depends on extract_linker, a go binary. + "robolectric-sqlite4java-native", // depends on soong_zip, a go binary + "robolectric_tzdata", // depends on soong_zip, a go binary + + "android_icu4j_srcgen_binary", // Bazel build error: deps not allowed without srcs; move to runtime_deps + "core-icu4j-for-host", // Bazel build error: deps not allowed without srcs; move to runtime_deps // java deps - "bin2c_fastdeployagent", // depends on deployagent, a java binary + "android_icu4j_srcgen", // depends on unconverted modules: currysrc + "bin2c_fastdeployagent", // depends on deployagent, a java binary + "currysrc", // depends on unconverted modules: currysrc_org.eclipse, guavalib, jopt-simple-4.9 + "robolectric-sqlite4java-0.282", // depends on unconverted modules: robolectric-sqlite4java-import, robolectric-sqlite4java-native + "timezone-host", // depends on unconverted modules: art.module.api.annotations + "truth-host-prebuilt", // depends on unconverted modules: truth-prebuilt + "truth-prebuilt", // depends on unconverted modules: asm-7.0, guava + + "generated_android_icu4j_resources", // depends on unconverted modules: android_icu4j_srcgen_binary, soong_zip + "generated_android_icu4j_test_resources", // depends on unconverted modules: android_icu4j_srcgen_binary, soong_zip + + "art-script", // depends on unconverted modules: dalvikvm, dex2oat + "dex2oat-script", // depends on unconverted modules: dex2oat } // Per-module denylist of cc_library modules to only generate the static @@ -696,6 +716,7 @@ func GetMainClassInManifest(c Config, filepath string) (string, error) { if err != nil { return "", err } + defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() diff --git a/android/module.go b/android/module.go index 00aed952f..d0807c3f6 100644 --- a/android/module.go +++ b/android/module.go @@ -1165,6 +1165,11 @@ func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *topDownMutator productConfigEnabledLabels, nil, }) + moduleSupportsDevice := mod.commonProperties.HostOrDeviceSupported&deviceSupported == deviceSupported + if mod.commonProperties.HostOrDeviceSupported != NeitherHostNorDeviceSupported && !moduleSupportsDevice { + enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(false)) + } + platformEnabledAttribute, err := enabledProperty.ToLabelListAttribute( bazel.LabelList{[]bazel.Label{bazel.Label{Label: "@platforms//:incompatible"}}, nil}, bazel.LabelList{[]bazel.Label{}, nil}) diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 91bbce68e..bd7364582 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -378,6 +378,7 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s ctx.PropertyErrorf("from", "failed to open %q: %s", from, err) return (map[string]blueprint.ModuleFactory)(nil) } + defer r.Close() mtDef, errs := soongconfig.Parse(r, from) if ctx.Config().runningAsBp2Build { diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index 8f44fc537..ce6b7f730 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -553,12 +553,66 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { `prebuilt_com.android.art`, }) + // The boot images are installed in the APEX by Soong, so there shouldn't be any dexpreopt-related Make modules. + ensureDoesNotContainRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "mybootclasspathfragment-dexpreopt-arm64-boot.art", + "mybootclasspathfragment-dexpreopt-arm64-boot.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot.art", + "mybootclasspathfragment-dexpreopt-arm-boot.oat", + "mybootclasspathfragment-dexpreopt-arm-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", + }) + // Make sure that the prebuilt bootclasspath_fragment copies its dex files to the predefined // locations for the art image. module := result.ModuleForTests("prebuilt_mybootclasspathfragment", "android_common_com.android.art") checkCopiesToPredefinedLocationForArt(t, result.Config, module, "bar", "foo") }) + t.Run("boot image files from preferred prebuilt no boot image in apex", func(t *testing.T) { + result := android.GroupFixturePreparers( + commonPreparer, + + // Configure some libraries in the art bootclasspath_fragment that match the source + // bootclasspath_fragment's contents property. + java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), + addSource("foo", "bar"), + + // Make sure that a preferred prebuilt with consistent contents doesn't affect the apex. + addPrebuilt(true, "foo", "bar"), + + java.FixtureSetBootImageInstallDirOnDevice("art", "system/framework"), + ).RunTest(t) + + ensureExactContents(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "etc/boot-image.prof", + "etc/classpaths/bootclasspath.pb", + "javalib/bar.jar", + "javalib/foo.jar", + }) + + ensureContainsRequiredDeps(t, result.TestContext, "com.android.art", "android_common_com.android.art_image", []string{ + "mybootclasspathfragment-dexpreopt-arm64-boot.art", + "mybootclasspathfragment-dexpreopt-arm64-boot.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm64-boot-bar.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot.art", + "mybootclasspathfragment-dexpreopt-arm-boot.oat", + "mybootclasspathfragment-dexpreopt-arm-boot.vdex", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.art", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.oat", + "mybootclasspathfragment-dexpreopt-arm-boot-bar.vdex", + }) + }) + t.Run("source with inconsistency between config and contents", func(t *testing.T) { android.GroupFixturePreparers( commonPreparer, @@ -631,6 +685,7 @@ func TestBootclasspathFragmentInPrebuiltArtApex(t *testing.T) { // Configure some libraries in the art bootclasspath_fragment. java.FixtureConfigureBootJars("com.android.art:foo", "com.android.art:bar"), + java.FixtureSetBootImageInstallDirOnDevice("art", "apex/com.android.art/javalib"), ) bp := ` diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 02d807516..158c8046f 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -65,7 +65,8 @@ type prebuiltCommon struct { // Installed locations of symlinks for backward compatibility. compatSymlinks android.InstallPaths - hostRequired []string + hostRequired []string + requiredModuleNames []string } type sanitizedPrebuilt interface { @@ -195,9 +196,19 @@ func (p *prebuiltCommon) initApexFilesForAndroidMk(ctx android.ModuleContext) { } p.apexFilesForAndroidMk = append(p.apexFilesForAndroidMk, af) } - } else if tag == exportedBootclasspathFragmentTag || - tag == exportedSystemserverclasspathFragmentTag { - // Visit the children of the bootclasspath_fragment and systemserver_fragment. + } else if tag == exportedBootclasspathFragmentTag { + bcpfModule, ok := child.(*java.PrebuiltBootclasspathFragmentModule) + if !ok { + ctx.PropertyErrorf("exported_bootclasspath_fragments", "%q is not a prebuilt_bootclasspath_fragment module", name) + return false + } + for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { + p.requiredModuleNames = append(p.requiredModuleNames, makeModuleName) + } + // Visit the children of the bootclasspath_fragment. + return true + } else if tag == exportedSystemserverclasspathFragmentTag { + // Visit the children of the systemserver_fragment. return true } @@ -211,6 +222,7 @@ func (p *prebuiltCommon) addRequiredModules(entries *android.AndroidMkEntries) { entries.AddStrings("LOCAL_TARGET_REQUIRED_MODULES", fi.targetRequiredModuleNames...) entries.AddStrings("LOCAL_HOST_REQUIRED_MODULES", fi.hostRequiredModuleNames...) } + entries.AddStrings("LOCAL_REQUIRED_MODULES", p.requiredModuleNames...) } func (p *prebuiltCommon) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go index a156480d2..f9adc78f8 100644 --- a/bp2build/cc_binary_conversion_test.go +++ b/bp2build/cc_binary_conversion_test.go @@ -84,13 +84,13 @@ func runCcHostBinaryTestCase(t *testing.T, tc ccBinaryBp2buildTestCase) { t.Helper() testCase := tc for i, tar := range testCase.targets { - if tar.typ != "cc_binary" { - continue - } - tar.attrs["target_compatible_with"] = `select({ + switch tar.typ { + case "cc_binary", "proto_library", "cc_lite_proto_library": + tar.attrs["target_compatible_with"] = `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], })` + } testCase.targets[i] = tar } moduleTypeUnderTest := "cc_binary_host" diff --git a/bp2build/genrule_conversion_test.go b/bp2build/genrule_conversion_test.go index 0666da778..9244b9922 100644 --- a/bp2build/genrule_conversion_test.go +++ b/bp2build/genrule_conversion_test.go @@ -97,13 +97,22 @@ func TestGenruleCliVariableReplacement(t *testing.T) { }` for _, tc := range testCases { + moduleAttrs := attrNameToString{ + "cmd": fmt.Sprintf(`"$(location :foo.tool) --genDir=%s arg $(SRCS) $(OUTS)"`, tc.genDir), + "outs": `["foo.out"]`, + "srcs": `["foo.in"]`, + "tools": `[":foo.tool"]`, + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ - "cmd": fmt.Sprintf(`"$(location :foo.tool) --genDir=%s arg $(SRCS) $(OUTS)"`, tc.genDir), - "outs": `["foo.out"]`, - "srcs": `["foo.in"]`, - "tools": `[":foo.tool"]`, - }), + makeBazelTarget("genrule", "foo", moduleAttrs), } t.Run(tc.moduleType, func(t *testing.T) { @@ -158,25 +167,36 @@ func TestGenruleLocationsLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := - []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ - "cmd": `"$(locations :foo.tools) -s $(OUTS) $(SRCS)"`, - "outs": `["foo.out"]`, - "srcs": `["foo.in"]`, - "tools": `[":foo.tools"]`, - }), - makeBazelTarget("genrule", "foo.tools", attrNameToString{ - "cmd": `"cp $(SRCS) $(OUTS)"`, - "outs": `[ + for _, tc := range testCases { + fooAttrs := attrNameToString{ + "cmd": `"$(locations :foo.tools) -s $(OUTS) $(SRCS)"`, + "outs": `["foo.out"]`, + "srcs": `["foo.in"]`, + "tools": `[":foo.tools"]`, + } + fooToolsAttrs := attrNameToString{ + "cmd": `"cp $(SRCS) $(OUTS)"`, + "outs": `[ "foo_tool.out", "foo_tool2.out", ]`, - "srcs": `["foo_tool.in"]`, - }), + "srcs": `["foo_tool.in"]`, + } + + if tc.moduleType == "java_genrule_host" { + compatibilityAttrs := `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + fooAttrs["target_compatible_with"] = compatibilityAttrs + fooToolsAttrs["target_compatible_with"] = compatibilityAttrs + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", fooAttrs), + makeBazelTarget("genrule", "foo.tools", fooToolsAttrs), } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -221,16 +241,25 @@ func TestGenruleLocationsAbsoluteLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(locations //other:foo.tool) -s $(OUTS) $(SRCS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, "tools": `["//other:foo.tool"]`, - }), - } + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -276,16 +305,25 @@ func TestGenruleSrcsLocationsAbsoluteLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(locations //other:foo.tool) -s $(OUTS) $(location //other:other.tool)"`, "outs": `["foo.out"]`, "srcs": `["//other:other.tool"]`, "tools": `["//other:foo.tool"]`, - }), - } + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -331,8 +369,8 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(location //other:foo.tool) -s $(OUTS) $(SRCS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, @@ -340,9 +378,19 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { "//other:foo.tool", "//other:other.tool", ]`, - })} + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -388,8 +436,8 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"$(locations //other:foo.tool) -s $(OUTS) $(SRCS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, @@ -397,9 +445,19 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { "//other:foo.tool", "//other:other.tool", ]`, - })} + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ @@ -444,14 +502,24 @@ func TestGenruleWithoutToolsOrToolFiles(t *testing.T) { bazel_module: { bp2build_available: true }, }` - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", attrNameToString{ + for _, tc := range testCases { + moduleAttrs := attrNameToString{ "cmd": `"cp $(SRCS) $(OUTS)"`, "outs": `["foo.out"]`, "srcs": `["foo.in"]`, - })} + } + + if tc.moduleType == "java_genrule_host" { + moduleAttrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + expectedBazelTargets := []string{ + makeBazelTarget("genrule", "foo", moduleAttrs), + } - for _, tc := range testCases { t.Run(tc.moduleType, func(t *testing.T) { runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, bp2buildTestCase{ diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go index c683b25f5..65136d9f1 100644 --- a/bp2build/java_binary_host_conversion_test.go +++ b/bp2build/java_binary_host_conversion_test.go @@ -59,6 +59,10 @@ func TestJavaBinaryHost(t *testing.T) { "deps": `["//other:jni-lib-1"]`, "jvm_flags": `["-Djava.library.path=$${RUNPATH}other"]`, "javacopts": `["-Xdoclint:all/protected"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) diff --git a/bp2build/java_library_host_conversion_test.go b/bp2build/java_library_host_conversion_test.go index 6ac82dbd2..73abdd2ea 100644 --- a/bp2build/java_library_host_conversion_test.go +++ b/bp2build/java_library_host_conversion_test.go @@ -48,9 +48,17 @@ java_library_host { makeBazelTarget("java_library", "java-lib-host-1", attrNameToString{ "srcs": `["a.java"]`, "deps": `[":java-lib-host-2"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), makeBazelTarget("java_library", "java-lib-host-2", attrNameToString{ "srcs": `["c.java"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) diff --git a/bp2build/python_binary_conversion_test.go b/bp2build/python_binary_conversion_test.go index 40c8ba1e9..dfa11d1be 100644 --- a/bp2build/python_binary_conversion_test.go +++ b/bp2build/python_binary_conversion_test.go @@ -51,6 +51,10 @@ func TestPythonBinaryHostSimple(t *testing.T) { "b/c.py", "b/d.py", ]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) @@ -80,6 +84,10 @@ func TestPythonBinaryHostPy2(t *testing.T) { makeBazelTarget("py_binary", "foo", attrNameToString{ "python_version": `"PY2"`, "srcs": `["a.py"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) @@ -109,6 +117,10 @@ func TestPythonBinaryHostPy3(t *testing.T) { // python_version is PY3 by default. makeBazelTarget("py_binary", "foo", attrNameToString{ "srcs": `["a.py"]`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) @@ -141,6 +153,10 @@ func TestPythonBinaryHostArchVariance(t *testing.T) { "//build/bazel/platforms/arch:x86": ["x86.py"], "//conditions:default": [], })`, + "target_compatible_with": `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })`, }), }, }) diff --git a/bp2build/python_library_conversion_test.go b/bp2build/python_library_conversion_test.go index 6b261052c..356d52e8c 100644 --- a/bp2build/python_library_conversion_test.go +++ b/bp2build/python_library_conversion_test.go @@ -11,19 +11,51 @@ import ( // TODO(alexmarquez): Should be lifted into a generic Bp2Build file type PythonLibBp2Build func(ctx android.TopDownMutatorContext) -func runPythonLibraryTestCase(t *testing.T, tc bp2buildTestCase) { +type pythonLibBp2BuildTestCase struct { + description string + filesystem map[string]string + blueprint string + expectedBazelTargets []testBazelTarget +} + +func convertPythonLibTestCaseToBp2build_Host(tc pythonLibBp2BuildTestCase) bp2buildTestCase { + for i := range tc.expectedBazelTargets { + tc.expectedBazelTargets[i].attrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + } + + return convertPythonLibTestCaseToBp2build(tc) +} + +func convertPythonLibTestCaseToBp2build(tc pythonLibBp2BuildTestCase) bp2buildTestCase { + var bp2BuildTargets []string + for _, t := range tc.expectedBazelTargets { + bp2BuildTargets = append(bp2BuildTargets, makeBazelTarget(t.typ, t.name, t.attrs)) + } + return bp2buildTestCase{ + description: tc.description, + filesystem: tc.filesystem, + blueprint: tc.blueprint, + expectedBazelTargets: bp2BuildTargets, + } +} + +func runPythonLibraryTestCase(t *testing.T, tc pythonLibBp2BuildTestCase) { t.Helper() - testCase := tc + testCase := convertPythonLibTestCaseToBp2build(tc) testCase.description = fmt.Sprintf(testCase.description, "python_library") testCase.blueprint = fmt.Sprintf(testCase.blueprint, "python_library") testCase.moduleTypeUnderTest = "python_library" testCase.moduleTypeUnderTestFactory = python.PythonLibraryFactory + runBp2BuildTestCaseSimple(t, testCase) } -func runPythonLibraryHostTestCase(t *testing.T, tc bp2buildTestCase) { +func runPythonLibraryHostTestCase(t *testing.T, tc pythonLibBp2BuildTestCase) { t.Helper() - testCase := tc + testCase := convertPythonLibTestCaseToBp2build_Host(tc) testCase.description = fmt.Sprintf(testCase.description, "python_library_host") testCase.blueprint = fmt.Sprintf(testCase.blueprint, "python_library_host") testCase.moduleTypeUnderTest = "python_library_host" @@ -34,14 +66,14 @@ func runPythonLibraryHostTestCase(t *testing.T, tc bp2buildTestCase) { testCase) } -func runPythonLibraryTestCases(t *testing.T, tc bp2buildTestCase) { +func runPythonLibraryTestCases(t *testing.T, tc pythonLibBp2BuildTestCase) { t.Helper() runPythonLibraryTestCase(t, tc) runPythonLibraryHostTestCase(t, tc) } func TestSimplePythonLib(t *testing.T) { - testCases := []bp2buildTestCase{ + testCases := []pythonLibBp2BuildTestCase{ { description: "simple %s converts to a native py_library", filesystem: map[string]string{ @@ -64,17 +96,21 @@ func TestSimplePythonLib(t *testing.T) { srcs: ["b/e.py"], bazel_module: { bp2build_available: false }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "data": `["files/data.txt"]`, - "deps": `[":bar"]`, - "srcs": `[ + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "data": `["files/data.txt"]`, + "deps": `[":bar"]`, + "srcs": `[ "a.py", "b/c.py", "b/d.py", ]`, - "srcs_version": `"PY3"`, - }), + "srcs_version": `"PY3"`, + }, + }, }, }, { @@ -93,11 +129,15 @@ func TestSimplePythonLib(t *testing.T) { bazel_module: { bp2build_available: true }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `["a.py"]`, - "srcs_version": `"PY2"`, - }), + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + "srcs_version": `"PY2"`, + }, + }, }, }, { @@ -116,11 +156,15 @@ func TestSimplePythonLib(t *testing.T) { bazel_module: { bp2build_available: true }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `["a.py"]`, - "srcs_version": `"PY3"`, - }), + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + "srcs_version": `"PY3"`, + }, + }, }, }, { @@ -139,11 +183,15 @@ func TestSimplePythonLib(t *testing.T) { bazel_module: { bp2build_available: true }, }`, - expectedBazelTargets: []string{ - // srcs_version is PY2ANDPY3 by default. - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `["a.py"]`, - }), + expectedBazelTargets: []testBazelTarget{ + { + // srcs_version is PY2ANDPY3 by default. + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + }, + }, }, }, } @@ -156,7 +204,7 @@ func TestSimplePythonLib(t *testing.T) { } func TestPythonArchVariance(t *testing.T) { - runPythonLibraryTestCases(t, bp2buildTestCase{ + runPythonLibraryTestCases(t, pythonLibBp2BuildTestCase{ description: "test %s arch variants", filesystem: map[string]string{ "dir/arm.py": "", @@ -173,15 +221,19 @@ func TestPythonArchVariance(t *testing.T) { }, }, }`, - expectedBazelTargets: []string{ - makeBazelTarget("py_library", "foo", attrNameToString{ - "srcs": `select({ + expectedBazelTargets: []testBazelTarget{ + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `select({ "//build/bazel/platforms/arch:arm": ["arm.py"], "//build/bazel/platforms/arch:x86": ["x86.py"], "//conditions:default": [], })`, - "srcs_version": `"PY3"`, - }), + "srcs_version": `"PY3"`, + }, + }, }, }) } diff --git a/cc/binary.go b/cc/binary.go index bf77d3deb..05923b1b8 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -605,19 +605,12 @@ func binaryBp2build(ctx android.TopDownMutatorContext, m *Module, typ string) { Features: baseAttrs.features, } - var enabledProperty bazel.BoolAttribute - if typ == "cc_binary_host" { - falseVal := false - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, android.Android.Name, &falseVal) - } - - ctx.CreateBazelTargetModuleWithRestrictions(bazel.BazelTargetModuleProperties{ + ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ Rule_class: "cc_binary", Bzl_load_location: "//build/bazel/rules:cc_binary.bzl", }, android.CommonAttributes{Name: m.Name()}, - attrs, - enabledProperty) + attrs) } // binaryAttributes contains Bazel attributes corresponding to a cc binary diff --git a/cc/bp2build.go b/cc/bp2build.go index c5eab0683..30c3c509b 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -844,10 +844,8 @@ func bp2BuildParseExportedIncludesHelper(ctx android.BazelConversionPathContext, func bazelLabelForStaticModule(ctx android.BazelConversionPathContext, m blueprint.Module) string { label := android.BazelModuleLabel(ctx, m) - if aModule, ok := m.(android.Module); ok { - if ctx.OtherModuleType(aModule) == "cc_library" && !android.GenerateCcLibraryStaticOnly(m.Name()) { - label += "_bp2build_cc_library_static" - } + if ccModule, ok := m.(*Module); ok && ccModule.typ() == fullLibrary && !android.GenerateCcLibraryStaticOnly(m.Name()) { + label += "_bp2build_cc_library_static" } return label } @@ -1751,7 +1751,7 @@ func (c *Module) setSubnameProperty(actx android.ModuleContext) { // Returns true if Bazel was successfully used for the analysis of this module. func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { var bazelModuleLabel string - if actx.ModuleType() == "cc_library" && c.static() { + if c.typ() == fullLibrary && c.static() { // cc_library is a special case in bp2build; two targets are generated -- one for each // of the shared and static variants. The shared variant keeps the module name, but the // static variant uses a different suffixed name. @@ -1759,6 +1759,7 @@ func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { } else { bazelModuleLabel = c.GetBazelLabel(actx, c) } + bazelActionsUsed := false // Mixed builds mode is disabled for modules outside of device OS. // TODO(b/200841190): Support non-device OS in mixed builds. @@ -3474,17 +3475,23 @@ func (c *Module) UniqueApexVariations() bool { var _ snapshot.RelativeInstallPath = (*Module)(nil) -// ConvertWithBp2build converts Module to Bazel for bp2build. -func (c *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - prebuilt := c.IsPrebuilt() +type moduleType int + +const ( + unknownType moduleType = iota + binary + object + fullLibrary + staticLibrary + sharedLibrary + headerLibrary +) + +func (c *Module) typ() moduleType { if c.Binary() { - if !prebuilt { - binaryBp2build(ctx, c, ctx.ModuleType()) - } + return binary } else if c.Object() { - if !prebuilt { - objectBp2Build(ctx, c) - } + return object } else if c.CcLibrary() { static := false shared := false @@ -3495,25 +3502,48 @@ func (c *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { static = library.MutatedProperties.BuildStatic shared = library.MutatedProperties.BuildShared } - if static && shared { - if !prebuilt { - libraryBp2Build(ctx, c) - } + return fullLibrary } else if !static && !shared { - libraryHeadersBp2Build(ctx, c) + return headerLibrary } else if static { - if prebuilt { - prebuiltLibraryStaticBp2Build(ctx, c) - } else { - sharedOrStaticLibraryBp2Build(ctx, c, true) - } - } else if shared { - if prebuilt { - prebuiltLibrarySharedBp2Build(ctx, c) - } else { - sharedOrStaticLibraryBp2Build(ctx, c, false) - } + return staticLibrary + } + return sharedLibrary + } + return unknownType +} + +// ConvertWithBp2build converts Module to Bazel for bp2build. +func (c *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + prebuilt := c.IsPrebuilt() + switch c.typ() { + case binary: + if !prebuilt { + binaryBp2build(ctx, c, ctx.ModuleType()) + } + case object: + if !prebuilt { + objectBp2Build(ctx, c) + } + case fullLibrary: + if !prebuilt { + libraryBp2Build(ctx, c) + } + case headerLibrary: + libraryHeadersBp2Build(ctx, c) + case staticLibrary: + + if prebuilt { + prebuiltLibraryStaticBp2Build(ctx, c) + } else { + sharedOrStaticLibraryBp2Build(ctx, c, true) + } + case sharedLibrary: + if prebuilt { + prebuiltLibrarySharedBp2Build(ctx, c) + } else { + sharedOrStaticLibraryBp2Build(ctx, c, false) } } } diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index 2d6bcb89d..979c82525 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -33,7 +33,9 @@ var ( }, "armv8-a-branchprot": []string{ "-march=armv8-a", - "-mbranch-protection=standard", + // Disable BTI until drm vendors stop using OS libraries as sources + // of gadgets (https://issuetracker.google.com/216395195). + "-mbranch-protection=pac-ret", }, "armv8-2a": []string{ "-march=armv8.2-a", diff --git a/cc/config/arm64_linux_host.go b/cc/config/arm64_linux_host.go index 853d818f1..5c7f926f8 100644 --- a/cc/config/arm64_linux_host.go +++ b/cc/config/arm64_linux_host.go @@ -41,7 +41,6 @@ var ( "-Wl,-z,relro", "-Wl,-z,now", "-Wl,--build-id=md5", - "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", "-Wl,--hash-style=gnu", "-Wl,--no-undefined-version", diff --git a/cc/config/global.go b/cc/config/global.go index 5acc7f52c..48a8b4805 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -95,6 +95,9 @@ var ( // Nested and array designated initialization is nice to have. "-Wno-c99-designator", + // Many old files still have GNU designator syntax. + "-Wno-gnu-designator", + // Warnings from clang-12 "-Wno-gnu-folding-constant", @@ -139,7 +142,6 @@ var ( "-Wl,-z,relro", "-Wl,-z,now", "-Wl,--build-id=md5", - "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", "-Wl,--no-undefined-version", // TODO: Eventually we should link against a libunwind.a with hidden symbols, and then these diff --git a/cc/config/x86_linux_bionic_host.go b/cc/config/x86_linux_bionic_host.go index 4b7ba6a85..976cc25be 100644 --- a/cc/config/x86_linux_bionic_host.go +++ b/cc/config/x86_linux_bionic_host.go @@ -47,7 +47,6 @@ var ( "-Wl,-z,relro", "-Wl,-z,now", "-Wl,--build-id=md5", - "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", "-Wl,--hash-style=gnu", "-Wl,--no-undefined-version", diff --git a/cc/library.go b/cc/library.go index cefbf6c44..de5d46fed 100644 --- a/cc/library.go +++ b/cc/library.go @@ -757,9 +757,10 @@ func GlobHeadersForSnapshot(ctx android.ModuleContext, paths android.Paths) andr if dir == "external/eigen" { // Only these two directories contains exported headers. for _, subdir := range []string{"Eigen", "unsupported/Eigen"} { - glob, err := ctx.GlobWithDeps("external/eigen/"+subdir+"/**/*", nil) + globDir := "external/eigen/" + subdir + "/**/*" + glob, err := ctx.GlobWithDeps(globDir, nil) if err != nil { - ctx.ModuleErrorf("glob failed: %#v", err) + ctx.ModuleErrorf("glob of %q failed: %s", globDir, err) return nil } for _, header := range glob { @@ -775,9 +776,10 @@ func GlobHeadersForSnapshot(ctx android.ModuleContext, paths android.Paths) andr } continue } - glob, err := ctx.GlobWithDeps(dir+"/**/*", nil) + globDir := dir + "/**/*" + glob, err := ctx.GlobWithDeps(globDir, nil) if err != nil { - ctx.ModuleErrorf("glob failed: %#v", err) + ctx.ModuleErrorf("glob of %q failed: %s", globDir, err) return nil } isLibcxx := strings.HasPrefix(dir, "external/libcxx/include") diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 13b20f42f..d80051c77 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -23,6 +23,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "time" "android/soong/shared" @@ -205,6 +206,15 @@ func main() { config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) { + var limits syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits) + if err != nil { + buildCtx.Verbosef("Failed to get file limit:", err) + } + buildCtx.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max) + } + + { // The order of the function calls is important. The last defer function call // is the first one that is executed to save the rbe metrics to a protobuf // file. The soong metrics file is then next. Bazel profiles are written diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 658e8e2dc..36513b64b 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -210,6 +210,34 @@ type ClassLoaderContext struct { Subcontexts []*ClassLoaderContext } +// excludeLibs excludes the libraries from this ClassLoaderContext. +// +// This treats the supplied context as being immutable (as it may come from a dependency). So, it +// implements copy-on-exclusion logic. That means that if any of the excluded libraries are used +// within this context then this will return a deep copy of this without those libraries. +// +// If this ClassLoaderContext matches one of the libraries to exclude then this returns (nil, true) +// to indicate that this context should be excluded from the containing list. +// +// If any of this ClassLoaderContext's Subcontexts reference the excluded libraries then this +// returns a pointer to a copy of this without the excluded libraries and true to indicate that this +// was copied. +// +// Otherwise, this returns a pointer to this and false to indicate that this was not copied. +func (c *ClassLoaderContext) excludeLibs(excludedLibs []string) (*ClassLoaderContext, bool) { + if android.InList(c.Name, excludedLibs) { + return nil, true + } + + if excludedList, modified := excludeLibsFromCLCList(c.Subcontexts, excludedLibs); modified { + clcCopy := *c + clcCopy.Subcontexts = excludedList + return &clcCopy, true + } + + return c, false +} + // ClassLoaderContextMap is a map from SDK version to CLC. There is a special entry with key // AnySdkVersion that stores unconditional CLC that is added regardless of the target SDK version. // @@ -408,6 +436,67 @@ func (clcMap ClassLoaderContextMap) Dump() string { return string(bytes) } +// excludeLibsFromCLCList excludes the libraries from the ClassLoaderContext in this list. +// +// This treats the supplied list as being immutable (as it may come from a dependency). So, it +// implements copy-on-exclusion logic. That means that if any of the excluded libraries are used +// within the contexts in the list then this will return a deep copy of the list without those +// libraries. +// +// If any of the ClassLoaderContext in the list reference the excluded libraries then this returns a +// copy of this list without the excluded libraries and true to indicate that this was copied. +// +// Otherwise, this returns the list and false to indicate that this was not copied. +func excludeLibsFromCLCList(clcList []*ClassLoaderContext, excludedLibs []string) ([]*ClassLoaderContext, bool) { + modifiedList := false + copiedList := make([]*ClassLoaderContext, 0, len(clcList)) + for _, clc := range clcList { + resultClc, modifiedClc := clc.excludeLibs(excludedLibs) + if resultClc != nil { + copiedList = append(copiedList, resultClc) + } + modifiedList = modifiedList || modifiedClc + } + + if modifiedList { + return copiedList, true + } else { + return clcList, false + } +} + +// ExcludeLibs excludes the libraries from the ClassLoaderContextMap. +// +// If the list o libraries is empty then this returns the ClassLoaderContextMap. +// +// This treats the ClassLoaderContextMap as being immutable (as it may come from a dependency). So, +// it implements copy-on-exclusion logic. That means that if any of the excluded libraries are used +// within the contexts in the map then this will return a deep copy of the map without those +// libraries. +// +// Otherwise, this returns the map unchanged. +func (clcMap ClassLoaderContextMap) ExcludeLibs(excludedLibs []string) ClassLoaderContextMap { + if len(excludedLibs) == 0 { + return clcMap + } + + excludedClcMap := make(ClassLoaderContextMap) + modifiedMap := false + for sdkVersion, clcList := range clcMap { + excludedList, modifiedList := excludeLibsFromCLCList(clcList, excludedLibs) + if len(excludedList) != 0 { + excludedClcMap[sdkVersion] = excludedList + } + modifiedMap = modifiedMap || modifiedList + } + + if modifiedMap { + return excludedClcMap + } else { + return clcMap + } +} + // Now that the full unconditional context is known, reconstruct conditional context. // Apply filters for individual libraries, mirroring what the PackageManager does when it // constructs class loader context on device. diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index 4a3d390c2..5d3a9d943 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -284,6 +284,111 @@ func TestCLCSdkVersionOrder(t *testing.T) { }) } +func TestCLCMExcludeLibs(t *testing.T) { + ctx := testContext() + const optional = false + const implicit = true + + excludeLibs := func(t *testing.T, m ClassLoaderContextMap, excluded_libs ...string) ClassLoaderContextMap { + // Dump the CLCM before creating a new copy that excludes a specific set of libraries. + before := m.Dump() + + // Create a new CLCM that excludes some libraries. + c := m.ExcludeLibs(excluded_libs) + + // Make sure that the original CLCM was not changed. + after := m.Dump() + android.AssertStringEquals(t, "input CLCM modified", before, after) + + return c + } + + t.Run("exclude nothing", func(t *testing.T) { + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + + a := excludeLibs(t, m) + + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Implicit": true, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [] + } + ] +}`, a.Dump()) + }) + + t.Run("one item from list", func(t *testing.T) { + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + + a := excludeLibs(t, m, "a") + + expected := `{ + "28": [ + { + "Name": "b", + "Optional": false, + "Implicit": true, + "Host": "out/soong/b.jar", + "Device": "/system/b.jar", + "Subcontexts": [] + } + ] +}` + android.AssertStringEquals(t, "output CLCM ", expected, a.Dump()) + }) + + t.Run("all items from a list", func(t *testing.T) { + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + + a := excludeLibs(t, m, "a", "b") + + android.AssertStringEquals(t, "output CLCM ", `{}`, a.Dump()) + }) + + t.Run("items from a subcontext", func(t *testing.T) { + s := make(ClassLoaderContextMap) + s.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + s.AddContext(ctx, AnySdkVersion, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), s) + + a := excludeLibs(t, m, "b") + + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Implicit": true, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [ + { + "Name": "c", + "Optional": false, + "Implicit": true, + "Host": "out/soong/c.jar", + "Device": "/system/c.jar", + "Subcontexts": [] + } + ] + } + ] +}`, a.Dump()) + }) +} + func checkError(t *testing.T, have error, want string) { if have == nil { t.Errorf("\nwant error: '%s'\nhave: none", want) diff --git a/finder/finder.go b/finder/finder.go index 5413fa6bb..b4834b16b 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -847,6 +847,7 @@ func (f *Finder) startFromExternalCache() (err error) { if err != nil { return errors.New("No data to load from database\n") } + defer reader.Close() bufferedReader := bufio.NewReader(reader) if !f.validateCacheHeader(bufferedReader) { return errors.New("Cache header does not match") diff --git a/finder/fs/test.go b/finder/fs/test.go index cb2140e75..ed981fd0e 100644 --- a/finder/fs/test.go +++ b/finder/fs/test.go @@ -74,6 +74,7 @@ func Read(t *testing.T, path string, filesystem *MockFs) string { if err != nil { t.Fatalf(err.Error()) } + defer reader.Close() bytes, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err.Error()) diff --git a/java/aar.go b/java/aar.go index 4687424c2..51aad8da0 100644 --- a/java/aar.go +++ b/java/aar.go @@ -267,11 +267,15 @@ var extractAssetsRule = pctx.AndroidStaticRule("extractAssets", }) func (a *aapt) buildActions(ctx android.ModuleContext, sdkContext android.SdkContext, - classLoaderContexts dexpreopt.ClassLoaderContextMap, extraLinkFlags ...string) { + classLoaderContexts dexpreopt.ClassLoaderContextMap, excludedLibs []string, + extraLinkFlags ...string) { transitiveStaticLibs, transitiveStaticLibManifests, staticRRODirs, assetPackages, libDeps, libFlags := aaptLibs(ctx, sdkContext, classLoaderContexts) + // Exclude any libraries from the supplied list. + classLoaderContexts = classLoaderContexts.ExcludeLibs(excludedLibs) + // App manifest file manifestFile := proptools.StringDefault(a.aaptProperties.Manifest, "AndroidManifest.xml") manifestSrcPath := android.PathForModuleSrc(ctx, manifestFile) @@ -530,7 +534,7 @@ func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.aapt.isLibrary = true a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) - a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts) + a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts, nil) a.hideApexVariantFromMake = !ctx.Provider(android.ApexInfoProvider).(android.ApexInfo).IsForPlatform() diff --git a/java/app.go b/java/app.go index 9f2f99a16..e4432ff4b 100755 --- a/java/app.go +++ b/java/app.go @@ -425,7 +425,8 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { a.aapt.splitNames = a.appProperties.Package_splits a.aapt.LoggingParent = String(a.overridableAppProperties.Logging_parent) - a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts, aaptLinkFlags...) + a.aapt.buildActions(ctx, android.SdkContext(a), a.classLoaderContexts, + a.usesLibraryProperties.Exclude_uses_libs, aaptLinkFlags...) // apps manifests are handled by aapt, don't let Module see them a.properties.Manifest = nil @@ -1211,6 +1212,23 @@ type UsesLibraryProperties struct { // libraries, because SDK ones are automatically picked up by Soong. The <uses-library> name // normally is the same as the module name, but there are exceptions. Provides_uses_lib *string + + // A list of shared library names to exclude from the classpath of the APK. Adding a library here + // will prevent it from being used when precompiling the APK and prevent it from being implicitly + // added to the APK's manifest's <uses-library> elements. + // + // Care must be taken when using this as it could result in runtime errors if the APK actually + // uses classes provided by the library and which are not provided in any other way. + // + // This is primarily intended for use by various CTS tests that check the runtime handling of the + // android.test.base shared library (and related libraries) but which depend on some common + // libraries that depend on the android.test.base library. Without this those tests will end up + // with a <uses-library android:name="android.test.base"/> in their manifest which would either + // render the tests worthless (as they would be testing the wrong behavior), or would break the + // test altogether by providing access to classes that the tests were not expecting. Those tests + // provide the android.test.base statically and use jarjar to rename them so they do not collide + // with the classes provided by the android.test.base library. + Exclude_uses_libs []string } // usesLibrary provides properties and helper functions for AndroidApp and AndroidAppImport to verify that the diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index 47941807c..a36bd6a63 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -1073,7 +1073,7 @@ type prebuiltBootclasspathFragmentProperties struct { // At the moment this is basically just a bootclasspath_fragment module that can be used as a // prebuilt. Eventually as more functionality is migrated into the bootclasspath_fragment module // type from the various singletons then this will diverge. -type prebuiltBootclasspathFragmentModule struct { +type PrebuiltBootclasspathFragmentModule struct { BootclasspathFragmentModule prebuilt android.Prebuilt @@ -1081,16 +1081,16 @@ type prebuiltBootclasspathFragmentModule struct { prebuiltProperties prebuiltBootclasspathFragmentProperties } -func (module *prebuiltBootclasspathFragmentModule) Prebuilt() *android.Prebuilt { +func (module *PrebuiltBootclasspathFragmentModule) Prebuilt() *android.Prebuilt { return &module.prebuilt } -func (module *prebuiltBootclasspathFragmentModule) Name() string { +func (module *PrebuiltBootclasspathFragmentModule) Name() string { return module.prebuilt.Name(module.ModuleBase.Name()) } // produceHiddenAPIOutput returns a path to the prebuilt all-flags.csv or nil if none is specified. -func (module *prebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput { +func (module *PrebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput { pathForOptionalSrc := func(src *string, defaultPath android.Path) android.Path { if src == nil { return defaultPath @@ -1131,7 +1131,7 @@ func (module *prebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx an } // produceBootImageFiles extracts the boot image files from the APEX if available. -func (module *prebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageFilesByArch { +func (module *PrebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig) bootImageFilesByArch { if !shouldCopyBootFilesToPredefinedLocations(ctx, imageConfig) { return nil } @@ -1141,37 +1141,53 @@ func (module *prebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx and return nil // An error has been reported by FindDeapexerProviderForModule. } - files := bootImageFilesByArch{} - for _, variant := range imageConfig.apexVariants() { - arch := variant.target.Arch.ArchType - for _, toPath := range variant.imagesDeps { - apexRelativePath := apexRootRelativePathToBootImageFile(arch, toPath.Base()) - // Get the path to the file that the deapexer extracted from the prebuilt apex file. - fromPath := di.PrebuiltExportPath(apexRelativePath) - - // Return the toPath as the calling code expects the paths in the returned map to be the - // paths predefined in the bootImageConfig. - files[arch] = append(files[arch], toPath) - - // Copy the file to the predefined location. - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: fromPath, - Output: toPath, - }) - } + profile := (android.WritablePath)(nil) + if imageConfig.profileInstallPathInApex != "" { + profile = di.PrebuiltExportPath(imageConfig.profileInstallPathInApex) } - // Build the boot image files for the host variants. These are built from the dex files provided - // by the contents of this module as prebuilt versions of the host boot image files are not - // available, i.e. there is no host specific prebuilt apex containing them. This has to be built - // without a profile as the prebuilt modules do not provide a profile. - buildBootImageVariantsForBuildOs(ctx, imageConfig, nil) + // Build the boot image files for the host variants. These are always built from the dex files + // provided by the contents of this module as prebuilt versions of the host boot image files are + // not available, i.e. there is no host specific prebuilt apex containing them. This has to be + // built without a profile as the prebuilt modules do not provide a profile. + buildBootImageVariantsForBuildOs(ctx, imageConfig, profile) - return files + if imageConfig.shouldInstallInApex() { + // If the boot image files for the android variants are in the prebuilt apex, we must use those + // rather than building new ones because those boot image files are going to be used on device. + files := bootImageFilesByArch{} + for _, variant := range imageConfig.apexVariants() { + arch := variant.target.Arch.ArchType + for _, toPath := range variant.imagesDeps { + apexRelativePath := apexRootRelativePathToBootImageFile(arch, toPath.Base()) + // Get the path to the file that the deapexer extracted from the prebuilt apex file. + fromPath := di.PrebuiltExportPath(apexRelativePath) + + // Return the toPath as the calling code expects the paths in the returned map to be the + // paths predefined in the bootImageConfig. + files[arch] = append(files[arch], toPath) + + // Copy the file to the predefined location. + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: fromPath, + Output: toPath, + }) + } + } + return files + } else { + if profile == nil { + ctx.ModuleErrorf("Unable to produce boot image files: neither boot image files nor profiles exists in the prebuilt apex") + return nil + } + // Build boot image files for the android variants from the dex files provided by the contents + // of this module. + return buildBootImageVariantsForAndroidOs(ctx, imageConfig, profile) + } } -var _ commonBootclasspathFragment = (*prebuiltBootclasspathFragmentModule)(nil) +var _ commonBootclasspathFragment = (*PrebuiltBootclasspathFragmentModule)(nil) // createBootImageTag creates the tag to uniquely identify the boot image file among all of the // files that a module requires from the prebuilt .apex file. @@ -1185,16 +1201,22 @@ func createBootImageTag(arch android.ArchType, baseName string) string { // // If there is no image config associated with this fragment then it returns nil. Otherwise, it // returns the files that are listed in the image config. -func (module *prebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex(ctx android.BaseModuleContext) []string { +func (module *PrebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex(ctx android.BaseModuleContext) []string { imageConfig := module.getImageConfig(ctx) if imageConfig != nil { - // Add the boot image files, e.g. .art, .oat and .vdex files. files := []string{} - for _, variant := range imageConfig.apexVariants() { - arch := variant.target.Arch.ArchType - for _, path := range variant.imagesDeps.Paths() { - base := path.Base() - files = append(files, apexRootRelativePathToBootImageFile(arch, base)) + if imageConfig.profileInstallPathInApex != "" { + // Add the boot image profile. + files = append(files, imageConfig.profileInstallPathInApex) + } + if imageConfig.shouldInstallInApex() { + // Add the boot image files, e.g. .art, .oat and .vdex files. + for _, variant := range imageConfig.apexVariants() { + arch := variant.target.Arch.ArchType + for _, path := range variant.imagesDeps.Paths() { + base := path.Base() + files = append(files, apexRootRelativePathToBootImageFile(arch, base)) + } } } return files @@ -1206,10 +1228,10 @@ func apexRootRelativePathToBootImageFile(arch android.ArchType, base string) str return filepath.Join("javalib", arch.String(), base) } -var _ android.RequiredFilesFromPrebuiltApex = (*prebuiltBootclasspathFragmentModule)(nil) +var _ android.RequiredFilesFromPrebuiltApex = (*PrebuiltBootclasspathFragmentModule)(nil) func prebuiltBootclasspathFragmentFactory() android.Module { - m := &prebuiltBootclasspathFragmentModule{} + m := &PrebuiltBootclasspathFragmentModule{} m.AddProperties(&m.properties, &m.prebuiltProperties) // This doesn't actually have any prebuilt files of its own so pass a placeholder for the srcs // array. diff --git a/java/plugin.go b/java/plugin.go index f1a5ec451..4b174b930 100644 --- a/java/plugin.go +++ b/java/plugin.go @@ -17,8 +17,6 @@ package java import ( "android/soong/android" "android/soong/bazel" - - "github.com/google/blueprint/proptools" ) func init() { @@ -78,12 +76,9 @@ func (p *Plugin) ConvertWithBp2build(ctx android.TopDownMutatorContext) { attrs.Processor_class = p.pluginProperties.Processor_class } - var enabledProperty bazel.BoolAttribute - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, android.Android.Name, proptools.BoolPtr(false)) - props := bazel.BazelTargetModuleProperties{ Rule_class: "java_plugin", } - ctx.CreateBazelTargetModuleWithRestrictions(props, android.CommonAttributes{Name: p.Name()}, attrs, enabledProperty) + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: p.Name()}, attrs) } diff --git a/java/rro.go b/java/rro.go index 0b4d0916a..be84afffc 100644 --- a/java/rro.go +++ b/java/rro.go @@ -139,7 +139,7 @@ func (r *RuntimeResourceOverlay) GenerateAndroidBuildActions(ctx android.ModuleC aaptLinkFlags = append(aaptLinkFlags, "--rename-overlay-target-package "+*r.overridableProperties.Target_package_name) } - r.aapt.buildActions(ctx, r, nil, aaptLinkFlags...) + r.aapt.buildActions(ctx, r, nil, nil, aaptLinkFlags...) // Sign the built package _, certificates := collectAppDeps(ctx, r, false, false) diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index d9b4e86b7..8e23a53c9 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -57,6 +57,7 @@ var ( cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file") traceCalls = flag.Bool("trace_calls", false, "trace function calls") inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables") + makefileList = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)") ) func init() { @@ -79,7 +80,7 @@ func init() { var backupSuffix string var tracedVariables []string var errorLogger = errorSink{data: make(map[string]datum)} -var makefileFinder = &LinuxMakefileFinder{} +var makefileFinder mk2rbc.MakefileFinder func main() { flag.Usage = func() { @@ -133,6 +134,16 @@ func main() { pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } + + if *makefileList != "" { + makefileFinder = &FileListMakefileFinder{ + cachedMakefiles: nil, + filePath: *makefileList, + } + } else { + makefileFinder = &FindCommandMakefileFinder{} + } + // Find out global variables getConfigVariables() getSoongVariables() @@ -519,17 +530,17 @@ func stringsWithFreq(items []string, topN int) (string, int) { return res, len(sorted) } -type LinuxMakefileFinder struct { +// FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that +// runs the unix find command to find all the makefiles in the source tree. +type FindCommandMakefileFinder struct { cachedRoot string cachedMakefiles []string } -func (l *LinuxMakefileFinder) Find(root string) []string { +func (l *FindCommandMakefileFinder) Find(root string) []string { if l.cachedMakefiles != nil && l.cachedRoot == root { return l.cachedMakefiles } - l.cachedRoot = root - l.cachedMakefiles = make([]string, 0) // Return all *.mk files but not in hidden directories. @@ -548,9 +559,60 @@ func (l *LinuxMakefileFinder) Find(root string) []string { panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err)) } scanner := bufio.NewScanner(stdout) + result := make([]string, 0) for scanner.Scan() { - l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./")) + result = append(result, strings.TrimPrefix(scanner.Text(), "./")) } stdout.Close() + err = scanner.Err() + if err != nil { + panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err)) + } + l.cachedRoot = root + l.cachedMakefiles = result + return l.cachedMakefiles +} + +// FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that +// reads a file containing the list of makefiles in the android source tree. +// This file is generated by soong's finder, so that it can be computed while +// soong is already walking the source tree looking for other files. If the root +// to find makefiles under is not the root of the android source tree, it will +// fall back to using FindCommandMakefileFinder. +type FileListMakefileFinder struct { + FindCommandMakefileFinder + cachedMakefiles []string + filePath string +} + +func (l *FileListMakefileFinder) Find(root string) []string { + root, err1 := filepath.Abs(root) + wd, err2 := filepath.Abs(*rootDir) + if root != wd || err1 != nil || err2 != nil { + return l.FindCommandMakefileFinder.Find(root) + } + if l.cachedMakefiles != nil { + return l.cachedMakefiles + } + + file, err := os.Open(l.filePath) + if err != nil { + panic(fmt.Errorf("Cannot read makefile list: %s\n", err)) + } + defer file.Close() + + result := make([]string, 0) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 { + result = append(result, line) + } + } + + if err = scanner.Err(); err != nil { + panic(fmt.Errorf("Cannot read makefile list: %s\n", err)) + } + l.cachedMakefiles = result return l.cachedMakefiles } diff --git a/rust/library.go b/rust/library.go index baac3f0bc..62eaefd68 100644 --- a/rust/library.go +++ b/rust/library.go @@ -773,9 +773,10 @@ func (l *libraryDecorator) collectHeadersForSnapshot(ctx android.ModuleContext, // Glob together the headers from the modules include_dirs property for _, path := range android.CopyOfPaths(l.includeDirs) { dir := path.String() - glob, err := ctx.GlobWithDeps(dir+"/**/*", nil) + globDir := dir + "/**/*" + glob, err := ctx.GlobWithDeps(globDir, nil) if err != nil { - ctx.ModuleErrorf("glob failed: %#v", err) + ctx.ModuleErrorf("glob of %q failed: %s", globDir, err) return } diff --git a/scripts/rbc-run b/scripts/rbc-run index 7243421ff..b8a6c0c73 100755 --- a/scripts/rbc-run +++ b/scripts/rbc-run @@ -9,9 +9,10 @@ declare -r output_root="${OUT_DIR:-out}" declare -r runner="${output_root}/soong/rbcrun" declare -r converter="${output_root}/soong/mk2rbc" declare -r launcher="${output_root}/rbc/launcher.rbc" +declare -r makefile_list="${output_root}/.module_paths/configuration.list" declare -r makefile="$1" declare -r input_variables="$2" shift 2 -"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" "${makefile}" +"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" --makefile_list="${makefile_list}" "${makefile}" "${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}" diff --git a/ui/build/cleanbuild.go b/ui/build/cleanbuild.go index a3a1aaf68..1c80cff31 100644 --- a/ui/build/cleanbuild.go +++ b/ui/build/cleanbuild.go @@ -171,6 +171,7 @@ func installClean(ctx Context, config Config) { productOut("recovery"), productOut("root"), productOut("system"), + productOut("system_dlkm"), productOut("system_other"), productOut("vendor"), productOut("vendor_dlkm"), diff --git a/ui/build/finder.go b/ui/build/finder.go index 8f74969fb..68efe2150 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -87,8 +87,8 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { // Bazel top-level file to mark a directory as a Bazel workspace. "WORKSPACE", }, - // Bazel Starlark configuration files. - IncludeSuffixes: []string{".bzl"}, + // Bazel Starlark configuration files and all .mk files for product/board configuration. + IncludeSuffixes: []string{".bzl", ".mk"}, } dumpDir := config.FileListDir() f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard), @@ -110,6 +110,19 @@ func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []s return entries.DirNames, matches } +func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { + matches := []string{} + for _, foundName := range entries.FileNames { + if foundName != "Android.mk" && + foundName != "AndroidProducts.mk" && + foundName != "CleanSpec.mk" && + strings.HasSuffix(foundName, ".mk") { + matches = append(matches, foundName) + } + } + return entries.DirNames, matches +} + // FindSources searches for source files known to <f> and writes them to the filesystem for // use later. func FindSources(ctx Context, config Config, f *finder.Finder) { @@ -172,6 +185,13 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { ctx.Fatalf("Could not find modules: %v", err) } + // Recursively look for all product/board config files. + configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles) + err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list")) + if err != nil { + ctx.Fatalf("Could not export product/board configuration list: %v", err) + } + if config.Dist() { f.WaitForDbDump() // Dist the files.db plain text database. |