diff options
128 files changed, 4742 insertions, 1274 deletions
diff --git a/Android.bp b/Android.bp index 42a8e5c91..63de01589 100644 --- a/Android.bp +++ b/Android.bp @@ -119,3 +119,14 @@ genrule { dexpreopt_systemserver_check { name: "dexpreopt_systemserver_check", } + +// buildinfo.prop contains common properties for system/build.prop, like ro.build.version.* +buildinfo_prop { + name: "buildinfo.prop", + + // not installable because this will be included to system/build.prop + installable: false, + + // Currently, only microdroid can refer to buildinfo.prop + visibility: ["//packages/modules/Virtualization/microdroid"], +} diff --git a/android/Android.bp b/android/Android.bp index 87b021fb6..d58370391 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -36,6 +36,7 @@ bootstrap_go_package { "bazel.go", "bazel_handler.go", "bazel_paths.go", + "buildinfo_prop.go", "config.go", "config_bp2build.go", "csuite_config.go", diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 72961e660..28dc8b452 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -104,12 +104,14 @@ var ( "external/google-benchmark": Bp2BuildDefaultTrueRecursively, "external/googletest": Bp2BuildDefaultTrueRecursively, "external/gwp_asan": Bp2BuildDefaultTrueRecursively, + "external/hamcrest": Bp2BuildDefaultTrueRecursively, "external/icu": Bp2BuildDefaultTrueRecursively, "external/icu/android_icu4j": Bp2BuildDefaultFalse, // java rules incomplete "external/icu/icu4j": Bp2BuildDefaultFalse, // java rules incomplete "external/javapoet": Bp2BuildDefaultTrueRecursively, "external/jemalloc_new": Bp2BuildDefaultTrueRecursively, "external/jsoncpp": Bp2BuildDefaultTrueRecursively, + "external/junit": Bp2BuildDefaultTrueRecursively, "external/libcap": Bp2BuildDefaultTrueRecursively, "external/libcxx": Bp2BuildDefaultTrueRecursively, "external/libcxxabi": Bp2BuildDefaultTrueRecursively, @@ -122,6 +124,7 @@ var ( "external/pcre": Bp2BuildDefaultTrueRecursively, "external/protobuf": Bp2BuildDefaultTrueRecursively, "external/python/six": Bp2BuildDefaultTrueRecursively, + "external/rappor": Bp2BuildDefaultTrueRecursively, "external/scudo": Bp2BuildDefaultTrueRecursively, "external/selinux/libselinux": Bp2BuildDefaultTrueRecursively, "external/selinux/libsepol": Bp2BuildDefaultTrueRecursively, @@ -313,6 +316,7 @@ var ( "host_bionic_linker_asm", // depends on extract_linker, a go binary. "host_bionic_linker_script", // depends on extract_linker, a go binary. "libc_musl_sysroot_bionic_arch_headers", // depends on soong_zip + "libc_musl_sysroot_zlib_headers", // depends on soong_zip and zip2zip "libc_musl_sysroot_bionic_headers", // 218405924, depends on soong_zip and generates duplicate srcs "libc_musl_sysroot_libc++_headers", "libc_musl_sysroot_libc++abi_headers", // depends on soong_zip, zip2zip "robolectric-sqlite4java-native", // depends on soong_zip, a go binary @@ -367,6 +371,10 @@ var ( "timezone-host", // depends on unconverted modules: art.module.api.annotations "truth-host-prebuilt", // depends on unconverted modules: truth-prebuilt "truth-prebuilt", // depends on unconverted modules: asm-7.0, guava + + // b/215723302; awaiting tz{data,_version} to then rename targets conflicting with srcs + "tzdata", + "tz_version", } Bp2buildCcLibraryStaticOnlyList = []string{} diff --git a/android/apex.go b/android/apex.go index b127f7410..555cbb55c 100644 --- a/android/apex.go +++ b/android/apex.go @@ -58,9 +58,6 @@ type ApexInfo struct { // to true. UsePlatformApis bool - // The list of SDK modules that the containing apexBundle depends on. - RequiredSdks SdkRefs - // List of Apex variant names that this module is associated with. This initially is the // same as the `ApexVariationName` field. Then when multiple apex variants are merged in // mergeApexVariations, ApexInfo struct of the merged variant holds the list of apexBundles @@ -110,9 +107,6 @@ func (i ApexInfo) AddJSONData(d *map[string]interface{}) { // thus wouldn't be merged. func (i ApexInfo) mergedName(ctx PathContext) string { name := "apex" + strconv.Itoa(i.MinSdkVersion.FinalOrFutureInt()) - for _, sdk := range i.RequiredSdks { - name += "_" + sdk.Name + "_" + sdk.Version - } return name } @@ -850,30 +844,13 @@ var minSdkVersionAllowlist = func(apiMap map[string]int) map[string]ApiLevel { } return list }(map[string]int{ - "android.net.ipsec.ike": 30, - "androidx.annotation_annotation-nodeps": 29, - "androidx.arch.core_core-common-nodeps": 29, - "androidx.collection_collection-nodeps": 29, - "androidx.collection_collection-ktx-nodeps": 30, - "androidx.concurrent_concurrent-futures-nodeps": 30, - "androidx.lifecycle_lifecycle-common-java8-nodeps": 30, - "androidx.lifecycle_lifecycle-common-nodeps": 29, - "androidx.room_room-common-nodeps": 30, "androidx-constraintlayout_constraintlayout-solver-nodeps": 29, "apache-commons-compress": 29, "bouncycastle_ike_digests": 30, "brotli-java": 29, - "captiveportal-lib": 28, - "error_prone_annotations": 30, "flatbuffer_headers": 30, - "framework-permission": 30, "gemmlowp_headers": 30, - "guava-listenablefuture-prebuilt-jar": 30, "ike-internals": 30, - "kotlinx-coroutines-android": 28, - "kotlinx-coroutines-android-nodeps": 30, - "kotlinx-coroutines-core": 28, - "kotlinx-coroutines-core-nodeps": 30, "libbrotli": 30, "libcrypto_static": 30, "libeigen": 30, @@ -888,14 +865,11 @@ var minSdkVersionAllowlist = func(apiMap map[string]int) map[string]ApiLevel { "libtextclassifier_hash_headers": 30, "libtextclassifier_hash_static": 30, "libtflite_kernel_utils": 30, - "libwatchdog": 29, "libzstd": 30, - "metrics-constants-protos": 28, "net-utils-framework-common": 29, "permissioncontroller-statsd": 28, "philox_random_headers": 30, "philox_random": 30, - "service-permission": 30, "tensorflow_headers": 30, "xz-java": 29, }) diff --git a/android/apex_test.go b/android/apex_test.go index 1e2f3bde4..0bf4c9c36 100644 --- a/android/apex_test.go +++ b/android/apex_test.go @@ -33,10 +33,10 @@ func Test_mergeApexVariations(t *testing.T) { { name: "single", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"foo", "apex10000"}, @@ -45,25 +45,25 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000_baz_1", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false}}, + {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false}}, wantAliases: [][2]string{ - {"bar", "apex10000_baz_1"}, - {"foo", "apex10000_baz_1"}, + {"bar", "apex10000"}, + {"foo", "apex10000"}, }, }, { name: "don't merge version", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", uncheckedFinalApiLevel(30), false, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex30", uncheckedFinalApiLevel(30), false, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - {"apex10000", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"apex30", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex30"}, @@ -73,11 +73,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge updatable", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, true, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -85,32 +85,17 @@ func Test_mergeApexVariations(t *testing.T) { }, }, { - name: "don't merge sdks", - in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, false, SdkRefs{{"baz", "2"}}, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - }, - wantMerged: []ApexInfo{ - {"apex10000_baz_2", FutureApiLevel, false, false, SdkRefs{{"baz", "2"}}, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - {"apex10000_baz_1", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - }, - wantAliases: [][2]string{ - {"bar", "apex10000_baz_2"}, - {"foo", "apex10000_baz_1"}, - }, - }, - { name: "don't merge when for prebuilt_apex", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, true, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, // This one should not be merged in with the others because it is for // a prebuilt_apex. - {"baz", FutureApiLevel, true, false, nil, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, + {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, - {"baz", FutureApiLevel, true, false, nil, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, + {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -120,11 +105,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge different UsePlatformApis but don't allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, true, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -134,11 +119,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge same UsePlatformApis and allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, true, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, true, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, true, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, true, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, true, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, diff --git a/android/bazel.go b/android/bazel.go index 4ef8d7881..67002ecf6 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -338,9 +338,19 @@ func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, di return false } -// MixedBuildsEnabled checks that a module is ready to be replaced by a +// MixedBuildsEnabled returns true if a module is ready to be replaced by a +// converted or handcrafted Bazel target. As a side effect, calling this +// method will also log whether this module is mixed build enabled for +// metrics reporting. +func MixedBuildsEnabled(ctx ModuleContext) bool { + mixedBuildEnabled := mixedBuildPossible(ctx) + ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled) + return mixedBuildEnabled +} + +// mixedBuildPossible returns true if a module is ready to be replaced by a // converted or handcrafted Bazel target. -func (b *BazelModuleBase) MixedBuildsEnabled(ctx ModuleContext) bool { +func mixedBuildPossible(ctx ModuleContext) bool { if ctx.Os() == Windows { // Windows toolchains are not currently supported. return false diff --git a/android/bazel_handler.go b/android/bazel_handler.go index d851a98bd..fa26fc87d 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -28,6 +28,7 @@ import ( "android/soong/bazel/cquery" "android/soong/shared" + "github.com/google/blueprint" "android/soong/bazel" ) @@ -101,6 +102,9 @@ type BazelContext interface { // Returns build statements which should get registered to reflect Bazel's outputs. BuildStatementsToRegister() []bazel.BuildStatement + + // Returns the depsets defined in Bazel's aquery response. + AqueryDepsets() []bazel.AqueryDepset } type bazelRunner interface { @@ -128,6 +132,9 @@ type bazelContext struct { // Build statements which should get registered to reflect Bazel's outputs. buildStatements []bazel.BuildStatement + + // Depsets which should be used for Bazel's build statements. + depsets []bazel.AqueryDepset } var _ BazelContext = &bazelContext{} @@ -175,6 +182,10 @@ func (m MockBazelContext) BuildStatementsToRegister() []bazel.BuildStatement { return []bazel.BuildStatement{} } +func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset { + return []bazel.AqueryDepset{} +} + var _ BazelContext = MockBazelContext{} func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { @@ -236,6 +247,10 @@ func (m noopBazelContext) BuildStatementsToRegister() []bazel.BuildStatement { return []bazel.BuildStatement{} } +func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset { + return []bazel.AqueryDepset{} +} + func NewBazelContext(c *config) (BazelContext, error) { // TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds" // are production ready. @@ -746,7 +761,7 @@ func (context *bazelContext) InvokeBazel() error { return err } - context.buildStatements, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) + context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) if err != nil { return err } @@ -772,6 +787,10 @@ func (context *bazelContext) BuildStatementsToRegister() []bazel.BuildStatement return context.buildStatements } +func (context *bazelContext) AqueryDepsets() []bazel.AqueryDepset { + return context.depsets +} + func (context *bazelContext) OutputBase() string { return context.paths.outputBase } @@ -804,6 +823,23 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { ctx.AddNinjaFileDeps(file) } + for _, depset := range ctx.Config().BazelContext.AqueryDepsets() { + var outputs []Path + for _, depsetDepId := range depset.TransitiveDepSetIds { + otherDepsetName := bazelDepsetName(depsetDepId) + outputs = append(outputs, PathForPhony(ctx, otherDepsetName)) + } + for _, artifactPath := range depset.DirectArtifacts { + outputs = append(outputs, PathForBazelOut(ctx, artifactPath)) + } + thisDepsetName := bazelDepsetName(depset.Id) + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)}, + Implicits: outputs, + }) + } + // Register bazel-owned build statements (obtained from the aquery invocation). for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { if len(buildStatement.Command) < 1 { @@ -838,6 +874,10 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { for _, inputPath := range buildStatement.InputPaths { cmd.Implicit(PathForBazelOut(ctx, inputPath)) } + for _, inputDepsetId := range buildStatement.InputDepsetIds { + otherDepsetName := bazelDepsetName(inputDepsetId) + cmd.Implicit(PathForPhony(ctx, otherDepsetName)) + } if depfile := buildStatement.Depfile; depfile != nil { cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile)) @@ -853,7 +893,8 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { // build statement have later timestamps than the outputs. rule.Restat() - rule.Build(fmt.Sprintf("bazel %d", index), buildStatement.Mnemonic) + desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) + rule.Build(fmt.Sprintf("bazel %d", index), desc) } } @@ -882,3 +923,7 @@ func GetConfigKey(ctx ModuleContext) configKey { osType: ctx.Os(), } } + +func bazelDepsetName(depsetId int) string { + return fmt.Sprintf("bazel_depset_%d", depsetId) +} diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go new file mode 100644 index 000000000..6339a7181 --- /dev/null +++ b/android/buildinfo_prop.go @@ -0,0 +1,182 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "strings" + + "github.com/google/blueprint/proptools" +) + +func init() { + ctx := InitRegistrationContext + ctx.RegisterSingletonModuleType("buildinfo_prop", buildinfoPropFactory) +} + +type buildinfoPropProperties struct { + // Whether this module is directly installable to one of the partitions. Default: true. + Installable *bool +} + +type buildinfoPropModule struct { + SingletonModuleBase + + properties buildinfoPropProperties + + outputFilePath OutputPath + installPath InstallPath +} + +var _ OutputFileProducer = (*buildinfoPropModule)(nil) + +func (p *buildinfoPropModule) installable() bool { + return proptools.BoolDefault(p.properties.Installable, true) +} + +// OutputFileProducer +func (p *buildinfoPropModule) OutputFiles(tag string) (Paths, error) { + if tag != "" { + return nil, fmt.Errorf("unsupported tag %q", tag) + } + return Paths{p.outputFilePath}, nil +} + +func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { + p.outputFilePath = PathForModuleOut(ctx, p.Name()).OutputPath + if !ctx.Config().KatiEnabled() { + WriteFileRule(ctx, p.outputFilePath, "# no buildinfo.prop if kati is disabled") + return + } + + rule := NewRuleBuilder(pctx, ctx) + cmd := rule.Command().Text("(") + + writeString := func(str string) { + cmd.Text(`echo "` + str + `" && `) + } + + writeString("# begin build properties") + writeString("# autogenerated by build/soong/android/buildinfo_prop.go") + + writeProp := func(key, value string) { + if strings.Contains(key, "=") { + panic(fmt.Errorf("wrong property key %q: key must not contain '='", key)) + } + writeString(key + "=" + value) + } + + config := ctx.Config() + + writeProp("ro.build.version.sdk", config.PlatformSdkVersion().String()) + writeProp("ro.build.version.preview_sdk", config.PlatformPreviewSdkVersion()) + writeProp("ro.build.version.codename", config.PlatformSdkCodename()) + writeProp("ro.build.version.all_codenames", strings.Join(config.PlatformVersionActiveCodenames(), ",")) + writeProp("ro.build.version.release", config.PlatformVersionLastStable()) + writeProp("ro.build.version.release_or_codename", config.PlatformVersionName()) + writeProp("ro.build.version.security_patch", config.PlatformSecurityPatch()) + writeProp("ro.build.version.base_os", config.PlatformBaseOS()) + writeProp("ro.build.version.min_supported_target_sdk", config.PlatformMinSupportedTargetSdkVersion()) + + if config.Eng() { + writeProp("ro.build.type", "eng") + } else if config.Debuggable() { + writeProp("ro.build.type", "userdebug") + } else { + writeProp("ro.build.type", "user") + } + + // Currently, only a few properties are implemented to unblock microdroid use case. + // TODO(b/189164487): support below properties as well and replace build/make/tools/buildinfo.sh + /* + if $BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT { + writeProp("ro.build.legacy.id", config.BuildID()) + } else { + writeProp("ro.build.id", config.BuildId()) + } + writeProp("ro.build.display.id", $BUILD_DISPLAY_ID) + writeProp("ro.build.version.incremental", $BUILD_NUMBER) + writeProp("ro.build.version.preview_sdk_fingerprint", $PLATFORM_PREVIEW_SDK_FINGERPRINT) + writeProp("ro.build.version.known_codenames", $PLATFORM_VERSION_KNOWN_CODENAMES) + writeProp("ro.build.version.release_or_preview_display", $PLATFORM_DISPLAY_VERSION) + writeProp("ro.build.date", `$DATE`) + writeProp("ro.build.date.utc", `$DATE +%s`) + writeProp("ro.build.user", $BUILD_USERNAME) + writeProp("ro.build.host", $BUILD_HOSTNAME) + writeProp("ro.build.tags", $BUILD_VERSION_TAGS) + writeProp("ro.build.flavor", $TARGET_BUILD_FLAVOR) + // These values are deprecated, use "ro.product.cpu.abilist" + // instead (see below). + writeString("# ro.product.cpu.abi and ro.product.cpu.abi2 are obsolete,") + writeString("# use ro.product.cpu.abilist instead.") + writeProp("ro.product.cpu.abi", $TARGET_CPU_ABI) + if [ -n "$TARGET_CPU_ABI2" ] { + writeProp("ro.product.cpu.abi2", $TARGET_CPU_ABI2) + } + + if [ -n "$PRODUCT_DEFAULT_LOCALE" ] { + writeProp("ro.product.locale", $PRODUCT_DEFAULT_LOCALE) + } + writeProp("ro.wifi.channels", $PRODUCT_DEFAULT_WIFI_CHANNELS) + writeString("# ro.build.product is obsolete; use ro.product.device") + writeProp("ro.build.product", $TARGET_DEVICE) + + writeString("# Do not try to parse description or thumbprint") + writeProp("ro.build.description", $PRIVATE_BUILD_DESC) + if [ -n "$BUILD_THUMBPRINT" ] { + writeProp("ro.build.thumbprint", $BUILD_THUMBPRINT) + } + */ + + writeString("# end build properties") + + cmd.Text("true) > ").Output(p.outputFilePath) + rule.Build("build.prop", "generating build.prop") + + if !p.installable() { + p.SkipInstall() + } + + p.installPath = PathForModuleInstall(ctx) + ctx.InstallFile(p.installPath, p.Name(), p.outputFilePath) +} + +func (f *buildinfoPropModule) GenerateSingletonBuildActions(ctx SingletonContext) { + // does nothing; buildinfo_prop is a singeton because two buildinfo modules don't make sense. +} + +func (p *buildinfoPropModule) AndroidMkEntries() []AndroidMkEntries { + return []AndroidMkEntries{AndroidMkEntries{ + Class: "ETC", + OutputFile: OptionalPathForPath(p.outputFilePath), + ExtraEntries: []AndroidMkExtraEntriesFunc{ + func(ctx AndroidMkExtraEntriesContext, entries *AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", p.installPath.String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.outputFilePath.Base()) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !p.installable()) + }, + }, + }} +} + +// buildinfo_prop module generates a build.prop file, which contains a set of common +// system/build.prop properties, such as ro.build.version.*. Not all properties are implemented; +// currently this module is only for microdroid. +func buildinfoPropFactory() SingletonModule { + module := &buildinfoPropModule{} + module.AddProperties(&module.properties) + InitAndroidModule(module) + return module +} diff --git a/android/config.go b/android/config.go index cb2fc6160..d69521747 100644 --- a/android/config.go +++ b/android/config.go @@ -170,6 +170,10 @@ type config struct { ninjaFileDepsSet sync.Map OncePer + + mixedBuildsLock sync.Mutex + mixedBuildEnabledModules map[string]struct{} + mixedBuildDisabledModules map[string]struct{} } type deviceConfig struct { @@ -375,7 +379,9 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string // passed to PathForSource or PathForModuleSrc. TestAllowNonExistentPaths: true, - BazelContext: noopBazelContext{}, + BazelContext: noopBazelContext{}, + mixedBuildDisabledModules: make(map[string]struct{}), + mixedBuildEnabledModules: make(map[string]struct{}), } config.deviceConfig = &deviceConfig{ config: config, @@ -466,8 +472,10 @@ func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir strin runGoTests: runGoTests, multilibConflicts: make(map[ArchType]bool), - moduleListFile: moduleListFile, - fs: pathtools.NewOsFs(absSrcDir), + moduleListFile: moduleListFile, + fs: pathtools.NewOsFs(absSrcDir), + mixedBuildDisabledModules: make(map[string]struct{}), + mixedBuildEnabledModules: make(map[string]struct{}), } config.deviceConfig = &deviceConfig{ @@ -777,8 +785,12 @@ func (c *config) PlatformBaseOS() string { return String(c.productVariables.Platform_base_os) } +func (c *config) PlatformVersionLastStable() string { + return String(c.productVariables.Platform_version_last_stable) +} + func (c *config) MinSupportedSdkVersion() ApiLevel { - return uncheckedFinalApiLevel(16) + return uncheckedFinalApiLevel(19) } func (c *config) FinalApiLevels() []ApiLevel { @@ -1482,6 +1494,10 @@ func (c *config) MissingUsesLibraries() []string { return c.productVariables.MissingUsesLibraries } +func (c *config) TargetMultitreeUpdateMeta() bool { + return c.productVariables.MultitreeUpdateMeta +} + func (c *deviceConfig) DeviceArch() string { return String(c.config.productVariables.DeviceArch) } @@ -2030,3 +2046,14 @@ func (c *config) RBEWrapper() string { func (c *config) UseHostMusl() bool { return Bool(c.productVariables.HostMusl) } + +func (c *config) LogMixedBuild(ctx ModuleContext, useBazel bool) { + moduleName := ctx.Module().Name() + c.mixedBuildsLock.Lock() + defer c.mixedBuildsLock.Unlock() + if useBazel { + c.mixedBuildEnabledModules[moduleName] = struct{}{} + } else { + c.mixedBuildDisabledModules[moduleName] = struct{}{} + } +} diff --git a/android/filegroup.go b/android/filegroup.go index 50356d1c3..1bf5e07c2 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -115,7 +115,7 @@ func FileGroupFactory() Module { } func (fg *fileGroup) maybeGenerateBazelBuildActions(ctx ModuleContext) { - if !fg.MixedBuildsEnabled(ctx) { + if !MixedBuildsEnabled(ctx) { return } diff --git a/android/hooks.go b/android/hooks.go index 5e3a4a7e7..2ad3b5fa5 100644 --- a/android/hooks.go +++ b/android/hooks.go @@ -89,8 +89,17 @@ func (l *loadHookContext) PrependProperties(props ...interface{}) { l.appendPrependHelper(props, proptools.PrependMatchingProperties) } -func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { - inherited := []interface{}{&l.Module().base().commonProperties} +func (l *loadHookContext) createModule(factory blueprint.ModuleFactory, name string, props ...interface{}) blueprint.Module { + return l.bp.CreateModule(factory, name, props...) +} + +type createModuleContext interface { + Module() Module + createModule(blueprint.ModuleFactory, string, ...interface{}) blueprint.Module +} + +func createModule(ctx createModuleContext, factory ModuleFactory, ext string, props ...interface{}) Module { + inherited := []interface{}{&ctx.Module().base().commonProperties} var typeName string if typeNameLookup, ok := ModuleTypeByFactory()[reflect.ValueOf(factory)]; ok { @@ -101,12 +110,12 @@ func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface filePath, _ := factoryFunc.FileLine(factoryPtr) typeName = fmt.Sprintf("%s_%s", path.Base(filePath), factoryFunc.Name()) } - typeName = typeName + "_loadHookModule" + typeName = typeName + "_" + ext - module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module) + module := ctx.createModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module) - if l.Module().base().variableProperties != nil && module.base().variableProperties != nil { - src := l.Module().base().variableProperties + if ctx.Module().base().variableProperties != nil && module.base().variableProperties != nil { + src := ctx.Module().base().variableProperties dst := []interface{}{ module.base().variableProperties, // Put an empty copy of the src properties into dst so that properties in src that are not in dst @@ -122,6 +131,10 @@ func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface return module } +func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { + return createModule(l, factory, "_loadHookModule", props...) +} + func (l *loadHookContext) registerScopedModuleType(name string, factory blueprint.ModuleFactory) { l.bp.RegisterScopedModuleType(name, factory) } diff --git a/android/metrics.go b/android/metrics.go index 9038bde9f..1580f82b1 100644 --- a/android/metrics.go +++ b/android/metrics.go @@ -17,6 +17,7 @@ package android import ( "io/ioutil" "runtime" + "sort" "github.com/google/blueprint/metrics" "google.golang.org/protobuf/proto" @@ -78,6 +79,23 @@ func collectMetrics(config Config, eventHandler metrics.EventHandler) *soong_met } metrics.Events = append(metrics.Events, &perfInfo) } + mixedBuildsInfo := soong_metrics_proto.MixedBuildsInfo{} + mixedBuildEnabledModules := make([]string, 0, len(config.mixedBuildEnabledModules)) + for module, _ := range config.mixedBuildEnabledModules { + mixedBuildEnabledModules = append(mixedBuildEnabledModules, module) + } + + mixedBuildDisabledModules := make([]string, 0, len(config.mixedBuildDisabledModules)) + for module, _ := range config.mixedBuildDisabledModules { + mixedBuildDisabledModules = append(mixedBuildDisabledModules, module) + } + // Sorted for deterministic output. + sort.Strings(mixedBuildEnabledModules) + sort.Strings(mixedBuildDisabledModules) + + mixedBuildsInfo.MixedBuildEnabledModules = mixedBuildEnabledModules + mixedBuildsInfo.MixedBuildDisabledModules = mixedBuildDisabledModules + metrics.MixedBuildsInfo = &mixedBuildsInfo return metrics } diff --git a/android/mutator.go b/android/mutator.go index 739e4ee6d..02a614353 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -15,12 +15,9 @@ package android import ( - "reflect" - "android/soong/bazel" "github.com/google/blueprint" - "github.com/google/blueprint/proptools" ) // Phases: @@ -553,29 +550,16 @@ func (t *topDownMutatorContext) Rename(name string) { t.Module().base().commonProperties.DebugName = name } -func (t *topDownMutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { - inherited := []interface{}{&t.Module().base().commonProperties} - module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module) - - if t.Module().base().variableProperties != nil && module.base().variableProperties != nil { - src := t.Module().base().variableProperties - dst := []interface{}{ - module.base().variableProperties, - // Put an empty copy of the src properties into dst so that properties in src that are not in dst - // don't cause a "failed to find property to extend" error. - proptools.CloneEmptyProperties(reflect.ValueOf(src)).Interface(), - } - err := proptools.AppendMatchingProperties(dst, src, nil) - if err != nil { - panic(err) - } - } +func (t *topDownMutatorContext) createModule(factory blueprint.ModuleFactory, name string, props ...interface{}) blueprint.Module { + return t.bp.CreateModule(factory, name, props...) +} - return module +func (t *topDownMutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { + return createModule(t, factory, "_topDownMutatorModule", props...) } func (t *topDownMutatorContext) createModuleWithoutInheritance(factory ModuleFactory, props ...interface{}) Module { - module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), props...).(Module) + module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), "", props...).(Module) return module } diff --git a/android/paths.go b/android/paths.go index e7829b961..e0e5ae580 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1057,7 +1057,8 @@ func safePathForSource(ctx PathContext, pathComponents ...string) (SourcePath, e } // absolute path already checked by validateSafePath - if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) { + // special-case api surface gen files for now + if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") { return ret, fmt.Errorf("source path %q is in output", ret.String()) } @@ -1073,7 +1074,8 @@ func pathForSource(ctx PathContext, pathComponents ...string) (SourcePath, error } // absolute path already checked by validatePath - if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) { + // special-case for now + if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") { return ret, fmt.Errorf("source path %q is in output", ret.String()) } diff --git a/android/sdk.go b/android/sdk.go index 1d63d7a94..3a5624030 100644 --- a/android/sdk.go +++ b/android/sdk.go @@ -23,24 +23,8 @@ import ( "github.com/google/blueprint/proptools" ) -// RequiredSdks provides access to the set of SDKs required by an APEX and its contents. -// -// Extracted from SdkAware to make it easier to define custom subsets of the -// SdkAware interface and improve code navigation within the IDE. -// -// In addition to its use in SdkAware this interface must also be implemented by -// APEX to specify the SDKs required by that module and its contents. e.g. APEX -// is expected to implement RequiredSdks() by reading its own properties like -// `uses_sdks`. -type RequiredSdks interface { - // RequiredSdks returns the set of SDKs required by an APEX and its contents. - RequiredSdks() SdkRefs -} - // sdkAwareWithoutModule is provided simply to improve code navigation with the IDE. type sdkAwareWithoutModule interface { - RequiredSdks - // SdkMemberComponentName will return the name to use for a component of this module based on the // base name of this module. // @@ -81,7 +65,6 @@ type sdkAwareWithoutModule interface { ContainingSdk() SdkRef MemberName() string - BuildWithSdks(sdks SdkRefs) } // SdkAware is the interface that must be supported by any module to become a member of SDK or to be @@ -150,9 +133,6 @@ type sdkProperties struct { // The SDK that this module is a member of. nil if it is not a member of any SDK ContainingSdk *SdkRef `blueprint:"mutated"` - // The list of SDK names and versions that are used to build this module - RequiredSdks SdkRefs `blueprint:"mutated"` - // Name of the module that this sdk member is representing Sdk_member_name *string } @@ -208,16 +188,6 @@ func (s *SdkBase) MemberName() string { return proptools.String(s.properties.Sdk_member_name) } -// BuildWithSdks is used to mark that this module has to be built with the given SDK(s). -func (s *SdkBase) BuildWithSdks(sdks SdkRefs) { - s.properties.RequiredSdks = sdks -} - -// RequiredSdks returns the SDK(s) that this module has to be built with -func (s *SdkBase) RequiredSdks() SdkRefs { - return s.properties.RequiredSdks -} - // InitSdkAwareModule initializes the SdkBase struct. This must be called by all modules including // SdkBase. func InitSdkAwareModule(m SdkAware) { diff --git a/android/variable.go b/android/variable.go index 077b81097..9478c0c9e 100644 --- a/android/variable.go +++ b/android/variable.go @@ -199,6 +199,7 @@ type productVariables struct { Platform_preview_sdk_version *string `json:",omitempty"` Platform_min_supported_target_sdk_version *string `json:",omitempty"` Platform_base_os *string `json:",omitempty"` + Platform_version_last_stable *string `json:",omitempty"` DeviceName *string `json:",omitempty"` DeviceProduct *string `json:",omitempty"` @@ -351,6 +352,8 @@ type productVariables struct { RecoverySnapshotDirsIncluded []string `json:",omitempty"` HostFakeSnapshotEnabled bool `json:",omitempty"` + MultitreeUpdateMeta bool `json:",omitempty"` + BoardVendorSepolicyDirs []string `json:",omitempty"` BoardOdmSepolicyDirs []string `json:",omitempty"` BoardReqdMaskPolicy []string `json:",omitempty"` @@ -460,12 +463,13 @@ func (v *productVariables) SetDefaultConfig() { *v = productVariables{ BuildNumberFile: stringPtr("build_number.txt"), - Platform_version_name: stringPtr("S"), - Platform_sdk_version: intPtr(30), - Platform_sdk_codename: stringPtr("S"), - Platform_sdk_final: boolPtr(false), - Platform_version_active_codenames: []string{"S"}, - Platform_vndk_version: stringPtr("S"), + Platform_version_name: stringPtr("S"), + Platform_base_sdk_extension_version: intPtr(30), + Platform_sdk_version: intPtr(30), + Platform_sdk_codename: stringPtr("S"), + Platform_sdk_final: boolPtr(false), + Platform_version_active_codenames: []string{"S"}, + Platform_vndk_version: stringPtr("S"), HostArch: stringPtr("x86_64"), HostSecondaryArch: stringPtr("x86"), diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go index 803032649..416e43005 100644 --- a/androidmk/parser/make_strings.go +++ b/androidmk/parser/make_strings.go @@ -279,7 +279,7 @@ func (ms *MakeString) TrimRightOne() { func (ms *MakeString) EndsWith(ch rune) bool { s := ms.Strings[len(ms.Strings)-1] - return s[len(s)-1] == uint8(ch) + return len(s) > 0 && s[len(s)-1] == uint8(ch) } func (ms *MakeString) ReplaceLiteral(input string, output string) { diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go index fbb289bb7..e243eceef 100644 --- a/androidmk/parser/make_strings_test.go +++ b/androidmk/parser/make_strings_test.go @@ -217,6 +217,36 @@ func TestMakeStringWords(t *testing.T) { } } +var endsWithTestCases = []struct { + in *MakeString + endsWith rune + expected bool +}{ + { + in: genMakeString("foo", "X", "bar ="), + endsWith: '=', + expected: true, + }, + { + in: genMakeString("foo", "X", "bar ="), + endsWith: ':', + expected: false, + }, + { + in: genMakeString("foo", "X", ""), + endsWith: '=', + expected: false, + }, +} + +func TestMakeStringEndsWith(t *testing.T) { + for _, test := range endsWithTestCases { + if test.in.EndsWith(test.endsWith) != test.expected { + t.Errorf("with:\n%q\nexpected:\n%t\ngot:\n%t", test.in.Dump(), test.expected, !test.expected) + } + } +} + func dumpArray(a []*MakeString) string { ret := make([]string, len(a)) diff --git a/apex/Android.bp b/apex/Android.bp index 41224ecd5..d3417c24d 100644 --- a/apex/Android.bp +++ b/apex/Android.bp @@ -14,6 +14,7 @@ bootstrap_go_package { "soong-cc", "soong-filesystem", "soong-java", + "soong-multitree", "soong-provenance", "soong-python", "soong-rust", diff --git a/apex/apex.go b/apex/apex.go index 62013cf63..b3398156d 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -34,6 +34,7 @@ import ( prebuilt_etc "android/soong/etc" "android/soong/filesystem" "android/soong/java" + "android/soong/multitree" "android/soong/python" "android/soong/rust" "android/soong/sh" @@ -159,12 +160,6 @@ type apexBundleProperties struct { // or else conflicting build rules may be created. Multi_install_skip_symbol_files *bool - // List of SDKs that are used to build this APEX. A reference to an SDK should be either - // `name#version` or `name` which is an alias for `name#current`. If left empty, - // `platform#current` is implied. This value affects all modules included in this APEX. In - // other words, they are also built with the SDKs specified here. - Uses_sdks []string - // The type of APEX to build. Controls what the APEX payload is. Either 'image', 'zip' or // 'both'. When set to image, contents are stored in a filesystem image inside a zip // container. When set to zip, contents are stored in a zip container directly. This type is @@ -359,6 +354,7 @@ type apexBundle struct { android.OverridableModuleBase android.SdkBase android.BazelModuleBase + multitree.ExportableModuleBase // Properties properties apexBundleProperties @@ -790,19 +786,6 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { commonVariation := ctx.Config().AndroidCommonTarget.Variations() ctx.AddFarVariationDependencies(commonVariation, fsTag, a.properties.Filesystems...) ctx.AddFarVariationDependencies(commonVariation, compatConfigTag, a.properties.Compat_configs...) - - // Marks that this APEX (in fact all the modules in it) has to be built with the given SDKs. - // This field currently isn't used. - // TODO(jiyong): consider dropping this feature - // TODO(jiyong): ensure that all apexes are with non-empty uses_sdks - if len(a.properties.Uses_sdks) > 0 { - sdkRefs := []android.SdkRef{} - for _, str := range a.properties.Uses_sdks { - parsed := android.ParseSdkRef(ctx, str, "uses_sdks") - sdkRefs = append(sdkRefs, parsed) - } - a.BuildWithSdks(sdkRefs) - } } // DepsMutator for the overridden properties. @@ -967,7 +950,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { apexInfo := android.ApexInfo{ ApexVariationName: apexVariationName, MinSdkVersion: minSdkVersion, - RequiredSdks: a.RequiredSdks(), Updatable: a.Updatable(), UsePlatformApis: a.UsePlatformApis(), InApexVariants: []string{apexVariationName}, @@ -1360,6 +1342,21 @@ func (a *apexBundle) OutputFiles(tag string) (android.Paths, error) { } } +var _ multitree.Exportable = (*apexBundle)(nil) + +func (a *apexBundle) Exportable() bool { + if a.properties.ApexType == flattenedApex { + return false + } + return true +} + +func (a *apexBundle) TaggedOutputs() map[string]android.Paths { + ret := make(map[string]android.Paths) + ret["apex"] = android.Paths{a.outputFile} + return ret +} + var _ cc.Coverage = (*apexBundle)(nil) // Implements cc.Coverage @@ -2386,6 +2383,7 @@ func newApexBundle() *apexBundle { android.InitSdkAwareModule(module) android.InitOverridableModule(module, &module.overridableProperties.Overrides) android.InitBazelModule(module) + multitree.InitExportableModule(module) return module } @@ -3519,7 +3517,7 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { props := bazel.BazelTargetModuleProperties{ Rule_class: "apex", - Bzl_load_location: "//build/bazel/rules:apex.bzl", + Bzl_load_location: "//build/bazel/rules/apex:apex.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: a.Name()}, attrs) diff --git a/apex/key.go b/apex/key.go index 829410ed2..9c5bb05e7 100644 --- a/apex/key.go +++ b/apex/key.go @@ -230,7 +230,7 @@ func apexKeyBp2BuildInternal(ctx android.TopDownMutatorContext, module *apexKey) props := bazel.BazelTargetModuleProperties{ Rule_class: "apex_key", - Bzl_load_location: "//build/bazel/rules:apex_key.bzl", + Bzl_load_location: "//build/bazel/rules/apex:apex_key.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs) diff --git a/bazel/aquery.go b/bazel/aquery.go index fd8cf677d..e05cbd6a8 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -43,6 +43,18 @@ type KeyValuePair struct { Value string } +// AqueryDepset is a depset definition from Bazel's aquery response. This is +// akin to the `depSetOfFiles` in the response proto, except that direct +// artifacts are enumerated by full path instead of by ID. +// A depset is a data structure for efficient transitive handling of artifact +// paths. A single depset consists of one or more artifact paths and one or +// more "child" depsets. +type AqueryDepset struct { + Id int + DirectArtifacts []string + TransitiveDepSetIds []int +} + // depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles. // Represents a data structure containing one or more files. Depsets in Bazel are an efficient // data structure for storing large numbers of file paths. @@ -79,21 +91,21 @@ type BuildStatement struct { Command string Depfile *string OutputPaths []string - InputPaths []string SymlinkPaths []string Env []KeyValuePair Mnemonic string + + // Inputs of this build statement, either as unexpanded depsets or expanded + // input paths. There should be no overlap between these fields; an input + // path should either be included as part of an unexpanded depset or a raw + // input path string, but not both. + InputDepsetIds []int + InputPaths []string } // A helper type for aquery processing which facilitates retrieval of path IDs from their // less readable Bazel structures (depset and path fragment). type aqueryArtifactHandler struct { - // Maps middleman artifact Id to input artifact depset ID. - // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, - // if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then, - // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for - // that action instead. - middlemanIdToDepsetIds map[int][]int // Maps depset Id to depset struct. depsetIdToDepset map[int]depSetOfFiles // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening @@ -132,12 +144,11 @@ func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler artifactIdToPath[artifact.Id] = artifactPath } - depsetIdToDepset := map[int]depSetOfFiles{} - for _, depset := range aqueryResult.DepSetOfFiles { - depsetIdToDepset[depset.Id] = depset - } - - // Do a pass through all actions to identify which artifacts are middleman artifacts. + // Map middleman artifact Id to input artifact depset ID. + // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, + // if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then, + // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for + // that action instead. middlemanIdToDepsetIds := map[int][]int{} for _, actionEntry := range aqueryResult.Actions { if actionEntry.Mnemonic == "Middleman" { @@ -146,14 +157,64 @@ func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler } } } + + // Store all depset IDs to validate all depset links are resolvable. + depsetIds := map[int]bool{} + for _, depset := range aqueryResult.DepSetOfFiles { + depsetIds[depset.Id] = true + } + + depsetIdToDepset := map[int]depSetOfFiles{} + // Validate and adjust aqueryResult.DepSetOfFiles values. + for _, depset := range aqueryResult.DepSetOfFiles { + filteredArtifactIds := []int{} + for _, artifactId := range depset.DirectArtifactIds { + path, pathExists := artifactIdToPath[artifactId] + if !pathExists { + return nil, fmt.Errorf("undefined input artifactId %d", artifactId) + } + // Filter out any inputs which are universally dropped, and swap middleman + // artifacts with their corresponding depsets. + if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[artifactId]; isMiddleman { + // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts. + depset.TransitiveDepSetIds = append(depset.TransitiveDepSetIds, depsetsToUse...) + } else if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) { + // Drop these artifacts. + // See go/python-binary-host-mixed-build for more details. + // 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of + // Bazel py_binary targets, so there is no Ninja build statements generated for creating it. + // 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, + // but it doesn't contain sufficient information so no Ninja build statements are generated + // for creating it. + // So in mixed build mode, when these two are used as input of some Ninja build statement, + // since there is no build statement to create them, they should be removed from input paths. + // TODO(b/197135294): Clean up this custom runfiles handling logic when + // SourceSymlinkManifest and SymlinkTree actions are supported. + } else { + // TODO(b/216194240): Filter out bazel tools. + filteredArtifactIds = append(filteredArtifactIds, artifactId) + } + } + depset.DirectArtifactIds = filteredArtifactIds + for _, childDepsetId := range depset.TransitiveDepSetIds { + if _, exists := depsetIds[childDepsetId]; !exists { + return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id) + } + } + depsetIdToDepset[depset.Id] = depset + } + return &aqueryArtifactHandler{ - middlemanIdToDepsetIds: middlemanIdToDepsetIds, depsetIdToDepset: depsetIdToDepset, depsetIdToArtifactIdsCache: map[int][]int{}, artifactIdToPath: artifactIdToPath, }, nil } +// getInputPaths flattens the depsets of the given IDs and returns all transitive +// input paths contained in these depsets. +// This is a potentially expensive operation, and should not be invoked except +// for actions which need specialized input handling. func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error) { inputPaths := []string{} @@ -163,48 +224,15 @@ func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error) return nil, err } for _, inputId := range inputArtifacts { - if middlemanInputDepsetIds, isMiddlemanArtifact := a.middlemanIdToDepsetIds[inputId]; isMiddlemanArtifact { - // Add all inputs from middleman actions which created middleman artifacts which are - // in the inputs for this action. - swappedInputPaths, err := a.getInputPaths(middlemanInputDepsetIds) - if err != nil { - return nil, err - } - inputPaths = append(inputPaths, swappedInputPaths...) - } else { - inputPath, exists := a.artifactIdToPath[inputId] - if !exists { - return nil, fmt.Errorf("undefined input artifactId %d", inputId) - } - inputPaths = append(inputPaths, inputPath) + inputPath, exists := a.artifactIdToPath[inputId] + if !exists { + return nil, fmt.Errorf("undefined input artifactId %d", inputId) } + inputPaths = append(inputPaths, inputPath) } } - // TODO(b/197135294): Clean up this custom runfiles handling logic when - // SourceSymlinkManifest and SymlinkTree actions are supported. - filteredInputPaths := filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths) - - return filteredInputPaths, nil -} - -// See go/python-binary-host-mixed-build for more details. -// 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of -// Bazel py_binary targets, so there is no Ninja build statements generated for creating it. -// 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, -// but it doesn't contain sufficient information so no Ninja build statements are generated -// for creating it. -// So in mixed build mode, when these two are used as input of some Ninja build statement, -// since there is no build statement to create them, they should be removed from input paths. -func filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths []string) []string { - filteredInputPaths := []string{} - for _, path := range inputPaths { - if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) { - continue - } - filteredInputPaths = append(filteredInputPaths, path) - } - return filteredInputPaths + return inputPaths, nil } func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) { @@ -227,115 +255,233 @@ func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, er } } -// AqueryBuildStatements returns an array of BuildStatements which should be registered (and output -// to a ninja file) to correspond one-to-one with the given action graph json proto (from a bazel -// aquery invocation). -func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) { +// AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset +// which should be registered (and output to a ninja file) to correspond with Bazel's +// action graph, as described by the given action graph json proto. +// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets +// are one-to-one with Bazel's depSetOfFiles objects. +func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) { buildStatements := []BuildStatement{} + depsets := []AqueryDepset{} var aqueryResult actionGraphContainer err := json.Unmarshal(aqueryJsonProto, &aqueryResult) if err != nil { - return nil, err + return nil, nil, err } aqueryHandler, err := newAqueryHandler(aqueryResult) if err != nil { - return nil, err + return nil, nil, err } for _, actionEntry := range aqueryResult.Actions { if shouldSkipAction(actionEntry) { continue } - outputPaths := []string{} - var depfile *string - for _, outputId := range actionEntry.OutputIds { - outputPath, exists := aqueryHandler.artifactIdToPath[outputId] - if !exists { - return nil, fmt.Errorf("undefined outputId %d", outputId) - } - ext := filepath.Ext(outputPath) - if ext == ".d" { - if depfile != nil { - return nil, fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) - } else { - depfile = &outputPath - } - } else { - outputPaths = append(outputPaths, outputPath) - } - } - inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - - buildStatement := BuildStatement{ - Command: strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " "), - Depfile: depfile, - OutputPaths: outputPaths, - InputPaths: inputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } + var buildStatement BuildStatement if isSymlinkAction(actionEntry) { - if len(inputPaths) != 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) - } - out := outputPaths[0] - outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) - out = proptools.ShellEscapeIncludingSpaces(out) - in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) - // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). - buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) - buildStatement.SymlinkPaths = outputPaths[:] + buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry) } else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 { - if len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) - } - expandedTemplateContent := expandTemplateContent(actionEntry) - // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, - // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might - // change \n to space and mess up the format of Python programs. - // sed is used to convert \\n back to \n before saving to output file. - // See go/python-binary-host-mixed-build for more details. - command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, - escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) - buildStatement.Command = command + buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry) } else if isPythonZipperAction(actionEntry) { - if len(inputPaths) < 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths) - } - buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement) - buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0]) - // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements. - // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input, - // which is not sufficient without the python zip file from which runfiles directory is created for py_binary. - // - // The following logic relies on that Bazel aquery output returns actions in the order that - // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions - // in that order, the following logic might not find the build statement generated for Python binary - // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output. - // See go/python-binary-host-mixed-build for more details. - pythonZipFilePath := outputPaths[0] - pyBinaryFound := false - for i, _ := range buildStatements { - if len(buildStatements[i].OutputPaths) == 1 && buildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath { - buildStatements[i].InputPaths = append(buildStatements[i].InputPaths, pythonZipFilePath) - pyBinaryFound = true - } - } - if !pyBinaryFound { - return nil, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths) - } + buildStatement, err = aqueryHandler.pythonZipperActionBuildStatement(actionEntry, buildStatements) } else if len(actionEntry.Arguments) < 1 { - return nil, fmt.Errorf("received action with no command: [%v]", buildStatement) + return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic) + } else { + buildStatement, err = aqueryHandler.normalActionBuildStatement(actionEntry) + } + + if err != nil { + return nil, nil, err } buildStatements = append(buildStatements, buildStatement) } - return buildStatements, nil + // Iterate over depset IDs in the initial aquery order to preserve determinism. + for _, depset := range aqueryResult.DepSetOfFiles { + // Use the depset in the aqueryHandler, as this contains the augmented depsets. + depset = aqueryHandler.depsetIdToDepset[depset.Id] + directPaths := []string{} + for _, artifactId := range depset.DirectArtifactIds { + pathString := aqueryHandler.artifactIdToPath[artifactId] + directPaths = append(directPaths, pathString) + } + aqueryDepset := AqueryDepset{ + Id: depset.Id, + DirectArtifacts: directPaths, + TransitiveDepSetIds: depset.TransitiveDepSetIds, + } + depsets = append(depsets, aqueryDepset) + } + return buildStatements, depsets, nil +} + +func (aqueryHandler *aqueryArtifactHandler) validateInputDepsets(inputDepsetIds []int) ([]int, error) { + // Validate input depsets correspond to real depsets. + for _, depsetId := range inputDepsetIds { + if _, exists := aqueryHandler.depsetIdToDepset[depsetId]; !exists { + return nil, fmt.Errorf("undefined input depsetId %d", depsetId) + } + } + return inputDepsetIds, nil +} + +func (aqueryHandler *aqueryArtifactHandler) normalActionBuildStatement(actionEntry action) (BuildStatement, error) { + command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ") + inputDepsetIds, err := aqueryHandler.validateInputDepsets(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputDepsetIds: inputDepsetIds, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + } + return buildStatement, nil +} + +func (aqueryHandler *aqueryArtifactHandler) pythonZipperActionBuildStatement(actionEntry action, prevBuildStatements []BuildStatement) (BuildStatement, error) { + inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + + if len(inputPaths) < 1 || len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths) + } + command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ") + inputPaths, command = removePy3wrapperScript(inputPaths, command) + command = addCommandForPyBinaryRunfilesDir(command, inputPaths[0], outputPaths[0]) + // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements. + // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input, + // which is not sufficient without the python zip file from which runfiles directory is created for py_binary. + // + // The following logic relies on that Bazel aquery output returns actions in the order that + // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions + // in that order, the following logic might not find the build statement generated for Python binary + // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output. + // See go/python-binary-host-mixed-build for more details. + pythonZipFilePath := outputPaths[0] + pyBinaryFound := false + for i, _ := range prevBuildStatements { + if len(prevBuildStatements[i].OutputPaths) == 1 && prevBuildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath { + prevBuildStatements[i].InputPaths = append(prevBuildStatements[i].InputPaths, pythonZipFilePath) + pyBinaryFound = true + } + } + if !pyBinaryFound { + return BuildStatement{}, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths) + } + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputPaths: inputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + } + return buildStatement, nil +} + +func (aqueryHandler *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + if len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) + } + expandedTemplateContent := expandTemplateContent(actionEntry) + // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, + // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might + // change \n to space and mess up the format of Python programs. + // sed is used to convert \\n back to \n before saving to output file. + // See go/python-binary-host-mixed-build for more details. + command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, + escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) + inputDepsetIds, err := aqueryHandler.validateInputDepsets(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputDepsetIds: inputDepsetIds, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + } + return buildStatement, nil +} + +func (aqueryHandler *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + + inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + if len(inputPaths) != 1 || len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) + } + out := outputPaths[0] + outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) + out = proptools.ShellEscapeIncludingSpaces(out) + in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) + // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). + command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) + symlinkPaths := outputPaths[:] + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputPaths: inputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + SymlinkPaths: symlinkPaths, + } + return buildStatement, nil +} + +func (aqueryHandler *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths []string, depfile *string, err error) { + for _, outputId := range actionEntry.OutputIds { + outputPath, exists := aqueryHandler.artifactIdToPath[outputId] + if !exists { + err = fmt.Errorf("undefined outputId %d", outputId) + return + } + ext := filepath.Ext(outputPath) + if ext == ".d" { + if depfile != nil { + err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) + return + } else { + depfile = &outputPath + } + } else { + outputPaths = append(outputPaths, outputPath) + } + } + return } // expandTemplateContent substitutes the tokens in a template. @@ -372,10 +518,10 @@ func escapeCommandlineArgument(str string) string { // removed from input paths and command of creating python zip file. // See go/python-binary-host-mixed-build for more details. // TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode. -func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) { +func removePy3wrapperScript(inputPaths []string, command string) (newInputPaths []string, newCommand string) { // Remove from inputs filteredInputPaths := []string{} - for _, path := range bs.InputPaths { + for _, path := range inputPaths { if !strings.HasSuffix(path, py3wrapperFileName) { filteredInputPaths = append(filteredInputPaths, path) } @@ -384,7 +530,7 @@ func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newComma // Remove from command line var re = regexp.MustCompile(`\S*` + py3wrapperFileName) - newCommand = re.ReplaceAllString(bs.Command, "") + newCommand = re.ReplaceAllString(command, "") return } @@ -395,14 +541,14 @@ func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newComma // so MANIFEST file could not be created, which also blocks the creation of runfiles directory. // See go/python-binary-host-mixed-build for more details. // TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action. -func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string { +func addCommandForPyBinaryRunfilesDir(oldCommand string, zipperCommandPath, zipFilePath string) string { // Unzip the zip file, zipFilePath looks like <python_binary>.zip runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles" command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName) // Create a symbolic link in <python_binary>.runfiles/, which is the expected structure // when running the python binary stub script. command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName) - return bs.Command + " && " + command + return oldCommand + " && " + command } func isSymlinkAction(a action) bool { diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index 68e50c21a..2328411e1 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -223,7 +223,7 @@ func TestAqueryMultiArchGenrule(t *testing.T) { "parentId": 13 }] }` - actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString)) + actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) expectedBuildStatements := []BuildStatement{} for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} { expectedBuildStatements = append(expectedBuildStatements, @@ -234,11 +234,7 @@ func TestAqueryMultiArchGenrule(t *testing.T) { OutputPaths: []string{ fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch), }, - InputPaths: []string{ - "../sourceroot/bionic/libc/SYSCALLS.TXT", - "../sourceroot/bionic/libc/tools/gensyscalls.py", - "../bazel_tools/tools/genrule/genrule-setup.sh", - }, + InputDepsetIds: []int{1}, Env: []KeyValuePair{ KeyValuePair{Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, }, @@ -246,6 +242,16 @@ func TestAqueryMultiArchGenrule(t *testing.T) { }) } assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) + + expectedFlattenedInputs := []string{ + "../sourceroot/bionic/libc/SYSCALLS.TXT", + "../sourceroot/bionic/libc/tools/gensyscalls.py", + "../bazel_tools/tools/genrule/genrule-setup.sh", + } + actualFlattenedInputs := flattenDepsets([]int{1}, actualDepsets) + if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { + t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) + } } func TestInvalidOutputId(t *testing.T) { @@ -280,11 +286,11 @@ func TestInvalidOutputId(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined outputId 3") } -func TestInvalidInputDepsetId(t *testing.T) { +func TestInvalidInputDepsetIdFromAction(t *testing.T) { const inputString = ` { "artifacts": [{ @@ -316,10 +322,47 @@ func TestInvalidInputDepsetId(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined input depsetId 2") } +func TestInvalidInputDepsetIdFromDepset(t *testing.T) { + const inputString = ` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 1 + }, { + "id": 2, + "pathFragmentId": 2 + }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "x", + "arguments": ["touch", "foo"], + "inputDepSetIds": [1], + "outputIds": [1], + "primaryOutputId": 1 + }], + "depSetOfFiles": [{ + "id": 1, + "directArtifactIds": [1, 2], + "transitiveDepSetIds": [42] + }], + "pathFragments": [{ + "id": 1, + "label": "one" + }, { + "id": 2, + "label": "two" + }] +}` + + _, _, err := AqueryBuildStatements([]byte(inputString)) + assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)") +} + func TestInvalidInputArtifactId(t *testing.T) { const inputString = ` { @@ -352,7 +395,7 @@ func TestInvalidInputArtifactId(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined input artifactId 3") } @@ -389,7 +432,7 @@ func TestInvalidPathFragmentId(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined path fragment id 3") } @@ -431,7 +474,7 @@ func TestDepfiles(t *testing.T) { }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } @@ -492,7 +535,7 @@ func TestMultipleDepfiles(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `found multiple potential depfiles "two.d", "other.d"`) } @@ -699,23 +742,31 @@ func TestTransitiveInputDepsets(t *testing.T) { }] }` - actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString)) + actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) + + expectedBuildStatements := []BuildStatement{ + BuildStatement{ + Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'", + OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, + InputDepsetIds: []int{1}, + Mnemonic: "Action", + }, + } + assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) + // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs // are given via a deep depset, but the depset is flattened when returned as a // BuildStatement slice. - inputPaths := []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root"} + expectedFlattenedInputs := []string{} for i := 1; i < 20; i++ { - inputPaths = append(inputPaths, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) + expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) } - expectedBuildStatements := []BuildStatement{ - BuildStatement{ - Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'", - OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, - InputPaths: inputPaths, - Mnemonic: "Action", - }, + expectedFlattenedInputs = append(expectedFlattenedInputs, "bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root") + + actualFlattenedInputs := flattenDepsets([]int{1}, actualDepsets) + if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { + t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) } - assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) } func TestMiddlemenAction(t *testing.T) { @@ -785,24 +836,74 @@ func TestMiddlemenAction(t *testing.T) { }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actualBuildStatements, actualDepsets, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } - if expected := 1; len(actual) != expected { - t.Fatalf("Expected %d build statements, got %d", expected, len(actual)) + if expected := 1; len(actualBuildStatements) != expected { + t.Fatalf("Expected %d build statements, got %d", expected, len(actualBuildStatements)) } - bs := actual[0] - expectedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"} - if !reflect.DeepEqual(bs.InputPaths, expectedInputs) { - t.Errorf("Expected main action inputs %q, but got %q", expectedInputs, bs.InputPaths) + bs := actualBuildStatements[0] + if len(bs.InputPaths) > 0 { + t.Errorf("Expected main action raw inputs to be empty, but got %q", bs.InputPaths) + } + + expectedInputDepsets := []int{2} + if !reflect.DeepEqual(bs.InputDepsetIds, expectedInputDepsets) { + t.Errorf("Expected main action depset IDs %v, but got %v", expectedInputDepsets, bs.InputDepsetIds) } expectedOutputs := []string{"output"} if !reflect.DeepEqual(bs.OutputPaths, expectedOutputs) { t.Errorf("Expected main action outputs %q, but got %q", expectedOutputs, bs.OutputPaths) } + + expectedAllDepsets := []AqueryDepset{ + { + Id: 1, + DirectArtifacts: []string{"middleinput_one", "middleinput_two"}, + }, + { + Id: 2, + DirectArtifacts: []string{"maininput_one", "maininput_two"}, + TransitiveDepSetIds: []int{1}, + }, + } + if !reflect.DeepEqual(actualDepsets, expectedAllDepsets) { + t.Errorf("Expected depsets %v, but got %v", expectedAllDepsets, actualDepsets) + } + + expectedFlattenedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"} + actualFlattenedInputs := flattenDepsets(bs.InputDepsetIds, actualDepsets) + + if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { + t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) + } +} + +// Returns the contents of given depsets in concatenated post order. +func flattenDepsets(depsetIdsToFlatten []int, allDepsets []AqueryDepset) []string { + depsetsById := map[int]AqueryDepset{} + for _, depset := range allDepsets { + depsetsById[depset.Id] = depset + } + result := []string{} + for _, depsetId := range depsetIdsToFlatten { + result = append(result, flattenDepset(depsetId, depsetsById)...) + } + return result +} + +// Returns the contents of a given depset in post order. +func flattenDepset(depsetIdToFlatten int, allDepsets map[int]AqueryDepset) []string { + depset := allDepsets[depsetIdToFlatten] + result := []string{} + for _, depsetId := range depset.TransitiveDepSetIds { + result = append(result, flattenDepset(depsetId, allDepsets)...) + } + result = append(result, depset.DirectArtifacts...) + return result } func TestSimpleSymlink(t *testing.T) { @@ -849,7 +950,7 @@ func TestSimpleSymlink(t *testing.T) { }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) @@ -913,7 +1014,7 @@ func TestSymlinkQuotesPaths(t *testing.T) { }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) @@ -970,7 +1071,7 @@ func TestSymlinkMultipleInputs(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]`) } @@ -1011,7 +1112,7 @@ func TestSymlinkMultipleOutputs(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`) } @@ -1045,7 +1146,7 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) { }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) @@ -1091,7 +1192,7 @@ func TestTemplateExpandActionNoOutput(t *testing.T) { }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1 output to template expand action, got: output []`) } @@ -1211,7 +1312,7 @@ func TestPythonZipperActionSuccess(t *testing.T) { "label": "python_binary" }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) @@ -1264,7 +1365,7 @@ func TestPythonZipperActionNoInput(t *testing.T) { "label": "python_binary.zip" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`) } @@ -1360,7 +1461,7 @@ func TestPythonZipperActionNoOutput(t *testing.T) { "parentId": 11 }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`) } diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go index 3824586c6..a216c9d11 100644 --- a/bp2build/android_app_conversion_test.go +++ b/bp2build/android_app_conversion_test.go @@ -74,7 +74,8 @@ android_app { package_name: "com.google", resource_dirs: ["resa", "resb"], manifest: "manifest/AndroidManifest.xml", - static_libs: ["static_lib_dep"] + static_libs: ["static_lib_dep"], + java_version: "7", } `, expectedBazelTargets: []string{ @@ -87,6 +88,7 @@ android_app { ]`, "custom_package": `"com.google"`, "deps": `[":static_lib_dep"]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }}) } diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index 22c9dfedf..7c2c10077 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -176,8 +176,8 @@ cc_library_shared { ":whole_static_lib_1", ":whole_static_lib_2", ]`, - "sdk_version": `"current"`, - "min_sdk_version": `"29"`, + "sdk_version": `"current"`, + "min_sdk_version": `"29"`, }), }, }) @@ -496,3 +496,27 @@ cc_library_shared { }, ) } + +func TestCcLibrarySharedSystemSharedLibsSharedEmpty(t *testing.T) { + runCcLibrarySharedTestCase(t, bp2buildTestCase{ + description: "cc_library_shared system_shared_libs empty shared default", + moduleTypeUnderTest: "cc_library_shared", + moduleTypeUnderTestFactory: cc.LibrarySharedFactory, + blueprint: soongCcLibrarySharedPreamble + ` +cc_defaults { + name: "empty_defaults", + shared: { + system_shared_libs: [], + }, + include_build_directory: false, +} +cc_library_shared { + name: "empty", + defaults: ["empty_defaults"], +} +`, + expectedBazelTargets: []string{makeBazelTarget("cc_library_shared", "empty", attrNameToString{ + "system_dynamic_deps": "[]", + })}, + }) +} diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go index 4fc07e014..d7a76a89f 100644 --- a/bp2build/java_binary_host_conversion_test.go +++ b/bp2build/java_binary_host_conversion_test.go @@ -52,6 +52,7 @@ func TestJavaBinaryHost(t *testing.T) { jni_libs: ["jni-lib-1"], javacflags: ["-Xdoclint:all/protected"], bazel_module: { bp2build_available: true }, + java_version: "8", }`, expectedBazelTargets: []string{ makeBazelTarget("java_binary", "java-binary-host-1", attrNameToString{ @@ -59,7 +60,10 @@ func TestJavaBinaryHost(t *testing.T) { "main_class": `"com.android.test.MainClass"`, "deps": `["//other:jni-lib-1"]`, "jvm_flags": `["-Djava.library.path=$${RUNPATH}other"]`, - "javacopts": `["-Xdoclint:all/protected"]`, + "javacopts": `[ + "-Xdoclint:all/protected", + "-source 1.8 -target 1.8", + ]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index ccc52ef12..3b6636975 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -158,6 +158,22 @@ java_plugin { }) } +func TestJavaLibraryJavaVersion(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java"], + java_version: "11", +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "srcs": `["a.java"]`, + "javacopts": `["-source 11 -target 11"]`, + }), + }, + }) +} + func TestJavaLibraryErrorproneJavacflagsEnabledManually(t *testing.T) { runJavaLibraryTestCase(t, bp2buildTestCase{ blueprint: `java_library { diff --git a/bp2build/java_library_host_conversion_test.go b/bp2build/java_library_host_conversion_test.go index 73abdd2ea..83cc5519a 100644 --- a/bp2build/java_library_host_conversion_test.go +++ b/bp2build/java_library_host_conversion_test.go @@ -43,6 +43,7 @@ java_library_host { name: "java-lib-host-2", srcs: ["c.java"], bazel_module: { bp2build_available: true }, + java_version: "9", }`, expectedBazelTargets: []string{ makeBazelTarget("java_library", "java-lib-host-1", attrNameToString{ @@ -54,7 +55,8 @@ java_library_host { })`, }), makeBazelTarget("java_library", "java-lib-host-2", attrNameToString{ - "srcs": `["c.java"]`, + "javacopts": `["-source 1.9 -target 1.9"]`, + "srcs": `["c.java"]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_plugin_conversion_test.go b/bp2build/java_plugin_conversion_test.go index c2a2182bd..dc763e763 100644 --- a/bp2build/java_plugin_conversion_test.go +++ b/bp2build/java_plugin_conversion_test.go @@ -39,6 +39,7 @@ func TestJavaPlugin(t *testing.T) { libs: ["java-lib-1"], static_libs: ["java-lib-2"], bazel_module: { bp2build_available: true }, + java_version: "7", } java_library { @@ -66,6 +67,7 @@ java_library { "a.java", "b.java", ]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }, }) diff --git a/bp2build/java_proto_conversion_test.go b/bp2build/java_proto_conversion_test.go index 67f80446e..c6feeb8dc 100644 --- a/bp2build/java_proto_conversion_test.go +++ b/bp2build/java_proto_conversion_test.go @@ -102,6 +102,7 @@ func TestJavaProtoDefault(t *testing.T) { blueprint: `java_library_static { name: "java-protos", srcs: ["a.proto"], + java_version: "7", } `, expectedBazelTargets: []string{ @@ -115,7 +116,8 @@ func TestJavaProtoDefault(t *testing.T) { "deps": `[":java-protos_proto"]`, }), makeBazelTarget("java_library", "java-protos", attrNameToString{ - "exports": `[":java-protos_java_proto_lite"]`, + "exports": `[":java-protos_java_proto_lite"]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }, }) diff --git a/bp2build/prebuilt_etc_conversion_test.go b/bp2build/prebuilt_etc_conversion_test.go index 3a5d5bb2a..2e4b22119 100644 --- a/bp2build/prebuilt_etc_conversion_test.go +++ b/bp2build/prebuilt_etc_conversion_test.go @@ -45,11 +45,11 @@ prebuilt_etc { } `, expectedBazelTargets: []string{ - makeBazelTarget("prebuilt_etc", "apex_tz_version", attrNameToString{ + makeBazelTarget("prebuilt_file", "apex_tz_version", attrNameToString{ "filename": `"tz_version"`, "installable": `False`, "src": `"version/tz_version"`, - "sub_dir": `"tz"`, + "dir": `"etc/tz"`, })}}) } @@ -75,7 +75,7 @@ prebuilt_etc { } `, expectedBazelTargets: []string{ - makeBazelTarget("prebuilt_etc", "apex_tz_version", attrNameToString{ + makeBazelTarget("prebuilt_file", "apex_tz_version", attrNameToString{ "filename": `"tz_version"`, "installable": `False`, "src": `select({ @@ -83,7 +83,7 @@ prebuilt_etc { "//build/bazel/platforms/arch:arm64": "arm64", "//conditions:default": "version/tz_version", })`, - "sub_dir": `"tz"`, + "dir": `"etc/tz"`, })}}) } @@ -114,7 +114,7 @@ prebuilt_etc { } `, expectedBazelTargets: []string{ - makeBazelTarget("prebuilt_etc", "apex_tz_version", attrNameToString{ + makeBazelTarget("prebuilt_file", "apex_tz_version", attrNameToString{ "filename": `"tz_version"`, "installable": `False`, "src": `select({ @@ -125,6 +125,59 @@ prebuilt_etc { "//build/bazel/platforms/os_arch:linux_bionic_arm64": "darwin_or_arm64", "//conditions:default": "version/tz_version", })`, - "sub_dir": `"tz"`, + "dir": `"etc/tz"`, + })}}) +} + +func runPrebuiltUsrShareTestCase(t *testing.T, tc bp2buildTestCase) { + t.Helper() + (&tc).moduleTypeUnderTest = "prebuilt_usr_share" + (&tc).moduleTypeUnderTestFactory = etc.PrebuiltUserShareFactory + runBp2BuildTestCase(t, registerPrebuiltEtcModuleTypes, tc) +} + +func registerPrebuiltUsrShareModuleTypes(ctx android.RegistrationContext) { +} + +func TestPrebuiltUsrShareSimple(t *testing.T) { + runPrebuiltUsrShareTestCase(t, bp2buildTestCase{ + description: "prebuilt_usr_share - simple example", + filesystem: map[string]string{}, + blueprint: ` +prebuilt_usr_share { + name: "apex_tz_version", + src: "version/tz_version", + filename: "tz_version", + sub_dir: "tz", + installable: false, +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("prebuilt_file", "apex_tz_version", attrNameToString{ + "filename": `"tz_version"`, + "installable": `False`, + "src": `"version/tz_version"`, + "dir": `"usr/share/tz"`, + })}}) +} + +func TestPrebuiltEtcNoSubdir(t *testing.T) { + runPrebuiltEtcTestCase(t, bp2buildTestCase{ + description: "prebuilt_etc - no subdir", + filesystem: map[string]string{}, + blueprint: ` +prebuilt_etc { + name: "apex_tz_version", + src: "version/tz_version", + filename: "tz_version", + installable: false, +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("prebuilt_file", "apex_tz_version", attrNameToString{ + "filename": `"tz_version"`, + "installable": `False`, + "src": `"version/tz_version"`, + "dir": `"etc"`, })}}) } diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go index 15a63356e..7d4819110 100644 --- a/bp2build/symlink_forest.go +++ b/bp2build/symlink_forest.go @@ -90,6 +90,20 @@ func symlinkIntoForest(topdir, dst, src string) { } } +func isDir(path string, fi os.FileInfo) bool { + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + return fi.IsDir() + } + + fi2, err := os.Stat(path) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot stat '%s': %s\n", path, err) + os.Exit(1) + } + + return fi2.IsDir() +} + // Recursively plants a symlink forest at forestDir. The symlink tree will // contain every file in buildFilesDir and srcDir excluding the files in // exclude. Collects every directory encountered during the traversal of srcDir @@ -145,8 +159,18 @@ func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir continue } + sDir := false + bDir := false + if sExists { + sDir = isDir(shared.JoinPath(topdir, srcChild), srcChildEntry) + } + + if bExists { + bDir = isDir(shared.JoinPath(topdir, buildFilesChild), buildFilesChildEntry) + } + if !sExists { - if buildFilesChildEntry.IsDir() && excludeChild != nil { + if bDir && excludeChild != nil { // Not in the source tree, but we have to exclude something from under // this subtree, so descend plantSymlinkForestRecursive(topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) @@ -155,7 +179,7 @@ func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir symlinkIntoForest(topdir, forestChild, buildFilesChild) } } else if !bExists { - if srcChildEntry.IsDir() && excludeChild != nil { + if sDir && excludeChild != nil { // Not in the build file tree, but we have to exclude something from // under this subtree, so descend plantSymlinkForestRecursive(topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) @@ -163,10 +187,10 @@ func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir // Not in the build file tree, symlink source tree, carry on symlinkIntoForest(topdir, forestChild, srcChild) } - } else if srcChildEntry.IsDir() && buildFilesChildEntry.IsDir() { + } else if sDir && bDir { // Both are directories. Descend. plantSymlinkForestRecursive(topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) - } else if !srcChildEntry.IsDir() && !buildFilesChildEntry.IsDir() { + } else if !sDir && !bDir { // Neither is a directory. Prioritize BUILD files generated by bp2build // over any BUILD file imported into external/. fmt.Fprintf(os.Stderr, "Both '%s' and '%s' exist, symlinking the former to '%s'\n", diff --git a/build_kzip.bash b/build_kzip.bash index aff2d6d03..6219021cf 100755 --- a/build_kzip.bash +++ b/build_kzip.bash @@ -36,7 +36,7 @@ export KYTHE_JAVA_SOURCE_BATCH_SIZE KYTHE_KZIP_ENCODING declare -r out="${OUT_DIR:-out}" # Build extraction files for C++ and Java. Build `merge_zips` which we use later. -build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java +build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java xref_rust # Build extraction file for Go the files in build/{blueprint,soong} directories. declare -r abspath_out=$(realpath "${out}") @@ -44,7 +44,7 @@ declare -r go_extractor=$(realpath prebuilts/build-tools/linux-x86/bin/go_extrac declare -r go_root=$(realpath prebuilts/go/linux-x86) declare -r source_root=$PWD -# TODO(asmundak): Until b/182183061 is fixed, default corpus has to be specified +# TODO(asmundak): Until b/182183061 is fixed, default corpus has to be specified # in the rules file. Generate this file on the fly with corpus value set from the # environment variable. for dir in blueprint soong; do diff --git a/build_test.bash b/build_test.bash index 1dc666034..8b91e2c9d 100755 --- a/build_test.bash +++ b/build_test.bash @@ -25,7 +25,8 @@ # Products that are broken or otherwise don't work with multiproduct_kati SKIPPED_PRODUCTS=( - # Both of these products are for soong-only builds, and will fail the kati stage. + # These products are for soong-only builds, and will fail the kati stage. + linux_bionic mainline_sdk ndk ) diff --git a/cc/Android.bp b/cc/Android.bp index 9103a48b4..60d329e62 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "soong-etc", "soong-fuzz", "soong-genrule", + "soong-multitree", "soong-snapshot", "soong-tradefed", ], @@ -65,6 +66,7 @@ bootstrap_go_package { "library.go", "library_headers.go", "library_sdk_member.go", + "library_stub.go", "native_bridge_sdk_trait.go", "object.go", "test.go", @@ -94,12 +96,14 @@ bootstrap_go_package { "gen_test.go", "genrule_test.go", "library_headers_test.go", + "library_stub_test.go", "library_test.go", "object_test.go", "prebuilt_test.go", "proto_test.go", "sanitize_test.go", "test_data_test.go", + "tidy_test.go", "vendor_public_library_test.go", "vendor_snapshot_test.go", ], @@ -1,4 +1,4 @@ per-file ndk_*.go = danalbert@google.com -per-file tidy.go = srhines@google.com, chh@google.com +per-file tidy*.go = srhines@google.com, chh@google.com per-file afdo.go,afdo_test.go,lto.go,pgo.go = srhines@google.com, pirama@google.com, yikong@google.com per-file coverage.go = pirama@google.com, srhines@google.com, allenhair@google.com diff --git a/cc/binary_test.go b/cc/binary_test.go index 8ec387182..1b8b4a83c 100644 --- a/cc/binary_test.go +++ b/cc/binary_test.go @@ -49,3 +49,23 @@ cc_binary { expectedUnStrippedFile := "outputbase/execroot/__main__/foo" android.AssertStringEquals(t, "Unstripped output file", expectedUnStrippedFile, unStrippedFilePath.String()) } + +func TestBinaryLinkerScripts(t *testing.T) { + result := PrepareForIntegrationTestWithCc.RunTestWithBp(t, ` + cc_binary { + name: "foo", + srcs: ["foo.cc"], + linker_scripts: ["foo.ld", "bar.ld"], + }`) + + binFoo := result.ModuleForTests("foo", "android_arm64_armv8-a").Rule("ld") + + android.AssertStringListContains(t, "missing dependency on linker_scripts", + binFoo.Implicits.Strings(), "foo.ld") + android.AssertStringListContains(t, "missing dependency on linker_scripts", + binFoo.Implicits.Strings(), "bar.ld") + android.AssertStringDoesContain(t, "missing flag for linker_scripts", + libfoo.Args["ldFlags"], "-Wl,--script,foo.ld") + android.AssertStringDoesContain(t, "missing flag for linker_scripts", + libfoo.Args["ldFlags"], "-Wl,--script,bar.ld") +} diff --git a/cc/bp2build.go b/cc/bp2build.go index cc378b3b4..19855fab8 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -167,21 +167,17 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs.System_dynamic_deps.ForceSpecifyEmptyList = true if isStatic { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &StaticProperties{}) { - for config, props := range configToProps { - if staticOrSharedProps, ok := props.(*StaticProperties); ok { - setAttrs(axis, config, staticOrSharedProps.Static) - } + bp2BuildPropParseHelper(ctx, module, &StaticProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if staticOrSharedProps, ok := props.(*StaticProperties); ok { + setAttrs(axis, config, staticOrSharedProps.Static) } - } + }) } else { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &SharedProperties{}) { - for config, props := range configToProps { - if staticOrSharedProps, ok := props.(*SharedProperties); ok { - setAttrs(axis, config, staticOrSharedProps.Shared) - } + bp2BuildPropParseHelper(ctx, module, &SharedProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if staticOrSharedProps, ok := props.(*SharedProperties); ok { + setAttrs(axis, config, staticOrSharedProps.Shared) } - } + }) } partitionedSrcs := groupSrcsByExtension(ctx, attrs.Srcs) @@ -359,21 +355,18 @@ func (ca *compilerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversi } func (ca *compilerAttributes) convertStlProps(ctx android.ArchVariantContext, module *Module) { - stlPropsByArch := module.GetArchVariantProperties(ctx, &StlProperties{}) - for _, configToProps := range stlPropsByArch { - for _, props := range configToProps { - if stlProps, ok := props.(*StlProperties); ok { - if stlProps.Stl == nil { - continue - } - if ca.stl == nil { - ca.stl = stlProps.Stl - } else if ca.stl != stlProps.Stl { - ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl) - } + bp2BuildPropParseHelper(ctx, module, &StlProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if stlProps, ok := props.(*StlProperties); ok { + if stlProps.Stl == nil { + return + } + if ca.stl == nil { + ca.stl = stlProps.Stl + } else if ca.stl != stlProps.Stl { + ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl) } } - } + }) } func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) { @@ -713,17 +706,15 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion } func (la *linkerAttributes) convertStripProps(ctx android.BazelConversionPathContext, module *Module) { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &StripProperties{}) { - for config, props := range configToProps { - if stripProperties, ok := props.(*StripProperties); ok { - la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols) - la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list) - la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame) - la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All) - la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None) - } + bp2BuildPropParseHelper(ctx, module, &StripProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if stripProperties, ok := props.(*StripProperties); ok { + la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols) + la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list) + la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame) + la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All) + la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None) } - } + }) } func (la *linkerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) { @@ -859,18 +850,16 @@ func bp2BuildParseExportedIncludesHelper(ctx android.BazelConversionPathContext, } else { exported = BazelIncludes{} } - for axis, configToProps := range module.GetArchVariantProperties(ctx, &FlagExporterProperties{}) { - for config, props := range configToProps { - if flagExporterProperties, ok := props.(*FlagExporterProperties); ok { - if len(flagExporterProperties.Export_include_dirs) > 0 { - exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...))) - } - if len(flagExporterProperties.Export_system_include_dirs) > 0 { - exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...))) - } + bp2BuildPropParseHelper(ctx, module, &FlagExporterProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if flagExporterProperties, ok := props.(*FlagExporterProperties); ok { + if len(flagExporterProperties.Export_include_dirs) > 0 { + exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...))) + } + if len(flagExporterProperties.Export_system_include_dirs) > 0 { + exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...))) } } - } + }) exported.AbsoluteIncludes.DeduplicateAxesFromBase() exported.Includes.DeduplicateAxesFromBase() exported.SystemIncludes.DeduplicateAxesFromBase() @@ -938,22 +927,19 @@ type binaryLinkerAttrs struct { func bp2buildBinaryLinkerProps(ctx android.BazelConversionPathContext, m *Module) binaryLinkerAttrs { attrs := binaryLinkerAttrs{} - archVariantProps := m.GetArchVariantProperties(ctx, &BinaryLinkerProperties{}) - for axis, configToProps := range archVariantProps { - for _, p := range configToProps { - props := p.(*BinaryLinkerProperties) - staticExecutable := props.Static_executable - if axis == bazel.NoConfigAxis { - if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared { - attrs.Linkshared = &linkBinaryShared - } - } else if staticExecutable != nil { - // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a - // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling - ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values") + bp2BuildPropParseHelper(ctx, m, &BinaryLinkerProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + linkerProps := props.(*BinaryLinkerProperties) + staticExecutable := linkerProps.Static_executable + if axis == bazel.NoConfigAxis { + if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared { + attrs.Linkshared = &linkBinaryShared } + } else if staticExecutable != nil { + // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a + // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling + ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values") } - } + }) return attrs } diff --git a/cc/builder.go b/cc/builder.go index 525b1a14f..107cd58e7 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -855,13 +855,24 @@ func transformObjToDynamicBinary(ctx android.ModuleContext, deps = append(deps, crtBegin...) deps = append(deps, crtEnd...) + var depFile android.WritablePath + var depFileLdFlags string + depsType := blueprint.DepsNone + if !ctx.Windows() && !ctx.Darwin() { + // lld only supports --dependency-file for elf files + depFile = outputFile.ReplaceExtension(ctx, "d") + depFileLdFlags = " -Wl,--dependency-file=" + depFile.String() + depsType = blueprint.DepsGCC + implicitOutputs = append(implicitOutputs, depFile) + } + rule := ld args := map[string]string{ "ldCmd": ldCmd, "crtBegin": strings.Join(crtBegin.Strings(), " "), "libFlags": strings.Join(libFlagsList, " "), "extraLibFlags": flags.extraLibFlags, - "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags, + "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags + depFileLdFlags, "crtEnd": strings.Join(crtEnd.Strings(), " "), } if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { @@ -872,6 +883,8 @@ func transformObjToDynamicBinary(ctx android.ModuleContext, ctx.Build(pctx, android.BuildParams{ Rule: rule, + Deps: depsType, + Depfile: depFile, Description: "link " + outputFile.Base(), Output: outputFile, ImplicitOutputs: implicitOutputs, @@ -1025,18 +1038,33 @@ func transformObjsToObj(ctx android.ModuleContext, objFiles android.Paths, ldCmd := "${config.ClangBin}/clang++" + var implicitOutputs android.WritablePaths + var depFile android.WritablePath + var depFileLdFlags string + depsType := blueprint.DepsNone + if !ctx.Windows() && !ctx.Darwin() { + // lld only supports --dependency-file for elf files + depFile = outputFile.ReplaceExtension(ctx, "d") + depFileLdFlags = " -Wl,--dependency-file=" + depFile.String() + depsType = blueprint.DepsGCC + implicitOutputs = append(implicitOutputs, depFile) + } + rule := partialLd args := map[string]string{ "ldCmd": ldCmd, - "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags, + "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags + depFileLdFlags, } if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { rule = partialLdRE args["inCommaList"] = strings.Join(objFiles.Strings(), ",") args["implicitInputs"] = strings.Join(deps.Strings(), ",") + args["implicitOutputs"] = strings.Join(implicitOutputs.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, + Deps: depsType, + Depfile: depFile, Description: "link " + outputFile.Base(), Output: outputFile, Inputs: objFiles, @@ -48,7 +48,6 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.BottomUp("vndk", VndkMutator).Parallel() ctx.BottomUp("link", LinkageMutator).Parallel() ctx.BottomUp("test_per_src", TestPerSrcMutator).Parallel() - ctx.BottomUp("version_selector", versionSelectorMutator).Parallel() ctx.BottomUp("version", versionMutator).Parallel() ctx.BottomUp("begin", BeginMutator).Parallel() ctx.BottomUp("sysprop_cc", SyspropMutator).Parallel() @@ -746,6 +745,7 @@ var ( runtimeDepTag = installDependencyTag{name: "runtime lib"} testPerSrcDepTag = dependencyTag{name: "test_per_src"} stubImplDepTag = dependencyTag{name: "stub_impl"} + JniFuzzLibTag = dependencyTag{name: "jni_fuzz_lib_tag"} ) func IsSharedDepTag(depTag blueprint.DependencyTag) bool { @@ -1788,7 +1788,7 @@ func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { bazelActionsUsed := false // Mixed builds mode is disabled for modules outside of device OS. // TODO(b/200841190): Support non-device OS in mixed builds. - if c.MixedBuildsEnabled(actx) && c.bazelHandler != nil { + if android.MixedBuildsEnabled(actx) && c.bazelHandler != nil { bazelActionsUsed = c.bazelHandler.GenerateBazelBuildActions(actx, bazelModuleLabel) } return bazelActionsUsed @@ -2116,7 +2116,7 @@ func AddSharedLibDependenciesWithVersions(ctx android.BottomUpMutatorContext, mo variations = append([]blueprint.Variation(nil), variations...) - if version != "" && CanBeOrLinkAgainstVersionVariants(mod) { + if version != "" && canBeOrLinkAgainstVersionVariants(mod) { // Version is explicitly specified. i.e. libFoo#30 variations = append(variations, blueprint.Variation{Mutator: "version", Variation: version}) if tag, ok := depTag.(libraryDependencyTag); ok { diff --git a/cc/check.go b/cc/check.go index a357a9751..3d290a94b 100644 --- a/cc/check.go +++ b/cc/check.go @@ -87,6 +87,8 @@ func CheckBadLinkerFlags(ctx BaseModuleContext, prop string, flags []string) { ctx.PropertyErrorf(prop, "Bad flag: `%s` is not allowed", flag) } else if strings.HasPrefix(flag, "-Wl,--version-script") { ctx.PropertyErrorf(prop, "Bad flag: `%s`, use version_script instead", flag) + } else if flag == "-T" || strings.HasPrefix(flag, "--script") { + ctx.PropertyErrorf(prop, "Bad flag: `%s`, use linker_scripts instead", flag) } else if flag == "--coverage" { ctx.PropertyErrorf(prop, "Bad flag: `%s`, use native_coverage instead", flag) } else if strings.Contains(flag, " ") { diff --git a/cc/config/global.go b/cc/config/global.go index 65bfbf0f9..3caf32792 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -286,8 +286,8 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r450784c" - ClangDefaultShortVersion = "14.0.5" + ClangDefaultVersion = "clang-r450784d" + ClangDefaultShortVersion = "14.0.6" // Directories with warnings from Android.bp files. WarningAllowedProjects = []string{ diff --git a/cc/installer.go b/cc/installer.go index 2522610d9..e2c0e7b8d 100644 --- a/cc/installer.go +++ b/cc/installer.go @@ -31,7 +31,7 @@ type InstallerProperties struct { Install_in_root *bool `android:"arch_variant"` // Install output directly in {partition}/xbin - Install_in_xbin *bool `android:"arch_vvariant"` + Install_in_xbin *bool `android:"arch_variant"` } type installLocation int diff --git a/cc/libbuildversion/tests/Android.bp b/cc/libbuildversion/tests/Android.bp index 0e97fedff..c616a3351 100644 --- a/cc/libbuildversion/tests/Android.bp +++ b/cc/libbuildversion/tests/Android.bp @@ -35,6 +35,16 @@ cc_defaults { dir: "host/", }, }, + linux_musl_x86: { + dist: { + dir: "host32/", + }, + }, + linux_musl_x86_64: { + dist: { + dir: "host/", + }, + }, linux_glibc_x86: { dist: { dir: "host32/", diff --git a/cc/library.go b/cc/library.go index 0abcb6f39..fdbbccbe7 100644 --- a/cc/library.go +++ b/cc/library.go @@ -2344,7 +2344,7 @@ func createPerApiVersionVariations(mctx android.BottomUpMutatorContext, minSdkVe } } -func CanBeOrLinkAgainstVersionVariants(module interface { +func canBeOrLinkAgainstVersionVariants(module interface { Host() bool InRamdisk() bool InVendorRamdisk() bool @@ -2352,15 +2352,14 @@ func CanBeOrLinkAgainstVersionVariants(module interface { return !module.Host() && !module.InRamdisk() && !module.InVendorRamdisk() } -func CanBeVersionVariant(module interface { +func canBeVersionVariant(module interface { Host() bool InRamdisk() bool InVendorRamdisk() bool - InRecovery() bool CcLibraryInterface() bool Shared() bool }) bool { - return CanBeOrLinkAgainstVersionVariants(module) && + return canBeOrLinkAgainstVersionVariants(module) && module.CcLibraryInterface() && module.Shared() } @@ -2371,37 +2370,41 @@ func moduleLibraryInterface(module blueprint.Module) libraryInterface { return nil } -// versionSelector normalizes the versions in the Stubs.Versions property into MutatedProperties.AllStubsVersions. -func versionSelectorMutator(mctx android.BottomUpMutatorContext) { - if library := moduleLibraryInterface(mctx.Module()); library != nil && CanBeVersionVariant(mctx.Module().(*Module)) { - if library.buildShared() { - versions := library.stubsVersions(mctx) - if len(versions) > 0 { - normalizeVersions(mctx, versions) - if mctx.Failed() { - return - } - // Set the versions on the pre-mutated module so they can be read by any llndk modules that - // depend on the implementation library and haven't been mutated yet. - library.setAllStubsVersions(versions) - } - } +// setStubsVersions normalizes the versions in the Stubs.Versions property into MutatedProperties.AllStubsVersions. +func setStubsVersions(mctx android.BottomUpMutatorContext, library libraryInterface, module *Module) { + if !library.buildShared() || !canBeVersionVariant(module) { + return + } + versions := library.stubsVersions(mctx) + if len(versions) <= 0 { + return + } + normalizeVersions(mctx, versions) + if mctx.Failed() { + return } + // Set the versions on the pre-mutated module so they can be read by any llndk modules that + // depend on the implementation library and haven't been mutated yet. + library.setAllStubsVersions(versions) } // versionMutator splits a module into the mandatory non-stubs variant // (which is unnamed) and zero or more stubs variants. func versionMutator(mctx android.BottomUpMutatorContext) { - if library := moduleLibraryInterface(mctx.Module()); library != nil && CanBeVersionVariant(mctx.Module().(*Module)) { + if mctx.Os() != android.Android { + return + } + + m, ok := mctx.Module().(*Module) + if library := moduleLibraryInterface(mctx.Module()); library != nil && canBeVersionVariant(m) { + setStubsVersions(mctx, library, m) + createVersionVariations(mctx, library.allStubsVersions()) return } - if m, ok := mctx.Module().(*Module); ok { + if ok { if m.SplitPerApiLevel() && m.IsSdkVariant() { - if mctx.Os() != android.Android { - return - } createPerApiVersionVariations(mctx, m.MinSdkVersion()) } } diff --git a/cc/library_stub.go b/cc/library_stub.go new file mode 100644 index 000000000..4d0148df4 --- /dev/null +++ b/cc/library_stub.go @@ -0,0 +1,163 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "android/soong/android" + "android/soong/multitree" +) + +func init() { + RegisterLibraryStubBuildComponents(android.InitRegistrationContext) +} + +func RegisterLibraryStubBuildComponents(ctx android.RegistrationContext) { + // cc_api_stub_library shares a lot of ndk_library, and this will be refactored later + ctx.RegisterModuleType("cc_api_stub_library", CcApiStubLibraryFactory) + ctx.RegisterModuleType("cc_api_contribution", CcApiContributionFactory) +} + +func CcApiStubLibraryFactory() android.Module { + module, decorator := NewLibrary(android.DeviceSupported) + apiStubDecorator := &apiStubDecorator{ + libraryDecorator: decorator, + } + apiStubDecorator.BuildOnlyShared() + + module.compiler = apiStubDecorator + module.linker = apiStubDecorator + module.installer = nil + module.library = apiStubDecorator + module.Properties.HideFromMake = true // TODO: remove + + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth) + module.AddProperties(&module.Properties, + &apiStubDecorator.properties, + &apiStubDecorator.MutatedProperties, + &apiStubDecorator.apiStubLibraryProperties) + return module +} + +type apiStubLiraryProperties struct { + Imported_includes []string `android:"path"` +} + +type apiStubDecorator struct { + *libraryDecorator + properties libraryProperties + apiStubLibraryProperties apiStubLiraryProperties +} + +func (compiler *apiStubDecorator) stubsVersions(ctx android.BaseMutatorContext) []string { + firstVersion := String(compiler.properties.First_version) + return ndkLibraryVersions(ctx, android.ApiLevelOrPanic(ctx, firstVersion)) +} + +func (decorator *apiStubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { + if decorator.stubsVersion() == "" { + decorator.setStubsVersion("current") + } // TODO: fix + symbolFile := String(decorator.properties.Symbol_file) + nativeAbiResult := parseNativeAbiDefinition(ctx, symbolFile, + android.ApiLevelOrPanic(ctx, decorator.stubsVersion()), + "") + return compileStubLibrary(ctx, flags, nativeAbiResult.stubSrc) +} + +func (decorator *apiStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objects Objects) android.Path { + decorator.reexportDirs(android.PathsForModuleSrc(ctx, decorator.apiStubLibraryProperties.Imported_includes)...) + return decorator.libraryDecorator.link(ctx, flags, deps, objects) +} + +func init() { + pctx.HostBinToolVariable("gen_api_surface_build_files", "gen_api_surface_build_files") +} + +type CcApiContribution struct { + android.ModuleBase + properties ccApiContributionProperties +} + +type ccApiContributionProperties struct { + Symbol_file *string `android:"path"` + First_version *string + Export_include_dir *string +} + +func CcApiContributionFactory() android.Module { + module := &CcApiContribution{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + return module +} + +// Do some simple validations +// Majority of the build rules will be created in the ctx of the api surface this module contributes to +func (contrib *CcApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if contrib.properties.Symbol_file == nil { + ctx.PropertyErrorf("symbol_file", "%v does not have symbol file", ctx.ModuleName()) + } + if contrib.properties.First_version == nil { + ctx.PropertyErrorf("first_version", "%v does not have first_version for stub variants", ctx.ModuleName()) + } +} + +// Path is out/soong/.export/ but will be different in final multi-tree layout +func outPathApiSurface(ctx android.ModuleContext, myModuleName string, pathComponent string) android.OutputPath { + return android.PathForOutput(ctx, ".export", ctx.ModuleName(), myModuleName, pathComponent) +} + +func (contrib *CcApiContribution) CopyFilesWithTag(apiSurfaceContext android.ModuleContext) map[string]android.Paths { + // copy map.txt for now + // hardlinks cannot be created since nsjail creates a different mountpoint for out/ + myDir := apiSurfaceContext.OtherModuleDir(contrib) + genMapTxt := outPathApiSurface(apiSurfaceContext, contrib.Name(), String(contrib.properties.Symbol_file)) + apiSurfaceContext.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Description: "import map.txt file", + Input: android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Symbol_file)), + Output: genMapTxt, + }) + + outputs := make(map[string]android.Paths) + outputs["map"] = []android.Path{genMapTxt} + + if contrib.properties.Export_include_dir != nil { + includeDir := android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Export_include_dir)) + outputs["export_include_dir"] = []android.Path{includeDir} + } + return outputs +} + +var _ multitree.ApiContribution = (*CcApiContribution)(nil) + +/* +func (contrib *CcApiContribution) GenerateBuildFiles(apiSurfaceContext android.ModuleContext) android.Paths { + genAndroidBp := outPathApiSurface(apiSurfaceContext, contrib.Name(), "Android.bp") + + // generate Android.bp + apiSurfaceContext.Build(pctx, android.BuildParams{ + Rule: genApiSurfaceBuildFiles, + Description: "generate API surface build files", + Outputs: []android.WritablePath{genAndroidBp}, + Args: map[string]string{ + "name": contrib.Name() + "." + apiSurfaceContext.ModuleName(), //e.g. liblog.ndk + "symbol_file": String(contrib.properties.Symbol_file), + "first_version": String(contrib.properties.First_version), + }, + }) + return []android.Path{genAndroidBp} +} +*/ diff --git a/cc/library_stub_test.go b/cc/library_stub_test.go new file mode 100644 index 000000000..15b56d227 --- /dev/null +++ b/cc/library_stub_test.go @@ -0,0 +1,108 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + _ "fmt" + _ "sort" + + "testing" + + "android/soong/android" + "android/soong/multitree" +) + +func TestCcApiStubLibraryOutputFiles(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := prepareForCcTest.RunTestWithBp(t, bp) + outputs := result.ModuleForTests("foo", "android_arm64_armv8-a_shared").AllOutputs() + expected_file_suffixes := []string{".c", "stub.map", ".o", ".so"} + for _, expected_file_suffix := range expected_file_suffixes { + android.AssertBoolEquals(t, expected_file_suffix+" file not found in output", true, android.SuffixInList(outputs, expected_file_suffix)) + } +} + +func TestCcApiStubLibraryVariants(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := prepareForCcTest.RunTestWithBp(t, bp) + variants := result.ModuleVariantsForTests("foo") + expected_variants := []string{"29", "30", "S", "Tiramisu"} //TODO: make this test deterministic by using fixtures + for _, expected_variant := range expected_variants { + android.AssertBoolEquals(t, expected_variant+" variant not found in foo", true, android.SubstringInList(variants, expected_variant)) + } +} + +func TestCcLibraryUsesCcApiStubLibrary(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + cc_library { + name: "foo_user", + shared_libs: [ + "foo#29", + ], + } + + ` + prepareForCcTest.RunTestWithBp(t, bp) +} + +func TestApiSurfaceOutputs(t *testing.T) { + bp := ` + api_surface { + name: "mysdk", + contributions: [ + "foo", + ], + } + + cc_api_contribution { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := android.GroupFixturePreparers( + prepareForCcTest, + multitree.PrepareForTestWithApiSurface, + ).RunTestWithBp(t, bp) + mysdk := result.ModuleForTests("mysdk", "") + + actual_surface_inputs := mysdk.Rule("phony").BuildParams.Inputs.Strings() + expected_file_suffixes := []string{"mysdk/foo/foo.map.txt"} + for _, expected_file_suffix := range expected_file_suffixes { + android.AssertBoolEquals(t, expected_file_suffix+" file not found in input", true, android.SuffixInList(actual_surface_inputs, expected_file_suffix)) + } + + // check args/inputs to rule + /*api_surface_gen_rule_args := result.ModuleForTests("mysdk", "").Rule("genApiSurfaceBuildFiles").Args + android.AssertStringEquals(t, "name", "foo.mysdk", api_surface_gen_rule_args["name"]) + android.AssertStringEquals(t, "symbol_file", "foo.map.txt", api_surface_gen_rule_args["symbol_file"])*/ +} diff --git a/cc/linker.go b/cc/linker.go index bea65d441..f34658485 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -227,6 +227,9 @@ type BaseLinkerProperties struct { // local file name to pass to the linker as --dynamic-list Dynamic_list *string `android:"path,arch_variant"` + // local files to pass to the linker as --script + Linker_scripts []string `android:"path,arch_variant"` + // list of static libs that should not be used to build this module Exclude_static_libs []string `android:"arch_variant"` @@ -602,6 +605,17 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { flags.LdFlagsDeps = append(flags.LdFlagsDeps, dynamicList.Path()) } } + + linkerScriptPaths := android.PathsForModuleSrc(ctx, linker.Properties.Linker_scripts) + if len(linkerScriptPaths) > 0 && (ctx.Darwin() || ctx.Windows()) { + ctx.PropertyErrorf("linker_scripts", "Only supported for ELF files") + } else { + for _, linkerScriptPath := range linkerScriptPaths { + flags.Local.LdFlags = append(flags.Local.LdFlags, + "-Wl,--script,"+linkerScriptPath.String()) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, linkerScriptPath) + } + } } return flags diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 5ef41eae5..0879257a5 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -93,7 +93,7 @@ var ( type libraryProperties struct { // Relative path to the symbol map. // An example file can be seen here: TODO(danalbert): Make an example. - Symbol_file *string + Symbol_file *string `android:"path"` // The first API level a library was available. A library will be generated // for every API level beginning with this one. @@ -284,6 +284,10 @@ func parseNativeAbiDefinition(ctx ModuleContext, symbolFile string, } func compileStubLibrary(ctx ModuleContext, flags Flags, src android.Path) Objects { + // libc/libm stubs libraries end up mismatching with clang's internal definition of these + // functions (which have noreturn attributes and other things). Because we just want to create a + // stub with symbol definitions, and types aren't important in C, ignore the mismatch. + flags.Local.ConlyFlags = append(flags.Local.ConlyFlags, "-fno-builtin") return compileObjs(ctx, flagsToBuilderFlags(flags), "", android.Paths{src}, nil, nil, nil, nil) } diff --git a/cc/sanitize.go b/cc/sanitize.go index 814fef6a9..53169de48 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -969,6 +969,22 @@ func sanitizerDepsMutator(t SanitizerType) func(android.TopDownMutatorContext) { }) } } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok { + // If it's a Java module with native dependencies through jni, + // set the sanitizer for them + if jniSanitizeable, ok := mctx.Module().(JniSanitizeable); ok { + if jniSanitizeable.IsSanitizerEnabledForJni(mctx, t.name()) { + mctx.VisitDirectDeps(func(child android.Module) { + if c, ok := child.(PlatformSanitizeable); ok && + mctx.OtherModuleDependencyTag(child) == JniFuzzLibTag && + c.SanitizePropDefined() && + !c.SanitizeNever() && + !c.IsSanitizerExplicitlyDisabled(t) { + c.SetSanitizeDep(true) + } + }) + } + } + // If an APEX module includes a lib which is enabled for a sanitizer T, then // the APEX module is also enabled for the same sanitizer type. mctx.VisitDirectDeps(func(child android.Module) { @@ -1280,6 +1296,11 @@ type Sanitizeable interface { AddSanitizerDependencies(ctx android.BottomUpMutatorContext, sanitizerName string) } +type JniSanitizeable interface { + android.Module + IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool +} + func (c *Module) MinimalRuntimeDep() bool { return c.sanitize.Properties.MinimalRuntimeDep } @@ -1407,7 +1428,7 @@ func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { } c.SetSanitizeDep(false) } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok && sanitizeable.IsSanitizerEnabled(mctx, t.name()) { - // APEX modules fall here + // APEX and Java fuzz modules fall here sanitizeable.AddSanitizerDependencies(mctx, t.name()) mctx.CreateVariations(t.variationName()) } else if c, ok := mctx.Module().(*Module); ok { diff --git a/cc/testing.go b/cc/testing.go index 32f7c6080..ecdae8b15 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -29,6 +29,7 @@ func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) { RegisterBinaryBuildComponents(ctx) RegisterLibraryBuildComponents(ctx) RegisterLibraryHeadersBuildComponents(ctx) + RegisterLibraryStubBuildComponents(ctx) ctx.RegisterModuleType("cc_benchmark", BenchmarkFactory) ctx.RegisterModuleType("cc_object", ObjectFactory) diff --git a/cc/tidy.go b/cc/tidy.go index 750e9de1e..03e967d7c 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -76,9 +76,10 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { // the global WITH_TIDY or module 'tidy' property is true. flags.Tidy = true - // If explicitly enabled, by global default or local tidy property, + // If explicitly enabled, by global WITH_TIDY or local tidy:true property, // set flags.NeedTidyFiles to make this module depend on .tidy files. - if ctx.Config().ClangTidy() || Bool(tidy.Properties.Tidy) { + // Note that locally set tidy:true is ignored if ALLOW_LOCAL_TIDY_TRUE is not set to true. + if ctx.Config().IsEnvTrue("WITH_TIDY") || (ctx.Config().IsEnvTrue("ALLOW_LOCAL_TIDY_TRUE") && Bool(tidy.Properties.Tidy)) { flags.NeedTidyFiles = true } diff --git a/cc/tidy_test.go b/cc/tidy_test.go new file mode 100644 index 000000000..339b30281 --- /dev/null +++ b/cc/tidy_test.go @@ -0,0 +1,98 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "fmt" + "testing" + + "android/soong/android" +) + +func TestWithTidy(t *testing.T) { + // When WITH_TIDY=1 or (ALLOW_LOCAL_TIDY_TRUE=1 and local tidy:true) + // a C++ library should depend on .tidy files. + testCases := []struct { + withTidy, allowLocalTidyTrue string // "_" means undefined + needTidyFile []bool // for {libfoo_0, libfoo_1} and {libbar_0, libbar_1} + }{ + {"_", "_", []bool{false, false, false}}, + {"_", "0", []bool{false, false, false}}, + {"_", "1", []bool{false, true, false}}, + {"_", "true", []bool{false, true, false}}, + {"0", "_", []bool{false, false, false}}, + {"0", "1", []bool{false, true, false}}, + {"1", "_", []bool{true, true, false}}, + {"1", "false", []bool{true, true, false}}, + {"1", "1", []bool{true, true, false}}, + {"true", "_", []bool{true, true, false}}, + } + bp := ` + cc_library_shared { + name: "libfoo_0", // depends on .tidy if WITH_TIDY=1 + srcs: ["foo.c"], + } + cc_library_shared { // depends on .tidy if WITH_TIDY=1 or ALLOW_LOCAL_TIDY_TRUE=1 + name: "libfoo_1", + srcs: ["foo.c"], + tidy: true, + } + cc_library_shared { // no .tidy + name: "libfoo_2", + srcs: ["foo.c"], + tidy: false, + } + cc_library_static { + name: "libbar_0", // depends on .tidy if WITH_TIDY=1 + srcs: ["bar.c"], + } + cc_library_static { // depends on .tidy if WITH_TIDY=1 or ALLOW_LOCAL_TIDY_TRUE=1 + name: "libbar_1", + srcs: ["bar.c"], + tidy: true, + } + cc_library_static { // no .tidy + name: "libbar_2", + srcs: ["bar.c"], + tidy: false, + }` + for index, test := range testCases { + testName := fmt.Sprintf("case%d,%v,%v", index, test.withTidy, test.allowLocalTidyTrue) + t.Run(testName, func(t *testing.T) { + testEnv := map[string]string{} + if test.withTidy != "_" { + testEnv["WITH_TIDY"] = test.withTidy + } + if test.allowLocalTidyTrue != "_" { + testEnv["ALLOW_LOCAL_TIDY_TRUE"] = test.allowLocalTidyTrue + } + ctx := android.GroupFixturePreparers(prepareForCcTest, android.FixtureMergeEnv(testEnv)).RunTestWithBp(t, bp) + for n := 0; n < 3; n++ { + checkLibraryRule := func(foo, variant, ruleName string) { + libName := fmt.Sprintf("lib%s_%d", foo, n) + tidyFile := "out/soong/.intermediates/" + libName + "/" + variant + "/obj/" + foo + ".tidy" + depFiles := ctx.ModuleForTests(libName, variant).Rule(ruleName).Validations.Strings() + if test.needTidyFile[n] { + android.AssertStringListContains(t, libName+" needs .tidy file", depFiles, tidyFile) + } else { + android.AssertStringListDoesNotContain(t, libName+" does not need .tidy file", depFiles, tidyFile) + } + } + checkLibraryRule("foo", "android_arm64_armv8-a_shared", "ld") + checkLibraryRule("bar", "android_arm64_armv8-a_static", "ar") + } + }) + } +} diff --git a/cc/util.go b/cc/util.go index b256b9ada..4e1003739 100644 --- a/cc/util.go +++ b/cc/util.go @@ -15,9 +15,7 @@ package cc import ( - "fmt" "path/filepath" - "regexp" "strings" "android/soong/android" @@ -30,30 +28,12 @@ func includeDirsToFlags(dirs android.Paths) string { return android.JoinWithPrefix(dirs.Strings(), "-I") } -func ldDirsToFlags(dirs []string) string { - return android.JoinWithPrefix(dirs, "-L") -} - -func libNamesToFlags(names []string) string { - return android.JoinWithPrefix(names, "-l") -} - var indexList = android.IndexList var inList = android.InList var filterList = android.FilterList var removeListFromList = android.RemoveListFromList var removeFromList = android.RemoveFromList -var libNameRegexp = regexp.MustCompile(`^lib(.*)$`) - -func moduleToLibName(module string) (string, error) { - matches := libNameRegexp.FindStringSubmatch(module) - if matches == nil { - return "", fmt.Errorf("Library module name %s does not start with lib", module) - } - return matches[1], nil -} - func flagsToBuilderFlags(in Flags) builderFlags { return builderFlags{ globalCommonFlags: strings.Join(in.Global.CommonFlags, " "), @@ -113,13 +93,6 @@ func addPrefix(list []string, prefix string) []string { return list } -func addSuffix(list []string, suffix string) []string { - for i := range list { - list[i] = list[i] + suffix - } - return list -} - // linkDirOnDevice/linkName -> target func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) string { dir := filepath.Join("$(PRODUCT_OUT)", linkDirOnDevice) diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 36513b64b..7bc9ab261 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -196,10 +196,6 @@ type ClassLoaderContext struct { // If the library is optional or required. Optional bool - // If the library is implicitly infered by Soong (as opposed to explicitly added via `uses_libs` - // or `optional_uses_libs`. - Implicit bool - // On-host build path to the library dex file (used in dex2oat argument --class-loader-context). Host android.Path @@ -290,9 +286,8 @@ const UnknownInstallLibraryPath = "error" const AnySdkVersion int = android.FutureApiLevelInt // Add class loader context for the given library to the map entry for the given SDK version. -func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathContext, sdkVer int, - lib string, optional, implicit bool, hostPath, installPath android.Path, - nestedClcMap ClassLoaderContextMap) error { +func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathContext, sdkVer int, lib string, + optional bool, hostPath, installPath android.Path, nestedClcMap ClassLoaderContextMap) error { // For prebuilts, library should have the same name as the source module. lib = android.RemoveOptionalPrebuiltPrefix(lib) @@ -341,7 +336,6 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont clcMap[sdkVer] = append(clcMap[sdkVer], &ClassLoaderContext{ Name: lib, Optional: optional, - Implicit: implicit, Host: hostPath, Device: devicePath, Subcontexts: subcontexts, @@ -354,10 +348,9 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont // about paths). For the subset of libraries that are used in dexpreopt, their build/install paths // are validated later before CLC is used (in validateClassLoaderContext). func (clcMap ClassLoaderContextMap) AddContext(ctx android.ModuleInstallPathContext, sdkVer int, - lib string, optional, implicit bool, hostPath, installPath android.Path, - nestedClcMap ClassLoaderContextMap) { + lib string, optional bool, hostPath, installPath android.Path, nestedClcMap ClassLoaderContextMap) { - err := clcMap.addContext(ctx, sdkVer, lib, optional, implicit, hostPath, installPath, nestedClcMap) + err := clcMap.addContext(ctx, sdkVer, lib, optional, hostPath, installPath, nestedClcMap) if err != nil { ctx.ModuleErrorf(err.Error()) } @@ -401,15 +394,13 @@ func (clcMap ClassLoaderContextMap) AddContextMap(otherClcMap ClassLoaderContext // included). This is the list of libraries that should be in the <uses-library> tags in the // manifest. Some of them may be present in the source manifest, others are added by manifest_fixer. // Required and optional libraries are in separate lists. -func (clcMap ClassLoaderContextMap) usesLibs(implicit bool) (required []string, optional []string) { +func (clcMap ClassLoaderContextMap) UsesLibs() (required []string, optional []string) { if clcMap != nil { clcs := clcMap[AnySdkVersion] required = make([]string, 0, len(clcs)) optional = make([]string, 0, len(clcs)) for _, clc := range clcs { - if implicit && !clc.Implicit { - // Skip, this is an explicit library and we need only the implicit ones. - } else if clc.Optional { + if clc.Optional { optional = append(optional, clc.Name) } else { required = append(required, clc.Name) @@ -419,14 +410,6 @@ func (clcMap ClassLoaderContextMap) usesLibs(implicit bool) (required []string, return required, optional } -func (clcMap ClassLoaderContextMap) UsesLibs() ([]string, []string) { - return clcMap.usesLibs(false) -} - -func (clcMap ClassLoaderContextMap) ImplicitUsesLibs() ([]string, []string) { - return clcMap.usesLibs(true) -} - func (clcMap ClassLoaderContextMap) Dump() string { jsonCLC := toJsonClassLoaderContext(clcMap) bytes, err := json.MarshalIndent(jsonCLC, "", " ") @@ -631,7 +614,6 @@ func computeClassLoaderContextRec(clcs []*ClassLoaderContext) (string, string, a type jsonClassLoaderContext struct { Name string Optional bool - Implicit bool Host string Device string Subcontexts []*jsonClassLoaderContext @@ -664,7 +646,6 @@ func fromJsonClassLoaderContextRec(ctx android.PathContext, jClcs []*jsonClassLo clcs = append(clcs, &ClassLoaderContext{ Name: clc.Name, Optional: clc.Optional, - Implicit: clc.Implicit, Host: constructPath(ctx, clc.Host), Device: clc.Device, Subcontexts: fromJsonClassLoaderContextRec(ctx, clc.Subcontexts), @@ -678,6 +659,9 @@ func toJsonClassLoaderContext(clcMap ClassLoaderContextMap) jsonClassLoaderConte jClcMap := make(jsonClassLoaderContextMap) for sdkVer, clcs := range clcMap { sdkVerStr := fmt.Sprintf("%d", sdkVer) + if sdkVer == AnySdkVersion { + sdkVerStr = "any" + } jClcMap[sdkVerStr] = toJsonClassLoaderContextRec(clcs) } return jClcMap @@ -697,7 +681,6 @@ func toJsonClassLoaderContextRec(clcs []*ClassLoaderContext) []*jsonClassLoaderC jClcs[i] = &jsonClassLoaderContext{ Name: clc.Name, Optional: clc.Optional, - Implicit: clc.Implicit, Host: host, Device: clc.Device, Subcontexts: toJsonClassLoaderContextRec(clc.Subcontexts), diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index 5d3a9d943..8b3c0130c 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -50,34 +50,33 @@ func TestCLC(t *testing.T) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) - m.AddContext(ctx, AnySdkVersion, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - m.AddContext(ctx, AnySdkVersion, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + m.AddContext(ctx, AnySdkVersion, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, AnySdkVersion, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) // Add some libraries with nested subcontexts. m1 := make(ClassLoaderContextMap) - m1.AddContext(ctx, AnySdkVersion, "a1", optional, implicit, buildPath(ctx, "a1"), installPath(ctx, "a1"), nil) - m1.AddContext(ctx, AnySdkVersion, "b1", optional, implicit, buildPath(ctx, "b1"), installPath(ctx, "b1"), nil) + m1.AddContext(ctx, AnySdkVersion, "a1", optional, buildPath(ctx, "a1"), installPath(ctx, "a1"), nil) + m1.AddContext(ctx, AnySdkVersion, "b1", optional, buildPath(ctx, "b1"), installPath(ctx, "b1"), nil) m2 := make(ClassLoaderContextMap) - m2.AddContext(ctx, AnySdkVersion, "a2", optional, implicit, buildPath(ctx, "a2"), installPath(ctx, "a2"), nil) - m2.AddContext(ctx, AnySdkVersion, "b2", optional, implicit, buildPath(ctx, "b2"), installPath(ctx, "b2"), nil) - m2.AddContext(ctx, AnySdkVersion, "c2", optional, implicit, buildPath(ctx, "c2"), installPath(ctx, "c2"), m1) + m2.AddContext(ctx, AnySdkVersion, "a2", optional, buildPath(ctx, "a2"), installPath(ctx, "a2"), nil) + m2.AddContext(ctx, AnySdkVersion, "b2", optional, buildPath(ctx, "b2"), installPath(ctx, "b2"), nil) + m2.AddContext(ctx, AnySdkVersion, "c2", optional, buildPath(ctx, "c2"), installPath(ctx, "c2"), m1) m3 := make(ClassLoaderContextMap) - m3.AddContext(ctx, AnySdkVersion, "a3", optional, implicit, buildPath(ctx, "a3"), installPath(ctx, "a3"), nil) - m3.AddContext(ctx, AnySdkVersion, "b3", optional, implicit, buildPath(ctx, "b3"), installPath(ctx, "b3"), nil) + m3.AddContext(ctx, AnySdkVersion, "a3", optional, buildPath(ctx, "a3"), installPath(ctx, "a3"), nil) + m3.AddContext(ctx, AnySdkVersion, "b3", optional, buildPath(ctx, "b3"), installPath(ctx, "b3"), nil) - m.AddContext(ctx, AnySdkVersion, "d", optional, implicit, buildPath(ctx, "d"), installPath(ctx, "d"), m2) + m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), m2) // When the same library is both in conditional and unconditional context, it should be removed // from conditional context. - m.AddContext(ctx, 42, "f", optional, implicit, buildPath(ctx, "f"), installPath(ctx, "f"), nil) - m.AddContext(ctx, AnySdkVersion, "f", optional, implicit, buildPath(ctx, "f"), installPath(ctx, "f"), nil) + m.AddContext(ctx, 42, "f", optional, buildPath(ctx, "f"), installPath(ctx, "f"), nil) + m.AddContext(ctx, AnySdkVersion, "f", optional, buildPath(ctx, "f"), installPath(ctx, "f"), nil) // Merge map with implicit root library that is among toplevel contexts => does nothing. m.AddContextMap(m1, "c") @@ -86,12 +85,12 @@ func TestCLC(t *testing.T) { m.AddContextMap(m3, "m_g") // Compatibility libraries with unknown install paths get default paths. - m.AddContext(ctx, 29, AndroidHidlManager, optional, implicit, buildPath(ctx, AndroidHidlManager), nil, nil) - m.AddContext(ctx, 29, AndroidHidlBase, optional, implicit, buildPath(ctx, AndroidHidlBase), nil, nil) + m.AddContext(ctx, 29, AndroidHidlManager, optional, buildPath(ctx, AndroidHidlManager), nil, nil) + m.AddContext(ctx, 29, AndroidHidlBase, optional, buildPath(ctx, AndroidHidlBase), nil, nil) // Add "android.test.mock" to conditional CLC, observe that is gets removed because it is only // needed as a compatibility library if "android.test.runner" is in CLC as well. - m.AddContext(ctx, 30, AndroidTestMock, optional, implicit, buildPath(ctx, AndroidTestMock), nil, nil) + m.AddContext(ctx, 30, AndroidTestMock, optional, buildPath(ctx, AndroidTestMock), nil, nil) valid, validationError := validateClassLoaderContext(m) @@ -165,12 +164,11 @@ func TestCLC(t *testing.T) { func TestCLCJson(t *testing.T) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 29, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - m.AddContext(ctx, 30, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) - m.AddContext(ctx, AnySdkVersion, "d", optional, implicit, buildPath(ctx, "d"), installPath(ctx, "d"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 29, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 30, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), nil) jsonCLC := toJsonClassLoaderContext(m) restored := fromJsonClassLoaderContext(ctx, jsonCLC) android.AssertIntEquals(t, "The size of the maps should be the same.", len(m), len(restored)) @@ -191,13 +189,12 @@ func TestCLCJson(t *testing.T) { func testCLCUnknownPath(t *testing.T, whichPath string) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) if whichPath == "build" { - m.AddContext(ctx, AnySdkVersion, "a", optional, implicit, nil, nil, nil) + m.AddContext(ctx, AnySdkVersion, "a", optional, nil, nil, nil) } else { - m.AddContext(ctx, AnySdkVersion, "a", optional, implicit, buildPath(ctx, "a"), nil, nil) + m.AddContext(ctx, AnySdkVersion, "a", optional, buildPath(ctx, "a"), nil, nil) } // The library should be added to <uses-library> tags by the manifest_fixer. @@ -232,11 +229,10 @@ func TestCLCUnknownInstallPath(t *testing.T) { func TestCLCNestedConditional(t *testing.T) { ctx := testContext() optional := false - implicit := true m1 := make(ClassLoaderContextMap) - m1.AddContext(ctx, 42, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m1.AddContext(ctx, 42, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) m := make(ClassLoaderContextMap) - err := m.addContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), m1) + err := m.addContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), m1) checkError(t, err, "nested class loader context shouldn't have conditional part") } @@ -245,12 +241,11 @@ func TestCLCNestedConditional(t *testing.T) { func TestCLCSdkVersionOrder(t *testing.T) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 29, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - m.AddContext(ctx, 30, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) - m.AddContext(ctx, AnySdkVersion, "d", optional, implicit, buildPath(ctx, "d"), installPath(ctx, "d"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 29, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 30, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), nil) valid, validationError := validateClassLoaderContext(m) @@ -287,7 +282,6 @@ func TestCLCSdkVersionOrder(t *testing.T) { func TestCLCMExcludeLibs(t *testing.T) { ctx := testContext() const optional = false - const implicit = true excludeLibs := func(t *testing.T, m ClassLoaderContextMap, excluded_libs ...string) ClassLoaderContextMap { // Dump the CLCM before creating a new copy that excludes a specific set of libraries. @@ -305,7 +299,7 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("exclude nothing", func(t *testing.T) { m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) a := excludeLibs(t, m) @@ -314,7 +308,6 @@ func TestCLCMExcludeLibs(t *testing.T) { { "Name": "a", "Optional": false, - "Implicit": true, "Host": "out/soong/a.jar", "Device": "/system/a.jar", "Subcontexts": [] @@ -325,8 +318,8 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("one item from list", func(t *testing.T) { m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) a := excludeLibs(t, m, "a") @@ -335,7 +328,6 @@ func TestCLCMExcludeLibs(t *testing.T) { { "Name": "b", "Optional": false, - "Implicit": true, "Host": "out/soong/b.jar", "Device": "/system/b.jar", "Subcontexts": [] @@ -347,8 +339,8 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("all items from a list", func(t *testing.T) { m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) a := excludeLibs(t, m, "a", "b") @@ -357,11 +349,11 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("items from a subcontext", func(t *testing.T) { s := make(ClassLoaderContextMap) - s.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - s.AddContext(ctx, AnySdkVersion, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + s.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + s.AddContext(ctx, AnySdkVersion, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), s) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), s) a := excludeLibs(t, m, "b") @@ -370,14 +362,12 @@ func TestCLCMExcludeLibs(t *testing.T) { { "Name": "a", "Optional": false, - "Implicit": true, "Host": "out/soong/a.jar", "Device": "/system/a.jar", "Subcontexts": [ { "Name": "c", "Optional": false, - "Implicit": true, "Host": "out/soong/c.jar", "Device": "/system/c.jar", "Subcontexts": [] @@ -389,6 +379,35 @@ func TestCLCMExcludeLibs(t *testing.T) { }) } +// Test that CLC is correctly serialized to JSON. +func TestCLCtoJSON(t *testing.T) { + ctx := testContext() + optional := false + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [] + } + ], + "any": [ + { + "Name": "b", + "Optional": false, + "Host": "out/soong/b.jar", + "Device": "/system/b.jar", + "Subcontexts": [] + } + ] +}`, m.Dump()) +} + func checkError(t *testing.T, have error, want string) { if have == nil { t.Errorf("\nwant error: '%s'\nhave: none", want) diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 153b0256f..ab9e418a0 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -36,8 +36,6 @@ type GlobalConfig struct { PreoptWithUpdatableBcp bool // If updatable boot jars are included in dexpreopt or not. - UseArtImage bool // use the art image (use other boot class path dex files without image) - HasSystemOther bool // store odex files that match PatternsOnSystemOther on the system_other partition PatternsOnSystemOther []string // patterns (using '%' to denote a prefix match) to put odex on the system_other partition diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index a142833dd..719771f4c 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -474,6 +474,7 @@ func PrebuiltUserShareFactory() android.Module { // This module is device-only android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) android.InitDefaultableModule(module) + android.InitBazelModule(module) return module } @@ -668,25 +669,17 @@ func generatePrebuiltSnapshot(s snapshot.SnapshotSingleton, ctx android.Singleto // For Bazel / bp2build -type bazelPrebuiltEtcAttributes struct { +type bazelPrebuiltFileAttributes struct { Src bazel.LabelAttribute Filename string - Sub_dir string + Dir string Installable bazel.BoolAttribute } // ConvertWithBp2build performs bp2build conversion of PrebuiltEtc -func (p *PrebuiltEtc) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - // All prebuilt_* modules are PrebuiltEtc, but at this time, we only convert prebuilt_etc modules. - if p.installDirBase != "etc" { - return - } - - prebuiltEtcBp2BuildInternal(ctx, p) -} - -func prebuiltEtcBp2BuildInternal(ctx android.TopDownMutatorContext, module *PrebuiltEtc) { - var srcLabelAttribute bazel.LabelAttribute +// All prebuilt_* modules are PrebuiltEtc, which we treat uniformily as *PrebuiltFile* +func (module *PrebuiltEtc) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + var src bazel.LabelAttribute for axis, configToProps := range module.GetArchVariantProperties(ctx, &prebuiltEtcProperties{}) { for config, p := range configToProps { props, ok := p.(*prebuiltEtcProperties) @@ -695,7 +688,7 @@ func prebuiltEtcBp2BuildInternal(ctx android.TopDownMutatorContext, module *Preb } if props.Src != nil { label := android.BazelLabelForModuleSrcSingle(ctx, *props.Src) - srcLabelAttribute.SetSelectValue(axis, config, label) + src.SetSelectValue(axis, config, label) } } } @@ -705,26 +698,30 @@ func prebuiltEtcBp2BuildInternal(ctx android.TopDownMutatorContext, module *Preb filename = *module.properties.Filename } - var subDir string - if module.subdirProperties.Sub_dir != nil { - subDir = *module.subdirProperties.Sub_dir + var dir = module.installDirBase + // prebuilt_file supports only `etc` or `usr/share` + if !(dir == "etc" || dir == "usr/share") { + return + } + if subDir := module.subdirProperties.Sub_dir; subDir != nil { + dir = dir + "/" + *subDir } - var installableBoolAttribute bazel.BoolAttribute - if module.properties.Installable != nil { - installableBoolAttribute.Value = module.properties.Installable + var installable bazel.BoolAttribute + if install := module.properties.Installable; install != nil { + installable.Value = install } - attrs := &bazelPrebuiltEtcAttributes{ - Src: srcLabelAttribute, + attrs := &bazelPrebuiltFileAttributes{ + Src: src, Filename: filename, - Sub_dir: subDir, - Installable: installableBoolAttribute, + Dir: dir, + Installable: installable, } props := bazel.BazelTargetModuleProperties{ - Rule_class: "prebuilt_etc", - Bzl_load_location: "//build/bazel/rules:prebuilt_etc.bzl", + Rule_class: "prebuilt_file", + Bzl_load_location: "//build/bazel/rules:prebuilt_file.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs) diff --git a/filesystem/Android.bp b/filesystem/Android.bp index 38684d3b2..857dfa7e8 100644 --- a/filesystem/Android.bp +++ b/filesystem/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "bootimg.go", "filesystem.go", "logical_partition.go", + "raw_binary.go", "system_image.go", "vbmeta.go", "testing.go", diff --git a/filesystem/bootimg.go b/filesystem/bootimg.go index 33beb37bf..352b45178 100644 --- a/filesystem/bootimg.go +++ b/filesystem/bootimg.go @@ -228,6 +228,15 @@ func (b *bootimg) signImage(ctx android.ModuleContext, unsignedImage android.Out return output } +// Calculates avb_salt from some input for deterministic output. +func (b *bootimg) salt() string { + var input []string + input = append(input, b.properties.Cmdline...) + input = append(input, proptools.StringDefault(b.properties.Partition_name, b.Name())) + input = append(input, proptools.String(b.properties.Header_version)) + return sha1sum(input) +} + func (b *bootimg) buildPropFile(ctx android.ModuleContext) (propFile android.OutputPath, toolDeps android.Paths) { var sb strings.Builder var deps android.Paths @@ -248,6 +257,7 @@ func (b *bootimg) buildPropFile(ctx android.ModuleContext) (propFile android.Out addStr("avb_add_hash_footer_args", "") // TODO(jiyong): add --rollback_index partitionName := proptools.StringDefault(b.properties.Partition_name, b.Name()) addStr("partition_name", partitionName) + addStr("avb_salt", b.salt()) propFile = android.PathForModuleOut(ctx, "prop").OutputPath android.WriteFileRule(ctx, propFile, sb.String()) diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index ccf9e9d3b..6e1e78a69 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -15,7 +15,9 @@ package filesystem import ( + "crypto/sha256" "fmt" + "io" "path/filepath" "strings" @@ -88,6 +90,13 @@ type filesystemProperties struct { // Symbolic links to be created under root with "ln -sf <target> <name>". Symlinks []symlinkDefinition + + // Seconds since unix epoch to override timestamps of file entries + Fake_timestamp *string + + // When set, passed to mkuserimg_mke2fs --mke2fs_uuid & --mke2fs_hash_seed. + // Otherwise, they'll be set as random which might cause indeterministic build output. + Uuid *string } // android_filesystem packages a set of modules and their transitive dependencies into a filesystem @@ -276,6 +285,11 @@ func (f *filesystem) buildFileContexts(ctx android.ModuleContext) android.Output return fcBin.OutputPath } +// Calculates avb_salt from entry list (sorted) for deterministic output. +func (f *filesystem) salt() string { + return sha1sum(f.entries) +} + func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android.OutputPath, toolDeps android.Paths) { type prop struct { name string @@ -321,12 +335,19 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android. addStr("avb_add_hashtree_footer_args", "--do_not_generate_fec") partitionName := proptools.StringDefault(f.properties.Partition_name, f.Name()) addStr("partition_name", partitionName) + addStr("avb_salt", f.salt()) } if proptools.String(f.properties.File_contexts) != "" { addPath("selinux_fc", f.buildFileContexts(ctx)) } - + if timestamp := proptools.String(f.properties.Fake_timestamp); timestamp != "" { + addStr("timestamp", timestamp) + } + if uuid := proptools.String(f.properties.Uuid); uuid != "" { + addStr("uuid", uuid) + addStr("hash_seed", uuid) + } propFile = android.PathForModuleOut(ctx, "prop").OutputPath builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("rm").Flag("-rf").Output(propFile) @@ -451,3 +472,11 @@ func (f *filesystem) gatherFilteredPackagingSpecs(ctx android.ModuleContext) map } return specs } + +func sha1sum(values []string) string { + h := sha256.New() + for _, value := range values { + io.WriteString(h, value) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/filesystem/raw_binary.go b/filesystem/raw_binary.go new file mode 100644 index 000000000..f7261240d --- /dev/null +++ b/filesystem/raw_binary.go @@ -0,0 +1,120 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystem + +import ( + "fmt" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +var ( + toRawBinary = pctx.AndroidStaticRule("toRawBinary", + blueprint.RuleParams{ + Command: "${objcopy} --output-target=binary ${in} ${out}", + CommandDeps: []string{"$objcopy"}, + }, + "objcopy") +) + +func init() { + pctx.Import("android/soong/cc/config") + + android.RegisterModuleType("raw_binary", rawBinaryFactory) +} + +type rawBinary struct { + android.ModuleBase + + properties rawBinaryProperties + + output android.OutputPath + installDir android.InstallPath +} + +type rawBinaryProperties struct { + // Set the name of the output. Defaults to <module_name>.bin. + Stem *string + + // Name of input executable. Can be a name of a target. + Src *string `android:"path,arch_variant"` +} + +func rawBinaryFactory() android.Module { + module := &rawBinary{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +func (r *rawBinary) DepsMutator(ctx android.BottomUpMutatorContext) { + // do nothing +} + +func (r *rawBinary) installFileName() string { + return proptools.StringDefault(r.properties.Stem, r.BaseModuleName()+".bin") +} + +func (r *rawBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + inputFile := android.PathForModuleSrc(ctx, proptools.String(r.properties.Src)) + outputFile := android.PathForModuleOut(ctx, r.installFileName()).OutputPath + + ctx.Build(pctx, android.BuildParams{ + Rule: toRawBinary, + Description: "prefix symbols " + outputFile.Base(), + Output: outputFile, + Input: inputFile, + Args: map[string]string{ + "objcopy": "${config.ClangBin}/llvm-objcopy", + }, + }) + + r.output = outputFile + r.installDir = android.PathForModuleInstall(ctx, "etc") + ctx.InstallFile(r.installDir, r.installFileName(), r.output) +} + +var _ android.AndroidMkEntriesProvider = (*rawBinary)(nil) + +// Implements android.AndroidMkEntriesProvider +func (r *rawBinary) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(r.output), + }} +} + +var _ Filesystem = (*rawBinary)(nil) + +func (r *rawBinary) OutputPath() android.Path { + return r.output +} + +func (r *rawBinary) SignedOutputPath() android.Path { + return nil +} + +var _ android.OutputFileProducer = (*rawBinary)(nil) + +// Implements android.OutputFileProducer +func (r *rawBinary) OutputFiles(tag string) (android.Paths, error) { + if tag == "" { + return []android.Path{r.output}, nil + } + return nil, fmt.Errorf("unsupported module reference tag %q", tag) +} diff --git a/finder/finder.go b/finder/finder.go index b4834b16b..c5196c8de 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -94,6 +94,10 @@ type CacheParams struct { // RootDirs are the root directories used to initiate the search RootDirs []string + // Whether symlinks are followed. If set, symlinks back to their own parent + // directory don't work. + FollowSymlinks bool + // ExcludeDirs are directory names that if encountered are removed from the search ExcludeDirs []string @@ -1415,9 +1419,14 @@ func (f *Finder) listDirSync(dir *pathMap) { // If stat fails this is probably a broken or dangling symlink, treat it as a file. subfiles = append(subfiles, child.Name()) } else if childStat.IsDir() { - // Skip symlink dirs. - // We don't have to support symlink dirs because - // that would cause duplicates. + // Skip symlink dirs if not requested otherwise. Android has a number + // of symlinks creating infinite source trees which would otherwise get + // us in an infinite loop. + // TODO(b/197349722): Revisit this once symlink loops are banned in the + // source tree. + if f.cacheMetadata.Config.FollowSymlinks { + subdirs = append(subdirs, child.Name()) + } } else { // We do have to support symlink files because the link name might be // different than the target name diff --git a/finder/finder_test.go b/finder/finder_test.go index 788dbdd35..8f73719a6 100644 --- a/finder/finder_test.go +++ b/finder/finder_test.go @@ -90,6 +90,7 @@ func runSimpleTest(t *testing.T, existentPaths []string, expectedMatches []strin CacheParams{ "/cwd", []string{root}, + false, nil, nil, []string{"findme.txt", "skipme.txt"}, @@ -121,6 +122,7 @@ func runTestWithSuffixes(t *testing.T, existentPaths []string, expectedMatches [ CacheParams{ "/cwd", []string{root}, + false, nil, nil, []string{"findme.txt", "skipme.txt"}, diff --git a/fuzz/fuzz_common.go b/fuzz/fuzz_common.go index 89f81879f..700cdf0a7 100644 --- a/fuzz/fuzz_common.go +++ b/fuzz/fuzz_common.go @@ -82,6 +82,9 @@ type FuzzConfig struct { Hwasan_options []string `json:"hwasan_options,omitempty"` // Additional options to be passed to HWASAN when running on host in Haiku. Asan_options []string `json:"asan_options,omitempty"` + // If there's a Java fuzzer with JNI, a different version of Jazzer would + // need to be added to the fuzzer package than one without JNI + IsJni *bool `json:"is_jni,omitempty"` } type FuzzProperties struct { diff --git a/genrule/genrule.go b/genrule/genrule.go index c52ddee53..3531ee6b8 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -578,7 +578,7 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { bazelModuleLabel := g.GetBazelLabel(ctx, g) bazelActionsUsed := false - if g.MixedBuildsEnabled(ctx) { + if android.MixedBuildsEnabled(ctx) { bazelActionsUsed = g.GenerateBazelBuildActions(ctx, bazelModuleLabel) } if !bazelActionsUsed { diff --git a/java/android_manifest.go b/java/android_manifest.go index 7772b7090..a297b2c10 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -82,8 +82,8 @@ func ManifestFixer(ctx android.ModuleContext, manifest android.Path, if minSdkVersion.FinalOrFutureInt() >= 23 { args = append(args, fmt.Sprintf("--extract-native-libs=%v", !params.UseEmbeddedNativeLibs)) } else if params.UseEmbeddedNativeLibs { - ctx.ModuleErrorf("module attempted to store uncompressed native libraries, but minSdkVersion=%d doesn't support it", - minSdkVersion) + ctx.ModuleErrorf("module attempted to store uncompressed native libraries, but minSdkVersion=%s doesn't support it", + minSdkVersion.String()) } } @@ -96,9 +96,9 @@ func ManifestFixer(ctx android.ModuleContext, manifest android.Path, } if params.ClassLoaderContexts != nil { - // manifest_fixer should add only the implicit SDK libraries inferred by Soong, not those added - // explicitly via `uses_libs`/`optional_uses_libs`. - requiredUsesLibs, optionalUsesLibs := params.ClassLoaderContexts.ImplicitUsesLibs() + // Libraries propagated via `uses_libs`/`optional_uses_libs` are also added (they may be + // propagated from dependencies). + requiredUsesLibs, optionalUsesLibs := params.ClassLoaderContexts.UsesLibs() for _, usesLib := range requiredUsesLibs { args = append(args, "--uses-library", usesLib) diff --git a/java/androidmk.go b/java/androidmk.go index 80b828d45..f6ea6a930 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -132,6 +132,16 @@ func (library *Library) AndroidMkEntries() []android.AndroidMkEntries { return entriesList } +func (j *JavaFuzzLibrary) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := j.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", "null-suite") + androidMkWriteTestData(j.jniFilePaths, entries) + }) + return entriesList +} + // Called for modules that are a component of a test suite. func testSuiteComponent(entries *android.AndroidMkEntries, test_suites []string, perTestcaseDirectory bool) { entries.SetString("LOCAL_MODULE_TAGS", "tests") diff --git a/java/app.go b/java/app.go index 2b52eab15..768d9e923 100755 --- a/java/app.go +++ b/java/app.go @@ -900,6 +900,7 @@ func AndroidAppFactory() android.Module { module.Module.dexProperties.Optimize.Shrink = proptools.BoolPtr(true) module.Module.properties.Instrument = true + module.Module.properties.Supports_static_instrumentation = true module.Module.properties.Installable = proptools.BoolPtr(true) module.addHostAndDeviceProperties() @@ -1019,6 +1020,7 @@ func AndroidTestFactory() android.Module { module.Module.dexProperties.Optimize.EnabledByDefault = true module.Module.properties.Instrument = true + module.Module.properties.Supports_static_instrumentation = true module.Module.properties.Installable = proptools.BoolPtr(true) module.appProperties.Use_embedded_native_libs = proptools.BoolPtr(true) module.appProperties.AlwaysPackageNativeLibs = true @@ -1225,28 +1227,17 @@ func (u *usesLibrary) addLib(lib string, optional bool) { func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, hasFrameworkLibs bool) { if !ctx.Config().UnbundledBuild() || ctx.Config().UnbundledBuildImage() { - reqTag := makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, false, false) - ctx.AddVariationDependencies(nil, reqTag, u.usesLibraryProperties.Uses_libs...) - - optTag := makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, true, false) - ctx.AddVariationDependencies(nil, optTag, u.presentOptionalUsesLibs(ctx)...) - + ctx.AddVariationDependencies(nil, usesLibReqTag, u.usesLibraryProperties.Uses_libs...) + ctx.AddVariationDependencies(nil, usesLibOptTag, u.presentOptionalUsesLibs(ctx)...) // Only add these extra dependencies if the module depends on framework libs. This avoids // creating a cyclic dependency: // e.g. framework-res -> org.apache.http.legacy -> ... -> framework-res. if hasFrameworkLibs { - // Add implicit <uses-library> dependencies on compatibility libraries. Some of them are - // optional, and some required --- this depends on the most common usage of the library - // and may be wrong for some apps (they need explicit `uses_libs`/`optional_uses_libs`). - - compat28OptTag := makeUsesLibraryDependencyTag(28, true, true) - ctx.AddVariationDependencies(nil, compat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) - - compat29ReqTag := makeUsesLibraryDependencyTag(29, false, true) - ctx.AddVariationDependencies(nil, compat29ReqTag, dexpreopt.CompatUsesLibs29...) - - compat30OptTag := makeUsesLibraryDependencyTag(30, true, true) - ctx.AddVariationDependencies(nil, compat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) + // Dexpreopt needs paths to the dex jars of these libraries in order to construct + // class loader context for dex2oat. Add them as a dependency with a special tag. + ctx.AddVariationDependencies(nil, usesLibCompat29ReqTag, dexpreopt.CompatUsesLibs29...) + ctx.AddVariationDependencies(nil, usesLibCompat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) + ctx.AddVariationDependencies(nil, usesLibCompat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) } } } @@ -1305,7 +1296,7 @@ func (u *usesLibrary) classLoaderContextForUsesLibDeps(ctx android.ModuleContext replaceInList(u.usesLibraryProperties.Uses_libs, dep, libName) replaceInList(u.usesLibraryProperties.Optional_uses_libs, dep, libName) } - clcMap.AddContext(ctx, tag.sdkVersion, libName, tag.optional, tag.implicit, + clcMap.AddContext(ctx, tag.sdkVersion, libName, tag.optional, lib.DexJarBuildPath().PathOrNil(), lib.DexJarInstallPath(), lib.ClassLoaderContexts()) } else if ctx.Config().AllowMissingDependencies() { diff --git a/java/app_test.go b/java/app_test.go index 6a4508cd6..8324dff6c 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -2505,12 +2505,20 @@ func TestUsesLibraries(t *testing.T) { prebuilt := result.ModuleForTests("prebuilt", "android_common") // Test that implicit dependencies on java_sdk_library instances are passed to the manifest. - // This should not include explicit `uses_libs`/`optional_uses_libs` entries. + // These also include explicit `uses_libs`/`optional_uses_libs` entries, as they may be + // propagated from dependencies. actualManifestFixerArgs := app.Output("manifest_fixer/AndroidManifest.xml").Args["args"] expectManifestFixerArgs := `--extract-native-libs=true ` + `--uses-library qux ` + `--uses-library quuz ` + - `--uses-library runtime-library` + `--uses-library foo ` + + `--uses-library com.non.sdk.lib ` + + `--uses-library runtime-library ` + + `--uses-library runtime-required-x ` + + `--uses-library runtime-required-y ` + + `--optional-uses-library bar ` + + `--optional-uses-library runtime-optional-x ` + + `--optional-uses-library runtime-optional-y` android.AssertStringDoesContain(t, "manifest_fixer args", actualManifestFixerArgs, expectManifestFixerArgs) // Test that all libraries are verified (library order matters). diff --git a/java/base.go b/java/base.go index b925350a0..e60e54ec1 100644 --- a/java/base.go +++ b/java/base.go @@ -170,6 +170,9 @@ type CommonProperties struct { } Instrument bool `blueprint:"mutated"` + // If true, then the module supports statically including the jacocoagent + // into the library. + Supports_static_instrumentation bool `blueprint:"mutated"` // List of files to include in the META-INF/services folder of the resulting jar. Services []string `android:"path,arch_variant"` @@ -442,9 +445,6 @@ type Module struct { // manifest file to use instead of properties.Manifest overrideManifest android.OptionalPath - // map of SDK version to class loader context - classLoaderContexts dexpreopt.ClassLoaderContextMap - // list of plugins that this java module is exporting exportedPluginJars android.Paths @@ -605,7 +605,8 @@ func (j *Module) shouldInstrument(ctx android.BaseModuleContext) bool { } func (j *Module) shouldInstrumentStatic(ctx android.BaseModuleContext) bool { - return j.shouldInstrument(ctx) && + return j.properties.Supports_static_instrumentation && + j.shouldInstrument(ctx) && (ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_STATIC") || ctx.Config().UnbundledBuild()) } @@ -723,8 +724,10 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { if component, ok := dep.(SdkLibraryComponentDependency); ok { if lib := component.OptionalSdkLibraryImplementation(); lib != nil { // Add library as optional if it's one of the optional compatibility libs. - optional := android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) - tag := makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, optional, true) + tag := usesLibReqTag + if android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) { + tag = usesLibOptTag + } ctx.AddVariationDependencies(nil, tag, *lib) } } @@ -1481,11 +1484,11 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { } if ctx.Device() { - lintSDKVersionString := func(sdkSpec android.SdkSpec) string { + lintSDKVersion := func(sdkSpec android.SdkSpec) android.ApiLevel { if v := sdkSpec.ApiLevel; !v.IsPreview() { - return v.String() + return v } else { - return ctx.Config().DefaultAppTargetSdk(ctx).String() + return ctx.Config().DefaultAppTargetSdk(ctx) } } @@ -1494,9 +1497,9 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.linter.srcJars = srcJars j.linter.classpath = append(append(android.Paths(nil), flags.bootClasspath...), flags.classpath...) j.linter.classes = j.implementationJarFile - j.linter.minSdkVersion = lintSDKVersionString(j.MinSdkVersion(ctx)) - j.linter.targetSdkVersion = lintSDKVersionString(j.TargetSdkVersion(ctx)) - j.linter.compileSdkVersion = lintSDKVersionString(j.SdkVersion(ctx)) + j.linter.minSdkVersion = lintSDKVersion(j.MinSdkVersion(ctx)) + j.linter.targetSdkVersion = lintSDKVersion(j.TargetSdkVersion(ctx)) + j.linter.compileSdkVersion = lintSDKVersion(j.SdkVersion(ctx)) j.linter.compileSdkKind = j.SdkVersion(ctx).Kind j.linter.javaLanguageLevel = flags.javaVersion.String() j.linter.kotlinLanguageLevel = "1.3" diff --git a/java/config/config.go b/java/config/config.go index 95b841fa7..d7440022f 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -68,6 +68,11 @@ var ( "-J-XX:+TieredCompilation", "-J-XX:TieredStopAtLevel=1", } + dexerJavaVmFlagsList = []string{ + `-JXX:OnError="cat hs_err_pid%p.log"`, + "-JXX:CICompilerCount=6", + "-JXX:+UseDynamicNumberOfGCThreads", + } ) func init() { @@ -81,11 +86,16 @@ func init() { exportedVars.ExportStringStaticVariable("ErrorProneHeapSize", "4096M") exportedVars.ExportStringStaticVariable("ErrorProneHeapFlags", "-J-Xmx${ErrorProneHeapSize}") - exportedVars.ExportStringListStaticVariable("DexFlags", []string{ - `-JXX:OnError="cat hs_err_pid%p.log"`, - "-JXX:CICompilerCount=6", - "-JXX:+UseDynamicNumberOfGCThreads", - }) + // D8 invocations are shorter lived, so we restrict their JIT tiering relative to R8. + // Note that the `-JXX` prefix syntax is specific to the R8/D8 invocation wrappers. + exportedVars.ExportStringListStaticVariable("D8Flags", append([]string{ + "-JXmx2048M", + "-JXX:+TieredCompilation", + "-JXX:TieredStopAtLevel=1", + }, dexerJavaVmFlagsList...)) + exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ + "-JXmx2048M", + }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{ `-Xmaxerrs 9999999`, diff --git a/java/config/makevars.go b/java/config/makevars.go index df447a129..273aca0b4 100644 --- a/java/config/makevars.go +++ b/java/config/makevars.go @@ -43,9 +43,10 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("JAVADOC", "${JavadocCmd}") ctx.Strict("COMMON_JDK_FLAGS", "${CommonJdkFlags}") - ctx.Strict("DX", "${D8Cmd}") - ctx.Strict("DX_COMMAND", "${D8Cmd} -JXms16M -JXmx2048M") - ctx.Strict("R8_COMPAT_PROGUARD", "${R8Cmd}") + ctx.Strict("D8", "${D8Cmd}") + ctx.Strict("R8", "${R8Cmd}") + ctx.Strict("D8_COMMAND", "${D8Cmd} ${D8Flags}") + ctx.Strict("R8_COMMAND", "${R8Cmd} ${R8Flags}") ctx.Strict("TURBINE", "${TurbineJar}") @@ -78,8 +79,6 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("CLASS2NONSDKLIST", "${Class2NonSdkList}") ctx.Strict("HIDDENAPI", "${HiddenAPI}") - ctx.Strict("DEX_FLAGS", "${DexFlags}") - ctx.Strict("AIDL", "${AidlCmd}") ctx.Strict("AAPT2", "${Aapt2Cmd}") ctx.Strict("ZIPALIGN", "${ZipAlign}") diff --git a/java/dex.go b/java/dex.go index 84665e7b5..13d6e4a01 100644 --- a/java/dex.go +++ b/java/dex.go @@ -95,7 +95,7 @@ var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8", Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + `mkdir -p $$(dirname $tmpJar) && ` + `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + - `$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $tmpJar && ` + + `$d8Template${config.D8Cmd} ${config.D8Flags} --output $outDir $d8Flags $tmpJar && ` + `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`, CommandDeps: []string{ @@ -128,7 +128,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", `mkdir -p $$(dirname ${outUsage}) && ` + `mkdir -p $$(dirname $tmpJar) && ` + `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + - `$r8Template${config.R8Cmd} ${config.DexFlags} -injars $tmpJar --output $outDir ` + + `$r8Template${config.R8Cmd} ${config.R8Flags} -injars $tmpJar --output $outDir ` + `--no-data-resources ` + `-printmapping ${outDict} ` + `-printusage ${outUsage} ` + diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 7c5f055e6..0adaf9917 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -259,10 +259,6 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr isSystemServerJar := global.AllSystemServerJars(ctx).ContainsJar(moduleName(ctx)) bootImage := defaultBootImageConfig(ctx) - if global.UseArtImage { - bootImage = artBootImageConfig(ctx) - } - dexFiles, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) targets := ctx.MultiTargets() diff --git a/java/dexpreopt.go_v1 b/java/dexpreopt.go_v1 new file mode 100644 index 000000000..0adaf9917 --- /dev/null +++ b/java/dexpreopt.go_v1 @@ -0,0 +1,404 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/dexpreopt" +) + +type DexpreopterInterface interface { + IsInstallable() bool // Structs that embed dexpreopter must implement this. + dexpreoptDisabled(ctx android.BaseModuleContext) bool + DexpreoptBuiltInstalledForApex() []dexpreopterInstall + AndroidMkEntriesForApex() []android.AndroidMkEntries +} + +type dexpreopterInstall struct { + // A unique name to distinguish an output from others for the same java library module. Usually in + // the form of `<arch>-<encoded-path>.odex/vdex/art`. + name string + + // The name of the input java module. + moduleName string + + // The path to the dexpreopt output on host. + outputPathOnHost android.Path + + // The directory on the device for the output to install to. + installDirOnDevice android.InstallPath + + // The basename (the last segment of the path) for the output to install as. + installFileOnDevice string +} + +// The full module name of the output in the makefile. +func (install *dexpreopterInstall) FullModuleName() string { + return install.moduleName + install.SubModuleName() +} + +// The sub-module name of the output in the makefile (the name excluding the java module name). +func (install *dexpreopterInstall) SubModuleName() string { + return "-dexpreopt-" + install.name +} + +// Returns Make entries for installing the file. +// +// This function uses a value receiver rather than a pointer receiver to ensure that the object is +// safe to use in `android.AndroidMkExtraEntriesFunc`. +func (install dexpreopterInstall) ToMakeEntries() android.AndroidMkEntries { + return android.AndroidMkEntries{ + Class: "ETC", + SubName: install.SubModuleName(), + OutputFile: android.OptionalPathForPath(install.outputPathOnHost), + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", install.installDirOnDevice.String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", install.installFileOnDevice) + entries.SetString("LOCAL_NOT_AVAILABLE_FOR_PLATFORM", "false") + }, + }, + } +} + +type dexpreopter struct { + dexpreoptProperties DexpreoptProperties + + installPath android.InstallPath + uncompressedDex bool + isSDKLibrary bool + isApp bool + isTest bool + isPresignedPrebuilt bool + preventInstall bool + + manifestFile android.Path + statusFile android.WritablePath + enforceUsesLibs bool + classLoaderContexts dexpreopt.ClassLoaderContextMap + + // See the `dexpreopt` function for details. + builtInstalled string + builtInstalledForApex []dexpreopterInstall + + // The config is used for two purposes: + // - Passing dexpreopt information about libraries from Soong to Make. This is needed when + // a <uses-library> is defined in Android.bp, but used in Android.mk (see dex_preopt_config_merger.py). + // Note that dexpreopt.config might be needed even if dexpreopt is disabled for the library itself. + // - Dexpreopt post-processing (using dexpreopt artifacts from a prebuilt system image to incrementally + // dexpreopt another partition). + configPath android.WritablePath +} + +type DexpreoptProperties struct { + Dex_preopt struct { + // If false, prevent dexpreopting. Defaults to true. + Enabled *bool + + // If true, generate an app image (.art file) for this module. + App_image *bool + + // If true, use a checked-in profile to guide optimization. Defaults to false unless + // a matching profile is set or a profile is found in PRODUCT_DEX_PREOPT_PROFILE_DIR + // that matches the name of this module, in which case it is defaulted to true. + Profile_guided *bool + + // If set, provides the path to profile relative to the Android.bp file. If not set, + // defaults to searching for a file that matches the name of this module in the default + // profile location set by PRODUCT_DEX_PREOPT_PROFILE_DIR, or empty if not found. + Profile *string `android:"path"` + } +} + +func init() { + dexpreopt.DexpreoptRunningInSoong = true +} + +func isApexVariant(ctx android.BaseModuleContext) bool { + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + return !apexInfo.IsForPlatform() +} + +func forPrebuiltApex(ctx android.BaseModuleContext) bool { + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + return apexInfo.ForPrebuiltApex +} + +func moduleName(ctx android.BaseModuleContext) string { + // Remove the "prebuilt_" prefix if the module is from a prebuilt because the prefix is not + // expected by dexpreopter. + return android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()) +} + +func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { + if !ctx.Device() { + return true + } + + if d.isTest { + return true + } + + if !BoolDefault(d.dexpreoptProperties.Dex_preopt.Enabled, true) { + return true + } + + // If the module is from a prebuilt APEX, it shouldn't be installable, but it can still be + // dexpreopted. + if !ctx.Module().(DexpreopterInterface).IsInstallable() && !forPrebuiltApex(ctx) { + return true + } + + if !android.IsModulePreferred(ctx.Module()) { + return true + } + + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisablePreopt { + return true + } + + if inList(moduleName(ctx), global.DisablePreoptModules) { + return true + } + + isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + if isApexVariant(ctx) { + // Don't preopt APEX variant module unless the module is an APEX system server jar and we are + // building the entire system image. + if !isApexSystemServerJar || ctx.Config().UnbundledBuild() { + return true + } + } else { + // Don't preopt the platform variant of an APEX system server jar to avoid conflicts. + if isApexSystemServerJar { + return true + } + } + + // TODO: contains no java code + + return false +} + +func dexpreoptToolDepsMutator(ctx android.BottomUpMutatorContext) { + if d, ok := ctx.Module().(DexpreopterInterface); !ok || d.dexpreoptDisabled(ctx) { + return + } + dexpreopt.RegisterToolDeps(ctx) +} + +func (d *dexpreopter) odexOnSystemOther(ctx android.ModuleContext, installPath android.InstallPath) bool { + return dexpreopt.OdexOnSystemOtherByName(moduleName(ctx), android.InstallPathToOnDevicePath(ctx, installPath), dexpreopt.GetGlobalConfig(ctx)) +} + +// Returns the install path of the dex jar of a module. +// +// Do not rely on `ApexInfo.ApexVariationName` because it can be something like "apex1000", rather +// than the `name` in the path `/apex/<name>` as suggested in its comment. +// +// This function is on a best-effort basis. It cannot handle the case where an APEX jar is not a +// system server jar, which is fine because we currently only preopt system server jars for APEXes. +func (d *dexpreopter) getInstallPath( + ctx android.ModuleContext, defaultInstallPath android.InstallPath) android.InstallPath { + global := dexpreopt.GetGlobalConfig(ctx) + if global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) { + dexLocation := dexpreopt.GetSystemServerDexLocation(ctx, global, moduleName(ctx)) + return android.PathForModuleInPartitionInstall(ctx, "", strings.TrimPrefix(dexLocation, "/")) + } + if !d.dexpreoptDisabled(ctx) && isApexVariant(ctx) && + filepath.Base(defaultInstallPath.PartitionDir()) != "apex" { + ctx.ModuleErrorf("unable to get the install path of the dex jar for dexpreopt") + } + return defaultInstallPath +} + +func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.WritablePath) { + global := dexpreopt.GetGlobalConfig(ctx) + + // TODO(b/148690468): The check on d.installPath is to bail out in cases where + // the dexpreopter struct hasn't been fully initialized before we're called, + // e.g. in aar.go. This keeps the behaviour that dexpreopting is effectively + // disabled, even if installable is true. + if d.installPath.Base() == "." { + return + } + + dexLocation := android.InstallPathToOnDevicePath(ctx, d.installPath) + + providesUsesLib := moduleName(ctx) + if ulib, ok := ctx.Module().(ProvidesUsesLib); ok { + name := ulib.ProvidesUsesLib() + if name != nil { + providesUsesLib = *name + } + } + + // If it is test, make config files regardless of its dexpreopt setting. + // The config files are required for apps defined in make which depend on the lib. + if d.isTest && d.dexpreoptDisabled(ctx) { + return + } + + isSystemServerJar := global.AllSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + + bootImage := defaultBootImageConfig(ctx) + dexFiles, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) + + targets := ctx.MultiTargets() + if len(targets) == 0 { + // assume this is a java library, dexpreopt for all arches for now + for _, target := range ctx.Config().Targets[android.Android] { + if target.NativeBridge == android.NativeBridgeDisabled { + targets = append(targets, target) + } + } + if isSystemServerJar && !d.isSDKLibrary { + // If the module is not an SDK library and it's a system server jar, only preopt the primary arch. + targets = targets[:1] + } + } + + var archs []android.ArchType + var images android.Paths + var imagesDeps []android.OutputPaths + for _, target := range targets { + archs = append(archs, target.Arch.ArchType) + variant := bootImage.getVariant(target) + images = append(images, variant.imagePathOnHost) + imagesDeps = append(imagesDeps, variant.imagesDeps) + } + // The image locations for all Android variants are identical. + hostImageLocations, deviceImageLocations := bootImage.getAnyAndroidVariant().imageLocations() + + var profileClassListing android.OptionalPath + var profileBootListing android.OptionalPath + profileIsTextListing := false + if BoolDefault(d.dexpreoptProperties.Dex_preopt.Profile_guided, true) { + // If dex_preopt.profile_guided is not set, default it based on the existence of the + // dexprepot.profile option or the profile class listing. + if String(d.dexpreoptProperties.Dex_preopt.Profile) != "" { + profileClassListing = android.OptionalPathForPath( + android.PathForModuleSrc(ctx, String(d.dexpreoptProperties.Dex_preopt.Profile))) + profileBootListing = android.ExistentPathForSource(ctx, + ctx.ModuleDir(), String(d.dexpreoptProperties.Dex_preopt.Profile)+"-boot") + profileIsTextListing = true + } else if global.ProfileDir != "" { + profileClassListing = android.ExistentPathForSource(ctx, + global.ProfileDir, moduleName(ctx)+".prof") + } + } + + // Full dexpreopt config, used to create dexpreopt build rules. + dexpreoptConfig := &dexpreopt.ModuleConfig{ + Name: moduleName(ctx), + DexLocation: dexLocation, + BuildPath: android.PathForModuleOut(ctx, "dexpreopt", moduleName(ctx)+".jar").OutputPath, + DexPath: dexJarFile, + ManifestPath: android.OptionalPathForPath(d.manifestFile), + UncompressedDex: d.uncompressedDex, + HasApkLibraries: false, + PreoptFlags: nil, + + ProfileClassListing: profileClassListing, + ProfileIsTextListing: profileIsTextListing, + ProfileBootListing: profileBootListing, + + EnforceUsesLibrariesStatusFile: dexpreopt.UsesLibrariesStatusFile(ctx), + EnforceUsesLibraries: d.enforceUsesLibs, + ProvidesUsesLibrary: providesUsesLib, + ClassLoaderContexts: d.classLoaderContexts, + + Archs: archs, + DexPreoptImagesDeps: imagesDeps, + DexPreoptImageLocationsOnHost: hostImageLocations, + DexPreoptImageLocationsOnDevice: deviceImageLocations, + + PreoptBootClassPathDexFiles: dexFiles.Paths(), + PreoptBootClassPathDexLocations: dexLocations, + + PreoptExtractedApk: false, + + NoCreateAppImage: !BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, true), + ForceCreateAppImage: BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, false), + + PresignedPrebuilt: d.isPresignedPrebuilt, + } + + d.configPath = android.PathForModuleOut(ctx, "dexpreopt", "dexpreopt.config") + dexpreopt.WriteModuleConfig(ctx, dexpreoptConfig, d.configPath) + + if d.dexpreoptDisabled(ctx) { + return + } + + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + + dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, globalSoong, global, dexpreoptConfig) + if err != nil { + ctx.ModuleErrorf("error generating dexpreopt rule: %s", err.Error()) + return + } + + dexpreoptRule.Build("dexpreopt", "dexpreopt") + + isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + + for _, install := range dexpreoptRule.Installs() { + // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. + installDir := strings.TrimPrefix(filepath.Dir(install.To), "/") + installBase := filepath.Base(install.To) + arch := filepath.Base(installDir) + installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) + + if isApexSystemServerJar { + // APEX variants of java libraries are hidden from Make, so their dexpreopt + // outputs need special handling. Currently, for APEX variants of java + // libraries, only those in the system server classpath are handled here. + // Preopting of boot classpath jars in the ART APEX are handled in + // java/dexpreopt_bootjars.go, and other APEX jars are not preopted. + // The installs will be handled by Make as sub-modules of the java library. + d.builtInstalledForApex = append(d.builtInstalledForApex, dexpreopterInstall{ + name: arch + "-" + installBase, + moduleName: moduleName(ctx), + outputPathOnHost: install.From, + installDirOnDevice: installPath, + installFileOnDevice: installBase, + }) + } else if !d.preventInstall { + ctx.InstallFile(installPath, installBase, install.From) + } + } + + if !isApexSystemServerJar { + d.builtInstalled = dexpreoptRule.Installs().String() + } +} + +func (d *dexpreopter) DexpreoptBuiltInstalledForApex() []dexpreopterInstall { + return d.builtInstalledForApex +} + +func (d *dexpreopter) AndroidMkEntriesForApex() []android.AndroidMkEntries { + var entries []android.AndroidMkEntries + for _, install := range d.builtInstalledForApex { + entries = append(entries, install.ToMakeEntries()) + } + return entries +} diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 3d91aec91..7c4da3ef6 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -213,12 +213,6 @@ import ( // writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names, // paths and so on. // -// 2.5. JIT-Zygote configuration -// ----------------------------- -// -// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling -// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage). -// var artApexNames = []string{ "com.android.art", @@ -938,11 +932,8 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(dexLocations, " ")) var imageNames []string - // TODO: the primary ART boot image should not be exposed to Make, as it is installed in a - // different way as a part of the ART APEX. However, there is a special JIT-Zygote build - // configuration which uses the primary ART image instead of the Framework boot image - // extension, and it relies on the ART image being exposed to Make. To fix this, it is - // necessary to rework the logic in makefiles. + // The primary ART boot image is exposed to Make for testing (gtests) and benchmarking + // (golem) purposes. for _, current := range append(d.otherImages, image) { imageNames = append(imageNames, current.name) for _, variant := range current.variants { diff --git a/java/dexpreopt_bootjars.go_v1 b/java/dexpreopt_bootjars.go_v1 new file mode 100644 index 000000000..07a357bb5 --- /dev/null +++ b/java/dexpreopt_bootjars.go_v1 @@ -0,0 +1,952 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/dexpreopt" + + "github.com/google/blueprint/proptools" +) + +// ================================================================================================= +// WIP - see http://b/177892522 for details +// +// The build support for boot images is currently being migrated away from singleton to modules so +// the documentation may not be strictly accurate. Rather than update the documentation at every +// step which will create a lot of churn the changes that have been made will be listed here and the +// documentation will be updated once it is closer to the final result. +// +// Changes: +// 1) dex_bootjars is now a singleton module and not a plain singleton. +// 2) Boot images are now represented by the boot_image module type. +// 3) The art boot image is called "art-boot-image", the framework boot image is called +// "framework-boot-image". +// 4) They are defined in art/build/boot/Android.bp and frameworks/base/boot/Android.bp +// respectively. +// 5) Each boot_image retrieves the appropriate boot image configuration from the map returned by +// genBootImageConfigs() using the image_name specified in the boot_image module. +// ================================================================================================= + +// This comment describes: +// 1. ART boot images in general (their types, structure, file layout, etc.) +// 2. build system support for boot images +// +// 1. ART boot images +// ------------------ +// +// A boot image in ART is a set of files that contain AOT-compiled native code and a heap snapshot +// of AOT-initialized classes for the bootclasspath Java libraries. A boot image is compiled from a +// set of DEX jars by the dex2oat compiler. A boot image is used for two purposes: 1) it is +// installed on device and loaded at runtime, and 2) other Java libraries and apps are compiled +// against it (compilation may take place either on host, known as "dexpreopt", or on device, known +// as "dexopt"). +// +// A boot image is not a single file, but a collection of interrelated files. Each boot image has a +// number of components that correspond to the Java libraries that constitute it. For each component +// there are multiple files: +// - *.oat or *.odex file with native code (architecture-specific, one per instruction set) +// - *.art file with pre-initialized Java classes (architecture-specific, one per instruction set) +// - *.vdex file with verification metadata for the DEX bytecode (architecture independent) +// +// *.vdex files for the boot images do not contain the DEX bytecode itself, because the +// bootclasspath DEX files are stored on disk in uncompressed and aligned form. Consequently a boot +// image is not self-contained and cannot be used without its DEX files. To simplify the management +// of boot image files, ART uses a certain naming scheme and associates the following metadata with +// each boot image: +// - A stem, which is a symbolic name that is prepended to boot image file names. +// - A location (on-device path to the boot image files). +// - A list of boot image locations (on-device paths to dependency boot images). +// - A set of DEX locations (on-device paths to the DEX files, one location for one DEX file used +// to compile the boot image). +// +// There are two kinds of boot images: +// - primary boot images +// - boot image extensions +// +// 1.1. Primary boot images +// ------------------------ +// +// A primary boot image is compiled for a core subset of bootclasspath Java libraries. It does not +// depend on any other images, and other boot images may depend on it. +// +// For example, assuming that the stem is "boot", the location is /apex/com.android.art/javalib/, +// the set of core bootclasspath libraries is A B C, and the boot image is compiled for ARM targets +// (32 and 64 bits), it will have three components with the following files: +// - /apex/com.android.art/javalib/{arm,arm64}/boot.{art,oat,vdex} +// - /apex/com.android.art/javalib/{arm,arm64}/boot-B.{art,oat,vdex} +// - /apex/com.android.art/javalib/{arm,arm64}/boot-C.{art,oat,vdex} +// +// The files of the first component are special: they do not have the component name appended after +// the stem. This naming convention dates back to the times when the boot image was not split into +// components, and there were just boot.oat and boot.art. The decision to split was motivated by +// licensing reasons for one of the bootclasspath libraries. +// +// As of November 2020 the only primary boot image in Android is the image in the ART APEX +// com.android.art. The primary ART boot image contains the Core libraries that are part of the ART +// module. When the ART module gets updated, the primary boot image will be updated with it, and all +// dependent images will get invalidated (the checksum of the primary image stored in dependent +// images will not match), unless they are updated in sync with the ART module. +// +// 1.2. Boot image extensions +// -------------------------- +// +// A boot image extension is compiled for a subset of bootclasspath Java libraries (in particular, +// this subset does not include the Core bootclasspath libraries that go into the primary boot +// image). A boot image extension depends on the primary boot image and optionally some other boot +// image extensions. Other images may depend on it. In other words, boot image extensions can form +// acyclic dependency graphs. +// +// The motivation for boot image extensions comes from the Mainline project. Consider a situation +// when the list of bootclasspath libraries is A B C, and both A and B are parts of the Android +// platform, but C is part of an updatable APEX com.android.C. When the APEX is updated, the Java +// code for C might have changed compared to the code that was used to compile the boot image. +// Consequently, the whole boot image is obsolete and invalidated (even though the code for A and B +// that does not depend on C is up to date). To avoid this, the original monolithic boot image is +// split in two parts: the primary boot image that contains A B, and the boot image extension that +// contains C and depends on the primary boot image (extends it). +// +// For example, assuming that the stem is "boot", the location is /system/framework, the set of +// bootclasspath libraries is D E (where D is part of the platform and is located in +// /system/framework, and E is part of a non-updatable APEX com.android.E and is located in +// /apex/com.android.E/javalib), and the boot image is compiled for ARM targets (32 and 64 bits), +// it will have two components with the following files: +// - /system/framework/{arm,arm64}/boot-D.{art,oat,vdex} +// - /system/framework/{arm,arm64}/boot-E.{art,oat,vdex} +// +// As of November 2020 the only boot image extension in Android is the Framework boot image +// extension. It extends the primary ART boot image and contains Framework libraries and other +// bootclasspath libraries from the platform and non-updatable APEXes that are not included in the +// ART image. The Framework boot image extension is updated together with the platform. In the +// future other boot image extensions may be added for some updatable modules. +// +// +// 2. Build system support for boot images +// --------------------------------------- +// +// The primary ART boot image needs to be compiled with one dex2oat invocation that depends on DEX +// jars for the core libraries. Framework boot image extension needs to be compiled with one dex2oat +// invocation that depends on the primary ART boot image and all bootclasspath DEX jars except the +// core libraries as they are already part of the primary ART boot image. +// +// 2.1. Libraries that go in the boot images +// ----------------------------------------- +// +// The contents of each boot image are determined by the PRODUCT variables. The primary ART APEX +// boot image contains libraries listed in the ART_APEX_JARS variable in the AOSP makefiles. The +// Framework boot image extension contains libraries specified in the PRODUCT_BOOT_JARS and +// PRODUCT_BOOT_JARS_EXTRA variables. The AOSP makefiles specify some common Framework libraries, +// but more product-specific libraries can be added in the product makefiles. +// +// Each component of the PRODUCT_BOOT_JARS and PRODUCT_BOOT_JARS_EXTRA variables is a +// colon-separated pair <apex>:<library>, where <apex> is the variant name of a non-updatable APEX, +// "platform" if the library is a part of the platform in the system partition, or "system_ext" if +// it's in the system_ext partition. +// +// In these variables APEXes are identified by their "variant names", i.e. the names they get +// mounted as in /apex on device. In Soong modules that is the name set in the "apex_name" +// properties, which default to the "name" values. For example, many APEXes have both +// com.android.xxx and com.google.android.xxx modules in Soong, but take the same place +// /apex/com.android.xxx at runtime. In these cases the variant name is always com.android.xxx, +// regardless which APEX goes into the product. See also android.ApexInfo.ApexVariationName and +// apex.apexBundleProperties.Apex_name. +// +// A related variable PRODUCT_APEX_BOOT_JARS contains bootclasspath libraries that are in APEXes. +// They are not included in the boot image. The only exception here are ART jars and core-icu4j.jar +// that have been historically part of the boot image and are now in apexes; they are in boot images +// and core-icu4j.jar is generally treated as being part of PRODUCT_BOOT_JARS. +// +// One exception to the above rules are "coverage" builds (a special build flavor which requires +// setting environment variable EMMA_INSTRUMENT_FRAMEWORK=true). In coverage builds the Java code in +// boot image libraries is instrumented, which means that the instrumentation library (jacocoagent) +// needs to be added to the list of bootclasspath DEX jars. +// +// In general, there is a requirement that the source code for a boot image library must be +// available at build time (e.g. it cannot be a stub that has a separate implementation library). +// +// 2.2. Static configs +// ------------------- +// +// Because boot images are used to dexpreopt other Java modules, the paths to boot image files must +// be known by the time dexpreopt build rules for the dependent modules are generated. Boot image +// configs are constructed very early during the build, before build rule generation. The configs +// provide predefined paths to boot image files (these paths depend only on static build +// configuration, such as PRODUCT variables, and use hard-coded directory names). +// +// 2.3. Singleton +// -------------- +// +// Build rules for the boot images are generated with a Soong singleton. Because a singleton has no +// dependencies on other modules, it has to find the modules for the DEX jars using VisitAllModules. +// Soong loops through all modules and compares each module against a list of bootclasspath library +// names. Then it generates build rules that copy DEX jars from their intermediate module-specific +// locations to the hard-coded locations predefined in the boot image configs. +// +// It would be possible to use a module with proper dependencies instead, but that would require +// changes in the way Soong generates variables for Make: a singleton can use one MakeVars() method +// that writes variables to out/soong/make_vars-*.mk, which is included early by the main makefile, +// but module(s) would have to use out/soong/Android-*.mk which has a group of LOCAL_* variables +// for each module, and is included later. +// +// 2.4. Install rules +// ------------------ +// +// The primary boot image and the Framework extension are installed in different ways. The primary +// boot image is part of the ART APEX: it is copied into the APEX intermediate files, packaged +// together with other APEX contents, extracted and mounted on device. The Framework boot image +// extension is installed by the rules defined in makefiles (make/core/dex_preopt_libart.mk). Soong +// writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names, +// paths and so on. +// + +var artApexNames = []string{ + "com.android.art", + "com.android.art.debug", + "com.android.art.testing", + "com.google.android.art", + "com.google.android.art.debug", + "com.google.android.art.testing", +} + +func init() { + RegisterDexpreoptBootJarsComponents(android.InitRegistrationContext) +} + +// Target-independent description of a boot image. +type bootImageConfig struct { + // If this image is an extension, the image that it extends. + extends *bootImageConfig + + // Image name (used in directory names and ninja rule names). + name string + + // Basename of the image: the resulting filenames are <stem>[-<jar>].{art,oat,vdex}. + stem string + + // Output directory for the image files. + dir android.OutputPath + + // Output directory for the image files with debug symbols. + symbolsDir android.OutputPath + + // Subdirectory where the image files are installed. + installDirOnHost string + + // Subdirectory where the image files on device are installed. + installDirOnDevice string + + // Install path of the boot image profile if it needs to be installed in the APEX, or empty if not + // needed. + profileInstallPathInApex string + + // A list of (location, jar) pairs for the Java modules in this image. + modules android.ConfiguredJarList + + // File paths to jars. + dexPaths android.WritablePaths // for this image + dexPathsDeps android.WritablePaths // for the dependency images and in this image + + // Map from module name (without prebuilt_ prefix) to the predefined build path. + dexPathsByModule map[string]android.WritablePath + + // File path to a zip archive with all image files (or nil, if not needed). + zip android.WritablePath + + // Rules which should be used in make to install the outputs. + profileInstalls android.RuleBuilderInstalls + + // Path to the license metadata file for the module that built the profile. + profileLicenseMetadataFile android.OptionalPath + + // Path to the image profile file on host (or empty, if profile is not generated). + profilePathOnHost android.Path + + // Target-dependent fields. + variants []*bootImageVariant + + // Path of the preloaded classes file. + preloadedClassesFile string +} + +// Target-dependent description of a boot image. +type bootImageVariant struct { + *bootImageConfig + + // Target for which the image is generated. + target android.Target + + // The "locations" of jars. + dexLocations []string // for this image + dexLocationsDeps []string // for the dependency images and in this image + + // Paths to image files. + imagePathOnHost android.OutputPath // first image file path on host + imagePathOnDevice string // first image file path on device + + // All the files that constitute this image variant, i.e. .art, .oat and .vdex files. + imagesDeps android.OutputPaths + + // The path to the primary image variant's imagePathOnHost field, where primary image variant + // means the image variant that this extends. + // + // This is only set for a variant of an image that extends another image. + primaryImages android.OutputPath + + // The paths to the primary image variant's imagesDeps field, where primary image variant + // means the image variant that this extends. + // + // This is only set for a variant of an image that extends another image. + primaryImagesDeps android.Paths + + // Rules which should be used in make to install the outputs on host. + installs android.RuleBuilderInstalls + vdexInstalls android.RuleBuilderInstalls + unstrippedInstalls android.RuleBuilderInstalls + + // Rules which should be used in make to install the outputs on device. + deviceInstalls android.RuleBuilderInstalls + + // Path to the license metadata file for the module that built the image. + licenseMetadataFile android.OptionalPath +} + +// Get target-specific boot image variant for the given boot image config and target. +func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant { + for _, variant := range image.variants { + if variant.target.Os == target.Os && variant.target.Arch.ArchType == target.Arch.ArchType { + return variant + } + } + return nil +} + +// Return any (the first) variant which is for the device (as opposed to for the host). +func (image bootImageConfig) getAnyAndroidVariant() *bootImageVariant { + for _, variant := range image.variants { + if variant.target.Os == android.Android { + return variant + } + } + return nil +} + +// Return the name of a boot image module given a boot image config and a component (module) index. +// A module name is a combination of the Java library name, and the boot image stem (that is stored +// in the config). +func (image bootImageConfig) moduleName(ctx android.PathContext, idx int) string { + // The first module of the primary boot image is special: its module name has only the stem, but + // not the library name. All other module names are of the form <stem>-<library name> + m := image.modules.Jar(idx) + name := image.stem + if idx != 0 || image.extends != nil { + name += "-" + android.ModuleStem(m) + } + return name +} + +// Return the name of the first boot image module, or stem if the list of modules is empty. +func (image bootImageConfig) firstModuleNameOrStem(ctx android.PathContext) string { + if image.modules.Len() > 0 { + return image.moduleName(ctx, 0) + } else { + return image.stem + } +} + +// Return filenames for the given boot image component, given the output directory and a list of +// extensions. +func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.OutputPath, exts ...string) android.OutputPaths { + ret := make(android.OutputPaths, 0, image.modules.Len()*len(exts)) + for i := 0; i < image.modules.Len(); i++ { + name := image.moduleName(ctx, i) + for _, ext := range exts { + ret = append(ret, dir.Join(ctx, name+ext)) + } + } + return ret +} + +// apexVariants returns a list of all *bootImageVariant that could be included in an apex. +func (image *bootImageConfig) apexVariants() []*bootImageVariant { + variants := []*bootImageVariant{} + for _, variant := range image.variants { + // We also generate boot images for host (for testing), but we don't need those in the apex. + // TODO(b/177892522) - consider changing this to check Os.OsClass = android.Device + if variant.target.Os == android.Android { + variants = append(variants, variant) + } + } + return variants +} + +// Returns true if the boot image should be installed in the APEX. +func (image *bootImageConfig) shouldInstallInApex() bool { + return strings.HasPrefix(image.installDirOnDevice, "apex/") +} + +// Return boot image locations (as a list of symbolic paths). +// +// The image "location" is a symbolic path that, with multiarchitecture support, doesn't really +// exist on the device. Typically it is /apex/com.android.art/javalib/boot.art and should be the +// same for all supported architectures on the device. The concrete architecture specific files +// actually end up in architecture-specific sub-directory such as arm, arm64, x86, or x86_64. +// +// For example a physical file /apex/com.android.art/javalib/x86/boot.art has "image location" +// /apex/com.android.art/javalib/boot.art (which is not an actual file). +// +// For a primary boot image the list of locations has a single element. +// +// For a boot image extension the list of locations contains a location for all dependency images +// (including the primary image) and the location of the extension itself. For example, for the +// Framework boot image extension that depends on the primary ART boot image the list contains two +// elements. +// +// The location is passed as an argument to the ART tools like dex2oat instead of the real path. +// ART tools will then reconstruct the architecture-specific real path. +// +func (image *bootImageVariant) imageLocations() (imageLocationsOnHost []string, imageLocationsOnDevice []string) { + if image.extends != nil { + imageLocationsOnHost, imageLocationsOnDevice = image.extends.getVariant(image.target).imageLocations() + } + return append(imageLocationsOnHost, dexpreopt.PathToLocation(image.imagePathOnHost, image.target.Arch.ArchType)), + append(imageLocationsOnDevice, dexpreopt.PathStringToLocation(image.imagePathOnDevice, image.target.Arch.ArchType)) +} + +func dexpreoptBootJarsFactory() android.SingletonModule { + m := &dexpreoptBootJars{} + android.InitAndroidModule(m) + return m +} + +func RegisterDexpreoptBootJarsComponents(ctx android.RegistrationContext) { + ctx.RegisterSingletonModuleType("dex_bootjars", dexpreoptBootJarsFactory) +} + +func SkipDexpreoptBootJars(ctx android.PathContext) bool { + return dexpreopt.GetGlobalConfig(ctx).DisablePreoptBootImages +} + +// Singleton module for generating boot image build rules. +type dexpreoptBootJars struct { + android.SingletonModuleBase + + // Default boot image config (currently always the Framework boot image extension). It should be + // noted that JIT-Zygote builds use ART APEX image instead of the Framework boot image extension, + // but the switch is handled not here, but in the makefiles (triggered with + // DEXPREOPT_USE_ART_IMAGE=true). + defaultBootImage *bootImageConfig + + // Build path to a config file that Soong writes for Make (to be used in makefiles that install + // the default boot image). + dexpreoptConfigForMake android.WritablePath +} + +// Provide paths to boot images for use by modules that depend upon them. +// +// The build rules are created in GenerateSingletonBuildActions(). +func (d *dexpreoptBootJars) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Placeholder for now. +} + +// Generate build rules for boot images. +func (d *dexpreoptBootJars) GenerateSingletonBuildActions(ctx android.SingletonContext) { + if SkipDexpreoptBootJars(ctx) { + return + } + if dexpreopt.GetCachedGlobalSoongConfig(ctx) == nil { + // No module has enabled dexpreopting, so we assume there will be no boot image to make. + return + } + + d.dexpreoptConfigForMake = android.PathForOutput(ctx, ctx.Config().DeviceName(), "dexpreopt.config") + writeGlobalConfigForMake(ctx, d.dexpreoptConfigForMake) + + global := dexpreopt.GetGlobalConfig(ctx) + if !shouldBuildBootImages(ctx.Config(), global) { + return + } + + defaultImageConfig := defaultBootImageConfig(ctx) + d.defaultBootImage = defaultImageConfig +} + +// shouldBuildBootImages determines whether boot images should be built. +func shouldBuildBootImages(config android.Config, global *dexpreopt.GlobalConfig) bool { + // Skip recompiling the boot image for the second sanitization phase. We'll get separate paths + // and invalidate first-stage artifacts which are crucial to SANITIZE_LITE builds. + // Note: this is technically incorrect. Compiled code contains stack checks which may depend + // on ASAN settings. + if len(config.SanitizeDevice()) == 1 && config.SanitizeDevice()[0] == "address" && global.SanitizeLite { + return false + } + return true +} + +// copyBootJarsToPredefinedLocations generates commands that will copy boot jars to predefined +// paths in the global config. +func copyBootJarsToPredefinedLocations(ctx android.ModuleContext, srcBootDexJarsByModule bootDexJarByModule, dstBootJarsByModule map[string]android.WritablePath) { + // Create the super set of module names. + names := []string{} + names = append(names, android.SortedStringKeys(srcBootDexJarsByModule)...) + names = append(names, android.SortedStringKeys(dstBootJarsByModule)...) + names = android.SortedUniqueStrings(names) + for _, name := range names { + src := srcBootDexJarsByModule[name] + dst := dstBootJarsByModule[name] + + if src == nil { + // A dex boot jar should be provided by the source java module. It needs to be installable or + // have compile_dex=true - cf. assignments to java.Module.dexJarFile. + // + // However, the source java module may be either replaced or overridden (using prefer:true) by + // a prebuilt java module with the same name. In that case the dex boot jar needs to be + // provided by the corresponding prebuilt APEX module. That APEX is the one that refers + // through a exported_(boot|systemserver)classpath_fragments property to a + // prebuilt_(boot|systemserver)classpath_fragment module, which in turn lists the prebuilt + // java module in the contents property. If that chain is broken then this dependency will + // fail. + if !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf("module %s does not provide a dex boot jar (see comment next to this message in Soong for details)", name) + } else { + ctx.AddMissingDependencies([]string{name}) + } + } else if dst == nil { + ctx.ModuleErrorf("module %s is not part of the boot configuration", name) + } else { + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: src, + Output: dst, + }) + } + } +} + +// buildBootImageVariantsForAndroidOs generates rules to build the boot image variants for the +// android.Android OsType and returns a map from the architectures to the paths of the generated +// boot image files. +// +// The paths are returned because they are needed elsewhere in Soong, e.g. for populating an APEX. +func buildBootImageVariantsForAndroidOs(ctx android.ModuleContext, image *bootImageConfig, profile android.WritablePath) bootImageFilesByArch { + return buildBootImageForOsType(ctx, image, profile, android.Android) +} + +// buildBootImageVariantsForBuildOs generates rules to build the boot image variants for the +// config.BuildOS OsType, i.e. the type of OS on which the build is being running. +// +// The files need to be generated into their predefined location because they are used from there +// both within Soong and outside, e.g. for ART based host side testing and also for use by some +// cloud based tools. However, they are not needed by callers of this function and so the paths do +// not need to be returned from this func, unlike the buildBootImageVariantsForAndroidOs func. +func buildBootImageVariantsForBuildOs(ctx android.ModuleContext, image *bootImageConfig, profile android.WritablePath) { + buildBootImageForOsType(ctx, image, profile, ctx.Config().BuildOS) +} + +// buildBootImageForOsType takes a bootImageConfig, a profile file and an android.OsType +// boot image files are required for and it creates rules to build the boot image +// files for all the required architectures for them. +// +// It returns a map from android.ArchType to the predefined paths of the boot image files. +func buildBootImageForOsType(ctx android.ModuleContext, image *bootImageConfig, profile android.WritablePath, requiredOsType android.OsType) bootImageFilesByArch { + filesByArch := bootImageFilesByArch{} + for _, variant := range image.variants { + if variant.target.Os == requiredOsType { + buildBootImageVariant(ctx, variant, profile) + filesByArch[variant.target.Arch.ArchType] = variant.imagesDeps.Paths() + } + } + + return filesByArch +} + +// buildBootImageZipInPredefinedLocation generates a zip file containing all the boot image files. +// +// The supplied filesByArch is nil when the boot image files have not been generated. Otherwise, it +// is a map from android.ArchType to the predefined locations. +func buildBootImageZipInPredefinedLocation(ctx android.ModuleContext, image *bootImageConfig, filesByArch bootImageFilesByArch) { + if filesByArch == nil { + return + } + + // Compute the list of files from all the architectures. + zipFiles := android.Paths{} + for _, archType := range android.ArchTypeList() { + zipFiles = append(zipFiles, filesByArch[archType]...) + } + + rule := android.NewRuleBuilder(pctx, ctx) + rule.Command(). + BuiltTool("soong_zip"). + FlagWithOutput("-o ", image.zip). + FlagWithArg("-C ", image.dir.Join(ctx, android.Android.String()).String()). + FlagWithInputList("-f ", zipFiles, " -f ") + + rule.Build("zip_"+image.name, "zip "+image.name+" image") +} + +// Generate boot image build rules for a specific target. +func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, profile android.Path) { + + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + arch := image.target.Arch.ArchType + os := image.target.Os.String() // We need to distinguish host-x86 and device-x86. + symbolsDir := image.symbolsDir.Join(ctx, os, image.installDirOnHost, arch.String()) + symbolsFile := symbolsDir.Join(ctx, image.stem+".oat") + outputDir := image.dir.Join(ctx, os, image.installDirOnHost, arch.String()) + outputPath := outputDir.Join(ctx, image.stem+".oat") + oatLocation := dexpreopt.PathToLocation(outputPath, arch) + imagePath := outputPath.ReplaceExtension(ctx, "art") + + rule := android.NewRuleBuilder(pctx, ctx) + + rule.Command().Text("mkdir").Flag("-p").Flag(symbolsDir.String()) + rule.Command().Text("rm").Flag("-f"). + Flag(symbolsDir.Join(ctx, "*.art").String()). + Flag(symbolsDir.Join(ctx, "*.oat").String()). + Flag(symbolsDir.Join(ctx, "*.invocation").String()) + rule.Command().Text("rm").Flag("-f"). + Flag(outputDir.Join(ctx, "*.art").String()). + Flag(outputDir.Join(ctx, "*.oat").String()). + Flag(outputDir.Join(ctx, "*.invocation").String()) + + cmd := rule.Command() + + extraFlags := ctx.Config().Getenv("ART_BOOT_IMAGE_EXTRA_ARGS") + if extraFlags == "" { + // Use ANDROID_LOG_TAGS to suppress most logging by default... + cmd.Text(`ANDROID_LOG_TAGS="*:e"`) + } else { + // ...unless the boot image is generated specifically for testing, then allow all logging. + cmd.Text(`ANDROID_LOG_TAGS="*:v"`) + } + + invocationPath := outputPath.ReplaceExtension(ctx, "invocation") + + cmd.Tool(globalSoong.Dex2oat). + Flag("--avoid-storing-invocation"). + FlagWithOutput("--write-invocation-to=", invocationPath).ImplicitOutput(invocationPath). + Flag("--runtime-arg").FlagWithArg("-Xms", global.Dex2oatImageXms). + Flag("--runtime-arg").FlagWithArg("-Xmx", global.Dex2oatImageXmx) + + if profile != nil { + cmd.FlagWithInput("--profile-file=", profile) + } + + dirtyImageFile := "frameworks/base/config/dirty-image-objects" + dirtyImagePath := android.ExistentPathForSource(ctx, dirtyImageFile) + if dirtyImagePath.Valid() { + cmd.FlagWithInput("--dirty-image-objects=", dirtyImagePath.Path()) + } + + if image.extends != nil { + // It is a boot image extension, so it needs the boot image it depends on (in this case the + // primary ART APEX image). + artImage := image.primaryImages + cmd. + Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). + Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", image.dexLocationsDeps, ":"). + // Add the path to the first file in the boot image with the arch specific directory removed, + // dex2oat will reconstruct the path to the actual file when it needs it. As the actual path + // to the file cannot be passed to the command make sure to add the actual path as an Implicit + // dependency to ensure that it is built before the command runs. + FlagWithArg("--boot-image=", dexpreopt.PathToLocation(artImage, arch)).Implicit(artImage). + // Similarly, the dex2oat tool will automatically find the paths to other files in the base + // boot image so make sure to add them as implicit dependencies to ensure that they are built + // before this command is run. + Implicits(image.primaryImagesDeps) + } else { + // It is a primary image, so it needs a base address. + cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress()) + } + + // We always expect a preloaded classes file to be available. However, if we cannot find it, it's + // OK to not pass the flag to dex2oat. + preloadedClassesPath := android.ExistentPathForSource(ctx, image.preloadedClassesFile) + if preloadedClassesPath.Valid() { + cmd.FlagWithInput("--preloaded-classes=", preloadedClassesPath.Path()) + } + + cmd. + FlagForEachInput("--dex-file=", image.dexPaths.Paths()). + FlagForEachArg("--dex-location=", image.dexLocations). + Flag("--generate-debug-info"). + Flag("--generate-build-id"). + Flag("--image-format=lz4hc"). + FlagWithArg("--oat-symbols=", symbolsFile.String()). + Flag("--strip"). + FlagWithArg("--oat-file=", outputPath.String()). + FlagWithArg("--oat-location=", oatLocation). + FlagWithArg("--image=", imagePath.String()). + FlagWithArg("--instruction-set=", arch.String()). + FlagWithArg("--android-root=", global.EmptyDirectory). + FlagWithArg("--no-inline-from=", "core-oj.jar"). + Flag("--force-determinism"). + Flag("--abort-on-hard-verifier-error") + + // Use the default variant/features for host builds. + // The map below contains only device CPU info (which might be x86 on some devices). + if image.target.Os == android.Android { + cmd.FlagWithArg("--instruction-set-variant=", global.CpuVariant[arch]) + cmd.FlagWithArg("--instruction-set-features=", global.InstructionSetFeatures[arch]) + } + + if global.BootFlags != "" { + cmd.Flag(global.BootFlags) + } + + if extraFlags != "" { + cmd.Flag(extraFlags) + } + + cmd.Textf(`|| ( echo %s ; false )`, proptools.ShellEscape(failureMessage)) + + installDir := filepath.Join("/", image.installDirOnHost, arch.String()) + + var vdexInstalls android.RuleBuilderInstalls + var unstrippedInstalls android.RuleBuilderInstalls + var deviceInstalls android.RuleBuilderInstalls + + for _, artOrOat := range image.moduleFiles(ctx, outputDir, ".art", ".oat") { + cmd.ImplicitOutput(artOrOat) + + // Install the .oat and .art files + rule.Install(artOrOat, filepath.Join(installDir, artOrOat.Base())) + } + + for _, vdex := range image.moduleFiles(ctx, outputDir, ".vdex") { + cmd.ImplicitOutput(vdex) + + // Note that the vdex files are identical between architectures. + // Make rules will create symlinks to share them between architectures. + vdexInstalls = append(vdexInstalls, + android.RuleBuilderInstall{vdex, filepath.Join(installDir, vdex.Base())}) + } + + for _, unstrippedOat := range image.moduleFiles(ctx, symbolsDir, ".oat") { + cmd.ImplicitOutput(unstrippedOat) + + // Install the unstripped oat files. The Make rules will put these in $(TARGET_OUT_UNSTRIPPED) + unstrippedInstalls = append(unstrippedInstalls, + android.RuleBuilderInstall{unstrippedOat, filepath.Join(installDir, unstrippedOat.Base())}) + } + + if image.installDirOnHost != image.installDirOnDevice && !image.shouldInstallInApex() && !ctx.Config().UnbundledBuild() { + installDirOnDevice := filepath.Join("/", image.installDirOnDevice, arch.String()) + for _, file := range image.moduleFiles(ctx, outputDir, ".art", ".oat", ".vdex") { + deviceInstalls = append(deviceInstalls, + android.RuleBuilderInstall{file, filepath.Join(installDirOnDevice, file.Base())}) + } + } + + rule.Build(image.name+"JarsDexpreopt_"+image.target.String(), "dexpreopt "+image.name+" jars "+arch.String()) + + // save output and installed files for makevars + image.installs = rule.Installs() + image.vdexInstalls = vdexInstalls + image.unstrippedInstalls = unstrippedInstalls + image.deviceInstalls = deviceInstalls + image.licenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) +} + +const failureMessage = `ERROR: Dex2oat failed to compile a boot image. +It is likely that the boot classpath is inconsistent. +Rebuild with ART_BOOT_IMAGE_EXTRA_ARGS="--runtime-arg -verbose:verifier" to see verification errors.` + +func bootImageProfileRule(ctx android.ModuleContext, image *bootImageConfig) android.WritablePath { + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisableGenerateProfile { + return nil + } + + defaultProfile := "frameworks/base/config/boot-image-profile.txt" + + rule := android.NewRuleBuilder(pctx, ctx) + + var bootImageProfile android.Path + if len(global.BootImageProfiles) > 1 { + combinedBootImageProfile := image.dir.Join(ctx, "boot-image-profile.txt") + rule.Command().Text("cat").Inputs(global.BootImageProfiles).Text(">").Output(combinedBootImageProfile) + bootImageProfile = combinedBootImageProfile + } else if len(global.BootImageProfiles) == 1 { + bootImageProfile = global.BootImageProfiles[0] + } else if path := android.ExistentPathForSource(ctx, defaultProfile); path.Valid() { + bootImageProfile = path.Path() + } else { + // No profile (not even a default one, which is the case on some branches + // like master-art-host that don't have frameworks/base). + // Return nil and continue without profile. + return nil + } + + profile := image.dir.Join(ctx, "boot.prof") + + rule.Command(). + Text(`ANDROID_LOG_TAGS="*:e"`). + Tool(globalSoong.Profman). + Flag("--output-profile-type=boot"). + FlagWithInput("--create-profile-from=", bootImageProfile). + FlagForEachInput("--apk=", image.dexPathsDeps.Paths()). + FlagForEachArg("--dex-location=", image.getAnyAndroidVariant().dexLocationsDeps). + FlagWithOutput("--reference-profile-file=", profile) + + if image == defaultBootImageConfig(ctx) { + rule.Install(profile, "/system/etc/boot-image.prof") + image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + image.profileLicenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) + } + + rule.Build("bootJarsProfile", "profile boot jars") + + image.profilePathOnHost = profile + + return profile +} + +// bootFrameworkProfileRule generates the rule to create the boot framework profile and +// returns a path to the generated file. +func bootFrameworkProfileRule(ctx android.ModuleContext, image *bootImageConfig) android.WritablePath { + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisableGenerateProfile || ctx.Config().UnbundledBuild() { + return nil + } + + defaultProfile := "frameworks/base/config/boot-profile.txt" + bootFrameworkProfile := android.PathForSource(ctx, defaultProfile) + + profile := image.dir.Join(ctx, "boot.bprof") + + rule := android.NewRuleBuilder(pctx, ctx) + rule.Command(). + Text(`ANDROID_LOG_TAGS="*:e"`). + Tool(globalSoong.Profman). + Flag("--output-profile-type=bprof"). + FlagWithInput("--create-profile-from=", bootFrameworkProfile). + FlagForEachInput("--apk=", image.dexPathsDeps.Paths()). + FlagForEachArg("--dex-location=", image.getAnyAndroidVariant().dexLocationsDeps). + FlagWithOutput("--reference-profile-file=", profile) + + rule.Install(profile, "/system/etc/boot-image.bprof") + rule.Build("bootFrameworkProfile", "profile boot framework jars") + image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + image.profileLicenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) + + return profile +} + +func dumpOatRules(ctx android.ModuleContext, image *bootImageConfig) { + var allPhonies android.Paths + for _, image := range image.variants { + arch := image.target.Arch.ArchType + suffix := arch.String() + // Host and target might both use x86 arch. We need to ensure the names are unique. + if image.target.Os.Class == android.Host { + suffix = "host-" + suffix + } + // Create a rule to call oatdump. + output := android.PathForOutput(ctx, "boot."+suffix+".oatdump.txt") + rule := android.NewRuleBuilder(pctx, ctx) + imageLocationsOnHost, _ := image.imageLocations() + rule.Command(). + BuiltTool("oatdump"). + FlagWithInputList("--runtime-arg -Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). + FlagWithList("--runtime-arg -Xbootclasspath-locations:", image.dexLocationsDeps, ":"). + FlagWithArg("--image=", strings.Join(imageLocationsOnHost, ":")).Implicits(image.imagesDeps.Paths()). + FlagWithOutput("--output=", output). + FlagWithArg("--instruction-set=", arch.String()) + rule.Build("dump-oat-boot-"+suffix, "dump oat boot "+arch.String()) + + // Create a phony rule that depends on the output file and prints the path. + phony := android.PathForPhony(ctx, "dump-oat-boot-"+suffix) + rule = android.NewRuleBuilder(pctx, ctx) + rule.Command(). + Implicit(output). + ImplicitOutput(phony). + Text("echo").FlagWithArg("Output in ", output.String()) + rule.Build("phony-dump-oat-boot-"+suffix, "dump oat boot "+arch.String()) + + allPhonies = append(allPhonies, phony) + } + + phony := android.PathForPhony(ctx, "dump-oat-boot") + ctx.Build(pctx, android.BuildParams{ + Rule: android.Phony, + Output: phony, + Inputs: allPhonies, + Description: "dump-oat-boot", + }) +} + +func writeGlobalConfigForMake(ctx android.SingletonContext, path android.WritablePath) { + data := dexpreopt.GetGlobalConfigRawData(ctx) + + android.WriteFileRule(ctx, path, string(data)) +} + +// Define Make variables for boot image names, paths, etc. These variables are used in makefiles +// (make/core/dex_preopt_libart.mk) to generate install rules that copy boot image files to the +// correct output directories. +func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { + if d.dexpreoptConfigForMake != nil { + ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String()) + ctx.Strict("DEX_PREOPT_SOONG_CONFIG_FOR_MAKE", android.PathForOutput(ctx, "dexpreopt_soong.config").String()) + } + + image := d.defaultBootImage + if image == nil { + return + } + + ctx.Strict("DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED", image.profileInstalls.String()) + if image.profileLicenseMetadataFile.Valid() { + ctx.Strict("DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA", image.profileLicenseMetadataFile.String()) + } + + global := dexpreopt.GetGlobalConfig(ctx) + dexPaths, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) + ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(dexPaths.Strings(), " ")) + ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(dexLocations, " ")) + + for _, variant := range image.variants { + suffix := "" + if variant.target.Os.Class == android.Host { + suffix = "_host" + } + sfx := suffix + "_" + variant.target.Arch.ArchType.String() + ctx.Strict("DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_"+sfx, variant.vdexInstalls.String()) + ctx.Strict("DEXPREOPT_IMAGE_"+sfx, variant.imagePathOnHost.String()) + ctx.Strict("DEXPREOPT_IMAGE_DEPS_"+sfx, strings.Join(variant.imagesDeps.Strings(), " ")) + ctx.Strict("DEXPREOPT_IMAGE_BUILT_INSTALLED_"+sfx, variant.installs.String()) + ctx.Strict("DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_"+sfx, variant.unstrippedInstalls.String()) + if variant.licenseMetadataFile.Valid() { + ctx.Strict("DEXPREOPT_IMAGE_LICENSE_METADATA_"+sfx, variant.licenseMetadataFile.String()) + } + } + imageLocationsOnHost, imageLocationsOnDevice := image.getAnyAndroidVariant().imageLocations() + ctx.Strict("DEXPREOPT_IMAGE_LOCATIONS_ON_HOST", strings.Join(imageLocationsOnHost, ":")) + ctx.Strict("DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICE", strings.Join(imageLocationsOnDevice, ":")) + ctx.Strict("DEXPREOPT_IMAGE_ZIP", image.zip.String()) + + // There used to be multiple images for JIT-Zygote mode, not there's only one. + ctx.Strict("DEXPREOPT_IMAGE_NAMES", image.name) +} diff --git a/java/dexpreopt_config.go_v1 b/java/dexpreopt_config.go_v1 new file mode 100644 index 000000000..d71e2bbfd --- /dev/null +++ b/java/dexpreopt_config.go_v1 @@ -0,0 +1,215 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/dexpreopt" +) + +// dexpreoptTargets returns the list of targets that are relevant to dexpreopting, which excludes architectures +// supported through native bridge. +func dexpreoptTargets(ctx android.PathContext) []android.Target { + var targets []android.Target + for _, target := range ctx.Config().Targets[android.Android] { + if target.NativeBridge == android.NativeBridgeDisabled { + targets = append(targets, target) + } + } + // We may also need the images on host in order to run host-based tests. + for _, target := range ctx.Config().Targets[ctx.Config().BuildOS] { + targets = append(targets, target) + } + + return targets +} + +var ( + bootImageConfigKey = android.NewOnceKey("bootImageConfig") + bootImageConfigRawKey = android.NewOnceKey("bootImageConfigRaw") + artBootImageName = "art" + frameworkBootImageName = "boot" +) + +func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig { + return ctx.Config().Once(bootImageConfigRawKey, func() interface{} { + global := dexpreopt.GetGlobalConfig(ctx) + + artModules := global.ArtApexJars + frameworkModules := global.BootJars.RemoveList(artModules) + + // ART config for the primary boot image in the ART apex. + // It includes the Core Libraries. + artCfg := bootImageConfig{ + name: artBootImageName, + stem: "boot", + installDirOnHost: "apex/art_boot_images/javalib", + installDirOnDevice: "system/framework", + profileInstallPathInApex: "etc/boot-image.prof", + modules: artModules, + preloadedClassesFile: "art/build/boot/preloaded-classes", + } + + // Framework config for the boot image extension. + // It includes framework libraries and depends on the ART config. + frameworkSubdir := "system/framework" + frameworkCfg := bootImageConfig{ + extends: &artCfg, + name: frameworkBootImageName, + stem: "boot", + installDirOnHost: frameworkSubdir, + installDirOnDevice: frameworkSubdir, + modules: frameworkModules, + preloadedClassesFile: "frameworks/base/config/preloaded-classes", + } + + return map[string]*bootImageConfig{ + artBootImageName: &artCfg, + frameworkBootImageName: &frameworkCfg, + } + }).(map[string]*bootImageConfig) +} + +// Construct the global boot image configs. +func genBootImageConfigs(ctx android.PathContext) map[string]*bootImageConfig { + return ctx.Config().Once(bootImageConfigKey, func() interface{} { + targets := dexpreoptTargets(ctx) + deviceDir := android.PathForOutput(ctx, ctx.Config().DeviceName()) + + configs := genBootImageConfigRaw(ctx) + artCfg := configs[artBootImageName] + frameworkCfg := configs[frameworkBootImageName] + + // common to all configs + for _, c := range configs { + c.dir = deviceDir.Join(ctx, "dex_"+c.name+"jars") + c.symbolsDir = deviceDir.Join(ctx, "dex_"+c.name+"jars_unstripped") + + // expands to <stem>.art for primary image and <stem>-<1st module>.art for extension + imageName := c.firstModuleNameOrStem(ctx) + ".art" + + // The path to bootclasspath dex files needs to be known at module + // GenerateAndroidBuildAction time, before the bootclasspath modules have been compiled. + // Set up known paths for them, the singleton rules will copy them there. + // TODO(b/143682396): use module dependencies instead + inputDir := deviceDir.Join(ctx, "dex_"+c.name+"jars_input") + c.dexPaths = c.modules.BuildPaths(ctx, inputDir) + c.dexPathsByModule = c.modules.BuildPathsByModule(ctx, inputDir) + c.dexPathsDeps = c.dexPaths + + // Create target-specific variants. + for _, target := range targets { + arch := target.Arch.ArchType + imageDir := c.dir.Join(ctx, target.Os.String(), c.installDirOnHost, arch.String()) + variant := &bootImageVariant{ + bootImageConfig: c, + target: target, + imagePathOnHost: imageDir.Join(ctx, imageName), + imagePathOnDevice: filepath.Join("/", c.installDirOnDevice, arch.String(), imageName), + imagesDeps: c.moduleFiles(ctx, imageDir, ".art", ".oat", ".vdex"), + dexLocations: c.modules.DevicePaths(ctx.Config(), target.Os), + } + variant.dexLocationsDeps = variant.dexLocations + c.variants = append(c.variants, variant) + } + + c.zip = c.dir.Join(ctx, c.name+".zip") + } + + // specific to the framework config + frameworkCfg.dexPathsDeps = append(artCfg.dexPathsDeps, frameworkCfg.dexPathsDeps...) + for i := range targets { + frameworkCfg.variants[i].primaryImages = artCfg.variants[i].imagePathOnHost + frameworkCfg.variants[i].primaryImagesDeps = artCfg.variants[i].imagesDeps.Paths() + frameworkCfg.variants[i].dexLocationsDeps = append(artCfg.variants[i].dexLocations, frameworkCfg.variants[i].dexLocationsDeps...) + } + + return configs + }).(map[string]*bootImageConfig) +} + +func defaultBootImageConfig(ctx android.PathContext) *bootImageConfig { + return genBootImageConfigs(ctx)[frameworkBootImageName] +} + +// Apex boot config allows to access build/install paths of apex boot jars without going +// through the usual trouble of registering dependencies on those modules and extracting build paths +// from those dependencies. +type apexBootConfig struct { + // A list of apex boot jars. + modules android.ConfiguredJarList + + // A list of predefined build paths to apex boot jars. They are configured very early, + // before the modules for these jars are processed and the actual paths are generated, and + // later on a singleton adds commands to copy actual jars to the predefined paths. + dexPaths android.WritablePaths + + // Map from module name (without prebuilt_ prefix) to the predefined build path. + dexPathsByModule map[string]android.WritablePath + + // A list of dex locations (a.k.a. on-device paths) to the boot jars. + dexLocations []string +} + +var updatableBootConfigKey = android.NewOnceKey("apexBootConfig") + +// Returns apex boot config. +func GetApexBootConfig(ctx android.PathContext) apexBootConfig { + return ctx.Config().Once(updatableBootConfigKey, func() interface{} { + apexBootJars := dexpreopt.GetGlobalConfig(ctx).ApexBootJars + + dir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "apex_bootjars") + dexPaths := apexBootJars.BuildPaths(ctx, dir) + dexPathsByModuleName := apexBootJars.BuildPathsByModule(ctx, dir) + + dexLocations := apexBootJars.DevicePaths(ctx.Config(), android.Android) + + return apexBootConfig{apexBootJars, dexPaths, dexPathsByModuleName, dexLocations} + }).(apexBootConfig) +} + +// Returns a list of paths and a list of locations for the boot jars used in dexpreopt (to be +// passed in -Xbootclasspath and -Xbootclasspath-locations arguments for dex2oat). +func bcpForDexpreopt(ctx android.PathContext, withUpdatable bool) (android.WritablePaths, []string) { + // Non-updatable boot jars (they are used both in the boot image and in dexpreopt). + bootImage := defaultBootImageConfig(ctx) + dexPaths := bootImage.dexPathsDeps + // The dex locations for all Android variants are identical. + dexLocations := bootImage.getAnyAndroidVariant().dexLocationsDeps + + if withUpdatable { + // Apex boot jars (they are used only in dexpreopt, but not in the boot image). + apexBootConfig := GetApexBootConfig(ctx) + dexPaths = append(dexPaths, apexBootConfig.dexPaths...) + dexLocations = append(dexLocations, apexBootConfig.dexLocations...) + } + + return dexPaths, dexLocations +} + +var defaultBootclasspathKey = android.NewOnceKey("defaultBootclasspath") + +var copyOf = android.CopyOf + +func init() { + android.RegisterMakeVarsProvider(pctx, dexpreoptConfigMakevars) +} + +func dexpreoptConfigMakevars(ctx android.MakeVarsContext) { + ctx.Strict("DEXPREOPT_BOOT_JARS_MODULES", strings.Join(defaultBootImageConfig(ctx).modules.CopyOfApexJarPairs(), ":")) +} diff --git a/java/fuzz.go b/java/fuzz.go index 257f34356..584c80b0c 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -15,14 +15,25 @@ package java import ( - "github.com/google/blueprint/proptools" "sort" "strings" + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + "android/soong/android" + "android/soong/cc" "android/soong/fuzz" ) +type jniProperties struct { + // list of jni libs + Jni_libs []string + + // sanitization + Sanitizers []string +} + func init() { RegisterJavaFuzzBuildComponents(android.InitRegistrationContext) } @@ -35,11 +46,60 @@ func RegisterJavaFuzzBuildComponents(ctx android.RegistrationContext) { type JavaFuzzLibrary struct { Library fuzzPackagedModule fuzz.FuzzPackagedModule + jniProperties jniProperties + jniFilePaths android.Paths } -func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { - j.Library.GenerateAndroidBuildActions(ctx) +// IsSanitizerEnabled implemented to make JavaFuzzLibrary implement +// cc.Sanitizeable +func (j *JavaFuzzLibrary) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool { + for _, s := range j.jniProperties.Sanitizers { + if sanitizerName == s { + return true + } + } + return false +} + +// IsSanitizerEnabledForJni implemented to make JavaFuzzLibrary implement +// cc.JniSanitizeable. It returns a bool for whether a cc dependency should be +// sanitized for the given sanitizer or not. +func (j *JavaFuzzLibrary) IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool { + return j.IsSanitizerEnabled(ctx, sanitizerName) +} + +// EnableSanitizer implemented to make JavaFuzzLibrary implement +// cc.Sanitizeable +func (j *JavaFuzzLibrary) EnableSanitizer(sanitizerName string) { +} + +// AddSanitizerDependencies implemented to make JavaFuzzLibrary implement +// cc.Sanitizeable +func (j *JavaFuzzLibrary) AddSanitizerDependencies(mctx android.BottomUpMutatorContext, sanitizerName string) { +} + +// To verify that JavaFuzzLibrary implements cc.Sanitizeable +var _ cc.Sanitizeable = (*JavaFuzzLibrary)(nil) + +func (j *JavaFuzzLibrary) DepsMutator(mctx android.BottomUpMutatorContext) { + if len(j.jniProperties.Jni_libs) > 0 { + if j.fuzzPackagedModule.FuzzProperties.Fuzz_config == nil { + config := &fuzz.FuzzConfig{} + j.fuzzPackagedModule.FuzzProperties.Fuzz_config = config + } + // this will be used by the ingestion pipeline to determine the version + // of jazzer to add to the fuzzer package + j.fuzzPackagedModule.FuzzProperties.Fuzz_config.IsJni = proptools.BoolPtr(true) + + for _, target := range mctx.MultiTargets() { + sharedLibVariations := append(target.Variations(), blueprint.Variation{Mutator: "link", Variation: "shared"}) + mctx.AddFarVariationDependencies(sharedLibVariations, cc.JniFuzzLibTag, j.jniProperties.Jni_libs...) + } + } + j.Library.DepsMutator(mctx) +} +func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { if j.fuzzPackagedModule.FuzzProperties.Corpus != nil { j.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, j.fuzzPackagedModule.FuzzProperties.Corpus) } @@ -55,6 +115,23 @@ func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) android.WriteFileRule(ctx, configPath, j.fuzzPackagedModule.FuzzProperties.Fuzz_config.String()) j.fuzzPackagedModule.Config = configPath } + + ctx.VisitDirectDepsWithTag(cc.JniFuzzLibTag, func(dep android.Module) { + sharedLibInfo := ctx.OtherModuleProvider(dep, cc.SharedLibraryInfoProvider).(cc.SharedLibraryInfo) + if sharedLibInfo.SharedLibrary != nil { + libPath := android.PathForModuleOut(ctx, sharedLibInfo.SharedLibrary.Base()) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: sharedLibInfo.SharedLibrary, + Output: libPath, + }) + j.jniFilePaths = append(j.jniFilePaths, libPath) + } else { + ctx.PropertyErrorf("jni_libs", "%q of type %q is not supported", dep.Name(), ctx.OtherModuleType(dep)) + } + }) + + j.Library.GenerateAndroidBuildActions(ctx) } // java_fuzz builds and links sources into a `.jar` file for the host. @@ -65,7 +142,8 @@ func FuzzFactory() android.Module { module := &JavaFuzzLibrary{} module.addHostProperties() - module.Module.properties.Installable = proptools.BoolPtr(false) + module.AddProperties(&module.jniProperties) + module.Module.properties.Installable = proptools.BoolPtr(true) module.AddProperties(&module.fuzzPackagedModule.FuzzProperties) // java_fuzz packaging rules collide when both linux_glibc and linux_bionic are enabled, disable the linux_bionic variants. @@ -83,7 +161,7 @@ func FuzzFactory() android.Module { module.initModuleAndImport(module) android.InitSdkAwareModule(module) - InitJavaModule(module, android.HostSupported) + InitJavaModuleMultiTargets(module, android.HostSupported) return module } @@ -106,26 +184,26 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { ctx.VisitAllModules(func(module android.Module) { // Discard non-fuzz targets. - javaModule, ok := module.(*JavaFuzzLibrary) + javaFuzzModule, ok := module.(*JavaFuzzLibrary) if !ok { return } fuzzModuleValidator := fuzz.FuzzModule{ - javaModule.ModuleBase, - javaModule.DefaultableModuleBase, - javaModule.ApexModuleBase, + javaFuzzModule.ModuleBase, + javaFuzzModule.DefaultableModuleBase, + javaFuzzModule.ApexModuleBase, } - if ok := fuzz.IsValid(fuzzModuleValidator); !ok || *javaModule.Module.properties.Installable { + if ok := fuzz.IsValid(fuzzModuleValidator); !ok { return } hostOrTargetString := "target" - if javaModule.Host() { + if javaFuzzModule.Host() { hostOrTargetString = "host" } - archString := javaModule.Arch().ArchType.String() + archString := javaFuzzModule.Arch().ArchType.String() archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) archOs := fuzz.ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()} @@ -134,12 +212,17 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { builder := android.NewRuleBuilder(pctx, ctx) // Package the artifacts (data, corpus, config and dictionary into a zipfile. - files = s.PackageArtifacts(ctx, module, javaModule.fuzzPackagedModule, archDir, builder) + files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder) // Add .jar - files = append(files, fuzz.FileToZip{javaModule.outputFile, ""}) + files = append(files, fuzz.FileToZip{javaFuzzModule.outputFile, ""}) + + // Add jni .so files + for _, fPath := range javaFuzzModule.jniFilePaths { + files = append(files, fuzz.FileToZip{fPath, ""}) + } - archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaModule.fuzzPackagedModule, files, builder, archDir, archString, "host", archOs, archDirs) + archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaFuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs) if !ok { return } diff --git a/java/fuzz_test.go b/java/fuzz_test.go index cf063ebe2..0a2c94517 100644 --- a/java/fuzz_test.go +++ b/java/fuzz_test.go @@ -15,13 +15,17 @@ package java import ( - "android/soong/android" "path/filepath" + "runtime" "testing" + + "android/soong/android" + "android/soong/cc" ) var prepForJavaFuzzTest = android.GroupFixturePreparers( PrepareForTestWithJavaDefaultModules, + cc.PrepareForTestWithCcBuildComponents, android.FixtureRegisterWithContext(RegisterJavaFuzzBuildComponents), ) @@ -32,6 +36,13 @@ func TestJavaFuzz(t *testing.T) { srcs: ["a.java"], libs: ["bar"], static_libs: ["baz"], + jni_libs: [ + "libjni", + ], + sanitizers: [ + "address", + "fuzzer", + ], } java_library_host { @@ -42,11 +53,21 @@ func TestJavaFuzz(t *testing.T) { java_library_host { name: "baz", srcs: ["c.java"], - }`) + } + + cc_library_shared { + name: "libjni", + host_supported: true, + device_supported: false, + stl: "none", + } + `) osCommonTarget := result.Config.BuildOSCommonTarget.String() - javac := result.ModuleForTests("foo", osCommonTarget).Rule("javac") - combineJar := result.ModuleForTests("foo", osCommonTarget).Description("for javac") + + osCommonTargetWithSan := osCommonTarget + "_asan" + "_fuzzer" + javac := result.ModuleForTests("foo", osCommonTargetWithSan).Rule("javac") + combineJar := result.ModuleForTests("foo", osCommonTargetWithSan).Description("for javac") if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" { t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs) @@ -62,4 +83,18 @@ func TestJavaFuzz(t *testing.T) { if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != baz { t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, baz) } + + ctx := result.TestContext + foo := ctx.ModuleForTests("foo", osCommonTargetWithSan).Module().(*JavaFuzzLibrary) + + expected := "libjni.so" + if runtime.GOOS == "darwin" { + expected = "libjni.dylib" + } + + fooJniFilePaths := foo.jniFilePaths + if len(fooJniFilePaths) != 1 || fooJniFilePaths[0].Rel() != expected { + t.Errorf(`expected foo test data relative path [%q], got %q`, + expected, fooJniFilePaths.Strings()) + } } diff --git a/java/hiddenapi.go b/java/hiddenapi.go index 3af5f1c7b..cf9c7ad7a 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -65,6 +65,8 @@ func (h *hiddenAPI) uncompressDex() *bool { type hiddenAPIModule interface { android.Module hiddenAPIIntf + + MinSdkVersion(ctx android.EarlyModuleContext) android.SdkSpec } type hiddenAPIIntf interface { @@ -148,7 +150,7 @@ func (h *hiddenAPI) hiddenAPIEncodeDex(ctx android.ModuleContext, dexJar android // Create a copy of the dex jar which has been encoded with hiddenapi flags. flagsCSV := hiddenAPISingletonPaths(ctx).flags outputDir := android.PathForModuleOut(ctx, "hiddenapi").OutputPath - encodedDex := hiddenAPIEncodeDex(ctx, dexJar, flagsCSV, uncompressDex, outputDir) + encodedDex := hiddenAPIEncodeDex(ctx, dexJar, flagsCSV, uncompressDex, android.SdkSpecNone, outputDir) // Use the encoded dex jar from here onwards. return encodedDex @@ -246,7 +248,7 @@ var hiddenAPIEncodeDexRule = pctx.AndroidStaticRule("hiddenAPIEncodeDex", bluepr // The encode dex rule requires unzipping, encoding and rezipping the classes.dex files along with // all the resources from the input jar. It also ensures that if it was uncompressed in the input // it stays uncompressed in the output. -func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Path, uncompressDex bool, outputDir android.OutputPath) android.OutputPath { +func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Path, uncompressDex bool, minSdkVersion android.SdkSpec, outputDir android.OutputPath) android.OutputPath { // The output file has the same name as the input file and is in the output directory. output := outputDir.Join(ctx, dexInput.Base()) @@ -274,6 +276,15 @@ func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Pa hiddenapiFlags = "--no-force-assign-all" } + // If the library is targeted for Q and/or R then make sure that they do not + // have any S+ flags encoded as that will break the runtime. + minApiLevel := minSdkVersion.ApiLevel + if !minApiLevel.IsNone() { + if minApiLevel.LessThanOrEqualTo(android.ApiLevelOrPanic(ctx, "R")) { + hiddenapiFlags = hiddenapiFlags + " --max-hiddenapi-level=max-target-r" + } + } + ctx.Build(pctx, android.BuildParams{ Rule: hiddenAPIEncodeDexRule, Description: "hiddenapi encode dex", diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index 534a8145f..c90b2ff97 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -1104,7 +1104,7 @@ func hiddenAPIRulesForBootclasspathFragment(ctx android.ModuleContext, contents for _, name := range android.SortedStringKeys(bootDexInfoByModule) { bootDexInfo := bootDexInfoByModule[name] unencodedDex := bootDexInfo.path - encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, outputDir) + encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, bootDexInfo.minSdkVersion, outputDir) encodedBootDexJarsByModule[name] = encodedDex } @@ -1188,6 +1188,9 @@ type bootDexInfo struct { // Indicates whether the dex jar needs uncompressing before encoding. uncompressDex bool + + // The minimum sdk version that the dex jar will be used on. + minSdkVersion android.SdkSpec } // bootDexInfoByModule is a map from module name (as returned by module.Name()) to the boot dex @@ -1213,6 +1216,7 @@ func extractBootDexInfoFromModules(ctx android.ModuleContext, contents []android bootDexJarsByModule[module.Name()] = bootDexInfo{ path: bootDexJar, uncompressDex: *hiddenAPIModule.uncompressDex(), + minSdkVersion: hiddenAPIModule.MinSdkVersion(ctx), } } diff --git a/java/java.go b/java/java.go index b34d6de8a..13f4c807e 100644 --- a/java/java.go +++ b/java/java.go @@ -300,19 +300,11 @@ var _ android.LicenseAnnotationsDependencyTag = dependencyTag{} type usesLibraryDependencyTag struct { dependencyTag - - // SDK version in which the library appared as a standalone library. - sdkVersion int - - // If the dependency is optional or required. - optional bool - - // Whether this is an implicit dependency inferred by Soong, or an explicit one added via - // `uses_libs`/`optional_uses_libs` properties. - implicit bool + sdkVersion int // SDK version in which the library appared as a standalone library. + optional bool // If the dependency is optional or required. } -func makeUsesLibraryDependencyTag(sdkVersion int, optional bool, implicit bool) usesLibraryDependencyTag { +func makeUsesLibraryDependencyTag(sdkVersion int, optional bool) usesLibraryDependencyTag { return usesLibraryDependencyTag{ dependencyTag: dependencyTag{ name: fmt.Sprintf("uses-library-%d", sdkVersion), @@ -320,7 +312,6 @@ func makeUsesLibraryDependencyTag(sdkVersion int, optional bool, implicit bool) }, sdkVersion: sdkVersion, optional: optional, - implicit: implicit, } } @@ -351,6 +342,11 @@ var ( syspropPublicStubDepTag = dependencyTag{name: "sysprop public stub"} jniInstallTag = installDependencyTag{name: "jni install"} binaryInstallTag = installDependencyTag{name: "binary install"} + usesLibReqTag = makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, false) + usesLibOptTag = makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, true) + usesLibCompat28OptTag = makeUsesLibraryDependencyTag(28, true) + usesLibCompat29ReqTag = makeUsesLibraryDependencyTag(29, false) + usesLibCompat30OptTag = makeUsesLibraryDependencyTag(30, true) ) func IsLibDepTag(depTag blueprint.DependencyTag) bool { @@ -1996,10 +1992,8 @@ func addCLCFromDep(ctx android.ModuleContext, depModule android.Module, depTag := ctx.OtherModuleDependencyTag(depModule) if depTag == libTag { // Ok, propagate <uses-library> through non-static library dependencies. - } else if tag, ok := depTag.(usesLibraryDependencyTag); ok && - tag.sdkVersion == dexpreopt.AnySdkVersion && tag.implicit { - // Ok, propagate <uses-library> through non-compatibility implicit <uses-library> - // dependencies. + } else if tag, ok := depTag.(usesLibraryDependencyTag); ok && tag.sdkVersion == dexpreopt.AnySdkVersion { + // Ok, propagate <uses-library> through non-compatibility <uses-library> dependencies. } else if depTag == staticLibTag { // Propagate <uses-library> through static library dependencies, unless it is a component // library (such as stubs). Component libraries have a dependency on their SDK library, @@ -2017,7 +2011,7 @@ func addCLCFromDep(ctx android.ModuleContext, depModule android.Module, // <uses_library> and should not be added to CLC, but the transitive <uses-library> dependencies // from its CLC should be added to the current CLC. if sdkLib != nil { - clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, false, true, + clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, false, dep.DexJarBuildPath().PathOrNil(), dep.DexJarInstallPath(), dep.ClassLoaderContexts()) } else { clcMap.AddContextMap(dep.ClassLoaderContexts(), depName) @@ -2089,6 +2083,11 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) if m.properties.Javacflags != nil { javacopts = append(javacopts, m.properties.Javacflags...) } + if m.properties.Java_version != nil { + javaVersion := normalizeJavaVersion(ctx, *m.properties.Java_version).String() + javacopts = append(javacopts, fmt.Sprintf("-source %s -target %s", javaVersion, javaVersion)) + } + epEnabled := m.properties.Errorprone.Enabled //TODO(b/227504307) add configuration that depends on RUN_ERROR_PRONE environment variable if Bool(epEnabled) { diff --git a/java/kotlin.go b/java/kotlin.go index eff5bb53f..903c6249b 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -175,6 +175,7 @@ func kotlinKapt(ctx android.ModuleContext, srcJarOutputFile, resJarOutputFile an var deps android.Paths deps = append(deps, flags.kotlincClasspath...) + deps = append(deps, flags.kotlincDeps...) deps = append(deps, srcJars...) deps = append(deps, flags.processorPath...) deps = append(deps, commonSrcFiles...) diff --git a/java/kotlin_test.go b/java/kotlin_test.go index f9ff98229..435d78294 100644 --- a/java/kotlin_test.go +++ b/java/kotlin_test.go @@ -325,6 +325,7 @@ func TestKotlinCompose(t *testing.T) { java_library { name: "withcompose", srcs: ["a.kt"], + plugins: ["plugin"], static_libs: ["androidx.compose.runtime_runtime"], } @@ -332,6 +333,10 @@ func TestKotlinCompose(t *testing.T) { name: "nocompose", srcs: ["a.kt"], } + + java_plugin { + name: "plugin", + } `) buildOS := result.Config.BuildOS.String() @@ -346,6 +351,9 @@ func TestKotlinCompose(t *testing.T) { android.AssertStringDoesContain(t, "missing compose compiler plugin", withCompose.VariablesForTestsRelativeToTop()["kotlincFlags"], "-Xplugin="+composeCompiler.String()) + android.AssertStringListContains(t, "missing kapt compose compiler dependency", + withCompose.Rule("kapt").Implicits.Strings(), composeCompiler.String()) + android.AssertStringListDoesNotContain(t, "unexpected compose compiler dependency", noCompose.Rule("kotlinc").Implicits.Strings(), composeCompiler.String()) diff --git a/java/lint.go b/java/lint.go index e97c9c225..426a2af25 100644 --- a/java/lint.go +++ b/java/lint.go @@ -75,9 +75,9 @@ type linter struct { extraLintCheckJars android.Paths test bool library bool - minSdkVersion string - targetSdkVersion string - compileSdkVersion string + minSdkVersion android.ApiLevel + targetSdkVersion android.ApiLevel + compileSdkVersion android.ApiLevel compileSdkKind android.SdkKind javaLanguageLevel string kotlinLanguageLevel string @@ -300,7 +300,7 @@ func (l *linter) generateManifest(ctx android.ModuleContext, rule *android.RuleB Text(`echo "<manifest xmlns:android='http://schemas.android.com/apk/res/android'" &&`). Text(`echo " android:versionCode='1' android:versionName='1' >" &&`). Textf(`echo " <uses-sdk android:minSdkVersion='%s' android:targetSdkVersion='%s'/>" &&`, - l.minSdkVersion, l.targetSdkVersion). + l.minSdkVersion.String(), l.targetSdkVersion.String()). Text(`echo "</manifest>"`). Text(") >").Output(manifestPath) @@ -325,7 +325,7 @@ func (l *linter) lint(ctx android.ModuleContext) { return } - if l.minSdkVersion != l.compileSdkVersion { + if l.minSdkVersion.CompareTo(l.compileSdkVersion) == -1 { l.extraMainlineLintErrors = append(l.extraMainlineLintErrors, updatabilityChecks...) _, filtered := android.FilterList(l.properties.Lint.Warning_checks, updatabilityChecks) if len(filtered) != 0 { @@ -427,7 +427,7 @@ func (l *linter) lint(ctx android.ModuleContext) { FlagWithOutput("--html ", html). FlagWithOutput("--text ", text). FlagWithOutput("--xml ", xml). - FlagWithArg("--compile-sdk-version ", l.compileSdkVersion). + FlagWithArg("--compile-sdk-version ", l.compileSdkVersion.String()). FlagWithArg("--java-language-level ", l.javaLanguageLevel). FlagWithArg("--kotlin-language-level ", l.kotlinLanguageLevel). FlagWithArg("--url ", fmt.Sprintf(".=.,%s=out", android.PathForOutput(ctx).String())). diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt index 4bc0c5fd4..1eee354cd 100644 --- a/java/lint_defaults.txt +++ b/java/lint_defaults.txt @@ -28,6 +28,11 @@ --disable_check SuspiciousImport --disable_check UnusedResources --disable_check ViewConstructor +# Disable NewApi checks for the platform since platform is the one that implements +# the API. This prevents noisy lint warnings like b/228956345#1 +# NewApi checks will continue to be enforced for apex deps since +# lint.strict_updatability_linting will be true for those Soong modules +--disable_check NewApi # Downgrade existing errors to warnings --warning_check AppCompatResource # 55 occurences in 10 modules @@ -66,7 +71,6 @@ --warning_check MissingTvBanner # 3 occurences in 3 modules --warning_check NamespaceTypo # 3 occurences in 3 modules --warning_check NetworkSecurityConfig # 46 occurences in 12 modules ---warning_check NewApi # 1996 occurences in 122 modules --warning_check NotSibling # 15 occurences in 10 modules --warning_check ObjectAnimatorBinding # 14 occurences in 5 modules --warning_check OnClick # 49 occurences in 21 modules diff --git a/java/sdk_library.go b/java/sdk_library.go index c37ed1a27..cd8e8755e 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -377,6 +377,9 @@ type sdkLibraryProperties struct { // List of Java libraries that will be in the classpath when building the implementation lib Impl_only_libs []string `android:"arch_variant"` + // List of Java libraries that will included in the implementation lib. + Impl_only_static_libs []string `android:"arch_variant"` + // List of Java libraries that will be in the classpath when building stubs Stub_only_libs []string `android:"arch_variant"` @@ -1346,10 +1349,12 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) visibility := childModuleVisibility(module.sdkLibraryProperties.Impl_library_visibility) props := struct { - Name *string - Visibility []string - Instrument bool - Libs []string + Name *string + Visibility []string + Instrument bool + Libs []string + Static_libs []string + Apex_available []string }{ Name: proptools.StringPtr(module.implLibraryModuleName()), Visibility: visibility, @@ -1358,6 +1363,12 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) // Set the impl_only libs. Note that the module's "Libs" get appended as well, via the // addition of &module.properties below. Libs: module.sdkLibraryProperties.Impl_only_libs, + // Set the impl_only static libs. Note that the module's "static_libs" get appended as well, via the + // addition of &module.properties below. + Static_libs: module.sdkLibraryProperties.Impl_only_static_libs, + // Pass the apex_available settings down so that the impl library can be statically + // embedded within a library that is added to an APEX. Needed for updatable-media. + Apex_available: module.ApexAvailable(), } properties := []interface{}{ @@ -1814,8 +1825,9 @@ func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookCont *javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName()) } - // Add the impl_only_libs *after* we're done using the Libs prop in submodules. + // Add the impl_only_libs and impl_only_static_libs *after* we're done using them in submodules. module.properties.Libs = append(module.properties.Libs, module.sdkLibraryProperties.Impl_only_libs...) + module.properties.Static_libs = append(module.properties.Static_libs, module.sdkLibraryProperties.Impl_only_static_libs...) } func (module *SdkLibrary) InitSdkLibraryProperties() { @@ -2211,8 +2223,23 @@ func (module *SdkLibraryImport) UniqueApexVariations() bool { return module.uniqueApexVariations() } +// MinSdkVersion - Implements hiddenAPIModule +func (module *SdkLibraryImport) MinSdkVersion(ctx android.EarlyModuleContext) android.SdkSpec { + return android.SdkSpecNone +} + +var _ hiddenAPIModule = (*SdkLibraryImport)(nil) + func (module *SdkLibraryImport) OutputFiles(tag string) (android.Paths, error) { - return module.commonOutputFiles(tag) + paths, err := module.commonOutputFiles(tag) + if paths != nil || err != nil { + return paths, err + } + if module.implLibraryModule != nil { + return module.implLibraryModule.OutputFiles(tag) + } else { + return nil, nil + } } func (module *SdkLibraryImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index e84eacd86..cc83430a8 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -173,7 +173,7 @@ func main() { } ok := true for _, mkFile := range files { - ok = convertOne(mkFile) && ok + ok = convertOne(mkFile, []string{}) && ok } if *launcher != "" { @@ -183,7 +183,7 @@ func main() { if *inputVariables == "" { quit(fmt.Errorf("the product launcher requires an input variables file")) } - if !convertOne(*inputVariables) { + if !convertOne(*inputVariables, []string{}) { quit(fmt.Errorf("the product launcher input variables file failed to convert")) } @@ -201,7 +201,7 @@ func main() { if *inputVariables == "" { quit(fmt.Errorf("the board launcher requires an input variables file")) } - if !convertOne(*inputVariables) { + if !convertOne(*inputVariables, []string{}) { quit(fmt.Errorf("the board launcher input variables file failed to convert")) } err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher( @@ -310,9 +310,13 @@ const copyright = `# // the output hierarchy, or to the stdout. // Optionally, recursively convert the files this one includes by // $(call inherit-product) or an include statement. -func convertOne(mkFile string) (ok bool) { +func convertOne(mkFile string, loadStack []string) (ok bool) { if v, ok := converted[mkFile]; ok { - return v != nil + if v == nil { + fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile) + return false + } + return true } converted[mkFile] = nil defer func() { @@ -356,6 +360,7 @@ func convertOne(mkFile string) (ok bool) { return false } } + loadStack = append(loadStack, mkFile) ok = true if *recurse { for _, sub := range ss.SubConfigFiles() { @@ -363,7 +368,7 @@ func convertOne(mkFile string) (ok bool) { if _, err := os.Stat(sub); os.IsNotExist(err) { continue } - ok = convertOne(sub) && ok + ok = convertOne(sub, loadStack) && ok } } converted[mkFile] = ss diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go index 92665207d..6a6eb460a 100644 --- a/mk2rbc/expr.go +++ b/mk2rbc/expr.go @@ -741,8 +741,8 @@ func (_ *badExpr) typ() starlarkType { return starlarkTypeUnknown } -func (_ *badExpr) emitListVarCopy(_ *generationContext) { - panic("implement me") +func (b *badExpr) emitListVarCopy(gctx *generationContext) { + b.emit(gctx) } func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 8f4fea4b1..1eae63f11 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -86,7 +86,7 @@ var knownFunctions = map[string]interface { "find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList}, "filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList}, "filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList}, - "firstword": &firstOrLastwordCallParser{isLastWord: false}, + "firstword": &simpleCallParser{name: baseName + ".first_word", returnType: starlarkTypeString}, "foreach": &foreachCallParser{}, "if": &ifCallParser{}, "info": &makeControlFuncParser{name: baseName + ".mkinfo"}, @@ -97,7 +97,7 @@ var knownFunctions = map[string]interface { "is-product-in-list": &isProductInListCallParser{}, "is-vendor-board-platform": &isVendorBoardPlatformCallParser{}, "is-vendor-board-qcom": &isVendorBoardQcomCallParser{}, - "lastword": &firstOrLastwordCallParser{isLastWord: true}, + "lastword": &simpleCallParser{name: baseName + ".last_word", returnType: starlarkTypeString}, "notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString}, "math_max": &mathMaxOrMinCallParser{function: "max"}, "math_min": &mathMaxOrMinCallParser{function: "min"}, @@ -116,6 +116,7 @@ var knownFunctions = map[string]interface { "subst": &substCallParser{fname: "subst"}, "warning": &makeControlFuncParser{name: baseName + ".mkwarning"}, "word": &wordCallParser{}, + "words": &wordsCallParser{}, "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList}, } @@ -130,6 +131,14 @@ var knownNodeFunctions = map[string]interface { "foreach": &foreachCallNodeParser{}, } +// These look like variables, but are actually functions, and would give +// undefined variable errors if we converted them as variables. Instead, +// emit an error instead of converting them. +var unsupportedFunctions = map[string]bool{ + "local-generated-sources-dir": true, + "local-intermediates-dir": true, +} + // These are functions that we don't implement conversions for, but // we allow seeing their definitions in the product config files. var ignoredDefines = map[string]bool{ @@ -459,6 +468,7 @@ func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { predefined := []struct{ name, value string }{ {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")}, {"LOCAL_PATH", filepath.Dir(ss.mkFile)}, + {"MAKEFILE_LIST", ss.mkFile}, {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk // TODO(asmundak): maybe read it from build/make/core/envsetup.mk? {"TARGET_COPY_OUT_SYSTEM", "system"}, @@ -531,7 +541,7 @@ func (ctx *parseContext) backNode() { func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode { // Handle only simple variables - if !a.Name.Const() { + if !a.Name.Const() || a.Target != nil { return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")} } name := a.Name.Strings[0] @@ -542,6 +552,12 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if strings.HasPrefix(name, "override ") { return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")} } + if name == ".KATI_READONLY" { + // Skip assignments to .KATI_READONLY. If it was in the output file, it + // would be an error because it would be sorted before the definition of + // the variable it's trying to make readonly. + return []starlarkNode{} + } // Soong configuration if strings.HasPrefix(name, soongNsPrefix) { @@ -556,9 +572,6 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs.valueType() == starlarkTypeUnknown { // Try to divine variable type from the RHS asgn.value = ctx.parseMakeString(a, a.Value) - if xBad, ok := asgn.value.(*badExpr); ok { - return []starlarkNode{&exprNode{xBad}} - } inferred_type := asgn.value.typ() if inferred_type != starlarkTypeUnknown { lhs.setValueType(inferred_type) @@ -567,21 +580,19 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs.valueType() == starlarkTypeList { xConcat, xBad := ctx.buildConcatExpr(a) if xBad != nil { - return []starlarkNode{&exprNode{expr: xBad}} - } - switch len(xConcat.items) { - case 0: - asgn.value = &listExpr{} - case 1: - asgn.value = xConcat.items[0] - default: - asgn.value = xConcat + asgn.value = xBad + } else { + switch len(xConcat.items) { + case 0: + asgn.value = &listExpr{} + case 1: + asgn.value = xConcat.items[0] + default: + asgn.value = xConcat + } } } else { asgn.value = ctx.parseMakeString(a, a.Value) - if xBad, ok := asgn.value.(*badExpr); ok { - return []starlarkNode{&exprNode{expr: xBad}} - } } if asgn.lhs.valueType() == starlarkTypeString && @@ -811,35 +822,40 @@ func (ctx *parseContext) handleSubConfig( // rblf.inherit(handle, _e[0], _e[1]) // var matchingPaths []string - varPath, ok := pathExpr.(*interpolateExpr) - if !ok { - return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} - } - - pathPattern := []string{varPath.chunks[0]} - for _, chunk := range varPath.chunks[1:] { - if chunk != "" { - pathPattern = append(pathPattern, chunk) + var needsWarning = false + if interpolate, ok := pathExpr.(*interpolateExpr); ok { + pathPattern := []string{interpolate.chunks[0]} + for _, chunk := range interpolate.chunks[1:] { + if chunk != "" { + pathPattern = append(pathPattern, chunk) + } } - } - if pathPattern[0] == "" && len(ctx.includeTops) > 0 { - // If pattern starts from the top. restrict it to the directories where - // we know inherit-product uses dynamically calculated path. + if pathPattern[0] == "" && len(ctx.includeTops) > 0 { + // If pattern starts from the top. restrict it to the directories where + // we know inherit-product uses dynamically calculated path. + for _, p := range ctx.includeTops { + pathPattern[0] = p + matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...) + } + } else { + matchingPaths = ctx.findMatchingPaths(pathPattern) + } + needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0 + } else if len(ctx.includeTops) > 0 { for _, p := range ctx.includeTops { - pathPattern[0] = p - matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...) + matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{p, ""})...) } } else { - matchingPaths = ctx.findMatchingPaths(pathPattern) + return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} } + // Safeguard against $(call inherit-product,$(PRODUCT_PATH)) const maxMatchingFiles = 150 if len(matchingPaths) > maxMatchingFiles { return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)} } - needsWarning := pathPattern[0] == "" && len(ctx.includeTops) == 0 - res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} + res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} for _, p := range matchingPaths { // A product configuration files discovered dynamically may attempt to inherit // from another one which does not exist in this source tree. Prevent load errors @@ -889,8 +905,9 @@ func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, arg }) } -func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) []starlarkNode { - return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode { +func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode { + loadAlways := v.Name[0] != '-' + return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode { return &includeNode{im, loadAlways} }) } @@ -1068,6 +1085,18 @@ func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { return otherOperand } } + if otherOperand.typ() == starlarkTypeList { + fields := strings.Fields(stringOperand) + elements := make([]starlarkExpr, len(fields)) + for i, s := range fields { + elements[i] = &stringLiteralExpr{literal: s} + } + return &eqExpr{ + left: otherOperand, + right: &listExpr{elements}, + isEq: isEq, + } + } if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt { return &eqExpr{ left: otherOperand, @@ -1113,8 +1142,6 @@ func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, switch call.name { case baseName + ".filter": return ctx.parseCompareFilterFuncResult(directive, call, value, isEq) - case baseName + ".expand_wildcard": - return ctx.parseCompareWildcardFuncResult(directive, call, value, !isEq), true case baseName + ".findstring": return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true case baseName + ".strip": @@ -1159,22 +1186,6 @@ func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive, } } -func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive, - xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { - if !isEmptyString(xValue) { - return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue) - } - callFunc := baseName + ".file_wildcard_exists" - if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") { - callFunc = baseName + ".file_exists" - } - var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool} - if !negate { - cc = ¬Expr{cc} - } - return cc -} - func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive, xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { if isEmptyString(xValue) { @@ -1262,6 +1273,12 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt return ctx.newBadExpr(node, "reference is too complex: %s", refDump) } + if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok { + if _, unsupported := unsupportedFunctions[name]; unsupported { + return ctx.newBadExpr(node, "%s is not supported", refDump) + } + } + // If it is a single word, it can be a simple variable // reference or a function call if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" { @@ -1309,9 +1326,8 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt } else { return ctx.newBadExpr(node, "cannot handle invoking %s", name) } - } else { - return ctx.newBadExpr(node, "cannot handle %s", refDump) } + return ctx.newBadExpr(node, "cannot handle %s", refDump) } type simpleCallParser struct { @@ -1587,6 +1603,16 @@ func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starla for _, n := range a.actions { transformNode(n, transformer) } + case *inheritNode: + if b, ok := a.module.(inheritedDynamicModule); ok { + b.path = b.path.transform(transformer) + a.module = b + } + case *includeNode: + if b, ok := a.module.(inheritedDynamicModule); ok { + b.path = b.path.transform(transformer) + a.module = b + } } } @@ -1637,9 +1663,11 @@ func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa if len(words) != 2 { return ctx.newBadExpr(node, "word function should have 2 arguments") } - var index uint64 = 0 + var index = 0 if words[0].Const() { - index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64) + if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil { + index = i + } } if index < 1 { return ctx.newBadExpr(node, "word index should be constant positive integer") @@ -1647,35 +1675,40 @@ func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa words[1].TrimLeftSpaces() words[1].TrimRightSpaces() array := ctx.parseMakeString(node, words[1]) - if xBad, ok := array.(*badExpr); ok { - return xBad + if bad, ok := array.(*badExpr); ok { + return bad } if array.typ() != starlarkTypeList { - array = &callExpr{object: array, name: "split", returnType: starlarkTypeList} + array = &callExpr{ + name: baseName + ".words", + args: []starlarkExpr{array}, + returnType: starlarkTypeList, + } } - return &indexExpr{array, &intLiteralExpr{int(index - 1)}} + return &indexExpr{array, &intLiteralExpr{index - 1}} } -type firstOrLastwordCallParser struct { - isLastWord bool -} +type wordsCallParser struct{} -func (p *firstOrLastwordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { - arg := ctx.parseMakeString(node, args) - if bad, ok := arg.(*badExpr); ok { +func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { + args.TrimLeftSpaces() + args.TrimRightSpaces() + array := ctx.parseMakeString(node, args) + if bad, ok := array.(*badExpr); ok { return bad } - index := &intLiteralExpr{0} - if p.isLastWord { - if v, ok := arg.(*variableRefExpr); ok && v.ref.name() == "MAKEFILE_LIST" { - return &stringLiteralExpr{ctx.script.mkFile} + if array.typ() != starlarkTypeList { + array = &callExpr{ + name: baseName + ".words", + args: []starlarkExpr{array}, + returnType: starlarkTypeList, } - index.literal = -1 } - if arg.typ() == starlarkTypeList { - return &indexExpr{arg, index} + return &callExpr{ + name: "len", + args: []starlarkExpr{array}, + returnType: starlarkTypeInt, } - return &indexExpr{&callExpr{object: arg, name: "split", returnType: starlarkTypeList}, index} } func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) { @@ -1759,10 +1792,23 @@ func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa } case *mkparser.Comment: return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}} + case *mkparser.Directive: + if n.Name == "include" || n.Name == "-include" { + return ctx.handleInclude(n) + } + case *mkparser.Variable: + // Technically inherit-product(-if-exists) don't need to be put inside + // an eval, but some makefiles do it, presumably because they copy+pasted + // from a $(eval include ...) + if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok { + if name == "inherit-product" || name == "inherit-product-if-exists" { + return ctx.handleVariable(n) + } + } } } - return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments and comments are supported")} + return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")} } func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr { @@ -1822,7 +1868,7 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNod result = []starlarkNode{res} } case "include", "-include": - result = ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') + result = ctx.handleInclude(x) case "ifeq", "ifneq", "ifdef", "ifndef": result = []starlarkNode{ctx.handleIfBlock(x)} default: diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index de7512927..7f236bbc7 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -117,8 +117,8 @@ PRODUCT_NAME := $(call foo0) def init(g, handle): cfg = rblf.cfg(handle) - rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1") - rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0") + cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1") + cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0") `, }, { @@ -568,14 +568,18 @@ ifeq (,$(wildcard foo.mk)) endif ifneq (,$(wildcard foo*.mk)) endif +ifeq (foo1.mk foo2.mk barxyz.mk,$(wildcard foo*.mk bar*.mk)) +endif `, expected: `load("//build/make/core:product_config.rbc", "rblf") def init(g, handle): cfg = rblf.cfg(handle) - if not rblf.file_exists("foo.mk"): + if not rblf.expand_wildcard("foo.mk"): + pass + if rblf.expand_wildcard("foo*.mk"): pass - if rblf.file_wildcard_exists("foo*.mk"): + if rblf.expand_wildcard("foo*.mk bar*.mk") == ["foo1.mk", "foo2.mk", "barxyz.mk"]: pass `, }, @@ -808,6 +812,10 @@ def init(g, handle): PRODUCT_COPY_FILES := $(addprefix pfx-,a b c) PRODUCT_COPY_FILES := $(addsuffix .sff, a b c) PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM))) +ifeq (1,$(words $(SOME_UNKNOWN_VARIABLE))) +endif +ifeq ($(words $(SOME_OTHER_VARIABLE)),$(SOME_INT_VARIABLE)) +endif $(info $(patsubst %.pub,$(PRODUCT_NAME)%,$(PRODUCT_ADB_KEYS))) $(info $$(dir foo/bar): $(dir foo/bar)) $(info $(firstword $(PRODUCT_COPY_FILES))) @@ -830,14 +838,18 @@ def init(g, handle): cfg = rblf.cfg(handle) cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c") cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c") - cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0] + cfg["PRODUCT_NAME"] = rblf.words((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " "))[0] + if len(rblf.words(g.get("SOME_UNKNOWN_VARIABLE", ""))) == 1: + pass + if ("%d" % (len(rblf.words(g.get("SOME_OTHER_VARIABLE", ""))))) == g.get("SOME_INT_VARIABLE", ""): + pass rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%s%%" % cfg["PRODUCT_NAME"], g.get("PRODUCT_ADB_KEYS", ""))) rblf.mkinfo("product.mk", "$(dir foo/bar): %s" % rblf.dir("foo/bar")) - rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][0]) - rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][-1]) - rblf.mkinfo("product.mk", rblf.dir("product.mk")) - rblf.mkinfo("product.mk", rblf.dir(cfg["PRODUCT_COPY_FILES"][-1])) - rblf.mkinfo("product.mk", rblf.dir((_foobar).split()[-1])) + rblf.mkinfo("product.mk", rblf.first_word(cfg["PRODUCT_COPY_FILES"])) + rblf.mkinfo("product.mk", rblf.last_word(cfg["PRODUCT_COPY_FILES"])) + rblf.mkinfo("product.mk", rblf.dir(rblf.last_word("product.mk"))) + rblf.mkinfo("product.mk", rblf.dir(rblf.last_word(cfg["PRODUCT_COPY_FILES"]))) + rblf.mkinfo("product.mk", rblf.dir(rblf.last_word(_foobar))) rblf.mkinfo("product.mk", rblf.abspath("foo/bar")) rblf.mkinfo("product.mk", rblf.notdir("foo/bar")) rblf.soong_config_namespace(g, "snsconfig") @@ -975,7 +987,7 @@ def init(g, handle): rblf.soong_config_namespace(g, "cvd") rblf.soong_config_set(g, "cvd", "launch_configs", "cvd_config_auto.json") rblf.soong_config_append(g, "cvd", "grub_config", "grub.cfg") - rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config") + _x = rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config") `, }, { desc: "soong namespace accesses", @@ -1142,6 +1154,11 @@ def init(g, handle): MY_PATH:=foo #RBC# include_top vendor/foo1 $(call inherit-product,$(MY_PATH)/cfg.mk) +#RBC# include_top vendor/foo1 +$(call inherit-product,$(MY_OTHER_PATH)) +#RBC# include_top vendor/foo1 +$(foreach f,$(MY_MAKEFILES), \ + $(call inherit-product,$(f))) `, expected: `load("//build/make/core:product_config.rbc", "rblf") load("//vendor/foo1:cfg.star|init", _cfg_init = "init") @@ -1156,6 +1173,21 @@ def init(g, handle): if not _varmod_init: rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"])) rblf.inherit(handle, _varmod, _varmod_init) + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get(g.get("MY_OTHER_PATH", "")) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % (g.get("MY_OTHER_PATH", ""))) + rblf.inherit(handle, _varmod, _varmod_init) + for f in rblf.words(g.get("MY_MAKEFILES", "")): + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get(f) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % (f)) + rblf.inherit(handle, _varmod, _varmod_init) `, }, { @@ -1242,13 +1274,15 @@ def init(g, handle): desc: "Ignore make rules", mkname: "product.mk", in: ` +foo: PRIVATE_VARIABLE = some_tool $< $@ foo: foo.c gcc -o $@ $*`, expected: `load("//build/make/core:product_config.rbc", "rblf") def init(g, handle): cfg = rblf.cfg(handle) - rblf.mk2rbc_error("product.mk:2", "unsupported line rule: foo: foo.c\n#gcc -o $@ $*") + rblf.mk2rbc_error("product.mk:2", "Only simple variables are handled") + rblf.mk2rbc_error("product.mk:3", "unsupported line rule: foo: foo.c\n#gcc -o $@ $*") `, }, { @@ -1269,6 +1303,7 @@ def init(g, handle): in: ` ifeq (,$(call foobar)) endif +my_sources := $(local-generated-sources-dir) `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -1276,6 +1311,7 @@ def init(g, handle): cfg = rblf.cfg(handle) if rblf.mk2rbc_error("build/product.mk:2", "cannot handle invoking foobar"): pass + _my_sources = rblf.mk2rbc_error("build/product.mk:4", "local-generated-sources-dir is not supported") `, }, { @@ -1509,24 +1545,54 @@ $(eval) $(eval MY_VAR := foo) $(eval # This is a test of eval functions) $(eval $(TOO_COMPLICATED) := bar) +$(eval include foo/font.mk) +$(eval $(call inherit-product,vendor/foo1/cfg.mk)) + $(foreach x,$(MY_LIST_VAR), \ $(eval PRODUCT_COPY_FILES += foo/bar/$(x):$(TARGET_COPY_OUT_VENDOR)/etc/$(x)) \ - $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x))) \ -) + $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x)))) +$(foreach x,$(MY_LIST_VAR), \ + $(eval include foo/$(x).mk)) `, expected: `load("//build/make/core:product_config.rbc", "rblf") +load("//foo:font.star", _font_init = "init") +load("//vendor/foo1:cfg.star", _cfg_init = "init") def init(g, handle): cfg = rblf.cfg(handle) g["MY_VAR"] = "foo" # This is a test of eval functions - rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments and comments are supported") + rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported") + _font_init(g, handle) + rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init) for x in rblf.words(g.get("MY_LIST_VAR", "")): rblf.setdefault(handle, "PRODUCT_COPY_FILES") cfg["PRODUCT_COPY_FILES"] += ("foo/bar/%s:%s/etc/%s" % (x, g.get("TARGET_COPY_OUT_VENDOR", ""), x)).split() if g.get("MY_OTHER_VAR", ""): cfg["PRODUCT_COPY_FILES"] += ("%s:foo/bar/%s" % (g.get("MY_OTHER_VAR", ""), x)).split() + for x in rblf.words(g.get("MY_LIST_VAR", "")): + _entry = { + "foo/font.mk": ("foo/font", _font_init), + }.get("foo/%s.mk" % _x) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % ("foo/%s.mk" % _x)) + _varmod_init(g, handle) +`, + }, + { + desc: ".KATI_READONLY", + mkname: "product.mk", + in: ` +MY_VAR := foo +.KATI_READONLY := MY_VAR +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["MY_VAR"] = "foo" `, }, } diff --git a/mk2rbc/node.go b/mk2rbc/node.go index 7c39b9ead..a01abd8ac 100644 --- a/mk2rbc/node.go +++ b/mk2rbc/node.go @@ -83,7 +83,7 @@ func (im inheritedStaticModule) needsLoadCheck() bool { } type inheritedDynamicModule struct { - path interpolateExpr + path starlarkExpr candidateModules []*moduleInfo loadAlways bool location ErrorLocation @@ -120,7 +120,7 @@ func (i inheritedDynamicModule) emitSelect(gctx *generationContext) { } func (i inheritedDynamicModule) pathExpr() starlarkExpr { - return &i.path + return i.path } func (i inheritedDynamicModule) needsLoadCheck() bool { diff --git a/multitree/Android.bp b/multitree/Android.bp new file mode 100644 index 000000000..9b16d2021 --- /dev/null +++ b/multitree/Android.bp @@ -0,0 +1,19 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-multitree", + pkgPath: "android/soong/multitree", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "api_surface.go", + "export.go", + "metadata.go", + "import.go", + ], + pluginFor: ["soong_build"], +} diff --git a/multitree/api_surface.go b/multitree/api_surface.go new file mode 100644 index 000000000..f739a2430 --- /dev/null +++ b/multitree/api_surface.go @@ -0,0 +1,119 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + "fmt" + + "github.com/google/blueprint" +) + +var ( + pctx = android.NewPackageContext("android/soong/multitree") +) + +func init() { + RegisterApiSurfaceBuildComponents(android.InitRegistrationContext) +} + +var PrepareForTestWithApiSurface = android.FixtureRegisterWithContext(RegisterApiSurfaceBuildComponents) + +func RegisterApiSurfaceBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("api_surface", ApiSurfaceFactory) +} + +type ApiSurface struct { + android.ModuleBase + ExportableModuleBase + properties apiSurfaceProperties + + allOutputs android.Paths + taggedOutputs map[string]android.Paths +} + +type apiSurfaceProperties struct { + Contributions []string +} + +func ApiSurfaceFactory() android.Module { + module := &ApiSurface{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + InitExportableModule(module) + return module +} + +func (surface *ApiSurface) DepsMutator(ctx android.BottomUpMutatorContext) { + if surface.properties.Contributions != nil { + ctx.AddVariationDependencies(nil, nil, surface.properties.Contributions...) + } + +} +func (surface *ApiSurface) GenerateAndroidBuildActions(ctx android.ModuleContext) { + contributionFiles := make(map[string]android.Paths) + var allOutputs android.Paths + ctx.WalkDeps(func(child, parent android.Module) bool { + if contribution, ok := child.(ApiContribution); ok { + copied := contribution.CopyFilesWithTag(ctx) + for tag, files := range copied { + contributionFiles[child.Name()+"#"+tag] = files + } + for _, paths := range copied { + allOutputs = append(allOutputs, paths...) + } + return false // no transitive dependencies + } + return false + }) + + // phony target + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: android.PathForPhony(ctx, ctx.ModuleName()), + Inputs: allOutputs, + }) + + surface.allOutputs = allOutputs + surface.taggedOutputs = contributionFiles +} + +func (surface *ApiSurface) OutputFiles(tag string) (android.Paths, error) { + if tag != "" { + return nil, fmt.Errorf("unknown tag: %q", tag) + } + return surface.allOutputs, nil +} + +func (surface *ApiSurface) TaggedOutputs() map[string]android.Paths { + return surface.taggedOutputs +} + +func (surface *ApiSurface) Exportable() bool { + return true +} + +var _ android.OutputFileProducer = (*ApiSurface)(nil) +var _ Exportable = (*ApiSurface)(nil) + +type ApiContribution interface { + // copy files necessaryt to construct an API surface + // For C, it will be map.txt and .h files + // For Java, it will be api.txt + CopyFilesWithTag(ctx android.ModuleContext) map[string]android.Paths // output paths + + // Generate Android.bp in out/ to use the exported .txt files + // GenerateBuildFiles(ctx ModuleContext) Paths //output paths +} diff --git a/multitree/export.go b/multitree/export.go new file mode 100644 index 000000000..aecade58d --- /dev/null +++ b/multitree/export.go @@ -0,0 +1,67 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + + "github.com/google/blueprint/proptools" +) + +type moduleExportProperty struct { + // True if the module is exported to the other components in a multi-tree. + // Any components in the multi-tree can import this module to use. + Export *bool +} + +type ExportableModuleBase struct { + properties moduleExportProperty +} + +type Exportable interface { + // Properties for the exporable module. + exportableModuleProps() *moduleExportProperty + + // Check if this module can be exported. + // If this returns false, the module will not be exported regardless of the 'export' value. + Exportable() bool + + // Returns 'true' if this module has 'export: true' + // This module will not be exported if it returns 'false' to 'Exportable()' interface even if + // it has 'export: true'. + IsExported() bool + + // Map from tags to outputs. + // Each module can tag their outputs for convenience. + TaggedOutputs() map[string]android.Paths +} + +type ExportableModule interface { + android.Module + android.OutputFileProducer + Exportable +} + +func InitExportableModule(module ExportableModule) { + module.AddProperties(module.exportableModuleProps()) +} + +func (m *ExportableModuleBase) exportableModuleProps() *moduleExportProperty { + return &m.properties +} + +func (m *ExportableModuleBase) IsExported() bool { + return proptools.Bool(m.properties.Export) +} diff --git a/multitree/import.go b/multitree/import.go new file mode 100644 index 000000000..1e5c421bc --- /dev/null +++ b/multitree/import.go @@ -0,0 +1,96 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" +) + +var ( + nameSuffix = ".imported" +) + +type MultitreeImportedModuleInterface interface { + GetMultitreeImportedModuleName() string +} + +func init() { + android.RegisterModuleType("imported_filegroup", importedFileGroupFactory) + + android.PreArchMutators(RegisterMultitreePreArchMutators) +} + +type importedFileGroupProperties struct { + // Imported modules from the other components in a multi-tree + Imported []string +} + +type importedFileGroup struct { + android.ModuleBase + + properties importedFileGroupProperties + srcs android.Paths +} + +func (ifg *importedFileGroup) Name() string { + return ifg.BaseModuleName() + nameSuffix +} + +func importedFileGroupFactory() android.Module { + module := &importedFileGroup{} + module.AddProperties(&module.properties) + + android.InitAndroidModule(module) + return module +} + +var _ MultitreeImportedModuleInterface = (*importedFileGroup)(nil) + +func (ifg *importedFileGroup) GetMultitreeImportedModuleName() string { + // The base module name of the imported filegroup is used as the imported module name + return ifg.BaseModuleName() +} + +var _ android.SourceFileProducer = (*importedFileGroup)(nil) + +func (ifg *importedFileGroup) Srcs() android.Paths { + return ifg.srcs +} + +func (ifg *importedFileGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // srcs from this module must not be used. Adding a dot path to avoid the empty + // source failure. Still soong returns error when a module wants to build against + // this source, which is intended. + ifg.srcs = android.PathsForModuleSrc(ctx, []string{"."}) +} + +func RegisterMultitreePreArchMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("multitree_imported_rename", MultitreeImportedRenameMutator).Parallel() +} + +func MultitreeImportedRenameMutator(ctx android.BottomUpMutatorContext) { + if m, ok := ctx.Module().(MultitreeImportedModuleInterface); ok { + name := m.GetMultitreeImportedModuleName() + if !ctx.OtherModuleExists(name) { + // Provide an empty filegroup not to break the build while updating the metadata. + // In other cases, soong will report an error to guide users to run 'm update-meta' + // first. + if !ctx.Config().TargetMultitreeUpdateMeta() { + ctx.ModuleErrorf("\"%s\" filegroup must be imported.\nRun 'm update-meta' first to import the filegroup.", name) + } + ctx.Rename(name) + } + } +} diff --git a/multitree/metadata.go b/multitree/metadata.go new file mode 100644 index 000000000..3fd721599 --- /dev/null +++ b/multitree/metadata.go @@ -0,0 +1,74 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + "encoding/json" +) + +func init() { + android.RegisterSingletonType("update-meta", UpdateMetaSingleton) +} + +func UpdateMetaSingleton() android.Singleton { + return &updateMetaSingleton{} +} + +type jsonImported struct { + FileGroups map[string][]string `json:",omitempty"` +} + +type metadataJsonFlags struct { + Imported jsonImported `json:",omitempty"` + Exported map[string][]string `json:",omitempty"` +} + +type updateMetaSingleton struct { + importedModules []string + generatedMetadataFile android.OutputPath +} + +func (s *updateMetaSingleton) GenerateBuildActions(ctx android.SingletonContext) { + metadata := metadataJsonFlags{ + Imported: jsonImported{ + FileGroups: make(map[string][]string), + }, + Exported: make(map[string][]string), + } + ctx.VisitAllModules(func(module android.Module) { + if ifg, ok := module.(*importedFileGroup); ok { + metadata.Imported.FileGroups[ifg.BaseModuleName()] = ifg.properties.Imported + } + if e, ok := module.(ExportableModule); ok { + if e.IsExported() && e.Exportable() { + for tag, files := range e.TaggedOutputs() { + // TODO(b/219846705): refactor this to a dictionary + metadata.Exported[e.Name()+":"+tag] = append(metadata.Exported[e.Name()+":"+tag], files.Strings()...) + } + } + } + }) + jsonStr, err := json.Marshal(metadata) + if err != nil { + ctx.Errorf(err.Error()) + } + s.generatedMetadataFile = android.PathForOutput(ctx, "multitree", "metadata.json") + android.WriteFileRule(ctx, s.generatedMetadataFile, string(jsonStr)) +} + +func (s *updateMetaSingleton) MakeVars(ctx android.MakeVarsContext) { + ctx.Strict("MULTITREE_METADATA", s.generatedMetadataFile.String()) +} diff --git a/rust/binary.go b/rust/binary.go index 0dc320e5f..41110f92f 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -128,11 +128,11 @@ func (binary *binaryDecorator) preferRlib() bool { return Bool(binary.baseCompiler.Properties.Prefer_rlib) || Bool(binary.Properties.Static_executable) } -func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { fileName := binary.getStem(ctx) + ctx.toolchain().ExecutableSuffix() srcPath, _ := srcPathFromModuleSrcs(ctx, binary.baseCompiler.Properties.Srcs) outputFile := android.PathForModuleOut(ctx, fileName) - ret := outputFile + ret := buildOutput{outputFile: outputFile} flags.RustFlags = append(flags.RustFlags, deps.depFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...) @@ -147,8 +147,7 @@ func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps Path } binary.baseCompiler.unstrippedOutputFile = outputFile - TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile) - + ret.kytheFile = TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile).kytheFile return ret } diff --git a/rust/bindgen.go b/rust/bindgen.go index c2b05129a..b4626a048 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -30,7 +30,7 @@ var ( defaultBindgenFlags = []string{""} // bindgen should specify its own Clang revision so updating Clang isn't potentially blocked on bindgen failures. - bindgenClangVersion = "clang-r445002" + bindgenClangVersion = "clang-r450784d" _ = pctx.VariableFunc("bindgenClangVersion", func(ctx android.PackageVarContext) string { if override := ctx.Config().Getenv("LLVM_BINDGEN_PREBUILTS_VERSION"); override != "" { diff --git a/rust/builder.go b/rust/builder.go index 20ca5dbee..7dd9dd276 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -83,10 +83,37 @@ var ( RspfileContent: "$in", }, "outDir") + + // Cross-referencing: + _ = pctx.SourcePathVariable("rustExtractor", + "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/rust_extractor") + _ = pctx.VariableFunc("kytheCorpus", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() }) + _ = pctx.VariableFunc("kytheCuEncoding", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCuEncoding() }) + _ = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json") + kytheExtract = pctx.AndroidStaticRule("kythe", + blueprint.RuleParams{ + Command: `KYTHE_CORPUS=${kytheCorpus} ` + + `KYTHE_OUTPUT_FILE=$out ` + + `KYTHE_VNAMES=$kytheVnames ` + + `KYTHE_KZIP_ENCODING=${kytheCuEncoding} ` + + `KYTHE_CANONICALIZE_VNAME_PATHS=prefer-relative ` + + `$rustExtractor $envVars ` + + `$rustcCmd ` + + `-C linker=${config.RustLinker} ` + + `-C link-args="${crtBegin} ${config.RustLinkerArgs} ${linkFlags} ${crtEnd}" ` + + `$in ${libFlags} $rustcFlags`, + CommandDeps: []string{"$rustExtractor", "$kytheVnames"}, + Rspfile: "${out}.rsp", + RspfileContent: "$in", + }, + "rustcFlags", "linkFlags", "libFlags", "crtBegin", "crtEnd", "envVars") ) type buildOutput struct { outputFile android.Path + kytheFile android.Path } func init() { @@ -324,6 +351,25 @@ func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, fl }, }) + if flags.EmitXrefs { + kytheFile := android.PathForModuleOut(ctx, outputFile.Base()+".kzip") + ctx.Build(pctx, android.BuildParams{ + Rule: kytheExtract, + Description: "Xref Rust extractor " + main.Rel(), + Output: kytheFile, + Inputs: inputs, + Implicits: implicits, + Args: map[string]string{ + "rustcFlags": strings.Join(rustcFlags, " "), + "linkFlags": strings.Join(linkFlags, " "), + "libFlags": strings.Join(libFlags, " "), + "crtBegin": strings.Join(deps.CrtBegin.Strings(), " "), + "crtEnd": strings.Join(deps.CrtEnd.Strings(), " "), + "envVars": strings.Join(envVars, " "), + }, + }) + output.kytheFile = kytheFile + } return output } diff --git a/rust/compiler.go b/rust/compiler.go index 19499fa36..bcd58c82b 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -304,6 +304,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag flags.GlobalRustFlags = append(flags.GlobalRustFlags, config.GlobalRustFlags...) flags.GlobalRustFlags = append(flags.GlobalRustFlags, ctx.toolchain().ToolchainRustFlags()) flags.GlobalLinkFlags = append(flags.GlobalLinkFlags, ctx.toolchain().ToolchainLinkFlags()) + flags.EmitXrefs = ctx.Config().EmitXrefRules() if ctx.Host() && !ctx.Windows() { rpathPrefix := `\$$ORIGIN/` @@ -324,7 +325,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag return flags } -func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { panic(fmt.Errorf("baseCrater doesn't know how to crate things!")) } diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go index 802e1da7f..746857916 100644 --- a/rust/config/allowed_list.go +++ b/rust/config/allowed_list.go @@ -8,6 +8,7 @@ var ( RustAllowedPaths = []string{ "device/google/cuttlefish", "external/adhd", + "external/boringssl", "external/crosvm", "external/libchromeos-rs", "external/minijail", diff --git a/rust/config/global.go b/rust/config/global.go index 2d5fa991b..d11665c23 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.59.0" + RustDefaultVersion = "1.60.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ @@ -50,6 +50,7 @@ var ( "-C force-unwind-tables=yes", // Use v0 mangling to distinguish from C++ symbols "-C symbol-mangling-version=v0", + "--color always", } deviceGlobalRustFlags = []string{ diff --git a/rust/library.go b/rust/library.go index 62eaefd68..1286549c6 100644 --- a/rust/library.go +++ b/rust/library.go @@ -474,8 +474,9 @@ func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags) F return flags } -func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { - var outputFile, ret android.ModuleOutPath +func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { + var outputFile android.ModuleOutPath + var ret buildOutput var fileName string srcPath := library.srcPath(ctx, deps) @@ -487,19 +488,19 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa if library.rlib() { fileName = library.getStem(ctx) + ctx.toolchain().RlibSuffix() outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } else if library.dylib() { fileName = library.getStem(ctx) + ctx.toolchain().DylibSuffix() outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } else if library.static() { fileName = library.getStem(ctx) + ctx.toolchain().StaticLibSuffix() outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } else if library.shared() { fileName = library.sharedLibFilename(ctx) outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } if !library.rlib() && !library.static() && library.stripper.NeedsStrip(ctx) { @@ -524,13 +525,13 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa // Call the appropriate builder for this library type if library.rlib() { - TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile).kytheFile } else if library.dylib() { - TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile).kytheFile } else if library.static() { - TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile).kytheFile } else if library.shared() { - TransformSrctoShared(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoShared(ctx, srcPath, deps, flags, outputFile).kytheFile } if library.rlib() || library.dylib() { @@ -572,7 +573,7 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa return ret } -func (library *libraryDecorator) srcPath(ctx ModuleContext, deps PathDeps) android.Path { +func (library *libraryDecorator) srcPath(ctx ModuleContext, _ PathDeps) android.Path { if library.sourceProvider != nil { // Assume the first source from the source provider is the library entry point. return library.sourceProvider.Srcs()[0] diff --git a/rust/prebuilt.go b/rust/prebuilt.go index 6cdd07de9..fe9d0b5dd 100644 --- a/rust/prebuilt.go +++ b/rust/prebuilt.go @@ -145,7 +145,7 @@ func (prebuilt *prebuiltLibraryDecorator) compilerProps() []interface{} { &prebuilt.Properties) } -func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { prebuilt.flagExporter.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...) prebuilt.flagExporter.setProvider(ctx) @@ -154,7 +154,7 @@ func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)") } prebuilt.baseCompiler.unstrippedOutputFile = srcPath - return srcPath + return buildOutput{outputFile: srcPath} } func (prebuilt *prebuiltLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags, @@ -202,7 +202,7 @@ func (prebuilt *prebuiltProcMacroDecorator) compilerProps() []interface{} { &prebuilt.Properties) } -func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { prebuilt.flagExporter.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...) prebuilt.flagExporter.setProvider(ctx) @@ -211,7 +211,7 @@ func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Fla ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)") } prebuilt.baseCompiler.unstrippedOutputFile = srcPath - return srcPath + return buildOutput{outputFile: srcPath} } func (prebuilt *prebuiltProcMacroDecorator) rustdoc(ctx ModuleContext, flags Flags, diff --git a/rust/proc_macro.go b/rust/proc_macro.go index f8a4bbded..832b62c36 100644 --- a/rust/proc_macro.go +++ b/rust/proc_macro.go @@ -70,14 +70,14 @@ func (procMacro *procMacroDecorator) compilerFlags(ctx ModuleContext, flags Flag return flags } -func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { fileName := procMacro.getStem(ctx) + ctx.toolchain().ProcMacroSuffix() outputFile := android.PathForModuleOut(ctx, fileName) srcPath, _ := srcPathFromModuleSrcs(ctx, procMacro.baseCompiler.Properties.Srcs) - TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile) + ret := TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile) procMacro.baseCompiler.unstrippedOutputFile = outputFile - return outputFile + return ret } func (procMacro *procMacroDecorator) getStem(ctx ModuleContext) string { diff --git a/rust/rust.go b/rust/rust.go index c4fd14859..48419eb8e 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -15,6 +15,7 @@ package rust import ( + "android/soong/bloaty" "fmt" "strings" @@ -22,7 +23,6 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" - "android/soong/bloaty" "android/soong/cc" cc_config "android/soong/cc/config" "android/soong/fuzz" @@ -52,6 +52,7 @@ func init() { }) pctx.Import("android/soong/rust/config") pctx.ImportAs("cc_config", "android/soong/cc/config") + android.InitRegistrationContext.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory) } type Flags struct { @@ -64,6 +65,7 @@ type Flags struct { Toolchain config.Toolchain Coverage bool Clippy bool + EmitXrefs bool // If true, emit rules to aid cross-referencing } type BaseProperties struct { @@ -161,6 +163,9 @@ type Module struct { // Output file to be installed, may be stripped or unstripped. outputFile android.OptionalPath + // Cross-reference input file + kytheFiles android.Paths + docTimestampFile android.OptionalPath hideApexVariantFromMake bool @@ -394,6 +399,10 @@ func (mod *Module) SplitPerApiLevel() bool { return false } +func (mod *Module) XrefRustFiles() android.Paths { + return mod.kytheFiles +} + type Deps struct { Dylibs []string Rlibs []string @@ -457,7 +466,7 @@ type compiler interface { cfgFlags(ctx ModuleContext, flags Flags) Flags featureFlags(ctx ModuleContext, flags Flags) Flags compilerProps() []interface{} - compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path + compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput compilerDeps(ctx DepsContext, deps Deps) Deps crateName() string rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath @@ -493,6 +502,10 @@ type exportedFlagsProducer interface { exportLinkObjects(...string) } +type xref interface { + XrefRustFiles() android.Paths +} + type flagExporter struct { linkDirs []string linkObjects []string @@ -904,11 +917,14 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if mod.compiler != nil && !mod.compiler.Disabled() { mod.compiler.initialize(ctx) - outputFile := mod.compiler.compile(ctx, flags, deps) + buildOutput := mod.compiler.compile(ctx, flags, deps) if ctx.Failed() { return } - mod.outputFile = android.OptionalPathForPath(outputFile) + mod.outputFile = android.OptionalPathForPath(buildOutput.outputFile) + if buildOutput.kytheFile != nil { + mod.kytheFiles = append(mod.kytheFiles, buildOutput.kytheFile) + } bloaty.MeasureSizeForPaths(ctx, mod.compiler.strippedOutputFilePath(), android.OptionalPathForPath(mod.compiler.unstrippedOutputFilePath())) mod.docTimestampFile = mod.compiler.rustdoc(ctx, flags, deps) @@ -1618,6 +1634,25 @@ func libNameFromFilePath(filepath android.Path) (string, bool) { return "", false } +func kytheExtractRustFactory() android.Singleton { + return &kytheExtractRustSingleton{} +} + +type kytheExtractRustSingleton struct { +} + +func (k kytheExtractRustSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var xrefTargets android.Paths + ctx.VisitAllModules(func(module android.Module) { + if rustModule, ok := module.(xref); ok { + xrefTargets = append(xrefTargets, rustModule.XrefRustFiles()...) + } + }) + if len(xrefTargets) > 0 { + ctx.Phony("xref_rust", xrefTargets...) + } +} + var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String diff --git a/rust/snapshot_prebuilt.go b/rust/snapshot_prebuilt.go index dfbc1d1e7..2f79cc5c3 100644 --- a/rust/snapshot_prebuilt.go +++ b/rust/snapshot_prebuilt.go @@ -69,7 +69,7 @@ func snapshotLibraryFactory(image cc.SnapshotImage, moduleSuffix string) (*Modul return module, prebuilt } -func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { var variant string if library.static() { variant = cc.SnapshotStaticSuffix @@ -85,11 +85,11 @@ func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, } if !library.MatchesWithDevice(ctx.DeviceConfig()) { - return nil + return buildOutput{} } outputFile := android.PathForModuleSrc(ctx, *library.properties.Src) library.unstrippedOutputFile = outputFile - return outputFile + return buildOutput{outputFile: outputFile} } func (library *snapshotLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath { diff --git a/rust/test.go b/rust/test.go index 250b7657e..6e5393595 100644 --- a/rust/test.go +++ b/rust/test.go @@ -15,6 +15,8 @@ package rust import ( + "path/filepath" + "github.com/google/blueprint/proptools" "android/soong/android" @@ -151,9 +153,15 @@ func (test *testDecorator) install(ctx ModuleContext) { ctx.ModuleErrorf("data_lib %q is not a linkable module", depName) } if linkableDep.OutputFile().Valid() { + // Copy the output in "lib[64]" so that it's compatible with + // the default rpath values. + libDir := "lib" + if linkableDep.Target().Arch.ArchType.Multilib == "lib64" { + libDir = "lib64" + } test.data = append(test.data, android.DataPath{SrcPath: linkableDep.OutputFile().Path(), - RelativeInstallPath: linkableDep.RelativeInstallPath()}) + RelativeInstallPath: filepath.Join(libDir, linkableDep.RelativeInstallPath())}) } }) diff --git a/rust/test_test.go b/rust/test_test.go index 112417673..8906f1cb0 100644 --- a/rust/test_test.go +++ b/rust/test_test.go @@ -187,12 +187,12 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { t.Errorf("expected test output file to be 'main_test', but was '%s'", outputPath) } entries := android.AndroidMkEntriesForTest(t, ctx, module)[0] - if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][0], ":test_lib.so:foo/bar/baz") { - t.Errorf("expected LOCAL_TEST_DATA to end with `:test_lib.so:foo/bar/baz`,"+ + if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][0], ":test_lib.so:lib64/foo/bar/baz") { + t.Errorf("expected LOCAL_TEST_DATA to end with `:test_lib.so:lib64/foo/bar/baz`,"+ " but was '%s'", entries.EntryMap["LOCAL_TEST_DATA"][0]) } - if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][1], ":librust_test_lib.so:foo/bar/baz") { - t.Errorf("expected LOCAL_TEST_DATA to end with `:librust_test_lib.so:foo/bar/baz`,"+ + if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][1], ":librust_test_lib.so:lib64/foo/bar/baz") { + t.Errorf("expected LOCAL_TEST_DATA to end with `:librust_test_lib.so:lib64/foo/bar/baz`,"+ " but was '%s'", entries.EntryMap["LOCAL_TEST_DATA"][1]) } if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][2], ":rusty:foo/bar/baz") { diff --git a/rust/testing.go b/rust/testing.go index cb98bed65..4796f691a 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -194,6 +194,7 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.BottomUp("rust_begin", BeginMutator).Parallel() }) ctx.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton) + ctx.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory) ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() }) diff --git a/scripts/test_config_fixer.py b/scripts/test_config_fixer.py index c150e8cfd..3dbc22ec2 100644 --- a/scripts/test_config_fixer.py +++ b/scripts/test_config_fixer.py @@ -28,6 +28,8 @@ from manifest import parse_manifest from manifest import parse_test_config from manifest import write_xml +KNOWN_PREPARERS = ['com.android.tradefed.targetprep.TestAppInstallSetup', + 'com.android.tradefed.targetprep.suite.SuiteApkInstaller'] def parse_args(): """Parse commandline arguments.""" @@ -64,7 +66,7 @@ def overwrite_test_file_name(test_config_doc, test_file_name): tests = get_children_with_tag(test_config, 'target_preparer') for test in tests: - if test.getAttribute('class') == "com.android.tradefed.targetprep.TestAppInstallSetup": + if test.getAttribute('class') in KNOWN_PREPARERS: options = get_children_with_tag(test, 'option') for option in options: if option.getAttribute('name') == "test-file-name": diff --git a/scripts/test_config_fixer_test.py b/scripts/test_config_fixer_test.py index d00a59315..39ce5b3c3 100644 --- a/scripts/test_config_fixer_test.py +++ b/scripts/test_config_fixer_test.py @@ -70,7 +70,7 @@ class OverwritePackageNameTest(unittest.TestCase): class OverwriteTestFileNameTest(unittest.TestCase): """ Unit tests for overwrite_test_file_name function """ - test_config = ( + test_config_test_app_install_setup = ( '<?xml version="1.0" encoding="utf-8"?>\n' '<configuration description="Runs some tests.">\n' ' <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup">\n' @@ -82,15 +82,38 @@ class OverwriteTestFileNameTest(unittest.TestCase): ' </test>\n' '</configuration>\n') - def test_all(self): - doc = minidom.parseString(self.test_config % ("foo.apk")) + test_config_suite_apk_installer = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<configuration description="Runs some tests.">\n' + ' <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">\n' + ' <option name="test-file-name" value="%s"/>\n' + ' </target_preparer>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="com.android.foo"/>\n' + ' <option name="runtime-hint" value="20s"/>\n' + ' </test>\n' + '</configuration>\n') + + def test_testappinstallsetup(self): + doc = minidom.parseString(self.test_config_test_app_install_setup % ("foo.apk")) + + test_config_fixer.overwrite_test_file_name(doc, "bar.apk") + output = io.StringIO() + test_config_fixer.write_xml(output, doc) + + # Only the matching package name in a test node should be updated. + expected = self.test_config_test_app_install_setup % ("bar.apk") + self.assertEqual(expected, output.getvalue()) + + def test_suiteapkinstaller(self): + doc = minidom.parseString(self.test_config_suite_apk_installer % ("foo.apk")) test_config_fixer.overwrite_test_file_name(doc, "bar.apk") output = io.StringIO() test_config_fixer.write_xml(output, doc) # Only the matching package name in a test node should be updated. - expected = self.test_config % ("bar.apk") + expected = self.test_config_suite_apk_installer % ("bar.apk") self.assertEqual(expected, output.getvalue()) diff --git a/sdk/cc_sdk_test.go b/sdk/cc_sdk_test.go index cd63dac8c..571d21420 100644 --- a/sdk/cc_sdk_test.go +++ b/sdk/cc_sdk_test.go @@ -193,114 +193,6 @@ sdk_snapshot { `)) } -func TestBasicSdkWithCc(t *testing.T) { - result := testSdkWithCc(t, ` - sdk { - name: "mysdk", - native_shared_libs: ["sdkmember"], - } - - cc_library_shared { - name: "sdkmember", - system_shared_libs: [], - stl: "none", - apex_available: ["mysdkapex"], - } - - sdk_snapshot { - name: "mysdk@1", - native_shared_libs: ["sdkmember_mysdk@1"], - } - - sdk_snapshot { - name: "mysdk@2", - native_shared_libs: ["sdkmember_mysdk@2"], - } - - cc_prebuilt_library_shared { - name: "sdkmember", - srcs: ["libfoo.so"], - prefer: false, - system_shared_libs: [], - stl: "none", - } - - cc_prebuilt_library_shared { - name: "sdkmember_mysdk@1", - sdk_member_name: "sdkmember", - srcs: ["libfoo.so"], - system_shared_libs: [], - stl: "none", - // TODO: remove //apex_available:platform - apex_available: [ - "//apex_available:platform", - "myapex", - ], - } - - cc_prebuilt_library_shared { - name: "sdkmember_mysdk@2", - sdk_member_name: "sdkmember", - srcs: ["libfoo.so"], - system_shared_libs: [], - stl: "none", - // TODO: remove //apex_available:platform - apex_available: [ - "//apex_available:platform", - "myapex2", - ], - } - - cc_library_shared { - name: "mycpplib", - srcs: ["Test.cpp"], - shared_libs: ["sdkmember"], - system_shared_libs: [], - stl: "none", - apex_available: [ - "myapex", - "myapex2", - ], - } - - apex { - name: "myapex", - native_shared_libs: ["mycpplib"], - uses_sdks: ["mysdk@1"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - - apex { - name: "myapex2", - native_shared_libs: ["mycpplib"], - uses_sdks: ["mysdk@2"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - - apex { - name: "mysdkapex", - native_shared_libs: ["sdkmember"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - `) - - sdkMemberV1 := result.ModuleForTests("sdkmember_mysdk@1", "android_arm64_armv8-a_shared_apex10000_mysdk_1").Rule("toc").Output - sdkMemberV2 := result.ModuleForTests("sdkmember_mysdk@2", "android_arm64_armv8-a_shared_apex10000_mysdk_2").Rule("toc").Output - - cpplibForMyApex := result.ModuleForTests("mycpplib", "android_arm64_armv8-a_shared_apex10000_mysdk_1") - cpplibForMyApex2 := result.ModuleForTests("mycpplib", "android_arm64_armv8-a_shared_apex10000_mysdk_2") - - // Depending on the uses_sdks value, different libs are linked - ensureListContains(t, pathsToStrings(cpplibForMyApex.Rule("ld").Implicits), sdkMemberV1.String()) - ensureListContains(t, pathsToStrings(cpplibForMyApex2.Rule("ld").Implicits), sdkMemberV2.String()) -} - // Make sure the sdk can use host specific cc libraries static/shared and both. func TestHostSdkWithCc(t *testing.T) { testSdkWithCc(t, ` @@ -2835,11 +2727,6 @@ func TestNoSanitizerMembers(t *testing.T) { } `) - // Mixing the snapshot with the source (irrespective of which one is preferred) causes a problem - // due to missing variants. - // TODO(b/183204176): Remove this and fix the cause. - snapshotWithSourceErrorHandler := android.FixtureExpectsAtLeastOneErrorMatchingPattern(`\QReplaceDependencies could not find identical variant {os:android,image:,arch:arm64_armv8-a,sdk:,link:shared,version:} for module mynativelib\E`) - CheckSnapshot(t, result, "mysdk", "", checkUnversionedAndroidBpContents(` // This is auto-generated. DO NOT EDIT. @@ -2866,7 +2753,5 @@ myinclude/Test.h -> include/myinclude/Test.h arm64/include/Arm64Test.h -> arm64/include/arm64/include/Arm64Test.h .intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so `), - snapshotTestErrorHandler(checkSnapshotWithSourcePreferred, snapshotWithSourceErrorHandler), - snapshotTestErrorHandler(checkSnapshotPreferredWithSource, snapshotWithSourceErrorHandler), ) } diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index f0d3b35d7..a99fa1ff0 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -71,90 +71,6 @@ func TestSdkDependsOnSourceEvenWhenPrebuiltPreferred(t *testing.T) { ) } -func TestBasicSdkWithJavaLibrary(t *testing.T) { - result := android.GroupFixturePreparers( - prepareForSdkTestWithJava, - prepareForSdkTestWithApex, - ).RunTestWithBp(t, ` - sdk { - name: "mysdk", - java_header_libs: ["sdkmember"], - } - - sdk_snapshot { - name: "mysdk@1", - java_header_libs: ["sdkmember_mysdk@1"], - } - - sdk_snapshot { - name: "mysdk@2", - java_header_libs: ["sdkmember_mysdk@2"], - } - - java_library { - name: "sdkmember", - srcs: ["Test.java"], - system_modules: "none", - sdk_version: "none", - host_supported: true, - } - - java_import { - name: "sdkmember_mysdk@1", - sdk_member_name: "sdkmember", - host_supported: true, - } - - java_import { - name: "sdkmember_mysdk@2", - sdk_member_name: "sdkmember", - host_supported: true, - } - - java_library { - name: "myjavalib", - srcs: ["Test.java"], - libs: ["sdkmember"], - system_modules: "none", - sdk_version: "none", - compile_dex: true, - host_supported: true, - apex_available: [ - "myapex", - "myapex2", - ], - } - - apex { - name: "myapex", - java_libs: ["myjavalib"], - uses_sdks: ["mysdk@1"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - - apex { - name: "myapex2", - java_libs: ["myjavalib"], - uses_sdks: ["mysdk@2"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - `) - - sdkMemberV1 := result.ModuleForTests("sdkmember_mysdk@1", "android_common").Rule("combineJar").Output - sdkMemberV2 := result.ModuleForTests("sdkmember_mysdk@2", "android_common").Rule("combineJar").Output - - javalibForMyApex := result.ModuleForTests("myjavalib", "android_common_apex10000_mysdk_1") - javalibForMyApex2 := result.ModuleForTests("myjavalib", "android_common_apex10000_mysdk_2") - - // Depending on the uses_sdks value, different libs are linked - ensureListContains(t, pathsToStrings(javalibForMyApex.Rule("javac").Implicits), sdkMemberV1.String()) - ensureListContains(t, pathsToStrings(javalibForMyApex2.Rule("javac").Implicits), sdkMemberV2.String()) -} - func TestSnapshotWithJavaHeaderLibrary(t *testing.T) { result := android.GroupFixturePreparers( prepareForSdkTestWithJava, diff --git a/sdk/sdk.go b/sdk/sdk.go index 84c9a96e4..c8c7b79df 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -39,7 +39,6 @@ func registerSdkBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("sdk", SdkModuleFactory) ctx.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory) ctx.PreDepsMutators(RegisterPreDepsMutators) - ctx.PostDepsMutators(RegisterPostDepsMutators) } type sdk struct { @@ -278,20 +277,6 @@ func RegisterPreDepsMutators(ctx android.RegisterMutatorsContext) { ctx.BottomUp("SdkMemberInterVersion", memberInterVersionMutator).Parallel() } -// RegisterPostDepsMutators registers post-deps mutators to support modules implementing SdkAware -// interface and the sdk module type. This function has been made public to be called by tests -// outside of the sdk package -func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { - // These must run AFTER apexMutator. Note that the apex package is imported even though there is - // no direct dependency to the package here. sdkDepsMutator sets the SDK requirements from an - // APEX to its dependents. Since different versions of the same SDK can be used by different - // APEXes, the apex and its dependents (which includes the dependencies to the sdk members) - // should have been mutated for the apex before the SDK requirements are set. - ctx.TopDown("SdkDepsMutator", sdkDepsMutator).Parallel() - ctx.BottomUp("SdkDepsReplaceMutator", sdkDepsReplaceMutator).Parallel() - ctx.TopDown("SdkRequirementCheck", sdkRequirementsMutator).Parallel() -} - type dependencyTag struct { blueprint.BaseDependencyTag } @@ -413,103 +398,4 @@ func memberInterVersionMutator(mctx android.BottomUpMutatorContext) { type sdkAndApexModule interface { android.Module android.DepIsInSameApex - android.RequiredSdks -} - -// Step 4: transitively ripple down the SDK requirements from the root modules like APEX to its -// descendants -func sdkDepsMutator(mctx android.TopDownMutatorContext) { - if parent, ok := mctx.Module().(sdkAndApexModule); ok { - // Module types for Mainline modules (e.g. APEX) are expected to implement RequiredSdks() - // by reading its own properties like `uses_sdks`. - requiredSdks := parent.RequiredSdks() - if len(requiredSdks) > 0 { - mctx.VisitDirectDeps(func(m android.Module) { - // Only propagate required sdks from the apex onto its contents. - if dep, ok := m.(android.SdkAware); ok && android.IsDepInSameApex(mctx, parent, dep) { - dep.BuildWithSdks(requiredSdks) - } - }) - } - } -} - -// Step 5: if libfoo.mysdk.11 is in the context where version 11 of mysdk is requested, the -// versioned module is used instead of the un-versioned (in-development) module libfoo -func sdkDepsReplaceMutator(mctx android.BottomUpMutatorContext) { - if versionedSdkMember, ok := mctx.Module().(android.SdkAware); ok && versionedSdkMember.IsInAnySdk() && versionedSdkMember.IsVersioned() { - if sdk := versionedSdkMember.ContainingSdk(); !sdk.Unversioned() { - // Only replace dependencies to <sdkmember> with <sdkmember@required-version> - // if the depending module requires it. e.g. - // foo -> sdkmember - // will be transformed to: - // foo -> sdkmember@1 - // if and only if foo is a member of an APEX that requires version 1 of the - // sdk containing sdkmember. - memberName := versionedSdkMember.MemberName() - - // Convert a panic into a normal error to allow it to be more easily tested for. This is a - // temporary workaround, once http://b/183204176 has been fixed this can be removed. - // TODO(b/183204176): Remove this after fixing. - defer func() { - if r := recover(); r != nil { - mctx.ModuleErrorf("sdkDepsReplaceMutator %s", r) - } - }() - - // Replace dependencies on sdkmember with a dependency on the current module which - // is a versioned prebuilt of the sdkmember if required. - mctx.ReplaceDependenciesIf(memberName, func(from blueprint.Module, tag blueprint.DependencyTag, to blueprint.Module) bool { - // from - foo - // to - sdkmember - replace := false - if parent, ok := from.(android.RequiredSdks); ok { - replace = parent.RequiredSdks().Contains(sdk) - } - return replace - }) - } - } -} - -// Step 6: ensure that the dependencies outside of the APEX are all from the required SDKs -func sdkRequirementsMutator(mctx android.TopDownMutatorContext) { - if m, ok := mctx.Module().(sdkAndApexModule); ok { - requiredSdks := m.RequiredSdks() - if len(requiredSdks) == 0 { - return - } - mctx.VisitDirectDeps(func(dep android.Module) { - tag := mctx.OtherModuleDependencyTag(dep) - if tag == android.DefaultsDepTag { - // dependency to defaults is always okay - return - } - - // Ignore the dependency from the unversioned member to any versioned members as an - // apex that depends on the unversioned member will not also be depending on a versioned - // member. - if _, ok := tag.(sdkMemberVersionedDepTag); ok { - return - } - - // If the dep is outside of the APEX, but is not in any of the required SDKs, we know that the - // dep is a violation. - if sa, ok := dep.(android.SdkAware); ok { - // It is not an error if a dependency that is excluded from the apex due to the tag is not - // in one of the required SDKs. That is because all of the existing tags that implement it - // do not depend on modules which can or should belong to an sdk_snapshot. - if _, ok := tag.(android.ExcludeFromApexContentsTag); ok { - // The tag defines a dependency that never requires the child module to be part of the - // same apex. - return - } - - if !m.DepIsInSameApex(mctx, dep) && !requiredSdks.Contains(sa.ContainingSdk()) { - mctx.ModuleErrorf("depends on %q (in SDK %q) that isn't part of the required SDKs: %v", - sa.Name(), sa.ContainingSdk(), requiredSdks) - } - } - }) - } } diff --git a/sdk/sdk_test.go b/sdk/sdk_test.go index 83294f6ab..40de15027 100644 --- a/sdk/sdk_test.go +++ b/sdk/sdk_test.go @@ -37,64 +37,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestDepNotInRequiredSdks(t *testing.T) { - testSdkError(t, `module "myjavalib".*depends on "otherlib".*that isn't part of the required SDKs:.*`, ` - sdk { - name: "mysdk", - java_header_libs: ["sdkmember"], - } - - sdk_snapshot { - name: "mysdk@1", - java_header_libs: ["sdkmember_mysdk_1"], - } - - java_import { - name: "sdkmember", - prefer: false, - host_supported: true, - } - - java_import { - name: "sdkmember_mysdk_1", - sdk_member_name: "sdkmember", - host_supported: true, - } - - java_library { - name: "myjavalib", - srcs: ["Test.java"], - libs: [ - "sdkmember", - "otherlib", - ], - system_modules: "none", - sdk_version: "none", - compile_dex: true, - host_supported: true, - apex_available: ["myapex"], - } - - // this lib is no in mysdk - java_library { - name: "otherlib", - srcs: ["Test.java"], - system_modules: "none", - sdk_version: "none", - compile_dex: true, - host_supported: true, - } - - apex { - name: "myapex", - java_libs: ["myjavalib"], - uses_sdks: ["mysdk@1"], - key: "myapex.key", - certificate: ":myapex.cert", - } - `) -} - // Ensure that prebuilt modules have the same effective visibility as the source // modules. func TestSnapshotVisibility(t *testing.T) { diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh index 4f37c2bff..74e49aabe 100755 --- a/tests/bp2build_bazel_test.sh +++ b/tests/bp2build_bazel_test.sh @@ -115,3 +115,57 @@ EOF } test_bp2build_generates_all_buildfiles + +function test_cc_correctness { + setup + create_mock_bazel + + mkdir -p a + cat > a/Android.bp <<EOF +cc_object { + name: "qq", + srcs: ["qq.cc"], + bazel_module: { + bp2build_available: true, + }, + stl: "none", + system_shared_libs: [], +} +EOF + + cat > a/qq.cc <<EOF +#include "qq.h" +int qq() { + return QQ; +} +EOF + + cat > a/qq.h <<EOF +#define QQ 1 +EOF + + run_soong bp2build + + run_bazel build --package_path=out/soong/workspace //a:qq + local output_mtime1=$(stat -c "%y" bazel-bin/a/_objs/qq/qq.o) + + run_bazel build --package_path=out/soong/workspace //a:qq + local output_mtime2=$(stat -c "%y" bazel-bin/a/_objs/qq/qq.o) + + if [[ "$output_mtime1" != "$output_mtime2" ]]; then + fail "output changed on null build" + fi + + cat > a/qq.h <<EOF +#define QQ 2 +EOF + + run_bazel build --package_path=out/soong/workspace //a:qq + local output_mtime3=$(stat -c "%y" bazel-bin/a/_objs/qq/qq.o) + + if [[ "$output_mtime1" == "$output_mtime3" ]]; then + fail "output not changed when included header changed" + fi +} + +test_cc_correctness diff --git a/tests/lib.sh b/tests/lib.sh index 1bb2df915..7fd970a40 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -85,6 +85,7 @@ function create_mock_soong { copy_directory build/soong copy_directory build/make/tools/rbcrun + symlink_directory prebuilts/sdk symlink_directory prebuilts/go symlink_directory prebuilts/build-tools symlink_directory prebuilts/clang/host @@ -115,8 +116,10 @@ function create_mock_bazel() { copy_directory build/bazel symlink_directory prebuilts/bazel + symlink_directory prebuilts/clang symlink_directory prebuilts/jdk symlink_directory external/bazel-skylib + symlink_directory external/bazelbuild-rules_android symlink_file WORKSPACE symlink_file BUILD @@ -136,4 +139,5 @@ info "Mock top: $MOCK_TOP" export ALLOW_MISSING_DEPENDENCIES=true +export ALLOW_BP_UNDER_SYMLINKS=true warmup_mock_top diff --git a/ui/build/build.go b/ui/build/build.go index d261f8947..aadf4af4e 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "text/template" "android/soong/ui/metrics" @@ -205,6 +206,8 @@ func Build(ctx Context, config Config) { return } + defer waitForDist(ctx) + // checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree. checkProblematicFiles(ctx) @@ -329,8 +332,18 @@ func Build(ctx Context, config Config) { } } +var distWaitGroup sync.WaitGroup + +// waitForDist waits for all backgrounded distGzipFile and distFile writes to finish +func waitForDist(ctx Context) { + ctx.BeginTrace("soong_ui", "dist") + defer ctx.EndTrace() + + distWaitGroup.Wait() +} + // distGzipFile writes a compressed copy of src to the distDir if dist is enabled. Failures -// are printed but non-fatal. +// are printed but non-fatal. Uses the distWaitGroup func for backgrounding (optimization). func distGzipFile(ctx Context, config Config, src string, subDirs ...string) { if !config.Dist() { return @@ -343,13 +356,17 @@ func distGzipFile(ctx Context, config Config, src string, subDirs ...string) { ctx.Printf("failed to mkdir %s: %s", destDir, err.Error()) } - if err := gzipFileToDir(src, destDir); err != nil { - ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error()) - } + distWaitGroup.Add(1) + go func() { + defer distWaitGroup.Done() + if err := gzipFileToDir(src, destDir); err != nil { + ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error()) + } + }() } // distFile writes a copy of src to the distDir if dist is enabled. Failures are printed but -// non-fatal. +// non-fatal. Uses the distWaitGroup func for backgrounding (optimization). func distFile(ctx Context, config Config, src string, subDirs ...string) { if !config.Dist() { return @@ -362,7 +379,11 @@ func distFile(ctx Context, config Config, src string, subDirs ...string) { ctx.Printf("failed to mkdir %s: %s", destDir, err.Error()) } - if _, err := copyFile(src, filepath.Join(destDir, filepath.Base(src))); err != nil { - ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error()) - } + distWaitGroup.Add(1) + go func() { + defer distWaitGroup.Done() + if _, err := copyFile(src, filepath.Join(destDir, filepath.Base(src))); err != nil { + ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error()) + } + }() } diff --git a/ui/build/config.go b/ui/build/config.go index e271bfca2..0092ff1ca 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -1223,6 +1223,21 @@ func (c *configImpl) rbeAuth() (string, string) { return "RBE_use_application_default_credentials", "true" } +func (c *configImpl) IsGooglerEnvironment() bool { + cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" + if v, ok := c.environ.Get(cf); ok { + return v == "googler" + } + return false +} + +func (c *configImpl) GoogleProdCredsExist() bool { + if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil { + return false + } + return true +} + func (c *configImpl) UseRemoteBuild() bool { return c.UseGoma() || c.UseRBE() } diff --git a/ui/build/finder.go b/ui/build/finder.go index 262de3de7..4d6ad426f 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -64,6 +64,7 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { cacheParams := finder.CacheParams{ WorkingDirectory: dir, RootDirs: []string{"."}, + FollowSymlinks: config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"), ExcludeDirs: []string{".git", ".repo"}, PruneFiles: pruneFiles, IncludeFiles: []string{ diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 8f9a69991..78d37b4c7 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -119,6 +119,7 @@ func startRBE(ctx Context, config Config) { } func stopRBE(ctx Context, config Config) { + defer checkProdCreds(ctx, config) cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown") output, err := cmd.CombinedOutput() if err != nil { @@ -131,6 +132,15 @@ func stopRBE(ctx Context, config Config) { } } +func checkProdCreds(ctx Context, config Config) { + if !config.IsGooglerEnvironment() || config.GoogleProdCredsExist() { + return + } + fmt.Fprintln(ctx.Writer, "") + fmt.Fprintln(ctx.Writer, "\033[33mWARNING: Missing LOAS credentials, please run `gcert`. This will result in failing RBE builds in the future, see go/build-fast#authentication.\033[0m") + fmt.Fprintln(ctx.Writer, "") +} + // DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics. // The protobuf file is created if RBE is enabled and the proxy service has // started. The proxy service is shutdown in order to dump the RBE metrics to the diff --git a/ui/build/soong.go b/ui/build/soong.go index c7f22f946..8992b4f07 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -15,7 +15,9 @@ package build import ( + "errors" "fmt" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -491,10 +493,14 @@ func runSoong(ctx Context, config Config) { ninja("bootstrap", "bootstrap.ninja", targets...) - var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics if shouldCollectBuildSoongMetrics(config) { soongBuildMetrics := loadSoongBuildMetrics(ctx, config) - logSoongBuildMetrics(ctx, soongBuildMetrics) + if soongBuildMetrics != nil { + logSoongBuildMetrics(ctx, soongBuildMetrics) + if ctx.Metrics != nil { + ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics) + } + } } distGzipFile(ctx, config, config.SoongNinjaFile(), "soong") @@ -504,9 +510,6 @@ func runSoong(ctx Context, config Config) { distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong") } - if shouldCollectBuildSoongMetrics(config) && ctx.Metrics != nil { - ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics) - } if config.JsonModuleGraph() { distGzipFile(ctx, config, config.ModuleGraphFile(), "soong") } @@ -538,8 +541,12 @@ func shouldCollectBuildSoongMetrics(config Config) bool { func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics { soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb") - buf, err := ioutil.ReadFile(soongBuildMetricsFile) - if err != nil { + buf, err := os.ReadFile(soongBuildMetricsFile) + if errors.Is(err, fs.ErrNotExist) { + // Soong may not have run during this invocation + ctx.Verbosef("Failed to read metrics file, %s: %s", soongBuildMetricsFile, err) + return nil + } else if err != nil { ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err) } soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{} diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index 69f5689b9..9f9b863ba 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -14,7 +14,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 +// protoc-gen-go v1.28.0 // protoc v3.9.1 // source: metrics.proto @@ -954,9 +954,9 @@ type ModuleTypeInfo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The build system, eg. Soong or Make. + // The build system, e.g. Soong or Make. BuildSystem *ModuleTypeInfo_BuildSystem `protobuf:"varint,1,opt,name=build_system,json=buildSystem,enum=soong_build_metrics.ModuleTypeInfo_BuildSystem,def=0" json:"build_system,omitempty"` - // The module type, eg. java_library, cc_binary, and etc. + // The module type, e.g. java_library, cc_binary, and etc. ModuleType *string `protobuf:"bytes,2,opt,name=module_type,json=moduleType" json:"module_type,omitempty"` // The number of logical modules. NumOfModules *uint32 `protobuf:"varint,3,opt,name=num_of_modules,json=numOfModules" json:"num_of_modules,omitempty"` @@ -1142,6 +1142,8 @@ type SoongBuildMetrics struct { MaxHeapSize *uint64 `protobuf:"varint,5,opt,name=max_heap_size,json=maxHeapSize" json:"max_heap_size,omitempty"` // Runtime metrics for soong_build execution. Events []*PerfInfo `protobuf:"bytes,6,rep,name=events" json:"events,omitempty"` + // Mixed Builds information + MixedBuildsInfo *MixedBuildsInfo `protobuf:"bytes,7,opt,name=mixed_builds_info,json=mixedBuildsInfo" json:"mixed_builds_info,omitempty"` } func (x *SoongBuildMetrics) Reset() { @@ -1218,6 +1220,13 @@ func (x *SoongBuildMetrics) GetEvents() []*PerfInfo { return nil } +func (x *SoongBuildMetrics) GetMixedBuildsInfo() *MixedBuildsInfo { + if x != nil { + return x.MixedBuildsInfo + } + return nil +} + type ExpConfigFetcher struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1287,6 +1296,63 @@ func (x *ExpConfigFetcher) GetMicros() uint64 { return 0 } +type MixedBuildsInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Modules that are enabled for Mixed Builds. + MixedBuildEnabledModules []string `protobuf:"bytes,1,rep,name=mixed_build_enabled_modules,json=mixedBuildEnabledModules" json:"mixed_build_enabled_modules,omitempty"` + // Modules that are not currently eligible for MixedBuilds + MixedBuildDisabledModules []string `protobuf:"bytes,2,rep,name=mixed_build_disabled_modules,json=mixedBuildDisabledModules" json:"mixed_build_disabled_modules,omitempty"` +} + +func (x *MixedBuildsInfo) Reset() { + *x = MixedBuildsInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_metrics_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MixedBuildsInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MixedBuildsInfo) ProtoMessage() {} + +func (x *MixedBuildsInfo) ProtoReflect() protoreflect.Message { + mi := &file_metrics_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MixedBuildsInfo.ProtoReflect.Descriptor instead. +func (*MixedBuildsInfo) Descriptor() ([]byte, []int) { + return file_metrics_proto_rawDescGZIP(), []int{10} +} + +func (x *MixedBuildsInfo) GetMixedBuildEnabledModules() []string { + if x != nil { + return x.MixedBuildEnabledModules + } + return nil +} + +func (x *MixedBuildsInfo) GetMixedBuildDisabledModules() []string { + if x != nil { + return x.MixedBuildDisabledModules + } + return nil +} + var File_metrics_proto protoreflect.FileDescriptor var file_metrics_proto_rawDesc = []byte{ @@ -1491,7 +1557,7 @@ var file_metrics_proto_rawDesc = []byte{ 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x43, 0x72, 0x69, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x6f, 0x75, 0x72, 0x6e, 0x65, 0x79, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, - 0x04, 0x63, 0x75, 0x6a, 0x73, 0x22, 0xfa, 0x01, 0x0a, 0x11, 0x53, 0x6f, 0x6f, 0x6e, 0x67, 0x42, + 0x04, 0x63, 0x75, 0x6a, 0x73, 0x22, 0xcc, 0x02, 0x0a, 0x11, 0x53, 0x6f, 0x6f, 0x6e, 0x67, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, @@ -1507,22 +1573,36 @@ var file_metrics_proto_rawDesc = []byte{ 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x73, 0x22, 0xc8, 0x01, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x45, 0x78, - 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x22, 0x34, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x4f, - 0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, - 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x42, 0x28, 0x5a, - 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, - 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x74, 0x73, 0x12, 0x50, 0x0a, 0x11, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x73, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, + 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x2e, 0x4d, 0x69, 0x78, 0x65, 0x64, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x73, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x73, + 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc8, 0x01, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, + 0x45, 0x78, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, + 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x22, 0x34, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, + 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4e, 0x46, + 0x49, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x22, + 0x91, 0x01, 0x0a, 0x0f, 0x4d, 0x69, 0x78, 0x65, 0x64, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x73, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x3d, 0x0a, 0x1b, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x18, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x1c, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x19, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, + 0x6c, 0x65, 0x73, 0x42, 0x28, 0x5a, 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, + 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -1538,7 +1618,7 @@ func file_metrics_proto_rawDescGZIP() []byte { } var file_metrics_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_metrics_proto_goTypes = []interface{}{ (MetricsBase_BuildVariant)(0), // 0: soong_build_metrics.MetricsBase.BuildVariant (MetricsBase_Arch)(0), // 1: soong_build_metrics.MetricsBase.Arch @@ -1554,6 +1634,7 @@ var file_metrics_proto_goTypes = []interface{}{ (*CriticalUserJourneysMetrics)(nil), // 11: soong_build_metrics.CriticalUserJourneysMetrics (*SoongBuildMetrics)(nil), // 12: soong_build_metrics.SoongBuildMetrics (*ExpConfigFetcher)(nil), // 13: soong_build_metrics.ExpConfigFetcher + (*MixedBuildsInfo)(nil), // 14: soong_build_metrics.MixedBuildsInfo } var file_metrics_proto_depIdxs = []int32{ 0, // 0: soong_build_metrics.MetricsBase.target_build_variant:type_name -> soong_build_metrics.MetricsBase.BuildVariant @@ -1575,12 +1656,13 @@ var file_metrics_proto_depIdxs = []int32{ 4, // 16: soong_build_metrics.CriticalUserJourneyMetrics.metrics:type_name -> soong_build_metrics.MetricsBase 10, // 17: soong_build_metrics.CriticalUserJourneysMetrics.cujs:type_name -> soong_build_metrics.CriticalUserJourneyMetrics 7, // 18: soong_build_metrics.SoongBuildMetrics.events:type_name -> soong_build_metrics.PerfInfo - 3, // 19: soong_build_metrics.ExpConfigFetcher.status:type_name -> soong_build_metrics.ExpConfigFetcher.ConfigStatus - 20, // [20:20] is the sub-list for method output_type - 20, // [20:20] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 14, // 19: soong_build_metrics.SoongBuildMetrics.mixed_builds_info:type_name -> soong_build_metrics.MixedBuildsInfo + 3, // 20: soong_build_metrics.ExpConfigFetcher.status:type_name -> soong_build_metrics.ExpConfigFetcher.ConfigStatus + 21, // [21:21] is the sub-list for method output_type + 21, // [21:21] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_metrics_proto_init() } @@ -1709,6 +1791,18 @@ func file_metrics_proto_init() { return nil } } + file_metrics_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MixedBuildsInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -1716,7 +1810,7 @@ func file_metrics_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_metrics_proto_rawDesc, NumEnums: 4, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 0, }, diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index 814eb67c8..51dd523d2 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -200,10 +200,10 @@ message ModuleTypeInfo { SOONG = 1; MAKE = 2; } - // The build system, eg. Soong or Make. + // The build system, e.g. Soong or Make. optional BuildSystem build_system = 1 [default = UNKNOWN]; - // The module type, eg. java_library, cc_binary, and etc. + // The module type, e.g. java_library, cc_binary, and etc. optional string module_type = 2; // The number of logical modules. @@ -241,6 +241,9 @@ message SoongBuildMetrics { // Runtime metrics for soong_build execution. repeated PerfInfo events = 6; + + // Mixed Builds information + optional MixedBuildsInfo mixed_builds_info = 7; } message ExpConfigFetcher { @@ -261,3 +264,25 @@ message ExpConfigFetcher { // Time, in microseconds, taken by the expconfigfetcher optional uint64 micros = 3; } + +message MixedBuildsInfo{ + // Modules may be listed below as both enabled for Mixed Builds + // and disabled for Mixed Builds. This implies that some variants + // of the module are handled by Bazel in a Mixed Build, and other + // variants of the same module are handled by Soong. + + // Modules that are enabled for Mixed Builds. + repeated string mixed_build_enabled_modules = 1; + + // Modules that are not currently eligible to be handled + // by Bazel in a Mixed Build. + // Note that not all modules exempt from Bazel handling are + // listed. This list includes only modules which are of a + // Mixed-Build supported module type but are nevertheless not + // handled by Bazel. This may occur due to being present in + // the mixed build denylist, or as part of an unsupported + // mixed build variant type such as Windows. + + // Modules that are not enabled for MixedBuilds + repeated string mixed_build_disabled_modules = 2; +} |