diff options
62 files changed, 1669 insertions, 287 deletions
diff --git a/Android.bp b/Android.bp index 42a8e5c91..63de01589 100644 --- a/Android.bp +++ b/Android.bp @@ -119,3 +119,14 @@ genrule { dexpreopt_systemserver_check { name: "dexpreopt_systemserver_check", } + +// buildinfo.prop contains common properties for system/build.prop, like ro.build.version.* +buildinfo_prop { + name: "buildinfo.prop", + + // not installable because this will be included to system/build.prop + installable: false, + + // Currently, only microdroid can refer to buildinfo.prop + visibility: ["//packages/modules/Virtualization/microdroid"], +} diff --git a/android/Android.bp b/android/Android.bp index 87b021fb6..d58370391 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -36,6 +36,7 @@ bootstrap_go_package { "bazel.go", "bazel_handler.go", "bazel_paths.go", + "buildinfo_prop.go", "config.go", "config_bp2build.go", "csuite_config.go", diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 4fd4f134b..28dc8b452 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -104,12 +104,14 @@ var ( "external/google-benchmark": Bp2BuildDefaultTrueRecursively, "external/googletest": Bp2BuildDefaultTrueRecursively, "external/gwp_asan": Bp2BuildDefaultTrueRecursively, + "external/hamcrest": Bp2BuildDefaultTrueRecursively, "external/icu": Bp2BuildDefaultTrueRecursively, "external/icu/android_icu4j": Bp2BuildDefaultFalse, // java rules incomplete "external/icu/icu4j": Bp2BuildDefaultFalse, // java rules incomplete "external/javapoet": Bp2BuildDefaultTrueRecursively, "external/jemalloc_new": Bp2BuildDefaultTrueRecursively, "external/jsoncpp": Bp2BuildDefaultTrueRecursively, + "external/junit": Bp2BuildDefaultTrueRecursively, "external/libcap": Bp2BuildDefaultTrueRecursively, "external/libcxx": Bp2BuildDefaultTrueRecursively, "external/libcxxabi": Bp2BuildDefaultTrueRecursively, @@ -122,6 +124,7 @@ var ( "external/pcre": Bp2BuildDefaultTrueRecursively, "external/protobuf": Bp2BuildDefaultTrueRecursively, "external/python/six": Bp2BuildDefaultTrueRecursively, + "external/rappor": Bp2BuildDefaultTrueRecursively, "external/scudo": Bp2BuildDefaultTrueRecursively, "external/selinux/libselinux": Bp2BuildDefaultTrueRecursively, "external/selinux/libsepol": Bp2BuildDefaultTrueRecursively, @@ -313,6 +316,7 @@ var ( "host_bionic_linker_asm", // depends on extract_linker, a go binary. "host_bionic_linker_script", // depends on extract_linker, a go binary. "libc_musl_sysroot_bionic_arch_headers", // depends on soong_zip + "libc_musl_sysroot_zlib_headers", // depends on soong_zip and zip2zip "libc_musl_sysroot_bionic_headers", // 218405924, depends on soong_zip and generates duplicate srcs "libc_musl_sysroot_libc++_headers", "libc_musl_sysroot_libc++abi_headers", // depends on soong_zip, zip2zip "robolectric-sqlite4java-native", // depends on soong_zip, a go binary diff --git a/android/api_levels.go b/android/api_levels.go index 81638940c..aa55aa149 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -54,6 +54,14 @@ type ApiLevel struct { isPreview bool } +func (this ApiLevel) FinalInt() int { + if this.IsPreview() { + panic("Requested a final int from a non-final ApiLevel") + } else { + return this.number + } +} + func (this ApiLevel) FinalOrFutureInt() int { if this.IsPreview() { return FutureApiLevelInt diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go new file mode 100644 index 000000000..6339a7181 --- /dev/null +++ b/android/buildinfo_prop.go @@ -0,0 +1,182 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "strings" + + "github.com/google/blueprint/proptools" +) + +func init() { + ctx := InitRegistrationContext + ctx.RegisterSingletonModuleType("buildinfo_prop", buildinfoPropFactory) +} + +type buildinfoPropProperties struct { + // Whether this module is directly installable to one of the partitions. Default: true. + Installable *bool +} + +type buildinfoPropModule struct { + SingletonModuleBase + + properties buildinfoPropProperties + + outputFilePath OutputPath + installPath InstallPath +} + +var _ OutputFileProducer = (*buildinfoPropModule)(nil) + +func (p *buildinfoPropModule) installable() bool { + return proptools.BoolDefault(p.properties.Installable, true) +} + +// OutputFileProducer +func (p *buildinfoPropModule) OutputFiles(tag string) (Paths, error) { + if tag != "" { + return nil, fmt.Errorf("unsupported tag %q", tag) + } + return Paths{p.outputFilePath}, nil +} + +func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { + p.outputFilePath = PathForModuleOut(ctx, p.Name()).OutputPath + if !ctx.Config().KatiEnabled() { + WriteFileRule(ctx, p.outputFilePath, "# no buildinfo.prop if kati is disabled") + return + } + + rule := NewRuleBuilder(pctx, ctx) + cmd := rule.Command().Text("(") + + writeString := func(str string) { + cmd.Text(`echo "` + str + `" && `) + } + + writeString("# begin build properties") + writeString("# autogenerated by build/soong/android/buildinfo_prop.go") + + writeProp := func(key, value string) { + if strings.Contains(key, "=") { + panic(fmt.Errorf("wrong property key %q: key must not contain '='", key)) + } + writeString(key + "=" + value) + } + + config := ctx.Config() + + writeProp("ro.build.version.sdk", config.PlatformSdkVersion().String()) + writeProp("ro.build.version.preview_sdk", config.PlatformPreviewSdkVersion()) + writeProp("ro.build.version.codename", config.PlatformSdkCodename()) + writeProp("ro.build.version.all_codenames", strings.Join(config.PlatformVersionActiveCodenames(), ",")) + writeProp("ro.build.version.release", config.PlatformVersionLastStable()) + writeProp("ro.build.version.release_or_codename", config.PlatformVersionName()) + writeProp("ro.build.version.security_patch", config.PlatformSecurityPatch()) + writeProp("ro.build.version.base_os", config.PlatformBaseOS()) + writeProp("ro.build.version.min_supported_target_sdk", config.PlatformMinSupportedTargetSdkVersion()) + + if config.Eng() { + writeProp("ro.build.type", "eng") + } else if config.Debuggable() { + writeProp("ro.build.type", "userdebug") + } else { + writeProp("ro.build.type", "user") + } + + // Currently, only a few properties are implemented to unblock microdroid use case. + // TODO(b/189164487): support below properties as well and replace build/make/tools/buildinfo.sh + /* + if $BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT { + writeProp("ro.build.legacy.id", config.BuildID()) + } else { + writeProp("ro.build.id", config.BuildId()) + } + writeProp("ro.build.display.id", $BUILD_DISPLAY_ID) + writeProp("ro.build.version.incremental", $BUILD_NUMBER) + writeProp("ro.build.version.preview_sdk_fingerprint", $PLATFORM_PREVIEW_SDK_FINGERPRINT) + writeProp("ro.build.version.known_codenames", $PLATFORM_VERSION_KNOWN_CODENAMES) + writeProp("ro.build.version.release_or_preview_display", $PLATFORM_DISPLAY_VERSION) + writeProp("ro.build.date", `$DATE`) + writeProp("ro.build.date.utc", `$DATE +%s`) + writeProp("ro.build.user", $BUILD_USERNAME) + writeProp("ro.build.host", $BUILD_HOSTNAME) + writeProp("ro.build.tags", $BUILD_VERSION_TAGS) + writeProp("ro.build.flavor", $TARGET_BUILD_FLAVOR) + // These values are deprecated, use "ro.product.cpu.abilist" + // instead (see below). + writeString("# ro.product.cpu.abi and ro.product.cpu.abi2 are obsolete,") + writeString("# use ro.product.cpu.abilist instead.") + writeProp("ro.product.cpu.abi", $TARGET_CPU_ABI) + if [ -n "$TARGET_CPU_ABI2" ] { + writeProp("ro.product.cpu.abi2", $TARGET_CPU_ABI2) + } + + if [ -n "$PRODUCT_DEFAULT_LOCALE" ] { + writeProp("ro.product.locale", $PRODUCT_DEFAULT_LOCALE) + } + writeProp("ro.wifi.channels", $PRODUCT_DEFAULT_WIFI_CHANNELS) + writeString("# ro.build.product is obsolete; use ro.product.device") + writeProp("ro.build.product", $TARGET_DEVICE) + + writeString("# Do not try to parse description or thumbprint") + writeProp("ro.build.description", $PRIVATE_BUILD_DESC) + if [ -n "$BUILD_THUMBPRINT" ] { + writeProp("ro.build.thumbprint", $BUILD_THUMBPRINT) + } + */ + + writeString("# end build properties") + + cmd.Text("true) > ").Output(p.outputFilePath) + rule.Build("build.prop", "generating build.prop") + + if !p.installable() { + p.SkipInstall() + } + + p.installPath = PathForModuleInstall(ctx) + ctx.InstallFile(p.installPath, p.Name(), p.outputFilePath) +} + +func (f *buildinfoPropModule) GenerateSingletonBuildActions(ctx SingletonContext) { + // does nothing; buildinfo_prop is a singeton because two buildinfo modules don't make sense. +} + +func (p *buildinfoPropModule) AndroidMkEntries() []AndroidMkEntries { + return []AndroidMkEntries{AndroidMkEntries{ + Class: "ETC", + OutputFile: OptionalPathForPath(p.outputFilePath), + ExtraEntries: []AndroidMkExtraEntriesFunc{ + func(ctx AndroidMkExtraEntriesContext, entries *AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", p.installPath.String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.outputFilePath.Base()) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !p.installable()) + }, + }, + }} +} + +// buildinfo_prop module generates a build.prop file, which contains a set of common +// system/build.prop properties, such as ro.build.version.*. Not all properties are implemented; +// currently this module is only for microdroid. +func buildinfoPropFactory() SingletonModule { + module := &buildinfoPropModule{} + module.AddProperties(&module.properties) + InitAndroidModule(module) + return module +} diff --git a/android/config.go b/android/config.go index 9f1fd6a1b..ee058e809 100644 --- a/android/config.go +++ b/android/config.go @@ -777,6 +777,10 @@ func (c *config) PlatformBaseOS() string { return String(c.productVariables.Platform_base_os) } +func (c *config) PlatformVersionLastStable() string { + return String(c.productVariables.Platform_version_last_stable) +} + func (c *config) MinSupportedSdkVersion() ApiLevel { return uncheckedFinalApiLevel(19) } @@ -1482,6 +1486,10 @@ func (c *config) MissingUsesLibraries() []string { return c.productVariables.MissingUsesLibraries } +func (c *config) TargetMultitreeUpdateMeta() bool { + return c.productVariables.MultitreeUpdateMeta +} + func (c *deviceConfig) DeviceArch() string { return String(c.config.productVariables.DeviceArch) } diff --git a/android/hooks.go b/android/hooks.go index 5e3a4a7e7..2ad3b5fa5 100644 --- a/android/hooks.go +++ b/android/hooks.go @@ -89,8 +89,17 @@ func (l *loadHookContext) PrependProperties(props ...interface{}) { l.appendPrependHelper(props, proptools.PrependMatchingProperties) } -func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { - inherited := []interface{}{&l.Module().base().commonProperties} +func (l *loadHookContext) createModule(factory blueprint.ModuleFactory, name string, props ...interface{}) blueprint.Module { + return l.bp.CreateModule(factory, name, props...) +} + +type createModuleContext interface { + Module() Module + createModule(blueprint.ModuleFactory, string, ...interface{}) blueprint.Module +} + +func createModule(ctx createModuleContext, factory ModuleFactory, ext string, props ...interface{}) Module { + inherited := []interface{}{&ctx.Module().base().commonProperties} var typeName string if typeNameLookup, ok := ModuleTypeByFactory()[reflect.ValueOf(factory)]; ok { @@ -101,12 +110,12 @@ func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface filePath, _ := factoryFunc.FileLine(factoryPtr) typeName = fmt.Sprintf("%s_%s", path.Base(filePath), factoryFunc.Name()) } - typeName = typeName + "_loadHookModule" + typeName = typeName + "_" + ext - module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module) + module := ctx.createModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module) - if l.Module().base().variableProperties != nil && module.base().variableProperties != nil { - src := l.Module().base().variableProperties + if ctx.Module().base().variableProperties != nil && module.base().variableProperties != nil { + src := ctx.Module().base().variableProperties dst := []interface{}{ module.base().variableProperties, // Put an empty copy of the src properties into dst so that properties in src that are not in dst @@ -122,6 +131,10 @@ func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface return module } +func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { + return createModule(l, factory, "_loadHookModule", props...) +} + func (l *loadHookContext) registerScopedModuleType(name string, factory blueprint.ModuleFactory) { l.bp.RegisterScopedModuleType(name, factory) } diff --git a/android/mutator.go b/android/mutator.go index 739e4ee6d..02a614353 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -15,12 +15,9 @@ package android import ( - "reflect" - "android/soong/bazel" "github.com/google/blueprint" - "github.com/google/blueprint/proptools" ) // Phases: @@ -553,29 +550,16 @@ func (t *topDownMutatorContext) Rename(name string) { t.Module().base().commonProperties.DebugName = name } -func (t *topDownMutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { - inherited := []interface{}{&t.Module().base().commonProperties} - module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module) - - if t.Module().base().variableProperties != nil && module.base().variableProperties != nil { - src := t.Module().base().variableProperties - dst := []interface{}{ - module.base().variableProperties, - // Put an empty copy of the src properties into dst so that properties in src that are not in dst - // don't cause a "failed to find property to extend" error. - proptools.CloneEmptyProperties(reflect.ValueOf(src)).Interface(), - } - err := proptools.AppendMatchingProperties(dst, src, nil) - if err != nil { - panic(err) - } - } +func (t *topDownMutatorContext) createModule(factory blueprint.ModuleFactory, name string, props ...interface{}) blueprint.Module { + return t.bp.CreateModule(factory, name, props...) +} - return module +func (t *topDownMutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { + return createModule(t, factory, "_topDownMutatorModule", props...) } func (t *topDownMutatorContext) createModuleWithoutInheritance(factory ModuleFactory, props ...interface{}) Module { - module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), props...).(Module) + module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), "", props...).(Module) return module } diff --git a/android/paths.go b/android/paths.go index e7829b961..e0e5ae580 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1057,7 +1057,8 @@ func safePathForSource(ctx PathContext, pathComponents ...string) (SourcePath, e } // absolute path already checked by validateSafePath - if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) { + // special-case api surface gen files for now + if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") { return ret, fmt.Errorf("source path %q is in output", ret.String()) } @@ -1073,7 +1074,8 @@ func pathForSource(ctx PathContext, pathComponents ...string) (SourcePath, error } // absolute path already checked by validatePath - if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) { + // special-case for now + if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") { return ret, fmt.Errorf("source path %q is in output", ret.String()) } diff --git a/android/variable.go b/android/variable.go index 077b81097..90cb6ff15 100644 --- a/android/variable.go +++ b/android/variable.go @@ -199,6 +199,7 @@ type productVariables struct { Platform_preview_sdk_version *string `json:",omitempty"` Platform_min_supported_target_sdk_version *string `json:",omitempty"` Platform_base_os *string `json:",omitempty"` + Platform_version_last_stable *string `json:",omitempty"` DeviceName *string `json:",omitempty"` DeviceProduct *string `json:",omitempty"` @@ -351,6 +352,8 @@ type productVariables struct { RecoverySnapshotDirsIncluded []string `json:",omitempty"` HostFakeSnapshotEnabled bool `json:",omitempty"` + MultitreeUpdateMeta bool `json:",omitempty"` + BoardVendorSepolicyDirs []string `json:",omitempty"` BoardOdmSepolicyDirs []string `json:",omitempty"` BoardReqdMaskPolicy []string `json:",omitempty"` diff --git a/apex/Android.bp b/apex/Android.bp index 41224ecd5..d3417c24d 100644 --- a/apex/Android.bp +++ b/apex/Android.bp @@ -14,6 +14,7 @@ bootstrap_go_package { "soong-cc", "soong-filesystem", "soong-java", + "soong-multitree", "soong-provenance", "soong-python", "soong-rust", diff --git a/apex/apex.go b/apex/apex.go index 76af1b82e..f16b72d96 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -33,6 +33,7 @@ import ( prebuilt_etc "android/soong/etc" "android/soong/filesystem" "android/soong/java" + "android/soong/multitree" "android/soong/python" "android/soong/rust" "android/soong/sh" @@ -358,6 +359,7 @@ type apexBundle struct { android.OverridableModuleBase android.SdkBase android.BazelModuleBase + multitree.ExportableModuleBase // Properties properties apexBundleProperties @@ -1359,6 +1361,21 @@ func (a *apexBundle) OutputFiles(tag string) (android.Paths, error) { } } +var _ multitree.Exportable = (*apexBundle)(nil) + +func (a *apexBundle) Exportable() bool { + if a.properties.ApexType == flattenedApex { + return false + } + return true +} + +func (a *apexBundle) TaggedOutputs() map[string]android.Paths { + ret := make(map[string]android.Paths) + ret["apex"] = android.Paths{a.outputFile} + return ret +} + var _ cc.Coverage = (*apexBundle)(nil) // Implements cc.Coverage @@ -2372,6 +2389,7 @@ func newApexBundle() *apexBundle { android.InitSdkAwareModule(module) android.InitOverridableModule(module, &module.overridableProperties.Overrides) android.InitBazelModule(module) + multitree.InitExportableModule(module) return module } diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go index 3824586c6..a216c9d11 100644 --- a/bp2build/android_app_conversion_test.go +++ b/bp2build/android_app_conversion_test.go @@ -74,7 +74,8 @@ android_app { package_name: "com.google", resource_dirs: ["resa", "resb"], manifest: "manifest/AndroidManifest.xml", - static_libs: ["static_lib_dep"] + static_libs: ["static_lib_dep"], + java_version: "7", } `, expectedBazelTargets: []string{ @@ -87,6 +88,7 @@ android_app { ]`, "custom_package": `"com.google"`, "deps": `[":static_lib_dep"]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }}) } diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go index 4fc07e014..d7a76a89f 100644 --- a/bp2build/java_binary_host_conversion_test.go +++ b/bp2build/java_binary_host_conversion_test.go @@ -52,6 +52,7 @@ func TestJavaBinaryHost(t *testing.T) { jni_libs: ["jni-lib-1"], javacflags: ["-Xdoclint:all/protected"], bazel_module: { bp2build_available: true }, + java_version: "8", }`, expectedBazelTargets: []string{ makeBazelTarget("java_binary", "java-binary-host-1", attrNameToString{ @@ -59,7 +60,10 @@ func TestJavaBinaryHost(t *testing.T) { "main_class": `"com.android.test.MainClass"`, "deps": `["//other:jni-lib-1"]`, "jvm_flags": `["-Djava.library.path=$${RUNPATH}other"]`, - "javacopts": `["-Xdoclint:all/protected"]`, + "javacopts": `[ + "-Xdoclint:all/protected", + "-source 1.8 -target 1.8", + ]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index ccc52ef12..3b6636975 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -158,6 +158,22 @@ java_plugin { }) } +func TestJavaLibraryJavaVersion(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java"], + java_version: "11", +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "srcs": `["a.java"]`, + "javacopts": `["-source 11 -target 11"]`, + }), + }, + }) +} + func TestJavaLibraryErrorproneJavacflagsEnabledManually(t *testing.T) { runJavaLibraryTestCase(t, bp2buildTestCase{ blueprint: `java_library { diff --git a/bp2build/java_library_host_conversion_test.go b/bp2build/java_library_host_conversion_test.go index 73abdd2ea..83cc5519a 100644 --- a/bp2build/java_library_host_conversion_test.go +++ b/bp2build/java_library_host_conversion_test.go @@ -43,6 +43,7 @@ java_library_host { name: "java-lib-host-2", srcs: ["c.java"], bazel_module: { bp2build_available: true }, + java_version: "9", }`, expectedBazelTargets: []string{ makeBazelTarget("java_library", "java-lib-host-1", attrNameToString{ @@ -54,7 +55,8 @@ java_library_host { })`, }), makeBazelTarget("java_library", "java-lib-host-2", attrNameToString{ - "srcs": `["c.java"]`, + "javacopts": `["-source 1.9 -target 1.9"]`, + "srcs": `["c.java"]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_plugin_conversion_test.go b/bp2build/java_plugin_conversion_test.go index c2a2182bd..dc763e763 100644 --- a/bp2build/java_plugin_conversion_test.go +++ b/bp2build/java_plugin_conversion_test.go @@ -39,6 +39,7 @@ func TestJavaPlugin(t *testing.T) { libs: ["java-lib-1"], static_libs: ["java-lib-2"], bazel_module: { bp2build_available: true }, + java_version: "7", } java_library { @@ -66,6 +67,7 @@ java_library { "a.java", "b.java", ]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }, }) diff --git a/bp2build/java_proto_conversion_test.go b/bp2build/java_proto_conversion_test.go index 67f80446e..c6feeb8dc 100644 --- a/bp2build/java_proto_conversion_test.go +++ b/bp2build/java_proto_conversion_test.go @@ -102,6 +102,7 @@ func TestJavaProtoDefault(t *testing.T) { blueprint: `java_library_static { name: "java-protos", srcs: ["a.proto"], + java_version: "7", } `, expectedBazelTargets: []string{ @@ -115,7 +116,8 @@ func TestJavaProtoDefault(t *testing.T) { "deps": `[":java-protos_proto"]`, }), makeBazelTarget("java_library", "java-protos", attrNameToString{ - "exports": `[":java-protos_java_proto_lite"]`, + "exports": `[":java-protos_java_proto_lite"]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }, }) diff --git a/cc/Android.bp b/cc/Android.bp index 9103a48b4..60d329e62 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "soong-etc", "soong-fuzz", "soong-genrule", + "soong-multitree", "soong-snapshot", "soong-tradefed", ], @@ -65,6 +66,7 @@ bootstrap_go_package { "library.go", "library_headers.go", "library_sdk_member.go", + "library_stub.go", "native_bridge_sdk_trait.go", "object.go", "test.go", @@ -94,12 +96,14 @@ bootstrap_go_package { "gen_test.go", "genrule_test.go", "library_headers_test.go", + "library_stub_test.go", "library_test.go", "object_test.go", "prebuilt_test.go", "proto_test.go", "sanitize_test.go", "test_data_test.go", + "tidy_test.go", "vendor_public_library_test.go", "vendor_snapshot_test.go", ], @@ -1,4 +1,4 @@ per-file ndk_*.go = danalbert@google.com -per-file tidy.go = srhines@google.com, chh@google.com +per-file tidy*.go = srhines@google.com, chh@google.com per-file afdo.go,afdo_test.go,lto.go,pgo.go = srhines@google.com, pirama@google.com, yikong@google.com per-file coverage.go = pirama@google.com, srhines@google.com, allenhair@google.com diff --git a/cc/bp2build.go b/cc/bp2build.go index cc378b3b4..19855fab8 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -167,21 +167,17 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs.System_dynamic_deps.ForceSpecifyEmptyList = true if isStatic { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &StaticProperties{}) { - for config, props := range configToProps { - if staticOrSharedProps, ok := props.(*StaticProperties); ok { - setAttrs(axis, config, staticOrSharedProps.Static) - } + bp2BuildPropParseHelper(ctx, module, &StaticProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if staticOrSharedProps, ok := props.(*StaticProperties); ok { + setAttrs(axis, config, staticOrSharedProps.Static) } - } + }) } else { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &SharedProperties{}) { - for config, props := range configToProps { - if staticOrSharedProps, ok := props.(*SharedProperties); ok { - setAttrs(axis, config, staticOrSharedProps.Shared) - } + bp2BuildPropParseHelper(ctx, module, &SharedProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if staticOrSharedProps, ok := props.(*SharedProperties); ok { + setAttrs(axis, config, staticOrSharedProps.Shared) } - } + }) } partitionedSrcs := groupSrcsByExtension(ctx, attrs.Srcs) @@ -359,21 +355,18 @@ func (ca *compilerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversi } func (ca *compilerAttributes) convertStlProps(ctx android.ArchVariantContext, module *Module) { - stlPropsByArch := module.GetArchVariantProperties(ctx, &StlProperties{}) - for _, configToProps := range stlPropsByArch { - for _, props := range configToProps { - if stlProps, ok := props.(*StlProperties); ok { - if stlProps.Stl == nil { - continue - } - if ca.stl == nil { - ca.stl = stlProps.Stl - } else if ca.stl != stlProps.Stl { - ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl) - } + bp2BuildPropParseHelper(ctx, module, &StlProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if stlProps, ok := props.(*StlProperties); ok { + if stlProps.Stl == nil { + return + } + if ca.stl == nil { + ca.stl = stlProps.Stl + } else if ca.stl != stlProps.Stl { + ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl) } } - } + }) } func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) { @@ -713,17 +706,15 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion } func (la *linkerAttributes) convertStripProps(ctx android.BazelConversionPathContext, module *Module) { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &StripProperties{}) { - for config, props := range configToProps { - if stripProperties, ok := props.(*StripProperties); ok { - la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols) - la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list) - la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame) - la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All) - la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None) - } + bp2BuildPropParseHelper(ctx, module, &StripProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if stripProperties, ok := props.(*StripProperties); ok { + la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols) + la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list) + la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame) + la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All) + la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None) } - } + }) } func (la *linkerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) { @@ -859,18 +850,16 @@ func bp2BuildParseExportedIncludesHelper(ctx android.BazelConversionPathContext, } else { exported = BazelIncludes{} } - for axis, configToProps := range module.GetArchVariantProperties(ctx, &FlagExporterProperties{}) { - for config, props := range configToProps { - if flagExporterProperties, ok := props.(*FlagExporterProperties); ok { - if len(flagExporterProperties.Export_include_dirs) > 0 { - exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...))) - } - if len(flagExporterProperties.Export_system_include_dirs) > 0 { - exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...))) - } + bp2BuildPropParseHelper(ctx, module, &FlagExporterProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if flagExporterProperties, ok := props.(*FlagExporterProperties); ok { + if len(flagExporterProperties.Export_include_dirs) > 0 { + exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...))) + } + if len(flagExporterProperties.Export_system_include_dirs) > 0 { + exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...))) } } - } + }) exported.AbsoluteIncludes.DeduplicateAxesFromBase() exported.Includes.DeduplicateAxesFromBase() exported.SystemIncludes.DeduplicateAxesFromBase() @@ -938,22 +927,19 @@ type binaryLinkerAttrs struct { func bp2buildBinaryLinkerProps(ctx android.BazelConversionPathContext, m *Module) binaryLinkerAttrs { attrs := binaryLinkerAttrs{} - archVariantProps := m.GetArchVariantProperties(ctx, &BinaryLinkerProperties{}) - for axis, configToProps := range archVariantProps { - for _, p := range configToProps { - props := p.(*BinaryLinkerProperties) - staticExecutable := props.Static_executable - if axis == bazel.NoConfigAxis { - if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared { - attrs.Linkshared = &linkBinaryShared - } - } else if staticExecutable != nil { - // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a - // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling - ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values") + bp2BuildPropParseHelper(ctx, m, &BinaryLinkerProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + linkerProps := props.(*BinaryLinkerProperties) + staticExecutable := linkerProps.Static_executable + if axis == bazel.NoConfigAxis { + if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared { + attrs.Linkshared = &linkBinaryShared } + } else if staticExecutable != nil { + // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a + // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling + ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values") } - } + }) return attrs } diff --git a/cc/builder.go b/cc/builder.go index 525b1a14f..107cd58e7 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -855,13 +855,24 @@ func transformObjToDynamicBinary(ctx android.ModuleContext, deps = append(deps, crtBegin...) deps = append(deps, crtEnd...) + var depFile android.WritablePath + var depFileLdFlags string + depsType := blueprint.DepsNone + if !ctx.Windows() && !ctx.Darwin() { + // lld only supports --dependency-file for elf files + depFile = outputFile.ReplaceExtension(ctx, "d") + depFileLdFlags = " -Wl,--dependency-file=" + depFile.String() + depsType = blueprint.DepsGCC + implicitOutputs = append(implicitOutputs, depFile) + } + rule := ld args := map[string]string{ "ldCmd": ldCmd, "crtBegin": strings.Join(crtBegin.Strings(), " "), "libFlags": strings.Join(libFlagsList, " "), "extraLibFlags": flags.extraLibFlags, - "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags, + "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags + depFileLdFlags, "crtEnd": strings.Join(crtEnd.Strings(), " "), } if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { @@ -872,6 +883,8 @@ func transformObjToDynamicBinary(ctx android.ModuleContext, ctx.Build(pctx, android.BuildParams{ Rule: rule, + Deps: depsType, + Depfile: depFile, Description: "link " + outputFile.Base(), Output: outputFile, ImplicitOutputs: implicitOutputs, @@ -1025,18 +1038,33 @@ func transformObjsToObj(ctx android.ModuleContext, objFiles android.Paths, ldCmd := "${config.ClangBin}/clang++" + var implicitOutputs android.WritablePaths + var depFile android.WritablePath + var depFileLdFlags string + depsType := blueprint.DepsNone + if !ctx.Windows() && !ctx.Darwin() { + // lld only supports --dependency-file for elf files + depFile = outputFile.ReplaceExtension(ctx, "d") + depFileLdFlags = " -Wl,--dependency-file=" + depFile.String() + depsType = blueprint.DepsGCC + implicitOutputs = append(implicitOutputs, depFile) + } + rule := partialLd args := map[string]string{ "ldCmd": ldCmd, - "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags, + "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags + depFileLdFlags, } if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CXX_LINKS") { rule = partialLdRE args["inCommaList"] = strings.Join(objFiles.Strings(), ",") args["implicitInputs"] = strings.Join(deps.Strings(), ",") + args["implicitOutputs"] = strings.Join(implicitOutputs.Strings(), ",") } ctx.Build(pctx, android.BuildParams{ Rule: rule, + Deps: depsType, + Depfile: depFile, Description: "link " + outputFile.Base(), Output: outputFile, Inputs: objFiles, @@ -746,6 +746,7 @@ var ( runtimeDepTag = installDependencyTag{name: "runtime lib"} testPerSrcDepTag = dependencyTag{name: "test_per_src"} stubImplDepTag = dependencyTag{name: "stub_impl"} + JniFuzzLibTag = dependencyTag{name: "jni_fuzz_lib_tag"} ) func IsSharedDepTag(depTag blueprint.DependencyTag) bool { diff --git a/cc/config/global.go b/cc/config/global.go index 3caf32792..dc6310c81 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -286,8 +286,8 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r450784d" - ClangDefaultShortVersion = "14.0.6" + ClangDefaultVersion = "clang-r450784e" + ClangDefaultShortVersion = "14.0.7" // Directories with warnings from Android.bp files. WarningAllowedProjects = []string{ diff --git a/cc/libbuildversion/tests/Android.bp b/cc/libbuildversion/tests/Android.bp index 0e97fedff..c616a3351 100644 --- a/cc/libbuildversion/tests/Android.bp +++ b/cc/libbuildversion/tests/Android.bp @@ -35,6 +35,16 @@ cc_defaults { dir: "host/", }, }, + linux_musl_x86: { + dist: { + dir: "host32/", + }, + }, + linux_musl_x86_64: { + dist: { + dir: "host/", + }, + }, linux_glibc_x86: { dist: { dir: "host32/", diff --git a/cc/library_stub.go b/cc/library_stub.go new file mode 100644 index 000000000..4d0148df4 --- /dev/null +++ b/cc/library_stub.go @@ -0,0 +1,163 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "android/soong/android" + "android/soong/multitree" +) + +func init() { + RegisterLibraryStubBuildComponents(android.InitRegistrationContext) +} + +func RegisterLibraryStubBuildComponents(ctx android.RegistrationContext) { + // cc_api_stub_library shares a lot of ndk_library, and this will be refactored later + ctx.RegisterModuleType("cc_api_stub_library", CcApiStubLibraryFactory) + ctx.RegisterModuleType("cc_api_contribution", CcApiContributionFactory) +} + +func CcApiStubLibraryFactory() android.Module { + module, decorator := NewLibrary(android.DeviceSupported) + apiStubDecorator := &apiStubDecorator{ + libraryDecorator: decorator, + } + apiStubDecorator.BuildOnlyShared() + + module.compiler = apiStubDecorator + module.linker = apiStubDecorator + module.installer = nil + module.library = apiStubDecorator + module.Properties.HideFromMake = true // TODO: remove + + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth) + module.AddProperties(&module.Properties, + &apiStubDecorator.properties, + &apiStubDecorator.MutatedProperties, + &apiStubDecorator.apiStubLibraryProperties) + return module +} + +type apiStubLiraryProperties struct { + Imported_includes []string `android:"path"` +} + +type apiStubDecorator struct { + *libraryDecorator + properties libraryProperties + apiStubLibraryProperties apiStubLiraryProperties +} + +func (compiler *apiStubDecorator) stubsVersions(ctx android.BaseMutatorContext) []string { + firstVersion := String(compiler.properties.First_version) + return ndkLibraryVersions(ctx, android.ApiLevelOrPanic(ctx, firstVersion)) +} + +func (decorator *apiStubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { + if decorator.stubsVersion() == "" { + decorator.setStubsVersion("current") + } // TODO: fix + symbolFile := String(decorator.properties.Symbol_file) + nativeAbiResult := parseNativeAbiDefinition(ctx, symbolFile, + android.ApiLevelOrPanic(ctx, decorator.stubsVersion()), + "") + return compileStubLibrary(ctx, flags, nativeAbiResult.stubSrc) +} + +func (decorator *apiStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objects Objects) android.Path { + decorator.reexportDirs(android.PathsForModuleSrc(ctx, decorator.apiStubLibraryProperties.Imported_includes)...) + return decorator.libraryDecorator.link(ctx, flags, deps, objects) +} + +func init() { + pctx.HostBinToolVariable("gen_api_surface_build_files", "gen_api_surface_build_files") +} + +type CcApiContribution struct { + android.ModuleBase + properties ccApiContributionProperties +} + +type ccApiContributionProperties struct { + Symbol_file *string `android:"path"` + First_version *string + Export_include_dir *string +} + +func CcApiContributionFactory() android.Module { + module := &CcApiContribution{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + return module +} + +// Do some simple validations +// Majority of the build rules will be created in the ctx of the api surface this module contributes to +func (contrib *CcApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if contrib.properties.Symbol_file == nil { + ctx.PropertyErrorf("symbol_file", "%v does not have symbol file", ctx.ModuleName()) + } + if contrib.properties.First_version == nil { + ctx.PropertyErrorf("first_version", "%v does not have first_version for stub variants", ctx.ModuleName()) + } +} + +// Path is out/soong/.export/ but will be different in final multi-tree layout +func outPathApiSurface(ctx android.ModuleContext, myModuleName string, pathComponent string) android.OutputPath { + return android.PathForOutput(ctx, ".export", ctx.ModuleName(), myModuleName, pathComponent) +} + +func (contrib *CcApiContribution) CopyFilesWithTag(apiSurfaceContext android.ModuleContext) map[string]android.Paths { + // copy map.txt for now + // hardlinks cannot be created since nsjail creates a different mountpoint for out/ + myDir := apiSurfaceContext.OtherModuleDir(contrib) + genMapTxt := outPathApiSurface(apiSurfaceContext, contrib.Name(), String(contrib.properties.Symbol_file)) + apiSurfaceContext.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Description: "import map.txt file", + Input: android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Symbol_file)), + Output: genMapTxt, + }) + + outputs := make(map[string]android.Paths) + outputs["map"] = []android.Path{genMapTxt} + + if contrib.properties.Export_include_dir != nil { + includeDir := android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Export_include_dir)) + outputs["export_include_dir"] = []android.Path{includeDir} + } + return outputs +} + +var _ multitree.ApiContribution = (*CcApiContribution)(nil) + +/* +func (contrib *CcApiContribution) GenerateBuildFiles(apiSurfaceContext android.ModuleContext) android.Paths { + genAndroidBp := outPathApiSurface(apiSurfaceContext, contrib.Name(), "Android.bp") + + // generate Android.bp + apiSurfaceContext.Build(pctx, android.BuildParams{ + Rule: genApiSurfaceBuildFiles, + Description: "generate API surface build files", + Outputs: []android.WritablePath{genAndroidBp}, + Args: map[string]string{ + "name": contrib.Name() + "." + apiSurfaceContext.ModuleName(), //e.g. liblog.ndk + "symbol_file": String(contrib.properties.Symbol_file), + "first_version": String(contrib.properties.First_version), + }, + }) + return []android.Path{genAndroidBp} +} +*/ diff --git a/cc/library_stub_test.go b/cc/library_stub_test.go new file mode 100644 index 000000000..15b56d227 --- /dev/null +++ b/cc/library_stub_test.go @@ -0,0 +1,108 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + _ "fmt" + _ "sort" + + "testing" + + "android/soong/android" + "android/soong/multitree" +) + +func TestCcApiStubLibraryOutputFiles(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := prepareForCcTest.RunTestWithBp(t, bp) + outputs := result.ModuleForTests("foo", "android_arm64_armv8-a_shared").AllOutputs() + expected_file_suffixes := []string{".c", "stub.map", ".o", ".so"} + for _, expected_file_suffix := range expected_file_suffixes { + android.AssertBoolEquals(t, expected_file_suffix+" file not found in output", true, android.SuffixInList(outputs, expected_file_suffix)) + } +} + +func TestCcApiStubLibraryVariants(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := prepareForCcTest.RunTestWithBp(t, bp) + variants := result.ModuleVariantsForTests("foo") + expected_variants := []string{"29", "30", "S", "Tiramisu"} //TODO: make this test deterministic by using fixtures + for _, expected_variant := range expected_variants { + android.AssertBoolEquals(t, expected_variant+" variant not found in foo", true, android.SubstringInList(variants, expected_variant)) + } +} + +func TestCcLibraryUsesCcApiStubLibrary(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + cc_library { + name: "foo_user", + shared_libs: [ + "foo#29", + ], + } + + ` + prepareForCcTest.RunTestWithBp(t, bp) +} + +func TestApiSurfaceOutputs(t *testing.T) { + bp := ` + api_surface { + name: "mysdk", + contributions: [ + "foo", + ], + } + + cc_api_contribution { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := android.GroupFixturePreparers( + prepareForCcTest, + multitree.PrepareForTestWithApiSurface, + ).RunTestWithBp(t, bp) + mysdk := result.ModuleForTests("mysdk", "") + + actual_surface_inputs := mysdk.Rule("phony").BuildParams.Inputs.Strings() + expected_file_suffixes := []string{"mysdk/foo/foo.map.txt"} + for _, expected_file_suffix := range expected_file_suffixes { + android.AssertBoolEquals(t, expected_file_suffix+" file not found in input", true, android.SuffixInList(actual_surface_inputs, expected_file_suffix)) + } + + // check args/inputs to rule + /*api_surface_gen_rule_args := result.ModuleForTests("mysdk", "").Rule("genApiSurfaceBuildFiles").Args + android.AssertStringEquals(t, "name", "foo.mysdk", api_surface_gen_rule_args["name"]) + android.AssertStringEquals(t, "symbol_file", "foo.map.txt", api_surface_gen_rule_args["symbol_file"])*/ +} diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 5ef41eae5..c031b14ca 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -93,7 +93,7 @@ var ( type libraryProperties struct { // Relative path to the symbol map. // An example file can be seen here: TODO(danalbert): Make an example. - Symbol_file *string + Symbol_file *string `android:"path"` // The first API level a library was available. A library will be generated // for every API level beginning with this one. diff --git a/cc/sanitize.go b/cc/sanitize.go index 814fef6a9..53169de48 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -969,6 +969,22 @@ func sanitizerDepsMutator(t SanitizerType) func(android.TopDownMutatorContext) { }) } } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok { + // If it's a Java module with native dependencies through jni, + // set the sanitizer for them + if jniSanitizeable, ok := mctx.Module().(JniSanitizeable); ok { + if jniSanitizeable.IsSanitizerEnabledForJni(mctx, t.name()) { + mctx.VisitDirectDeps(func(child android.Module) { + if c, ok := child.(PlatformSanitizeable); ok && + mctx.OtherModuleDependencyTag(child) == JniFuzzLibTag && + c.SanitizePropDefined() && + !c.SanitizeNever() && + !c.IsSanitizerExplicitlyDisabled(t) { + c.SetSanitizeDep(true) + } + }) + } + } + // If an APEX module includes a lib which is enabled for a sanitizer T, then // the APEX module is also enabled for the same sanitizer type. mctx.VisitDirectDeps(func(child android.Module) { @@ -1280,6 +1296,11 @@ type Sanitizeable interface { AddSanitizerDependencies(ctx android.BottomUpMutatorContext, sanitizerName string) } +type JniSanitizeable interface { + android.Module + IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool +} + func (c *Module) MinimalRuntimeDep() bool { return c.sanitize.Properties.MinimalRuntimeDep } @@ -1407,7 +1428,7 @@ func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { } c.SetSanitizeDep(false) } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok && sanitizeable.IsSanitizerEnabled(mctx, t.name()) { - // APEX modules fall here + // APEX and Java fuzz modules fall here sanitizeable.AddSanitizerDependencies(mctx, t.name()) mctx.CreateVariations(t.variationName()) } else if c, ok := mctx.Module().(*Module); ok { diff --git a/cc/testing.go b/cc/testing.go index 32f7c6080..ecdae8b15 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -29,6 +29,7 @@ func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) { RegisterBinaryBuildComponents(ctx) RegisterLibraryBuildComponents(ctx) RegisterLibraryHeadersBuildComponents(ctx) + RegisterLibraryStubBuildComponents(ctx) ctx.RegisterModuleType("cc_benchmark", BenchmarkFactory) ctx.RegisterModuleType("cc_object", ObjectFactory) diff --git a/cc/tidy.go b/cc/tidy.go index 750e9de1e..03e967d7c 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -76,9 +76,10 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { // the global WITH_TIDY or module 'tidy' property is true. flags.Tidy = true - // If explicitly enabled, by global default or local tidy property, + // If explicitly enabled, by global WITH_TIDY or local tidy:true property, // set flags.NeedTidyFiles to make this module depend on .tidy files. - if ctx.Config().ClangTidy() || Bool(tidy.Properties.Tidy) { + // Note that locally set tidy:true is ignored if ALLOW_LOCAL_TIDY_TRUE is not set to true. + if ctx.Config().IsEnvTrue("WITH_TIDY") || (ctx.Config().IsEnvTrue("ALLOW_LOCAL_TIDY_TRUE") && Bool(tidy.Properties.Tidy)) { flags.NeedTidyFiles = true } diff --git a/cc/tidy_test.go b/cc/tidy_test.go new file mode 100644 index 000000000..339b30281 --- /dev/null +++ b/cc/tidy_test.go @@ -0,0 +1,98 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "fmt" + "testing" + + "android/soong/android" +) + +func TestWithTidy(t *testing.T) { + // When WITH_TIDY=1 or (ALLOW_LOCAL_TIDY_TRUE=1 and local tidy:true) + // a C++ library should depend on .tidy files. + testCases := []struct { + withTidy, allowLocalTidyTrue string // "_" means undefined + needTidyFile []bool // for {libfoo_0, libfoo_1} and {libbar_0, libbar_1} + }{ + {"_", "_", []bool{false, false, false}}, + {"_", "0", []bool{false, false, false}}, + {"_", "1", []bool{false, true, false}}, + {"_", "true", []bool{false, true, false}}, + {"0", "_", []bool{false, false, false}}, + {"0", "1", []bool{false, true, false}}, + {"1", "_", []bool{true, true, false}}, + {"1", "false", []bool{true, true, false}}, + {"1", "1", []bool{true, true, false}}, + {"true", "_", []bool{true, true, false}}, + } + bp := ` + cc_library_shared { + name: "libfoo_0", // depends on .tidy if WITH_TIDY=1 + srcs: ["foo.c"], + } + cc_library_shared { // depends on .tidy if WITH_TIDY=1 or ALLOW_LOCAL_TIDY_TRUE=1 + name: "libfoo_1", + srcs: ["foo.c"], + tidy: true, + } + cc_library_shared { // no .tidy + name: "libfoo_2", + srcs: ["foo.c"], + tidy: false, + } + cc_library_static { + name: "libbar_0", // depends on .tidy if WITH_TIDY=1 + srcs: ["bar.c"], + } + cc_library_static { // depends on .tidy if WITH_TIDY=1 or ALLOW_LOCAL_TIDY_TRUE=1 + name: "libbar_1", + srcs: ["bar.c"], + tidy: true, + } + cc_library_static { // no .tidy + name: "libbar_2", + srcs: ["bar.c"], + tidy: false, + }` + for index, test := range testCases { + testName := fmt.Sprintf("case%d,%v,%v", index, test.withTidy, test.allowLocalTidyTrue) + t.Run(testName, func(t *testing.T) { + testEnv := map[string]string{} + if test.withTidy != "_" { + testEnv["WITH_TIDY"] = test.withTidy + } + if test.allowLocalTidyTrue != "_" { + testEnv["ALLOW_LOCAL_TIDY_TRUE"] = test.allowLocalTidyTrue + } + ctx := android.GroupFixturePreparers(prepareForCcTest, android.FixtureMergeEnv(testEnv)).RunTestWithBp(t, bp) + for n := 0; n < 3; n++ { + checkLibraryRule := func(foo, variant, ruleName string) { + libName := fmt.Sprintf("lib%s_%d", foo, n) + tidyFile := "out/soong/.intermediates/" + libName + "/" + variant + "/obj/" + foo + ".tidy" + depFiles := ctx.ModuleForTests(libName, variant).Rule(ruleName).Validations.Strings() + if test.needTidyFile[n] { + android.AssertStringListContains(t, libName+" needs .tidy file", depFiles, tidyFile) + } else { + android.AssertStringListDoesNotContain(t, libName+" does not need .tidy file", depFiles, tidyFile) + } + } + checkLibraryRule("foo", "android_arm64_armv8-a_shared", "ld") + checkLibraryRule("bar", "android_arm64_armv8-a_static", "ar") + } + }) + } +} diff --git a/cc/util.go b/cc/util.go index b256b9ada..4e1003739 100644 --- a/cc/util.go +++ b/cc/util.go @@ -15,9 +15,7 @@ package cc import ( - "fmt" "path/filepath" - "regexp" "strings" "android/soong/android" @@ -30,30 +28,12 @@ func includeDirsToFlags(dirs android.Paths) string { return android.JoinWithPrefix(dirs.Strings(), "-I") } -func ldDirsToFlags(dirs []string) string { - return android.JoinWithPrefix(dirs, "-L") -} - -func libNamesToFlags(names []string) string { - return android.JoinWithPrefix(names, "-l") -} - var indexList = android.IndexList var inList = android.InList var filterList = android.FilterList var removeListFromList = android.RemoveListFromList var removeFromList = android.RemoveFromList -var libNameRegexp = regexp.MustCompile(`^lib(.*)$`) - -func moduleToLibName(module string) (string, error) { - matches := libNameRegexp.FindStringSubmatch(module) - if matches == nil { - return "", fmt.Errorf("Library module name %s does not start with lib", module) - } - return matches[1], nil -} - func flagsToBuilderFlags(in Flags) builderFlags { return builderFlags{ globalCommonFlags: strings.Join(in.Global.CommonFlags, " "), @@ -113,13 +93,6 @@ func addPrefix(list []string, prefix string) []string { return list } -func addSuffix(list []string, suffix string) []string { - for i := range list { - list[i] = list[i] + suffix - } - return list -} - // linkDirOnDevice/linkName -> target func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) string { dir := filepath.Join("$(PRODUCT_OUT)", linkDirOnDevice) diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 36513b64b..d0a6a39df 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -678,6 +678,9 @@ func toJsonClassLoaderContext(clcMap ClassLoaderContextMap) jsonClassLoaderConte jClcMap := make(jsonClassLoaderContextMap) for sdkVer, clcs := range clcMap { sdkVerStr := fmt.Sprintf("%d", sdkVer) + if sdkVer == AnySdkVersion { + sdkVerStr = "any" + } jClcMap[sdkVerStr] = toJsonClassLoaderContextRec(clcs) } return jClcMap diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index 5d3a9d943..614681f50 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -389,6 +389,38 @@ func TestCLCMExcludeLibs(t *testing.T) { }) } +// Test that CLC is correctly serialized to JSON. +func TestCLCtoJSON(t *testing.T) { + ctx := testContext() + optional := false + implicit := true + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Implicit": true, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [] + } + ], + "any": [ + { + "Name": "b", + "Optional": false, + "Implicit": true, + "Host": "out/soong/b.jar", + "Device": "/system/b.jar", + "Subcontexts": [] + } + ] +}`, m.Dump()) +} + func checkError(t *testing.T, have error, want string) { if have == nil { t.Errorf("\nwant error: '%s'\nhave: none", want) diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 153b0256f..ab9e418a0 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -36,8 +36,6 @@ type GlobalConfig struct { PreoptWithUpdatableBcp bool // If updatable boot jars are included in dexpreopt or not. - UseArtImage bool // use the art image (use other boot class path dex files without image) - HasSystemOther bool // store odex files that match PatternsOnSystemOther on the system_other partition PatternsOnSystemOther []string // patterns (using '%' to denote a prefix match) to put odex on the system_other partition diff --git a/fuzz/fuzz_common.go b/fuzz/fuzz_common.go index 89f81879f..700cdf0a7 100644 --- a/fuzz/fuzz_common.go +++ b/fuzz/fuzz_common.go @@ -82,6 +82,9 @@ type FuzzConfig struct { Hwasan_options []string `json:"hwasan_options,omitempty"` // Additional options to be passed to HWASAN when running on host in Haiku. Asan_options []string `json:"asan_options,omitempty"` + // If there's a Java fuzzer with JNI, a different version of Jazzer would + // need to be added to the fuzzer package than one without JNI + IsJni *bool `json:"is_jni,omitempty"` } type FuzzProperties struct { diff --git a/java/androidmk.go b/java/androidmk.go index 80b828d45..f6ea6a930 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -132,6 +132,16 @@ func (library *Library) AndroidMkEntries() []android.AndroidMkEntries { return entriesList } +func (j *JavaFuzzLibrary) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := j.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", "null-suite") + androidMkWriteTestData(j.jniFilePaths, entries) + }) + return entriesList +} + // Called for modules that are a component of a test suite. func testSuiteComponent(entries *android.AndroidMkEntries, test_suites []string, perTestcaseDirectory bool) { entries.SetString("LOCAL_MODULE_TAGS", "tests") diff --git a/java/base.go b/java/base.go index b925350a0..4932c4831 100644 --- a/java/base.go +++ b/java/base.go @@ -442,9 +442,6 @@ type Module struct { // manifest file to use instead of properties.Manifest overrideManifest android.OptionalPath - // map of SDK version to class loader context - classLoaderContexts dexpreopt.ClassLoaderContextMap - // list of plugins that this java module is exporting exportedPluginJars android.Paths @@ -1481,11 +1478,30 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { } if ctx.Device() { - lintSDKVersionString := func(sdkSpec android.SdkSpec) string { + lintSDKVersion := func(sdkSpec android.SdkSpec) int { if v := sdkSpec.ApiLevel; !v.IsPreview() { - return v.String() + return v.FinalInt() } else { - return ctx.Config().DefaultAppTargetSdk(ctx).String() + // When running metalava, we pass --version-codename. When that value + // is not REL, metalava will add 1 to the --current-version argument. + // On old branches, PLATFORM_SDK_VERSION is the latest version (for that + // branch) and the codename is REL, except potentially on the most + // recent non-master branch. On that branch, it goes through two other + // phases before it gets to the phase previously described: + // - PLATFORM_SDK_VERSION has not been updated yet, and the codename + // is not rel. This happens for most of the internal branch's life + // while the branch has been cut but is still under active development. + // - PLATFORM_SDK_VERSION has been set, but the codename is still not + // REL. This happens briefly during the release process. During this + // state the code to add --current-version is commented out, and then + // that commenting out is reverted after the codename is set to REL. + // On the master branch, the PLATFORM_SDK_VERSION always represents a + // prior version and the codename is always non-REL. + // + // We need to add one here to match metalava adding 1. Technically + // this means that in the state described in the second bullet point + // above, this number is 1 higher than it should be. + return ctx.Config().PlatformSdkVersion().FinalInt() + 1 } } @@ -1494,9 +1510,9 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.linter.srcJars = srcJars j.linter.classpath = append(append(android.Paths(nil), flags.bootClasspath...), flags.classpath...) j.linter.classes = j.implementationJarFile - j.linter.minSdkVersion = lintSDKVersionString(j.MinSdkVersion(ctx)) - j.linter.targetSdkVersion = lintSDKVersionString(j.TargetSdkVersion(ctx)) - j.linter.compileSdkVersion = lintSDKVersionString(j.SdkVersion(ctx)) + j.linter.minSdkVersion = lintSDKVersion(j.MinSdkVersion(ctx)) + j.linter.targetSdkVersion = lintSDKVersion(j.TargetSdkVersion(ctx)) + j.linter.compileSdkVersion = lintSDKVersion(j.SdkVersion(ctx)) j.linter.compileSdkKind = j.SdkVersion(ctx).Kind j.linter.javaLanguageLevel = flags.javaVersion.String() j.linter.kotlinLanguageLevel = "1.3" diff --git a/java/config/config.go b/java/config/config.go index 46c91a2f6..d7440022f 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -68,6 +68,11 @@ var ( "-J-XX:+TieredCompilation", "-J-XX:TieredStopAtLevel=1", } + dexerJavaVmFlagsList = []string{ + `-JXX:OnError="cat hs_err_pid%p.log"`, + "-JXX:CICompilerCount=6", + "-JXX:+UseDynamicNumberOfGCThreads", + } ) func init() { @@ -83,19 +88,14 @@ func init() { // D8 invocations are shorter lived, so we restrict their JIT tiering relative to R8. // Note that the `-JXX` prefix syntax is specific to the R8/D8 invocation wrappers. - exportedVars.ExportStringListStaticVariable("D8Flags", []string{ - `-JXX:OnError="cat hs_err_pid%p.log"`, - "-JXX:CICompilerCount=6", - "-JXX:+UseDynamicNumberOfGCThreads", + exportedVars.ExportStringListStaticVariable("D8Flags", append([]string{ + "-JXmx2048M", "-JXX:+TieredCompilation", "-JXX:TieredStopAtLevel=1", - }) - - exportedVars.ExportStringListStaticVariable("R8Flags", []string{ - `-JXX:OnError="cat hs_err_pid%p.log"`, - "-JXX:CICompilerCount=6", - "-JXX:+UseDynamicNumberOfGCThreads", - }) + }, dexerJavaVmFlagsList...)) + exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ + "-JXmx2048M", + }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{ `-Xmaxerrs 9999999`, diff --git a/java/config/makevars.go b/java/config/makevars.go index bc6848fc3..273aca0b4 100644 --- a/java/config/makevars.go +++ b/java/config/makevars.go @@ -43,9 +43,10 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("JAVADOC", "${JavadocCmd}") ctx.Strict("COMMON_JDK_FLAGS", "${CommonJdkFlags}") - ctx.Strict("DX", "${D8Cmd}") - ctx.Strict("DX_COMMAND", "${D8Cmd} -JXms16M -JXmx2048M") - ctx.Strict("R8_COMPAT_PROGUARD", "${R8Cmd}") + ctx.Strict("D8", "${D8Cmd}") + ctx.Strict("R8", "${R8Cmd}") + ctx.Strict("D8_COMMAND", "${D8Cmd} ${D8Flags}") + ctx.Strict("R8_COMMAND", "${R8Cmd} ${R8Flags}") ctx.Strict("TURBINE", "${TurbineJar}") @@ -78,9 +79,6 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("CLASS2NONSDKLIST", "${Class2NonSdkList}") ctx.Strict("HIDDENAPI", "${HiddenAPI}") - ctx.Strict("D8_FLAGS", "${D8Flags}") - ctx.Strict("R8_FLAGS", "${R8Flags}") - ctx.Strict("AIDL", "${AidlCmd}") ctx.Strict("AAPT2", "${Aapt2Cmd}") ctx.Strict("ZIPALIGN", "${ZipAlign}") diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 7c5f055e6..0adaf9917 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -259,10 +259,6 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr isSystemServerJar := global.AllSystemServerJars(ctx).ContainsJar(moduleName(ctx)) bootImage := defaultBootImageConfig(ctx) - if global.UseArtImage { - bootImage = artBootImageConfig(ctx) - } - dexFiles, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) targets := ctx.MultiTargets() diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 3d91aec91..7c4da3ef6 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -213,12 +213,6 @@ import ( // writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names, // paths and so on. // -// 2.5. JIT-Zygote configuration -// ----------------------------- -// -// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling -// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage). -// var artApexNames = []string{ "com.android.art", @@ -938,11 +932,8 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(dexLocations, " ")) var imageNames []string - // TODO: the primary ART boot image should not be exposed to Make, as it is installed in a - // different way as a part of the ART APEX. However, there is a special JIT-Zygote build - // configuration which uses the primary ART image instead of the Framework boot image - // extension, and it relies on the ART image being exposed to Make. To fix this, it is - // necessary to rework the logic in makefiles. + // The primary ART boot image is exposed to Make for testing (gtests) and benchmarking + // (golem) purposes. for _, current := range append(d.otherImages, image) { imageNames = append(imageNames, current.name) for _, variant := range current.variants { diff --git a/java/fuzz.go b/java/fuzz.go index 257f34356..584c80b0c 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -15,14 +15,25 @@ package java import ( - "github.com/google/blueprint/proptools" "sort" "strings" + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + "android/soong/android" + "android/soong/cc" "android/soong/fuzz" ) +type jniProperties struct { + // list of jni libs + Jni_libs []string + + // sanitization + Sanitizers []string +} + func init() { RegisterJavaFuzzBuildComponents(android.InitRegistrationContext) } @@ -35,11 +46,60 @@ func RegisterJavaFuzzBuildComponents(ctx android.RegistrationContext) { type JavaFuzzLibrary struct { Library fuzzPackagedModule fuzz.FuzzPackagedModule + jniProperties jniProperties + jniFilePaths android.Paths } -func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { - j.Library.GenerateAndroidBuildActions(ctx) +// IsSanitizerEnabled implemented to make JavaFuzzLibrary implement +// cc.Sanitizeable +func (j *JavaFuzzLibrary) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool { + for _, s := range j.jniProperties.Sanitizers { + if sanitizerName == s { + return true + } + } + return false +} + +// IsSanitizerEnabledForJni implemented to make JavaFuzzLibrary implement +// cc.JniSanitizeable. It returns a bool for whether a cc dependency should be +// sanitized for the given sanitizer or not. +func (j *JavaFuzzLibrary) IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool { + return j.IsSanitizerEnabled(ctx, sanitizerName) +} + +// EnableSanitizer implemented to make JavaFuzzLibrary implement +// cc.Sanitizeable +func (j *JavaFuzzLibrary) EnableSanitizer(sanitizerName string) { +} + +// AddSanitizerDependencies implemented to make JavaFuzzLibrary implement +// cc.Sanitizeable +func (j *JavaFuzzLibrary) AddSanitizerDependencies(mctx android.BottomUpMutatorContext, sanitizerName string) { +} + +// To verify that JavaFuzzLibrary implements cc.Sanitizeable +var _ cc.Sanitizeable = (*JavaFuzzLibrary)(nil) + +func (j *JavaFuzzLibrary) DepsMutator(mctx android.BottomUpMutatorContext) { + if len(j.jniProperties.Jni_libs) > 0 { + if j.fuzzPackagedModule.FuzzProperties.Fuzz_config == nil { + config := &fuzz.FuzzConfig{} + j.fuzzPackagedModule.FuzzProperties.Fuzz_config = config + } + // this will be used by the ingestion pipeline to determine the version + // of jazzer to add to the fuzzer package + j.fuzzPackagedModule.FuzzProperties.Fuzz_config.IsJni = proptools.BoolPtr(true) + + for _, target := range mctx.MultiTargets() { + sharedLibVariations := append(target.Variations(), blueprint.Variation{Mutator: "link", Variation: "shared"}) + mctx.AddFarVariationDependencies(sharedLibVariations, cc.JniFuzzLibTag, j.jniProperties.Jni_libs...) + } + } + j.Library.DepsMutator(mctx) +} +func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { if j.fuzzPackagedModule.FuzzProperties.Corpus != nil { j.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, j.fuzzPackagedModule.FuzzProperties.Corpus) } @@ -55,6 +115,23 @@ func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) android.WriteFileRule(ctx, configPath, j.fuzzPackagedModule.FuzzProperties.Fuzz_config.String()) j.fuzzPackagedModule.Config = configPath } + + ctx.VisitDirectDepsWithTag(cc.JniFuzzLibTag, func(dep android.Module) { + sharedLibInfo := ctx.OtherModuleProvider(dep, cc.SharedLibraryInfoProvider).(cc.SharedLibraryInfo) + if sharedLibInfo.SharedLibrary != nil { + libPath := android.PathForModuleOut(ctx, sharedLibInfo.SharedLibrary.Base()) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: sharedLibInfo.SharedLibrary, + Output: libPath, + }) + j.jniFilePaths = append(j.jniFilePaths, libPath) + } else { + ctx.PropertyErrorf("jni_libs", "%q of type %q is not supported", dep.Name(), ctx.OtherModuleType(dep)) + } + }) + + j.Library.GenerateAndroidBuildActions(ctx) } // java_fuzz builds and links sources into a `.jar` file for the host. @@ -65,7 +142,8 @@ func FuzzFactory() android.Module { module := &JavaFuzzLibrary{} module.addHostProperties() - module.Module.properties.Installable = proptools.BoolPtr(false) + module.AddProperties(&module.jniProperties) + module.Module.properties.Installable = proptools.BoolPtr(true) module.AddProperties(&module.fuzzPackagedModule.FuzzProperties) // java_fuzz packaging rules collide when both linux_glibc and linux_bionic are enabled, disable the linux_bionic variants. @@ -83,7 +161,7 @@ func FuzzFactory() android.Module { module.initModuleAndImport(module) android.InitSdkAwareModule(module) - InitJavaModule(module, android.HostSupported) + InitJavaModuleMultiTargets(module, android.HostSupported) return module } @@ -106,26 +184,26 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { ctx.VisitAllModules(func(module android.Module) { // Discard non-fuzz targets. - javaModule, ok := module.(*JavaFuzzLibrary) + javaFuzzModule, ok := module.(*JavaFuzzLibrary) if !ok { return } fuzzModuleValidator := fuzz.FuzzModule{ - javaModule.ModuleBase, - javaModule.DefaultableModuleBase, - javaModule.ApexModuleBase, + javaFuzzModule.ModuleBase, + javaFuzzModule.DefaultableModuleBase, + javaFuzzModule.ApexModuleBase, } - if ok := fuzz.IsValid(fuzzModuleValidator); !ok || *javaModule.Module.properties.Installable { + if ok := fuzz.IsValid(fuzzModuleValidator); !ok { return } hostOrTargetString := "target" - if javaModule.Host() { + if javaFuzzModule.Host() { hostOrTargetString = "host" } - archString := javaModule.Arch().ArchType.String() + archString := javaFuzzModule.Arch().ArchType.String() archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) archOs := fuzz.ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()} @@ -134,12 +212,17 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { builder := android.NewRuleBuilder(pctx, ctx) // Package the artifacts (data, corpus, config and dictionary into a zipfile. - files = s.PackageArtifacts(ctx, module, javaModule.fuzzPackagedModule, archDir, builder) + files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder) // Add .jar - files = append(files, fuzz.FileToZip{javaModule.outputFile, ""}) + files = append(files, fuzz.FileToZip{javaFuzzModule.outputFile, ""}) + + // Add jni .so files + for _, fPath := range javaFuzzModule.jniFilePaths { + files = append(files, fuzz.FileToZip{fPath, ""}) + } - archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaModule.fuzzPackagedModule, files, builder, archDir, archString, "host", archOs, archDirs) + archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaFuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs) if !ok { return } diff --git a/java/fuzz_test.go b/java/fuzz_test.go index cf063ebe2..0a2c94517 100644 --- a/java/fuzz_test.go +++ b/java/fuzz_test.go @@ -15,13 +15,17 @@ package java import ( - "android/soong/android" "path/filepath" + "runtime" "testing" + + "android/soong/android" + "android/soong/cc" ) var prepForJavaFuzzTest = android.GroupFixturePreparers( PrepareForTestWithJavaDefaultModules, + cc.PrepareForTestWithCcBuildComponents, android.FixtureRegisterWithContext(RegisterJavaFuzzBuildComponents), ) @@ -32,6 +36,13 @@ func TestJavaFuzz(t *testing.T) { srcs: ["a.java"], libs: ["bar"], static_libs: ["baz"], + jni_libs: [ + "libjni", + ], + sanitizers: [ + "address", + "fuzzer", + ], } java_library_host { @@ -42,11 +53,21 @@ func TestJavaFuzz(t *testing.T) { java_library_host { name: "baz", srcs: ["c.java"], - }`) + } + + cc_library_shared { + name: "libjni", + host_supported: true, + device_supported: false, + stl: "none", + } + `) osCommonTarget := result.Config.BuildOSCommonTarget.String() - javac := result.ModuleForTests("foo", osCommonTarget).Rule("javac") - combineJar := result.ModuleForTests("foo", osCommonTarget).Description("for javac") + + osCommonTargetWithSan := osCommonTarget + "_asan" + "_fuzzer" + javac := result.ModuleForTests("foo", osCommonTargetWithSan).Rule("javac") + combineJar := result.ModuleForTests("foo", osCommonTargetWithSan).Description("for javac") if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" { t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs) @@ -62,4 +83,18 @@ func TestJavaFuzz(t *testing.T) { if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != baz { t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, baz) } + + ctx := result.TestContext + foo := ctx.ModuleForTests("foo", osCommonTargetWithSan).Module().(*JavaFuzzLibrary) + + expected := "libjni.so" + if runtime.GOOS == "darwin" { + expected = "libjni.dylib" + } + + fooJniFilePaths := foo.jniFilePaths + if len(fooJniFilePaths) != 1 || fooJniFilePaths[0].Rel() != expected { + t.Errorf(`expected foo test data relative path [%q], got %q`, + expected, fooJniFilePaths.Strings()) + } } diff --git a/java/java.go b/java/java.go index b34d6de8a..4e7e14c07 100644 --- a/java/java.go +++ b/java/java.go @@ -2089,6 +2089,11 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) if m.properties.Javacflags != nil { javacopts = append(javacopts, m.properties.Javacflags...) } + if m.properties.Java_version != nil { + javaVersion := normalizeJavaVersion(ctx, *m.properties.Java_version).String() + javacopts = append(javacopts, fmt.Sprintf("-source %s -target %s", javaVersion, javaVersion)) + } + epEnabled := m.properties.Errorprone.Enabled //TODO(b/227504307) add configuration that depends on RUN_ERROR_PRONE environment variable if Bool(epEnabled) { diff --git a/java/lint.go b/java/lint.go index e97c9c225..f09db955d 100644 --- a/java/lint.go +++ b/java/lint.go @@ -17,6 +17,7 @@ package java import ( "fmt" "sort" + "strconv" "strings" "github.com/google/blueprint/proptools" @@ -75,9 +76,9 @@ type linter struct { extraLintCheckJars android.Paths test bool library bool - minSdkVersion string - targetSdkVersion string - compileSdkVersion string + minSdkVersion int + targetSdkVersion int + compileSdkVersion int compileSdkKind android.SdkKind javaLanguageLevel string kotlinLanguageLevel string @@ -299,7 +300,7 @@ func (l *linter) generateManifest(ctx android.ModuleContext, rule *android.RuleB Text(`echo "<?xml version='1.0' encoding='utf-8'?>" &&`). Text(`echo "<manifest xmlns:android='http://schemas.android.com/apk/res/android'" &&`). Text(`echo " android:versionCode='1' android:versionName='1' >" &&`). - Textf(`echo " <uses-sdk android:minSdkVersion='%s' android:targetSdkVersion='%s'/>" &&`, + Textf(`echo " <uses-sdk android:minSdkVersion='%d' android:targetSdkVersion='%d'/>" &&`, l.minSdkVersion, l.targetSdkVersion). Text(`echo "</manifest>"`). Text(") >").Output(manifestPath) @@ -427,7 +428,7 @@ func (l *linter) lint(ctx android.ModuleContext) { FlagWithOutput("--html ", html). FlagWithOutput("--text ", text). FlagWithOutput("--xml ", xml). - FlagWithArg("--compile-sdk-version ", l.compileSdkVersion). + FlagWithArg("--compile-sdk-version ", strconv.Itoa(l.compileSdkVersion)). FlagWithArg("--java-language-level ", l.javaLanguageLevel). FlagWithArg("--kotlin-language-level ", l.kotlinLanguageLevel). FlagWithArg("--url ", fmt.Sprintf(".=.,%s=out", android.PathForOutput(ctx).String())). diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt index 4bc0c5fd4..1eee354cd 100644 --- a/java/lint_defaults.txt +++ b/java/lint_defaults.txt @@ -28,6 +28,11 @@ --disable_check SuspiciousImport --disable_check UnusedResources --disable_check ViewConstructor +# Disable NewApi checks for the platform since platform is the one that implements +# the API. This prevents noisy lint warnings like b/228956345#1 +# NewApi checks will continue to be enforced for apex deps since +# lint.strict_updatability_linting will be true for those Soong modules +--disable_check NewApi # Downgrade existing errors to warnings --warning_check AppCompatResource # 55 occurences in 10 modules @@ -66,7 +71,6 @@ --warning_check MissingTvBanner # 3 occurences in 3 modules --warning_check NamespaceTypo # 3 occurences in 3 modules --warning_check NetworkSecurityConfig # 46 occurences in 12 modules ---warning_check NewApi # 1996 occurences in 122 modules --warning_check NotSibling # 15 occurences in 10 modules --warning_check ObjectAnimatorBinding # 14 occurences in 5 modules --warning_check OnClick # 49 occurences in 21 modules diff --git a/java/sdk_library.go b/java/sdk_library.go index c37ed1a27..fb032559e 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -377,6 +377,9 @@ type sdkLibraryProperties struct { // List of Java libraries that will be in the classpath when building the implementation lib Impl_only_libs []string `android:"arch_variant"` + // List of Java libraries that will included in the implementation lib. + Impl_only_static_libs []string `android:"arch_variant"` + // List of Java libraries that will be in the classpath when building stubs Stub_only_libs []string `android:"arch_variant"` @@ -1346,10 +1349,12 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) visibility := childModuleVisibility(module.sdkLibraryProperties.Impl_library_visibility) props := struct { - Name *string - Visibility []string - Instrument bool - Libs []string + Name *string + Visibility []string + Instrument bool + Libs []string + Static_libs []string + Apex_available []string }{ Name: proptools.StringPtr(module.implLibraryModuleName()), Visibility: visibility, @@ -1358,6 +1363,12 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) // Set the impl_only libs. Note that the module's "Libs" get appended as well, via the // addition of &module.properties below. Libs: module.sdkLibraryProperties.Impl_only_libs, + // Set the impl_only static libs. Note that the module's "static_libs" get appended as well, via the + // addition of &module.properties below. + Static_libs: module.sdkLibraryProperties.Impl_only_static_libs, + // Pass the apex_available settings down so that the impl library can be statically + // embedded within a library that is added to an APEX. Needed for updatable-media. + Apex_available: module.ApexAvailable(), } properties := []interface{}{ @@ -1814,8 +1825,9 @@ func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookCont *javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName()) } - // Add the impl_only_libs *after* we're done using the Libs prop in submodules. + // Add the impl_only_libs and impl_only_static_libs *after* we're done using them in submodules. module.properties.Libs = append(module.properties.Libs, module.sdkLibraryProperties.Impl_only_libs...) + module.properties.Static_libs = append(module.properties.Static_libs, module.sdkLibraryProperties.Impl_only_static_libs...) } func (module *SdkLibrary) InitSdkLibraryProperties() { diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index e84eacd86..cc83430a8 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -173,7 +173,7 @@ func main() { } ok := true for _, mkFile := range files { - ok = convertOne(mkFile) && ok + ok = convertOne(mkFile, []string{}) && ok } if *launcher != "" { @@ -183,7 +183,7 @@ func main() { if *inputVariables == "" { quit(fmt.Errorf("the product launcher requires an input variables file")) } - if !convertOne(*inputVariables) { + if !convertOne(*inputVariables, []string{}) { quit(fmt.Errorf("the product launcher input variables file failed to convert")) } @@ -201,7 +201,7 @@ func main() { if *inputVariables == "" { quit(fmt.Errorf("the board launcher requires an input variables file")) } - if !convertOne(*inputVariables) { + if !convertOne(*inputVariables, []string{}) { quit(fmt.Errorf("the board launcher input variables file failed to convert")) } err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher( @@ -310,9 +310,13 @@ const copyright = `# // the output hierarchy, or to the stdout. // Optionally, recursively convert the files this one includes by // $(call inherit-product) or an include statement. -func convertOne(mkFile string) (ok bool) { +func convertOne(mkFile string, loadStack []string) (ok bool) { if v, ok := converted[mkFile]; ok { - return v != nil + if v == nil { + fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile) + return false + } + return true } converted[mkFile] = nil defer func() { @@ -356,6 +360,7 @@ func convertOne(mkFile string) (ok bool) { return false } } + loadStack = append(loadStack, mkFile) ok = true if *recurse { for _, sub := range ss.SubConfigFiles() { @@ -363,7 +368,7 @@ func convertOne(mkFile string) (ok bool) { if _, err := os.Stat(sub); os.IsNotExist(err) { continue } - ok = convertOne(sub) && ok + ok = convertOne(sub, loadStack) && ok } } converted[mkFile] = ss diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go index 92665207d..6a6eb460a 100644 --- a/mk2rbc/expr.go +++ b/mk2rbc/expr.go @@ -741,8 +741,8 @@ func (_ *badExpr) typ() starlarkType { return starlarkTypeUnknown } -func (_ *badExpr) emitListVarCopy(_ *generationContext) { - panic("implement me") +func (b *badExpr) emitListVarCopy(gctx *generationContext) { + b.emit(gctx) } func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 8f4fea4b1..104110833 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -116,6 +116,7 @@ var knownFunctions = map[string]interface { "subst": &substCallParser{fname: "subst"}, "warning": &makeControlFuncParser{name: baseName + ".mkwarning"}, "word": &wordCallParser{}, + "words": &wordsCallParser{}, "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList}, } @@ -130,6 +131,14 @@ var knownNodeFunctions = map[string]interface { "foreach": &foreachCallNodeParser{}, } +// These look like variables, but are actually functions, and would give +// undefined variable errors if we converted them as variables. Instead, +// emit an error instead of converting them. +var unsupportedFunctions = map[string]bool{ + "local-generated-sources-dir": true, + "local-intermediates-dir": true, +} + // These are functions that we don't implement conversions for, but // we allow seeing their definitions in the product config files. var ignoredDefines = map[string]bool{ @@ -531,7 +540,7 @@ func (ctx *parseContext) backNode() { func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode { // Handle only simple variables - if !a.Name.Const() { + if !a.Name.Const() || a.Target != nil { return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")} } name := a.Name.Strings[0] @@ -542,6 +551,12 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if strings.HasPrefix(name, "override ") { return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")} } + if name == ".KATI_READONLY" { + // Skip assignments to .KATI_READONLY. If it was in the output file, it + // would be an error because it would be sorted before the definition of + // the variable it's trying to make readonly. + return []starlarkNode{} + } // Soong configuration if strings.HasPrefix(name, soongNsPrefix) { @@ -556,9 +571,6 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs.valueType() == starlarkTypeUnknown { // Try to divine variable type from the RHS asgn.value = ctx.parseMakeString(a, a.Value) - if xBad, ok := asgn.value.(*badExpr); ok { - return []starlarkNode{&exprNode{xBad}} - } inferred_type := asgn.value.typ() if inferred_type != starlarkTypeUnknown { lhs.setValueType(inferred_type) @@ -567,21 +579,19 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs.valueType() == starlarkTypeList { xConcat, xBad := ctx.buildConcatExpr(a) if xBad != nil { - return []starlarkNode{&exprNode{expr: xBad}} - } - switch len(xConcat.items) { - case 0: - asgn.value = &listExpr{} - case 1: - asgn.value = xConcat.items[0] - default: - asgn.value = xConcat + asgn.value = xBad + } else { + switch len(xConcat.items) { + case 0: + asgn.value = &listExpr{} + case 1: + asgn.value = xConcat.items[0] + default: + asgn.value = xConcat + } } } else { asgn.value = ctx.parseMakeString(a, a.Value) - if xBad, ok := asgn.value.(*badExpr); ok { - return []starlarkNode{&exprNode{expr: xBad}} - } } if asgn.lhs.valueType() == starlarkTypeString && @@ -811,35 +821,40 @@ func (ctx *parseContext) handleSubConfig( // rblf.inherit(handle, _e[0], _e[1]) // var matchingPaths []string - varPath, ok := pathExpr.(*interpolateExpr) - if !ok { - return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} - } - - pathPattern := []string{varPath.chunks[0]} - for _, chunk := range varPath.chunks[1:] { - if chunk != "" { - pathPattern = append(pathPattern, chunk) + var needsWarning = false + if interpolate, ok := pathExpr.(*interpolateExpr); ok { + pathPattern := []string{interpolate.chunks[0]} + for _, chunk := range interpolate.chunks[1:] { + if chunk != "" { + pathPattern = append(pathPattern, chunk) + } } - } - if pathPattern[0] == "" && len(ctx.includeTops) > 0 { - // If pattern starts from the top. restrict it to the directories where - // we know inherit-product uses dynamically calculated path. + if pathPattern[0] == "" && len(ctx.includeTops) > 0 { + // If pattern starts from the top. restrict it to the directories where + // we know inherit-product uses dynamically calculated path. + for _, p := range ctx.includeTops { + pathPattern[0] = p + matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...) + } + } else { + matchingPaths = ctx.findMatchingPaths(pathPattern) + } + needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0 + } else if len(ctx.includeTops) > 0 { for _, p := range ctx.includeTops { - pathPattern[0] = p - matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...) + matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{p, ""})...) } } else { - matchingPaths = ctx.findMatchingPaths(pathPattern) + return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} } + // Safeguard against $(call inherit-product,$(PRODUCT_PATH)) const maxMatchingFiles = 150 if len(matchingPaths) > maxMatchingFiles { return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)} } - needsWarning := pathPattern[0] == "" && len(ctx.includeTops) == 0 - res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} + res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} for _, p := range matchingPaths { // A product configuration files discovered dynamically may attempt to inherit // from another one which does not exist in this source tree. Prevent load errors @@ -889,8 +904,9 @@ func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, arg }) } -func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) []starlarkNode { - return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode { +func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode { + loadAlways := v.Name[0] != '-' + return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode { return &includeNode{im, loadAlways} }) } @@ -1068,6 +1084,18 @@ func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { return otherOperand } } + if otherOperand.typ() == starlarkTypeList { + fields := strings.Fields(stringOperand) + elements := make([]starlarkExpr, len(fields)) + for i, s := range fields { + elements[i] = &stringLiteralExpr{literal: s} + } + return &eqExpr{ + left: otherOperand, + right: &listExpr{elements}, + isEq: isEq, + } + } if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt { return &eqExpr{ left: otherOperand, @@ -1113,8 +1141,6 @@ func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, switch call.name { case baseName + ".filter": return ctx.parseCompareFilterFuncResult(directive, call, value, isEq) - case baseName + ".expand_wildcard": - return ctx.parseCompareWildcardFuncResult(directive, call, value, !isEq), true case baseName + ".findstring": return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true case baseName + ".strip": @@ -1159,22 +1185,6 @@ func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive, } } -func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive, - xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { - if !isEmptyString(xValue) { - return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue) - } - callFunc := baseName + ".file_wildcard_exists" - if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") { - callFunc = baseName + ".file_exists" - } - var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool} - if !negate { - cc = ¬Expr{cc} - } - return cc -} - func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive, xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { if isEmptyString(xValue) { @@ -1262,6 +1272,12 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt return ctx.newBadExpr(node, "reference is too complex: %s", refDump) } + if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok { + if _, unsupported := unsupportedFunctions[name]; unsupported { + return ctx.newBadExpr(node, "%s is not supported", refDump) + } + } + // If it is a single word, it can be a simple variable // reference or a function call if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" { @@ -1309,9 +1325,8 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt } else { return ctx.newBadExpr(node, "cannot handle invoking %s", name) } - } else { - return ctx.newBadExpr(node, "cannot handle %s", refDump) } + return ctx.newBadExpr(node, "cannot handle %s", refDump) } type simpleCallParser struct { @@ -1587,6 +1602,16 @@ func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starla for _, n := range a.actions { transformNode(n, transformer) } + case *inheritNode: + if b, ok := a.module.(inheritedDynamicModule); ok { + b.path = b.path.transform(transformer) + a.module = b + } + case *includeNode: + if b, ok := a.module.(inheritedDynamicModule); ok { + b.path = b.path.transform(transformer) + a.module = b + } } } @@ -1637,9 +1662,11 @@ func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa if len(words) != 2 { return ctx.newBadExpr(node, "word function should have 2 arguments") } - var index uint64 = 0 + var index = 0 if words[0].Const() { - index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64) + if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil { + index = i + } } if index < 1 { return ctx.newBadExpr(node, "word index should be constant positive integer") @@ -1647,13 +1674,40 @@ func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa words[1].TrimLeftSpaces() words[1].TrimRightSpaces() array := ctx.parseMakeString(node, words[1]) - if xBad, ok := array.(*badExpr); ok { - return xBad + if bad, ok := array.(*badExpr); ok { + return bad + } + if array.typ() != starlarkTypeList { + array = &callExpr{ + name: baseName + ".words", + args: []starlarkExpr{array}, + returnType: starlarkTypeList, + } + } + return &indexExpr{array, &intLiteralExpr{index - 1}} +} + +type wordsCallParser struct{} + +func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { + args.TrimLeftSpaces() + args.TrimRightSpaces() + array := ctx.parseMakeString(node, args) + if bad, ok := array.(*badExpr); ok { + return bad } if array.typ() != starlarkTypeList { - array = &callExpr{object: array, name: "split", returnType: starlarkTypeList} + array = &callExpr{ + name: baseName + ".words", + args: []starlarkExpr{array}, + returnType: starlarkTypeList, + } + } + return &callExpr{ + name: "len", + args: []starlarkExpr{array}, + returnType: starlarkTypeInt, } - return &indexExpr{array, &intLiteralExpr{int(index - 1)}} } type firstOrLastwordCallParser struct { @@ -1759,10 +1813,23 @@ func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa } case *mkparser.Comment: return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}} + case *mkparser.Directive: + if n.Name == "include" || n.Name == "-include" { + return ctx.handleInclude(n) + } + case *mkparser.Variable: + // Technically inherit-product(-if-exists) don't need to be put inside + // an eval, but some makefiles do it, presumably because they copy+pasted + // from a $(eval include ...) + if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok { + if name == "inherit-product" || name == "inherit-product-if-exists" { + return ctx.handleVariable(n) + } + } } } - return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments and comments are supported")} + return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")} } func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr { @@ -1822,7 +1889,7 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNod result = []starlarkNode{res} } case "include", "-include": - result = ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') + result = ctx.handleInclude(x) case "ifeq", "ifneq", "ifdef", "ifndef": result = []starlarkNode{ctx.handleIfBlock(x)} default: diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index de7512927..de2dc3c4c 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -117,8 +117,8 @@ PRODUCT_NAME := $(call foo0) def init(g, handle): cfg = rblf.cfg(handle) - rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1") - rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0") + cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1") + cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0") `, }, { @@ -568,14 +568,18 @@ ifeq (,$(wildcard foo.mk)) endif ifneq (,$(wildcard foo*.mk)) endif +ifeq (foo1.mk foo2.mk barxyz.mk,$(wildcard foo*.mk bar*.mk)) +endif `, expected: `load("//build/make/core:product_config.rbc", "rblf") def init(g, handle): cfg = rblf.cfg(handle) - if not rblf.file_exists("foo.mk"): + if not rblf.expand_wildcard("foo.mk"): + pass + if rblf.expand_wildcard("foo*.mk"): pass - if rblf.file_wildcard_exists("foo*.mk"): + if rblf.expand_wildcard("foo*.mk bar*.mk") == ["foo1.mk", "foo2.mk", "barxyz.mk"]: pass `, }, @@ -808,6 +812,10 @@ def init(g, handle): PRODUCT_COPY_FILES := $(addprefix pfx-,a b c) PRODUCT_COPY_FILES := $(addsuffix .sff, a b c) PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM))) +ifeq (1,$(words $(SOME_UNKNOWN_VARIABLE))) +endif +ifeq ($(words $(SOME_OTHER_VARIABLE)),$(SOME_INT_VARIABLE)) +endif $(info $(patsubst %.pub,$(PRODUCT_NAME)%,$(PRODUCT_ADB_KEYS))) $(info $$(dir foo/bar): $(dir foo/bar)) $(info $(firstword $(PRODUCT_COPY_FILES))) @@ -830,7 +838,11 @@ def init(g, handle): cfg = rblf.cfg(handle) cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c") cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c") - cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0] + cfg["PRODUCT_NAME"] = rblf.words((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " "))[0] + if len(rblf.words(g.get("SOME_UNKNOWN_VARIABLE", ""))) == 1: + pass + if ("%d" % (len(rblf.words(g.get("SOME_OTHER_VARIABLE", ""))))) == g.get("SOME_INT_VARIABLE", ""): + pass rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%s%%" % cfg["PRODUCT_NAME"], g.get("PRODUCT_ADB_KEYS", ""))) rblf.mkinfo("product.mk", "$(dir foo/bar): %s" % rblf.dir("foo/bar")) rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][0]) @@ -975,7 +987,7 @@ def init(g, handle): rblf.soong_config_namespace(g, "cvd") rblf.soong_config_set(g, "cvd", "launch_configs", "cvd_config_auto.json") rblf.soong_config_append(g, "cvd", "grub_config", "grub.cfg") - rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config") + _x = rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config") `, }, { desc: "soong namespace accesses", @@ -1142,6 +1154,11 @@ def init(g, handle): MY_PATH:=foo #RBC# include_top vendor/foo1 $(call inherit-product,$(MY_PATH)/cfg.mk) +#RBC# include_top vendor/foo1 +$(call inherit-product,$(MY_OTHER_PATH)) +#RBC# include_top vendor/foo1 +$(foreach f,$(MY_MAKEFILES), \ + $(call inherit-product,$(f))) `, expected: `load("//build/make/core:product_config.rbc", "rblf") load("//vendor/foo1:cfg.star|init", _cfg_init = "init") @@ -1156,6 +1173,21 @@ def init(g, handle): if not _varmod_init: rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"])) rblf.inherit(handle, _varmod, _varmod_init) + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get(g.get("MY_OTHER_PATH", "")) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % (g.get("MY_OTHER_PATH", ""))) + rblf.inherit(handle, _varmod, _varmod_init) + for f in rblf.words(g.get("MY_MAKEFILES", "")): + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get(f) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % (f)) + rblf.inherit(handle, _varmod, _varmod_init) `, }, { @@ -1242,13 +1274,15 @@ def init(g, handle): desc: "Ignore make rules", mkname: "product.mk", in: ` +foo: PRIVATE_VARIABLE = some_tool $< $@ foo: foo.c gcc -o $@ $*`, expected: `load("//build/make/core:product_config.rbc", "rblf") def init(g, handle): cfg = rblf.cfg(handle) - rblf.mk2rbc_error("product.mk:2", "unsupported line rule: foo: foo.c\n#gcc -o $@ $*") + rblf.mk2rbc_error("product.mk:2", "Only simple variables are handled") + rblf.mk2rbc_error("product.mk:3", "unsupported line rule: foo: foo.c\n#gcc -o $@ $*") `, }, { @@ -1269,6 +1303,7 @@ def init(g, handle): in: ` ifeq (,$(call foobar)) endif +my_sources := $(local-generated-sources-dir) `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -1276,6 +1311,7 @@ def init(g, handle): cfg = rblf.cfg(handle) if rblf.mk2rbc_error("build/product.mk:2", "cannot handle invoking foobar"): pass + _my_sources = rblf.mk2rbc_error("build/product.mk:4", "local-generated-sources-dir is not supported") `, }, { @@ -1509,24 +1545,54 @@ $(eval) $(eval MY_VAR := foo) $(eval # This is a test of eval functions) $(eval $(TOO_COMPLICATED) := bar) +$(eval include foo/font.mk) +$(eval $(call inherit-product,vendor/foo1/cfg.mk)) + $(foreach x,$(MY_LIST_VAR), \ $(eval PRODUCT_COPY_FILES += foo/bar/$(x):$(TARGET_COPY_OUT_VENDOR)/etc/$(x)) \ - $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x))) \ -) + $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x)))) +$(foreach x,$(MY_LIST_VAR), \ + $(eval include foo/$(x).mk)) `, expected: `load("//build/make/core:product_config.rbc", "rblf") +load("//foo:font.star", _font_init = "init") +load("//vendor/foo1:cfg.star", _cfg_init = "init") def init(g, handle): cfg = rblf.cfg(handle) g["MY_VAR"] = "foo" # This is a test of eval functions - rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments and comments are supported") + rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported") + _font_init(g, handle) + rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init) for x in rblf.words(g.get("MY_LIST_VAR", "")): rblf.setdefault(handle, "PRODUCT_COPY_FILES") cfg["PRODUCT_COPY_FILES"] += ("foo/bar/%s:%s/etc/%s" % (x, g.get("TARGET_COPY_OUT_VENDOR", ""), x)).split() if g.get("MY_OTHER_VAR", ""): cfg["PRODUCT_COPY_FILES"] += ("%s:foo/bar/%s" % (g.get("MY_OTHER_VAR", ""), x)).split() + for x in rblf.words(g.get("MY_LIST_VAR", "")): + _entry = { + "foo/font.mk": ("foo/font", _font_init), + }.get("foo/%s.mk" % _x) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % ("foo/%s.mk" % _x)) + _varmod_init(g, handle) +`, + }, + { + desc: ".KATI_READONLY", + mkname: "product.mk", + in: ` +MY_VAR := foo +.KATI_READONLY := MY_VAR +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["MY_VAR"] = "foo" `, }, } diff --git a/mk2rbc/node.go b/mk2rbc/node.go index 7c39b9ead..a01abd8ac 100644 --- a/mk2rbc/node.go +++ b/mk2rbc/node.go @@ -83,7 +83,7 @@ func (im inheritedStaticModule) needsLoadCheck() bool { } type inheritedDynamicModule struct { - path interpolateExpr + path starlarkExpr candidateModules []*moduleInfo loadAlways bool location ErrorLocation @@ -120,7 +120,7 @@ func (i inheritedDynamicModule) emitSelect(gctx *generationContext) { } func (i inheritedDynamicModule) pathExpr() starlarkExpr { - return &i.path + return i.path } func (i inheritedDynamicModule) needsLoadCheck() bool { diff --git a/multitree/Android.bp b/multitree/Android.bp new file mode 100644 index 000000000..9b16d2021 --- /dev/null +++ b/multitree/Android.bp @@ -0,0 +1,19 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-multitree", + pkgPath: "android/soong/multitree", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "api_surface.go", + "export.go", + "metadata.go", + "import.go", + ], + pluginFor: ["soong_build"], +} diff --git a/multitree/api_surface.go b/multitree/api_surface.go new file mode 100644 index 000000000..f739a2430 --- /dev/null +++ b/multitree/api_surface.go @@ -0,0 +1,119 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + "fmt" + + "github.com/google/blueprint" +) + +var ( + pctx = android.NewPackageContext("android/soong/multitree") +) + +func init() { + RegisterApiSurfaceBuildComponents(android.InitRegistrationContext) +} + +var PrepareForTestWithApiSurface = android.FixtureRegisterWithContext(RegisterApiSurfaceBuildComponents) + +func RegisterApiSurfaceBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("api_surface", ApiSurfaceFactory) +} + +type ApiSurface struct { + android.ModuleBase + ExportableModuleBase + properties apiSurfaceProperties + + allOutputs android.Paths + taggedOutputs map[string]android.Paths +} + +type apiSurfaceProperties struct { + Contributions []string +} + +func ApiSurfaceFactory() android.Module { + module := &ApiSurface{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + InitExportableModule(module) + return module +} + +func (surface *ApiSurface) DepsMutator(ctx android.BottomUpMutatorContext) { + if surface.properties.Contributions != nil { + ctx.AddVariationDependencies(nil, nil, surface.properties.Contributions...) + } + +} +func (surface *ApiSurface) GenerateAndroidBuildActions(ctx android.ModuleContext) { + contributionFiles := make(map[string]android.Paths) + var allOutputs android.Paths + ctx.WalkDeps(func(child, parent android.Module) bool { + if contribution, ok := child.(ApiContribution); ok { + copied := contribution.CopyFilesWithTag(ctx) + for tag, files := range copied { + contributionFiles[child.Name()+"#"+tag] = files + } + for _, paths := range copied { + allOutputs = append(allOutputs, paths...) + } + return false // no transitive dependencies + } + return false + }) + + // phony target + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: android.PathForPhony(ctx, ctx.ModuleName()), + Inputs: allOutputs, + }) + + surface.allOutputs = allOutputs + surface.taggedOutputs = contributionFiles +} + +func (surface *ApiSurface) OutputFiles(tag string) (android.Paths, error) { + if tag != "" { + return nil, fmt.Errorf("unknown tag: %q", tag) + } + return surface.allOutputs, nil +} + +func (surface *ApiSurface) TaggedOutputs() map[string]android.Paths { + return surface.taggedOutputs +} + +func (surface *ApiSurface) Exportable() bool { + return true +} + +var _ android.OutputFileProducer = (*ApiSurface)(nil) +var _ Exportable = (*ApiSurface)(nil) + +type ApiContribution interface { + // copy files necessaryt to construct an API surface + // For C, it will be map.txt and .h files + // For Java, it will be api.txt + CopyFilesWithTag(ctx android.ModuleContext) map[string]android.Paths // output paths + + // Generate Android.bp in out/ to use the exported .txt files + // GenerateBuildFiles(ctx ModuleContext) Paths //output paths +} diff --git a/multitree/export.go b/multitree/export.go new file mode 100644 index 000000000..aecade58d --- /dev/null +++ b/multitree/export.go @@ -0,0 +1,67 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + + "github.com/google/blueprint/proptools" +) + +type moduleExportProperty struct { + // True if the module is exported to the other components in a multi-tree. + // Any components in the multi-tree can import this module to use. + Export *bool +} + +type ExportableModuleBase struct { + properties moduleExportProperty +} + +type Exportable interface { + // Properties for the exporable module. + exportableModuleProps() *moduleExportProperty + + // Check if this module can be exported. + // If this returns false, the module will not be exported regardless of the 'export' value. + Exportable() bool + + // Returns 'true' if this module has 'export: true' + // This module will not be exported if it returns 'false' to 'Exportable()' interface even if + // it has 'export: true'. + IsExported() bool + + // Map from tags to outputs. + // Each module can tag their outputs for convenience. + TaggedOutputs() map[string]android.Paths +} + +type ExportableModule interface { + android.Module + android.OutputFileProducer + Exportable +} + +func InitExportableModule(module ExportableModule) { + module.AddProperties(module.exportableModuleProps()) +} + +func (m *ExportableModuleBase) exportableModuleProps() *moduleExportProperty { + return &m.properties +} + +func (m *ExportableModuleBase) IsExported() bool { + return proptools.Bool(m.properties.Export) +} diff --git a/multitree/import.go b/multitree/import.go new file mode 100644 index 000000000..1e5c421bc --- /dev/null +++ b/multitree/import.go @@ -0,0 +1,96 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" +) + +var ( + nameSuffix = ".imported" +) + +type MultitreeImportedModuleInterface interface { + GetMultitreeImportedModuleName() string +} + +func init() { + android.RegisterModuleType("imported_filegroup", importedFileGroupFactory) + + android.PreArchMutators(RegisterMultitreePreArchMutators) +} + +type importedFileGroupProperties struct { + // Imported modules from the other components in a multi-tree + Imported []string +} + +type importedFileGroup struct { + android.ModuleBase + + properties importedFileGroupProperties + srcs android.Paths +} + +func (ifg *importedFileGroup) Name() string { + return ifg.BaseModuleName() + nameSuffix +} + +func importedFileGroupFactory() android.Module { + module := &importedFileGroup{} + module.AddProperties(&module.properties) + + android.InitAndroidModule(module) + return module +} + +var _ MultitreeImportedModuleInterface = (*importedFileGroup)(nil) + +func (ifg *importedFileGroup) GetMultitreeImportedModuleName() string { + // The base module name of the imported filegroup is used as the imported module name + return ifg.BaseModuleName() +} + +var _ android.SourceFileProducer = (*importedFileGroup)(nil) + +func (ifg *importedFileGroup) Srcs() android.Paths { + return ifg.srcs +} + +func (ifg *importedFileGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // srcs from this module must not be used. Adding a dot path to avoid the empty + // source failure. Still soong returns error when a module wants to build against + // this source, which is intended. + ifg.srcs = android.PathsForModuleSrc(ctx, []string{"."}) +} + +func RegisterMultitreePreArchMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("multitree_imported_rename", MultitreeImportedRenameMutator).Parallel() +} + +func MultitreeImportedRenameMutator(ctx android.BottomUpMutatorContext) { + if m, ok := ctx.Module().(MultitreeImportedModuleInterface); ok { + name := m.GetMultitreeImportedModuleName() + if !ctx.OtherModuleExists(name) { + // Provide an empty filegroup not to break the build while updating the metadata. + // In other cases, soong will report an error to guide users to run 'm update-meta' + // first. + if !ctx.Config().TargetMultitreeUpdateMeta() { + ctx.ModuleErrorf("\"%s\" filegroup must be imported.\nRun 'm update-meta' first to import the filegroup.", name) + } + ctx.Rename(name) + } + } +} diff --git a/multitree/metadata.go b/multitree/metadata.go new file mode 100644 index 000000000..3fd721599 --- /dev/null +++ b/multitree/metadata.go @@ -0,0 +1,74 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + "encoding/json" +) + +func init() { + android.RegisterSingletonType("update-meta", UpdateMetaSingleton) +} + +func UpdateMetaSingleton() android.Singleton { + return &updateMetaSingleton{} +} + +type jsonImported struct { + FileGroups map[string][]string `json:",omitempty"` +} + +type metadataJsonFlags struct { + Imported jsonImported `json:",omitempty"` + Exported map[string][]string `json:",omitempty"` +} + +type updateMetaSingleton struct { + importedModules []string + generatedMetadataFile android.OutputPath +} + +func (s *updateMetaSingleton) GenerateBuildActions(ctx android.SingletonContext) { + metadata := metadataJsonFlags{ + Imported: jsonImported{ + FileGroups: make(map[string][]string), + }, + Exported: make(map[string][]string), + } + ctx.VisitAllModules(func(module android.Module) { + if ifg, ok := module.(*importedFileGroup); ok { + metadata.Imported.FileGroups[ifg.BaseModuleName()] = ifg.properties.Imported + } + if e, ok := module.(ExportableModule); ok { + if e.IsExported() && e.Exportable() { + for tag, files := range e.TaggedOutputs() { + // TODO(b/219846705): refactor this to a dictionary + metadata.Exported[e.Name()+":"+tag] = append(metadata.Exported[e.Name()+":"+tag], files.Strings()...) + } + } + } + }) + jsonStr, err := json.Marshal(metadata) + if err != nil { + ctx.Errorf(err.Error()) + } + s.generatedMetadataFile = android.PathForOutput(ctx, "multitree", "metadata.json") + android.WriteFileRule(ctx, s.generatedMetadataFile, string(jsonStr)) +} + +func (s *updateMetaSingleton) MakeVars(ctx android.MakeVarsContext) { + ctx.Strict("MULTITREE_METADATA", s.generatedMetadataFile.String()) +} diff --git a/rust/config/global.go b/rust/config/global.go index 2d5fa991b..6bfa9cf94 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.59.0" + RustDefaultVersion = "1.60.0" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ diff --git a/ui/build/config.go b/ui/build/config.go index e271bfca2..0092ff1ca 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -1223,6 +1223,21 @@ func (c *configImpl) rbeAuth() (string, string) { return "RBE_use_application_default_credentials", "true" } +func (c *configImpl) IsGooglerEnvironment() bool { + cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" + if v, ok := c.environ.Get(cf); ok { + return v == "googler" + } + return false +} + +func (c *configImpl) GoogleProdCredsExist() bool { + if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil { + return false + } + return true +} + func (c *configImpl) UseRemoteBuild() bool { return c.UseGoma() || c.UseRBE() } diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 8f9a69991..78d37b4c7 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -119,6 +119,7 @@ func startRBE(ctx Context, config Config) { } func stopRBE(ctx Context, config Config) { + defer checkProdCreds(ctx, config) cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown") output, err := cmd.CombinedOutput() if err != nil { @@ -131,6 +132,15 @@ func stopRBE(ctx Context, config Config) { } } +func checkProdCreds(ctx Context, config Config) { + if !config.IsGooglerEnvironment() || config.GoogleProdCredsExist() { + return + } + fmt.Fprintln(ctx.Writer, "") + fmt.Fprintln(ctx.Writer, "\033[33mWARNING: Missing LOAS credentials, please run `gcert`. This will result in failing RBE builds in the future, see go/build-fast#authentication.\033[0m") + fmt.Fprintln(ctx.Writer, "") +} + // DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics. // The protobuf file is created if RBE is enabled and the proxy service has // started. The proxy service is shutdown in order to dump the RBE metrics to the |