diff options
103 files changed, 2607 insertions, 783 deletions
diff --git a/android/Android.bp b/android/Android.bp index 6450a06ca..cfa2be38f 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -19,6 +19,9 @@ bootstrap_go_package { "soong-ui-metrics_proto", "golang-protobuf-proto", "golang-protobuf-encoding-prototext", + + // Only used for tests. + "androidmk-parser", ], srcs: [ "androidmk.go", diff --git a/android/androidmk.go b/android/androidmk.go index b6b04a6bc..9025a746a 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -25,7 +25,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" "path/filepath" "reflect" @@ -35,6 +34,7 @@ import ( "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" + "github.com/google/blueprint/pathtools" ) func init() { @@ -690,7 +690,7 @@ func (c *androidMkSingleton) GenerateBuildActions(ctx SingletonContext) { }) } -func translateAndroidMk(ctx SingletonContext, mkFile string, mods []blueprint.Module) error { +func translateAndroidMk(ctx SingletonContext, absMkFile string, mods []blueprint.Module) error { buf := &bytes.Buffer{} fmt.Fprintln(buf, "LOCAL_MODULE_MAKEFILE := $(lastword $(MAKEFILE_LIST))") @@ -699,7 +699,7 @@ func translateAndroidMk(ctx SingletonContext, mkFile string, mods []blueprint.Mo for _, mod := range mods { err := translateAndroidMkModule(ctx, buf, mod) if err != nil { - os.Remove(mkFile) + os.Remove(absMkFile) return err } @@ -719,27 +719,7 @@ func translateAndroidMk(ctx SingletonContext, mkFile string, mods []blueprint.Mo fmt.Fprintf(buf, "STATS.SOONG_MODULE_TYPE.%s := %d\n", mod_type, typeStats[mod_type]) } - // Don't write to the file if it hasn't changed - if _, err := os.Stat(absolutePath(mkFile)); !os.IsNotExist(err) { - if data, err := ioutil.ReadFile(absolutePath(mkFile)); err == nil { - matches := buf.Len() == len(data) - - if matches { - for i, value := range buf.Bytes() { - if value != data[i] { - matches = false - break - } - } - } - - if matches { - return nil - } - } - } - - return ioutil.WriteFile(absolutePath(mkFile), buf.Bytes(), 0666) + return pathtools.WriteFileIfChanged(absMkFile, buf.Bytes(), 0666) } func translateAndroidMkModule(ctx SingletonContext, w io.Writer, mod blueprint.Module) error { diff --git a/android/bazel.go b/android/bazel.go index cf27cb464..26d70e48f 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -15,6 +15,7 @@ package android import ( + "android/soong/bazel" "fmt" "io/ioutil" "path/filepath" @@ -24,34 +25,33 @@ import ( "github.com/google/blueprint/proptools" ) -type bazelModuleProperties struct { - // The label of the Bazel target replacing this Soong module. When run in conversion mode, this - // will import the handcrafted build target into the autogenerated file. Note: this may result in - // a conflict due to duplicate targets if bp2build_available is also set. - Label *string - - // If true, bp2build will generate the converted Bazel target for this module. Note: this may - // cause a conflict due to the duplicate targets if label is also set. - // - // This is a bool pointer to support tristates: true, false, not set. - // - // To opt-in a module, set bazel_module: { bp2build_available: true } - // To opt-out a module, set bazel_module: { bp2build_available: false } - // To defer the default setting for the directory, do not set the value. - Bp2build_available *bool -} - // Properties contains common module properties for Bazel migration purposes. type properties struct { // In USE_BAZEL_ANALYSIS=1 mode, this represents the Bazel target replacing // this Soong module. - Bazel_module bazelModuleProperties + Bazel_module bazel.BazelModuleProperties } +type namespacedVariableProperties map[string]interface{} + // BazelModuleBase contains the property structs with metadata for modules which can be converted to // Bazel. type BazelModuleBase struct { bazelProperties properties + + // namespacedVariableProperties is used for soong_config_module_type support + // in bp2build. Soong config modules allow users to set module properties + // based on custom product variables defined in Android.bp files. These + // variables are namespaced to prevent clobbering, especially when set from + // Makefiles. + namespacedVariableProperties namespacedVariableProperties + + // baseModuleType is set when this module was created from a module type + // defined by a soong_config_module_type. Every soong_config_module_type + // "wraps" another module type, e.g. a soong_config_module_type can wrap a + // cc_defaults to a custom_cc_defaults, or cc_binary to a custom_cc_binary. + // This baseModuleType is set to the wrapped module type. + baseModuleType string } // Bazelable is specifies the interface for modules that can be converted to Bazel. @@ -63,6 +63,12 @@ type Bazelable interface { ConvertWithBp2build(ctx BazelConversionContext) bool convertWithBp2build(ctx BazelConversionContext, module blueprint.Module) bool GetBazelBuildFileContents(c Config, path, name string) (string, error) + + // For namespaced config variable support + namespacedVariableProps() namespacedVariableProperties + setNamespacedVariableProps(props namespacedVariableProperties) + BaseModuleType() string + SetBaseModuleType(string) } // BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules. @@ -82,6 +88,22 @@ func (b *BazelModuleBase) bazelProps() *properties { return &b.bazelProperties } +func (b *BazelModuleBase) namespacedVariableProps() namespacedVariableProperties { + return b.namespacedVariableProperties +} + +func (b *BazelModuleBase) setNamespacedVariableProps(props namespacedVariableProperties) { + b.namespacedVariableProperties = props +} + +func (b *BazelModuleBase) BaseModuleType() string { + return b.baseModuleType +} + +func (b *BazelModuleBase) SetBaseModuleType(baseModuleType string) { + b.baseModuleType = baseModuleType +} + // HasHandcraftedLabel returns whether this module has a handcrafted Bazel label. func (b *BazelModuleBase) HasHandcraftedLabel() bool { return b.bazelProperties.Bazel_module.Label != nil @@ -166,6 +188,7 @@ var ( "packages/apps/QuickSearchBox":/* recursive = */ true, "packages/apps/WallpaperPicker":/* recursive = */ false, + "prebuilts/gcc":/* recursive = */ true, "prebuilts/sdk":/* recursive = */ false, "prebuilts/sdk/current/extras/app-toolkit":/* recursive = */ false, "prebuilts/sdk/current/support":/* recursive = */ false, @@ -228,6 +251,8 @@ var ( // Per-module denylist to always opt modules out of both bp2build and mixed builds. bp2buildModuleDoNotConvertList = []string{ + "libprotobuf-cpp-full", "libprotobuf-cpp-lite", // Unsupported product&vendor suffix. b/204811222 and b/204810610. + "libc_malloc_debug", // depends on libunwindstack, which depends on unsupported module art_cc_library_statics "libbase_ndk", // http://b/186826477, fails to link libctscamera2_jni for device (required for CtsCameraTestCases) @@ -248,14 +273,8 @@ var ( "platform_tools_properties", "build_tools_source_properties", - // //external/libcap/... - "cap_names.h", // http://b/196105070 host toolchain misconfigurations for mixed builds - "libcap", // http://b/196105070 host toolchain misconfigurations for mixed builds - - "libminijail", // depends on unconverted modules: libcap - "getcap", // depends on unconverted modules: libcap - "setcap", // depends on unconverted modules: libcap - "minijail0", // depends on unconverted modules: libcap, libminijail + "libminijail", // b/202491296: Uses unsupported c_std property. + "minijail0", // depends on unconverted modules: libminijail "drop_privs", // depends on unconverted modules: libminijail // Tests. Handle later. @@ -359,7 +378,11 @@ func ShouldKeepExistingBuildFileForDir(dir string) bool { // MixedBuildsEnabled checks that a module is ready to be replaced by a // converted or handcrafted Bazel target. -func (b *BazelModuleBase) MixedBuildsEnabled(ctx BazelConversionPathContext) bool { +func (b *BazelModuleBase) MixedBuildsEnabled(ctx ModuleContext) bool { + if ctx.Os() == Windows { + // Windows toolchains are not currently supported. + return false + } if !ctx.Config().BazelContext.BazelEnabled() { return false } @@ -399,7 +422,15 @@ func (b *BazelModuleBase) convertWithBp2build(ctx BazelConversionContext, module // prevents mixed builds from using auto-converted modules just by matching // the package dir; it also has to have a bp2build mutator as well. if ctx.Config().bp2buildModuleTypeConfig[ctx.OtherModuleType(module)] == false { - return false + if b, ok := module.(Bazelable); ok && b.BaseModuleType() != "" { + // For modules with custom types from soong_config_module_types, + // check that their _base module type_ has a bp2build mutator. + if ctx.Config().bp2buildModuleTypeConfig[b.BaseModuleType()] == false { + return false + } + } else { + return false + } } packagePath := ctx.OtherModuleDir(module) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 3c6212e0c..0052551f4 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -602,8 +602,6 @@ def get_arch(target): platform_name = build_options(target)["//command_line_option:platforms"][0].name if platform_name == "host": return "HOST" - elif platform_name.startswith("linux_glibc_"): - return platform_name[len("linux_glibc_"):] + "|" + platform_name[:len("linux_glibc_")-1] elif platform_name.startswith("android_"): return platform_name[len("android_"):] + "|" + platform_name[:len("android_")-1] elif platform_name.startswith("linux_"): @@ -865,7 +863,7 @@ func getConfigString(key cqueryKey) string { arch = "x86_64" } os := key.configKey.osType.Name - if len(os) == 0 || os == "common_os" { + if len(os) == 0 || os == "common_os" || os == "linux_glibc" { // Use host OS, which is currently hardcoded to be linux. os = "linux" } diff --git a/android/config.go b/android/config.go index 78d43c6d4..6f055650b 100644 --- a/android/config.go +++ b/android/config.go @@ -155,6 +155,7 @@ type config struct { fs pathtools.FileSystem mockBpList string + runningAsBp2Build bool bp2buildPackageConfig Bp2BuildConfig bp2buildModuleTypeConfig map[string]bool @@ -333,10 +334,8 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string ShippingApiLevel: stringPtr("30"), }, - outDir: buildDir, - // soongOutDir is inconsistent with production (it should be buildDir + "/soong") - // but a lot of tests assume this :( - soongOutDir: buildDir, + outDir: buildDir, + soongOutDir: filepath.Join(buildDir, "soong"), captureBuild: true, env: envCopy, @@ -564,11 +563,18 @@ func (c *config) SetAllowMissingDependencies() { // BlueprintToolLocation returns the directory containing build system tools // from Blueprint, like soong_zip and merge_zips. func (c *config) HostToolDir() string { - return filepath.Join(c.soongOutDir, "host", c.PrebuiltOS(), "bin") + if c.KatiEnabled() { + return filepath.Join(c.outDir, "host", c.PrebuiltOS(), "bin") + } else { + return filepath.Join(c.soongOutDir, "host", c.PrebuiltOS(), "bin") + } } func (c *config) HostToolPath(ctx PathContext, tool string) Path { path := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "bin", false, tool) + if ctx.Config().KatiEnabled() { + path = path.ToMakePath() + } return path } @@ -578,11 +584,18 @@ func (c *config) HostJNIToolPath(ctx PathContext, lib string) Path { ext = ".dylib" } path := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "lib64", false, lib+ext) + if ctx.Config().KatiEnabled() { + path = path.ToMakePath() + } return path } -func (c *config) HostJavaToolPath(ctx PathContext, path string) Path { - return PathForOutput(ctx, "host", c.PrebuiltOS(), "framework", path) +func (c *config) HostJavaToolPath(ctx PathContext, tool string) Path { + path := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "framework", false, tool) + if ctx.Config().KatiEnabled() { + path = path.ToMakePath() + } + return path } // PrebuiltOS returns the name of the host OS used in prebuilts directories. diff --git a/android/makevars.go b/android/makevars.go index 20db65a50..665d57625 100644 --- a/android/makevars.go +++ b/android/makevars.go @@ -142,15 +142,19 @@ type SingletonMakeVarsProvider interface { var singletonMakeVarsProvidersKey = NewOnceKey("singletonMakeVarsProvidersKey") +func getSingletonMakevarsProviders(config Config) *[]makeVarsProvider { + return config.Once(singletonMakeVarsProvidersKey, func() interface{} { + return &[]makeVarsProvider{} + }).(*[]makeVarsProvider) +} + // registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to // the list of MakeVarsProviders to run. func registerSingletonMakeVarsProvider(config Config, singleton SingletonMakeVarsProvider) { // Singletons are registered on the Context and may be different between different Contexts, // for example when running multiple tests. Store the SingletonMakeVarsProviders in the // Config so they are attached to the Context. - singletonMakeVarsProviders := config.Once(singletonMakeVarsProvidersKey, func() interface{} { - return &[]makeVarsProvider{} - }).(*[]makeVarsProvider) + singletonMakeVarsProviders := getSingletonMakevarsProviders(config) *singletonMakeVarsProviders = append(*singletonMakeVarsProviders, makeVarsProvider{pctx, singletonMakeVarsProviderAdapter(singleton)}) @@ -175,7 +179,9 @@ func makeVarsSingletonFunc() Singleton { return &makeVarsSingleton{} } -type makeVarsSingleton struct{} +type makeVarsSingleton struct { + installsForTesting []byte +} type makeVarsProvider struct { pctx PackageContext @@ -238,7 +244,7 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { var katiSymlinks []katiInstall providers := append([]makeVarsProvider(nil), makeVarsInitProviders...) - providers = append(providers, *ctx.Config().Get(singletonMakeVarsProvidersKey).(*[]makeVarsProvider)...) + providers = append(providers, *getSingletonMakevarsProviders(ctx.Config())...) for _, provider := range providers { mctx := &makeVarsContext{ @@ -313,6 +319,8 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { if err := pathtools.WriteFileIfChanged(installsFile, installsBytes, 0666); err != nil { ctx.Errorf(err.Error()) } + + s.installsForTesting = installsBytes } func (s *makeVarsSingleton) writeVars(vars []makeVarsVariable) []byte { @@ -414,6 +422,7 @@ func (s *makeVarsSingleton) writeLate(phonies []phony, dists []dist) []byte { fmt.Fprintln(buf) for _, dist := range dists { + fmt.Fprintf(buf, ".PHONY: %s\n", strings.Join(dist.goals, " ")) fmt.Fprintf(buf, "$(call dist-for-goals,%s,%s)\n", strings.Join(dist.goals, " "), strings.Join(dist.paths, " ")) } @@ -465,15 +474,19 @@ func (s *makeVarsSingleton) writeInstalls(installs, symlinks []katiInstall) []by for _, symlink := range symlinks { fmt.Fprintf(buf, "%s:", symlink.to.String()) + if symlink.from != nil { + // The symlink doesn't need updating when the target is modified, but we sometimes + // have a dependency on a symlink to a binary instead of to the binary directly, and + // the mtime of the symlink must be updated when the binary is modified, so use a + // normal dependency here instead of an order-only dependency. + fmt.Fprintf(buf, " %s", symlink.from.String()) + } for _, dep := range symlink.implicitDeps { fmt.Fprintf(buf, " %s", dep.String()) } - if symlink.from != nil || len(symlink.orderOnlyDeps) > 0 { + if len(symlink.orderOnlyDeps) > 0 { fmt.Fprintf(buf, " |") } - if symlink.from != nil { - fmt.Fprintf(buf, " %s", symlink.from.String()) - } for _, dep := range symlink.orderOnlyDeps { fmt.Fprintf(buf, " %s", dep.String()) } diff --git a/android/module.go b/android/module.go index 3447f2b51..c778078d0 100644 --- a/android/module.go +++ b/android/module.go @@ -2953,6 +2953,10 @@ func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, src to: fullInstallPath, }) } else { + // The symlink doesn't need updating when the target is modified, but we sometimes + // have a dependency on a symlink to a binary instead of to the binary directly, and + // the mtime of the symlink must be updated when the binary is modified, so use a + // normal dependency here instead of an order-only dependency. m.Build(pctx, BuildParams{ Rule: Symlink, Description: "install symlink " + fullInstallPath.Base(), diff --git a/android/module_test.go b/android/module_test.go index 9e2b0ca29..8607a8d34 100644 --- a/android/module_test.go +++ b/android/module_test.go @@ -15,7 +15,12 @@ package android import ( + "bytes" + "path/filepath" + "runtime" "testing" + + mkparser "android/soong/androidmk/parser" ) func TestSrcIsModule(t *testing.T) { @@ -199,17 +204,28 @@ type depsModule struct { } } +func (m *depsModule) InstallBypassMake() bool { + return true +} + func (m *depsModule) GenerateAndroidBuildActions(ctx ModuleContext) { + outputFile := PathForModuleOut(ctx, ctx.ModuleName()) + ctx.Build(pctx, BuildParams{ + Rule: Touch, + Output: outputFile, + }) + installFile := ctx.InstallFile(PathForModuleInstall(ctx), ctx.ModuleName(), outputFile) + ctx.InstallSymlink(PathForModuleInstall(ctx, "symlinks"), ctx.ModuleName(), installFile) } func (m *depsModule) DepsMutator(ctx BottomUpMutatorContext) { - ctx.AddDependency(ctx.Module(), nil, m.props.Deps...) + ctx.AddDependency(ctx.Module(), installDepTag{}, m.props.Deps...) } func depsModuleFactory() Module { m := &depsModule{} m.AddProperties(&m.props) - InitAndroidModule(m) + InitAndroidArchModule(m, HostAndDeviceDefault, MultilibCommon) return m } @@ -320,3 +336,286 @@ func TestDistErrorChecking(t *testing.T) { ExtendWithErrorHandler(FixtureExpectsAllErrorsToMatchAPattern(expectedErrs)). RunTestWithBp(t, bp) } + +func TestInstall(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + bp := ` + deps { + name: "foo", + deps: ["bar"], + } + + deps { + name: "bar", + deps: ["baz", "qux"], + } + + deps { + name: "baz", + deps: ["qux"], + } + + deps { + name: "qux", + } + ` + + result := GroupFixturePreparers( + prepareForModuleTests, + PrepareForTestWithArchMutator, + ).RunTestWithBp(t, bp) + + module := func(name string, host bool) TestingModule { + variant := "android_common" + if host { + variant = result.Config.BuildOSCommonTarget.String() + } + return result.ModuleForTests(name, variant) + } + + outputRule := func(name string) TestingBuildParams { return module(name, false).Output(name) } + + installRule := func(name string) TestingBuildParams { + return module(name, false).Output(filepath.Join("out/soong/target/product/test_device/system", name)) + } + + symlinkRule := func(name string) TestingBuildParams { + return module(name, false).Output(filepath.Join("out/soong/target/product/test_device/system/symlinks", name)) + } + + hostOutputRule := func(name string) TestingBuildParams { return module(name, true).Output(name) } + + hostInstallRule := func(name string) TestingBuildParams { + return module(name, true).Output(filepath.Join("out/soong/host/linux-x86", name)) + } + + hostSymlinkRule := func(name string) TestingBuildParams { + return module(name, true).Output(filepath.Join("out/soong/host/linux-x86/symlinks", name)) + } + + assertInputs := func(params TestingBuildParams, inputs ...Path) { + t.Helper() + AssertArrayString(t, "expected inputs", Paths(inputs).Strings(), + append(PathsIfNonNil(params.Input), params.Inputs...).Strings()) + } + + assertImplicits := func(params TestingBuildParams, implicits ...Path) { + t.Helper() + AssertArrayString(t, "expected implicit dependencies", Paths(implicits).Strings(), + append(PathsIfNonNil(params.Implicit), params.Implicits...).Strings()) + } + + assertOrderOnlys := func(params TestingBuildParams, orderonlys ...Path) { + t.Helper() + AssertArrayString(t, "expected orderonly dependencies", Paths(orderonlys).Strings(), + params.OrderOnly.Strings()) + } + + // Check host install rule dependencies + assertInputs(hostInstallRule("foo"), hostOutputRule("foo").Output) + assertImplicits(hostInstallRule("foo"), + hostInstallRule("bar").Output, + hostSymlinkRule("bar").Output, + hostInstallRule("baz").Output, + hostSymlinkRule("baz").Output, + hostInstallRule("qux").Output, + hostSymlinkRule("qux").Output, + ) + assertOrderOnlys(hostInstallRule("foo")) + + // Check host symlink rule dependencies. Host symlinks must use a normal dependency, not an + // order-only dependency, so that the tool gets updated when the symlink is depended on. + assertInputs(hostSymlinkRule("foo"), hostInstallRule("foo").Output) + assertImplicits(hostSymlinkRule("foo")) + assertOrderOnlys(hostSymlinkRule("foo")) + + // Check device install rule dependencies + assertInputs(installRule("foo"), outputRule("foo").Output) + assertImplicits(installRule("foo")) + assertOrderOnlys(installRule("foo"), + installRule("bar").Output, + symlinkRule("bar").Output, + installRule("baz").Output, + symlinkRule("baz").Output, + installRule("qux").Output, + symlinkRule("qux").Output, + ) + + // Check device symlink rule dependencies. Device symlinks could use an order-only dependency, + // but the current implementation uses a normal dependency. + assertInputs(symlinkRule("foo"), installRule("foo").Output) + assertImplicits(symlinkRule("foo")) + assertOrderOnlys(symlinkRule("foo")) +} + +func TestInstallBypassMake(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + bp := ` + deps { + name: "foo", + deps: ["bar"], + } + + deps { + name: "bar", + deps: ["baz", "qux"], + } + + deps { + name: "baz", + deps: ["qux"], + } + + deps { + name: "qux", + } + ` + + result := GroupFixturePreparers( + prepareForModuleTests, + PrepareForTestWithArchMutator, + FixtureModifyConfig(SetKatiEnabledForTests), + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterSingletonType("makevars", makeVarsSingletonFunc) + }), + ).RunTestWithBp(t, bp) + + installs := result.SingletonForTests("makevars").Singleton().(*makeVarsSingleton).installsForTesting + buf := bytes.NewBuffer(append([]byte(nil), installs...)) + parser := mkparser.NewParser("makevars", buf) + + nodes, errs := parser.Parse() + if len(errs) > 0 { + t.Fatalf("error parsing install rules: %s", errs[0]) + } + + rules := parseMkRules(t, result.Config, nodes) + + module := func(name string, host bool) TestingModule { + variant := "android_common" + if host { + variant = result.Config.BuildOSCommonTarget.String() + } + return result.ModuleForTests(name, variant) + } + + outputRule := func(name string) TestingBuildParams { return module(name, false).Output(name) } + + ruleForOutput := func(output string) installMakeRule { + for _, rule := range rules { + if rule.target == output { + return rule + } + } + t.Fatalf("no make install rule for %s", output) + return installMakeRule{} + } + + installRule := func(name string) installMakeRule { + return ruleForOutput(filepath.Join("out/target/product/test_device/system", name)) + } + + symlinkRule := func(name string) installMakeRule { + return ruleForOutput(filepath.Join("out/target/product/test_device/system/symlinks", name)) + } + + hostOutputRule := func(name string) TestingBuildParams { return module(name, true).Output(name) } + + hostInstallRule := func(name string) installMakeRule { + return ruleForOutput(filepath.Join("out/host/linux-x86", name)) + } + + hostSymlinkRule := func(name string) installMakeRule { + return ruleForOutput(filepath.Join("out/host/linux-x86/symlinks", name)) + } + + assertDeps := func(rule installMakeRule, deps ...string) { + t.Helper() + AssertArrayString(t, "expected inputs", deps, rule.deps) + } + + assertOrderOnlys := func(rule installMakeRule, orderonlys ...string) { + t.Helper() + AssertArrayString(t, "expected orderonly dependencies", orderonlys, rule.orderOnlyDeps) + } + + // Check host install rule dependencies + assertDeps(hostInstallRule("foo"), + hostOutputRule("foo").Output.String(), + hostInstallRule("bar").target, + hostSymlinkRule("bar").target, + hostInstallRule("baz").target, + hostSymlinkRule("baz").target, + hostInstallRule("qux").target, + hostSymlinkRule("qux").target, + ) + assertOrderOnlys(hostInstallRule("foo")) + + // Check host symlink rule dependencies. Host symlinks must use a normal dependency, not an + // order-only dependency, so that the tool gets updated when the symlink is depended on. + assertDeps(hostSymlinkRule("foo"), hostInstallRule("foo").target) + assertOrderOnlys(hostSymlinkRule("foo")) + + // Check device install rule dependencies + assertDeps(installRule("foo"), outputRule("foo").Output.String()) + assertOrderOnlys(installRule("foo"), + installRule("bar").target, + symlinkRule("bar").target, + installRule("baz").target, + symlinkRule("baz").target, + installRule("qux").target, + symlinkRule("qux").target, + ) + + // Check device symlink rule dependencies. Device symlinks could use an order-only dependency, + // but the current implementation uses a normal dependency. + assertDeps(symlinkRule("foo"), installRule("foo").target) + assertOrderOnlys(symlinkRule("foo")) +} + +type installMakeRule struct { + target string + deps []string + orderOnlyDeps []string +} + +func parseMkRules(t *testing.T, config Config, nodes []mkparser.Node) []installMakeRule { + var rules []installMakeRule + for _, node := range nodes { + if mkParserRule, ok := node.(*mkparser.Rule); ok { + var rule installMakeRule + + if targets := mkParserRule.Target.Words(); len(targets) == 0 { + t.Fatalf("no targets for rule %s", mkParserRule.Dump()) + } else if len(targets) > 1 { + t.Fatalf("unsupported multiple targets for rule %s", mkParserRule.Dump()) + } else if !targets[0].Const() { + t.Fatalf("unsupported non-const target for rule %s", mkParserRule.Dump()) + } else { + rule.target = normalizeStringRelativeToTop(config, targets[0].Value(nil)) + } + + prereqList := &rule.deps + for _, prereq := range mkParserRule.Prerequisites.Words() { + if !prereq.Const() { + t.Fatalf("unsupported non-const prerequisite for rule %s", mkParserRule.Dump()) + } + + if prereq.Value(nil) == "|" { + prereqList = &rule.orderOnlyDeps + continue + } + + *prereqList = append(*prereqList, normalizeStringRelativeToTop(config, prereq.Value(nil))) + } + + rules = append(rules, rule) + } + } + + return rules +} diff --git a/android/paths.go b/android/paths.go index 69ab5f75c..e68106ce9 100644 --- a/android/paths.go +++ b/android/paths.go @@ -465,6 +465,9 @@ func (p OutputPaths) Strings() []string { // PathForGoBinary returns the path to the installed location of a bootstrap_go_binary module. func PathForGoBinary(ctx PathContext, goBinary bootstrap.GoBinaryTool) Path { goBinaryInstallDir := pathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "bin", false) + if ctx.Config().KatiEnabled() { + goBinaryInstallDir = goBinaryInstallDir.ToMakePath() + } rel := Rel(ctx, goBinaryInstallDir.String(), goBinary.InstallPath()) return goBinaryInstallDir.Join(ctx, rel) } @@ -1658,6 +1661,12 @@ func PathForModuleInstall(ctx ModuleInstallPathContext, pathComponents ...string return makePathForInstall(ctx, os, arch, partition, ctx.Debug(), pathComponents...) } +// PathForHostDexInstall returns an InstallPath representing the install path for the +// module appended with paths... +func PathForHostDexInstall(ctx ModuleInstallPathContext, pathComponents ...string) InstallPath { + return makePathForInstall(ctx, ctx.Config().BuildOS, ctx.Config().BuildArch, "", ctx.Debug(), pathComponents...) +} + // PathForModuleInPartitionInstall is similar to PathForModuleInstall but partition is provided by the caller func PathForModuleInPartitionInstall(ctx ModuleInstallPathContext, partition string, pathComponents ...string) InstallPath { os, arch := osAndArch(ctx) @@ -2055,7 +2064,12 @@ func maybeRelErr(basePath string, targetPath string) (string, bool, error) { // Writes a file to the output directory. Attempting to write directly to the output directory // will fail due to the sandbox of the soong_build process. func WriteFileToOutputDir(path WritablePath, data []byte, perm os.FileMode) error { - return ioutil.WriteFile(absolutePath(path.String()), data, perm) + absPath := absolutePath(path.String()) + err := os.MkdirAll(filepath.Dir(absPath), 0777) + if err != nil { + return err + } + return ioutil.WriteFile(absPath, data, perm) } func RemoveAllOutputDir(path WritablePath) error { diff --git a/android/paths_test.go b/android/paths_test.go index 3f4625dba..3cad8529d 100644 --- a/android/paths_test.go +++ b/android/paths_test.go @@ -993,7 +993,7 @@ func TestPathForSource(t *testing.T) { { name: "in out dir", buildDir: "out", - src: "out/a/b/c", + src: "out/soong/a/b/c", err: "is in output", }, } @@ -1525,7 +1525,7 @@ func ExampleOutputPath_ReplaceExtension() { fmt.Println(p.Rel(), p2.Rel()) // Output: - // out/system/framework/boot.art out/system/framework/boot.oat + // out/soong/system/framework/boot.art out/soong/system/framework/boot.oat // boot.art boot.oat } @@ -1539,7 +1539,7 @@ func ExampleOutputPath_InSameDir() { fmt.Println(p.Rel(), p2.Rel()) // Output: - // out/system/framework/boot.art out/system/framework/oat/arm/boot.vdex + // out/soong/system/framework/boot.art out/soong/system/framework/oat/arm/boot.vdex // boot.art oat/arm/boot.vdex } diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index a1f8e6367..fa40d1fb3 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -510,9 +510,9 @@ func registerTestPrebuiltModules(ctx RegistrationContext) { ctx.RegisterModuleType("prebuilt", newPrebuiltModule) ctx.RegisterModuleType("source", newSourceModule) ctx.RegisterModuleType("override_source", newOverrideSourceModule) - ctx.RegisterModuleType("soong_config_module_type", soongConfigModuleTypeFactory) - ctx.RegisterModuleType("soong_config_string_variable", soongConfigStringVariableDummyFactory) - ctx.RegisterModuleType("soong_config_bool_variable", soongConfigBoolVariableDummyFactory) + ctx.RegisterModuleType("soong_config_module_type", SoongConfigModuleTypeFactory) + ctx.RegisterModuleType("soong_config_string_variable", SoongConfigStringVariableDummyFactory) + ctx.RegisterModuleType("soong_config_bool_variable", SoongConfigBoolVariableDummyFactory) } type prebuiltModule struct { diff --git a/android/register.go b/android/register.go index 59848627a..424439806 100644 --- a/android/register.go +++ b/android/register.go @@ -161,6 +161,10 @@ func NewContext(config Config) *Context { return ctx } +func (ctx *Context) SetRunningAsBp2build() { + ctx.config.runningAsBp2Build = true +} + // RegisterForBazelConversion registers an alternate shadow pipeline of // singletons, module types and mutators to register for converting Blueprint // files to semantically equivalent BUILD files. diff --git a/android/rule_builder.go b/android/rule_builder.go index 1c6b1c086..f8de5fb96 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -839,6 +839,14 @@ func sboxPathForToolRel(ctx BuilderContext, path Path) string { // The tool is in the Soong output directory, it will be copied to __SBOX_OUT_DIR__/tools/out return filepath.Join(sboxToolsSubDir, "out", relOutSoong) } + if ctx.Config().KatiEnabled() { + toolDir = toolDir.ToMakePath() + relOut, isRelOut, _ := maybeRelErr(toolDir.String(), path.String()) + if isRelOut { + // The tool is in the Make output directory, it will be copied to __SBOX_OUT_DIR__/tools/out + return filepath.Join(sboxToolsSubDir, "out", relOut) + } + } // The tool is in the source directory, it will be copied to __SBOX_OUT_DIR__/tools/src return filepath.Join(sboxToolsSubDir, "src", path.String()) } diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index feee90f29..3766bb03b 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -64,10 +64,10 @@ func ExampleRuleBuilder() { fmt.Printf("outputs: %q\n", rule.Outputs()) // Output: - // commands: "ld a.o b.o -o out/linked && echo success" + // commands: "ld a.o b.o -o out/soong/linked && echo success" // tools: ["ld"] // inputs: ["a.o" "b.o"] - // outputs: ["out/linked"] + // outputs: ["out/soong/linked"] } func ExampleRuleBuilder_SymlinkOutputs() { @@ -79,7 +79,7 @@ func ExampleRuleBuilder_SymlinkOutputs() { Tool(PathForSource(ctx, "ln")). FlagWithInput("-s ", PathForTesting("a.o")). SymlinkOutput(PathForOutput(ctx, "a")) - rule.Command().Text("cp out/a out/b"). + rule.Command().Text("cp out/soong/a out/soong/b"). ImplicitSymlinkOutput(PathForOutput(ctx, "b")) fmt.Printf("commands: %q\n", strings.Join(rule.Commands(), " && ")) @@ -89,11 +89,11 @@ func ExampleRuleBuilder_SymlinkOutputs() { fmt.Printf("symlink_outputs: %q\n", rule.SymlinkOutputs()) // Output: - // commands: "ln -s a.o out/a && cp out/a out/b" + // commands: "ln -s a.o out/soong/a && cp out/soong/a out/soong/b" // tools: ["ln"] // inputs: ["a.o"] - // outputs: ["out/a" "out/b"] - // symlink_outputs: ["out/a" "out/b"] + // outputs: ["out/soong/a" "out/soong/b"] + // symlink_outputs: ["out/soong/a" "out/soong/b"] } func ExampleRuleBuilder_Temporary() { @@ -117,10 +117,10 @@ func ExampleRuleBuilder_Temporary() { fmt.Printf("outputs: %q\n", rule.Outputs()) // Output: - // commands: "cp a out/b && cp out/b out/c" + // commands: "cp a out/soong/b && cp out/soong/b out/soong/c" // tools: ["cp"] // inputs: ["a"] - // outputs: ["out/c"] + // outputs: ["out/soong/c"] } func ExampleRuleBuilder_DeleteTemporaryFiles() { @@ -145,10 +145,10 @@ func ExampleRuleBuilder_DeleteTemporaryFiles() { fmt.Printf("outputs: %q\n", rule.Outputs()) // Output: - // commands: "cp a out/b && cp out/b out/c && rm -f out/b" + // commands: "cp a out/soong/b && cp out/soong/b out/soong/c && rm -f out/soong/b" // tools: ["cp"] // inputs: ["a"] - // outputs: ["out/c"] + // outputs: ["out/soong/c"] } func ExampleRuleBuilder_Installs() { @@ -168,7 +168,7 @@ func ExampleRuleBuilder_Installs() { fmt.Printf("rule.Installs().String() = %q\n", rule.Installs().String()) // Output: - // rule.Installs().String() = "out/linked:/bin/linked out/linked:/sbin/linked" + // rule.Installs().String() = "out/soong/linked:/bin/linked out/soong/linked:/sbin/linked" } func ExampleRuleBuilderCommand() { @@ -271,7 +271,7 @@ func ExampleRuleBuilderCommand_FlagWithRspFileInputList() { FlagWithRspFileInputList("@", PathForOutput(ctx, "foo.rsp"), PathsForTesting("a.java", "b.java")). String()) // Output: - // javac @out/foo.rsp + // javac @out/soong/foo.rsp } func ExampleRuleBuilderCommand_String() { @@ -371,15 +371,15 @@ func TestRuleBuilder(t *testing.T) { addCommands(rule) wantCommands := []string{ - "out_local/module/DepFile Flag FlagWithArg=arg FlagWithDepFile=out_local/module/depfile " + - "FlagWithInput=input FlagWithOutput=out_local/module/output FlagWithRspFileInputList=out_local/rsp " + - "Input out_local/module/Output out_local/module/SymlinkOutput Text Tool after command2 old cmd", - "command2 out_local/module/depfile2 input2 out_local/module/output2 tool2", - "command3 input3 out_local/module/output2 out_local/module/output3 input3 out_local/module/output2", + "out_local/soong/module/DepFile Flag FlagWithArg=arg FlagWithDepFile=out_local/soong/module/depfile " + + "FlagWithInput=input FlagWithOutput=out_local/soong/module/output FlagWithRspFileInputList=out_local/soong/rsp " + + "Input out_local/soong/module/Output out_local/soong/module/SymlinkOutput Text Tool after command2 old cmd", + "command2 out_local/soong/module/depfile2 input2 out_local/soong/module/output2 tool2", + "command3 input3 out_local/soong/module/output2 out_local/soong/module/output3 input3 out_local/soong/module/output2", } - wantDepMergerCommand := "out_local/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer " + - "out_local/module/DepFile out_local/module/depfile out_local/module/ImplicitDepFile out_local/module/depfile2" + wantDepMergerCommand := "out_local/soong/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer " + + "out_local/soong/module/DepFile out_local/soong/module/depfile out_local/soong/module/ImplicitDepFile out_local/soong/module/depfile2" AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands()) @@ -403,13 +403,13 @@ func TestRuleBuilder(t *testing.T) { wantCommands := []string{ "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile " + "FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output " + - "FlagWithRspFileInputList=out_local/rsp Input __SBOX_SANDBOX_DIR__/out/Output " + + "FlagWithRspFileInputList=out_local/soong/rsp Input __SBOX_SANDBOX_DIR__/out/Output " + "__SBOX_SANDBOX_DIR__/out/SymlinkOutput Text Tool after command2 old cmd", "command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 tool2", "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3 input3 __SBOX_SANDBOX_DIR__/out/output2", } - wantDepMergerCommand := "out_local/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2" + wantDepMergerCommand := "out_local/soong/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2" AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands()) @@ -433,7 +433,7 @@ func TestRuleBuilder(t *testing.T) { wantCommands := []string{ "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile " + "FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output " + - "FlagWithRspFileInputList=out_local/rsp Input __SBOX_SANDBOX_DIR__/out/Output " + + "FlagWithRspFileInputList=out_local/soong/rsp Input __SBOX_SANDBOX_DIR__/out/Output " + "__SBOX_SANDBOX_DIR__/out/SymlinkOutput Text __SBOX_SANDBOX_DIR__/tools/src/Tool after command2 old cmd", "command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/tools/src/tool2", "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3 input3 __SBOX_SANDBOX_DIR__/out/output2", diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 17f6d66a8..065440d1f 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -31,10 +31,10 @@ import ( ) func init() { - RegisterModuleType("soong_config_module_type_import", soongConfigModuleTypeImportFactory) - RegisterModuleType("soong_config_module_type", soongConfigModuleTypeFactory) - RegisterModuleType("soong_config_string_variable", soongConfigStringVariableDummyFactory) - RegisterModuleType("soong_config_bool_variable", soongConfigBoolVariableDummyFactory) + RegisterModuleType("soong_config_module_type_import", SoongConfigModuleTypeImportFactory) + RegisterModuleType("soong_config_module_type", SoongConfigModuleTypeFactory) + RegisterModuleType("soong_config_string_variable", SoongConfigStringVariableDummyFactory) + RegisterModuleType("soong_config_bool_variable", SoongConfigBoolVariableDummyFactory) } type soongConfigModuleTypeImport struct { @@ -153,7 +153,7 @@ type soongConfigModuleTypeImportProperties struct { // Then libacme_foo would build with cflags: // "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT". -func soongConfigModuleTypeImportFactory() Module { +func SoongConfigModuleTypeImportFactory() Module { module := &soongConfigModuleTypeImport{} module.AddProperties(&module.properties) @@ -179,6 +179,7 @@ func (*soongConfigModuleTypeImport) GenerateAndroidBuildActions(ModuleContext) { type soongConfigModuleTypeModule struct { ModuleBase + BazelModuleBase properties soongconfig.ModuleTypeProperties } @@ -262,7 +263,7 @@ type soongConfigModuleTypeModule struct { // SOONG_CONFIG_acme_width := 200 // // Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE". -func soongConfigModuleTypeFactory() Module { +func SoongConfigModuleTypeFactory() Module { module := &soongConfigModuleTypeModule{} module.AddProperties(&module.properties) @@ -296,7 +297,7 @@ type soongConfigBoolVariableDummyModule struct { // soong_config_string_variable defines a variable and a set of possible string values for use // in a soong_config_module_type definition. -func soongConfigStringVariableDummyFactory() Module { +func SoongConfigStringVariableDummyFactory() Module { module := &soongConfigStringVariableDummyModule{} module.AddProperties(&module.properties, &module.stringProperties) initAndroidModuleBase(module) @@ -305,7 +306,7 @@ func soongConfigStringVariableDummyFactory() Module { // soong_config_string_variable defines a variable with true or false values for use // in a soong_config_module_type definition. -func soongConfigBoolVariableDummyFactory() Module { +func SoongConfigBoolVariableDummyFactory() Module { module := &soongConfigBoolVariableDummyModule{} module.AddProperties(&module.properties) initAndroidModuleBase(module) @@ -324,6 +325,9 @@ func (m *soongConfigBoolVariableDummyModule) Name() string { func (*soongConfigBoolVariableDummyModule) Nameless() {} func (*soongConfigBoolVariableDummyModule) GenerateAndroidBuildActions(ctx ModuleContext) {} +// importModuleTypes registers the module factories for a list of module types defined +// in an Android.bp file. These module factories are scoped for the current Android.bp +// file only. func importModuleTypes(ctx LoadHookContext, from string, moduleTypes ...string) { from = filepath.Clean(from) if filepath.Ext(from) != ".bp" { @@ -389,7 +393,7 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s for name, moduleType := range mtDef.ModuleTypes { factory := globalModuleTypes[moduleType.BaseModuleType] if factory != nil { - factories[name] = soongConfigModuleFactory(factory, moduleType) + factories[name] = configModuleFactory(factory, moduleType, ctx.Config().runningAsBp2Build) } else { reportErrors(ctx, from, fmt.Errorf("missing global module type factory for %q", moduleType.BaseModuleType)) @@ -404,20 +408,40 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s }).(map[string]blueprint.ModuleFactory) } -// soongConfigModuleFactory takes an existing soongConfigModuleFactory and a ModuleType and returns -// a new soongConfigModuleFactory that wraps the existing soongConfigModuleFactory and adds conditional on Soong config -// variables. -func soongConfigModuleFactory(factory blueprint.ModuleFactory, - moduleType *soongconfig.ModuleType) blueprint.ModuleFactory { - +// configModuleFactory takes an existing soongConfigModuleFactory and a +// ModuleType to create a new ModuleFactory that uses a custom loadhook. +func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType, bp2build bool) blueprint.ModuleFactory { conditionalFactoryProps := soongconfig.CreateProperties(factory, moduleType) - if conditionalFactoryProps.IsValid() { - return func() (blueprint.Module, []interface{}) { - module, props := factory() - - conditionalProps := proptools.CloneEmptyProperties(conditionalFactoryProps) - props = append(props, conditionalProps.Interface()) - + if !conditionalFactoryProps.IsValid() { + return factory + } + useBp2buildHook := bp2build && proptools.BoolDefault(moduleType.Bp2buildAvailable, false) + + return func() (blueprint.Module, []interface{}) { + module, props := factory() + conditionalProps := proptools.CloneEmptyProperties(conditionalFactoryProps) + props = append(props, conditionalProps.Interface()) + + if useBp2buildHook { + // The loadhook is different for bp2build, since we don't want to set a specific + // set of property values based on a vendor var -- we want __all of them__ to + // generate select statements, so we put the entire soong_config_variables + // struct, together with the namespace representing those variables, while + // creating the custom module with the factory. + AddLoadHook(module, func(ctx LoadHookContext) { + if m, ok := module.(Bazelable); ok { + m.SetBaseModuleType(moduleType.BaseModuleType) + // Instead of applying all properties, keep the entire conditionalProps struct as + // part of the custom module so dependent modules can create the selects accordingly + m.setNamespacedVariableProps(namespacedVariableProperties{ + moduleType.ConfigNamespace: conditionalProps.Interface(), + }) + } + }) + } else { + // Regular Soong operation wraps the existing module factory with a + // conditional on Soong config variables by reading the product + // config variables from Make. AddLoadHook(module, func(ctx LoadHookContext) { config := ctx.Config().VendorConfig(moduleType.ConfigNamespace) newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config) @@ -429,10 +453,7 @@ func soongConfigModuleFactory(factory blueprint.ModuleFactory, ctx.AppendProperties(ps) } }) - - return module, props } - } else { - return factory + return module, props } } diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go index 0ec9bcbd5..acb9d180e 100644 --- a/android/soong_config_modules_test.go +++ b/android/soong_config_modules_test.go @@ -310,10 +310,10 @@ func TestSoongConfigModule(t *testing.T) { tc.preparer, PrepareForTestWithDefaults, FixtureRegisterWithContext(func(ctx RegistrationContext) { - ctx.RegisterModuleType("soong_config_module_type_import", soongConfigModuleTypeImportFactory) - ctx.RegisterModuleType("soong_config_module_type", soongConfigModuleTypeFactory) - ctx.RegisterModuleType("soong_config_string_variable", soongConfigStringVariableDummyFactory) - ctx.RegisterModuleType("soong_config_bool_variable", soongConfigBoolVariableDummyFactory) + ctx.RegisterModuleType("soong_config_module_type_import", SoongConfigModuleTypeImportFactory) + ctx.RegisterModuleType("soong_config_module_type", SoongConfigModuleTypeFactory) + ctx.RegisterModuleType("soong_config_string_variable", SoongConfigStringVariableDummyFactory) + ctx.RegisterModuleType("soong_config_bool_variable", SoongConfigBoolVariableDummyFactory) ctx.RegisterModuleType("test_defaults", soongConfigTestDefaultsModuleFactory) ctx.RegisterModuleType("test", soongConfigTestModuleFactory) }), @@ -372,10 +372,10 @@ func TestNonExistentPropertyInSoongConfigModule(t *testing.T) { fixtureForVendorVars(map[string]map[string]string{"acme": {"feature1": "1"}}), PrepareForTestWithDefaults, FixtureRegisterWithContext(func(ctx RegistrationContext) { - ctx.RegisterModuleType("soong_config_module_type_import", soongConfigModuleTypeImportFactory) - ctx.RegisterModuleType("soong_config_module_type", soongConfigModuleTypeFactory) - ctx.RegisterModuleType("soong_config_string_variable", soongConfigStringVariableDummyFactory) - ctx.RegisterModuleType("soong_config_bool_variable", soongConfigBoolVariableDummyFactory) + ctx.RegisterModuleType("soong_config_module_type_import", SoongConfigModuleTypeImportFactory) + ctx.RegisterModuleType("soong_config_module_type", SoongConfigModuleTypeFactory) + ctx.RegisterModuleType("soong_config_string_variable", SoongConfigStringVariableDummyFactory) + ctx.RegisterModuleType("soong_config_bool_variable", SoongConfigBoolVariableDummyFactory) ctx.RegisterModuleType("test_defaults", soongConfigTestDefaultsModuleFactory) ctx.RegisterModuleType("test", soongConfigTestModuleFactory) }), diff --git a/android/soongconfig/Android.bp b/android/soongconfig/Android.bp index e7fa5a036..9bf334480 100644 --- a/android/soongconfig/Android.bp +++ b/android/soongconfig/Android.bp @@ -9,6 +9,7 @@ bootstrap_go_package { "blueprint", "blueprint-parser", "blueprint-proptools", + "soong-bazel", ], srcs: [ "config.go", diff --git a/android/soongconfig/modules.go b/android/soongconfig/modules.go index 34b180d7d..1af89ba5d 100644 --- a/android/soongconfig/modules.go +++ b/android/soongconfig/modules.go @@ -15,6 +15,7 @@ package soongconfig import ( + "android/soong/bazel" "fmt" "io" "reflect" @@ -28,7 +29,7 @@ import ( const conditionsDefault = "conditions_default" -var soongConfigProperty = proptools.FieldNameForProperty("soong_config_variables") +var SoongConfigProperty = proptools.FieldNameForProperty("soong_config_variables") // loadSoongConfigModuleTypeDefinition loads module types from an Android.bp file. It caches the // result so each file is only parsed once. @@ -120,6 +121,8 @@ type ModuleTypeProperties struct { // the list of properties that this module type will extend. Properties []string + + Bazel_module bazel.BazelModuleProperties } func processModuleTypeDef(v *SoongConfigDefinition, def *parser.Module) (errs []error) { @@ -271,12 +274,12 @@ func CreateProperties(factory blueprint.ModuleFactory, moduleType *ModuleType) r } typ := reflect.StructOf([]reflect.StructField{{ - Name: soongConfigProperty, + Name: SoongConfigProperty, Type: reflect.StructOf(fields), }}) props := reflect.New(typ) - structConditions := props.Elem().FieldByName(soongConfigProperty) + structConditions := props.Elem().FieldByName(SoongConfigProperty) for i, c := range moduleType.Variables { c.initializeProperties(structConditions.Field(i), affectablePropertiesType) @@ -415,7 +418,7 @@ func typeForPropertyFromPropertyStruct(ps interface{}, property string) reflect. // soong_config_variables are expected to be in the same order as moduleType.Variables. func PropertiesToApply(moduleType *ModuleType, props reflect.Value, config SoongConfig) ([]interface{}, error) { var ret []interface{} - props = props.Elem().FieldByName(soongConfigProperty) + props = props.Elem().FieldByName(SoongConfigProperty) for i, c := range moduleType.Variables { if ps, err := c.PropertiesToApply(config, props.Field(i)); err != nil { return nil, err @@ -433,6 +436,7 @@ type ModuleType struct { affectableProperties []string variableNames []string + Bp2buildAvailable *bool } func newModuleType(props *ModuleTypeProperties) (*ModuleType, []error) { @@ -441,6 +445,7 @@ func newModuleType(props *ModuleTypeProperties) (*ModuleType, []error) { ConfigNamespace: props.Config_namespace, BaseModuleType: props.Module_type, variableNames: props.Variables, + Bp2buildAvailable: props.Bazel_module.Bp2build_available, } for _, name := range props.Bool_variables { diff --git a/android/testing.go b/android/testing.go index b9d8fa878..6290d4317 100644 --- a/android/testing.go +++ b/android/testing.go @@ -458,6 +458,7 @@ func (ctx *TestContext) Register() { // RegisterForBazelConversion prepares a test context for bp2build conversion. func (ctx *TestContext) RegisterForBazelConversion() { + ctx.SetRunningAsBp2build() RegisterMutatorsForBazelConversion(ctx.Context, ctx.bp2buildPreArch, ctx.bp2buildMutators) } diff --git a/android/variable.go b/android/variable.go index e9436401c..6ad58c3f2 100644 --- a/android/variable.go +++ b/android/variable.go @@ -15,6 +15,8 @@ package android import ( + "android/soong/android/soongconfig" + "android/soong/bazel" "fmt" "reflect" "runtime" @@ -487,13 +489,97 @@ type ProductConfigContext interface { // ProductConfigProperty contains the information for a single property (may be a struct) paired // with the appropriate ProductConfigVariable. type ProductConfigProperty struct { + // The name of the product variable, e.g. "safestack", "malloc_not_svelte", + // "board" ProductConfigVariable string - FullConfig string - Property interface{} + + // Namespace of the variable, if this is a soong_config_module_type variable + // e.g. "acme", "ANDROID", "vendor_nae" + Namespace string // for soong config variables + + // Unique configuration to identify this product config property (i.e. a + // primary key), as just using the product variable name is not sufficient. + // + // For product variables, this is the product variable name + optional + // archvariant information. e.g. + // + // product_variables: { + // foo: { + // cflags: ["-Dfoo"], + // }, + // }, + // + // FullConfig would be "foo". + // + // target: { + // android: { + // product_variables: { + // foo: { + // cflags: ["-Dfoo-android"], + // }, + // }, + // }, + // }, + // + // FullConfig would be "foo-android". + // + // For soong config variables, this is the namespace + product variable name + // + value of the variable, if applicable. The value can also be + // conditions_default. + // + // e.g. + // + // soong_config_variables: { + // feature1: { + // conditions_default: { + // cflags: ["-DDEFAULT1"], + // }, + // cflags: ["-DFEATURE1"], + // }, + // } + // + // where feature1 is created in the "acme" namespace, so FullConfig would be + // "acme__feature1" and "acme__feature1__conditions_default". + // + // e.g. + // + // soong_config_variables: { + // board: { + // soc_a: { + // cflags: ["-DSOC_A"], + // }, + // soc_b: { + // cflags: ["-DSOC_B"], + // }, + // soc_c: {}, + // conditions_default: { + // cflags: ["-DSOC_DEFAULT"] + // }, + // }, + // } + // + // where board is created in the "acme" namespace, so FullConfig would be + // "acme__board__soc_a", "acme__board__soc_b", and + // "acme__board__conditions_default" + FullConfig string + + // The actual property value: list, bool, string.. + Property interface{} +} + +func (p *ProductConfigProperty) ConfigurationAxis() bazel.ConfigurationAxis { + if p.Namespace == "" { + return bazel.ProductVariableConfigurationAxis(p.FullConfig) + } else { + // Soong config variables can be uniquely identified by the namespace + // (e.g. acme, android) and the product variable name (e.g. board, size) + return bazel.ProductVariableConfigurationAxis(p.Namespace + "__" + p.ProductConfigVariable) + } } -// ProductConfigProperties is a map of property name to a slice of ProductConfigProperty such that -// all it all product variable-specific versions of a property are easily accessed together +// ProductConfigProperties is a map of property name to a slice of +// ProductConfigProperty such that all product variable-specific versions of a +// property are easily accessed together type ProductConfigProperties map[string]map[string]ProductConfigProperty // ProductVariableProperties returns a ProductConfigProperties containing only the properties which @@ -504,36 +590,165 @@ func ProductVariableProperties(ctx BazelConversionPathContext) ProductConfigProp productConfigProperties := ProductConfigProperties{} - if moduleBase.variableProperties == nil { - return productConfigProperties + if moduleBase.variableProperties != nil { + productVariablesProperty := proptools.FieldNameForProperty("product_variables") + productVariableValues( + productVariablesProperty, + moduleBase.variableProperties, + "", + "", + &productConfigProperties) + + for _, configToProps := range moduleBase.GetArchVariantProperties(ctx, moduleBase.variableProperties) { + for config, props := range configToProps { + // GetArchVariantProperties is creating an instance of the requested type + // and productVariablesValues expects an interface, so no need to cast + productVariableValues( + productVariablesProperty, + props, + "", + config, + &productConfigProperties) + } + } } - productVariableValues(moduleBase.variableProperties, "", &productConfigProperties) - - for _, configToProps := range moduleBase.GetArchVariantProperties(ctx, moduleBase.variableProperties) { - for config, props := range configToProps { - // GetArchVariantProperties is creating an instance of the requested type - // and productVariablesValues expects an interface, so no need to cast - productVariableValues(props, config, &productConfigProperties) + if m, ok := module.(Bazelable); ok && m.namespacedVariableProps() != nil { + for namespace, namespacedVariableProp := range m.namespacedVariableProps() { + productVariableValues( + soongconfig.SoongConfigProperty, + namespacedVariableProp, + namespace, + "", + &productConfigProperties) } } return productConfigProperties } -func productVariableValues(variableProps interface{}, suffix string, productConfigProperties *ProductConfigProperties) { +func (p *ProductConfigProperties) AddProductConfigProperty( + propertyName, namespace, productVariableName, config string, property interface{}) { + if (*p)[propertyName] == nil { + (*p)[propertyName] = make(map[string]ProductConfigProperty) + } + + // Normalize config to be all lowercase. It's the "primary key" of this + // unique property value. This can be the conditions_default value of the + // product variable as well. + config = strings.ToLower(config) + (*p)[propertyName][config] = ProductConfigProperty{ + Namespace: namespace, // e.g. acme, android + ProductConfigVariable: productVariableName, // e.g. size, feature1, feature2, FEATURE3, board + FullConfig: config, // e.g. size, feature1-x86, size__conditions_default + Property: property, // e.g. ["-O3"] + } +} + +var ( + conditionsDefaultField string = proptools.FieldNameForProperty(bazel.ConditionsDefaultConfigKey) +) + +// maybeExtractConfigVarProp attempts to read this value as a config var struct +// wrapped by interfaces and ptrs. If it's not the right type, the second return +// value is false. +func maybeExtractConfigVarProp(v reflect.Value) (reflect.Value, bool) { + if v.Kind() == reflect.Interface { + // The conditions_default value can be either + // 1) an ptr to an interface of a struct (bool config variables and product variables) + // 2) an interface of 1) (config variables with nested structs, like string vars) + v = v.Elem() + } + if v.Kind() != reflect.Ptr { + return v, false + } + v = reflect.Indirect(v) + if v.Kind() == reflect.Interface { + // Extract the struct from the interface + v = v.Elem() + } + + if !v.IsValid() { + return v, false + } + + if v.Kind() != reflect.Struct { + return v, false + } + return v, true +} + +// productVariableValues uses reflection to convert a property struct for +// product_variables and soong_config_variables to structs that can be generated +// as select statements. +func productVariableValues( + fieldName string, variableProps interface{}, namespace, suffix string, productConfigProperties *ProductConfigProperties) { if suffix != "" { suffix = "-" + suffix } - variableValues := reflect.ValueOf(variableProps).Elem().FieldByName("Product_variables") + + // variableValues represent the product_variables or soong_config_variables + // struct. + variableValues := reflect.ValueOf(variableProps).Elem().FieldByName(fieldName) + + // Example of product_variables: + // + // product_variables: { + // malloc_not_svelte: { + // shared_libs: ["malloc_not_svelte_shared_lib"], + // whole_static_libs: ["malloc_not_svelte_whole_static_lib"], + // exclude_static_libs: [ + // "malloc_not_svelte_static_lib_excludes", + // "malloc_not_svelte_whole_static_lib_excludes", + // ], + // }, + // }, + // + // Example of soong_config_variables: + // + // soong_config_variables: { + // feature1: { + // conditions_default: { + // ... + // }, + // cflags: ... + // }, + // feature2: { + // cflags: ... + // conditions_default: { + // ... + // }, + // }, + // board: { + // soc_a: { + // ... + // }, + // soc_a: { + // ... + // }, + // soc_c: {}, + // conditions_default: { + // ... + // }, + // }, + // } for i := 0; i < variableValues.NumField(); i++ { + // e.g. Platform_sdk_version, Unbundled_build, Malloc_not_svelte, etc. + productVariableName := variableValues.Type().Field(i).Name + variableValue := variableValues.Field(i) // Check if any properties were set for the module if variableValue.IsZero() { + // e.g. feature1: {}, malloc_not_svelte: {} continue } - // e.g. Platform_sdk_version, Unbundled_build, Malloc_not_svelte, etc. - productVariableName := variableValues.Type().Field(i).Name + + // Unlike product variables, config variables require a few more + // indirections to extract the struct from the reflect.Value. + if v, ok := maybeExtractConfigVarProp(variableValue); ok { + variableValue = v + } + for j := 0; j < variableValue.NumField(); j++ { property := variableValue.Field(j) // If the property wasn't set, no need to pass it along @@ -543,14 +758,57 @@ func productVariableValues(variableProps interface{}, suffix string, productConf // e.g. Asflags, Cflags, Enabled, etc. propertyName := variableValue.Type().Field(j).Name - if (*productConfigProperties)[propertyName] == nil { - (*productConfigProperties)[propertyName] = make(map[string]ProductConfigProperty) - } - config := productVariableName + suffix - (*productConfigProperties)[propertyName][config] = ProductConfigProperty{ - ProductConfigVariable: productVariableName, - FullConfig: config, - Property: property.Interface(), + + if v, ok := maybeExtractConfigVarProp(property); ok { + // The field is a struct, which is used by: + // 1) soong_config_string_variables + // + // soc_a: { + // cflags: ..., + // } + // + // soc_b: { + // cflags: ..., + // } + // + // 2) conditions_default structs for all soong config variable types. + // + // conditions_default: { + // cflags: ..., + // static_libs: ... + // } + field := v + for k := 0; k < field.NumField(); k++ { + // Iterate over fields of this struct prop. + if field.Field(k).IsZero() { + continue + } + productVariableValue := proptools.PropertyNameForField(propertyName) + config := strings.Join([]string{namespace, productVariableName, productVariableValue}, "__") + actualPropertyName := field.Type().Field(k).Name + + productConfigProperties.AddProductConfigProperty( + actualPropertyName, // e.g. cflags, static_libs + namespace, // e.g. acme, android + productVariableName, // e.g. size, feature1, FEATURE2, board + config, + field.Field(k).Interface(), // e.g. ["-DDEFAULT"], ["foo", "bar"] + ) + } + } else { + // Not a conditions_default or a struct prop, i.e. regular + // product variables, or not a string-typed config var. + config := productVariableName + suffix + if namespace != "" { + config = namespace + "__" + config + } + productConfigProperties.AddProductConfigProperty( + propertyName, + namespace, + productVariableName, + config, + property.Interface(), + ) } } } diff --git a/apex/androidmk.go b/apex/androidmk.go index 17b2849f8..4f37fc3d5 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -326,6 +326,9 @@ func (a *apexBundle) writeRequiredModules(w io.Writer, apexBundleName string) { var required []string var targetRequired []string var hostRequired []string + required = append(required, a.RequiredModuleNames()...) + targetRequired = append(targetRequired, a.TargetRequiredModuleNames()...) + hostRequired = append(hostRequired, a.HostRequiredModuleNames()...) installMapSet := make(map[string]bool) // set of dependency module:location mappings for _, fi := range a.filesInfo { required = append(required, fi.requiredModuleNames...) diff --git a/apex/apex_test.go b/apex/apex_test.go index 8aaa31afa..59ea206f4 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -8350,6 +8350,46 @@ func TestAndroidMk_DexpreoptBuiltInstalledForApex_Prebuilt(t *testing.T) { }) } +func TestAndroidMk_RequiredModules(t *testing.T) { + ctx := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + updatable: false, + java_libs: ["foo"], + required: ["otherapex"], + } + + apex { + name: "otherapex", + key: "myapex.key", + updatable: false, + java_libs: ["foo"], + required: ["otherapex"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_library { + name: "foo", + srcs: ["foo.java"], + apex_available: ["myapex", "otherapex"], + installable: true, + } + `) + + apexBundle := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + data := android.AndroidMkDataForTest(t, ctx, apexBundle) + var builder strings.Builder + data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES += otherapex") +} + func TestMain(m *testing.M) { os.Exit(m.Run()) } diff --git a/bazel/aquery.go b/bazel/aquery.go index 0dedcf492..6d96b1ca1 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -245,9 +245,9 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) { out := outputPaths[0] outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) out = proptools.ShellEscapeIncludingSpaces(out) - in := proptools.ShellEscapeIncludingSpaces(inputPaths[0]) - // Use hard links, because some soong actions expect real files (for example, `cp -d`). - buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -f %[3]s %[2]s", outDir, out, in) + in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) + // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). + buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) buildStatement.SymlinkPaths = outputPaths[:] } else if len(actionEntry.Arguments) < 1 { return nil, fmt.Errorf("received action with no command: [%v]", buildStatement) diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index 88066c8fe..69f11152e 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -859,7 +859,7 @@ func TestSimpleSymlink(t *testing.T) { BuildStatement{ Command: "mkdir -p one/symlink_subdir && " + "rm -f one/symlink_subdir/symlink && " + - "ln -f one/file_subdir/file one/symlink_subdir/symlink", + "ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink", InputPaths: []string{"one/file_subdir/file"}, OutputPaths: []string{"one/symlink_subdir/symlink"}, SymlinkPaths: []string{"one/symlink_subdir/symlink"}, @@ -923,14 +923,14 @@ func TestSymlinkQuotesPaths(t *testing.T) { BuildStatement{ Command: "mkdir -p 'one/symlink subdir' && " + "rm -f 'one/symlink subdir/symlink' && " + - "ln -f 'one/file subdir/file' 'one/symlink subdir/symlink'", + "ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'", InputPaths: []string{"one/file subdir/file"}, OutputPaths: []string{"one/symlink subdir/symlink"}, SymlinkPaths: []string{"one/symlink subdir/symlink"}, Mnemonic: "SolibSymlink", }, } - assertBuildStatements(t, actual, expectedBuildStatements) + assertBuildStatements(t, expectedBuildStatements, actual) } func TestSymlinkMultipleInputs(t *testing.T) { diff --git a/bazel/configurability.go b/bazel/configurability.go index f05c8e55c..1993f76fc 100644 --- a/bazel/configurability.go +++ b/bazel/configurability.go @@ -158,9 +158,9 @@ func (ct configurationType) validateConfig(config string) { } // SelectKey returns the Bazel select key for a given configurationType and config string. -func (ct configurationType) SelectKey(config string) string { - ct.validateConfig(config) - switch ct { +func (ca ConfigurationAxis) SelectKey(config string) string { + ca.validateConfig(config) + switch ca.configurationType { case noConfig: panic(fmt.Errorf("SelectKey is unnecessary for noConfig ConfigurationType ")) case arch: @@ -170,12 +170,13 @@ func (ct configurationType) SelectKey(config string) string { case osArch: return platformOsArchMap[config] case productVariables: - if config == ConditionsDefaultConfigKey { + if strings.HasSuffix(config, ConditionsDefaultConfigKey) { + // e.g. "acme__feature1__conditions_default" or "android__board__conditions_default" return ConditionsDefaultSelectKey } - return fmt.Sprintf("%s:%s", productVariableBazelPackage, strings.ToLower(config)) + return fmt.Sprintf("%s:%s", productVariableBazelPackage, config) default: - panic(fmt.Errorf("Unrecognized ConfigurationType %d", ct)) + panic(fmt.Errorf("Unrecognized ConfigurationType %d", ca.configurationType)) } } diff --git a/bazel/properties.go b/bazel/properties.go index facbedd9a..a438481be 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -24,6 +24,23 @@ import ( "github.com/google/blueprint" ) +type BazelModuleProperties struct { + // The label of the Bazel target replacing this Soong module. When run in conversion mode, this + // will import the handcrafted build target into the autogenerated file. Note: this may result in + // a conflict due to duplicate targets if bp2build_available is also set. + Label *string + + // If true, bp2build will generate the converted Bazel target for this module. Note: this may + // cause a conflict due to the duplicate targets if label is also set. + // + // This is a bool pointer to support tristates: true, false, not set. + // + // To opt-in a module, set bazel_module: { bp2build_available: true } + // To opt-out a module, set bazel_module: { bp2build_available: false } + // To defer the default setting for the directory, do not set the value. + Bp2build_available *bool +} + // BazelTargetModuleProperties contain properties and metadata used for // Blueprint to BUILD file conversion. type BazelTargetModuleProperties struct { diff --git a/bp2build/soong_config_module_type_conversion_test.go b/bp2build/soong_config_module_type_conversion_test.go new file mode 100644 index 000000000..41184794f --- /dev/null +++ b/bp2build/soong_config_module_type_conversion_test.go @@ -0,0 +1,257 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "android/soong/cc" + "testing" +) + +func runSoongConfigModuleTypeTest(t *testing.T, tc bp2buildTestCase) { + t.Helper() + runBp2BuildTestCase(t, registerSoongConfigModuleTypes, tc) +} + +func registerSoongConfigModuleTypes(ctx android.RegistrationContext) { + cc.RegisterCCBuildComponents(ctx) + + ctx.RegisterModuleType("soong_config_module_type_import", android.SoongConfigModuleTypeImportFactory) + ctx.RegisterModuleType("soong_config_module_type", android.SoongConfigModuleTypeFactory) + ctx.RegisterModuleType("soong_config_string_variable", android.SoongConfigStringVariableDummyFactory) + ctx.RegisterModuleType("soong_config_bool_variable", android.SoongConfigBoolVariableDummyFactory) +} + +func TestSoongConfigModuleType(t *testing.T) { + bp := ` +soong_config_module_type { + name: "custom_cc_library_static", + module_type: "cc_library_static", + config_namespace: "acme", + bool_variables: ["feature1"], + properties: ["cflags"], + bazel_module: { bp2build_available: true }, +} + +custom_cc_library_static { + name: "foo", + bazel_module: { bp2build_available: true }, + soong_config_variables: { + feature1: { + conditions_default: { + cflags: ["-DDEFAULT1"], + }, + cflags: ["-DFEATURE1"], + }, + }, +} +` + + runSoongConfigModuleTypeTest(t, bp2buildTestCase{ + description: "soong config variables - soong_config_module_type is supported in bp2build", + moduleTypeUnderTest: "cc_library_static", + moduleTypeUnderTestFactory: cc.LibraryStaticFactory, + moduleTypeUnderTestBp2BuildMutator: cc.CcLibraryStaticBp2Build, + blueprint: bp, + expectedBazelTargets: []string{`cc_library_static( + name = "foo", + copts = select({ + "//build/bazel/product_variables:acme__feature1": ["-DFEATURE1"], + "//conditions:default": ["-DDEFAULT1"], + }), + local_includes = ["."], +)`}}) +} + +func TestSoongConfigModuleTypeImport(t *testing.T) { + configBp := ` +soong_config_module_type { + name: "custom_cc_library_static", + module_type: "cc_library_static", + config_namespace: "acme", + bool_variables: ["feature1"], + properties: ["cflags"], + bazel_module: { bp2build_available: true }, +} +` + bp := ` +soong_config_module_type_import { + from: "foo/bar/SoongConfig.bp", + module_types: ["custom_cc_library_static"], +} + +custom_cc_library_static { + name: "foo", + bazel_module: { bp2build_available: true }, + soong_config_variables: { + feature1: { + conditions_default: { + cflags: ["-DDEFAULT1"], + }, + cflags: ["-DFEATURE1"], + }, + }, +} +` + + runSoongConfigModuleTypeTest(t, bp2buildTestCase{ + description: "soong config variables - soong_config_module_type_import is supported in bp2build", + moduleTypeUnderTest: "cc_library_static", + moduleTypeUnderTestFactory: cc.LibraryStaticFactory, + moduleTypeUnderTestBp2BuildMutator: cc.CcLibraryStaticBp2Build, + filesystem: map[string]string{ + "foo/bar/SoongConfig.bp": configBp, + }, + blueprint: bp, + expectedBazelTargets: []string{`cc_library_static( + name = "foo", + copts = select({ + "//build/bazel/product_variables:acme__feature1": ["-DFEATURE1"], + "//conditions:default": ["-DDEFAULT1"], + }), + local_includes = ["."], +)`}}) +} + +func TestSoongConfigModuleType_StringVar(t *testing.T) { + bp := ` +soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b", "soc_c"], +} + +soong_config_module_type { + name: "custom_cc_library_static", + module_type: "cc_library_static", + config_namespace: "acme", + variables: ["board"], + properties: ["cflags"], + bazel_module: { bp2build_available: true }, +} + +custom_cc_library_static { + name: "foo", + bazel_module: { bp2build_available: true }, + soong_config_variables: { + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + soc_b: { + cflags: ["-DSOC_B"], + }, + soc_c: {}, + conditions_default: { + cflags: ["-DSOC_DEFAULT"] + }, + }, + }, +} +` + + runSoongConfigModuleTypeTest(t, bp2buildTestCase{ + description: "soong config variables - generates selects for string vars", + moduleTypeUnderTest: "cc_library_static", + moduleTypeUnderTestFactory: cc.LibraryStaticFactory, + moduleTypeUnderTestBp2BuildMutator: cc.CcLibraryStaticBp2Build, + blueprint: bp, + expectedBazelTargets: []string{`cc_library_static( + name = "foo", + copts = select({ + "//build/bazel/product_variables:acme__board__soc_a": ["-DSOC_A"], + "//build/bazel/product_variables:acme__board__soc_b": ["-DSOC_B"], + "//conditions:default": ["-DSOC_DEFAULT"], + }), + local_includes = ["."], +)`}}) +} + +func TestSoongConfigModuleType_StringAndBoolVar(t *testing.T) { + bp := ` +soong_config_bool_variable { + name: "feature1", +} + +soong_config_bool_variable { + name: "feature2", +} + +soong_config_string_variable { + name: "board", + values: ["soc_a", "soc_b", "soc_c"], +} + +soong_config_module_type { + name: "custom_cc_library_static", + module_type: "cc_library_static", + config_namespace: "acme", + variables: ["feature1", "feature2", "board"], + properties: ["cflags"], + bazel_module: { bp2build_available: true }, +} + +custom_cc_library_static { + name: "foo", + bazel_module: { bp2build_available: true }, + soong_config_variables: { + feature1: { + conditions_default: { + cflags: ["-DDEFAULT1"], + }, + cflags: ["-DFEATURE1"], + }, + feature2: { + cflags: ["-DFEATURE2"], + conditions_default: { + cflags: ["-DDEFAULT2"], + }, + }, + board: { + soc_a: { + cflags: ["-DSOC_A"], + }, + soc_b: { + cflags: ["-DSOC_B"], + }, + soc_c: {}, + conditions_default: { + cflags: ["-DSOC_DEFAULT"] + }, + }, + }, +}` + + runSoongConfigModuleTypeTest(t, bp2buildTestCase{ + description: "soong config variables - generates selects for multiple variable types", + moduleTypeUnderTest: "cc_library_static", + moduleTypeUnderTestFactory: cc.LibraryStaticFactory, + moduleTypeUnderTestBp2BuildMutator: cc.CcLibraryStaticBp2Build, + blueprint: bp, + expectedBazelTargets: []string{`cc_library_static( + name = "foo", + copts = select({ + "//build/bazel/product_variables:acme__board__soc_a": ["-DSOC_A"], + "//build/bazel/product_variables:acme__board__soc_b": ["-DSOC_B"], + "//conditions:default": ["-DSOC_DEFAULT"], + }) + select({ + "//build/bazel/product_variables:acme__feature1": ["-DFEATURE1"], + "//conditions:default": ["-DDEFAULT1"], + }) + select({ + "//build/bazel/product_variables:acme__feature2": ["-DFEATURE2"], + "//conditions:default": ["-DDEFAULT2"], + }), + local_includes = ["."], +)`}}) +} diff --git a/build_kzip.bash b/build_kzip.bash index 56550677f..aff2d6d03 100755 --- a/build_kzip.bash +++ b/build_kzip.bash @@ -61,5 +61,5 @@ declare -r kzip_count=$(find "$out" -name '*.kzip' | wc -l) # Pack # TODO(asmundak): this should be done by soong. declare -r allkzip="$KZIP_NAME.kzip" -"$out/soong/host/linux-x86/bin/merge_zips" "$DIST_DIR/$allkzip" @<(find "$out" -name '*.kzip') +"$out/host/linux-x86/bin/merge_zips" "$DIST_DIR/$allkzip" @<(find "$out" -name '*.kzip') diff --git a/cc/androidmk.go b/cc/androidmk.go index 93283d09d..5c4ef1705 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -450,11 +450,6 @@ func (installer *baseInstaller) AndroidMkEntries(ctx AndroidMkContext, entries * if installer.path == (android.InstallPath{}) { return } - // Soong installation is only supported for host modules. Have Make - // installation trigger Soong installation. - if ctx.Target().Os.Class == android.Host { - entries.OutputFile = android.OptionalPathForPath(installer.path) - } entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { path, file := filepath.Split(installer.path.ToMakePath().String()) diff --git a/cc/bp2build.go b/cc/bp2build.go index 1b1385480..2059f5e53 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -327,7 +327,7 @@ func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversio ctx.ModuleErrorf("Could not convert product variable %s property", proptools.PropertyNameForField(propName)) } newFlags, _ := bazel.TryVariableSubstitutions(flags, prop.ProductConfigVariable) - attr.SetSelectValue(bazel.ProductVariableConfigurationAxis(prop.FullConfig), prop.FullConfig, newFlags) + attr.SetSelectValue(prop.ConfigurationAxis(), prop.FullConfig, newFlags) } } } @@ -611,7 +611,7 @@ func (la *linkerAttributes) convertProductVariables(ctx android.BazelConversionP ctx.ModuleErrorf("Could not convert product variable %s property", dep.excludesField) } - dep.attribute.SetSelectValue(bazel.ProductVariableConfigurationAxis(config), config, dep.depResolutionFunc(ctx, android.FirstUniqueStrings(includes), excludes)) + dep.attribute.SetSelectValue(prop.ConfigurationAxis(), config, dep.depResolutionFunc(ctx, android.FirstUniqueStrings(includes), excludes)) } } } diff --git a/cc/builder.go b/cc/builder.go index abd5f1db9..72c2fa555 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -949,8 +949,7 @@ func sourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceD } // Generate a rule for extracting a table of contents from a shared library (.so) -func transformSharedObjectToToc(ctx android.ModuleContext, inputFile android.Path, - outputFile android.WritablePath, flags builderFlags) { +func TransformSharedObjectToToc(ctx android.ModuleContext, inputFile android.Path, outputFile android.WritablePath) { var format string if ctx.Darwin() { @@ -1365,6 +1365,8 @@ func (c *Module) InstallInRoot() bool { return c.installer != nil && c.installer.installInRoot() } +func (c *Module) InstallBypassMake() bool { return true } + type baseModuleContext struct { android.BaseModuleContext moduleContextImpl @@ -1715,7 +1717,7 @@ func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { bazelActionsUsed := false // Mixed builds mode is disabled for modules outside of device OS. // TODO(b/200841190): Support non-device OS in mixed builds. - if c.MixedBuildsEnabled(actx) && c.bazelHandler != nil && actx.Os().Class == android.Device { + if c.MixedBuildsEnabled(actx) && c.bazelHandler != nil { bazelActionsUsed = c.bazelHandler.GenerateBazelBuildActions(actx, bazelModuleLabel) } return bazelActionsUsed diff --git a/cc/cc_test.go b/cc/cc_test.go index 4c9f5799b..ad2ccae42 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -346,7 +346,7 @@ func checkVndkOutput(t *testing.T, ctx *android.TestContext, output string, expe func checkVndkLibrariesOutput(t *testing.T, ctx *android.TestContext, module string, expected []string) { t.Helper() - got := ctx.ModuleForTests(module, "").Module().(*vndkLibrariesTxt).fileNames + got := ctx.ModuleForTests(module, "android_common").Module().(*vndkLibrariesTxt).fileNames assertArrayString(t, got, expected) } @@ -532,11 +532,11 @@ func TestVndk(t *testing.T) { CheckSnapshot(t, ctx, snapshotSingleton, "libllndk", "libllndk.so", llndkLib2ndPath, variant2nd) snapshotConfigsPath := filepath.Join(snapshotVariantPath, "configs") - CheckSnapshot(t, ctx, snapshotSingleton, "llndk.libraries.txt", "llndk.libraries.txt", snapshotConfigsPath, "") - CheckSnapshot(t, ctx, snapshotSingleton, "vndkcore.libraries.txt", "vndkcore.libraries.txt", snapshotConfigsPath, "") - CheckSnapshot(t, ctx, snapshotSingleton, "vndksp.libraries.txt", "vndksp.libraries.txt", snapshotConfigsPath, "") - CheckSnapshot(t, ctx, snapshotSingleton, "vndkprivate.libraries.txt", "vndkprivate.libraries.txt", snapshotConfigsPath, "") - CheckSnapshot(t, ctx, snapshotSingleton, "vndkproduct.libraries.txt", "vndkproduct.libraries.txt", snapshotConfigsPath, "") + CheckSnapshot(t, ctx, snapshotSingleton, "llndk.libraries.txt", "llndk.libraries.txt", snapshotConfigsPath, "android_common") + CheckSnapshot(t, ctx, snapshotSingleton, "vndkcore.libraries.txt", "vndkcore.libraries.txt", snapshotConfigsPath, "android_common") + CheckSnapshot(t, ctx, snapshotSingleton, "vndksp.libraries.txt", "vndksp.libraries.txt", snapshotConfigsPath, "android_common") + CheckSnapshot(t, ctx, snapshotSingleton, "vndkprivate.libraries.txt", "vndkprivate.libraries.txt", snapshotConfigsPath, "android_common") + CheckSnapshot(t, ctx, snapshotSingleton, "vndkproduct.libraries.txt", "vndkproduct.libraries.txt", snapshotConfigsPath, "android_common") checkVndkOutput(t, ctx, "vndk/vndk.libraries.txt", []string{ "LLNDK: libc.so", @@ -614,7 +614,7 @@ func TestVndkLibrariesTxtAndroidMk(t *testing.T) { config.TestProductVariables.Platform_vndk_version = StringPtr("29") ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("llndk.libraries.txt", "") + module := ctx.ModuleForTests("llndk.libraries.txt", "android_common") entries := android.AndroidMkEntriesForTest(t, ctx, module.Module())[0] assertArrayString(t, entries.EntryMap["LOCAL_MODULE_STEM"], []string{"llndk.libraries.29.txt"}) } @@ -3585,6 +3585,58 @@ func TestAidlFlagsPassedToTheAidlCompiler(t *testing.T) { } } +func TestAidlFlagsWithMinSdkVersion(t *testing.T) { + for _, tc := range []struct { + name string + sdkVersion string + variant string + expected string + }{ + { + name: "default is current", + sdkVersion: "", + variant: "android_arm64_armv8-a_static", + expected: "platform_apis", + }, + { + name: "use sdk_version", + sdkVersion: `sdk_version: "29"`, + variant: "android_arm64_armv8-a_static", + expected: "platform_apis", + }, + { + name: "use sdk_version(sdk variant)", + sdkVersion: `sdk_version: "29"`, + variant: "android_arm64_armv8-a_sdk_static", + expected: "29", + }, + { + name: "use min_sdk_version", + sdkVersion: `min_sdk_version: "29"`, + variant: "android_arm64_armv8-a_static", + expected: "29", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := testCc(t, ` + cc_library { + name: "libfoo", + stl: "none", + srcs: ["a/Foo.aidl"], + `+tc.sdkVersion+` + } + `) + libfoo := ctx.ModuleForTests("libfoo", tc.variant) + manifest := android.RuleBuilderSboxProtoForTests(t, libfoo.Output("aidl.sbox.textproto")) + aidlCommand := manifest.Commands[0].GetCommand() + expectedAidlFlag := "--min_sdk_version=" + tc.expected + if !strings.Contains(aidlCommand, expectedAidlFlag) { + t.Errorf("aidl command %q does not contain %q", aidlCommand, expectedAidlFlag) + } + }) + } +} + func TestMinSdkVersionInClangTriple(t *testing.T) { ctx := testCc(t, ` cc_library_shared { diff --git a/cc/ccdeps.go b/cc/ccdeps.go index b96d8b007..75e1faf0b 100644 --- a/cc/ccdeps.go +++ b/cc/ccdeps.go @@ -44,11 +44,9 @@ type ccdepsGeneratorSingleton struct { var _ android.SingletonMakeVarsProvider = (*ccdepsGeneratorSingleton)(nil) const ( - // Environment variables used to control the behavior of this singleton. - envVariableCollectCCDeps = "SOONG_COLLECT_CC_DEPS" - ccdepsJsonFileName = "module_bp_cc_deps.json" - cClang = "clang" - cppClang = "clang++" + ccdepsJsonFileName = "module_bp_cc_deps.json" + cClang = "clang" + cppClang = "clang++" ) type ccIdeInfo struct { @@ -83,10 +81,7 @@ type ccDeps struct { } func (c *ccdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonContext) { - if !ctx.Config().IsEnvTrue(envVariableCollectCCDeps) { - return - } - + // (b/204397180) Generate module_bp_cc_deps.json by default. moduleDeps := ccDeps{} moduleInfos := map[string]ccIdeInfo{} diff --git a/cc/compiler.go b/cc/compiler.go index 00df66912..ffe8b2e36 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -552,6 +552,12 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.aidlFlags = append(flags.aidlFlags, "-t") } + aidlMinSdkVersion := ctx.minSdkVersion() + if aidlMinSdkVersion == "" { + aidlMinSdkVersion = "platform_apis" + } + flags.aidlFlags = append(flags.aidlFlags, "--min_sdk_version="+aidlMinSdkVersion) + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-I"+android.PathForModuleGen(ctx, "aidl").String()) } diff --git a/cc/library.go b/cc/library.go index c3f7305bf..dbf927d61 100644 --- a/cc/library.go +++ b/cc/library.go @@ -1377,7 +1377,7 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, // depending on a table of contents file instead of the library itself. tocFile := outputFile.ReplaceExtension(ctx, flags.Toolchain.ShlibSuffix()[1:]+".toc") library.tocFile = android.OptionalPathForPath(tocFile) - transformSharedObjectToToc(ctx, outputFile, tocFile, builderFlags) + TransformSharedObjectToToc(ctx, outputFile, tocFile) stripFlags := flagsToStripFlags(flags) needsStrip := library.stripper.NeedsStrip(ctx) @@ -2358,9 +2358,6 @@ func ccSharedOrStaticBp2BuildMutator(ctx android.TopDownMutatorContext, modType if !module.ConvertWithBp2build(ctx) { return } - if ctx.ModuleType() != modType { - return - } ccSharedOrStaticBp2BuildMutatorInternal(ctx, module, modType) } @@ -2498,7 +2495,15 @@ type bazelCcLibraryStaticAttributes struct { } func CcLibraryStaticBp2Build(ctx android.TopDownMutatorContext) { - ccSharedOrStaticBp2BuildMutator(ctx, "cc_library_static") + isLibraryStatic := ctx.ModuleType() == "cc_library_static" + if b, ok := ctx.Module().(android.Bazelable); ok { + // This is created by a custom soong config module type, so its ctx.ModuleType() is not + // cc_library_static. Check its BaseModuleType. + isLibraryStatic = isLibraryStatic || b.BaseModuleType() == "cc_library_static" + } + if isLibraryStatic { + ccSharedOrStaticBp2BuildMutator(ctx, "cc_library_static") + } } // TODO(b/199902614): Can this be factored to share with the other Attributes? @@ -2529,5 +2534,13 @@ type bazelCcLibrarySharedAttributes struct { } func CcLibrarySharedBp2Build(ctx android.TopDownMutatorContext) { - ccSharedOrStaticBp2BuildMutator(ctx, "cc_library_shared") + isLibraryShared := ctx.ModuleType() == "cc_library_shared" + if b, ok := ctx.Module().(android.Bazelable); ok { + // This is created by a custom soong config module type, so its ctx.ModuleType() is not + // cc_library_shared. Check its BaseModuleType. + isLibraryShared = isLibraryShared || b.BaseModuleType() == "cc_library_shared" + } + if isLibraryShared { + ccSharedOrStaticBp2BuildMutator(ctx, "cc_library_shared") + } } diff --git a/cc/prebuilt.go b/cc/prebuilt.go index 3401e3658..16945ac69 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -114,8 +114,6 @@ func (p *prebuiltLibraryLinker) link(ctx ModuleContext, // TODO(ccross): verify shared library dependencies srcs := p.prebuiltSrcs(ctx) if len(srcs) > 0 { - builderFlags := flagsToBuilderFlags(flags) - if len(srcs) > 1 { ctx.PropertyErrorf("srcs", "multiple prebuilt source files") return nil @@ -152,7 +150,7 @@ func (p *prebuiltLibraryLinker) link(ctx ModuleContext, // depending on a table of contents file instead of the library itself. tocFile := android.PathForModuleOut(ctx, libName+".toc") p.tocFile = android.OptionalPathForPath(tocFile) - transformSharedObjectToToc(ctx, outputFile, tocFile, builderFlags) + TransformSharedObjectToToc(ctx, outputFile, tocFile) if ctx.Windows() && p.properties.Windows_import_lib != nil { // Consumers of this library actually links to the import library in build diff --git a/cc/sanitize.go b/cc/sanitize.go index f6a9d5bc4..93d4b4c54 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -39,8 +39,15 @@ var ( } asanLdflags = []string{"-Wl,-u,__asan_preinit"} - hwasanCflags = []string{"-fno-omit-frame-pointer", "-Wno-frame-larger-than=", + hwasanCflags = []string{ + "-fno-omit-frame-pointer", + "-Wno-frame-larger-than=", "-fsanitize-hwaddress-abi=platform", + } + + // ThinLTO performs codegen during link time, thus these flags need to + // passed to both CFLAGS and LDFLAGS. + hwasanCommonflags = []string{ // The following improves debug location information // availability at the cost of its accuracy. It increases // the likelihood of a stack variable's frame offset @@ -48,11 +55,11 @@ var ( // for the quality of hwasan reports. The downside is a // higher number of "optimized out" stack variables. // b/112437883. - "-mllvm", "-instcombine-lower-dbg-declare=0", + "-instcombine-lower-dbg-declare=0", // TODO(b/159343917): HWASan and GlobalISel don't play nicely, and // GlobalISel is the default at -O0 on aarch64. - "-mllvm", "--aarch64-enable-global-isel-at-O=-1", - "-mllvm", "-fast-isel=false", + "--aarch64-enable-global-isel-at-O=-1", + "-fast-isel=false", } cfiCflags = []string{"-flto", "-fsanitize-cfi-cross-dso", @@ -81,7 +88,7 @@ const ( intOverflow scs Fuzzer - memtag_heap + Memtag_heap cfi // cfi is last to prevent it running before incompatible mutators ) @@ -92,7 +99,7 @@ var Sanitizers = []SanitizerType{ intOverflow, scs, Fuzzer, - memtag_heap, + Memtag_heap, cfi, // cfi is last to prevent it running before incompatible mutators } @@ -111,7 +118,7 @@ func (t SanitizerType) variationName() string { return "cfi" case scs: return "scs" - case memtag_heap: + case Memtag_heap: return "memtag_heap" case Fuzzer: return "fuzzer" @@ -127,7 +134,7 @@ func (t SanitizerType) name() string { return "address" case Hwasan: return "hwaddress" - case memtag_heap: + case Memtag_heap: return "memtag_heap" case tsan: return "thread" @@ -149,7 +156,7 @@ func (t SanitizerType) registerMutators(ctx android.RegisterMutatorsContext) { case Asan, Hwasan, Fuzzer, scs, tsan, cfi: ctx.TopDown(t.variationName()+"_deps", sanitizerDepsMutator(t)) ctx.BottomUp(t.variationName(), sanitizerMutator(t)) - case memtag_heap, intOverflow: + case Memtag_heap, intOverflow: // do nothing default: panic(fmt.Errorf("unknown SanitizerType %d", t)) @@ -172,6 +179,8 @@ func (*Module) SanitizerSupported(t SanitizerType) bool { return true case Fuzzer: return true + case Memtag_heap: + return true default: return false } @@ -460,7 +469,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Scs = nil } - // memtag_heap is only implemented on AArch64. + // Memtag_heap is only implemented on AArch64. if ctx.Arch().ArchType != android.Arm64 { s.Memtag_heap = nil } @@ -629,6 +638,14 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { if Bool(sanitize.Properties.Sanitize.Hwaddress) { flags.Local.CFlags = append(flags.Local.CFlags, hwasanCflags...) + + for _, flag := range hwasanCommonflags { + flags.Local.CFlags = append(flags.Local.CFlags, "-mllvm", flag) + } + for _, flag := range hwasanCommonflags { + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-mllvm,"+flag) + } + if Bool(sanitize.Properties.Sanitize.Writeonly) { flags.Local.CFlags = append(flags.Local.CFlags, "-mllvm", "-hwasan-instrument-reads=0") } @@ -798,7 +815,7 @@ func (sanitize *sanitize) getSanitizerBoolPtr(t SanitizerType) *bool { return sanitize.Properties.Sanitize.Cfi case scs: return sanitize.Properties.Sanitize.Scs - case memtag_heap: + case Memtag_heap: return sanitize.Properties.Sanitize.Memtag_heap case Fuzzer: return sanitize.Properties.Sanitize.Fuzzer @@ -814,7 +831,7 @@ func (sanitize *sanitize) isUnsanitizedVariant() bool { !sanitize.isSanitizerEnabled(tsan) && !sanitize.isSanitizerEnabled(cfi) && !sanitize.isSanitizerEnabled(scs) && - !sanitize.isSanitizerEnabled(memtag_heap) && + !sanitize.isSanitizerEnabled(Memtag_heap) && !sanitize.isSanitizerEnabled(Fuzzer) } @@ -844,7 +861,7 @@ func (sanitize *sanitize) SetSanitizer(t SanitizerType, b bool) { sanitize.Properties.Sanitize.Cfi = bPtr case scs: sanitize.Properties.Sanitize.Scs = bPtr - case memtag_heap: + case Memtag_heap: sanitize.Properties.Sanitize.Memtag_heap = bPtr case Fuzzer: sanitize.Properties.Sanitize.Fuzzer = bPtr @@ -1133,7 +1150,7 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { if lib, ok := snapshot.StaticLibs[noteDep]; ok { noteDep = lib } - depTag := libraryDependencyTag{Kind: staticLibraryDependency, wholeStatic: true} + depTag := StaticDepTag(true) variations := append(mctx.Target().Variations(), blueprint.Variation{Mutator: "link", Variation: "static"}) if c.Device() { @@ -1303,6 +1320,10 @@ var _ PlatformSanitizeable = (*Module)(nil) func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { return func(mctx android.BottomUpMutatorContext) { if c, ok := mctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { + + // Make sure we're not setting CFI to any value if it's not supported. + cfiSupported := mctx.Module().(PlatformSanitizeable).SanitizerSupported(cfi) + if c.Binary() && c.IsSanitizerEnabled(t) { modules := mctx.CreateVariations(t.variationName()) modules[0].(PlatformSanitizeable).SetSanitizer(t, true) @@ -1323,7 +1344,6 @@ func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { // is redirected to the sanitized variant of the dependent module. defaultVariation := t.variationName() // Not all PlatformSanitizeable modules support the CFI sanitizer - cfiSupported := mctx.Module().(PlatformSanitizeable).SanitizerSupported(cfi) mctx.SetDefaultDependencyVariation(&defaultVariation) modules := mctx.CreateVariations("", t.variationName()) @@ -1370,7 +1390,7 @@ func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { modules[0].(PlatformSanitizeable).SetInSanitizerDir() } - if mctx.Device() && t.incompatibleWithCfi() { + if mctx.Device() && t.incompatibleWithCfi() && cfiSupported { // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that // are incompatible with cfi modules[0].(PlatformSanitizeable).SetSanitizer(cfi, false) diff --git a/cc/snapshot_prebuilt.go b/cc/snapshot_prebuilt.go index 9570664c7..253a11bb6 100644 --- a/cc/snapshot_prebuilt.go +++ b/cc/snapshot_prebuilt.go @@ -66,7 +66,7 @@ func (recoverySnapshotImage) moduleNameSuffix() string { // Override existing vendor and recovery snapshot for cc module specific extra functions var VendorSnapshotImageSingleton vendorSnapshotImage = vendorSnapshotImage{&snapshot.VendorSnapshotImageSingleton} -var recoverySnapshotImageSingleton recoverySnapshotImage = recoverySnapshotImage{&snapshot.RecoverySnapshotImageSingleton} +var RecoverySnapshotImageSingleton recoverySnapshotImage = recoverySnapshotImage{&snapshot.RecoverySnapshotImageSingleton} func RegisterVendorSnapshotModules(ctx android.RegistrationContext) { ctx.RegisterModuleType("vendor_snapshot", vendorSnapshotFactory) @@ -231,7 +231,7 @@ func vendorSnapshotFactory() android.Module { } func recoverySnapshotFactory() android.Module { - return snapshotFactory(recoverySnapshotImageSingleton) + return snapshotFactory(RecoverySnapshotImageSingleton) } func snapshotFactory(image SnapshotImage) android.Module { @@ -326,7 +326,7 @@ func (p *BaseSnapshotDecorator) SetSnapshotAndroidMkSuffix(ctx android.ModuleCon return } - images := []SnapshotImage{VendorSnapshotImageSingleton, recoverySnapshotImageSingleton} + images := []SnapshotImage{VendorSnapshotImageSingleton, RecoverySnapshotImageSingleton} for _, image := range images { if p.Image == image { @@ -476,13 +476,12 @@ func (p *snapshotLibraryDecorator) link(ctx ModuleContext, flags Flags, deps Pat if p.shared() { libName := in.Base() - builderFlags := flagsToBuilderFlags(flags) // Optimize out relinking against shared libraries whose interface hasn't changed by // depending on a table of contents file instead of the library itself. tocFile := android.PathForModuleOut(ctx, libName+".toc") p.tocFile = android.OptionalPathForPath(tocFile) - transformSharedObjectToToc(ctx, in, tocFile, builderFlags) + TransformSharedObjectToToc(ctx, in, tocFile) ctx.SetProvider(SharedLibraryInfoProvider, SharedLibraryInfo{ SharedLibrary: in, @@ -584,7 +583,7 @@ func VendorSnapshotSharedFactory() android.Module { // overrides the recovery variant of the cc shared library with the same name, if BOARD_VNDK_VERSION // is set. func RecoverySnapshotSharedFactory() android.Module { - module, prebuilt := snapshotLibraryFactory(recoverySnapshotImageSingleton, SnapshotSharedSuffix) + module, prebuilt := snapshotLibraryFactory(RecoverySnapshotImageSingleton, SnapshotSharedSuffix) prebuilt.libraryDecorator.BuildOnlyShared() return module.Init() } @@ -604,7 +603,7 @@ func VendorSnapshotStaticFactory() android.Module { // overrides the recovery variant of the cc static library with the same name, if BOARD_VNDK_VERSION // is set. func RecoverySnapshotStaticFactory() android.Module { - module, prebuilt := snapshotLibraryFactory(recoverySnapshotImageSingleton, SnapshotStaticSuffix) + module, prebuilt := snapshotLibraryFactory(RecoverySnapshotImageSingleton, SnapshotStaticSuffix) prebuilt.libraryDecorator.BuildOnlyStatic() return module.Init() } @@ -624,7 +623,7 @@ func VendorSnapshotHeaderFactory() android.Module { // overrides the recovery variant of the cc header library with the same name, if BOARD_VNDK_VERSION // is set. func RecoverySnapshotHeaderFactory() android.Module { - module, prebuilt := snapshotLibraryFactory(recoverySnapshotImageSingleton, snapshotHeaderSuffix) + module, prebuilt := snapshotLibraryFactory(RecoverySnapshotImageSingleton, snapshotHeaderSuffix) prebuilt.libraryDecorator.HeaderOnly() return module.Init() } @@ -699,7 +698,7 @@ func VendorSnapshotBinaryFactory() android.Module { // development/vendor_snapshot/update.py. As a part of recovery snapshot, recovery_snapshot_binary // overrides the recovery variant of the cc binary with the same name, if BOARD_VNDK_VERSION is set. func RecoverySnapshotBinaryFactory() android.Module { - return snapshotBinaryFactory(recoverySnapshotImageSingleton, snapshotBinarySuffix) + return snapshotBinaryFactory(RecoverySnapshotImageSingleton, snapshotBinarySuffix) } func snapshotBinaryFactory(image SnapshotImage, moduleSuffix string) android.Module { @@ -801,7 +800,7 @@ func RecoverySnapshotObjectFactory() android.Module { } module.linker = prebuilt - prebuilt.Init(module, recoverySnapshotImageSingleton, snapshotObjectSuffix) + prebuilt.Init(module, RecoverySnapshotImageSingleton, snapshotObjectSuffix) module.AddProperties(&prebuilt.properties) return module.Init() } diff --git a/cc/snapshot_utils.go b/cc/snapshot_utils.go index 24abcce55..de50ef50f 100644 --- a/cc/snapshot_utils.go +++ b/cc/snapshot_utils.go @@ -113,7 +113,7 @@ func ShouldCollectHeadersForSnapshot(ctx android.ModuleContext, m LinkableInterf return ctx.Config().VndkSnapshotBuildArtifacts() } - for _, image := range []SnapshotImage{VendorSnapshotImageSingleton, recoverySnapshotImageSingleton} { + for _, image := range []SnapshotImage{VendorSnapshotImageSingleton, RecoverySnapshotImageSingleton} { if isSnapshotAware(ctx.DeviceConfig(), m, image.IsProprietaryPath(ctx.ModuleDir(), ctx.DeviceConfig()), apexInfo, image) { return true } diff --git a/cc/vndk.go b/cc/vndk.go index 1ae79de05..c9c9f2c08 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -495,7 +495,7 @@ func newVndkLibrariesWithMakeVarFilter(lister moduleListerFunc, makeVarName stri filterOutFromMakeVar: filter, } m.AddProperties(&m.properties) - android.InitAndroidModule(m) + android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibCommon) return m } diff --git a/cc/vndk_prebuilt.go b/cc/vndk_prebuilt.go index da34f3684..31b6d10dd 100644 --- a/cc/vndk_prebuilt.go +++ b/cc/vndk_prebuilt.go @@ -144,7 +144,6 @@ func (p *vndkPrebuiltLibraryDecorator) link(ctx ModuleContext, // current VNDK prebuilts are only shared libs. in := p.singleSourcePath(ctx) - builderFlags := flagsToBuilderFlags(flags) p.unstrippedOutputFile = in libName := in.Base() if p.stripper.NeedsStrip(ctx) { @@ -158,7 +157,7 @@ func (p *vndkPrebuiltLibraryDecorator) link(ctx ModuleContext, // depending on a table of contents file instead of the library itself. tocFile := android.PathForModuleOut(ctx, libName+".toc") p.tocFile = android.OptionalPathForPath(tocFile) - transformSharedObjectToToc(ctx, in, tocFile, builderFlags) + TransformSharedObjectToToc(ctx, in, tocFile) p.androidMkSuffix = p.NameSuffix() diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index e9eabd37c..dfc4eae7a 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -464,6 +464,11 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { // conversion for Bazel conversion. bp2buildCtx := android.NewContext(configuration) + // Soong internals like LoadHooks behave differently when running as + // bp2build. This is the bit to differentiate between Soong-as-Soong and + // Soong-as-bp2build. + bp2buildCtx.SetRunningAsBp2build() + // Propagate "allow misssing dependencies" bit. This is normally set in // newContext(), but we create bp2buildCtx without calling that method. bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) diff --git a/compliance/build_license_metadata/build_license_metadata.go b/compliance/build_license_metadata/build_license_metadata.go index ba3cc3ef8..8b1fe589e 100644 --- a/compliance/build_license_metadata/build_license_metadata.go +++ b/compliance/build_license_metadata/build_license_metadata.go @@ -119,7 +119,7 @@ func convertInstalledMap(installMaps []string) []*license_metadata_proto.Install for _, installMap := range installMaps { components := strings.Split(installMap, ":") if len(components) != 2 { - panic(fmt.Errorf("install map entry %q contains %d colons, expected 1", len(components)-1)) + panic(fmt.Errorf("install map entry %q contains %d colons, expected 1", installMap, len(components)-1)) } ret = append(ret, &license_metadata_proto.InstallMap{ FromPath: proto.String(components[0]), @@ -140,7 +140,7 @@ func convertDependencies(deps []string) []*license_metadata_proto.AnnotatedDepen dep := components[0] components = components[1:] ad := &license_metadata_proto.AnnotatedDependency{ - File: proto.String(dep), + File: proto.String(dep), Annotations: make([]string, 0, len(components)), } for _, ann := range components { diff --git a/cuj/run_cuj_tests.sh b/cuj/run_cuj_tests.sh index b4f9f8895..a746bd59e 100755 --- a/cuj/run_cuj_tests.sh +++ b/cuj/run_cuj_tests.sh @@ -18,11 +18,10 @@ readonly ANDROID_TOP="$(cd $(dirname $0)/../../..; pwd)" cd "$ANDROID_TOP" export OUT_DIR="${OUT_DIR:-out}" -readonly SOONG_OUT="${OUT_DIR}/soong" -build/soong/soong_ui.bash --make-mode "${SOONG_OUT}/host/${OS}-x86/bin/cuj_tests" +build/soong/soong_ui.bash --make-mode "${OUT_DIR}/host/${OS}-x86/bin/cuj_tests" -"${SOONG_OUT}/host/${OS}-x86/bin/cuj_tests" || true +"${OUT_DIR}/host/${OS}-x86/bin/cuj_tests" || true if [ -n "${DIST_DIR}" ]; then cp -r "${OUT_DIR}/cuj_tests/logs" "${DIST_DIR}" diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index d81ac2cbf..4a3d390c2 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -115,16 +115,16 @@ func TestCLC(t *testing.T) { // Test that class loader context structure is correct. t.Run("string", func(t *testing.T) { wantStr := " --host-context-for-sdk 29 " + - "PCL[out/" + AndroidHidlManager + ".jar]#" + - "PCL[out/" + AndroidHidlBase + ".jar]" + + "PCL[out/soong/" + AndroidHidlManager + ".jar]#" + + "PCL[out/soong/" + AndroidHidlBase + ".jar]" + " --target-context-for-sdk 29 " + "PCL[/system/framework/" + AndroidHidlManager + ".jar]#" + "PCL[/system/framework/" + AndroidHidlBase + ".jar]" + " --host-context-for-sdk any " + - "PCL[out/a.jar]#PCL[out/b.jar]#PCL[out/c.jar]#PCL[out/d.jar]" + - "{PCL[out/a2.jar]#PCL[out/b2.jar]#PCL[out/c2.jar]" + - "{PCL[out/a1.jar]#PCL[out/b1.jar]}}#" + - "PCL[out/f.jar]#PCL[out/a3.jar]#PCL[out/b3.jar]" + + "PCL[out/soong/a.jar]#PCL[out/soong/b.jar]#PCL[out/soong/c.jar]#PCL[out/soong/d.jar]" + + "{PCL[out/soong/a2.jar]#PCL[out/soong/b2.jar]#PCL[out/soong/c2.jar]" + + "{PCL[out/soong/a1.jar]#PCL[out/soong/b1.jar]}}#" + + "PCL[out/soong/f.jar]#PCL[out/soong/a3.jar]#PCL[out/soong/b3.jar]" + " --target-context-for-sdk any " + "PCL[/system/a.jar]#PCL[/system/b.jar]#PCL[/system/c.jar]#PCL[/system/d.jar]" + "{PCL[/system/a2.jar]#PCL[/system/b2.jar]#PCL[/system/c2.jar]" + @@ -138,11 +138,11 @@ func TestCLC(t *testing.T) { // Test that all expected build paths are gathered. t.Run("paths", func(t *testing.T) { wantPaths := []string{ - "out/android.hidl.manager-V1.0-java.jar", "out/android.hidl.base-V1.0-java.jar", - "out/a.jar", "out/b.jar", "out/c.jar", "out/d.jar", - "out/a2.jar", "out/b2.jar", "out/c2.jar", - "out/a1.jar", "out/b1.jar", - "out/f.jar", "out/a3.jar", "out/b3.jar", + "out/soong/android.hidl.manager-V1.0-java.jar", "out/soong/android.hidl.base-V1.0-java.jar", + "out/soong/a.jar", "out/soong/b.jar", "out/soong/c.jar", "out/soong/d.jar", + "out/soong/a2.jar", "out/soong/b2.jar", "out/soong/c2.jar", + "out/soong/a1.jar", "out/soong/b1.jar", + "out/soong/f.jar", "out/soong/a3.jar", "out/soong/b3.jar", } if !reflect.DeepEqual(wantPaths, havePaths.Strings()) { t.Errorf("\nwant paths: %s\nhave paths: %s", wantPaths, havePaths) @@ -270,13 +270,13 @@ func TestCLCSdkVersionOrder(t *testing.T) { // Test that class loader context structure is correct. t.Run("string", func(t *testing.T) { - wantStr := " --host-context-for-sdk 30 PCL[out/c.jar]" + + wantStr := " --host-context-for-sdk 30 PCL[out/soong/c.jar]" + " --target-context-for-sdk 30 PCL[/system/c.jar]" + - " --host-context-for-sdk 29 PCL[out/b.jar]" + + " --host-context-for-sdk 29 PCL[out/soong/b.jar]" + " --target-context-for-sdk 29 PCL[/system/b.jar]" + - " --host-context-for-sdk 28 PCL[out/a.jar]" + + " --host-context-for-sdk 28 PCL[out/soong/a.jar]" + " --target-context-for-sdk 28 PCL[/system/a.jar]" + - " --host-context-for-sdk any PCL[out/d.jar]" + + " --host-context-for-sdk any PCL[out/soong/d.jar]" + " --target-context-for-sdk any PCL[/system/d.jar]" if wantStr != haveStr { t.Errorf("\nwant class loader context: %s\nhave class loader context: %s", wantStr, haveStr) diff --git a/java/androidmk.go b/java/androidmk.go index eca5caacb..4c115d58c 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -47,6 +47,7 @@ func (library *Library) AndroidMkEntriesHostDex() android.AndroidMkEntries { if library.dexJarFile.IsSet() { entries.SetPath("LOCAL_SOONG_DEX_JAR", library.dexJarFile.Path()) } + entries.SetPath("LOCAL_SOONG_INSTALLED_MODULE", library.hostdexInstallFile) entries.SetPath("LOCAL_SOONG_HEADER_JAR", library.headerJarFile) entries.SetPath("LOCAL_SOONG_CLASSES_JAR", library.implementationAndResourcesJar) entries.SetString("LOCAL_MODULE_STEM", library.Stem()+"-hostdex") @@ -285,11 +286,6 @@ func (binary *Binary) AndroidMkEntries() []android.AndroidMkEntries { }} } else { outputFile := binary.wrapperFile - // Have Make installation trigger Soong installation by using Soong's install path as - // the output file. - if binary.Host() { - outputFile = binary.binaryFile - } return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "EXECUTABLES", diff --git a/java/app.go b/java/app.go index 6554d6693..c08ec0697 100755 --- a/java/app.go +++ b/java/app.go @@ -471,6 +471,7 @@ func (a *AndroidApp) dexBuildActions(ctx android.ModuleContext) android.Path { a.dexpreopter.enforceUsesLibs = a.usesLibrary.enforceUsesLibraries() a.dexpreopter.classLoaderContexts = a.classLoaderContexts a.dexpreopter.manifestFile = a.mergedManifestFile + a.dexpreopter.preventInstall = a.appProperties.PreventInstall if ctx.ModuleName() != "framework-res" { a.Module.compile(ctx, a.aaptSrcJar) @@ -720,11 +721,15 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) // Install the app package. - if (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() { - ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile) + if (Bool(a.Module.properties.Installable) || ctx.Host()) && apexInfo.IsForPlatform() && + !a.appProperties.PreventInstall { + + var extraInstalledPaths android.Paths for _, extra := range a.extraOutputFiles { - ctx.InstallFile(a.installDir, extra.Base(), extra) + installed := ctx.InstallFile(a.installDir, extra.Base(), extra) + extraInstalledPaths = append(extraInstalledPaths, installed) } + ctx.InstallFile(a.installDir, a.outputFile.Base(), a.outputFile, extraInstalledPaths...) } a.buildAppDependencyInfo(ctx) diff --git a/java/app_import.go b/java/app_import.go index 3e5f972a4..4d1969ebb 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -108,6 +108,8 @@ func (a *AndroidAppImport) IsInstallable() bool { return true } +func (a *AndroidAppImport) InstallBypassMake() bool { return true } + // Updates properties with variant-specific values. func (a *AndroidAppImport) processVariants(ctx android.LoadHookContext) { config := ctx.Config() diff --git a/java/base.go b/java/base.go index ca34f2eef..6930bcd17 100644 --- a/java/base.go +++ b/java/base.go @@ -407,6 +407,9 @@ type Module struct { // installed file for binary dependency installFile android.Path + // installed file for hostdex copy + hostdexInstallFile android.InstallPath + // list of .java files and srcjars that was passed to javac compiledJavaSrcs android.Paths compiledSrcJars android.Paths @@ -784,6 +787,9 @@ func (j *Module) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.Opt flags = append(flags, "--transaction_names") } + aidlMinSdkVersion := j.MinSdkVersion(ctx).ApiLevel.String() + flags = append(flags, "--min_sdk_version="+aidlMinSdkVersion) + return strings.Join(flags, " "), deps } diff --git a/java/dexpreopt.go b/java/dexpreopt.go index e9dc982c7..7c081b6e5 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -66,6 +66,7 @@ type dexpreopter struct { isApp bool isTest bool isPresignedPrebuilt bool + preventInstall bool manifestFile android.Path statusFile android.WritablePath @@ -335,17 +336,19 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr dexpreoptRule.Build("dexpreopt", "dexpreopt") - if global.ApexSystemServerJars.ContainsJar(moduleName(ctx)) { - // APEX variants of java libraries are hidden from Make, so their dexpreopt outputs need special - // handling. Currently, for APEX variants of java libraries, only those in the system server - // classpath are handled here. Preopting of boot classpath jars in the ART APEX are handled in - // java/dexpreopt_bootjars.go, and other APEX jars are not preopted. - for _, install := range dexpreoptRule.Installs() { - // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. - installDir := strings.TrimPrefix(filepath.Dir(install.To), "/") - installBase := filepath.Base(install.To) - arch := filepath.Base(installDir) - installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) + for _, install := range dexpreoptRule.Installs() { + // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. + installDir := strings.TrimPrefix(filepath.Dir(install.To), "/") + installBase := filepath.Base(install.To) + arch := filepath.Base(installDir) + installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) + + if global.ApexSystemServerJars.ContainsJar(moduleName(ctx)) { + // APEX variants of java libraries are hidden from Make, so their dexpreopt + // outputs need special handling. Currently, for APEX variants of java + // libraries, only those in the system server classpath are handled here. + // Preopting of boot classpath jars in the ART APEX are handled in + // java/dexpreopt_bootjars.go, and other APEX jars are not preopted. // The installs will be handled by Make as sub-modules of the java library. d.builtInstalledForApex = append(d.builtInstalledForApex, dexpreopterInstall{ name: arch + "-" + installBase, @@ -354,10 +357,12 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr installDirOnDevice: installPath, installFileOnDevice: installBase, }) + } else if !d.preventInstall { + ctx.InstallFile(installPath, installBase, install.From) } - } else { - // The installs will be handled by Make as LOCAL_SOONG_BUILT_INSTALLED of the java library - // module. + } + + if !global.ApexSystemServerJars.ContainsJar(moduleName(ctx)) { d.builtInstalled = dexpreoptRule.Installs().String() } } diff --git a/java/droiddoc.go b/java/droiddoc.go index 869a5982d..c84a15c1f 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -769,8 +769,8 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { d.Javadoc.docZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"docs.zip") - jsilver := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", "jsilver.jar") - doclava := android.PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", "doclava.jar") + jsilver := ctx.Config().HostJavaToolPath(ctx, "jsilver.jar") + doclava := ctx.Config().HostJavaToolPath(ctx, "doclava.jar") outDir := android.PathForModuleOut(ctx, "out") srcJarDir := android.PathForModuleOut(ctx, "srcjars") diff --git a/java/droidstubs.go b/java/droidstubs.go index 0c66ccf40..7ad316fcf 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -16,6 +16,7 @@ package java import ( "fmt" + "path/filepath" "strings" "github.com/google/blueprint/proptools" @@ -805,7 +806,7 @@ type PrebuiltStubsSources struct { properties PrebuiltStubsSourcesProperties - stubsSrcJar android.ModuleOutPath + stubsSrcJar android.Path } func (p *PrebuiltStubsSources) OutputFiles(tag string) (android.Paths, error) { @@ -822,35 +823,39 @@ func (d *PrebuiltStubsSources) StubsSrcJar() android.Path { } func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleContext) { - p.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") - if len(p.properties.Srcs) != 1 { - ctx.PropertyErrorf("srcs", "must only specify one directory path, contains %d paths", len(p.properties.Srcs)) + ctx.PropertyErrorf("srcs", "must only specify one directory path or srcjar, contains %d paths", len(p.properties.Srcs)) return } - localSrcDir := p.properties.Srcs[0] - // Although PathForModuleSrc can return nil if either the path doesn't exist or - // the path components are invalid it won't in this case because no components - // are specified and the module directory must exist in order to get this far. - srcDir := android.PathForModuleSrc(ctx).(android.SourcePath).Join(ctx, localSrcDir) - - // Glob the contents of the directory just in case the directory does not exist. - srcGlob := localSrcDir + "/**/*" - srcPaths := android.PathsForModuleSrc(ctx, []string{srcGlob}) + src := p.properties.Srcs[0] + if filepath.Ext(src) == ".srcjar" { + // This is a srcjar. We can use it directly. + p.stubsSrcJar = android.PathForModuleSrc(ctx, src) + } else { + outPath := android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") - rule := android.NewRuleBuilder(pctx, ctx) - rule.Command(). - BuiltTool("soong_zip"). - Flag("-write_if_changed"). - Flag("-jar"). - FlagWithOutput("-o ", p.stubsSrcJar). - FlagWithArg("-C ", srcDir.String()). - FlagWithRspFileInputList("-r ", p.stubsSrcJar.ReplaceExtension(ctx, "rsp"), srcPaths) + // This is a directory. Glob the contents just in case the directory does not exist. + srcGlob := src + "/**/*" + srcPaths := android.PathsForModuleSrc(ctx, []string{srcGlob}) - rule.Restat() + // Although PathForModuleSrc can return nil if either the path doesn't exist or + // the path components are invalid it won't in this case because no components + // are specified and the module directory must exist in order to get this far. + srcDir := android.PathForModuleSrc(ctx).(android.SourcePath).Join(ctx, src) - rule.Build("zip src", "Create srcjar from prebuilt source") + rule := android.NewRuleBuilder(pctx, ctx) + rule.Command(). + BuiltTool("soong_zip"). + Flag("-write_if_changed"). + Flag("-jar"). + FlagWithOutput("-o ", outPath). + FlagWithArg("-C ", srcDir.String()). + FlagWithRspFileInputList("-r ", outPath.ReplaceExtension(ctx, "rsp"), srcPaths) + rule.Restat() + rule.Build("zip src", "Create srcjar from prebuilt source") + p.stubsSrcJar = outPath + } } func (p *PrebuiltStubsSources) Prebuilt() *android.Prebuilt { diff --git a/java/java.go b/java/java.go index 854097b0a..2f9e03a80 100644 --- a/java/java.go +++ b/java/java.go @@ -265,6 +265,8 @@ func (j *Module) XrefJavaFiles() android.Paths { return j.kytheFiles } +func (j *Module) InstallBypassMake() bool { return true } + type dependencyTag struct { blueprint.BaseDependencyTag name string @@ -567,8 +569,23 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { if j.InstallMixin != nil { extraInstallDeps = j.InstallMixin(ctx, j.outputFile) } - j.installFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), - j.Stem()+".jar", j.outputFile, extraInstallDeps...) + hostDexNeeded := Bool(j.deviceProperties.Hostdex) && !ctx.Host() + if hostDexNeeded { + j.hostdexInstallFile = ctx.InstallFile( + android.PathForHostDexInstall(ctx, "framework"), + j.Stem()+"-hostdex.jar", j.outputFile) + } + var installDir android.InstallPath + if ctx.InstallInTestcases() { + var archDir string + if !ctx.Host() { + archDir = ctx.DeviceConfig().DeviceArch() + } + installDir = android.PathForModuleInstall(ctx, ctx.ModuleName(), archDir) + } else { + installDir = android.PathForModuleInstall(ctx, "framework") + } + j.installFile = ctx.InstallFile(installDir, j.Stem()+".jar", j.outputFile, extraInstallDeps...) } } @@ -842,6 +859,20 @@ type JavaTestImport struct { dexJarFile android.Path } +func (j *Test) InstallInTestcases() bool { + // Host java tests install into $(HOST_OUT_JAVA_LIBRARIES), and then are copied into + // testcases by base_rules.mk. + return !j.Host() +} + +func (j *TestHelperLibrary) InstallInTestcases() bool { + return true +} + +func (j *JavaTestImport) InstallInTestcases() bool { + return true +} + func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { if len(j.testHostProperties.Data_native_bins) > 0 { for _, target := range ctx.MultiTargets() { @@ -1376,8 +1407,17 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { }) if Bool(j.properties.Installable) { - ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"), - jarName, outputFile) + var installDir android.InstallPath + if ctx.InstallInTestcases() { + var archDir string + if !ctx.Host() { + archDir = ctx.DeviceConfig().DeviceArch() + } + installDir = android.PathForModuleInstall(ctx, ctx.ModuleName(), archDir) + } else { + installDir = android.PathForModuleInstall(ctx, "framework") + } + ctx.InstallFile(installDir, jarName, outputFile) } j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.properties.Aidl.Export_include_dirs) diff --git a/java/java_test.go b/java/java_test.go index bc9b40964..c039f7246 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -1357,6 +1357,36 @@ func TestAidlFlagsArePassedToTheAidlCompiler(t *testing.T) { } } +func TestAidlFlagsWithMinSdkVersion(t *testing.T) { + fixture := android.GroupFixturePreparers( + prepareForJavaTest, FixtureWithPrebuiltApis(map[string][]string{"14": {"foo"}})) + + for _, tc := range []struct { + name string + sdkVersion string + expected string + }{ + {"default is current", "", "current"}, + {"use sdk_version", `sdk_version: "14"`, "14"}, + {"system_current", `sdk_version: "system_current"`, "current"}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := fixture.RunTestWithBp(t, ` + java_library { + name: "foo", + srcs: ["aidl/foo/IFoo.aidl"], + `+tc.sdkVersion+` + } + `) + aidlCommand := ctx.ModuleForTests("foo", "android_common").Rule("aidl").RuleParams.Command + expectedAidlFlag := "--min_sdk_version=" + tc.expected + if !strings.Contains(aidlCommand, expectedAidlFlag) { + t.Errorf("aidl command %q does not contain %q", aidlCommand, expectedAidlFlag) + } + }) + } +} + func TestDataNativeBinaries(t *testing.T) { ctx, _ := testJava(t, ` java_test_host { diff --git a/java/jdeps.go b/java/jdeps.go index 0ab2e422b..eff9a3174 100644 --- a/java/jdeps.go +++ b/java/jdeps.go @@ -40,16 +40,11 @@ type jdepsGeneratorSingleton struct { var _ android.SingletonMakeVarsProvider = (*jdepsGeneratorSingleton)(nil) const ( - // Environment variables used to modify behavior of this singleton. - envVariableCollectJavaDeps = "SOONG_COLLECT_JAVA_DEPS" - jdepsJsonFileName = "module_bp_java_deps.json" + jdepsJsonFileName = "module_bp_java_deps.json" ) func (j *jdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonContext) { - if !ctx.Config().IsEnvTrue(envVariableCollectJavaDeps) { - return - } - + // (b/204397180) Generate module_bp_java_deps.json by default. moduleInfos := make(map[string]android.IdeInfo) ctx.VisitAllModules(func(module android.Module) { diff --git a/java/sdk.go b/java/sdk.go index 697deb1ec..e6bf220b4 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -141,11 +141,13 @@ func decodeSdkDep(ctx android.EarlyModuleContext, sdkContext android.SdkContext) } } - toModule := func(systemModules string, module string, aidl android.Path) sdkDep { + toModule := func(module string, aidl android.Path) sdkDep { + // Select the kind of system modules needed for the sdk version. + systemModulesKind := systemModuleKind(sdkVersion.Kind, android.FutureApiLevel) return sdkDep{ useModule: true, bootclasspath: []string{module, config.DefaultLambdaStubsLibrary}, - systemModules: systemModules, + systemModules: fmt.Sprintf("core-%s-stubs-system-modules", systemModulesKind), java9Classpath: []string{module}, frameworkResModule: "framework-res", aidl: android.OptionalPathForPath(aidl), @@ -186,11 +188,11 @@ func decodeSdkDep(ctx android.EarlyModuleContext, sdkContext android.SdkContext) noFrameworksLibs: true, } case android.SdkPublic: - return toModule("core-public-stubs-system-modules", "android_stubs_current", sdkFrameworkAidlPath(ctx)) + return toModule("android_stubs_current", sdkFrameworkAidlPath(ctx)) case android.SdkSystem: - return toModule("core-public-stubs-system-modules", "android_system_stubs_current", sdkFrameworkAidlPath(ctx)) + return toModule("android_system_stubs_current", sdkFrameworkAidlPath(ctx)) case android.SdkTest: - return toModule("core-public-stubs-system-modules", "android_test_stubs_current", sdkFrameworkAidlPath(ctx)) + return toModule("android_test_stubs_current", sdkFrameworkAidlPath(ctx)) case android.SdkCore: return sdkDep{ useModule: true, @@ -200,10 +202,10 @@ func decodeSdkDep(ctx android.EarlyModuleContext, sdkContext android.SdkContext) } case android.SdkModule: // TODO(146757305): provide .apk and .aidl that have more APIs for modules - return toModule("core-module-lib-stubs-system-modules", "android_module_lib_stubs_current", nonUpdatableFrameworkAidlPath(ctx)) + return toModule("android_module_lib_stubs_current", nonUpdatableFrameworkAidlPath(ctx)) case android.SdkSystemServer: // TODO(146757305): provide .apk and .aidl that have more APIs for modules - return toModule("core-module-lib-stubs-system-modules", "android_system_server_stubs_current", sdkFrameworkAidlPath(ctx)) + return toModule("android_system_server_stubs_current", sdkFrameworkAidlPath(ctx)) default: panic(fmt.Errorf("invalid sdk %q", sdkVersion.Raw)) } diff --git a/java/sdk_library.go b/java/sdk_library.go index 273efec13..d7f14d659 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -361,13 +361,14 @@ type ApiScopeProperties struct { // The sdk_version to use for building the stubs. // // If not specified then it will use an sdk_version determined as follows: + // // 1) If the sdk_version specified on the java_sdk_library is none then this - // will be none. This is used for java_sdk_library instances that are used - // to create stubs that contribute to the core_current sdk version. - // 2) Otherwise, it is assumed that this library extends but does not contribute - // directly to a specific sdk_version and so this uses the sdk_version appropriate - // for the api scope. e.g. public will use sdk_version: current, system will use - // sdk_version: system_current, etc. + // will be none. This is used for java_sdk_library instances that are used + // to create stubs that contribute to the core_current sdk version. + // 2) Otherwise, it is assumed that this library extends but does not + // contribute directly to a specific sdk_version and so this uses the + // sdk_version appropriate for the api scope. e.g. public will use + // sdk_version: current, system will use sdk_version: system_current, etc. // // This does not affect the sdk_version used for either generating the stubs source // or the API file. They both have to use the same sdk_version as is used for diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index b089f912a..7bb0246be 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -46,17 +46,19 @@ var ( dryRun = flag.Bool("dry_run", false, "dry run") recurse = flag.Bool("convert_dependents", false, "convert all dependent files") mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`) - warn = flag.Bool("warnings", false, "warn about partially failed conversions") + noWarn = flag.Bool("no_warnings", false, "don't warn about partially failed conversions") verbose = flag.Bool("v", false, "print summary") errstat = flag.Bool("error_stat", false, "print error statistics") traceVar = flag.String("trace", "", "comma-separated list of variables to trace") // TODO(asmundak): this option is for debugging allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //") outputTop = flag.String("outdir", "", "write output files into this directory hierarchy") - launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_") + launcher = flag.String("launcher", "", "generated launcher path.") + boardlauncher = flag.String("boardlauncher", "", "generated board configuration launcher path.") printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit") cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file") traceCalls = flag.Bool("trace_calls", false, "trace function calls") + inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables") ) func init() { @@ -73,13 +75,13 @@ func init() { flagAlias("root", "d") flagAlias("dry_run", "n") flagAlias("convert_dependents", "r") - flagAlias("warnings", "w") + flagAlias("no_warnings", "w") flagAlias("error_stat", "e") } var backupSuffix string var tracedVariables []string -var errorLogger = errorsByType{data: make(map[string]datum)} +var errorLogger = errorSink{data: make(map[string]datum)} var makefileFinder = &LinuxMakefileFinder{} var versionDefaultsMk = filepath.Join("build", "make", "core", "version_defaults.mk") @@ -87,8 +89,7 @@ func main() { flag.Usage = func() { cmd := filepath.Base(os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), - "Usage: %[1]s flags file...\n"+ - "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd) + "Usage: %[1]s flags file...\n", cmd) flag.PrintDefaults() } flag.Parse() @@ -177,14 +178,31 @@ func main() { versionDefaultsPath := outputFilePath(versionDefaultsMk) err = writeGenerated(versionDefaultsPath, versionDefaults) if err != nil { - fmt.Fprintf(os.Stderr, "%s:%s", files[0], err) + fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) ok = false } err = writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(files[0]), versionDefaultsPath, mk2rbc.MakePath2ModuleName(files[0]))) if err != nil { - fmt.Fprintf(os.Stderr, "%s:%s", files[0], err) + fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) + ok = false + } + } + if *boardlauncher != "" { + if len(files) != 1 { + quit(fmt.Errorf("a launcher can be generated only for a single product")) + } + if *inputVariables == "" { + quit(fmt.Errorf("the board launcher requires an input variables file")) + } + if !convertOne(*inputVariables) { + quit(fmt.Errorf("the board launcher input variables file failed to convert")) + } + err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher( + outputFilePath(files[0]), outputFilePath(*inputVariables))) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) ok = false } } @@ -318,12 +336,10 @@ func convertOne(mkFile string) (ok bool) { OutputSuffix: *suffix, TracedVariables: tracedVariables, TraceCalls: *traceCalls, - WarnPartialSuccess: *warn, + WarnPartialSuccess: !*noWarn, SourceFS: os.DirFS(*rootDir), MakefileFinder: makefileFinder, - } - if *errstat { - mk2starRequest.ErrorLogger = errorLogger + ErrorLogger: errorLogger, } ss, err := mk2rbc.Convert(mk2starRequest) if err != nil { @@ -401,7 +417,7 @@ func writeGenerated(path string, contents string) error { func printStats() { var sortedFiles []string - if !*warn && !*verbose { + if *noWarn && !*verbose { return } for p := range converted { @@ -419,7 +435,7 @@ func printStats() { nOk++ } } - if *warn { + if !*noWarn { if nPartial > 0 { fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n") for _, f := range sortedFiles { @@ -438,10 +454,8 @@ func printStats() { } } } - if *verbose { - fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk) - fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial) - fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed) + if *verbose && (nPartial > 0 || nFailed > 0) { + fmt.Fprintln(os.Stderr, "Succeeded: ", nOk, " Partial: ", nPartial, " Failed: ", nFailed) } } @@ -450,11 +464,18 @@ type datum struct { formattingArgs []string } -type errorsByType struct { +type errorSink struct { data map[string]datum } -func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) { +func (ebt errorSink) NewError(sourceFile string, sourceLine int, node parser.Node, message string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "%s:%d ", sourceFile, sourceLine) + fmt.Fprintf(os.Stderr, message, args...) + fmt.Fprintln(os.Stderr) + if !*errstat { + return + } + v, exists := ebt.data[message] if exists { v.count++ @@ -479,7 +500,7 @@ func (ebt errorsByType) NewError(message string, node parser.Node, args ...inter ebt.data[message] = v } -func (ebt errorsByType) printStatistics() { +func (ebt errorSink) printStatistics() { if len(ebt.data) > 0 { fmt.Fprintln(os.Stderr, "Error counts:") } diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 6227164e1..7ce183495 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -52,7 +52,9 @@ const ( // And here are the functions and variables: cfnGetCfg = baseName + ".cfg" cfnMain = baseName + ".product_configuration" + cfnBoardMain = baseName + ".board_configuration" cfnPrintVars = baseName + ".printvars" + cfnPrintGlobals = baseName + ".printglobals" cfnWarning = baseName + ".warning" cfnLocalAppend = baseName + ".local_append" cfnLocalSetDefault = baseName + ".local_set_default" @@ -70,6 +72,7 @@ const ( soongConfigVarSetOld = "add_soong_config_var_value" soongConfigAppend = "soong_config_append" soongConfigAssign = "soong_config_set" + soongConfigGet = "soong_config_get" wildcardExistsPhony = "$wildcard_exists" ) @@ -93,6 +96,7 @@ var knownFunctions = map[string]struct { soongConfigVarSetOld: {baseName + ".soong_config_set", starlarkTypeVoid, hiddenArgGlobal}, soongConfigAssign: {baseName + ".soong_config_set", starlarkTypeVoid, hiddenArgGlobal}, soongConfigAppend: {baseName + ".soong_config_append", starlarkTypeVoid, hiddenArgGlobal}, + soongConfigGet: {baseName + ".soong_config_get", starlarkTypeString, hiddenArgGlobal}, "add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList, hiddenArgNone}, "addprefix": {baseName + ".addprefix", starlarkTypeList, hiddenArgNone}, "addsuffix": {baseName + ".addsuffix", starlarkTypeList, hiddenArgNone}, @@ -157,7 +161,7 @@ type Request struct { RootDir string // root directory path used to resolve included files OutputSuffix string // generated Starlark files suffix OutputDir string // if set, root of the output hierarchy - ErrorLogger ErrorMonitorCB + ErrorLogger ErrorLogger TracedVariables []string // trace assignment to these variables TraceCalls bool WarnPartialSuccess bool @@ -165,10 +169,10 @@ type Request struct { MakefileFinder MakefileFinder } -// An error sink allowing to gather error statistics. -// NewError is called on every error encountered during processing. -type ErrorMonitorCB interface { - NewError(s string, node mkparser.Node, args ...interface{}) +// ErrorLogger prints errors and gathers error statistics. +// Its NewError function is called on every error encountered during the conversion. +type ErrorLogger interface { + NewError(sourceFile string, sourceLine int, node mkparser.Node, text string, args ...interface{}) } // Derives module name for a given file. It is base name @@ -377,6 +381,7 @@ type StarlarkScript struct { warnPartialSuccess bool sourceFS fs.FS makefileFinder MakefileFinder + nodeLocator func(pos mkparser.Pos) int } func (ss *StarlarkScript) newNode(node starlarkNode) { @@ -402,7 +407,7 @@ type parseContext struct { fatalError error builtinMakeVars map[string]starlarkExpr outputSuffix string - errorLogger ErrorMonitorCB + errorLogger ErrorLogger tracedVariables map[string]bool // variables to be traced in the generated script variables map[string]variable varAssignments *varAssignmentScope @@ -536,7 +541,15 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) { return } name := a.Name.Strings[0] - // Soong confuguration + // The `override` directive + // override FOO := + // is parsed as an assignment to a variable named `override FOO`. + // There are very few places where `override` is used, just flag it. + if strings.HasPrefix(name, "override ") { + ctx.errorf(a, "cannot handle override directive") + } + + // Soong configuration if strings.HasPrefix(name, soongNsPrefix) { ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a) return @@ -637,7 +650,7 @@ func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Ass // Upon seeing // SOONG_CONFIG_x_y = v // find a namespace called `x` and act as if we encountered - // $(call add_config_var_value(x,y,v) + // $(call soong_config_set,x,y,v) // or check that `x_y` is a namespace, and then add the RHS of this assignment as variables in // it. // Emit an error in the ambiguous situation (namespaces `foo_bar` with a variable `baz` @@ -678,7 +691,7 @@ func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Ass ctx.errorf(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName) return } - fname := soongConfigVarSetOld + fname := soongConfigAssign if asgn.Type == "+=" { fname = soongConfigAppend } @@ -961,25 +974,16 @@ func (ctx *parseContext) processBranch(check *mkparser.Directive) { ctx.pushReceiver(&block) for ctx.hasNodes() { node := ctx.getNode() - if ctx.handleSimpleStatement(node) { - continue - } - switch d := node.(type) { - case *mkparser.Directive: + if d, ok := node.(*mkparser.Directive); ok { switch d.Name { case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif": ctx.popReceiver() ctx.receiver.newNode(&block) ctx.backNode() return - case "ifdef", "ifndef", "ifeq", "ifneq": - ctx.handleIfBlock(d) - default: - ctx.errorf(d, "unexpected directive %s", d.Name) } - default: - ctx.errorf(node, "unexpected statement") } + ctx.handleSimpleStatement(node) } ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump()) ctx.popReceiver() @@ -1019,7 +1023,7 @@ func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr { message := fmt.Sprintf(text, args...) if ctx.errorLogger != nil { - ctx.errorLogger.NewError(text, node, args) + ctx.errorLogger.NewError(ctx.script.mkFile, ctx.script.nodeLocator(node.Pos()), node, text, args...) } ctx.script.hasErrors = true return &badExpr{node, message} @@ -1167,10 +1171,12 @@ func (ctx *parseContext) parseCheckFunctionCallResult(directive *mkparser.Direct return ctx.newBadExpr(directive, fmt.Sprintf("the result of %s can be compared only to empty", x.name)), true } + // if the expression is ifneq (,$(call is-vendor-board-platform,...)), negate==true, + // so we should set inExpr.isNot to false return &inExpr{ expr: &variableRefExpr{ctx.addVariable("TARGET_BOARD_PLATFORM"), false}, list: &variableRefExpr{ctx.addVariable("QCOM_BOARD_PLATFORMS"), true}, - isNot: negate, + isNot: !negate, }, true default: return ctx.newBadExpr(directive, "Unknown function in ifeq: %s", x.name), true @@ -1318,7 +1324,7 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt } if strings.HasPrefix(refDump, soongNsPrefix) { // TODO (asmundak): if we find many, maybe handle them. - return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced: %s", refDump) + return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: %s", refDump) } if v := ctx.addVariable(refDump); v != nil { return &variableRefExpr{v, ctx.lastAssignment(v.name()) != nil} @@ -1384,11 +1390,14 @@ func (ctx *parseContext) parseSubstFunc(node mkparser.Node, fname string, args * if len(words) != 3 { return ctx.newBadExpr(node, "%s function should have 3 arguments", fname) } - if !words[0].Const() || !words[1].Const() { - return ctx.newBadExpr(node, "%s function's from and to arguments should be constant", fname) + from := ctx.parseMakeString(node, words[0]) + if xBad, ok := from.(*badExpr); ok { + return xBad + } + to := ctx.parseMakeString(node, words[1]) + if xBad, ok := to.(*badExpr); ok { + return xBad } - from := words[0].Strings[0] - to := words[1].Strings[0] words[2].TrimLeftSpaces() words[2].TrimRightSpaces() obj := ctx.parseMakeString(node, words[2]) @@ -1398,13 +1407,13 @@ func (ctx *parseContext) parseSubstFunc(node mkparser.Node, fname string, args * return &callExpr{ object: obj, name: "replace", - args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}}, + args: []starlarkExpr{from, to}, returnType: typ, } } return &callExpr{ name: fname, - args: []starlarkExpr{&stringLiteralExpr{from}, &stringLiteralExpr{to}, obj}, + args: []starlarkExpr{from, to, obj}, returnType: obj.typ(), } } @@ -1476,9 +1485,7 @@ func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeSt // Handles the statements whose treatment is the same in all contexts: comment, // assignment, variable (which is a macro call in reality) and all constructs that // do not handle in any context ('define directive and any unrecognized stuff). -// Return true if we handled it. -func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool { - handled := true +func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) { switch x := node.(type) { case *mkparser.Comment: ctx.maybeHandleAnnotation(x) @@ -1493,13 +1500,14 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) bool { ctx.handleDefine(x) case "include", "-include": ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') + case "ifeq", "ifneq", "ifdef", "ifndef": + ctx.handleIfBlock(x) default: - handled = false + ctx.errorf(x, "unexpected directive %s", x.Name) } default: ctx.errorf(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#")) } - return handled } // Processes annotation. An annotation is a comment that starts with #RBC# and provides @@ -1537,11 +1545,12 @@ func (ctx *parseContext) carryAsComment(failedNode mkparser.Node) { // records that the given node failed to be converted and includes an explanatory message func (ctx *parseContext) errorf(failedNode mkparser.Node, message string, args ...interface{}) { if ctx.errorLogger != nil { - ctx.errorLogger.NewError(message, failedNode, args...) + ctx.errorLogger.NewError(ctx.script.mkFile, ctx.script.nodeLocator(failedNode.Pos()), failedNode, message, args...) } message = fmt.Sprintf(message, args...) ctx.insertComment(fmt.Sprintf("# MK2RBC TRANSLATION ERROR: %s", message)) ctx.carryAsComment(failedNode) + ctx.script.hasErrors = true } @@ -1658,6 +1667,7 @@ func Convert(req Request) (*StarlarkScript, error) { warnPartialSuccess: req.WarnPartialSuccess, sourceFS: req.SourceFS, makefileFinder: req.MakefileFinder, + nodeLocator: func(pos mkparser.Pos) int { return parser.Unpack(pos).Line }, } ctx := newParseContext(starScript, nodes) ctx.outputSuffix = req.OutputSuffix @@ -1671,21 +1681,7 @@ func Convert(req Request) (*StarlarkScript, error) { } ctx.pushReceiver(starScript) for ctx.hasNodes() && ctx.fatalError == nil { - node := ctx.getNode() - if ctx.handleSimpleStatement(node) { - continue - } - switch x := node.(type) { - case *mkparser.Directive: - switch x.Name { - case "ifeq", "ifneq", "ifdef", "ifndef": - ctx.handleIfBlock(x) - default: - ctx.errorf(x, "unexpected directive %s", x.Name) - } - default: - ctx.errorf(x, "unsupported line") - } + ctx.handleSimpleStatement(ctx.getNode()) } if ctx.fatalError != nil { return nil, ctx.fatalError @@ -1702,6 +1698,17 @@ func Launcher(mainModuleUri, versionDefaultsUri, mainModuleName string) string { return buf.String() } +func BoardLauncher(mainModuleUri string, inputVariablesUri string) string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName) + fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri) + fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri) + fmt.Fprintf(&buf, "globals, cfg, globals_base = %s(init, input_variables_init)\n", cfnBoardMain) + fmt.Fprintf(&buf, "# TODO: Some product config variables need to be printed, but most are readonly so we can't just print cfg here.\n") + fmt.Fprintf(&buf, "%s(globals, globals_base)\n", cfnPrintGlobals) + return buf.String() +} + func MakePath2ModuleName(mkPath string) string { return strings.TrimSuffix(mkPath, filepath.Ext(mkPath)) } diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index 511bb0f87..1e79552cd 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -226,6 +226,9 @@ ifdef PRODUCT_NAME PRODUCT_NAME = gizmo else endif +local_var := +ifdef local_var +endif `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -235,6 +238,9 @@ def init(g, handle): cfg["PRODUCT_NAME"] = "gizmo" else: pass + _local_var = "" + if _local_var: + pass `, }, { @@ -579,7 +585,7 @@ def init(g, handle): pass elif not rblf.board_platform_is(g, "copper"): pass - elif g.get("TARGET_BOARD_PLATFORM", "") not in g["QCOM_BOARD_PLATFORMS"]: + elif g.get("TARGET_BOARD_PLATFORM", "") in g["QCOM_BOARD_PLATFORMS"]: pass `, }, @@ -691,7 +697,7 @@ def init(g, handle): PRODUCT_COPY_FILES := $(addprefix pfx-,a b c) PRODUCT_COPY_FILES := $(addsuffix .sff, a b c) PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM))) -$(info $(patsubst %.pub,%,$(PRODUCT_ADB_KEYS))) +$(info $(patsubst %.pub,$(PRODUCT_NAME)%,$(PRODUCT_ADB_KEYS))) $(info $(dir foo/bar)) $(info $(firstword $(PRODUCT_COPY_FILES))) $(info $(lastword $(PRODUCT_COPY_FILES))) @@ -714,7 +720,7 @@ def init(g, handle): cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c") cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c") cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0] - rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%", g.get("PRODUCT_ADB_KEYS", ""))) + rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%s%%" % cfg["PRODUCT_NAME"], g.get("PRODUCT_ADB_KEYS", ""))) rblf.mkinfo("product.mk", rblf.dir("foo/bar")) rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][0]) rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][-1]) @@ -821,10 +827,30 @@ def init(g, handle): rblf.soong_config_namespace(g, "cvd") rblf.soong_config_set(g, "cvd", "launch_configs", "cvd_config_auto.json") rblf.soong_config_append(g, "cvd", "grub_config", "grub.cfg") - # MK2RBC TRANSLATION ERROR: SOONG_CONFIG_ variables cannot be referenced: SOONG_CONFIG_cvd_grub_config + # MK2RBC TRANSLATION ERROR: SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config # x := $(SOONG_CONFIG_cvd_grub_config) rblf.warning("product.mk", "partially successful conversion") `, + }, { + desc: "soong namespace accesses", + mkname: "product.mk", + in: ` +SOONG_CONFIG_NAMESPACES += cvd +SOONG_CONFIG_cvd += launch_configs +SOONG_CONFIG_cvd_launch_configs = cvd_config_auto.json +SOONG_CONFIG_cvd += grub_config +SOONG_CONFIG_cvd_grub_config += grub.cfg +x := $(call soong_config_get,cvd,grub_config) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.soong_config_namespace(g, "cvd") + rblf.soong_config_set(g, "cvd", "launch_configs", "cvd_config_auto.json") + rblf.soong_config_append(g, "cvd", "grub_config", "grub.cfg") + _x = rblf.soong_config_get(g, "cvd", "grub_config") +`, }, { desc: "string split", @@ -1001,6 +1027,21 @@ def init(g, handle): rblf.warning("product.mk", "partially successful conversion") `, }, + { + desc: "Flag override", + mkname: "product.mk", + in: ` +override FOO:=`, + expected: `# MK2RBC TRANSLATION ERROR: cannot handle override directive +# override FOO := +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["override FOO"] = "" + rblf.warning("product.mk", "partially successful conversion") +`, + }, } var known_variables = []struct { diff --git a/mk2rbc/variable.go b/mk2rbc/variable.go index 4bb9ed52d..ded07fed6 100644 --- a/mk2rbc/variable.go +++ b/mk2rbc/variable.go @@ -177,8 +177,8 @@ type localVariable struct { baseVariable } -func (lv localVariable) emitDefined(_ *generationContext) { - panic("implement me") +func (lv localVariable) emitDefined(gctx *generationContext) { + gctx.writef(lv.String()) } func (lv localVariable) String() string { diff --git a/rust/Android.bp b/rust/Android.bp index 0ee673de4..cda2dbc6a 100644 --- a/rust/Android.bp +++ b/rust/Android.bp @@ -54,6 +54,7 @@ bootstrap_go_package { "project_json_test.go", "protobuf_test.go", "rust_test.go", + "sanitize_test.go", "source_provider_test.go", "test_test.go", "vendor_snapshot_test.go", diff --git a/rust/androidmk.go b/rust/androidmk.go index 630805a85..c9f648632 100644 --- a/rust/androidmk.go +++ b/rust/androidmk.go @@ -50,7 +50,7 @@ func (mod *Module) AndroidMkEntries() []android.AndroidMkEntries { } ret := android.AndroidMkEntries{ - OutputFile: mod.unstrippedOutputFile, + OutputFile: android.OptionalPathForPath(mod.UnstrippedOutputFile()), Include: "$(BUILD_SYSTEM)/soong_rust_prebuilt.mk", ExtraEntries: []android.AndroidMkExtraEntriesFunc{ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { @@ -137,10 +137,15 @@ func (library *libraryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.An } else if library.shared() { ret.Class = "SHARED_LIBRARIES" } - if library.distFile.Valid() { ret.DistFiles = android.MakeDefaultDistFiles(library.distFile.Path()) } + ret.ExtraEntries = append(ret.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + if library.tocFile.Valid() { + entries.SetString("LOCAL_SOONG_TOC", library.tocFile.String()) + } + }) } func (procMacro *procMacroDecorator) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkEntries) { @@ -181,18 +186,13 @@ func (compiler *baseCompiler) AndroidMk(ctx AndroidMkContext, ret *android.Andro return } - var unstrippedOutputFile android.OptionalPath - // Soong installation is only supported for host modules. Have Make - // installation trigger Soong installation. - if ctx.Target().Os.Class == android.Host { - ret.OutputFile = android.OptionalPathForPath(compiler.path) - } else if compiler.strippedOutputFile.Valid() { - unstrippedOutputFile = ret.OutputFile + if compiler.strippedOutputFile.Valid() { ret.OutputFile = compiler.strippedOutputFile } + ret.ExtraEntries = append(ret.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.SetOptionalPath("LOCAL_SOONG_UNSTRIPPED_BINARY", unstrippedOutputFile) + entries.SetPath("LOCAL_SOONG_UNSTRIPPED_BINARY", compiler.unstrippedOutputFile) path, file := filepath.Split(compiler.path.ToMakePath().String()) stem, suffix, _ := android.SplitFileExt(file) entries.SetString("LOCAL_MODULE_SUFFIX", suffix) diff --git a/rust/binary.go b/rust/binary.go index 7c18730c6..db91ccb9a 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -34,6 +34,7 @@ type BinaryCompilerProperties struct { type binaryInterface interface { binary() bool staticallyLinked() bool + testBinary() bool } type binaryDecorator struct { @@ -122,20 +123,24 @@ func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps Path fileName := binary.getStem(ctx) + ctx.toolchain().ExecutableSuffix() srcPath, _ := srcPathFromModuleSrcs(ctx, binary.baseCompiler.Properties.Srcs) outputFile := android.PathForModuleOut(ctx, fileName) + ret := outputFile flags.RustFlags = append(flags.RustFlags, deps.depFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.linkObjects...) - TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile) - if binary.stripper.NeedsStrip(ctx) { - strippedOutputFile := android.PathForModuleOut(ctx, "stripped", fileName) + strippedOutputFile := outputFile + outputFile = android.PathForModuleOut(ctx, "unstripped", fileName) binary.stripper.StripExecutableOrSharedLib(ctx, outputFile, strippedOutputFile) - binary.strippedOutputFile = android.OptionalPathForPath(strippedOutputFile) + + binary.baseCompiler.strippedOutputFile = android.OptionalPathForPath(strippedOutputFile) } + binary.baseCompiler.unstrippedOutputFile = outputFile - return outputFile + TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile) + + return ret } func (binary *binaryDecorator) autoDep(ctx android.BottomUpMutatorContext) autoDep { @@ -168,3 +173,7 @@ func (binary *binaryDecorator) binary() bool { func (binary *binaryDecorator) staticallyLinked() bool { return Bool(binary.Properties.Static_executable) } + +func (binary *binaryDecorator) testBinary() bool { + return false +} diff --git a/rust/binary_test.go b/rust/binary_test.go index 968c0c1ff..7dac2490a 100644 --- a/rust/binary_test.go +++ b/rust/binary_test.go @@ -106,7 +106,7 @@ func TestBinaryFlags(t *testing.T) { srcs: ["foo.rs"], }`) - fizzBuzz := ctx.ModuleForTests("fizz-buzz", "linux_glibc_x86_64").Output("fizz-buzz") + fizzBuzz := ctx.ModuleForTests("fizz-buzz", "linux_glibc_x86_64").Rule("rustc") flags := fizzBuzz.Args["rustcFlags"] if strings.Contains(flags, "--test") { @@ -139,7 +139,7 @@ func TestStaticBinaryFlags(t *testing.T) { static_executable: true, }`) - fizzOut := ctx.ModuleForTests("fizz", "android_arm64_armv8-a").Output("fizz") + fizzOut := ctx.ModuleForTests("fizz", "android_arm64_armv8-a").Rule("rustc") fizzMod := ctx.ModuleForTests("fizz", "android_arm64_armv8-a").Module().(*Module) flags := fizzOut.Args["rustcFlags"] @@ -173,7 +173,7 @@ func TestLinkObjects(t *testing.T) { name: "libfoo", }`) - fizzBuzz := ctx.ModuleForTests("fizz-buzz", "android_arm64_armv8-a").Output("fizz-buzz") + fizzBuzz := ctx.ModuleForTests("fizz-buzz", "android_arm64_armv8-a").Rule("rustc") linkFlags := fizzBuzz.Args["linkFlags"] if !strings.Contains(linkFlags, "/libfoo.so") { t.Errorf("missing shared dependency 'libfoo.so' in linkFlags: %#v", linkFlags) @@ -197,15 +197,17 @@ func TestStrippedBinary(t *testing.T) { `) foo := ctx.ModuleForTests("foo", "android_arm64_armv8-a") - foo.Output("stripped/foo") + foo.Output("unstripped/foo") + foo.Output("foo") + // Check that the `cp` rules is using the stripped version as input. cp := foo.Rule("android.Cp") - if !strings.HasSuffix(cp.Input.String(), "stripped/foo") { + if strings.HasSuffix(cp.Input.String(), "unstripped/foo") { t.Errorf("installed binary not based on stripped version: %v", cp.Input) } - fizzBar := ctx.ModuleForTests("bar", "android_arm64_armv8-a").MaybeOutput("stripped/bar") + fizzBar := ctx.ModuleForTests("bar", "android_arm64_armv8-a").MaybeOutput("unstripped/bar") if fizzBar.Rule != nil { - t.Errorf("stripped version of bar has been generated") + t.Errorf("unstripped binary exists, so stripped binary has incorrectly been generated") } } diff --git a/rust/bindgen.go b/rust/bindgen.go index 845f2586d..32d02e4f6 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -15,6 +15,7 @@ package rust import ( + "fmt" "strings" "github.com/google/blueprint" @@ -147,6 +148,31 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr cflags = append(cflags, strings.ReplaceAll(ccToolchain.Cflags(), "${config.", "${cc_config.")) cflags = append(cflags, strings.ReplaceAll(ccToolchain.ToolchainCflags(), "${config.", "${cc_config.")) + if ctx.RustModule().UseVndk() { + cflags = append(cflags, "-D__ANDROID_VNDK__") + if ctx.RustModule().InVendor() { + cflags = append(cflags, "-D__ANDROID_VENDOR__") + } else if ctx.RustModule().InProduct() { + cflags = append(cflags, "-D__ANDROID_PRODUCT__") + } + } + + if ctx.RustModule().InRecovery() { + cflags = append(cflags, "-D__ANDROID_RECOVERY__") + } + + if mctx, ok := ctx.(*moduleContext); ok && mctx.apexVariationName() != "" { + cflags = append(cflags, "-D__ANDROID_APEX__") + if ctx.Device() { + cflags = append(cflags, fmt.Sprintf("-D__ANDROID_APEX_MIN_SDK_VERSION__=%d", + ctx.RustModule().apexSdkVersion.FinalOrFutureInt())) + } + } + + if ctx.Target().NativeBridge == android.NativeBridgeEnabled { + cflags = append(cflags, "-D__ANDROID_NATIVE_BRIDGE__") + } + // Dependency clang flags and include paths cflags = append(cflags, deps.depClangFlags...) for _, include := range deps.depIncludePaths { diff --git a/rust/builder.go b/rust/builder.go index f79cf9b12..60b5926e8 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -185,7 +185,7 @@ func rustEnvVars(ctx ModuleContext, deps PathDeps) []string { } func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, flags Flags, - outputFile android.WritablePath, crate_type string) buildOutput { + outputFile android.WritablePath, crateType string) buildOutput { var inputs android.Paths var implicits android.Paths @@ -204,7 +204,7 @@ func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, fl // Collect rustc flags rustcFlags = append(rustcFlags, flags.GlobalRustFlags...) rustcFlags = append(rustcFlags, flags.RustFlags...) - rustcFlags = append(rustcFlags, "--crate-type="+crate_type) + rustcFlags = append(rustcFlags, "--crate-type="+crateType) if crateName != "" { rustcFlags = append(rustcFlags, "--crate-name="+crateName) } @@ -361,7 +361,7 @@ func Rustdoc(ctx ModuleContext, main android.Path, deps PathDeps, Description: "rustdoc " + main.Rel(), Output: docTimestampFile, Input: main, - Implicit: ctx.RustModule().unstrippedOutputFile.Path(), + Implicit: ctx.RustModule().UnstrippedOutputFile(), Args: map[string]string{ "rustdocFlags": strings.Join(rustdocFlags, " "), "outDir": docDir.String(), diff --git a/rust/compiler.go b/rust/compiler.go index cada9854a..3040e5d9e 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -181,7 +181,11 @@ type baseCompiler struct { sanitize *sanitize distFile android.OptionalPath - // Stripped output file. If Valid(), this file will be installed instead of outputFile. + + // unstripped output file. + unstrippedOutputFile android.Path + + // stripped output file. strippedOutputFile android.OptionalPath // If a crate has a source-generated dependency, a copy of the source file @@ -340,6 +344,10 @@ func (compiler *baseCompiler) CargoPkgVersion() string { return String(compiler.Properties.Cargo_pkg_version) } +func (compiler *baseCompiler) unstrippedOutputFilePath() android.Path { + return compiler.unstrippedOutputFile +} + func (compiler *baseCompiler) strippedOutputFilePath() android.OptionalPath { return compiler.strippedOutputFile } @@ -355,9 +363,9 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { if !Bool(compiler.Properties.No_stdlibs) { for _, stdlib := range config.Stdlibs { - // If we're building for the primary arch of the build host, use the compiler's stdlibs + // If we're building for the build host, use the prebuilt stdlibs if ctx.Target().Os == ctx.Config().BuildOS { - stdlib = stdlib + "_" + ctx.toolchain().RustTriple() + stdlib = "prebuilt_" + stdlib } deps.Stdlibs = append(deps.Stdlibs, stdlib) } diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go index 8182c327a..0d0b712c2 100644 --- a/rust/config/allowed_list.go +++ b/rust/config/allowed_list.go @@ -11,17 +11,19 @@ var ( "external/crosvm", "external/libchromeos-rs", "external/minijail", + "external/open-dice", "external/rust", "external/selinux/libselinux", "external/uwb", "external/vm_tools/p9", "frameworks/native/libs/binder/rust", "frameworks/proto_logging/stats", + "hardware/interfaces/security", + "packages/modules/Bluetooth", "packages/modules/DnsResolver", "packages/modules/Uwb", "packages/modules/Virtualization", "prebuilts/rust", - "system/bt", "system/core/libstats/pull_rust", "system/extras/profcollectd", "system/extras/simpleperf", diff --git a/rust/config/global.go b/rust/config/global.go index b163bb6ac..ebddb75ce 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.55.0" + RustDefaultVersion = "1.56.1" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2018" Stdlibs = []string{ diff --git a/rust/fuzz.go b/rust/fuzz.go index a628b6158..55921ba4b 100644 --- a/rust/fuzz.go +++ b/rust/fuzz.go @@ -36,7 +36,7 @@ type fuzzDecorator struct { fuzzPackagedModule fuzz.FuzzPackagedModule } -var _ compiler = (*binaryDecorator)(nil) +var _ compiler = (*fuzzDecorator)(nil) // rust_binary produces a binary that is runnable on a device. func RustFuzzFactory() android.Module { @@ -147,7 +147,7 @@ func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { files = s.PackageArtifacts(ctx, module, fuzzModule.fuzzPackagedModule, archDir, builder) // The executable. - files = append(files, fuzz.FileToZip{rustModule.unstrippedOutputFile.Path(), ""}) + files = append(files, fuzz.FileToZip{rustModule.UnstrippedOutputFile(), ""}) // Grab the list of required shared libraries. sharedLibraries := fuzz.CollectAllSharedDependencies(ctx, module, cc.UnstrippedOutputFile, cc.IsValidSharedDependency) diff --git a/rust/fuzz_test.go b/rust/fuzz_test.go index 2524f9176..98be7c20d 100644 --- a/rust/fuzz_test.go +++ b/rust/fuzz_test.go @@ -45,7 +45,7 @@ func TestRustFuzz(t *testing.T) { } // Check that compiler flags are set appropriately . - fuzz_libtest := ctx.ModuleForTests("fuzz_libtest", "android_arm64_armv8-a_fuzzer").Output("fuzz_libtest") + fuzz_libtest := ctx.ModuleForTests("fuzz_libtest", "android_arm64_armv8-a_fuzzer").Rule("rustc") if !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-Z sanitizer=hwaddress") || !strings.Contains(fuzz_libtest.Args["rustcFlags"], "-C passes='sancov'") || !strings.Contains(fuzz_libtest.Args["rustcFlags"], "--cfg fuzzing") { diff --git a/rust/library.go b/rust/library.go index 38dae4d33..07843fe32 100644 --- a/rust/library.go +++ b/rust/library.go @@ -102,6 +102,9 @@ type libraryDecorator struct { sourceProvider SourceProvider collectedSnapshotHeaders android.Paths + + // table-of-contents file for cdylib crates to optimize out relinking when possible + tocFile android.OptionalPath } type libraryInterface interface { @@ -137,12 +140,18 @@ type libraryInterface interface { BuildOnlyDylib() BuildOnlyStatic() BuildOnlyShared() + + toc() android.OptionalPath } func (library *libraryDecorator) nativeCoverage() bool { return true } +func (library *libraryDecorator) toc() android.OptionalPath { + return library.tocFile +} + func (library *libraryDecorator) rlib() bool { return library.MutatedProperties.VariantIsRlib } @@ -460,7 +469,7 @@ func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags) F } func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { - var outputFile android.ModuleOutPath + var outputFile, ret android.ModuleOutPath var fileName string srcPath := library.srcPath(ctx, deps) @@ -468,6 +477,34 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa deps.srcProviderFiles = append(deps.srcProviderFiles, library.sourceProvider.Srcs()...) } + // Calculate output filename + if library.rlib() { + fileName = library.getStem(ctx) + ctx.toolchain().RlibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + ret = outputFile + } else if library.dylib() { + fileName = library.getStem(ctx) + ctx.toolchain().DylibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + ret = outputFile + } else if library.static() { + fileName = library.getStem(ctx) + ctx.toolchain().StaticLibSuffix() + outputFile = android.PathForModuleOut(ctx, fileName) + ret = outputFile + } else if library.shared() { + fileName = library.sharedLibFilename(ctx) + outputFile = android.PathForModuleOut(ctx, fileName) + ret = outputFile + } + + if !library.rlib() && !library.static() && library.stripper.NeedsStrip(ctx) { + strippedOutputFile := outputFile + outputFile = android.PathForModuleOut(ctx, "unstripped", fileName) + library.stripper.StripExecutableOrSharedLib(ctx, outputFile, strippedOutputFile) + + library.baseCompiler.strippedOutputFile = android.OptionalPathForPath(strippedOutputFile) + } + library.baseCompiler.unstrippedOutputFile = outputFile + flags.RustFlags = append(flags.RustFlags, deps.depFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.linkObjects...) @@ -479,34 +516,17 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa flags.RustFlags = append(flags.RustFlags, "-C prefer-dynamic") } + // Call the appropriate builder for this library type if library.rlib() { - fileName = library.getStem(ctx) + ctx.toolchain().RlibSuffix() - outputFile = android.PathForModuleOut(ctx, fileName) - TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile) } else if library.dylib() { - fileName = library.getStem(ctx) + ctx.toolchain().DylibSuffix() - outputFile = android.PathForModuleOut(ctx, fileName) - TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile) } else if library.static() { - fileName = library.getStem(ctx) + ctx.toolchain().StaticLibSuffix() - outputFile = android.PathForModuleOut(ctx, fileName) - TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile) } else if library.shared() { - fileName = library.sharedLibFilename(ctx) - outputFile = android.PathForModuleOut(ctx, fileName) - TransformSrctoShared(ctx, srcPath, deps, flags, outputFile) } - if !library.rlib() && !library.static() && library.stripper.NeedsStrip(ctx) { - strippedOutputFile := android.PathForModuleOut(ctx, "stripped", fileName) - library.stripper.StripExecutableOrSharedLib(ctx, outputFile, strippedOutputFile) - library.strippedOutputFile = android.OptionalPathForPath(strippedOutputFile) - } - if library.rlib() || library.dylib() { library.flagExporter.exportLinkDirs(deps.linkDirs...) library.flagExporter.exportLinkObjects(deps.linkObjects...) @@ -519,9 +539,16 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa } if library.shared() { + // Optimize out relinking against shared libraries whose interface hasn't changed by + // depending on a table of contents file instead of the library itself. + tocFile := outputFile.ReplaceExtension(ctx, flags.Toolchain.SharedLibSuffix()[1:]+".toc") + library.tocFile = android.OptionalPathForPath(tocFile) + cc.TransformSharedObjectToToc(ctx, outputFile, tocFile) + ctx.SetProvider(cc.SharedLibraryInfoProvider, cc.SharedLibraryInfo{ - SharedLibrary: outputFile, - Target: ctx.Target(), + TableOfContents: android.OptionalPathForPath(tocFile), + SharedLibrary: outputFile, + Target: ctx.Target(), }) } @@ -536,7 +563,7 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa library.flagExporter.setProvider(ctx) - return outputFile + return ret } func (library *libraryDecorator) srcPath(ctx ModuleContext, deps PathDeps) android.Path { diff --git a/rust/library_test.go b/rust/library_test.go index cb4ef7eec..d78dcdd97 100644 --- a/rust/library_test.go +++ b/rust/library_test.go @@ -37,10 +37,10 @@ func TestLibraryVariants(t *testing.T) { }`) // Test all variants are being built. - libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib_rlib-std").Output("libfoo.rlib") - libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Output("libfoo.dylib.so") - libfooStatic := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_static").Output("libfoo.ffi.a") - libfooShared := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_shared").Output("libfoo.ffi.so") + libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib_rlib-std").Rule("rustc") + libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Rule("rustc") + libfooStatic := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_static").Rule("rustc") + libfooShared := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_shared").Rule("rustc") rlibCrateType := "rlib" dylibCrateType := "dylib" @@ -78,7 +78,7 @@ func TestDylibPreferDynamic(t *testing.T) { crate_name: "foo", }`) - libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Output("libfoo.dylib.so") + libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Rule("rustc") if !strings.Contains(libfooDylib.Args["rustcFlags"], "prefer-dynamic") { t.Errorf("missing prefer-dynamic flag for libfoo dylib, rustcFlags: %#v", libfooDylib.Args["rustcFlags"]) @@ -94,7 +94,7 @@ func TestAndroidDylib(t *testing.T) { crate_name: "foo", }`) - libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Output("libfoo.dylib.so") + libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Rule("rustc") if !strings.Contains(libfooDylib.Args["rustcFlags"], "--cfg 'android_dylib'") { t.Errorf("missing android_dylib cfg flag for libfoo dylib, rustcFlags: %#v", libfooDylib.Args["rustcFlags"]) @@ -148,7 +148,7 @@ func TestSharedLibrary(t *testing.T) { libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared") - libfooOutput := libfoo.Output("libfoo.so") + libfooOutput := libfoo.Rule("rustc") if !strings.Contains(libfooOutput.Args["linkFlags"], "-Wl,-soname=libfoo.so") { t.Errorf("missing expected -Wl,-soname linker flag for libfoo shared lib, linkFlags: %#v", libfooOutput.Args["linkFlags"]) @@ -160,6 +160,26 @@ func TestSharedLibrary(t *testing.T) { } } +func TestSharedLibraryToc(t *testing.T) { + ctx := testRust(t, ` + rust_ffi_shared { + name: "libfoo", + srcs: ["foo.rs"], + crate_name: "foo", + } + cc_binary { + name: "fizzbuzz", + shared_libs: ["libfoo"], + }`) + + fizzbuzz := ctx.ModuleForTests("fizzbuzz", "android_arm64_armv8-a").Rule("ld") + + if !android.SuffixInList(fizzbuzz.Implicits.Strings(), "libfoo.so.toc") { + t.Errorf("missing expected libfoo.so.toc implicit dependency, instead found: %#v", + fizzbuzz.Implicits.Strings()) + } +} + func TestStaticLibraryLinkage(t *testing.T) { ctx := testRust(t, ` rust_ffi_static { @@ -242,16 +262,17 @@ func TestStrippedLibrary(t *testing.T) { `) foo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_dylib") - foo.Output("stripped/libfoo.dylib.so") + foo.Output("libfoo.dylib.so") + foo.Output("unstripped/libfoo.dylib.so") // Check that the `cp` rule is using the stripped version as input. cp := foo.Rule("android.Cp") - if !strings.HasSuffix(cp.Input.String(), "stripped/libfoo.dylib.so") { - t.Errorf("installed binary not based on stripped version: %v", cp.Input) + if strings.HasSuffix(cp.Input.String(), "unstripped/libfoo.dylib.so") { + t.Errorf("installed library not based on stripped version: %v", cp.Input) } - fizzBar := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_dylib").MaybeOutput("stripped/libbar.dylib.so") + fizzBar := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_dylib").MaybeOutput("unstripped/libbar.dylib.so") if fizzBar.Rule != nil { - t.Errorf("stripped version of bar has been generated") + t.Errorf("unstripped library exists, so stripped library has incorrectly been generated") } } diff --git a/rust/prebuilt.go b/rust/prebuilt.go index 49f3c0f79..6f17272f7 100644 --- a/rust/prebuilt.go +++ b/rust/prebuilt.go @@ -32,6 +32,8 @@ type PrebuiltProperties struct { } type prebuiltLibraryDecorator struct { + android.Prebuilt + *libraryDecorator Properties PrebuiltProperties } @@ -54,6 +56,13 @@ func PrebuiltRlibFactory() android.Module { return module.Init() } +func addSrcSupplier(module android.PrebuiltInterface, prebuilt *prebuiltLibraryDecorator) { + srcsSupplier := func(_ android.BaseModuleContext, _ android.Module) []string { + return prebuilt.prebuiltSrcs() + } + android.InitPrebuiltModuleWithSrcSupplier(module, srcsSupplier, "srcs") +} + func NewPrebuiltLibrary(hod android.HostOrDeviceSupported) (*Module, *prebuiltLibraryDecorator) { module, library := NewRustLibrary(hod) library.BuildOnlyRust() @@ -62,6 +71,9 @@ func NewPrebuiltLibrary(hod android.HostOrDeviceSupported) (*Module, *prebuiltLi libraryDecorator: library, } module.compiler = prebuilt + + addSrcSupplier(module, prebuilt) + return module, prebuilt } @@ -73,6 +85,9 @@ func NewPrebuiltDylib(hod android.HostOrDeviceSupported) (*Module, *prebuiltLibr libraryDecorator: library, } module.compiler = prebuilt + + addSrcSupplier(module, prebuilt) + return module, prebuilt } @@ -84,6 +99,9 @@ func NewPrebuiltRlib(hod android.HostOrDeviceSupported) (*Module, *prebuiltLibra libraryDecorator: library, } module.compiler = prebuilt + + addSrcSupplier(module, prebuilt) + return module, prebuilt } @@ -100,6 +118,7 @@ func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags if len(paths) > 0 { ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)") } + prebuilt.baseCompiler.unstrippedOutputFile = srcPath return srcPath } @@ -129,3 +148,7 @@ func (prebuilt *prebuiltLibraryDecorator) prebuiltSrcs() []string { return srcs } + +func (prebuilt *prebuiltLibraryDecorator) prebuilt() *android.Prebuilt { + return &prebuilt.Prebuilt +} diff --git a/rust/proc_macro.go b/rust/proc_macro.go index 804d79fe9..974c096c4 100644 --- a/rust/proc_macro.go +++ b/rust/proc_macro.go @@ -75,6 +75,7 @@ func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, dep srcPath, _ := srcPathFromModuleSrcs(ctx, procMacro.baseCompiler.Properties.Srcs) TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile) + procMacro.baseCompiler.unstrippedOutputFile = outputFile return outputFile } diff --git a/rust/rust.go b/rust/rust.go index c465cb609..3cc786878 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -156,13 +156,15 @@ type Module struct { sourceProvider SourceProvider subAndroidMkOnce map[SubAndroidMkProvider]bool - // Unstripped output. This is usually used when this module is linked to another module - // as a library. The stripped output which is used for installation can be found via - // compiler.strippedOutputFile if it exists. - unstrippedOutputFile android.OptionalPath - docTimestampFile android.OptionalPath + // Output file to be installed, may be stripped or unstripped. + outputFile android.OptionalPath + + docTimestampFile android.OptionalPath hideApexVariantFromMake bool + + // For apex variants, this is set as apex.min_sdk_version + apexSdkVersion android.ApiLevel } func (mod *Module) Header() bool { @@ -281,8 +283,8 @@ func (mod *Module) Object() bool { func (mod *Module) Toc() android.OptionalPath { if mod.compiler != nil { - if _, ok := mod.compiler.(libraryInterface); ok { - return android.OptionalPath{} + if lib, ok := mod.compiler.(libraryInterface); ok { + return lib.toc() } } panic(fmt.Errorf("Toc() called on non-library module: %q", mod.BaseModuleName())) @@ -465,6 +467,7 @@ type compiler interface { stdLinkage(ctx *depsContext) RustLinkage + unstrippedOutputFilePath() android.Path strippedOutputFilePath() android.OptionalPath } @@ -593,8 +596,8 @@ func (mod *Module) CcLibraryInterface() bool { } func (mod *Module) UnstrippedOutputFile() android.Path { - if mod.unstrippedOutputFile.Valid() { - return mod.unstrippedOutputFile.Path() + if mod.compiler != nil { + return mod.compiler.unstrippedOutputFilePath() } return nil } @@ -651,10 +654,7 @@ func (mod *Module) Module() android.Module { } func (mod *Module) OutputFile() android.OptionalPath { - if mod.compiler != nil && mod.compiler.strippedOutputFilePath().Valid() { - return mod.compiler.strippedOutputFilePath() - } - return mod.unstrippedOutputFile + return mod.outputFile } func (mod *Module) CoverageFiles() android.Paths { @@ -678,6 +678,10 @@ func (mod *Module) installable(apexInfo android.ApexInfo) bool { return mod.OutputFile().Valid() && !mod.Properties.PreventInstall } +func (ctx moduleContext) apexVariationName() string { + return ctx.Provider(android.ApexInfoProvider).(android.ApexInfo).ApexVariationName +} + var _ cc.LinkableInterface = (*Module)(nil) func (mod *Module) Init() android.Module { @@ -885,9 +889,12 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if mod.compiler != nil && !mod.compiler.Disabled() { mod.compiler.initialize(ctx) - unstrippedOutputFile := mod.compiler.compile(ctx, flags, deps) - mod.unstrippedOutputFile = android.OptionalPathForPath(unstrippedOutputFile) - bloaty.MeasureSizeForPaths(ctx, mod.compiler.strippedOutputFilePath(), mod.unstrippedOutputFile) + outputFile := mod.compiler.compile(ctx, flags, deps) + if ctx.Failed() { + return + } + mod.outputFile = android.OptionalPathForPath(outputFile) + bloaty.MeasureSizeForPaths(ctx, mod.compiler.strippedOutputFilePath(), android.OptionalPathForPath(mod.compiler.unstrippedOutputFilePath())) mod.docTimestampFile = mod.compiler.rustdoc(ctx, flags, deps) @@ -1012,6 +1019,13 @@ func (mod *Module) begin(ctx BaseModuleContext) { } } +func (mod *Module) Prebuilt() *android.Prebuilt { + if p, ok := mod.compiler.(*prebuiltLibraryDecorator); ok { + return p.prebuilt() + } + return nil +} + func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { var depPaths PathDeps @@ -1023,6 +1037,20 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { directSrcProvidersDeps := []*Module{} directSrcDeps := [](android.SourceFileProducer){} + // For the dependency from platform to apex, use the latest stubs + mod.apexSdkVersion = android.FutureApiLevel + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + if !apexInfo.IsForPlatform() { + mod.apexSdkVersion = apexInfo.MinSdkVersion + } + + if android.InList("hwaddress", ctx.Config().SanitizeDevice()) { + // In hwasan build, we override apexSdkVersion to the FutureApiLevel(10000) + // so that even Q(29/Android10) apexes could use the dynamic unwinder by linking the newer stubs(e.g libc(R+)). + // (b/144430859) + mod.apexSdkVersion = android.FutureApiLevel + } + ctx.VisitDirectDeps(func(dep android.Module) { depName := ctx.OtherModuleName(dep) depTag := ctx.OtherModuleDependencyTag(dep) @@ -1083,13 +1111,8 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { } if depTag == dylibDepTag || depTag == rlibDepTag || depTag == procMacroDepTag { - linkFile := rustDep.unstrippedOutputFile - if !linkFile.Valid() { - ctx.ModuleErrorf("Invalid output file when adding dep %q to %q", - depName, ctx.ModuleName()) - return - } - linkDir := linkPathFromFilePath(linkFile.Path()) + linkFile := rustDep.UnstrippedOutputFile() + linkDir := linkPathFromFilePath(linkFile) if lib, ok := mod.compiler.(exportedFlagsProducer); ok { lib.exportLinkDirs(linkDir) } @@ -1198,15 +1221,15 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { var rlibDepFiles RustLibraries for _, dep := range directRlibDeps { - rlibDepFiles = append(rlibDepFiles, RustLibrary{Path: dep.unstrippedOutputFile.Path(), CrateName: dep.CrateName()}) + rlibDepFiles = append(rlibDepFiles, RustLibrary{Path: dep.UnstrippedOutputFile(), CrateName: dep.CrateName()}) } var dylibDepFiles RustLibraries for _, dep := range directDylibDeps { - dylibDepFiles = append(dylibDepFiles, RustLibrary{Path: dep.unstrippedOutputFile.Path(), CrateName: dep.CrateName()}) + dylibDepFiles = append(dylibDepFiles, RustLibrary{Path: dep.UnstrippedOutputFile(), CrateName: dep.CrateName()}) } var procMacroDepFiles RustLibraries for _, dep := range directProcMacroDeps { - procMacroDepFiles = append(procMacroDepFiles, RustLibrary{Path: dep.unstrippedOutputFile.Path(), CrateName: dep.CrateName()}) + procMacroDepFiles = append(procMacroDepFiles, RustLibrary{Path: dep.UnstrippedOutputFile(), CrateName: dep.CrateName()}) } var staticLibDepFiles android.Paths @@ -1273,6 +1296,10 @@ func (mod *Module) InstallInRecovery() bool { return mod.InRecovery() } +func (mod *Module) InstallBypassMake() bool { + return true +} + func linkPathFromFilePath(filepath android.Path) string { return strings.Split(filepath.String(), filepath.Base())[0] } diff --git a/rust/rust_test.go b/rust/rust_test.go index 80f693eb6..b99b1e63b 100644 --- a/rust/rust_test.go +++ b/rust/rust_test.go @@ -439,6 +439,13 @@ func TestLibrarySizes(t *testing.T) { }`) m := ctx.SingletonForTests("file_metrics") + m.Output("unstripped/libwaldo.dylib.so.bloaty.csv") m.Output("libwaldo.dylib.so.bloaty.csv") - m.Output("stripped/libwaldo.dylib.so.bloaty.csv") +} + +func assertString(t *testing.T, got, expected string) { + t.Helper() + if got != expected { + t.Errorf("expected %q got %q", expected, got) + } } diff --git a/rust/sanitize.go b/rust/sanitize.go index baa383da6..fdb342d40 100644 --- a/rust/sanitize.go +++ b/rust/sanitize.go @@ -15,20 +15,39 @@ package rust import ( + "fmt" + "strings" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + "android/soong/android" "android/soong/cc" "android/soong/rust/config" - "fmt" - "github.com/google/blueprint" ) +// TODO: When Rust has sanitizer-parity with CC, deduplicate this struct type SanitizeProperties struct { // enable AddressSanitizer, HWAddressSanitizer, and others. Sanitize struct { Address *bool `android:"arch_variant"` Hwaddress *bool `android:"arch_variant"` - Fuzzer *bool `android:"arch_variant"` - Never *bool `android:"arch_variant"` + + // Memory-tagging, only available on arm64 + // if diag.memtag unset or false, enables async memory tagging + Memtag_heap *bool `android:"arch_variant"` + Fuzzer *bool `android:"arch_variant"` + Never *bool `android:"arch_variant"` + + // Sanitizers to run in the diagnostic mode (as opposed to the release mode). + // Replaces abort() on error with a human-readable error message. + // Address and Thread sanitizers always run in diagnostic mode. + Diag struct { + // Memory-tagging, only available on arm64 + // requires sanitizer.memtag: true + // if set, enables sync memory tagging + Memtag_heap *bool `android:"arch_variant"` + } } SanitizerEnabled bool `blueprint:"mutated"` SanitizeDep bool `blueprint:"mutated"` @@ -59,9 +78,18 @@ var asanFlags = []string{ "-Z sanitizer=address", } +// See cc/sanitize.go's hwasanGlobalOptions for global hwasan options. var hwasanFlags = []string{ "-Z sanitizer=hwaddress", "-C target-feature=+tagged-globals", + + // Flags from cc/sanitize.go hwasanFlags + "-C llvm-args=--aarch64-enable-global-isel-at-O=-1", + "-C llvm-args=-fast-isel=false", + "-C llvm-args=-instcombine-lower-dbg-declare=0", + + // Additional flags for HWASAN-ified Rust/C interop + "-C llvm-args=--hwasan-with-ifunc", } func boolPtr(v bool) *bool { @@ -79,7 +107,85 @@ func (sanitize *sanitize) props() []interface{} { } func (sanitize *sanitize) begin(ctx BaseModuleContext) { - s := sanitize.Properties.Sanitize + s := &sanitize.Properties.Sanitize + + // Never always wins. + if Bool(s.Never) { + return + } + + // rust_test targets default to SYNC MemTag unless explicitly set to ASYNC (via diag: {Memtag_heap}). + if binary, ok := ctx.RustModule().compiler.(binaryInterface); ok && binary.testBinary() { + if s.Memtag_heap == nil { + s.Memtag_heap = proptools.BoolPtr(true) + } + if s.Diag.Memtag_heap == nil { + s.Diag.Memtag_heap = proptools.BoolPtr(true) + } + } + + var globalSanitizers []string + var globalSanitizersDiag []string + + if ctx.Host() { + if !ctx.Windows() { + globalSanitizers = ctx.Config().SanitizeHost() + } + } else { + arches := ctx.Config().SanitizeDeviceArch() + if len(arches) == 0 || android.InList(ctx.Arch().ArchType.Name, arches) { + globalSanitizers = ctx.Config().SanitizeDevice() + globalSanitizersDiag = ctx.Config().SanitizeDeviceDiag() + } + } + + if len(globalSanitizers) > 0 { + var found bool + + // Global Sanitizers + if found, globalSanitizers = android.RemoveFromList("hwaddress", globalSanitizers); found && s.Hwaddress == nil { + // TODO(b/180495975): HWASan for static Rust binaries isn't supported yet. + if !ctx.RustModule().StaticExecutable() { + s.Hwaddress = proptools.BoolPtr(true) + } + } + + if found, globalSanitizers = android.RemoveFromList("memtag_heap", globalSanitizers); found && s.Memtag_heap == nil { + if !ctx.Config().MemtagHeapDisabledForPath(ctx.ModuleDir()) { + s.Memtag_heap = proptools.BoolPtr(true) + } + } + + if found, globalSanitizers = android.RemoveFromList("address", globalSanitizers); found && s.Address == nil { + s.Address = proptools.BoolPtr(true) + } + + if found, globalSanitizers = android.RemoveFromList("fuzzer", globalSanitizers); found && s.Fuzzer == nil { + s.Fuzzer = proptools.BoolPtr(true) + } + + // Global Diag Sanitizers + if found, globalSanitizersDiag = android.RemoveFromList("memtag_heap", globalSanitizersDiag); found && + s.Diag.Memtag_heap == nil && Bool(s.Memtag_heap) { + s.Diag.Memtag_heap = proptools.BoolPtr(true) + } + } + + // Enable Memtag for all components in the include paths (for Aarch64 only) + if ctx.Arch().ArchType == android.Arm64 { + if ctx.Config().MemtagHeapSyncEnabledForPath(ctx.ModuleDir()) { + if s.Memtag_heap == nil { + s.Memtag_heap = proptools.BoolPtr(true) + } + if s.Diag.Memtag_heap == nil { + s.Diag.Memtag_heap = proptools.BoolPtr(true) + } + } else if ctx.Config().MemtagHeapAsyncEnabledForPath(ctx.ModuleDir()) { + if s.Memtag_heap == nil { + s.Memtag_heap = proptools.BoolPtr(true) + } + } + } // TODO:(b/178369775) // For now sanitizing is only supported on devices @@ -96,7 +202,22 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { s.Hwaddress = nil } - if ctx.Os() == android.Android && Bool(s.Hwaddress) { + // HWASan ramdisk (which is built from recovery) goes over some bootloader limit. + // Keep libc instrumented so that ramdisk / vendor_ramdisk / recovery can run hwasan-instrumented code if necessary. + if (ctx.RustModule().InRamdisk() || ctx.RustModule().InVendorRamdisk() || ctx.RustModule().InRecovery()) && !strings.HasPrefix(ctx.ModuleDir(), "bionic/libc") { + s.Hwaddress = nil + } + + if Bool(s.Hwaddress) { + s.Address = nil + } + + // Memtag_heap is only implemented on AArch64. + if ctx.Arch().ArchType != android.Arm64 { + s.Memtag_heap = nil + } + + if ctx.Os() == android.Android && (Bool(s.Hwaddress) || Bool(s.Address) || Bool(s.Memtag_heap)) { sanitize.Properties.SanitizerEnabled = true } } @@ -136,6 +257,26 @@ func rustSanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { return } + if Bool(mod.sanitize.Properties.Sanitize.Memtag_heap) && mod.Binary() { + noteDep := "note_memtag_heap_async" + if Bool(mod.sanitize.Properties.Sanitize.Diag.Memtag_heap) { + noteDep = "note_memtag_heap_sync" + } + // If we're using snapshots, redirect to snapshot whenever possible + // TODO(b/178470649): clean manual snapshot redirections + snapshot := mctx.Provider(cc.SnapshotInfoProvider).(cc.SnapshotInfo) + if lib, ok := snapshot.StaticLibs[noteDep]; ok { + noteDep = lib + } + depTag := cc.StaticDepTag(true) + variations := append(mctx.Target().Variations(), + blueprint.Variation{Mutator: "link", Variation: "static"}) + if mod.Device() { + variations = append(variations, mod.ImageVariation()) + } + mctx.AddFarVariationDependencies(variations, depTag, noteDep) + } + variations := mctx.Target().Variations() var depTag blueprint.DependencyTag var deps []string @@ -149,26 +290,23 @@ func rustSanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { } else if mod.IsSanitizerEnabled(cc.Hwasan) || (mod.IsSanitizerEnabled(cc.Fuzzer) && mctx.Arch().ArchType == android.Arm64) { // TODO(b/180495975): HWASan for static Rust binaries isn't supported yet. - if binary, ok := mod.compiler.(*binaryDecorator); ok { - if Bool(binary.Properties.Static_executable) { + if binary, ok := mod.compiler.(binaryInterface); ok { + if binary.staticallyLinked() { mctx.ModuleErrorf("HWASan is not supported for static Rust executables yet.") } } - if mod.StaticallyLinked() { - variations = append(variations, - blueprint.Variation{Mutator: "link", Variation: "static"}) - depTag = cc.StaticDepTag(false) - deps = []string{config.LibclangRuntimeLibrary(mod.toolchain(mctx), "hwasan_static")} - } else { - variations = append(variations, - blueprint.Variation{Mutator: "link", Variation: "shared"}) - depTag = cc.SharedDepTag() - deps = []string{config.LibclangRuntimeLibrary(mod.toolchain(mctx), "hwasan")} - } + // Always link against the shared library -- static binaries will pull in the static + // library during final link if necessary + variations = append(variations, + blueprint.Variation{Mutator: "link", Variation: "shared"}) + depTag = cc.SharedDepTag() + deps = []string{config.LibclangRuntimeLibrary(mod.toolchain(mctx), "hwasan")} } - mctx.AddFarVariationDependencies(variations, depTag, deps...) + if len(deps) > 0 { + mctx.AddFarVariationDependencies(variations, depTag, deps...) + } } } @@ -184,6 +322,9 @@ func (sanitize *sanitize) SetSanitizer(t cc.SanitizerType, b bool) { case cc.Hwasan: sanitize.Properties.Sanitize.Hwaddress = boolPtr(b) sanitizerSet = true + case cc.Memtag_heap: + sanitize.Properties.Sanitize.Memtag_heap = boolPtr(b) + sanitizerSet = true default: panic(fmt.Errorf("setting unsupported sanitizerType %d", t)) } @@ -243,6 +384,8 @@ func (sanitize *sanitize) getSanitizerBoolPtr(t cc.SanitizerType) *bool { return sanitize.Properties.Sanitize.Address case cc.Hwasan: return sanitize.Properties.Sanitize.Hwaddress + case cc.Memtag_heap: + return sanitize.Properties.Sanitize.Memtag_heap default: return nil } @@ -268,6 +411,12 @@ func (mod *Module) SanitizerSupported(t cc.SanitizerType) bool { case cc.Asan: return true case cc.Hwasan: + // TODO(b/180495975): HWASan for static Rust binaries isn't supported yet. + if mod.StaticExecutable() { + return false + } + return true + case cc.Memtag_heap: return true default: return false diff --git a/rust/sanitize_test.go b/rust/sanitize_test.go new file mode 100644 index 000000000..d6a14b295 --- /dev/null +++ b/rust/sanitize_test.go @@ -0,0 +1,365 @@ +package rust + +import ( + "fmt" + "strings" + "testing" + + "android/soong/android" +) + +type MemtagNoteType int + +const ( + None MemtagNoteType = iota + 1 + Sync + Async +) + +func (t MemtagNoteType) str() string { + switch t { + case None: + return "none" + case Sync: + return "sync" + case Async: + return "async" + default: + panic("type_note_invalid") + } +} + +func checkHasMemtagNote(t *testing.T, m android.TestingModule, expected MemtagNoteType) { + t.Helper() + note_async := "note_memtag_heap_async" + note_sync := "note_memtag_heap_sync" + + found := None + implicits := m.Rule("rustc").Implicits + for _, lib := range implicits { + if strings.Contains(lib.Rel(), note_async) { + found = Async + break + } else if strings.Contains(lib.Rel(), note_sync) { + found = Sync + break + } + } + + if found != expected { + t.Errorf("Wrong Memtag note in target %q: found %q, expected %q", m.Module().(*Module).Name(), found.str(), expected.str()) + } +} + +var prepareForTestWithMemtagHeap = android.GroupFixturePreparers( + android.FixtureModifyMockFS(func(fs android.MockFS) { + templateBp := ` + rust_test { + name: "unset_test_%[1]s", + srcs: ["foo.rs"], + } + + rust_test { + name: "no_memtag_test_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: false }, + } + + rust_test { + name: "set_memtag_test_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: true }, + } + + rust_test { + name: "set_memtag_set_async_test_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: true, diag: { memtag_heap: false } }, + } + + rust_test { + name: "set_memtag_set_sync_test_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: true, diag: { memtag_heap: true } }, + } + + rust_test { + name: "unset_memtag_set_sync_test_%[1]s", + srcs: ["foo.rs"], + sanitize: { diag: { memtag_heap: true } }, + } + + rust_binary { + name: "unset_binary_%[1]s", + srcs: ["foo.rs"], + } + + rust_binary { + name: "no_memtag_binary_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: false }, + } + + rust_binary { + name: "set_memtag_binary_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: true }, + } + + rust_binary { + name: "set_memtag_set_async_binary_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: true, diag: { memtag_heap: false } }, + } + + rust_binary { + name: "set_memtag_set_sync_binary_%[1]s", + srcs: ["foo.rs"], + sanitize: { memtag_heap: true, diag: { memtag_heap: true } }, + } + + rust_binary { + name: "unset_memtag_set_sync_binary_%[1]s", + srcs: ["foo.rs"], + sanitize: { diag: { memtag_heap: true } }, + } + ` + subdirNoOverrideBp := fmt.Sprintf(templateBp, "no_override") + subdirOverrideDefaultDisableBp := fmt.Sprintf(templateBp, "override_default_disable") + subdirSyncBp := fmt.Sprintf(templateBp, "override_default_sync") + subdirAsyncBp := fmt.Sprintf(templateBp, "override_default_async") + + fs.Merge(android.MockFS{ + "subdir_no_override/Android.bp": []byte(subdirNoOverrideBp), + "subdir_override_default_disable/Android.bp": []byte(subdirOverrideDefaultDisableBp), + "subdir_sync/Android.bp": []byte(subdirSyncBp), + "subdir_async/Android.bp": []byte(subdirAsyncBp), + }) + }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.MemtagHeapExcludePaths = []string{"subdir_override_default_disable"} + // "subdir_override_default_disable" is covered by both include and override_default_disable paths. override_default_disable wins. + variables.MemtagHeapSyncIncludePaths = []string{"subdir_sync", "subdir_override_default_disable"} + variables.MemtagHeapAsyncIncludePaths = []string{"subdir_async", "subdir_override_default_disable"} + }), +) + +func TestSanitizeMemtagHeap(t *testing.T) { + variant := "android_arm64_armv8-a" + + result := android.GroupFixturePreparers( + prepareForRustTest, + prepareForTestWithMemtagHeap, + ).RunTest(t) + ctx := result.TestContext + + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_async", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_sync", variant), None) + + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_async", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_sync", variant), None) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_sync", variant), Async) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_sync", variant), Async) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_sync", variant), Sync) +} + +func TestSanitizeMemtagHeapWithSanitizeDevice(t *testing.T) { + variant := "android_arm64_armv8-a" + + result := android.GroupFixturePreparers( + prepareForRustTest, + prepareForTestWithMemtagHeap, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.SanitizeDevice = []string{"memtag_heap"} + }), + ).RunTest(t) + ctx := result.TestContext + + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_async", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_sync", variant), None) + + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_async", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_sync", variant), None) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_sync", variant), Async) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_sync", variant), Async) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_sync", variant), Sync) + + // should sanitize: { diag: { memtag: true } } result in Sync instead of None here? + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_async", variant), Sync) + // should sanitize: { diag: { memtag: true } } result in Sync instead of None here? + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_sync", variant), Sync) +} + +func TestSanitizeMemtagHeapWithSanitizeDeviceDiag(t *testing.T) { + variant := "android_arm64_armv8-a" + + result := android.GroupFixturePreparers( + prepareForRustTest, + prepareForTestWithMemtagHeap, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.SanitizeDevice = []string{"memtag_heap"} + variables.SanitizeDeviceDiag = []string{"memtag_heap"} + }), + ).RunTest(t) + ctx := result.TestContext + + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_async", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_binary_override_default_sync", variant), None) + + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_no_override", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_async", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("no_memtag_test_override_default_sync", variant), None) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_binary_override_default_sync", variant), Async) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_no_override", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_async", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_disable", variant), Async) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_async_test_override_default_sync", variant), Async) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("set_memtag_set_sync_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_async", variant), Sync) + // should sanitize: { diag: { memtag: true } } result in Sync instead of None here? + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_memtag_set_sync_test_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_disable", variant), None) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_binary_override_default_sync", variant), Sync) + + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_no_override", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_async", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_disable", variant), Sync) + checkHasMemtagNote(t, ctx.ModuleForTests("unset_test_override_default_sync", variant), Sync) +} diff --git a/rust/snapshot_prebuilt.go b/rust/snapshot_prebuilt.go index 79eaab382..dfbc1d1e7 100644 --- a/rust/snapshot_prebuilt.go +++ b/rust/snapshot_prebuilt.go @@ -44,6 +44,8 @@ func init() { func registerRustSnapshotModules(ctx android.RegistrationContext) { cc.VendorSnapshotImageSingleton.RegisterAdditionalModule(ctx, "vendor_snapshot_rlib", VendorSnapshotRlibFactory) + cc.RecoverySnapshotImageSingleton.RegisterAdditionalModule(ctx, + "recovery_snapshot_rlib", RecoverySnapshotRlibFactory) } func snapshotLibraryFactory(image cc.SnapshotImage, moduleSuffix string) (*Module, *snapshotLibraryDecorator) { @@ -85,8 +87,9 @@ func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, if !library.MatchesWithDevice(ctx.DeviceConfig()) { return nil } - - return android.PathForModuleSrc(ctx, *library.properties.Src) + outputFile := android.PathForModuleSrc(ctx, *library.properties.Src) + library.unstrippedOutputFile = outputFile + return outputFile } func (library *snapshotLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath { @@ -104,6 +107,13 @@ func VendorSnapshotRlibFactory() android.Module { return module.Init() } +func RecoverySnapshotRlibFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(cc.RecoverySnapshotImageSingleton, cc.SnapshotRlibSuffix) + prebuilt.libraryDecorator.BuildOnlyRlib() + prebuilt.libraryDecorator.setNoStdlibs() + return module.Init() +} + func (library *snapshotLibraryDecorator) MatchesWithDevice(config android.DeviceConfig) bool { arches := config.Arches() if len(arches) == 0 || arches[0].ArchType.String() != library.Arch() { diff --git a/rust/test.go b/rust/test.go index 56da509b5..bb877a9a5 100644 --- a/rust/test.go +++ b/rust/test.go @@ -196,3 +196,7 @@ func (test *testDecorator) compilerDeps(ctx DepsContext, deps Deps) Deps { return deps } + +func (test *testDecorator) testBinary() bool { + return true +} diff --git a/rust/testing.go b/rust/testing.go index 94cdd9dcd..9f8ed54d9 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -53,74 +53,14 @@ var PrepareForTestWithRustIncludeVndk = android.GroupFixturePreparers( func GatherRequiredDepsForTest() string { bp := ` rust_prebuilt_library { - name: "libstd_x86_64-unknown-linux-gnu", - crate_name: "std", - rlib: { - srcs: ["libstd.rlib"], - }, - dylib: { - srcs: ["libstd.so"], - }, - host_supported: true, - sysroot: true, - } - rust_prebuilt_library { - name: "libtest_x86_64-unknown-linux-gnu", - crate_name: "test", - rlib: { - srcs: ["libtest.rlib"], - }, - dylib: { - srcs: ["libtest.so"], - }, - host_supported: true, - sysroot: true, - } - rust_prebuilt_library { - name: "libstd_i686-unknown-linux-gnu", - crate_name: "std", - rlib: { - srcs: ["libstd.rlib"], - }, - dylib: { - srcs: ["libstd.so"], - }, - host_supported: true, - sysroot: true, - } - rust_prebuilt_library { - name: "libtest_i686-unknown-linux-gnu", - crate_name: "test", - rlib: { - srcs: ["libtest.rlib"], - }, - dylib: { - srcs: ["libtest.so"], - }, - host_supported: true, - sysroot: true, - } - rust_prebuilt_library { - name: "libstd_x86_64-apple-darwin", - crate_name: "std", - rlib: { - srcs: ["libstd.rlib"], - }, - dylib: { - srcs: ["libstd.so"], - }, - host_supported: true, - sysroot: true, - } - rust_prebuilt_library { - name: "libtest_x86_64-apple-darwin", - crate_name: "test", - rlib: { - srcs: ["libtest.rlib"], - }, - dylib: { - srcs: ["libtest.so"], - }, + name: "libstd", + crate_name: "std", + rlib: { + srcs: ["libstd.rlib"], + }, + dylib: { + srcs: ["libstd.so"], + }, host_supported: true, sysroot: true, } @@ -151,7 +91,12 @@ func GatherRequiredDepsForTest() string { no_libcrt: true, nocrt: true, system_shared_libs: [], - export_include_dirs: ["libprotobuf-cpp-full-includes"], + } + cc_library { + name: "libclang_rt.hwasan_static-aarch64-android", + no_libcrt: true, + nocrt: true, + system_shared_libs: [], } rust_library { name: "libstd", @@ -246,5 +191,8 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.BottomUp("rust_begin", BeginMutator).Parallel() }) ctx.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton) + ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() + }) registerRustSnapshotModules(ctx) } diff --git a/rust/vendor_snapshot_test.go b/rust/vendor_snapshot_test.go index 60ddb653f..bfa6f361a 100644 --- a/rust/vendor_snapshot_test.go +++ b/rust/vendor_snapshot_test.go @@ -562,6 +562,7 @@ func TestVendorSnapshotUse(t *testing.T) { "libvendor", "libvndk", "libclang_rt.builtins-aarch64-android", + "note_memtag_heap_sync", ], shared_libs: [ "libvendor_available", @@ -853,6 +854,20 @@ func TestVendorSnapshotUse(t *testing.T) { }, } + // Test sanitizers use the snapshot libraries + rust_binary { + name: "memtag_binary", + srcs: ["vendor/bin.rs"], + vendor: true, + compile_multilib: "64", + sanitize: { + memtag_heap: true, + diag: { + memtag_heap: true, + } + }, + } + // old snapshot module which has to be ignored vendor_snapshot_binary { name: "bin", @@ -880,11 +895,25 @@ func TestVendorSnapshotUse(t *testing.T) { }, }, } + + vendor_snapshot_static { + name: "note_memtag_heap_sync", + vendor: true, + target_arch: "arm64", + version: "30", + arch: { + arm64: { + src: "note_memtag_heap_sync.a", + }, + }, + } + ` mockFS := android.MockFS{ "framework/Android.bp": []byte(frameworkBp), "framework/bin.rs": nil, + "note_memtag_heap_sync.a": nil, "vendor/Android.bp": []byte(vendorProprietaryBp), "vendor/bin": nil, "vendor/bin32": nil, @@ -993,4 +1022,9 @@ func TestVendorSnapshotUse(t *testing.T) { if android.InList(binaryVariant, binVariants) { t.Errorf("bin must not have variant %#v, but it does", sharedVariant) } + + memtagStaticLibs := ctx.ModuleForTests("memtag_binary", "android_vendor.30_arm64_armv8-a").Module().(*Module).Properties.AndroidMkStaticLibs + if g, w := memtagStaticLibs, []string{"libclang_rt.builtins-aarch64-android.vendor", "note_memtag_heap_sync.vendor"}; !reflect.DeepEqual(g, w) { + t.Errorf("wanted memtag_binary AndroidMkStaticLibs %q, got %q", w, g) + } } diff --git a/scripts/Android.bp b/scripts/Android.bp index 635be10b3..730d7567e 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -1,5 +1,6 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], + default_visibility: ["//build/soong:__subpackages__"], } python_binary_host { @@ -8,14 +9,6 @@ python_binary_host { srcs: [ "check_boot_jars/check_boot_jars.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, } python_binary_host { @@ -24,14 +17,6 @@ python_binary_host { srcs: [ "manifest_fixer.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -45,11 +30,8 @@ python_test_host { "manifest_fixer.py", ], version: { - py2: { - enabled: true, - }, py3: { - enabled: false, + embedded_launcher: true, }, }, libs: [ @@ -67,12 +49,14 @@ python_library_host { ], version: { py2: { + // TODO(b/203436762) Remove when system/apex/apexer/apexer.py is converted enabled: true, }, py3: { - enabled: false, + enabled: true, }, }, + visibility: ["//system/apex/apexer:__pkg__"], } python_binary_host { @@ -81,14 +65,6 @@ python_binary_host { srcs: [ "manifest_check.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -101,14 +77,6 @@ python_test_host { "manifest_check_test.py", "manifest_check.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -123,14 +91,6 @@ python_binary_host { srcs: [ "jsonmodify.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, } python_binary_host { @@ -139,14 +99,6 @@ python_binary_host { srcs: [ "test_config_fixer.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -159,14 +111,6 @@ python_test_host { "test_config_fixer_test.py", "test_config_fixer.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -179,14 +123,6 @@ python_binary_host { srcs: [ "construct_context.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -199,14 +135,6 @@ python_test_host { "construct_context_test.py", "construct_context.py", ], - version: { - py2: { - enabled: true, - }, - py3: { - enabled: false, - }, - }, libs: [ "manifest_utils", ], @@ -253,11 +181,7 @@ python_binary_host { "conv_linker_config.py", ], version: { - py2: { - enabled: false, - }, py3: { - enabled: true, embedded_launcher: true, }, }, @@ -272,12 +196,4 @@ python_binary_host { srcs: [ "get_clang_version.py", ], - version: { - py2: { - enabled: false, - }, - py3: { - enabled: true, - }, - }, } diff --git a/scripts/diff_build_graphs.sh b/scripts/diff_build_graphs.sh index 81010f323..8d011240c 100755 --- a/scripts/diff_build_graphs.sh +++ b/scripts/diff_build_graphs.sh @@ -98,7 +98,7 @@ function run_build() { # or in case it is affected by some of the changes we're testing make blueprint_tools # find multiproduct_kati and have it build the ninja files for each product - builder="$(echo $OUT_DIR/soong/host/*/bin/multiproduct_kati)" + builder="$(echo $OUT_DIR/host/*/bin/multiproduct_kati)" BUILD_NUMBER=sample "$builder" $PRODUCTS_ARG --keep --out "$OUT_DIR_TEMP" || true echo } diff --git a/scripts/manifest.py b/scripts/manifest.py index 04f7405df..81f9c61a8 100755 --- a/scripts/manifest.py +++ b/scripts/manifest.py @@ -123,4 +123,4 @@ def get_indent(element, default_level): def write_xml(f, doc): f.write('<?xml version="1.0" encoding="utf-8"?>\n') for node in doc.childNodes: - f.write(node.toxml(encoding='utf-8') + '\n') + f.write(node.toxml() + '\n') diff --git a/scripts/manifest_check.py b/scripts/manifest_check.py index 8bed52a9f..c8d4f76c8 100755 --- a/scripts/manifest_check.py +++ b/scripts/manifest_check.py @@ -335,7 +335,7 @@ def main(): if is_apk: aapt = args.aapt if args.aapt is not None else 'aapt' manifest = subprocess.check_output( - [aapt, 'dump', 'badging', args.input]) + [aapt, 'dump', 'badging', args.input]).decode('utf-8') else: manifest = minidom.parse(args.input) @@ -381,7 +381,7 @@ def main(): if is_apk: raise RuntimeError('cannot save APK manifest as XML') - with open(args.output, 'wb') as f: + with open(args.output, 'w') as f: write_xml(f, manifest) # pylint: disable=broad-except diff --git a/scripts/manifest_fixer.py b/scripts/manifest_fixer.py index 55d0fd158..d80a617a1 100755 --- a/scripts/manifest_fixer.py +++ b/scripts/manifest_fixer.py @@ -352,7 +352,7 @@ def main(): if args.extract_native_libs is not None: add_extract_native_libs(doc, args.extract_native_libs) - with open(args.output, 'wb') as f: + with open(args.output, 'w') as f: write_xml(f, doc) # pylint: disable=broad-except diff --git a/scripts/manifest_fixer_test.py b/scripts/manifest_fixer_test.py index 3a0a25d1c..f6fcaafe5 100755 --- a/scripts/manifest_fixer_test.py +++ b/scripts/manifest_fixer_test.py @@ -16,16 +16,16 @@ # """Unit tests for manifest_fixer.py.""" -import StringIO +import io import sys import unittest from xml.dom import minidom +import xml.etree.ElementTree as ET import manifest_fixer sys.dont_write_bytecode = True - class CompareVersionGtTest(unittest.TestCase): """Unit tests for compare_version_gt function.""" @@ -59,7 +59,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): doc = minidom.parseString(input_manifest) manifest_fixer.raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -80,13 +80,16 @@ class RaiseMinSdkVersionTest(unittest.TestCase): attrs += ' ' + extra return ' <uses-sdk%s/>\n' % (attrs) + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def test_no_uses_sdk(self): """Tests inserting a uses-sdk element into a manifest.""" manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.uses_sdk(min='28', target='28') output = self.raise_min_sdk_version_test(manifest_input, '28', '28', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_no_min(self): """Tests inserting a minSdkVersion attribute into a uses-sdk element.""" @@ -95,7 +98,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): expected = self.manifest_tmpl % self.uses_sdk(min='28', target='28', extra='extra="foo"') output = self.raise_min_sdk_version_test(manifest_input, '28', '28', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_raise_min(self): """Tests inserting a minSdkVersion attribute into a uses-sdk element.""" @@ -103,7 +106,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='28') output = self.raise_min_sdk_version_test(manifest_input, '28', '28', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_raise(self): """Tests raising a minSdkVersion attribute.""" @@ -111,7 +114,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='28') output = self.raise_min_sdk_version_test(manifest_input, '28', '28', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_no_raise_min(self): """Tests a minSdkVersion that doesn't need raising.""" @@ -119,7 +122,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='28') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='27') output = self.raise_min_sdk_version_test(manifest_input, '27', '27', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_raise_codename(self): """Tests raising a minSdkVersion attribute to a codename.""" @@ -127,7 +130,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='28') expected = self.manifest_tmpl % self.uses_sdk(min='P', target='P') output = self.raise_min_sdk_version_test(manifest_input, 'P', 'P', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_no_raise_codename(self): """Tests a minSdkVersion codename that doesn't need raising.""" @@ -135,7 +138,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='P') expected = self.manifest_tmpl % self.uses_sdk(min='P', target='28') output = self.raise_min_sdk_version_test(manifest_input, '28', '28', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_target(self): """Tests an existing targetSdkVersion is preserved.""" @@ -143,7 +146,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='26', target='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='27') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_no_target(self): """Tests inserting targetSdkVersion when minSdkVersion exists.""" @@ -151,7 +154,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='29') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_target_no_min(self): """"Tests inserting targetSdkVersion when minSdkVersion exists.""" @@ -159,7 +162,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(target='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='27') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_no_target_no_min(self): """Tests inserting targetSdkVersion when minSdkVersion does not exist.""" @@ -167,7 +170,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.uses_sdk(min='28', target='29') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_library_no_target(self): """Tests inserting targetSdkVersion when minSdkVersion exists.""" @@ -175,7 +178,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(min='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='16') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', True) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_library_target_no_min(self): """Tests inserting targetSdkVersion when minSdkVersion exists.""" @@ -183,7 +186,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_sdk(target='27') expected = self.manifest_tmpl % self.uses_sdk(min='28', target='27') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', True) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_library_no_target_no_min(self): """Tests inserting targetSdkVersion when minSdkVersion does not exist.""" @@ -191,7 +194,7 @@ class RaiseMinSdkVersionTest(unittest.TestCase): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.uses_sdk(min='28', target='16') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', True) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_extra(self): """Tests that extra attributes and elements are maintained.""" @@ -204,12 +207,12 @@ class RaiseMinSdkVersionTest(unittest.TestCase): # pylint: disable=line-too-long expected = self.manifest_tmpl % ( ' <!-- comment -->\n' - ' <uses-sdk android:minSdkVersion="28" android:targetSdkVersion="29" extra="foo"/>\n' + ' <uses-sdk android:minSdkVersion="28" extra="foo" android:targetSdkVersion="29"/>\n' ' <application/>\n') output = self.raise_min_sdk_version_test(manifest_input, '28', '29', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_indent(self): """Tests that an inserted element copies the existing indentation.""" @@ -223,17 +226,20 @@ class RaiseMinSdkVersionTest(unittest.TestCase): output = self.raise_min_sdk_version_test(manifest_input, '28', '29', False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) class AddLoggingParentTest(unittest.TestCase): """Unit tests for add_logging_parent function.""" + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def add_logging_parent_test(self, input_manifest, logging_parent=None): doc = minidom.parseString(input_manifest) if logging_parent: manifest_fixer.add_logging_parent(doc, logging_parent) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -257,23 +263,26 @@ class AddLoggingParentTest(unittest.TestCase): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.uses_logging_parent() output = self.add_logging_parent_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_logging_parent(self): """Tests manifest_fixer with no logging_parent.""" manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.uses_logging_parent('FOO') output = self.add_logging_parent_test(manifest_input, 'FOO') - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) class AddUsesLibrariesTest(unittest.TestCase): """Unit tests for add_uses_libraries function.""" + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def run_test(self, input_manifest, new_uses_libraries): doc = minidom.parseString(input_manifest) manifest_fixer.add_uses_libraries(doc, new_uses_libraries, True) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -301,7 +310,7 @@ class AddUsesLibrariesTest(unittest.TestCase): ('bar', 'false')]) expected = manifest_input output = self.run_test(manifest_input, []) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_not_overwrite(self): """new_uses_libraries must not overwrite existing tags.""" @@ -310,7 +319,7 @@ class AddUsesLibrariesTest(unittest.TestCase): ('bar', 'false')]) expected = manifest_input output = self.run_test(manifest_input, ['foo', 'bar']) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_add(self): """New names are added with 'required:true'.""" @@ -323,7 +332,7 @@ class AddUsesLibrariesTest(unittest.TestCase): ('baz', 'true'), ('qux', 'true')]) output = self.run_test(manifest_input, ['bar', 'baz', 'qux']) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_no_application(self): """When there is no <application> tag, the tag is added.""" @@ -336,7 +345,7 @@ class AddUsesLibrariesTest(unittest.TestCase): ('foo', 'true'), ('bar', 'true')]) output = self.run_test(manifest_input, ['foo', 'bar']) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_empty_application(self): """Even when here is an empty <application/> tag, the libs are added.""" @@ -350,16 +359,19 @@ class AddUsesLibrariesTest(unittest.TestCase): ('foo', 'true'), ('bar', 'true')]) output = self.run_test(manifest_input, ['foo', 'bar']) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) class AddUsesNonSdkApiTest(unittest.TestCase): """Unit tests for add_uses_libraries function.""" + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def run_test(self, input_manifest): doc = minidom.parseString(input_manifest) manifest_fixer.add_uses_non_sdk_api(doc) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -377,23 +389,26 @@ class AddUsesNonSdkApiTest(unittest.TestCase): manifest_input = self.manifest_tmpl % self.uses_non_sdk_api(False) expected = self.manifest_tmpl % self.uses_non_sdk_api(True) output = self.run_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_already_set(self): """new_uses_libraries must not overwrite existing tags.""" manifest_input = self.manifest_tmpl % self.uses_non_sdk_api(True) expected = manifest_input output = self.run_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) class UseEmbeddedDexTest(unittest.TestCase): """Unit tests for add_use_embedded_dex function.""" + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def run_test(self, input_manifest): doc = minidom.parseString(input_manifest) manifest_fixer.add_use_embedded_dex(doc) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -410,13 +425,13 @@ class UseEmbeddedDexTest(unittest.TestCase): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.use_embedded_dex('true') output = self.run_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_manifest_with_use_embedded_dex(self): manifest_input = self.manifest_tmpl % self.use_embedded_dex('true') expected = manifest_input output = self.run_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_manifest_with_not_use_embedded_dex(self): manifest_input = self.manifest_tmpl % self.use_embedded_dex('false') @@ -426,10 +441,13 @@ class UseEmbeddedDexTest(unittest.TestCase): class AddExtractNativeLibsTest(unittest.TestCase): """Unit tests for add_extract_native_libs function.""" + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def run_test(self, input_manifest, value): doc = minidom.parseString(input_manifest) manifest_fixer.add_extract_native_libs(doc, value) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -446,19 +464,19 @@ class AddExtractNativeLibsTest(unittest.TestCase): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.extract_native_libs('true') output = self.run_test(manifest_input, True) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_set_false(self): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % self.extract_native_libs('false') output = self.run_test(manifest_input, False) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_match(self): manifest_input = self.manifest_tmpl % self.extract_native_libs('true') expected = manifest_input output = self.run_test(manifest_input, True) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_conflict(self): manifest_input = self.manifest_tmpl % self.extract_native_libs('true') @@ -468,10 +486,13 @@ class AddExtractNativeLibsTest(unittest.TestCase): class AddNoCodeApplicationTest(unittest.TestCase): """Unit tests for set_has_code_to_false function.""" + def assert_xml_equal(self, output, expected): + self.assertEqual(ET.canonicalize(output), ET.canonicalize(expected)) + def run_test(self, input_manifest): doc = minidom.parseString(input_manifest) manifest_fixer.set_has_code_to_false(doc) - output = StringIO.StringIO() + output = io.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -485,26 +506,26 @@ class AddNoCodeApplicationTest(unittest.TestCase): manifest_input = self.manifest_tmpl % '' expected = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' output = self.run_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_has_application_no_has_code(self): manifest_input = self.manifest_tmpl % ' <application/>\n' expected = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' output = self.run_test(manifest_input) - self.assertEqual(output, expected) + self.assert_xml_equal(output, expected) def test_has_application_has_code_false(self): """ Do nothing if there's already an application elemeent. """ manifest_input = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' output = self.run_test(manifest_input) - self.assertEqual(output, manifest_input) + self.assert_xml_equal(output, manifest_input) def test_has_application_has_code_true(self): """ Do nothing if there's already an application elemeent even if its hasCode attribute is true. """ manifest_input = self.manifest_tmpl % ' <application android:hasCode="true"/>\n' output = self.run_test(manifest_input) - self.assertEqual(output, manifest_input) + self.assert_xml_equal(output, manifest_input) if __name__ == '__main__': diff --git a/scripts/rbc-run b/scripts/rbc-run index a0907cfb4..235da7517 100755 --- a/scripts/rbc-run +++ b/scripts/rbc-run @@ -4,12 +4,13 @@ # that will be passed to rbcrun. [[ $# -gt 0 && -f "$1" ]] || { echo "Usage: ${0##*/} product.mk [Additional rbcrun arguments]" >&2; exit 1; } set -eu -declare -r output_root=${OUT_DIR:-out} -declare -r runner="$output_root/soong/.bootstrap/bin/rbcrun" -declare -r converter="$output_root/soong/.bootstrap/bin/mk2rbc" -declare -r launcher=$output_root/launchers/run.rbc -declare -r makefile=$1 + +declare -r output_root="${OUT_DIR:-out}" +declare -r runner="${output_root}/soong/rbcrun" +declare -r converter="${output_root}/soong/mk2rbc" +declare -r launcher="${output_root}/rbc/launcher.rbc" +declare -r makefile="$1" shift -$converter -mode=write -r --outdir $output_root --launcher=$launcher $makefile -$runner RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ $launcher +"${converter}" -mode=write -r --outdir "${output_root}/rbc" --launcher="${launcher}" "${makefile}" +"${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}" diff --git a/scripts/test_config_fixer.py b/scripts/test_config_fixer.py index 32d5b175e..c150e8cfd 100644 --- a/scripts/test_config_fixer.py +++ b/scripts/test_config_fixer.py @@ -86,7 +86,7 @@ def main(): if args.test_file_name: overwrite_test_file_name(doc, args.test_file_name) - with open(args.output, 'wb') as f: + with open(args.output, 'w') as f: write_xml(f, doc) # pylint: disable=broad-except diff --git a/scripts/test_config_fixer_test.py b/scripts/test_config_fixer_test.py index 1272c6b33..d00a59315 100644 --- a/scripts/test_config_fixer_test.py +++ b/scripts/test_config_fixer_test.py @@ -16,7 +16,7 @@ # """Unit tests for test_config_fixer.py.""" -import StringIO +import io import sys import unittest from xml.dom import minidom @@ -59,7 +59,7 @@ class OverwritePackageNameTest(unittest.TestCase): manifest = minidom.parseString(self.manifest) test_config_fixer.overwrite_package_name(doc, manifest, "com.soong.foo") - output = StringIO.StringIO() + output = io.StringIO() test_config_fixer.write_xml(output, doc) # Only the matching package name in a test node should be updated. @@ -86,7 +86,7 @@ class OverwriteTestFileNameTest(unittest.TestCase): doc = minidom.parseString(self.test_config % ("foo.apk")) test_config_fixer.overwrite_test_file_name(doc, "bar.apk") - output = StringIO.StringIO() + output = io.StringIO() test_config_fixer.write_xml(output, doc) # Only the matching package name in a test node should be updated. diff --git a/sh/sh_binary.go b/sh/sh_binary.go index b22a5b750..d5033ef84 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -168,6 +168,10 @@ type ShBinary struct { var _ android.HostToolProvider = (*ShBinary)(nil) +func (s *ShBinary) InstallBypassMake() bool { + return true +} + type ShTest struct { ShBinary @@ -271,6 +275,9 @@ func (s *ShBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { s.generateAndroidBuildActions(ctx) installDir := android.PathForModuleInstall(ctx, "bin", proptools.String(s.properties.Sub_dir)) s.installedFile = ctx.InstallExecutable(installDir, s.outputFilePath.Base(), s.outputFilePath) + for _, symlink := range s.Symlinks() { + ctx.InstallSymlink(installDir, symlink, s.installedFile) + } } func (s *ShBinary) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/snapshot/recovery_snapshot.go b/snapshot/recovery_snapshot.go index 9b3919c34..f1e31ca03 100644 --- a/snapshot/recovery_snapshot.go +++ b/snapshot/recovery_snapshot.go @@ -71,6 +71,10 @@ func (RecoverySnapshotImage) Init(ctx android.RegistrationContext) { ctx.RegisterSingletonType("recovery-snapshot", RecoverySnapshotSingleton) } +func (RecoverySnapshotImage) RegisterAdditionalModule(ctx android.RegistrationContext, name string, factory android.ModuleFactory) { + ctx.RegisterModuleType(name, factory) +} + func (RecoverySnapshotImage) shouldGenerateSnapshot(ctx android.SingletonContext) bool { // RECOVERY_SNAPSHOT_VERSION must be set to 'current' in order to generate a // snapshot. diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index a22adc5aa..e92a561f1 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -17,10 +17,10 @@ function test_smoke { function test_null_build() { setup run_soong - local bootstrap_mtime1=$(stat -c "%y" out/soong/.bootstrap/build.ninja) + local bootstrap_mtime1=$(stat -c "%y" out/soong/bootstrap.ninja) local output_mtime1=$(stat -c "%y" out/soong/build.ninja) run_soong - local bootstrap_mtime2=$(stat -c "%y" out/soong/.bootstrap/build.ninja) + local bootstrap_mtime2=$(stat -c "%y" out/soong/bootstrap.ninja) local output_mtime2=$(stat -c "%y" out/soong/build.ninja) if [[ "$bootstrap_mtime1" == "$bootstrap_mtime2" ]]; then @@ -36,12 +36,12 @@ function test_null_build() { function test_soong_build_rebuilt_if_blueprint_changes() { setup run_soong - local mtime1=$(stat -c "%y" out/soong/.bootstrap/build.ninja) + local mtime1=$(stat -c "%y" out/soong/bootstrap.ninja) sed -i 's/pluginGenSrcCmd/pluginGenSrcCmd2/g' build/blueprint/bootstrap/bootstrap.go run_soong - local mtime2=$(stat -c "%y" out/soong/.bootstrap/build.ninja) + local mtime2=$(stat -c "%y" out/soong/bootstrap.ninja) if [[ "$mtime1" == "$mtime2" ]]; then fail "Bootstrap Ninja file did not change" @@ -541,7 +541,7 @@ EOF function test_bp2build_smoke { setup run_soong bp2build - [[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created" + [[ -e out/soong/bp2build_workspace_marker ]] || fail "bp2build marker file not created" [[ -e out/soong/workspace ]] || fail "Bazel workspace not created" } @@ -551,7 +551,7 @@ function test_bp2build_generates_marker_file { run_soong bp2build - if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then + if [[ ! -f "./out/soong/bp2build_workspace_marker" ]]; then fail "Marker file was not generated" fi } @@ -592,10 +592,10 @@ function test_bp2build_null_build { setup run_soong bp2build - local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) run_soong bp2build - local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$mtime1" != "$mtime2" ]]; then fail "Output Ninja file changed on null build" @@ -626,7 +626,7 @@ EOF function test_multiple_soong_build_modes() { setup run_soong json-module-graph bp2build nothing - if [[ ! -f "out/soong/.bootstrap/bp2build_workspace_marker" ]]; then + if [[ ! -f "out/soong/bp2build_workspace_marker" ]]; then fail "bp2build marker file was not generated" fi @@ -780,11 +780,11 @@ function test_bp2build_back_and_forth_null_build { fail "Output Ninja file changed when switching to bp2build" fi - local marker_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local marker_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) run_soong local output_mtime3=$(stat -c "%y" out/soong/build.ninja) - local marker_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local marker_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime3" ]]; then fail "Output Ninja file changed when switching to regular build from bp2build" fi @@ -794,7 +794,7 @@ function test_bp2build_back_and_forth_null_build { run_soong bp2build local output_mtime4=$(stat -c "%y" out/soong/build.ninja) - local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local marker_mtime3=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime4" ]]; then fail "Output Ninja file changed when switching back to bp2build" fi diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh index 379eb6548..4f37c2bff 100755 --- a/tests/bp2build_bazel_test.sh +++ b/tests/bp2build_bazel_test.sh @@ -11,10 +11,10 @@ readonly GENERATED_BUILD_FILE_NAME="BUILD.bazel" function test_bp2build_null_build() { setup run_soong bp2build - local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local output_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) run_soong bp2build - local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local output_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime2" ]]; then fail "Output bp2build marker file changed on null build" @@ -36,10 +36,10 @@ EOF touch foo/bar/a.txt foo/bar/b.txt run_soong bp2build - local output_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local output_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) run_soong bp2build - local output_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker) + local output_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime2" ]]; then fail "Output bp2build marker file changed on null build" @@ -105,7 +105,7 @@ EOF # NOTE: We don't actually use the extra BUILD file for anything here run_bazel build --package_path=out/soong/workspace //foo/... - local the_answer_file="bazel-out/k8-fastbuild/bin/foo/convertible_soong_module/the_answer.txt" + local the_answer_file="bazel-out/android_target-fastbuild/bin/foo/convertible_soong_module/the_answer.txt" if [[ ! -f "${the_answer_file}" ]]; then fail "Expected '${the_answer_file}' to be generated, but was missing" fi diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index 6304a118c..a376e0698 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -7,3 +7,4 @@ TOP="$(readlink -f "$(dirname "$0")"/../../..)" "$TOP/build/soong/tests/mixed_mode_test.sh" "$TOP/build/soong/tests/bp2build_bazel_test.sh" "$TOP/build/soong/tests/soong_test.sh" +"$TOP/build/bazel/ci/rbc_product_config.sh" aosp_arm64-userdebug diff --git a/ui/build/config.go b/ui/build/config.go index 07ffb44f5..c30663349 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -784,12 +784,17 @@ func (c *configImpl) PrebuiltOS() string { panic("Unknown GOOS") } } + func (c *configImpl) HostToolDir() string { - return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") + if c.SkipKatiNinja() { + return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") + } else { + return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin") + } } func (c *configImpl) NamedGlobFile(name string) string { - return shared.JoinPath(c.SoongOutDir(), ".bootstrap/build-globs."+name+".ninja") + return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja") } func (c *configImpl) UsedEnvFile(tag string) string { @@ -797,7 +802,7 @@ func (c *configImpl) UsedEnvFile(tag string) string { } func (c *configImpl) Bp2BuildMarkerFile() string { - return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker") + return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker") } func (c *configImpl) SoongDocsHtml() string { diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index afec8293c..3f10f75e3 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -263,9 +263,9 @@ func runMakeProductConfig(ctx Context, config Config) { }, exportEnvVars...), BannerVars...) // We need Roboleaf converter and runner in the mixed mode - runMicrofactory(ctx, config, ".bootstrap/bin/mk2rbc", "android/soong/mk2rbc/cmd", + runMicrofactory(ctx, config, "mk2rbc", "android/soong/mk2rbc/cmd", map[string]string{"android/soong": "build/soong"}) - runMicrofactory(ctx, config, ".bootstrap/bin/rbcrun", "rbcrun/cmd", + runMicrofactory(ctx, config, "rbcrun", "rbcrun/cmd", map[string]string{"go.starlark.net": "external/starlark-go", "rbcrun": "build/make/tools/rbcrun"}) makeVars, err := dumpMakeVars(ctx, config, config.Arguments(), allVars, true, "") diff --git a/ui/build/soong.go b/ui/build/soong.go index 1c7fbac7e..4ced722d6 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -49,7 +49,7 @@ const ( // version of bootstrap and needs cleaning before continuing the build. Increment this for // incompatible changes, for example when moving the location of the bpglob binary that is // executed during bootstrap before the primary builder has had a chance to update the path. - bootstrapEpoch = 0 + bootstrapEpoch = 1 ) func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error { @@ -292,7 +292,7 @@ func bootstrapBlueprint(ctx Context, config Config) { var blueprintArgs bootstrap.Args blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") - blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") + blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja") blueprintArgs.EmptyNinjaFile = false blueprintCtx := blueprint.NewContext() @@ -314,7 +314,7 @@ func bootstrapBlueprint(ctx Context, config Config) { } bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig) - bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") + bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja.d") err := deptools.WriteDepFile(bootstrapDepFile, blueprintArgs.OutFile, bootstrapDeps) if err != nil { ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err) @@ -342,11 +342,6 @@ func runSoong(ctx Context, config Config) { // unused variables were changed? envFile := filepath.Join(config.SoongOutDir(), availableEnvFile) - dir := filepath.Join(config.SoongOutDir(), ".bootstrap") - if err := os.MkdirAll(dir, 0755); err != nil { - ctx.Fatalf("Cannot mkdir " + dir) - } - buildMode := config.bazelBuildMode() integratedBp2Build := buildMode == mixedBuild @@ -457,7 +452,7 @@ func runSoong(ctx Context, config Config) { targets = append(targets, config.SoongNinjaFile()) } - ninja("bootstrap", ".bootstrap/build.ninja", targets...) + ninja("bootstrap", "bootstrap.ninja", targets...) var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics if shouldCollectBuildSoongMetrics(config) { @@ -477,8 +472,7 @@ func runSoong(ctx Context, config Config) { } } -func runMicrofactory(ctx Context, config Config, relExePath string, pkg string, mapping map[string]string) { - name := filepath.Base(relExePath) +func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) { ctx.BeginTrace(metrics.RunSoong, name) defer ctx.EndTrace() cfg := microfactory.Config{TrimPath: absPath(ctx, ".")} @@ -486,7 +480,7 @@ func runMicrofactory(ctx Context, config Config, relExePath string, pkg string, cfg.Map(pkgPrefix, pathPrefix) } - exePath := filepath.Join(config.SoongOutDir(), relExePath) + exePath := filepath.Join(config.SoongOutDir(), name) dir := filepath.Dir(exePath) if err := os.MkdirAll(dir, 0777); err != nil { ctx.Fatalf("cannot create %s: %s", dir, err) diff --git a/ui/build/test_build.go b/ui/build/test_build.go index f9a60b604..86c85681a 100644 --- a/ui/build/test_build.go +++ b/ui/build/test_build.go @@ -63,7 +63,6 @@ func testForDanglingRules(ctx Context, config Config) { cmd.StartOrFatal() outDir := config.OutDir() - bootstrapDir := filepath.Join(outDir, "soong", ".bootstrap") modulePathsDir := filepath.Join(outDir, ".module_paths") variablesFilePath := filepath.Join(outDir, "soong", "soong.variables") @@ -77,6 +76,9 @@ func testForDanglingRules(ctx Context, config Config) { // out/build_date.txt is considered a "source file" buildDatetimeFilePath := filepath.Join(outDir, "build_date.txt") + // bpglob is built explicitly using Microfactory + bpglob := filepath.Join(config.SoongOutDir(), "bpglob") + danglingRules := make(map[string]bool) scanner := bufio.NewScanner(stdout) @@ -86,11 +88,11 @@ func testForDanglingRules(ctx Context, config Config) { // Leaf node is not in the out directory. continue } - if strings.HasPrefix(line, bootstrapDir) || - strings.HasPrefix(line, modulePathsDir) || + if strings.HasPrefix(line, modulePathsDir) || line == variablesFilePath || line == dexpreoptConfigFilePath || - line == buildDatetimeFilePath { + line == buildDatetimeFilePath || + line == bpglob { // Leaf node is in one of Soong's bootstrap directories, which do not have // full build rules in the primary build.ninja file. continue |