diff options
35 files changed, 1106 insertions, 402 deletions
@@ -26,6 +26,6 @@ yudiliu@google.com jingwen@google.com # EMEA -hansson@google.com +hansson@google.com #{LAST_RESORT_SUGGESTION} lberki@google.com -paulduffin@google.com +paulduffin@google.com #{LAST_RESORT_SUGGESTION} diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index a407b5e13..ccfad00b7 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -210,7 +210,6 @@ var ( "system/sepolicy/apex": Bp2BuildDefaultTrueRecursively, "system/timezone/apex": Bp2BuildDefaultTrueRecursively, "system/timezone/output_data": Bp2BuildDefaultTrueRecursively, - "system/unwinding/libbacktrace": Bp2BuildDefaultTrueRecursively, "system/unwinding/libunwindstack": Bp2BuildDefaultTrueRecursively, "tools/apksig": Bp2BuildDefaultTrue, "tools/platform-compat/java/android/compat": Bp2BuildDefaultTrueRecursively, @@ -227,12 +226,14 @@ var ( "build/bazel/ci/dist":/* recursive = */ false, "build/bazel/examples/android_app":/* recursive = */ true, "build/bazel/examples/java":/* recursive = */ true, + "build/bazel/examples/partitions":/* recursive = */ true, "build/bazel/bazel_skylib":/* recursive = */ true, "build/bazel/rules":/* recursive = */ true, "build/bazel/rules_cc":/* recursive = */ true, "build/bazel/scripts":/* recursive = */ true, "build/bazel/tests":/* recursive = */ true, "build/bazel/platforms":/* recursive = */ true, + "build/bazel/product_config":/* recursive = */ true, "build/bazel/product_variables":/* recursive = */ true, "build/bazel/vendor/google":/* recursive = */ true, "build/bazel_common_rules":/* recursive = */ true, diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 8834c1123..27255d194 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -528,7 +529,7 @@ config_node(name = "%s", configNodesSection := "" labelsByConfig := map[string][]string{} - for val, _ := range context.requests { + for val := range context.requests { labelString := fmt.Sprintf("\"@%s\"", val.label) configString := getConfigString(val) labelsByConfig[configString] = append(labelsByConfig[configString], labelString) @@ -566,7 +567,7 @@ func indent(original string) string { // request type. func (context *bazelContext) cqueryStarlarkFileContents() []byte { requestTypeToCqueryIdEntries := map[cqueryRequest][]string{} - for val, _ := range context.requests { + for val := range context.requests { cqueryId := getCqueryId(val) mapEntryString := fmt.Sprintf("%q : True", cqueryId) requestTypeToCqueryIdEntries[val.requestType] = @@ -870,61 +871,74 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { }) } - // Register bazel-owned build statements (obtained from the aquery invocation). + executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__") + bazelOutDir := path.Join(executionRoot, "bazel-out") for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { if len(buildStatement.Command) < 1 { panic(fmt.Sprintf("unhandled build statement: %v", buildStatement)) } rule := NewRuleBuilder(pctx, ctx) - cmd := rule.Command() - - // cd into Bazel's execution root, which is the action cwd. - cmd.Text(fmt.Sprintf("cd %s/execroot/__main__ &&", ctx.Config().BazelContext.OutputBase())) - - // Remove old outputs, as some actions might not rerun if the outputs are detected. - if len(buildStatement.OutputPaths) > 0 { - cmd.Text("rm -f") - for _, outputPath := range buildStatement.OutputPaths { - cmd.Text(outputPath) - } - cmd.Text("&&") - } + createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx) + // This is required to silence warnings pertaining to unexpected timestamps. Particularly, + // some Bazel builtins (such as files in the bazel_tools directory) have far-future + // timestamps. Without restat, Ninja would emit warnings that the input files of a + // build statement have later timestamps than the outputs. + rule.Restat() - for _, pair := range buildStatement.Env { - // Set per-action env variables, if any. - cmd.Flag(pair.Key + "=" + pair.Value) - } + desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) + rule.Build(fmt.Sprintf("bazel %d", index), desc) + } +} - // The actual Bazel action. - cmd.Text(buildStatement.Command) +// Register bazel-owned build statements (obtained from the aquery invocation). +func createCommand(cmd *RuleBuilderCommand, buildStatement bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx PathContext) { + // executionRoot is the action cwd. + cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot)) + // Remove old outputs, as some actions might not rerun if the outputs are detected. + if len(buildStatement.OutputPaths) > 0 { + cmd.Text("rm -f") for _, outputPath := range buildStatement.OutputPaths { - cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath)) - } - for _, inputPath := range buildStatement.InputPaths { - cmd.Implicit(PathForBazelOut(ctx, inputPath)) - } - for _, inputDepsetHash := range buildStatement.InputDepsetHashes { - otherDepsetName := bazelDepsetName(inputDepsetHash) - cmd.Implicit(PathForPhony(ctx, otherDepsetName)) + cmd.Text(outputPath) } + cmd.Text("&&") + } - if depfile := buildStatement.Depfile; depfile != nil { - cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile)) - } + for _, pair := range buildStatement.Env { + // Set per-action env variables, if any. + cmd.Flag(pair.Key + "=" + pair.Value) + } - for _, symlinkPath := range buildStatement.SymlinkPaths { - cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath)) - } + // The actual Bazel action. + cmd.Text(buildStatement.Command) - // This is required to silence warnings pertaining to unexpected timestamps. Particularly, - // some Bazel builtins (such as files in the bazel_tools directory) have far-future - // timestamps. Without restat, Ninja would emit warnings that the input files of a - // build statement have later timestamps than the outputs. - rule.Restat() + for _, outputPath := range buildStatement.OutputPaths { + cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath)) + } + for _, inputPath := range buildStatement.InputPaths { + cmd.Implicit(PathForBazelOut(ctx, inputPath)) + } + for _, inputDepsetHash := range buildStatement.InputDepsetHashes { + otherDepsetName := bazelDepsetName(inputDepsetHash) + cmd.Implicit(PathForPhony(ctx, otherDepsetName)) + } - desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) - rule.Build(fmt.Sprintf("bazel %d", index), desc) + if depfile := buildStatement.Depfile; depfile != nil { + // The paths in depfile are relative to `executionRoot`. + // Hence, they need to be corrected by replacing "bazel-out" + // with the full `bazelOutDir`. + // Otherwise, implicit outputs and implicit inputs under "bazel-out/" + // would be deemed missing. + // (Note: The regexp uses a capture group because the version of sed + // does not support a look-behind pattern.) + replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`, + bazelOutDir, *depfile) + cmd.Text(replacement) + cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile)) + } + + for _, symlinkPath := range buildStatement.SymlinkPaths { + cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath)) } } @@ -938,12 +952,12 @@ func getConfigString(key cqueryKey) string { // Use host platform, which is currently hardcoded to be x86_64. arch = "x86_64" } - os := key.configKey.osType.Name - if len(os) == 0 || os == "common_os" || os == "linux_glibc" { + osName := key.configKey.osType.Name + if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" { // Use host OS, which is currently hardcoded to be linux. - os = "linux" + osName = "linux" } - return arch + "|" + os + return arch + "|" + osName } func GetConfigKey(ctx BaseModuleContext) configKey { diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index dd9a7ed6d..935ce4e58 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -57,8 +57,13 @@ func TestInvokeBazelWritesBazelFiles(t *testing.T) { } func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ - bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: ` + type testCase struct { + input string + command string + } + + var testCases = []testCase{ + {` { "artifacts": [{ "id": 1, @@ -88,15 +93,60 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { "label": "two" }] }`, - }) - err := bazelContext.InvokeBazel(testConfig) - if err != nil { - t.Fatalf("Did not expect error invoking Bazel, but got %s", err) + "cd 'er' && rm -f one && touch foo", + }, {` +{ + "artifacts": [{ + "id": 1, + "pathFragmentId": 10 + }, { + "id": 2, + "pathFragmentId": 20 + }], + "actions": [{ + "targetId": 100, + "actionKey": "x", + "mnemonic": "x", + "arguments": ["bogus", "command"], + "outputIds": [1, 2], + "primaryOutputId": 1 + }], + "pathFragments": [{ + "id": 10, + "label": "one", + "parentId": 30 + }, { + "id": 20, + "label": "one.d", + "parentId": 30 + }, { + "id": 30, + "label": "parent" + }] +}`, + `cd 'er' && rm -f parent/one && bogus command && sed -i'' -E 's@(^|\s|")bazel-out/@\1bo/@g' 'parent/one.d'`, + }, } - got := bazelContext.BuildStatementsToRegister() - if want := 1; len(got) != want { - t.Errorf("Expected %d registered build statements, got %#v", want, got) + for _, testCase := range testCases { + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ + bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: testCase.input}) + + err := bazelContext.InvokeBazel(testConfig) + if err != nil { + t.Fatalf("Did not expect error invoking Bazel, but got %s", err) + } + + got := bazelContext.BuildStatementsToRegister() + if want := 1; len(got) != want { + t.Errorf("expected %d registered build statements, but got %#v", want, got) + } + + cmd := RuleBuilderCommand{} + createCommand(&cmd, got[0], "er", "bo", PathContextForTesting(TestConfig("out", nil, "", nil))) + if actual := cmd.buf.String(); testCase.command != actual { + t.Errorf("expected: [%s], actual: [%s]", testCase.command, actual) + } } } diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go index 6339a7181..acebdbbff 100644 --- a/android/buildinfo_prop.go +++ b/android/buildinfo_prop.go @@ -89,6 +89,7 @@ func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { writeProp("ro.build.version.security_patch", config.PlatformSecurityPatch()) writeProp("ro.build.version.base_os", config.PlatformBaseOS()) writeProp("ro.build.version.min_supported_target_sdk", config.PlatformMinSupportedTargetSdkVersion()) + writeProp("ro.build.version.known_codenames", config.PlatformVersionKnownCodenames()) if config.Eng() { writeProp("ro.build.type", "eng") @@ -109,7 +110,6 @@ func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { writeProp("ro.build.display.id", $BUILD_DISPLAY_ID) writeProp("ro.build.version.incremental", $BUILD_NUMBER) writeProp("ro.build.version.preview_sdk_fingerprint", $PLATFORM_PREVIEW_SDK_FINGERPRINT) - writeProp("ro.build.version.known_codenames", $PLATFORM_VERSION_KNOWN_CODENAMES) writeProp("ro.build.version.release_or_preview_display", $PLATFORM_DISPLAY_VERSION) writeProp("ro.build.date", `$DATE`) writeProp("ro.build.date.utc", `$DATE +%s`) diff --git a/android/config.go b/android/config.go index ef712926f..cd902cb86 100644 --- a/android/config.go +++ b/android/config.go @@ -793,6 +793,10 @@ func (c *config) PlatformVersionLastStable() string { return String(c.productVariables.Platform_version_last_stable) } +func (c *config) PlatformVersionKnownCodenames() string { + return String(c.productVariables.Platform_version_known_codenames) +} + func (c *config) MinSupportedSdkVersion() ApiLevel { return uncheckedFinalApiLevel(19) } diff --git a/android/deapexer.go b/android/deapexer.go index 265f5312b..6a93f6087 100644 --- a/android/deapexer.go +++ b/android/deapexer.go @@ -15,6 +15,8 @@ package android import ( + "strings" + "github.com/google/blueprint" ) @@ -148,12 +150,19 @@ type RequiresFilesFromPrebuiltApexTag interface { func FindDeapexerProviderForModule(ctx ModuleContext) *DeapexerInfo { var di *DeapexerInfo ctx.VisitDirectDepsWithTag(DeapexerTag, func(m Module) { - p := ctx.OtherModuleProvider(m, DeapexerProvider).(DeapexerInfo) + c := ctx.OtherModuleProvider(m, DeapexerProvider).(DeapexerInfo) + p := &c if di != nil { + // If two DeapexerInfo providers have been found then check if they are + // equivalent. If they are then use the selected one, otherwise fail. + if selected := equivalentDeapexerInfoProviders(di, p); selected != nil { + di = selected + return + } ctx.ModuleErrorf("Multiple installable prebuilt APEXes provide ambiguous deapexers: %s and %s", di.ApexModuleName(), p.ApexModuleName()) } - di = &p + di = p }) if di != nil { return di @@ -162,3 +171,33 @@ func FindDeapexerProviderForModule(ctx ModuleContext) *DeapexerInfo { ctx.ModuleErrorf("No prebuilt APEX provides a deapexer module for APEX variant %s", ai.ApexVariationName) return nil } + +// removeCompressedApexSuffix removes the _compressed suffix from the name if present. +func removeCompressedApexSuffix(name string) string { + return strings.TrimSuffix(name, "_compressed") +} + +// equivalentDeapexerInfoProviders checks to make sure that the two DeapexerInfo structures are +// equivalent. +// +// At the moment <x> and <x>_compressed APEXes are treated as being equivalent. +// +// If they are not equivalent then this returns nil, otherwise, this returns the DeapexerInfo that +// should be used by the build, which is always the uncompressed one. That ensures that the behavior +// of the build is not dependent on which prebuilt APEX is visited first. +func equivalentDeapexerInfoProviders(p1 *DeapexerInfo, p2 *DeapexerInfo) *DeapexerInfo { + n1 := removeCompressedApexSuffix(p1.ApexModuleName()) + n2 := removeCompressedApexSuffix(p2.ApexModuleName()) + + // If the names don't match then they are not equivalent. + if n1 != n2 { + return nil + } + + // Select the uncompressed APEX. + if n1 == removeCompressedApexSuffix(n1) { + return p1 + } else { + return p2 + } +} diff --git a/android/license_metadata.go b/android/license_metadata.go index 48c1383f6..f2ab0a44c 100644 --- a/android/license_metadata.go +++ b/android/license_metadata.go @@ -105,7 +105,7 @@ func buildLicenseMetadata(ctx ModuleContext, licenseMetadataFile WritablePath) { if p := base.commonProperties.Effective_package_name; p != nil { args = append(args, - `-p "`+proptools.NinjaAndShellEscape(*p)+`"`) + `-p `+proptools.NinjaAndShellEscapeIncludingSpaces(*p)) } args = append(args, diff --git a/android/mutator.go b/android/mutator.go index f06ecdab0..9e4aa59ad 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -93,6 +93,7 @@ type RegisterMutatorsContext interface { TopDown(name string, m TopDownMutator) MutatorHandle BottomUp(name string, m BottomUpMutator) MutatorHandle BottomUpBlueprint(name string, m blueprint.BottomUpMutator) MutatorHandle + Transition(name string, m TransitionMutator) } type RegisterMutatorFunc func(RegisterMutatorsContext) @@ -421,6 +422,182 @@ func (x *registerMutatorsContext) BottomUpBlueprint(name string, m blueprint.Bot return mutator } +type IncomingTransitionContext interface { + // Module returns the target of the dependency edge for which the transition + // is being computed + Module() Module + + // Config returns the configuration for the build. + Config() Config +} + +type OutgoingTransitionContext interface { + // Module returns the target of the dependency edge for which the transition + // is being computed + Module() Module + + // DepTag() Returns the dependency tag through which this dependency is + // reached + DepTag() blueprint.DependencyTag +} + +// Transition mutators implement a top-down mechanism where a module tells its +// direct dependencies what variation they should be built in but the dependency +// has the final say. +// +// When implementing a transition mutator, one needs to implement four methods: +// - Split() that tells what variations a module has by itself +// - OutgoingTransition() where a module tells what it wants from its +// dependency +// - IncomingTransition() where a module has the final say about its own +// variation +// - Mutate() that changes the state of a module depending on its variation +// +// That the effective variation of module B when depended on by module A is the +// composition the outgoing transition of module A and the incoming transition +// of module B. +// +// the outgoing transition should not take the properties of the dependency into +// account, only those of the module that depends on it. For this reason, the +// dependency is not even passed into it as an argument. Likewise, the incoming +// transition should not take the properties of the depending module into +// account and is thus not informed about it. This makes for a nice +// decomposition of the decision logic. +// +// A given transition mutator only affects its own variation; other variations +// stay unchanged along the dependency edges. +// +// Soong makes sure that all modules are created in the desired variations and +// that dependency edges are set up correctly. This ensures that "missing +// variation" errors do not happen and allows for more flexible changes in the +// value of the variation among dependency edges (as oppposed to bottom-up +// mutators where if module A in variation X depends on module B and module B +// has that variation X, A must depend on variation X of B) +// +// The limited power of the context objects passed to individual mutators +// methods also makes it more difficult to shoot oneself in the foot. Complete +// safety is not guaranteed because no one prevents individual transition +// mutators from mutating modules in illegal ways and for e.g. Split() or +// Mutate() to run their own visitations of the transitive dependency of the +// module and both of these are bad ideas, but it's better than no guardrails at +// all. +// +// This model is pretty close to Bazel's configuration transitions. The mapping +// between concepts in Soong and Bazel is as follows: +// - Module == configured target +// - Variant == configuration +// - Variation name == configuration flag +// - Variation == configuration flag value +// - Outgoing transition == attribute transition +// - Incoming transition == rule transition +// +// The Split() method does not have a Bazel equivalent and Bazel split +// transitions do not have a Soong equivalent. +// +// Mutate() does not make sense in Bazel due to the different models of the +// two systems: when creating new variations, Soong clones the old module and +// thus some way is needed to change it state whereas Bazel creates each +// configuration of a given configured target anew. +type TransitionMutator interface { + // Split returns the set of variations that should be created for a module no + // matter who depends on it. Used when Make depends on a particular variation + // or when the module knows its variations just based on information given to + // it in the Blueprint file. This method should not mutate the module it is + // called on. + Split(ctx BaseModuleContext) []string + + // Called on a module to determine which variation it wants from its direct + // dependencies. The dependency itself can override this decision. This method + // should not mutate the module itself. + OutgoingTransition(ctx OutgoingTransitionContext, sourceVariation string) string + + // Called on a module to determine which variation it should be in based on + // the variation modules that depend on it want. This gives the module a final + // say about its own variations. This method should not mutate the module + // itself. + IncomingTransition(ctx IncomingTransitionContext, incomingVariation string) string + + // Called after a module was split into multiple variations on each variation. + // It should not split the module any further but adding new dependencies is + // fine. Unlike all the other methods on TransitionMutator, this method is + // allowed to mutate the module. + Mutate(ctx BottomUpMutatorContext, variation string) +} + +type androidTransitionMutator struct { + finalPhase bool + bazelConversionMode bool + mutator TransitionMutator +} + +func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string { + if m, ok := ctx.Module().(Module); ok { + moduleContext := m.base().baseModuleContextFactory(ctx) + moduleContext.bazelConversionMode = a.bazelConversionMode + return a.mutator.Split(&moduleContext) + } else { + return []string{""} + } +} + +type outgoingTransitionContextImpl struct { + bp blueprint.OutgoingTransitionContext +} + +func (c *outgoingTransitionContextImpl) Module() Module { + return c.bp.Module().(Module) +} + +func (c *outgoingTransitionContextImpl) DepTag() blueprint.DependencyTag { + return c.bp.DepTag() +} + +func (a *androidTransitionMutator) OutgoingTransition(ctx blueprint.OutgoingTransitionContext, sourceVariation string) string { + if _, ok := ctx.Module().(Module); ok { + return a.mutator.OutgoingTransition(&outgoingTransitionContextImpl{bp: ctx}, sourceVariation) + } else { + return "" + } +} + +type incomingTransitionContextImpl struct { + bp blueprint.IncomingTransitionContext +} + +func (c *incomingTransitionContextImpl) Module() Module { + return c.bp.Module().(Module) +} + +func (c *incomingTransitionContextImpl) Config() Config { + return c.bp.Config().(Config) +} + +func (a *androidTransitionMutator) IncomingTransition(ctx blueprint.IncomingTransitionContext, incomingVariation string) string { + if _, ok := ctx.Module().(Module); ok { + return a.mutator.IncomingTransition(&incomingTransitionContextImpl{bp: ctx}, incomingVariation) + } else { + return "" + } +} + +func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, variation string) { + if am, ok := ctx.Module().(Module); ok { + a.mutator.Mutate(bottomUpMutatorContextFactory(ctx, am, a.finalPhase, a.bazelConversionMode), variation) + } +} + +func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { + atm := &androidTransitionMutator{ + finalPhase: x.finalPhase, + bazelConversionMode: x.bazelConversionMode, + mutator: m, + } + mutator := &mutator{ + name: name, + transitionMutator: atm} + x.mutators = append(x.mutators, mutator) +} + func (x *registerMutatorsContext) mutatorName(name string) string { if x.bazelConversionMode { return name + "_bp2build" @@ -456,6 +633,8 @@ func (mutator *mutator) register(ctx *Context) { handle = blueprintCtx.RegisterBottomUpMutator(mutator.name, mutator.bottomUpMutator) } else if mutator.topDownMutator != nil { handle = blueprintCtx.RegisterTopDownMutator(mutator.name, mutator.topDownMutator) + } else if mutator.transitionMutator != nil { + blueprintCtx.RegisterTransitionMutator(mutator.name, mutator.transitionMutator) } if mutator.parallel { handle.Parallel() diff --git a/android/register.go b/android/register.go index c50583322..4ff8fff97 100644 --- a/android/register.go +++ b/android/register.go @@ -96,10 +96,11 @@ var singletons sortableComponents var preSingletons sortableComponents type mutator struct { - name string - bottomUpMutator blueprint.BottomUpMutator - topDownMutator blueprint.TopDownMutator - parallel bool + name string + bottomUpMutator blueprint.BottomUpMutator + topDownMutator blueprint.TopDownMutator + transitionMutator blueprint.TransitionMutator + parallel bool } var _ sortableComponent = &mutator{} diff --git a/android/variable.go b/android/variable.go index 9478c0c9e..734ed1b68 100644 --- a/android/variable.go +++ b/android/variable.go @@ -200,6 +200,7 @@ type productVariables struct { Platform_min_supported_target_sdk_version *string `json:",omitempty"` Platform_base_os *string `json:",omitempty"` Platform_version_last_stable *string `json:",omitempty"` + Platform_version_known_codenames *string `json:",omitempty"` DeviceName *string `json:",omitempty"` DeviceProduct *string `json:",omitempty"` diff --git a/apex/apex.go b/apex/apex.go index beabbc90a..7e913e8fb 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1468,7 +1468,7 @@ func (a *apexBundle) EnableSanitizer(sanitizerName string) { } } -func (a *apexBundle) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool { +func (a *apexBundle) IsSanitizerEnabled(config android.Config, sanitizerName string) bool { if android.InList(sanitizerName, a.properties.SanitizerNames) { return true } @@ -1476,11 +1476,11 @@ func (a *apexBundle) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizer // Then follow the global setting globalSanitizerNames := []string{} if a.Host() { - globalSanitizerNames = ctx.Config().SanitizeHost() + globalSanitizerNames = config.SanitizeHost() } else { - arches := ctx.Config().SanitizeDeviceArch() + arches := config.SanitizeDeviceArch() if len(arches) == 0 || android.InList(a.Arch().ArchType.Name, arches) { - globalSanitizerNames = ctx.Config().SanitizeDevice() + globalSanitizerNames = config.SanitizeDevice() } } return android.InList(sanitizerName, globalSanitizerNames) diff --git a/apex/apex_test.go b/apex/apex_test.go index 7905710f2..dbe918010 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -7440,7 +7440,7 @@ func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, preparer android.F return result.TestContext } -func TestDuplicateDeapexeresFromPrebuiltApexes(t *testing.T) { +func TestDuplicateDeapexersFromPrebuiltApexes(t *testing.T) { preparers := android.GroupFixturePreparers( java.PrepareForTestWithJavaDefaultModules, PrepareForTestWithApexBuildComponents, @@ -7509,6 +7509,107 @@ func TestDuplicateDeapexeresFromPrebuiltApexes(t *testing.T) { }) } +func TestDuplicateButEquivalentDeapexersFromPrebuiltApexes(t *testing.T) { + preparers := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + PrepareForTestWithApexBuildComponents, + ) + + bpBase := ` + apex_set { + name: "com.android.myapex", + installable: true, + exported_bootclasspath_fragments: ["my-bootclasspath-fragment"], + set: "myapex.apks", + } + + apex_set { + name: "com.android.myapex_compressed", + apex_name: "com.android.myapex", + installable: true, + exported_bootclasspath_fragments: ["my-bootclasspath-fragment"], + set: "myapex_compressed.apks", + } + + prebuilt_bootclasspath_fragment { + name: "my-bootclasspath-fragment", + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + hidden_api: { + annotation_flags: "annotation-flags.csv", + metadata: "metadata.csv", + index: "index.csv", + signature_patterns: "signature_patterns.csv", + }, + %s + } + ` + + t.Run("java_import", func(t *testing.T) { + result := preparers.RunTestWithBp(t, + fmt.Sprintf(bpBase, `contents: ["libfoo"]`)+` + java_import { + name: "libfoo", + jars: ["libfoo.jar"], + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + } + `) + + module := result.Module("libfoo", "android_common_com.android.myapex") + usesLibraryDep := module.(java.UsesLibraryDependency) + android.AssertPathRelativeToTopEquals(t, "dex jar path", + "out/soong/.intermediates/com.android.myapex.deapexer/android_common/deapexer/javalib/libfoo.jar", + usesLibraryDep.DexJarBuildPath().Path()) + }) + + t.Run("java_sdk_library_import", func(t *testing.T) { + result := preparers.RunTestWithBp(t, + fmt.Sprintf(bpBase, `contents: ["libfoo"]`)+` + java_sdk_library_import { + name: "libfoo", + public: { + jars: ["libbar.jar"], + }, + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + compile_dex: true, + } + `) + + module := result.Module("libfoo", "android_common_com.android.myapex") + usesLibraryDep := module.(java.UsesLibraryDependency) + android.AssertPathRelativeToTopEquals(t, "dex jar path", + "out/soong/.intermediates/com.android.myapex.deapexer/android_common/deapexer/javalib/libfoo.jar", + usesLibraryDep.DexJarBuildPath().Path()) + }) + + t.Run("prebuilt_bootclasspath_fragment", func(t *testing.T) { + _ = preparers.RunTestWithBp(t, fmt.Sprintf(bpBase, ` + image_name: "art", + contents: ["libfoo"], + `)+` + java_sdk_library_import { + name: "libfoo", + public: { + jars: ["libbar.jar"], + }, + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + compile_dex: true, + } + `) + }) +} + func TestUpdatable_should_set_min_sdk_version(t *testing.T) { testApexError(t, `"myapex" .*: updatable: updatable APEXes should set min_sdk_version`, ` apex { diff --git a/bazel/aquery.go b/bazel/aquery.go index 5e4ebf8d2..6829698c0 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -167,7 +167,7 @@ func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler // Map middleman artifact ContentHash to input artifact depset ID. // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, - // if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then, + // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then, // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for // that action instead. middlemanIdToDepsetIds := map[artifactId][]depsetId{} @@ -348,7 +348,7 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey { // Two depsets collide on hash. Ensure that their contents are identical. if !reflect.DeepEqual(aqueryDepset, prevEntry) { - return nil, nil, fmt.Errorf("Two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset) + return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset) } } else { depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index c9c890930..1da6340a5 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -17,6 +17,7 @@ package bazel import ( "fmt" "reflect" + "sort" "testing" ) @@ -224,7 +225,7 @@ func TestAqueryMultiArchGenrule(t *testing.T) { }] }` actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) - expectedBuildStatements := []BuildStatement{} + var expectedBuildStatements []BuildStatement for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} { expectedBuildStatements = append(expectedBuildStatements, BuildStatement{ @@ -235,7 +236,7 @@ func TestAqueryMultiArchGenrule(t *testing.T) { fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch), }, Env: []KeyValuePair{ - KeyValuePair{Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, + {Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, }, Mnemonic: "Genrule", }) @@ -747,7 +748,7 @@ func TestTransitiveInputDepsets(t *testing.T) { actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'", OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, Mnemonic: "Action", @@ -758,7 +759,7 @@ func TestTransitiveInputDepsets(t *testing.T) { // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs // are given via a deep depset, but the depset is flattened when returned as a // BuildStatement slice. - expectedFlattenedInputs := []string{} + var expectedFlattenedInputs []string for i := 1; i < 20; i++ { expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) } @@ -876,7 +877,7 @@ func flattenDepsets(depsetHashesToFlatten []string, allDepsets []AqueryDepset) [ for _, depset := range allDepsets { depsetsByHash[depset.ContentHash] = depset } - result := []string{} + var result []string for _, depsetId := range depsetHashesToFlatten { result = append(result, flattenDepset(depsetId, depsetsByHash)...) } @@ -886,7 +887,7 @@ func flattenDepsets(depsetHashesToFlatten []string, allDepsets []AqueryDepset) [ // Returns the contents of a given depset in post order. func flattenDepset(depsetHashToFlatten string, allDepsets map[string]AqueryDepset) []string { depset := allDepsets[depsetHashToFlatten] - result := []string{} + var result []string for _, depsetId := range depset.TransitiveDepSetHashes { result = append(result, flattenDepset(depsetId, allDepsets)...) } @@ -897,7 +898,7 @@ func flattenDepset(depsetHashToFlatten string, allDepsets map[string]AqueryDepse func assertFlattenedDepsets(t *testing.T, actualDepsets []AqueryDepset, expectedDepsetFiles [][]string) { t.Helper() if len(actualDepsets) != len(expectedDepsetFiles) { - t.Errorf("Expected %s depsets, but got %s depsets", expectedDepsetFiles, actualDepsets) + t.Errorf("Expected %d depsets, but got %d depsets", len(expectedDepsetFiles), len(actualDepsets)) } for i, actualDepset := range actualDepsets { actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) @@ -958,7 +959,7 @@ func TestSimpleSymlink(t *testing.T) { } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "mkdir -p one/symlink_subdir && " + "rm -f one/symlink_subdir/symlink && " + "ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink", @@ -1022,7 +1023,7 @@ func TestSymlinkQuotesPaths(t *testing.T) { } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "mkdir -p 'one/symlink subdir' && " + "rm -f 'one/symlink subdir/symlink' && " + "ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'", @@ -1154,7 +1155,7 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) { } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " + "chmod a+x template_file'", OutputPaths: []string{"template_file"}, @@ -1320,14 +1321,14 @@ func TestPythonZipperActionSuccess(t *testing.T) { } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > python_binary && " + "chmod a+x python_binary'", InputPaths: []string{"python_binary.zip"}, OutputPaths: []string{"python_binary"}, Mnemonic: "TemplateExpand", }, - BuildStatement{ + { Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " + "__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py && " + "../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles", @@ -1484,50 +1485,54 @@ func assertBuildStatements(t *testing.T, expected []BuildStatement, actual []Bui len(expected), len(actual), expected, actual) return } -ACTUAL_LOOP: - for _, actualStatement := range actual { - for _, expectedStatement := range expected { - if buildStatementEquals(actualStatement, expectedStatement) { - continue ACTUAL_LOOP - } + type compareFn = func(i int, j int) bool + byCommand := func(slice []BuildStatement) compareFn { + return func(i int, j int) bool { + return slice[i].Command < slice[j].Command + } + } + sort.SliceStable(expected, byCommand(expected)) + sort.SliceStable(actual, byCommand(actual)) + for i, actualStatement := range actual { + expectedStatement := expected[i] + if differingField := buildStatementEquals(actualStatement, expectedStatement); differingField != "" { + t.Errorf("%s differs\nunexpected build statement %#v.\nexpected: %#v", + differingField, actualStatement, expected) + return } - t.Errorf("unexpected build statement %#v.\n expected: %#v", - actualStatement, expected) - return } } -func buildStatementEquals(first BuildStatement, second BuildStatement) bool { +func buildStatementEquals(first BuildStatement, second BuildStatement) string { if first.Mnemonic != second.Mnemonic { - return false + return "Mnemonic" } if first.Command != second.Command { - return false + return "Command" } // Ordering is significant for environment variables. if !reflect.DeepEqual(first.Env, second.Env) { - return false + return "Env" } // Ordering is irrelevant for input and output paths, so compare sets. - if !reflect.DeepEqual(stringSet(first.InputPaths), stringSet(second.InputPaths)) { - return false + if !reflect.DeepEqual(sortedStrings(first.InputPaths), sortedStrings(second.InputPaths)) { + return "InputPaths" } - if !reflect.DeepEqual(stringSet(first.OutputPaths), stringSet(second.OutputPaths)) { - return false + if !reflect.DeepEqual(sortedStrings(first.OutputPaths), sortedStrings(second.OutputPaths)) { + return "OutputPaths" } - if !reflect.DeepEqual(stringSet(first.SymlinkPaths), stringSet(second.SymlinkPaths)) { - return false + if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) { + return "SymlinkPaths" } if first.Depfile != second.Depfile { - return false + return "Depfile" } - return true + return "" } -func stringSet(stringSlice []string) map[string]struct{} { - stringMap := make(map[string]struct{}) - for _, s := range stringSlice { - stringMap[s] = struct{}{} - } - return stringMap +func sortedStrings(stringSlice []string) []string { + sorted := make([]string, len(stringSlice)) + copy(sorted, stringSlice) + sort.Strings(sorted) + return sorted } diff --git a/bp2build/cc_library_headers_conversion_test.go b/bp2build/cc_library_headers_conversion_test.go index e5bb12010..2b54d4592 100644 --- a/bp2build/cc_library_headers_conversion_test.go +++ b/bp2build/cc_library_headers_conversion_test.go @@ -84,18 +84,6 @@ func TestCcLibraryHeadersSimple(t *testing.T) { }, blueprint: soongCcLibraryHeadersPreamble + ` cc_library_headers { - name: "lib-1", - export_include_dirs: ["lib-1"], - bazel_module: { bp2build_available: false }, -} - -cc_library_headers { - name: "lib-2", - export_include_dirs: ["lib-2"], - bazel_module: { bp2build_available: false }, -} - -cc_library_headers { name: "foo_headers", export_include_dirs: ["dir-1", "dir-2"], header_libs: ["lib-1", "lib-2"], @@ -128,12 +116,8 @@ cc_library_headers { "//build/bazel/platforms/arch:x86_64": ["arch_x86_64_exported_include_dir"], "//conditions:default": [], })`, - "implementation_deps": `[ - ":lib-1", - ":lib-2", - ]`, - "sdk_version": `"current"`, - "min_sdk_version": `"29"`, + "sdk_version": `"current"`, + "min_sdk_version": `"29"`, }), }, }) @@ -173,18 +157,34 @@ cc_library_headers { cc_library_headers { name: "foo_headers", header_libs: ["base-lib"], + export_header_lib_headers: ["base-lib"], target: { - android: { header_libs: ["android-lib"] }, - darwin: { header_libs: ["darwin-lib"] }, - linux_bionic: { header_libs: ["linux_bionic-lib"] }, - linux_glibc: { header_libs: ["linux-lib"] }, - windows: { header_libs: ["windows-lib"] }, + android: { + header_libs: ["android-lib"], + export_header_lib_headers: ["android-lib"], + }, + darwin: { + header_libs: ["darwin-lib"], + export_header_lib_headers: ["darwin-lib"], + }, + linux_bionic: { + header_libs: ["linux_bionic-lib"], + export_header_lib_headers: ["linux_bionic-lib"], + }, + linux_glibc: { + header_libs: ["linux-lib"], + export_header_lib_headers: ["linux-lib"], + }, + windows: { + header_libs: ["windows-lib"], + export_header_lib_headers: ["windows-lib"], + }, }, include_build_directory: false, }`, expectedBazelTargets: []string{ makeBazelTarget("cc_library_headers", "foo_headers", attrNameToString{ - "implementation_deps": `[":base-lib"] + select({ + "deps": `[":base-lib"] + select({ "//build/bazel/platforms/os:android": [":android-lib"], "//build/bazel/platforms/os:darwin": [":darwin-lib"], "//build/bazel/platforms/os:linux": [":linux-lib"], @@ -228,10 +228,6 @@ cc_library_headers { "//build/bazel/platforms/os:android": [":exported-lib"], "//conditions:default": [], })`, - "implementation_deps": `select({ - "//build/bazel/platforms/os:android": [":android-lib"], - "//conditions:default": [], - })`, }), }, }) @@ -95,6 +95,10 @@ type Deps struct { HeaderLibs []string RuntimeLibs []string + // UnexportedStaticLibs are static libraries that are also passed to -Wl,--exclude-libs= to + // prevent automatically exporting symbols. + UnexportedStaticLibs []string + // Used for data dependencies adjacent to tests DataLibs []string DataBins []string @@ -156,6 +160,7 @@ type PathDeps struct { GeneratedDeps android.Paths Flags []string + LdFlags []string IncludeDirs android.Paths SystemIncludeDirs android.Paths ReexportedDirs android.Paths @@ -678,6 +683,9 @@ type libraryDependencyTag struct { // Whether or not this dependency has to be followed for the apex variants excludeInApex bool + + // If true, don't automatically export symbols from the static library into a shared library. + unexportedSymbols bool } // header returns true if the libraryDependencyTag is tagging a header lib dependency. @@ -982,6 +990,7 @@ func (c *Module) Shared() bool { return library.shared() } } + panic(fmt.Errorf("Shared() called on non-library module: %q", c.BaseModuleName())) } @@ -1921,6 +1930,8 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-isystem "+dir.String()) } + flags.Local.LdFlags = append(flags.Local.LdFlags, deps.LdFlags...) + c.flags = flags // We need access to all the flags seen by a source file. if c.sabi != nil { @@ -2368,6 +2379,13 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { }, depTag, RewriteSnapshotLib(lib, GetSnapshot(c, &snapshotInfo, actx).StaticLibs)) } + for _, lib := range deps.UnexportedStaticLibs { + depTag := libraryDependencyTag{Kind: staticLibraryDependency, Order: lateLibraryDependency, unexportedSymbols: true} + actx.AddVariationDependencies([]blueprint.Variation{ + {Mutator: "link", Variation: "static"}, + }, depTag, RewriteSnapshotLib(lib, GetSnapshot(c, &snapshotInfo, actx).StaticLibs)) + } + for _, lib := range deps.LateSharedLibs { if inList(lib, sharedLibNames) { // This is to handle the case that some of the late shared libs (libc, libdl, libm, ...) @@ -2866,6 +2884,10 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) } } + if libDepTag.unexportedSymbols { + depPaths.LdFlags = append(depPaths.LdFlags, + "-Wl,--exclude-libs="+staticLibraryInfo.StaticLibrary.Base()) + } } if libDepTag.static() && !libDepTag.wholeStatic { diff --git a/cc/library.go b/cc/library.go index 0fa01d78a..c445a42ca 100644 --- a/cc/library.go +++ b/cc/library.go @@ -925,7 +925,6 @@ func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Fla if ctx.Darwin() { f = append(f, "-dynamiclib", - "-single_module", "-install_name @rpath/"+libName+flags.Toolchain.ShlibSuffix(), ) if ctx.Arch().ArchType == android.X86 { diff --git a/cc/library_headers.go b/cc/library_headers.go index 7232290fd..77c25231c 100644 --- a/cc/library_headers.go +++ b/cc/library_headers.go @@ -130,7 +130,6 @@ func libraryHeadersBp2Build(ctx android.TopDownMutatorContext, module *Module) { Export_includes: exportedIncludes.Includes, Export_absolute_includes: exportedIncludes.AbsoluteIncludes, Export_system_includes: exportedIncludes.SystemIncludes, - Implementation_deps: linkerAttrs.implementationDeps, Deps: linkerAttrs.deps, System_dynamic_deps: linkerAttrs.systemDynamicDeps, Hdrs: baseAttributes.hdrs, diff --git a/cc/linkable.go b/cc/linkable.go index 04eab3927..c58bfdfbd 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -22,13 +22,6 @@ type PlatformSanitizeable interface { // than left undefined. IsSanitizerExplicitlyDisabled(t SanitizerType) bool - // SanitizeDep returns true if the module is statically linked into another that is sanitized - // with the given sanitizer. - SanitizeDep(t SanitizerType) bool - - // SetSanitizeDep marks a module as a static dependency of another module to be sanitized. - SetSanitizeDep(t SanitizerType) - // SetSanitizer enables or disables the specified sanitizer type if it's supported, otherwise this should panic. SetSanitizer(t SanitizerType, b bool) diff --git a/cc/linker.go b/cc/linker.go index 4e9404c0f..76a60ca7f 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -399,7 +399,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { if ctx.toolchain().Bionic() { // libclang_rt.builtins has to be last on the command line if !Bool(linker.Properties.No_libcrt) && !ctx.header() { - deps.LateStaticLibs = append(deps.LateStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) + deps.UnexportedStaticLibs = append(deps.UnexportedStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) } if inList("libdl", deps.SharedLibs) { @@ -422,7 +422,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { } } else if ctx.toolchain().Musl() { if !Bool(linker.Properties.No_libcrt) && !ctx.header() { - deps.LateStaticLibs = append(deps.LateStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) + deps.UnexportedStaticLibs = append(deps.UnexportedStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) } } @@ -436,11 +436,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { } func (linker *baseLinker) useClangLld(ctx ModuleContext) bool { - // Clang lld is not ready for for Darwin host executables yet. - // See https://lld.llvm.org/AtomLLD.html for status of lld for Mach-O. - if ctx.Darwin() { - return false - } if linker.Properties.Use_clang_lld != nil { return Bool(linker.Properties.Use_clang_lld) } @@ -530,10 +525,6 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { } } - if ctx.toolchain().LibclangRuntimeLibraryArch() != "" { - flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--exclude-libs="+config.BuiltinsRuntimeLibrary(ctx.toolchain())+".a") - } - CheckBadLinkerFlags(ctx, "ldflags", linker.Properties.Ldflags) flags.Local.LdFlags = append(flags.Local.LdFlags, proptools.NinjaAndShellEscapeList(linker.Properties.Ldflags)...) diff --git a/cc/sanitize.go b/cc/sanitize.go index 42a112ed4..92987f7f8 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -153,9 +153,10 @@ func (t SanitizerType) name() string { func (t SanitizerType) registerMutators(ctx android.RegisterMutatorsContext) { switch t { - case Asan, Hwasan, Fuzzer, scs, tsan, cfi: - ctx.TopDown(t.variationName()+"_deps", sanitizerDepsMutator(t)) - ctx.BottomUp(t.variationName(), sanitizerMutator(t)) + case cfi, Hwasan, Asan, tsan, Fuzzer, scs: + sanitizer := &sanitizerSplitMutator{t} + ctx.TopDown(t.variationName()+"_markapexes", sanitizer.markSanitizableApexesMutator) + ctx.Transition(t.variationName(), sanitizer) case Memtag_heap, intOverflow: // do nothing default: @@ -276,7 +277,6 @@ type SanitizeUserProps struct { type SanitizeProperties struct { Sanitize SanitizeUserProps `android:"arch_variant"` SanitizerEnabled bool `blueprint:"mutated"` - SanitizeDepTypes []SanitizerType `blueprint:"mutated"` MinimalRuntimeDep bool `blueprint:"mutated"` BuiltinsDep bool `blueprint:"mutated"` UbsanRuntimeDep bool `blueprint:"mutated"` @@ -588,13 +588,6 @@ func toDisableUnsignedShiftBaseChange(flags []string) bool { } func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { - minimalRuntimeLib := config.UndefinedBehaviorSanitizerMinimalRuntimeLibrary(ctx.toolchain()) + ".a" - - if sanitize.Properties.MinimalRuntimeDep { - flags.Local.LdFlags = append(flags.Local.LdFlags, - "-Wl,--exclude-libs,"+minimalRuntimeLib) - } - if !sanitize.Properties.SanitizerEnabled && !sanitize.Properties.UbsanRuntimeDep { return flags } @@ -722,11 +715,6 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=vptr,function") } - if enableMinimalRuntime(sanitize) { - flags.Local.CFlags = append(flags.Local.CFlags, strings.Join(minimalRuntimeFlags, " ")) - flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--exclude-libs,"+minimalRuntimeLib) - } - if Bool(sanitize.Properties.Sanitize.Fuzzer) { // When fuzzing, we wish to crash with diagnostics on any bug. flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize-trap=all", "-fno-sanitize-recover=all") @@ -735,6 +723,11 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { } else { flags.Local.CFlags = append(flags.Local.CFlags, "-fsanitize-trap=all", "-ftrap-function=abort") } + + if enableMinimalRuntime(sanitize) { + flags.Local.CFlags = append(flags.Local.CFlags, strings.Join(minimalRuntimeFlags, " ")) + } + // http://b/119329758, Android core does not boot up with this sanitizer yet. if toDisableImplicitIntegerChange(flags.Local.CFlags) { flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=implicit-integer-sign-change") @@ -906,7 +899,7 @@ func (m *Module) SanitizableDepTagChecker() SantizableDependencyTagChecker { // Determines if the current module is a static library going to be captured // as vendor snapshot. Such modules must create both cfi and non-cfi variants, // except for ones which explicitly disable cfi. -func needsCfiForVendorSnapshot(mctx android.TopDownMutatorContext) bool { +func needsCfiForVendorSnapshot(mctx android.BaseModuleContext) bool { if snapshot.IsVendorProprietaryModule(mctx) { return false } @@ -934,62 +927,232 @@ func needsCfiForVendorSnapshot(mctx android.TopDownMutatorContext) bool { !c.IsSanitizerExplicitlyDisabled(cfi) } -// Propagate sanitizer requirements down from binaries -func sanitizerDepsMutator(t SanitizerType) func(android.TopDownMutatorContext) { - return func(mctx android.TopDownMutatorContext) { - if c, ok := mctx.Module().(PlatformSanitizeable); ok { - enabled := c.IsSanitizerEnabled(t) - if t == cfi && needsCfiForVendorSnapshot(mctx) { - // We shouldn't change the result of isSanitizerEnabled(cfi) to correctly - // determine defaultVariation in sanitizerMutator below. - // Instead, just mark SanitizeDep to forcefully create cfi variant. +type sanitizerSplitMutator struct { + sanitizer SanitizerType +} + +// If an APEX is sanitized or not depends on whether it contains at least one +// sanitized module. Transition mutators cannot propagate information up the +// dependency graph this way, so we need an auxiliary mutator to do so. +func (s *sanitizerSplitMutator) markSanitizableApexesMutator(ctx android.TopDownMutatorContext) { + if sanitizeable, ok := ctx.Module().(Sanitizeable); ok { + enabled := sanitizeable.IsSanitizerEnabled(ctx.Config(), s.sanitizer.name()) + ctx.VisitDirectDeps(func(dep android.Module) { + if c, ok := dep.(*Module); ok && c.sanitize.isSanitizerEnabled(s.sanitizer) { enabled = true - c.SetSanitizeDep(t) } - if enabled { - isSanitizableDependencyTag := c.SanitizableDepTagChecker() - mctx.WalkDeps(func(child, parent android.Module) bool { - if !isSanitizableDependencyTag(mctx.OtherModuleDependencyTag(child)) { - return false - } - if d, ok := child.(PlatformSanitizeable); ok && d.SanitizePropDefined() && - !d.SanitizeNever() && - !d.IsSanitizerExplicitlyDisabled(t) { - if t == cfi || t == Hwasan || t == scs || t == Asan { - if d.StaticallyLinked() && d.SanitizerSupported(t) { - // Rust does not support some of these sanitizers, so we need to check if it's - // supported before setting this true. - d.SetSanitizeDep(t) - } - } else { - d.SetSanitizeDep(t) - } - } - return true - }) + }) + + if enabled { + sanitizeable.EnableSanitizer(s.sanitizer.name()) + } + } +} + +func (s *sanitizerSplitMutator) Split(ctx android.BaseModuleContext) []string { + if c, ok := ctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { + if s.sanitizer == cfi && needsCfiForVendorSnapshot(ctx) { + return []string{"", s.sanitizer.variationName()} + } + + // If the given sanitizer is not requested in the .bp file for a module, it + // won't automatically build the sanitized variation. + if !c.IsSanitizerEnabled(s.sanitizer) { + return []string{""} + } + + if c.Binary() { + // If a sanitizer is enabled for a binary, we do not build the version + // without the sanitizer + return []string{s.sanitizer.variationName()} + } else if c.StaticallyLinked() || c.Header() { + // For static libraries, we build both versions. Some Make modules + // apparently depend on this behavior. + return []string{"", s.sanitizer.variationName()} + } else { + // We only build the requested variation of dynamic libraries + return []string{s.sanitizer.variationName()} + } + } + + if _, ok := ctx.Module().(JniSanitizeable); ok { + // TODO: this should call into JniSanitizable.IsSanitizerEnabledForJni but + // that is short-circuited for now + return []string{""} + } + + // If an APEX has a sanitized dependency, we build the APEX in the sanitized + // variation. This is useful because such APEXes require extra dependencies. + if sanitizeable, ok := ctx.Module().(Sanitizeable); ok { + enabled := sanitizeable.IsSanitizerEnabled(ctx.Config(), s.sanitizer.name()) + if enabled { + return []string{s.sanitizer.variationName()} + } else { + return []string{""} + } + } + + if c, ok := ctx.Module().(*Module); ok { + //TODO: When Rust modules have vendor support, enable this path for PlatformSanitizeable + + // Check if it's a snapshot module supporting sanitizer + if ss, ok := c.linker.(snapshotSanitizer); ok && ss.isSanitizerEnabled(s.sanitizer) { + return []string{"", s.sanitizer.variationName()} + } else { + return []string{""} + } + } + + return []string{""} +} + +func (s *sanitizerSplitMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + if c, ok := ctx.Module().(PlatformSanitizeable); ok { + if !c.SanitizableDepTagChecker()(ctx.DepTag()) { + // If the dependency is through a non-sanitizable tag, use the + // non-sanitized variation + return "" + } + + return sourceVariation + } else if _, ok := ctx.Module().(JniSanitizeable); ok { + // TODO: this should call into JniSanitizable.IsSanitizerEnabledForJni but + // that is short-circuited for now + return "" + } else { + // Otherwise, do not rock the boat. + return sourceVariation + } +} + +func (s *sanitizerSplitMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if d, ok := ctx.Module().(PlatformSanitizeable); ok { + if dm, ok := ctx.Module().(*Module); ok { + if ss, ok := dm.linker.(snapshotSanitizer); ok && ss.isSanitizerEnabled(s.sanitizer) { + return incomingVariation } - } else if jniSanitizeable, ok := mctx.Module().(JniSanitizeable); ok { - // If it's a Java module with native dependencies through jni, - // set the sanitizer for them - if jniSanitizeable.IsSanitizerEnabledForJni(mctx, t.name()) { - mctx.VisitDirectDeps(func(child android.Module) { - if c, ok := child.(PlatformSanitizeable); ok && - mctx.OtherModuleDependencyTag(child) == JniFuzzLibTag && - c.SanitizePropDefined() && - !c.SanitizeNever() && - !c.IsSanitizerExplicitlyDisabled(t) { - c.SetSanitizeDep(t) - } - }) + } + + if !d.SanitizePropDefined() || + d.SanitizeNever() || + d.IsSanitizerExplicitlyDisabled(s.sanitizer) || + !d.SanitizerSupported(s.sanitizer) { + // If a module opts out of a sanitizer, use its non-sanitized variation + return "" + } + + // Binaries are always built in the variation they requested. + if d.Binary() { + if d.IsSanitizerEnabled(s.sanitizer) { + return s.sanitizer.variationName() + } else { + return "" + } + } + + // If a shared library requests to be sanitized, it will be built for that + // sanitizer. Otherwise, some sanitizers propagate through shared library + // dependency edges, some do not. + if !d.StaticallyLinked() && !d.Header() { + if d.IsSanitizerEnabled(s.sanitizer) { + return s.sanitizer.variationName() + } + + if s.sanitizer == cfi || s.sanitizer == Hwasan || s.sanitizer == scs || s.sanitizer == Asan { + return "" + } + } + + // Static and header libraries inherit whether they are sanitized from the + // module they are linked into + return incomingVariation + } else if d, ok := ctx.Module().(Sanitizeable); ok { + // If an APEX contains a sanitized module, it will be built in the variation + // corresponding to that sanitizer. + enabled := d.IsSanitizerEnabled(ctx.Config(), s.sanitizer.name()) + if enabled { + return s.sanitizer.variationName() + } + + return incomingVariation + } + + return "" +} + +func (s *sanitizerSplitMutator) Mutate(mctx android.BottomUpMutatorContext, variationName string) { + sanitizerVariation := variationName == s.sanitizer.variationName() + + if c, ok := mctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { + sanitizerEnabled := c.IsSanitizerEnabled(s.sanitizer) + + oneMakeVariation := false + if c.StaticallyLinked() || c.Header() { + if s.sanitizer != cfi && s.sanitizer != scs && s.sanitizer != Hwasan { + // These sanitizers export only one variation to Make. For the rest, + // Make targets can depend on both the sanitized and non-sanitized + // versions. + oneMakeVariation = true + } + } else if !c.Binary() { + // Shared library. These are the sanitizers that do propagate through shared + // library dependencies and therefore can cause multiple variations of a + // shared library to be built. + if s.sanitizer != cfi && s.sanitizer != Hwasan && s.sanitizer != scs && s.sanitizer != Asan { + oneMakeVariation = true + } + } + + if oneMakeVariation { + if sanitizerEnabled != sanitizerVariation { + c.SetPreventInstall() + c.SetHideFromMake() + } + } + + if sanitizerVariation { + c.SetSanitizer(s.sanitizer, true) + + // CFI is incompatible with ASAN so disable it in ASAN variations + if s.sanitizer.incompatibleWithCfi() { + cfiSupported := mctx.Module().(PlatformSanitizeable).SanitizerSupported(cfi) + if mctx.Device() && cfiSupported { + c.SetSanitizer(cfi, false) + } + } + + // locate the asan libraries under /data/asan + if !c.Binary() && !c.StaticallyLinked() && !c.Header() && mctx.Device() && s.sanitizer == Asan && sanitizerEnabled { + c.SetInSanitizerDir() } - } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok { - // If an APEX module includes a lib which is enabled for a sanitizer T, then - // the APEX module is also enabled for the same sanitizer type. - mctx.VisitDirectDeps(func(child android.Module) { - if c, ok := child.(*Module); ok && c.sanitize.isSanitizerEnabled(t) { - sanitizeable.EnableSanitizer(t.name()) + + if c.StaticallyLinked() && c.ExportedToMake() { + if s.sanitizer == Hwasan { + hwasanStaticLibs(mctx.Config()).add(c, c.Module().Name()) + } else if s.sanitizer == cfi { + cfiStaticLibs(mctx.Config()).add(c, c.Module().Name()) } - }) + } + } else if c.IsSanitizerEnabled(s.sanitizer) { + // Disable the sanitizer for the non-sanitized variation + c.SetSanitizer(s.sanitizer, false) + } + } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok { + // If an APEX has sanitized dependencies, it gets a few more dependencies + if sanitizerVariation { + sanitizeable.AddSanitizerDependencies(mctx, s.sanitizer.name()) + } + } else if c, ok := mctx.Module().(*Module); ok { + if ss, ok := c.linker.(snapshotSanitizer); ok && ss.isSanitizerEnabled(s.sanitizer) { + c.linker.(snapshotSanitizer).setSanitizerVariation(s.sanitizer, sanitizerVariation) + + // Export the static lib name to make + if c.static() && c.ExportedToMake() { + if s.sanitizer == cfi { + // use BaseModuleName which is the name for Make. + cfiStaticLibs(mctx.Config()).add(c, c.BaseModuleName()) + } + } } } } @@ -1217,7 +1380,7 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { } // static executable gets static runtime libs - depTag := libraryDependencyTag{Kind: staticLibraryDependency} + depTag := libraryDependencyTag{Kind: staticLibraryDependency, unexportedSymbols: true} variations := append(mctx.Target().Variations(), blueprint.Variation{Mutator: "link", Variation: "static"}) if c.Device() { @@ -1289,7 +1452,7 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { type Sanitizeable interface { android.Module - IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool + IsSanitizerEnabled(config android.Config, sanitizerName string) bool EnableSanitizer(sanitizerName string) AddSanitizerDependencies(ctx android.BottomUpMutatorContext, sanitizerName string) } @@ -1315,16 +1478,6 @@ func (c *Module) IsSanitizerEnabled(t SanitizerType) bool { return c.sanitize.isSanitizerEnabled(t) } -func (c *Module) SanitizeDep(t SanitizerType) bool { - for _, e := range c.sanitize.Properties.SanitizeDepTypes { - if t == e { - return true - } - } - - return false -} - func (c *Module) StaticallyLinked() bool { return c.static() } @@ -1341,123 +1494,8 @@ func (c *Module) SetSanitizer(t SanitizerType, b bool) { } } -func (c *Module) SetSanitizeDep(t SanitizerType) { - if !c.SanitizeDep(t) { - c.sanitize.Properties.SanitizeDepTypes = append(c.sanitize.Properties.SanitizeDepTypes, t) - } -} - var _ PlatformSanitizeable = (*Module)(nil) -// Create sanitized variants for modules that need them -func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { - return func(mctx android.BottomUpMutatorContext) { - if c, ok := mctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { - - // Make sure we're not setting CFI to any value if it's not supported. - cfiSupported := mctx.Module().(PlatformSanitizeable).SanitizerSupported(cfi) - - if c.Binary() && c.IsSanitizerEnabled(t) { - modules := mctx.CreateVariations(t.variationName()) - modules[0].(PlatformSanitizeable).SetSanitizer(t, true) - } else if c.IsSanitizerEnabled(t) || c.SanitizeDep(t) { - isSanitizerEnabled := c.IsSanitizerEnabled(t) - if c.StaticallyLinked() || c.Header() || t == Fuzzer { - // Static and header libs are split into non-sanitized and sanitized variants. - // Shared libs are not split. However, for asan and fuzzer, we split even for shared - // libs because a library sanitized for asan/fuzzer can't be linked from a library - // that isn't sanitized for asan/fuzzer. - // - // Note for defaultVariation: since we don't split for shared libs but for static/header - // libs, it is possible for the sanitized variant of a static/header lib to depend - // on non-sanitized variant of a shared lib. Such unfulfilled variation causes an - // error when the module is split. defaultVariation is the name of the variation that - // will be used when such a dangling dependency occurs during the split of the current - // module. By setting it to the name of the sanitized variation, the dangling dependency - // is redirected to the sanitized variant of the dependent module. - defaultVariation := t.variationName() - // Not all PlatformSanitizeable modules support the CFI sanitizer - mctx.SetDefaultDependencyVariation(&defaultVariation) - - modules := mctx.CreateVariations("", t.variationName()) - modules[0].(PlatformSanitizeable).SetSanitizer(t, false) - modules[1].(PlatformSanitizeable).SetSanitizer(t, true) - - if mctx.Device() && t.incompatibleWithCfi() && cfiSupported { - // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that - // are incompatible with cfi - modules[1].(PlatformSanitizeable).SetSanitizer(cfi, false) - } - - // For cfi/scs/hwasan, we can export both sanitized and un-sanitized variants - // to Make, because the sanitized version has a different suffix in name. - // For other types of sanitizers, suppress the variation that is disabled. - if t != cfi && t != scs && t != Hwasan { - if isSanitizerEnabled { - modules[0].(PlatformSanitizeable).SetPreventInstall() - modules[0].(PlatformSanitizeable).SetHideFromMake() - } else { - modules[1].(PlatformSanitizeable).SetPreventInstall() - modules[1].(PlatformSanitizeable).SetHideFromMake() - } - } - - // Export the static lib name to make - if c.StaticallyLinked() && c.ExportedToMake() { - if t == cfi { - cfiStaticLibs(mctx.Config()).add(c, c.Module().Name()) - } else if t == Hwasan { - hwasanStaticLibs(mctx.Config()).add(c, c.Module().Name()) - } - } - } else { - // Shared libs are not split. Only the sanitized variant is created. - modules := mctx.CreateVariations(t.variationName()) - modules[0].(PlatformSanitizeable).SetSanitizer(t, true) - - // locate the asan libraries under /data/asan - if mctx.Device() && t == Asan && isSanitizerEnabled { - modules[0].(PlatformSanitizeable).SetInSanitizerDir() - } - - if mctx.Device() && t.incompatibleWithCfi() && cfiSupported { - // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that - // are incompatible with cfi - modules[0].(PlatformSanitizeable).SetSanitizer(cfi, false) - } - } - } - } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok && sanitizeable.IsSanitizerEnabled(mctx, t.name()) { - // APEX fuzz modules fall here - sanitizeable.AddSanitizerDependencies(mctx, t.name()) - mctx.CreateVariations(t.variationName()) - } else if _, ok := mctx.Module().(JniSanitizeable); ok { - // Java fuzz modules fall here - mctx.CreateVariations(t.variationName()) - } else if c, ok := mctx.Module().(*Module); ok { - //TODO: When Rust modules have vendor support, enable this path for PlatformSanitizeable - - // Check if it's a snapshot module supporting sanitizer - if s, ok := c.linker.(snapshotSanitizer); ok && s.isSanitizerEnabled(t) { - // Set default variation as above. - defaultVariation := t.variationName() - mctx.SetDefaultDependencyVariation(&defaultVariation) - modules := mctx.CreateVariations("", t.variationName()) - modules[0].(*Module).linker.(snapshotSanitizer).setSanitizerVariation(t, false) - modules[1].(*Module).linker.(snapshotSanitizer).setSanitizerVariation(t, true) - - // Export the static lib name to make - if c.static() && c.ExportedToMake() { - if t == cfi { - // use BaseModuleName which is the name for Make. - cfiStaticLibs(mctx.Config()).add(c, c.BaseModuleName()) - } - } - } - } - } -} - type sanitizerStaticLibsMap struct { // libsMap contains one list of modules per each image and each arch. // e.g. libs[vendor]["arm"] contains arm modules installed to vendor diff --git a/cc/sanitize_test.go b/cc/sanitize_test.go index c1ca03408..5d7e7d80a 100644 --- a/cc/sanitize_test.go +++ b/cc/sanitize_test.go @@ -16,6 +16,7 @@ package cc import ( "fmt" + "runtime" "strings" "testing" @@ -201,6 +202,125 @@ func TestAsan(t *testing.T) { t.Run("device", func(t *testing.T) { check(t, result, "android_arm64_armv8-a") }) } +func TestUbsan(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + + bp := ` + cc_binary { + name: "bin_with_ubsan", + host_supported: true, + shared_libs: [ + "libshared", + ], + static_libs: [ + "libstatic", + "libnoubsan", + ], + sanitize: { + undefined: true, + } + } + + cc_binary { + name: "bin_depends_ubsan", + host_supported: true, + shared_libs: [ + "libshared", + ], + static_libs: [ + "libstatic", + "libubsan", + "libnoubsan", + ], + } + + cc_binary { + name: "bin_no_ubsan", + host_supported: true, + shared_libs: [ + "libshared", + ], + static_libs: [ + "libstatic", + "libnoubsan", + ], + } + + cc_library_shared { + name: "libshared", + host_supported: true, + shared_libs: ["libtransitive"], + } + + cc_library_shared { + name: "libtransitive", + host_supported: true, + } + + cc_library_static { + name: "libubsan", + host_supported: true, + sanitize: { + undefined: true, + } + } + + cc_library_static { + name: "libstatic", + host_supported: true, + } + + cc_library_static { + name: "libnoubsan", + host_supported: true, + } + ` + + result := android.GroupFixturePreparers( + prepareForCcTest, + ).RunTestWithBp(t, bp) + + check := func(t *testing.T, result *android.TestResult, variant string) { + staticVariant := variant + "_static" + + minimalRuntime := result.ModuleForTests("libclang_rt.ubsan_minimal", staticVariant) + + // The binaries, one with ubsan and one without + binWithUbsan := result.ModuleForTests("bin_with_ubsan", variant) + binDependsUbsan := result.ModuleForTests("bin_depends_ubsan", variant) + binNoUbsan := result.ModuleForTests("bin_no_ubsan", variant) + + android.AssertStringListContains(t, "missing libclang_rt.ubsan_minimal in bin_with_ubsan static libs", + strings.Split(binWithUbsan.Rule("ld").Args["libFlags"], " "), + minimalRuntime.OutputFiles(t, "")[0].String()) + + android.AssertStringListContains(t, "missing libclang_rt.ubsan_minimal in bin_depends_ubsan static libs", + strings.Split(binDependsUbsan.Rule("ld").Args["libFlags"], " "), + minimalRuntime.OutputFiles(t, "")[0].String()) + + android.AssertStringListDoesNotContain(t, "unexpected libclang_rt.ubsan_minimal in bin_no_ubsan static libs", + strings.Split(binNoUbsan.Rule("ld").Args["libFlags"], " "), + minimalRuntime.OutputFiles(t, "")[0].String()) + + android.AssertStringListContains(t, "missing -Wl,--exclude-libs for minimal runtime in bin_with_ubsan", + strings.Split(binWithUbsan.Rule("ld").Args["ldFlags"], " "), + "-Wl,--exclude-libs="+minimalRuntime.OutputFiles(t, "")[0].Base()) + + android.AssertStringListContains(t, "missing -Wl,--exclude-libs for minimal runtime in bin_depends_ubsan static libs", + strings.Split(binDependsUbsan.Rule("ld").Args["ldFlags"], " "), + "-Wl,--exclude-libs="+minimalRuntime.OutputFiles(t, "")[0].Base()) + + android.AssertStringListDoesNotContain(t, "unexpected -Wl,--exclude-libs for minimal runtime in bin_no_ubsan static libs", + strings.Split(binNoUbsan.Rule("ld").Args["ldFlags"], " "), + "-Wl,--exclude-libs="+minimalRuntime.OutputFiles(t, "")[0].Base()) + } + + t.Run("host", func(t *testing.T) { check(t, result, result.Config.BuildOSTarget.String()) }) + t.Run("device", func(t *testing.T) { check(t, result, "android_arm64_armv8-a") }) +} + type MemtagNoteType int const ( diff --git a/cc/testing.go b/cc/testing.go index ecdae8b15..077fcda3c 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -73,7 +73,6 @@ func commonDefaultModules() string { nocrt: true, system_shared_libs: [], stl: "none", - srcs: [""], check_elf_files: false, sanitize: { never: true, @@ -84,6 +83,7 @@ func commonDefaultModules() string { name: "libcompiler_rt-extras", defaults: ["toolchain_libs_defaults"], vendor_ramdisk_available: true, + srcs: [""], } cc_prebuilt_library_static { @@ -93,11 +93,13 @@ func commonDefaultModules() string { vendor_available: true, vendor_ramdisk_available: true, native_bridge_supported: true, + srcs: [""], } cc_prebuilt_library_shared { name: "libclang_rt.hwasan", defaults: ["toolchain_libs_defaults"], + srcs: [""], } cc_prebuilt_library_static { @@ -108,6 +110,7 @@ func commonDefaultModules() string { ], vendor_ramdisk_available: true, native_bridge_supported: true, + srcs: [""], } cc_prebuilt_library_static { @@ -116,17 +119,34 @@ func commonDefaultModules() string { "linux_bionic_supported", "toolchain_libs_defaults", ], + srcs: [""], } // Needed for sanitizer cc_prebuilt_library_shared { name: "libclang_rt.ubsan_standalone", defaults: ["toolchain_libs_defaults"], + srcs: [""], } cc_prebuilt_library_static { name: "libclang_rt.ubsan_minimal", defaults: ["toolchain_libs_defaults"], + host_supported: true, + target: { + android_arm64: { + srcs: ["libclang_rt.ubsan_minimal.android_arm64.a"], + }, + android_arm: { + srcs: ["libclang_rt.ubsan_minimal.android_arm.a"], + }, + linux_glibc_x86_64: { + srcs: ["libclang_rt.ubsan_minimal.x86_64.a"], + }, + linux_glibc_x86: { + srcs: ["libclang_rt.ubsan_minimal.x86.a"], + }, + }, } cc_library { @@ -546,6 +566,11 @@ var PrepareForTestWithCcDefaultModules = android.GroupFixturePreparers( "defaults/cc/common/crtend_so.c": nil, "defaults/cc/common/crtend.c": nil, "defaults/cc/common/crtbrand.c": nil, + + "defaults/cc/common/libclang_rt.ubsan_minimal.android_arm64.a": nil, + "defaults/cc/common/libclang_rt.ubsan_minimal.android_arm.a": nil, + "defaults/cc/common/libclang_rt.ubsan_minimal.x86_64.a": nil, + "defaults/cc/common/libclang_rt.ubsan_minimal.x86.a": nil, }.AddToFixture(), // Place the default cc test modules that are common to all platforms in a location that will not diff --git a/java/app.go b/java/app.go index c5d88e978..8ff5d91c2 100755 --- a/java/app.go +++ b/java/app.go @@ -583,6 +583,16 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) + var noticeAssetPath android.WritablePath + if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { + // The rule to create the notice file can't be generated yet, as the final output path + // for the apk isn't known yet. Add the path where the notice file will be generated to the + // aapt rules now before calling aaptBuildActions, the rule to create the notice file will + // be generated later. + noticeAssetPath = android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") + a.aapt.noticeFile = android.OptionalPathForPath(noticeAssetPath) + } + // Process all building blocks, from AAPT to certificates. a.aaptBuildActions(ctx) @@ -654,7 +664,8 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile) } - if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { + if a.aapt.noticeFile.Valid() { + // Generating the notice file rule has to be here after a.outputFile is known. noticeFile := android.PathForModuleOut(ctx, "NOTICE.html.gz") android.BuildNoticeHtmlOutputFromLicenseMetadata( ctx, noticeFile, "", "", @@ -663,13 +674,11 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { android.PathForModuleInstall(ctx).String() + "/", a.outputFile.String(), }) - noticeAssetPath := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("cp"). Input(noticeFile). Output(noticeAssetPath) builder.Build("notice_dir", "Building notice dir") - a.aapt.noticeFile = android.OptionalPathForPath(noticeAssetPath) } for _, split := range a.aapt.splits { diff --git a/java/config/config.go b/java/config/config.go index e728b7d00..1d4b242f9 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -159,7 +159,7 @@ func init() { pctx.HostBinToolVariable("ZipSyncCmd", "zipsync") pctx.HostBinToolVariable("ApiCheckCmd", "apicheck") pctx.HostBinToolVariable("D8Cmd", "d8") - pctx.HostBinToolVariable("R8Cmd", "r8-compat-proguard") + pctx.HostBinToolVariable("R8Cmd", "r8") pctx.HostBinToolVariable("HiddenAPICmd", "hiddenapi") pctx.HostBinToolVariable("ExtractApksCmd", "extract_apks") pctx.VariableFunc("TurbineJar", func(ctx android.PackageVarContext) string { @@ -177,7 +177,7 @@ func init() { pctx.HostJavaToolVariable("MetalavaJar", "metalava.jar") pctx.HostJavaToolVariable("DokkaJar", "dokka.jar") pctx.HostJavaToolVariable("JetifierJar", "jetifier.jar") - pctx.HostJavaToolVariable("R8Jar", "r8-compat-proguard.jar") + pctx.HostJavaToolVariable("R8Jar", "r8.jar") pctx.HostJavaToolVariable("D8Jar", "d8.jar") pctx.HostBinToolVariable("SoongJavacWrapper", "soong_javac_wrapper") diff --git a/java/java.go b/java/java.go index feb49ad1d..2897fd7f5 100644 --- a/java/java.go +++ b/java/java.go @@ -528,7 +528,7 @@ func normalizeJavaVersion(ctx android.BaseModuleContext, javaVersion string) jav case "11": return JAVA_VERSION_11 case "17": - return JAVA_VERSION_11 + return JAVA_VERSION_17 case "10", "12", "13", "14", "15", "16": ctx.PropertyErrorf("java_version", "Java language level %s is not supported", javaVersion) return JAVA_VERSION_UNSUPPORTED diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index e59146be1..2707f0c5e 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -830,21 +830,13 @@ func (ctx *parseContext) handleSubConfig( pathPattern = append(pathPattern, chunk) } } - if pathPattern[0] == "" && len(ctx.includeTops) > 0 { - // If pattern starts from the top. restrict it to the directories where - // we know inherit-product uses dynamically calculated path. - for _, p := range ctx.includeTops { - pathPattern[0] = p - matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...) - } - } else { - matchingPaths = ctx.findMatchingPaths(pathPattern) + if len(pathPattern) == 1 { + pathPattern = append(pathPattern, "") } + matchingPaths = ctx.findMatchingPaths(pathPattern) needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0 } else if len(ctx.includeTops) > 0 { - for _, p := range ctx.includeTops { - matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{p, ""})...) - } + matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{"", ""})...) } else { return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} } @@ -872,17 +864,31 @@ func (ctx *parseContext) findMatchingPaths(pattern []string) []string { } // Create regular expression from the pattern - s_regexp := "^" + regexp.QuoteMeta(pattern[0]) + regexString := "^" + regexp.QuoteMeta(pattern[0]) for _, s := range pattern[1:] { - s_regexp += ".*" + regexp.QuoteMeta(s) + regexString += ".*" + regexp.QuoteMeta(s) } - s_regexp += "$" - rex := regexp.MustCompile(s_regexp) + regexString += "$" + rex := regexp.MustCompile(regexString) + + includeTopRegexString := "" + if len(ctx.includeTops) > 0 { + for i, top := range ctx.includeTops { + if i > 0 { + includeTopRegexString += "|" + } + includeTopRegexString += "^" + regexp.QuoteMeta(top) + } + } else { + includeTopRegexString = ".*" + } + + includeTopRegex := regexp.MustCompile(includeTopRegexString) // Now match var res []string for _, p := range files { - if rex.MatchString(p) { + if rex.MatchString(p) && includeTopRegex.MatchString(p) { res = append(res, p) } } diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index a09764c17..31555d3f5 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -1157,6 +1157,8 @@ $(call inherit-product,$(MY_PATH)/cfg.mk) #RBC# include_top vendor/foo1 $(call inherit-product,$(MY_OTHER_PATH)) #RBC# include_top vendor/foo1 +$(call inherit-product,vendor/$(MY_OTHER_PATH)) +#RBC# include_top vendor/foo1 $(foreach f,$(MY_MAKEFILES), \ $(call inherit-product,$(f))) `, @@ -1180,6 +1182,13 @@ def init(g, handle): if not _varmod_init: rblf.mkerror("product.mk", "Cannot find %s" % (g.get("MY_OTHER_PATH", ""))) rblf.inherit(handle, _varmod, _varmod_init) + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get("vendor/%s" % g.get("MY_OTHER_PATH", "")) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % ("vendor/%s" % g.get("MY_OTHER_PATH", ""))) + rblf.inherit(handle, _varmod, _varmod_init) for f in rblf.words(g.get("MY_MAKEFILES", "")): _entry = { "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), diff --git a/provenance/provenance_singleton.go b/provenance/provenance_singleton.go index d1cbd8f42..fbb6212ff 100644 --- a/provenance/provenance_singleton.go +++ b/provenance/provenance_singleton.go @@ -35,10 +35,10 @@ var ( mergeProvenanceMetaData = pctx.AndroidStaticRule("mergeProvenanceMetaData", blueprint.RuleParams{ - Command: `rm -rf $out $out.temp && ` + + Command: `rm -rf $out && ` + `echo "# proto-file: build/soong/provenance/proto/provenance_metadata.proto" > $out && ` + `echo "# proto-message: ProvenanceMetaDataList" >> $out && ` + - `touch $out.temp && cat $out.temp $in | grep -v "^#.*" >> $out && rm -rf $out.temp`, + `for file in $in; do echo '' >> $out; echo 'metadata {' | cat - $$file | grep -Ev "^#.*|^$$" >> $out; echo '}' >> $out; done`, }) ) diff --git a/rust/config/global.go b/rust/config/global.go index 647a7cfdb..e9751fd7a 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.61.0.p1" + RustDefaultVersion = "1.61.0.p2" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ diff --git a/rust/sanitize.go b/rust/sanitize.go index aadc00f31..536fcbd17 100644 --- a/rust/sanitize.go +++ b/rust/sanitize.go @@ -49,8 +49,7 @@ type SanitizeProperties struct { Memtag_heap *bool `android:"arch_variant"` } } - SanitizerEnabled bool `blueprint:"mutated"` - SanitizeDepTypes []cc.SanitizerType `blueprint:"mutated"` + SanitizerEnabled bool `blueprint:"mutated"` // Used when we need to place libraries in their own directory, such as ASAN. InSanitizerDir bool `blueprint:"mutated"` @@ -444,28 +443,12 @@ func (mod *Module) IsSanitizerExplicitlyDisabled(t cc.SanitizerType) bool { return mod.sanitize.isSanitizerExplicitlyDisabled(t) } -func (mod *Module) SanitizeDep(t cc.SanitizerType) bool { - for _, e := range mod.sanitize.Properties.SanitizeDepTypes { - if t == e { - return true - } - } - - return false -} - func (mod *Module) SetSanitizer(t cc.SanitizerType, b bool) { if !Bool(mod.sanitize.Properties.Sanitize.Never) { mod.sanitize.SetSanitizer(t, b) } } -func (c *Module) SetSanitizeDep(t cc.SanitizerType) { - if !c.SanitizeDep(t) { - c.sanitize.Properties.SanitizeDepTypes = append(c.sanitize.Properties.SanitizeDepTypes, t) - } -} - func (mod *Module) StaticallyLinked() bool { if lib, ok := mod.compiler.(libraryInterface); ok { return lib.rlib() || lib.static() diff --git a/tests/apex_comparison_tests.sh b/tests/apex_comparison_tests.sh new file mode 100755 index 000000000..4b2f795de --- /dev/null +++ b/tests/apex_comparison_tests.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Soong/Bazel integration test for building unbundled apexes in the real source tree. +# +# These tests build artifacts from head and compares their contents. + +if [ ! -e "build/make/core/Makefile" ]; then + echo "$0 must be run from the top of the Android source tree." + exit 1 +fi + +############ +# Test Setup +############ + +OUTPUT_DIR="$(mktemp -d)" +SOONG_OUTPUT_DIR="$OUTPUT_DIR/soong" +BAZEL_OUTPUT_DIR="$OUTPUT_DIR/bazel" + +function cleanup { + # call bazel clean because some bazel outputs don't have w bits. + call_bazel clean + rm -rf "${OUTPUT_DIR}" +} +trap cleanup EXIT + +########### +# Run Soong +########### +export UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true # don't rely on prebuilts +export TARGET_BUILD_APPS="com.android.adbd com.android.tzdata build.bazel.examples.apex.minimal" +packages/modules/common/build/build_unbundled_mainline_module.sh \ + --product module_arm \ + --dist_dir "$SOONG_OUTPUT_DIR" + +###################### +# Run bp2build / Bazel +###################### +build/soong/soong_ui.bash --make-mode BP2BUILD_VERBOSE=1 --skip-soong-tests bp2build + +function call_bazel() { + tools/bazel --output_base="$BAZEL_OUTPUT_DIR" $@ +} +BAZEL_OUT="$(call_bazel info output_path)" + +call_bazel build --config=bp2build --config=ci --config=android_arm \ + //packages/modules/adb/apex:com.android.adbd \ + //system/timezone/apex:com.android.tzdata \ + //build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex + +# Build debugfs separately, as it's not a dep of apexer, but needs to be an explicit arg. +call_bazel build --config=bp2build --config=linux_x86_64 //external/e2fsprogs/debugfs +DEBUGFS_PATH="$BAZEL_OUT/linux_x86_64-fastbuild/bin/external/e2fsprogs/debugfs/debugfs" + +function run_deapexer() { + call_bazel run --config=bp2build --config=linux_x86_64 //system/apex/tools:deapexer \ + -- \ + --debugfs_path="$DEBUGFS_PATH" \ + $@ +} + +####### +# Tests +####### + +function compare_deapexer_list() { + local APEX_DIR=$1; shift + local APEX=$1; shift + + # Compare the outputs of `deapexer list`, which lists the contents of the apex filesystem image. + local SOONG_APEX="$SOONG_OUTPUT_DIR/$APEX" + local BAZEL_APEX="$BAZEL_OUT/android_arm-fastbuild/bin/$APEX_DIR/$APEX" + + local SOONG_LIST="$OUTPUT_DIR/soong.list" + local BAZEL_LIST="$OUTPUT_DIR/bazel.list" + + run_deapexer list "$SOONG_APEX" > "$SOONG_LIST" + run_deapexer list "$BAZEL_APEX" > "$BAZEL_LIST" + + if cmp -s "$SOONG_LIST" "$BAZEL_LIST" + then + echo "ok: $APEX" + else + echo "contents of $APEX are different between Soong and Bazel:" + echo + echo expected + echo + cat "$SOONG_LIST" + echo + echo got + echo + cat "$BAZEL_LIST" + exit 1 + fi +} + +compare_deapexer_list packages/modules/adb/apex com.android.adbd.apex +compare_deapexer_list system/timezone/apex com.android.tzdata.apex +compare_deapexer_list build/bazel/examples/apex/minimal build.bazel.examples.apex.minimal.apex diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index 76a918be4..1e0772797 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -9,3 +9,7 @@ TOP="$(readlink -f "$(dirname "$0")"/../../..)" "$TOP/build/soong/tests/bp2build_bazel_test.sh" "$TOP/build/soong/tests/soong_test.sh" "$TOP/build/bazel/ci/rbc_regression_test.sh" aosp_arm64-userdebug + +# The following tests build against the full source tree and don't rely on the +# mock client. +"$TOP/build/soong/tests/apex_comparison_tests.sh" diff --git a/ui/build/config.go b/ui/build/config.go index 5765f2123..59b01b30e 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -211,7 +211,7 @@ func loadEnvConfig(ctx Context, config *configImpl) error { } if err := fetchEnvConfig(ctx, config, bc); err != nil { - fmt.Fprintf(os.Stderr, "Failed to fetch config file: %v\n", err) + ctx.Verbosef("Failed to fetch config file: %v\n", err) } configDirs := []string{ |