diff options
53 files changed, 1395 insertions, 370 deletions
@@ -314,6 +314,9 @@ subpackages) can use this module. * `["//visibility:override"]`: Discards any rules inherited from defaults or a creating module. Can only be used at the beginning of a list of visibility rules. +* `["//visibility:any_partition"]`: Any modules of type android_filesystem +or android_system_image can use this module. Intended for modules that no one +should link against, but should still be included in soong-built partitions. * `["//some/package:__pkg__", "//other/package:__pkg__"]`: Only modules in `some/package` and `other/package` (defined in `some/package/*.bp` and `other/package/*.bp`) have access to this module. Note that sub-packages do not diff --git a/aconfig/aconfig_declarations.go b/aconfig/aconfig_declarations.go index 78f506a04..392e8193a 100644 --- a/aconfig/aconfig_declarations.go +++ b/aconfig/aconfig_declarations.go @@ -40,6 +40,9 @@ type DeclarationsModule struct { // Container(system/vendor/apex) that this module belongs to Container string + + // The flags will only be repackaged if this prop is true. + Exportable bool } intermediatePath android.WritablePath @@ -159,6 +162,7 @@ func (module *DeclarationsModule) GenerateAndroidBuildActions(ctx android.Module android.SetProvider(ctx, android.AconfigDeclarationsProviderKey, android.AconfigDeclarationsProviderData{ Package: module.properties.Package, Container: module.properties.Container, + Exportable: module.properties.Exportable, IntermediateCacheOutputPath: intermediateCacheFilePath, IntermediateDumpOutputPath: intermediateDumpFilePath, }) diff --git a/aconfig/aconfig_declarations_test.go b/aconfig/aconfig_declarations_test.go index d508af7a4..1fe3c862a 100644 --- a/aconfig/aconfig_declarations_test.go +++ b/aconfig/aconfig_declarations_test.go @@ -27,6 +27,7 @@ func TestAconfigDeclarations(t *testing.T) { name: "module_name", package: "com.example.package", container: "com.android.foo", + exportable: true, srcs: [ "foo.aconfig", "bar.aconfig", @@ -41,6 +42,7 @@ func TestAconfigDeclarations(t *testing.T) { depData, _ := android.SingletonModuleProvider(result, module, android.AconfigDeclarationsProviderKey) android.AssertStringEquals(t, "package", depData.Package, "com.example.package") android.AssertStringEquals(t, "container", depData.Container, "com.android.foo") + android.AssertBoolEquals(t, "exportable", depData.Exportable, true) if !strings.HasSuffix(depData.IntermediateCacheOutputPath.String(), "/intermediate.pb") { t.Errorf("Missing intermediates proto path in provider: %s", depData.IntermediateCacheOutputPath.String()) } @@ -48,3 +50,22 @@ func TestAconfigDeclarations(t *testing.T) { t.Errorf("Missing intermediates text path in provider: %s", depData.IntermediateDumpOutputPath.String()) } } + +func TestAconfigDeclarationsWithExportableUnset(t *testing.T) { + bp := ` + aconfig_declarations { + name: "module_name", + package: "com.example.package", + container: "com.android.foo", + srcs: [ + "foo.aconfig", + "bar.aconfig", + ], + } + ` + result := runTest(t, android.FixtureExpectsNoErrors, bp) + + module := result.ModuleForTests("module_name", "").Module().(*DeclarationsModule) + depData, _ := android.SingletonModuleProvider(result, module, android.AconfigDeclarationsProviderKey) + android.AssertBoolEquals(t, "exportable", depData.Exportable, false) +} diff --git a/aconfig/codegen/java_aconfig_library.go b/aconfig/codegen/java_aconfig_library.go index 4b8d3468e..d4c6da574 100644 --- a/aconfig/codegen/java_aconfig_library.go +++ b/aconfig/codegen/java_aconfig_library.go @@ -91,6 +91,12 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild if !isModeSupported(mode) { ctx.PropertyErrorf("mode", "%q is not a supported mode", mode) } + // TODO: uncomment this part after internal clean up + //if mode == "exported" && !declarations.Exportable { + // // if mode is exported, the corresponding aconfig_declaration must mark its + // // exportable property true + // ctx.PropertyErrorf("mode", "exported mode requires its aconfig_declaration has exportable prop true") + //} ctx.Build(pctx, android.BuildParams{ Rule: javaRule, @@ -102,12 +108,15 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild }, }) - // Mark our generated code as possibly needing jarjar repackaging - // TODO: Maybe control this with a property? - module.AddJarJarRenameRule(declarations.Package+".Flags", "") - module.AddJarJarRenameRule(declarations.Package+".FeatureFlags", "") - module.AddJarJarRenameRule(declarations.Package+".FeatureFlagsImpl", "") - module.AddJarJarRenameRule(declarations.Package+".FakeFeatureFlagsImpl", "") + if declarations.Exportable { + // Mark our generated code as possibly needing jarjar repackaging + // The repackaging only happens when the corresponding aconfig_declaration + // has property exportable true + module.AddJarJarRenameRule(declarations.Package+".Flags", "") + module.AddJarJarRenameRule(declarations.Package+".FeatureFlags", "") + module.AddJarJarRenameRule(declarations.Package+".FeatureFlagsImpl", "") + module.AddJarJarRenameRule(declarations.Package+".FakeFeatureFlagsImpl", "") + } return srcJarPath } diff --git a/aconfig/codegen/java_aconfig_library_test.go b/aconfig/codegen/java_aconfig_library_test.go index 85d2675d2..de45b5cc3 100644 --- a/aconfig/codegen/java_aconfig_library_test.go +++ b/aconfig/codegen/java_aconfig_library_test.go @@ -176,6 +176,7 @@ func testCodegenMode(t *testing.T, bpMode string, ruleMode string) { name: "my_aconfig_declarations", package: "com.example.package", srcs: ["foo.aconfig"], + exportable: true, } java_aconfig_library { diff --git a/aconfig/init.go b/aconfig/init.go index 3e9d29760..77f5ed363 100644 --- a/aconfig/init.go +++ b/aconfig/init.go @@ -43,7 +43,7 @@ var ( // For create-device-config-sysprops: Generate aconfig flag value map text file aconfigTextRule = pctx.AndroidStaticRule("aconfig_text", blueprint.RuleParams{ - Command: `${aconfig} dump-cache --format='{fully_qualified_name}={state:bool}'` + + Command: `${aconfig} dump-cache --dedup --format='{fully_qualified_name}={state:bool}'` + ` --cache ${in}` + ` --out ${out}.tmp` + ` && ( if cmp -s ${out}.tmp ${out} ; then rm ${out}.tmp ; else mv ${out}.tmp ${out} ; fi )`, @@ -56,7 +56,7 @@ var ( // For all_aconfig_declarations: Combine all parsed_flags proto files AllDeclarationsRule = pctx.AndroidStaticRule("All_aconfig_declarations_dump", blueprint.RuleParams{ - Command: `${aconfig} dump-cache --format protobuf --out ${out} ${cache_files}`, + Command: `${aconfig} dump-cache --dedup --format protobuf --out ${out} ${cache_files}`, CommandDeps: []string{ "${aconfig}", }, @@ -73,7 +73,7 @@ var ( blueprint.RuleParams{ Command: `rm -rf ${out}.tmp` + `&& for cache in ${cache_files}; do ` + - ` if [ -n "$$(${aconfig} dump-cache --cache $$cache --filter=is_exported:true --format='{fully_qualified_name}')" ]; then ` + + ` if [ -n "$$(${aconfig} dump-cache --dedup --cache $$cache --filter=is_exported:true --format='{fully_qualified_name}')" ]; then ` + ` ${aconfig} create-java-lib --cache $$cache --mode=exported --out ${out}.tmp; ` + ` fi ` + `done` + diff --git a/android/aconfig_providers.go b/android/aconfig_providers.go index 1444e7d8f..74c1a5ecc 100644 --- a/android/aconfig_providers.go +++ b/android/aconfig_providers.go @@ -35,6 +35,7 @@ var ( type AconfigDeclarationsProviderData struct { Package string Container string + Exportable bool IntermediateCacheOutputPath WritablePath IntermediateDumpOutputPath WritablePath } diff --git a/android/config.go b/android/config.go index d94a86f71..e57bc2cbd 100644 --- a/android/config.go +++ b/android/config.go @@ -28,6 +28,7 @@ import ( "strconv" "strings" "sync" + "unicode" "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" @@ -209,6 +210,12 @@ func (c Config) ReleaseDefaultModuleBuildFromSource() bool { Bool(c.config.productVariables.ReleaseDefaultModuleBuildFromSource) } +// Enables flagged apis annotated with READ_WRITE aconfig flags to be included in the stubs +// and hiddenapi flags so that they are accessible at runtime +func (c Config) ReleaseExportRuntimeApis() bool { + return c.config.productVariables.GetBuildFlagBool("RELEASE_EXPORT_RUNTIME_APIS") +} + // Enables ABI monitoring of NDK libraries func (c Config) ReleaseNdkAbiMonitored() bool { return c.config.productVariables.GetBuildFlagBool("RELEASE_NDK_ABI_MONITORED") @@ -320,6 +327,18 @@ func loadConfig(config *config) error { return loadFromConfigFile(&config.productVariables, absolutePath(config.ProductVariablesFileName)) } +// Checks if the string is a valid go identifier. This is equivalent to blueprint's definition +// of an identifier, so it will match the same identifiers as those that can be used in bp files. +func isGoIdentifier(ident string) bool { + for i, r := range ident { + valid := r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) && i > 0 + if !valid { + return false + } + } + return len(ident) > 0 +} + // loadFromConfigFile loads and decodes configuration options from a JSON file // in the current working directory. func loadFromConfigFile(configurable *ProductVariables, filename string) error { @@ -355,6 +374,20 @@ func loadFromConfigFile(configurable *ProductVariables, filename string) error { Bool(configurable.GcovCoverage) || Bool(configurable.ClangCoverage)) + // The go scanner's definition of identifiers is c-style identifiers, but allowing unicode's + // definition of letters and digits. This is the same scanner that blueprint uses, so it + // will allow the same identifiers as are valid in bp files. + for namespace := range configurable.VendorVars { + if !isGoIdentifier(namespace) { + return fmt.Errorf("soong config namespaces must be valid identifiers: %q", namespace) + } + for variable := range configurable.VendorVars[namespace] { + if !isGoIdentifier(variable) { + return fmt.Errorf("soong config variables must be valid identifiers: %q", variable) + } + } + } + // when Platform_sdk_final is true (or PLATFORM_VERSION_CODENAME is REL), use Platform_sdk_version; // if false (pre-released version, for example), use Platform_sdk_codename. if Bool(configurable.Platform_sdk_final) { @@ -1466,18 +1499,18 @@ func (c *deviceConfig) PgoAdditionalProfileDirs() []string { } // AfdoProfile returns fully qualified path associated to the given module name -func (c *deviceConfig) AfdoProfile(name string) (*string, error) { +func (c *deviceConfig) AfdoProfile(name string) (string, error) { for _, afdoProfile := range c.config.productVariables.AfdoProfiles { split := strings.Split(afdoProfile, ":") if len(split) != 3 { - return nil, fmt.Errorf("AFDO_PROFILES has invalid value: %s. "+ + return "", fmt.Errorf("AFDO_PROFILES has invalid value: %s. "+ "The expected format is <module>:<fully-qualified-path-to-fdo_profile>", afdoProfile) } if split[0] == name { - return proptools.StringPtr(strings.Join([]string{split[1], split[2]}, ":")), nil + return strings.Join([]string{split[1], split[2]}, ":"), nil } } - return nil, nil + return "", nil } func (c *deviceConfig) VendorSepolicyDirs() []string { @@ -1706,10 +1739,6 @@ func (c *deviceConfig) PlatformSepolicyVersion() string { return String(c.config.productVariables.PlatformSepolicyVersion) } -func (c *deviceConfig) TotSepolicyVersion() string { - return String(c.config.productVariables.TotSepolicyVersion) -} - func (c *deviceConfig) PlatformSepolicyCompatVersions() []string { return c.config.productVariables.PlatformSepolicyCompatVersions } diff --git a/android/packaging.go b/android/packaging.go index 8873540c5..a8fb28d30 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -85,6 +85,7 @@ type PackageModule interface { // GatherPackagingSpecs gathers PackagingSpecs of transitive dependencies. GatherPackagingSpecs(ctx ModuleContext) map[string]PackagingSpec + GatherPackagingSpecsWithFilter(ctx ModuleContext, filter func(PackagingSpec) bool) map[string]PackagingSpec // CopyDepsToZip zips the built artifacts of the dependencies into the given zip file and // returns zip entries in it. This is expected to be called in GenerateAndroidBuildActions, @@ -221,14 +222,18 @@ func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext, depTag blueprint.Dep } } -// See PackageModule.GatherPackagingSpecs -func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]PackagingSpec { +func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter func(PackagingSpec) bool) map[string]PackagingSpec { m := make(map[string]PackagingSpec) ctx.VisitDirectDeps(func(child Module) { if pi, ok := ctx.OtherModuleDependencyTag(child).(PackagingItem); !ok || !pi.IsPackagingItem() { return } for _, ps := range child.TransitivePackagingSpecs() { + if filter != nil { + if !filter(ps) { + continue + } + } if _, ok := m[ps.relPathInPackage]; !ok { m[ps.relPathInPackage] = ps } @@ -237,6 +242,11 @@ func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]Packa return m } +// See PackageModule.GatherPackagingSpecs +func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]PackagingSpec { + return p.GatherPackagingSpecsWithFilter(ctx, nil) +} + // CopySpecsToDir is a helper that will add commands to the rule builder to copy the PackagingSpec // entries into the specified directory. func (p *PackagingBase) CopySpecsToDir(ctx ModuleContext, builder *RuleBuilder, specs map[string]PackagingSpec, dir WritablePath) (entries []string) { diff --git a/android/rule_builder.go b/android/rule_builder.go index e8dbd48c6..85e29bd20 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -1157,11 +1157,15 @@ func (c *RuleBuilderCommand) Outputs(paths WritablePaths) *RuleBuilderCommand { // OutputDir adds the output directory to the command line. This is only available when used with RuleBuilder.Sbox, // and will be the temporary output directory managed by sbox, not the final one. -func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand { +func (c *RuleBuilderCommand) OutputDir(subPathComponents ...string) *RuleBuilderCommand { if !c.rule.sbox { panic("OutputDir only valid with Sbox") } - return c.Text(sboxOutDir) + path := sboxOutDir + if len(subPathComponents) > 0 { + path = filepath.Join(append([]string{sboxOutDir}, subPathComponents...)...) + } + return c.Text(path) } // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command diff --git a/android/sdk_version.go b/android/sdk_version.go index 9355667d1..b2ff960ee 100644 --- a/android/sdk_version.go +++ b/android/sdk_version.go @@ -393,6 +393,7 @@ func init() { // Export the name of the soong modules representing the various Java API surfaces. func javaSdkMakeVars(ctx MakeVarsContext) { ctx.Strict("ANDROID_PUBLIC_STUBS", SdkPublic.DefaultJavaLibraryName()) + ctx.Strict("ANDROID_PUBLIC_EXPORTABLE_STUBS", SdkPublic.DefaultExportableJavaLibraryName()) ctx.Strict("ANDROID_SYSTEM_STUBS", SdkSystem.DefaultJavaLibraryName()) ctx.Strict("ANDROID_TEST_STUBS", SdkTest.DefaultJavaLibraryName()) ctx.Strict("ANDROID_MODULE_LIB_STUBS", SdkModule.DefaultJavaLibraryName()) diff --git a/android/singleton.go b/android/singleton.go index e0e552ebc..ccddeafd2 100644 --- a/android/singleton.go +++ b/android/singleton.go @@ -250,8 +250,8 @@ func (s *singletonContextAdaptor) FinalModule(module Module) Module { } func (s *singletonContextAdaptor) ModuleVariantsFromName(referer Module, name string) []Module { - // get qualified module name for visibility enforcement - qualified := createQualifiedModuleName(s.ModuleName(referer), s.ModuleDir(referer)) + // get module reference for visibility enforcement + qualified := createVisibilityModuleReference(s.ModuleName(referer), s.ModuleDir(referer), s.ModuleType(referer)) modules := s.SingletonContext.ModuleVariantsFromName(referer, name) result := make([]Module, 0, len(modules)) @@ -262,7 +262,7 @@ func (s *singletonContextAdaptor) ModuleVariantsFromName(referer Module, name st depDir := s.ModuleDir(module) depQualified := qualifiedModuleName{depDir, depName} // Targets are always visible to other targets in their own package. - if depQualified.pkg != qualified.pkg { + if depQualified.pkg != qualified.name.pkg { rule := effectiveVisibilityRules(s.Config(), depQualified) if !rule.matches(qualified) { s.ModuleErrorf(referer, "module %q references %q which is not visible to this module\nYou may need to add %q to its visibility", diff --git a/android/test_suites.go b/android/test_suites.go index 9ded99832..adcc15a6e 100644 --- a/android/test_suites.go +++ b/android/test_suites.go @@ -24,6 +24,7 @@ func testSuiteFilesFactory() Singleton { type testSuiteFiles struct { robolectric WritablePath + ravenwood WritablePath } type TestSuiteModule interface { @@ -47,12 +48,15 @@ func (t *testSuiteFiles) GenerateBuildActions(ctx SingletonContext) { }) t.robolectric = robolectricTestSuite(ctx, files["robolectric-tests"]) - ctx.Phony("robolectric-tests", t.robolectric) + + t.ravenwood = ravenwoodTestSuite(ctx, files["ravenwood-tests"]) + ctx.Phony("ravenwood-tests", t.ravenwood) } func (t *testSuiteFiles) MakeVars(ctx MakeVarsContext) { ctx.DistForGoal("robolectric-tests", t.robolectric) + ctx.DistForGoal("ravenwood-tests", t.ravenwood) } func robolectricTestSuite(ctx SingletonContext, files map[string]InstallPaths) WritablePath { @@ -74,3 +78,23 @@ func robolectricTestSuite(ctx SingletonContext, files map[string]InstallPaths) W return outputFile } + +func ravenwoodTestSuite(ctx SingletonContext, files map[string]InstallPaths) WritablePath { + var installedPaths InstallPaths + for _, module := range SortedKeys(files) { + installedPaths = append(installedPaths, files[module]...) + } + testCasesDir := pathForInstall(ctx, ctx.Config().BuildOS, X86, "testcases") + + outputFile := PathForOutput(ctx, "packaging", "ravenwood-tests.zip") + rule := NewRuleBuilder(pctx, ctx) + rule.Command().BuiltTool("soong_zip"). + FlagWithOutput("-o ", outputFile). + FlagWithArg("-P ", "host/testcases"). + FlagWithArg("-C ", testCasesDir.String()). + FlagWithRspFileInputList("-r ", outputFile.ReplaceExtension(ctx, "rsp"), installedPaths.Paths()). + Flag("-sha256") + rule.Build("ravenwood_tests_zip", "ravenwood-tests.zip") + + return outputFile +} diff --git a/android/variable.go b/android/variable.go index a4917c54c..758967e23 100644 --- a/android/variable.go +++ b/android/variable.go @@ -383,7 +383,6 @@ type ProductVariables struct { BoardSepolicyVers *string `json:",omitempty"` PlatformSepolicyVersion *string `json:",omitempty"` - TotSepolicyVersion *string `json:",omitempty"` SystemExtSepolicyPrebuiltApiDir *string `json:",omitempty"` ProductSepolicyPrebuiltApiDir *string `json:",omitempty"` diff --git a/android/visibility.go b/android/visibility.go index 3130135f1..b3875620f 100644 --- a/android/visibility.go +++ b/android/visibility.go @@ -57,12 +57,29 @@ const ( var visibilityRuleRegexp = regexp.MustCompile(visibilityRulePattern) +type visibilityModuleReference struct { + name qualifiedModuleName + isPartitionModule bool +} + +func createVisibilityModuleReference(name, dir, typ string) visibilityModuleReference { + isPartitionModule := false + switch typ { + case "android_filesystem", "android_system_image": + isPartitionModule = true + } + return visibilityModuleReference{ + name: createQualifiedModuleName(name, dir), + isPartitionModule: isPartitionModule, + } +} + // A visibility rule is associated with a module and determines which other modules it is visible // to, i.e. which other modules can depend on the rule's module. type visibilityRule interface { // Check to see whether this rules matches m. // Returns true if it does, false otherwise. - matches(m qualifiedModuleName) bool + matches(m visibilityModuleReference) bool String() string } @@ -108,8 +125,10 @@ func (p visibilityPropertyImpl) getStrings() []string { // ["//visibility:private"]. type compositeRule []visibilityRule +var _ visibilityRule = compositeRule{} + // A compositeRule matches if and only if any of its rules matches. -func (c compositeRule) matches(m qualifiedModuleName) bool { +func (c compositeRule) matches(m visibilityModuleReference) bool { for _, r := range c { if r.matches(m) { return true @@ -135,8 +154,10 @@ type packageRule struct { pkg string } -func (r packageRule) matches(m qualifiedModuleName) bool { - return m.pkg == r.pkg +var _ visibilityRule = packageRule{} + +func (r packageRule) matches(m visibilityModuleReference) bool { + return m.name.pkg == r.pkg } func (r packageRule) String() string { @@ -149,8 +170,10 @@ type subpackagesRule struct { pkgPrefix string } -func (r subpackagesRule) matches(m qualifiedModuleName) bool { - return isAncestor(r.pkgPrefix, m.pkg) +var _ visibilityRule = subpackagesRule{} + +func (r subpackagesRule) matches(m visibilityModuleReference) bool { + return isAncestor(r.pkgPrefix, m.name.pkg) } func isAncestor(p1 string, p2 string) bool { @@ -168,7 +191,9 @@ func (r subpackagesRule) String() string { // visibilityRule for //visibility:public type publicRule struct{} -func (r publicRule) matches(_ qualifiedModuleName) bool { +var _ visibilityRule = publicRule{} + +func (r publicRule) matches(_ visibilityModuleReference) bool { return true } @@ -179,7 +204,9 @@ func (r publicRule) String() string { // visibilityRule for //visibility:private type privateRule struct{} -func (r privateRule) matches(_ qualifiedModuleName) bool { +var _ visibilityRule = privateRule{} + +func (r privateRule) matches(_ visibilityModuleReference) bool { return false } @@ -187,6 +214,19 @@ func (r privateRule) String() string { return "//visibility:private" } +// visibilityRule for //visibility:any_partition +type anyPartitionRule struct{} + +var _ visibilityRule = anyPartitionRule{} + +func (r anyPartitionRule) matches(m visibilityModuleReference) bool { + return m.isPartitionModule +} + +func (r anyPartitionRule) String() string { + return "//visibility:any_partition" +} + var visibilityRuleMap = NewOnceKey("visibilityRuleMap") // The map from qualifiedModuleName to visibilityRule. @@ -237,13 +277,10 @@ func RegisterVisibilityRuleEnforcer(ctx RegisterMutatorsContext) { // Checks the per-module visibility rule lists before defaults expansion. func visibilityRuleChecker(ctx BottomUpMutatorContext) { - qualified := createQualifiedModuleName(ctx.ModuleName(), ctx.ModuleDir()) - if m, ok := ctx.Module().(Module); ok { - visibilityProperties := m.visibilityProperties() - for _, p := range visibilityProperties { - if visibility := p.getStrings(); visibility != nil { - checkRules(ctx, qualified.pkg, p.getName(), visibility) - } + visibilityProperties := ctx.Module().visibilityProperties() + for _, p := range visibilityProperties { + if visibility := p.getStrings(); visibility != nil { + checkRules(ctx, ctx.ModuleDir(), p.getName(), visibility) } } } @@ -266,7 +303,7 @@ func checkRules(ctx BaseModuleContext, currentPkg, property string, visibility [ if pkg == "visibility" { switch name { - case "private", "public": + case "private", "public", "any_partition": case "legacy_public": ctx.PropertyErrorf(property, "//visibility:legacy_public must not be used") continue @@ -305,10 +342,7 @@ func checkRules(ctx BaseModuleContext, currentPkg, property string, visibility [ // // See ../README.md#Visibility for information on the format of the visibility rules. func visibilityRuleGatherer(ctx BottomUpMutatorContext) { - m, ok := ctx.Module().(Module) - if !ok { - return - } + m := ctx.Module() qualifiedModuleId := m.qualifiedModuleId(ctx) currentPkg := qualifiedModuleId.pkg @@ -355,6 +389,8 @@ func parseRules(ctx BaseModuleContext, currentPkg, property string, visibility [ hasNonPrivateRule = false // This does not actually create a rule so continue onto the next rule. continue + case "any_partition": + r = anyPartitionRule{} } } else { switch name { @@ -395,10 +431,7 @@ func parseRules(ctx BaseModuleContext, currentPkg, property string, visibility [ func isAllowedFromOutsideVendor(pkg string, name string) bool { if pkg == "vendor" { - if name == "__subpackages__" { - return true - } - return false + return name == "__subpackages__" } return !isAncestor("vendor", pkg) @@ -434,11 +467,7 @@ func splitRule(ctx BaseModuleContext, ruleExpression string, currentPkg, propert } func visibilityRuleEnforcer(ctx TopDownMutatorContext) { - if _, ok := ctx.Module().(Module); !ok { - return - } - - qualified := createQualifiedModuleName(ctx.ModuleName(), ctx.ModuleDir()) + qualified := createVisibilityModuleReference(ctx.ModuleName(), ctx.ModuleDir(), ctx.ModuleType()) // Visit all the dependencies making sure that this module has access to them all. ctx.VisitDirectDeps(func(dep Module) { @@ -453,7 +482,7 @@ func visibilityRuleEnforcer(ctx TopDownMutatorContext) { depQualified := qualifiedModuleName{depDir, depName} // Targets are always visible to other targets in their own package. - if depQualified.pkg == qualified.pkg { + if depQualified.pkg == qualified.name.pkg { return } @@ -478,7 +507,7 @@ func effectiveVisibilityRules(config Config, qualified qualifiedModuleName) comp if ok { rule = value.(compositeRule) } else { - rule = packageDefaultVisibility(config, qualified) + rule = packageDefaultVisibility(moduleToVisibilityRule, qualified) } // If no rule is specified then return the default visibility rule to avoid @@ -494,8 +523,7 @@ func createQualifiedModuleName(moduleName, dir string) qualifiedModuleName { return qualified } -func packageDefaultVisibility(config Config, moduleId qualifiedModuleName) compositeRule { - moduleToVisibilityRule := moduleToVisibilityRuleMap(config) +func packageDefaultVisibility(moduleToVisibilityRule *sync.Map, moduleId qualifiedModuleName) compositeRule { packageQualifiedId := moduleId.getContainingPackageId() for { value, ok := moduleToVisibilityRule.Load(packageQualifiedId) @@ -574,10 +602,12 @@ func EffectiveVisibilityRules(ctx BaseModuleContext, module Module) VisibilityRu rule := effectiveVisibilityRules(ctx.Config(), qualified) + currentModule := createVisibilityModuleReference(moduleName, dir, ctx.OtherModuleType(module)) + // Modules are implicitly visible to other modules in the same package, // without checking the visibility rules. Here we need to add that visibility // explicitly. - if !rule.matches(qualified) { + if !rule.matches(currentModule) { if len(rule) == 1 { if _, ok := rule[0].(privateRule); ok { // If the rule is //visibility:private we can't append another diff --git a/android/visibility_test.go b/android/visibility_test.go index a66f0b698..d4add7d32 100644 --- a/android/visibility_test.go +++ b/android/visibility_test.go @@ -1904,6 +1904,38 @@ var visibilityTests = []struct { }`), }, }, + { + name: "any_partition visibility works", + fs: MockFS{ + "top/Android.bp": []byte(` + android_filesystem { + name: "foo", + deps: ["bar"], + }`), + "top/nested/Android.bp": []byte(` + package(default_visibility=["//visibility:private"]) + mock_library { + name: "bar", + visibility: ["//visibility:any_partition"], + }`), + }, + }, + { + name: "any_partition visibility doesn't work for non-partitions", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "foo", + deps: ["bar"], + }`), + "top/nested/Android.bp": []byte(` + mock_library { + name: "bar", + visibility: ["//visibility:any_partition"], + }`), + }, + expectedErrors: []string{`module "foo" variant "android_common": depends on //top/nested:bar which is not visible to this module`}, + }, } func TestVisibility(t *testing.T) { @@ -1925,6 +1957,8 @@ func TestVisibility(t *testing.T) { ctx.RegisterModuleType("mock_library", newMockLibraryModule) ctx.RegisterModuleType("mock_parent", newMockParentFactory) ctx.RegisterModuleType("mock_defaults", defaultsFactory) + // For testing //visibility:any_partition. The module type doesn't matter, just that it's registered under the name "android_filesystem" + ctx.RegisterModuleType("android_filesystem", newMockLibraryModule) }), prepareForTestWithFakePrebuiltModules, // Add additional files to the mock filesystem diff --git a/apex/builder.go b/apex/builder.go index 1d86434d7..3d612196d 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -912,7 +912,7 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { var validations android.Paths validations = append(validations, runApexLinkerconfigValidation(ctx, unsignedOutputFile.OutputPath, imageDir.OutputPath)) // TODO(b/279688635) deapexer supports [ext4] - if suffix == imageApexSuffix && ext4 == a.payloadFsType { + if !a.testApex && suffix == imageApexSuffix && ext4 == a.payloadFsType { validations = append(validations, runApexSepolicyTests(ctx, unsignedOutputFile.OutputPath)) } if !a.testApex && len(a.properties.Unwanted_transitive_deps) > 0 { diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 399d9b92f..cebbae9a6 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -639,7 +639,7 @@ func (p *prebuiltCommon) createDeapexerModuleIfNeeded(ctx android.TopDownMutator return false } - name := android.RemoveOptionalPrebuiltPrefix(ctx.OtherModuleName(child)) + name := java.ModuleStemForDeapexing(child) if _, ok := tag.(android.RequiresFilesFromPrebuiltApexTag); ok { commonModules = append(commonModules, name) diff --git a/bin/soongdbg b/bin/soongdbg index 132a0f0b1..bfdbbde70 100755 --- a/bin/soongdbg +++ b/bin/soongdbg @@ -2,9 +2,12 @@ import argparse import fnmatch +import html +import io import json import os import pathlib +import subprocess import types import sys @@ -27,6 +30,7 @@ class Graph: dep = get_or_make_node(self.nodes, d.id, None) node.deps.add(dep) dep.rdeps.add(node) + node.dep_tags.setdefault(dep, list()).append(d) def find_paths(self, id1, id2): # Throws KeyError if one of the names isn't found @@ -60,6 +64,7 @@ class Node: self.module = module self.deps = set() self.rdeps = set() + self.dep_tags = {} PROVIDERS = [ @@ -68,21 +73,35 @@ PROVIDERS = [ ] -def format_node_label(node): - if not node.module: - return node.id - if node.module.debug: - module_debug = f"<tr><td>{node.module.debug}</td></tr>" - else: - module_debug = "" - - result = (f"<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">" - + f"<tr><td><b>{node.module.name}</b></td></tr>" - + module_debug - + f"<tr><td>{node.module.type}</td></tr>") - for p in node.module.providers: - if p.type in PROVIDERS: - result += "<tr><td><font color=\"#666666\">" + format_provider(p) + "</font></td></tr>" +def format_dep_label(node, dep): + tags = node.dep_tags.get(dep) + labels = [] + if tags: + labels = [tag.tag_type.split("/")[-1] for tag in tags] + labels = sorted(set(labels)) + if labels: + result = "<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">" + for label in labels: + result += f"<tr><td>{label}</td></tr>" + result += "</table>>" + return result + + +def format_node_label(node, module_formatter): + result = "<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">" + + # node name + result += f"<tr><td><b>{node.module.name if node.module else node.id}</b></td></tr>" + + if node.module: + # node_type + result += f"<tr><td>{node.module.type}</td></tr>" + + # module_formatter will return a list of rows + for row in module_formatter(node.module): + row = html.escape(row) + result += f"<tr><td><font color=\"#666666\">{row}</font></td></tr>" + result += "</table>>" return result @@ -190,22 +209,87 @@ def load_and_filter_modules(args): yield m -def print_nodes(nodes): - print("digraph {") +def print_args(parser): + parser.add_argument("--label", action="append", metavar="JQ_FILTER", + help="jq query for each module metadata") + parser.add_argument("--deptags", action="store_true", + help="show dependency tags (makes the graph much more complex)") + + group = parser.add_argument_group("output formats", + "If no format is provided, a dot file will be written to" + + " stdout.") + output = group.add_mutually_exclusive_group() + output.add_argument("--dot", type=str, metavar="FILENAME", + help="Write the graph to this file as dot (graphviz format)") + output.add_argument("--svg", type=str, metavar="FILENAME", + help="Write the graph to this file as svg") + + +def print_nodes(args, nodes, module_formatter): + # Generate the graphviz + dep_tag_id = 0 + dot = io.StringIO() + dot.write("digraph {\n") + dot.write("node [shape=box];") + for node in nodes: - print(f"\"{node.id}\"[label={format_node_label(node)}];") + dot.write(f"\"{node.id}\" [label={format_node_label(node, module_formatter)}];\n") for dep in node.deps: if dep in nodes: - print(f"\"{node.id}\" -> \"{dep.id}\";") - print("}") + if args.deptags: + dot.write(f"\"{node.id}\" -> \"__dep_tag_{dep_tag_id}\" [ arrowhead=none ];\n") + dot.write(f"\"__dep_tag_{dep_tag_id}\" -> \"{dep.id}\";\n") + dot.write(f"\"__dep_tag_{dep_tag_id}\"" + + f"[label={format_dep_label(node, dep)} shape=ellipse" + + " color=\"#666666\" fontcolor=\"#666666\"];\n") + else: + dot.write(f"\"{node.id}\" -> \"{dep.id}\";\n") + dep_tag_id += 1 + dot.write("}\n") + text = dot.getvalue() + + # Write it somewhere + if args.dot: + with open(args.dot, "w") as f: + f.write(text) + elif args.svg: + subprocess.run(["dot", "-Tsvg", "-o", args.svg], + input=text, text=True, check=True) + else: + sys.stdout.write(text) -def get_deps(nodes, root): +def get_deps(nodes, root, maxdepth, reverse): if root in nodes: return nodes.add(root) - for dep in root.deps: - get_deps(nodes, dep) + if maxdepth != 0: + for dep in (root.rdeps if reverse else root.deps): + get_deps(nodes, dep, maxdepth-1, reverse) + + +def new_module_formatter(args): + def module_formatter(module): + if not args.label: + return [] + result = [] + text = json.dumps(module, default=lambda o: o.__dict__) + for jq_filter in args.label: + proc = subprocess.run(["jq", jq_filter], + input=text, text=True, check=True, stdout=subprocess.PIPE) + if proc.stdout: + o = json.loads(proc.stdout) + if type(o) == list: + for row in o: + if row: + result.append(row) + elif type(o) == dict: + result.append(str(proc.stdout).strip()) + else: + if o: + result.append(str(o).strip()) + return result + return module_formatter class BetweenCommand: @@ -213,11 +297,13 @@ class BetweenCommand: def args(self, parser): parser.add_argument("module", nargs=2, - help="The two modules") + help="the two modules") + print_args(parser) def run(self, args): graph = load_graph() - print_nodes(graph.find_paths(args.module[0], args.module[1])) + print_nodes(args, graph.find_paths(args.module[0], args.module[1]), + new_module_formatter(args)) class DepsCommand: @@ -226,21 +312,26 @@ class DepsCommand: def args(self, parser): parser.add_argument("module", nargs="+", help="Module to print dependencies of") + parser.add_argument("--reverse", action="store_true", + help="traverse reverse dependencies") + parser.add_argument("--depth", type=int, default=-1, + help="max depth of dependencies (can keep the graph size reasonable)") + print_args(parser) def run(self, args): graph = load_graph() nodes = set() err = False - for id in sys.argv[3:]: + for id in args.module: root = graph.nodes.get(id) if not root: sys.stderr.write(f"error: Can't find root: {id}\n") err = True continue - get_deps(nodes, root) + get_deps(nodes, root, args.depth, args.reverse) if err: sys.exit(1) - print_nodes(nodes) + print_nodes(args, nodes, new_module_formatter(args)) class IdCommand: @@ -254,6 +345,25 @@ class IdCommand: print(m.id) +class JsonCommand: + help = "Print metadata about modules in json format" + + def args(self, parser): + module_selection_args(parser) + parser.add_argument("--list", action="store_true", + help="Print the results in a json list. If not set and multiple" + + " modules are matched, the output won't be valid json.") + + def run(self, args): + modules = load_and_filter_modules(args) + if args.list: + json.dump([m for m in modules], sys.stdout, indent=4, default=lambda o: o.__dict__) + else: + for m in modules: + json.dump(m, sys.stdout, indent=4, default=lambda o: o.__dict__) + print() + + class QueryCommand: help = "Query details about modules" @@ -275,6 +385,7 @@ COMMANDS = { "between": BetweenCommand(), "deps": DepsCommand(), "id": IdCommand(), + "json": JsonCommand(), "query": QueryCommand(), } diff --git a/cc/afdo.go b/cc/afdo.go index 79fbae157..00b22456f 100644 --- a/cc/afdo.go +++ b/cc/afdo.go @@ -21,29 +21,17 @@ import ( "android/soong/android" "github.com/google/blueprint" - "github.com/google/blueprint/proptools" ) // This flag needs to be in both CFlags and LdFlags to ensure correct symbol ordering const afdoFlagsFormat = "-fprofile-sample-use=%s -fprofile-sample-accurate" -func recordMissingAfdoProfileFile(ctx android.BaseModuleContext, missing string) { - getNamedMapForConfig(ctx.Config(), modulesMissingProfileFileKey).Store(missing, true) -} - -type afdoRdep struct { - VariationName *string - ProfilePath *string -} - type AfdoProperties struct { // Afdo allows developers self-service enroll for // automatic feedback-directed optimization using profile data. Afdo bool - FdoProfilePath *string `blueprint:"mutated"` - - AfdoRDeps []afdoRdep `blueprint:"mutated"` + AfdoDep bool `blueprint:"mutated"` } type afdo struct { @@ -62,13 +50,32 @@ func (afdo *afdo) begin(ctx BaseModuleContext) { } // afdoEnabled returns true for binaries and shared libraries -// that set afdo prop to True and there is a profile available +// that set afdo prop to True. func (afdo *afdo) afdoEnabled() bool { return afdo != nil && afdo.Properties.Afdo } +func (afdo *afdo) isAfdoCompile(ctx ModuleContext) bool { + fdoProfilePath := getFdoProfilePathFromDep(ctx) + return !ctx.Host() && (afdo.Properties.Afdo || afdo.Properties.AfdoDep) && (fdoProfilePath != "") +} + +func getFdoProfilePathFromDep(ctx ModuleContext) string { + fdoProfileDeps := ctx.GetDirectDepsWithTag(FdoProfileTag) + if len(fdoProfileDeps) > 0 && fdoProfileDeps[0] != nil { + if info, ok := android.OtherModuleProvider(ctx, fdoProfileDeps[0], FdoProfileProvider); ok { + return info.Path.String() + } + } + return "" +} + func (afdo *afdo) flags(ctx ModuleContext, flags Flags) Flags { - if afdo.Properties.Afdo { + if ctx.Host() { + return flags + } + + if afdo.Properties.Afdo || afdo.Properties.AfdoDep { // We use `-funique-internal-linkage-names` to associate profiles to the right internal // functions. This option should be used before generating a profile. Because a profile // generated for a binary without unique names doesn't work well building a binary with @@ -86,15 +93,15 @@ func (afdo *afdo) flags(ctx ModuleContext, flags Flags) Flags { // TODO(b/266595187): Remove the following feature once it is enabled in LLVM by default. flags.Local.CFlags = append([]string{"-mllvm", "-improved-fs-discriminator=true"}, flags.Local.CFlags...) } - if path := afdo.Properties.FdoProfilePath; path != nil { + if fdoProfilePath := getFdoProfilePathFromDep(ctx); fdoProfilePath != "" { // The flags are prepended to allow overriding. - profileUseFlag := fmt.Sprintf(afdoFlagsFormat, *path) + profileUseFlag := fmt.Sprintf(afdoFlagsFormat, fdoProfilePath) flags.Local.CFlags = append([]string{profileUseFlag}, flags.Local.CFlags...) flags.Local.LdFlags = append([]string{profileUseFlag, "-Wl,-mllvm,-no-warn-sample-unused=true"}, flags.Local.LdFlags...) // Update CFlagsDeps and LdFlagsDeps so the module is rebuilt // if profileFile gets updated - pathForSrc := android.PathForSource(ctx, *path) + pathForSrc := android.PathForSource(ctx, fdoProfilePath) flags.CFlagsDeps = append(flags.CFlagsDeps, pathForSrc) flags.LdFlagsDeps = append(flags.LdFlagsDeps, pathForSrc) } @@ -102,122 +109,86 @@ func (afdo *afdo) flags(ctx ModuleContext, flags Flags) Flags { return flags } -func (afdo *afdo) addDep(ctx BaseModuleContext, actx android.BottomUpMutatorContext) { - if ctx.Host() { - return +func (a *afdo) addDep(ctx android.BottomUpMutatorContext, fdoProfileTarget string) { + if fdoProfileName, err := ctx.DeviceConfig().AfdoProfile(fdoProfileTarget); fdoProfileName != "" && err == nil { + ctx.AddFarVariationDependencies( + []blueprint.Variation{ + {Mutator: "arch", Variation: ctx.Target().ArchVariation()}, + {Mutator: "os", Variation: "android"}, + }, + FdoProfileTag, + fdoProfileName) } +} - if ctx.static() && !ctx.staticBinary() { - return +func afdoPropagateViaDepTag(tag blueprint.DependencyTag) bool { + libTag, isLibTag := tag.(libraryDependencyTag) + // Do not recurse down non-static dependencies + if isLibTag { + return libTag.static() + } else { + return tag == objDepTag || tag == reuseObjTag || tag == staticVariantTag } +} - if c, ok := ctx.Module().(*Module); ok && c.Enabled() { - if fdoProfileName, err := actx.DeviceConfig().AfdoProfile(actx.ModuleName()); fdoProfileName != nil && err == nil { - actx.AddFarVariationDependencies( - []blueprint.Variation{ - {Mutator: "arch", Variation: actx.Target().ArchVariation()}, - {Mutator: "os", Variation: "android"}, - }, - FdoProfileTag, - []string{*fdoProfileName}..., - ) - } - } +// afdoTransitionMutator creates afdo variants of cc modules. +type afdoTransitionMutator struct{} + +func (a *afdoTransitionMutator) Split(ctx android.BaseModuleContext) []string { + return []string{""} } -// FdoProfileMutator reads the FdoProfileProvider from a direct dep with FdoProfileTag -// assigns FdoProfileInfo.Path to the FdoProfilePath mutated property -func (c *Module) fdoProfileMutator(ctx android.BottomUpMutatorContext) { - if !c.Enabled() { - return +func (a *afdoTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + if ctx.Host() { + return "" } - if !c.afdo.afdoEnabled() { - return - } + if m, ok := ctx.Module().(*Module); ok && m.afdo != nil { + if !afdoPropagateViaDepTag(ctx.DepTag()) { + return "" + } - ctx.VisitDirectDepsWithTag(FdoProfileTag, func(m android.Module) { - if info, ok := android.OtherModuleProvider(ctx, m, FdoProfileProvider); ok { - c.afdo.Properties.FdoProfilePath = proptools.StringPtr(info.Path.String()) + if sourceVariation != "" { + return sourceVariation } - }) -} - -var _ FdoProfileMutatorInterface = (*Module)(nil) - -// Propagate afdo requirements down from binaries and shared libraries -func afdoDepsMutator(mctx android.TopDownMutatorContext) { - if m, ok := mctx.Module().(*Module); ok && m.afdo.afdoEnabled() { - path := m.afdo.Properties.FdoProfilePath - mctx.WalkDeps(func(dep android.Module, parent android.Module) bool { - tag := mctx.OtherModuleDependencyTag(dep) - libTag, isLibTag := tag.(libraryDependencyTag) - - // Do not recurse down non-static dependencies - if isLibTag { - if !libTag.static() { - return false - } - } else { - if tag != objDepTag && tag != reuseObjTag { - return false - } - } - if dep, ok := dep.(*Module); ok { - dep.afdo.Properties.AfdoRDeps = append( - dep.afdo.Properties.AfdoRDeps, - afdoRdep{ - VariationName: proptools.StringPtr(encodeTarget(m.Name())), - ProfilePath: path, - }, - ) - } + if !m.afdo.afdoEnabled() { + return "" + } - return true - }) + // TODO(b/324141705): this is designed to prevent propagating AFDO from static libraries that have afdo: true set, but + // it should be m.static() && !m.staticBinary() so that static binaries use AFDO variants of dependencies. + if m.static() { + return "" + } + + return encodeTarget(ctx.Module().Name()) } + return "" } -// Create afdo variants for modules that need them -func afdoMutator(mctx android.BottomUpMutatorContext) { - if m, ok := mctx.Module().(*Module); ok && m.afdo != nil { - if !m.static() && m.afdo.Properties.Afdo { - mctx.SetDependencyVariation(encodeTarget(m.Name())) - return - } - - variationNames := []string{""} - - variantNameToProfilePath := make(map[string]*string) - - for _, afdoRDep := range m.afdo.Properties.AfdoRDeps { - variantName := *afdoRDep.VariationName - // An rdep can be set twice in AfdoRDeps because there can be - // more than one path from an afdo-enabled module to - // a static dep such as - // afdo_enabled_foo -> static_bar ----> static_baz - // \ ^ - // ----------------------| - // We only need to create one variant per unique rdep - if _, exists := variantNameToProfilePath[variantName]; !exists { - variationNames = append(variationNames, variantName) - variantNameToProfilePath[variantName] = afdoRDep.ProfilePath - } - } +func (a *afdoTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if m, ok := ctx.Module().(*Module); ok && m.afdo != nil { + return incomingVariation + } + return "" +} - if len(variationNames) > 1 { - modules := mctx.CreateVariations(variationNames...) - for i, name := range variationNames { - if name == "" { - continue - } - variation := modules[i].(*Module) - variation.Properties.PreventInstall = true - variation.Properties.HideFromMake = true - variation.afdo.Properties.Afdo = true - variation.afdo.Properties.FdoProfilePath = variantNameToProfilePath[name] +func (a *afdoTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + if m, ok := ctx.Module().(*Module); ok && m.afdo != nil { + if variation == "" { + // The empty variation is either a module that has enabled AFDO for itself, or the non-AFDO + // variant of a dependency. + if m.afdo.afdoEnabled() && !(m.static() && !m.staticBinary()) && !m.Host() { + m.afdo.addDep(ctx, ctx.ModuleName()) } + } else { + // The non-empty variation is the AFDO variant of a dependency of a module that enabled AFDO + // for itself. + m.Properties.PreventInstall = true + m.Properties.HideFromMake = true + m.afdo.Properties.AfdoDep = true + m.afdo.addDep(ctx, decodeTarget(variation)) } } } diff --git a/cc/afdo_test.go b/cc/afdo_test.go index b250ad1a1..0679d1386 100644 --- a/cc/afdo_test.go +++ b/cc/afdo_test.go @@ -15,6 +15,7 @@ package cc import ( + "runtime" "strings" "testing" @@ -42,19 +43,25 @@ func TestAfdoDeps(t *testing.T) { bp := ` cc_library_shared { name: "libTest", + host_supported: true, srcs: ["test.c"], static_libs: ["libFoo"], afdo: true, + lto: { + thin: true, + }, } cc_library_static { name: "libFoo", + host_supported: true, srcs: ["foo.c"], static_libs: ["libBar"], } cc_library_static { name: "libBar", + host_supported: true, srcs: ["bar.c"], } ` @@ -72,13 +79,20 @@ func TestAfdoDeps(t *testing.T) { "afdo_profiles_package/Android.bp": []byte(` fdo_profile { name: "libTest_afdo", - profile: "libTest.afdo", + arch: { + arm64: { + profile: "libTest.afdo", + }, + }, } `), }.AddToFixture(), ).RunTestWithBp(t, bp) - expectedCFlag := "-fprofile-sample-use=afdo_profiles_package/libTest.afdo" + profileSampleCFlag := "-fprofile-sample-use=afdo_profiles_package/libTest.afdo" + uniqueInternalLinkageNamesCFlag := "-funique-internal-linkage-names" + afdoLtoLdFlag := "-Wl,-plugin-opt,-import-instr-limit=40" + noAfdoLtoLdFlag := "-Wl,-plugin-opt,-import-instr-limit=5" libTest := result.ModuleForTests("libTest", "android_arm64_armv8-a_shared") libFooAfdoVariant := result.ModuleForTests("libFoo", "android_arm64_armv8-a_static_afdo-libTest") @@ -86,18 +100,32 @@ func TestAfdoDeps(t *testing.T) { // Check cFlags of afdo-enabled module and the afdo-variant of its static deps cFlags := libTest.Rule("cc").Args["cFlags"] - if !strings.Contains(cFlags, expectedCFlag) { - t.Errorf("Expected 'libTest' to enable afdo, but did not find %q in cflags %q", expectedCFlag, cFlags) + if !strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected 'libTest' to enable afdo profile, but did not find %q in cflags %q", profileSampleCFlag, cFlags) + } + if !strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected 'libTest' to enable afdo, but did not find %q in cflags %q", profileSampleCFlag, cFlags) + } + + ldFlags := libTest.Rule("ld").Args["ldFlags"] + if !strings.Contains(ldFlags, afdoLtoLdFlag) { + t.Errorf("Expected 'libTest' to enable afdo, but did not find %q in ldflags %q", afdoLtoLdFlag, ldFlags) } cFlags = libFooAfdoVariant.Rule("cc").Args["cFlags"] - if !strings.Contains(cFlags, expectedCFlag) { - t.Errorf("Expected 'libFooAfdoVariant' to enable afdo, but did not find %q in cflags %q", expectedCFlag, cFlags) + if !strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected 'libFooAfdoVariant' to enable afdo profile, but did not find %q in cflags %q", profileSampleCFlag, cFlags) + } + if !strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected 'libFooAfdoVariant' to enable afdo, but did not find %q in cflags %q", profileSampleCFlag, cFlags) } cFlags = libBarAfdoVariant.Rule("cc").Args["cFlags"] - if !strings.Contains(cFlags, expectedCFlag) { - t.Errorf("Expected 'libBarAfdoVariant' to enable afdo, but did not find %q in cflags %q", expectedCFlag, cFlags) + if !strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected 'libBarAfdoVariant' to enable afdo profile, but did not find %q in cflags %q", profileSampleCFlag, cFlags) + } + if !strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected 'libBarAfdoVariant' to enable afdo, but did not find %q in cflags %q", profileSampleCFlag, cFlags) } // Check dependency edge from afdo-enabled module to static deps @@ -114,12 +142,18 @@ func TestAfdoDeps(t *testing.T) { libBar := result.ModuleForTests("libBar", "android_arm64_armv8-a_static") cFlags = libFoo.Rule("cc").Args["cFlags"] - if strings.Contains(cFlags, expectedCFlag) { - t.Errorf("Expected 'libFoo' to not enable afdo, but found %q in cflags %q", expectedCFlag, cFlags) + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected 'libFoo' to not enable afdo profile, but found %q in cflags %q", profileSampleCFlag, cFlags) + } + if strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected 'libFoo' to not enable afdo, but found %q in cflags %q", profileSampleCFlag, cFlags) } cFlags = libBar.Rule("cc").Args["cFlags"] - if strings.Contains(cFlags, expectedCFlag) { - t.Errorf("Expected 'libBar' to not enable afdo, but found %q in cflags %q", expectedCFlag, cFlags) + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected 'libBar' to not enable afdo profile, but found %q in cflags %q", profileSampleCFlag, cFlags) + } + if strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected 'libBar' to not enable afdo, but found %q in cflags %q", profileSampleCFlag, cFlags) } // Check dependency edges of static deps @@ -130,6 +164,104 @@ func TestAfdoDeps(t *testing.T) { if !hasDirectDep(result, libFoo.Module(), libBar.Module()) { t.Errorf("libFoo missing dependency on non-afdo variant of libBar") } + + // Verify that the arm variant does not have FDO since the fdo_profile module only has a profile for arm64 + libTest32 := result.ModuleForTests("libTest", "android_arm_armv7-a-neon_shared") + libFooAfdoVariant32 := result.ModuleForTests("libFoo", "android_arm_armv7-a-neon_static_afdo-libTest_lto-thin") + libBarAfdoVariant32 := result.ModuleForTests("libBar", "android_arm_armv7-a-neon_static_afdo-libTest_lto-thin") + + cFlags = libTest32.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected arm32 'libTest' not to enable afdo, but found %q in cflags %q", profileSampleCFlag, cFlags) + } + + // TODO(b/324141705): when the fdo_profile module doesn't provide a source file the dependencies don't get + // -funique-internal-linkage-names but the module does. + if !strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected arm32 'libTest' to enable -funique-internal-linkage-names but did not find %q in cflags %q", + uniqueInternalLinkageNamesCFlag, cFlags) + } + + ldFlags = libTest32.Rule("ld").Args["ldFlags"] + if !strings.Contains(ldFlags, noAfdoLtoLdFlag) { + t.Errorf("Expected arm32 'libTest' to not enable afdo, but did not find %q in ldflags %q", noAfdoLtoLdFlag, ldFlags) + } + if strings.Contains(ldFlags, afdoLtoLdFlag) { + t.Errorf("Expected arm32 'libTest' to not enable afdo, but found %q in ldflags %q", afdoLtoLdFlag, ldFlags) + } + + // Check dependency edge from afdo-enabled module to static deps + if !hasDirectDep(result, libTest32.Module(), libFooAfdoVariant32.Module()) { + t.Errorf("arm32 libTest missing dependency on afdo variant of libFoo") + } + + if !hasDirectDep(result, libFooAfdoVariant32.Module(), libBarAfdoVariant32.Module()) { + t.Errorf("arm32 libTest missing dependency on afdo variant of libBar") + } + + cFlags = libFooAfdoVariant32.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected arm32 'libFoo' to not enable afdo profile, but found %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + if !strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected arm32 'libFoo' to enable afdo, but did not find %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + cFlags = libBarAfdoVariant32.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected arm32 'libBar' to not enable afdo profile, but found %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + if !strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected arm32 'libBar' to enable afdo, but did not find %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + + // Verify that the host variants don't enable afdo + libTestHost := result.ModuleForTests("libTest", result.Config.BuildOSTarget.String()+"_shared") + libFooHost := result.ModuleForTests("libFoo", result.Config.BuildOSTarget.String()+"_static_lto-thin") + libBarHost := result.ModuleForTests("libBar", result.Config.BuildOSTarget.String()+"_static_lto-thin") + + cFlags = libTestHost.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected host 'libTest' to not enable afdo profile, but found %q in cflags %q", profileSampleCFlag, cFlags) + } + + if strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected host 'libTest' to not enable afdo but found %q in cflags %q", + uniqueInternalLinkageNamesCFlag, cFlags) + } + + if runtime.GOOS != "darwin" { + ldFlags := libTestHost.Rule("ld").Args["ldFlags"] + if !strings.Contains(ldFlags, noAfdoLtoLdFlag) { + t.Errorf("Expected host 'libTest' to not enable afdo, but did not find %q in ldflags %q", noAfdoLtoLdFlag, ldFlags) + } + if strings.Contains(ldFlags, afdoLtoLdFlag) { + t.Errorf("Expected host 'libTest' to not enable afdo, but found %q in ldflags %q", afdoLtoLdFlag, ldFlags) + } + } + + // Check dependency edge from afdo-enabled module to static deps + if !hasDirectDep(result, libTestHost.Module(), libFooHost.Module()) { + t.Errorf("host libTest missing dependency on non-afdo variant of libFoo") + } + + if !hasDirectDep(result, libFooHost.Module(), libBarHost.Module()) { + t.Errorf("host libTest missing dependency on non-afdo variant of libBar") + } + + cFlags = libFooHost.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected host 'libFoo' to not enable afdo profile, but found %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + if strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected host 'libFoo' to not enable afdo, but found %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + cFlags = libBarHost.Rule("cc").Args["cFlags"] + if strings.Contains(cFlags, profileSampleCFlag) { + t.Errorf("Expected host 'libBar' to not enable afdo profile, but found %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } + if strings.Contains(cFlags, uniqueInternalLinkageNamesCFlag) { + t.Errorf("Expected host 'libBar' to not enable afdo, but found %q in cflags %q", uniqueInternalLinkageNamesCFlag, cFlags) + } } func TestAfdoEnabledOnStaticDepNoAfdo(t *testing.T) { @@ -174,11 +306,11 @@ func TestAfdoEnabledOnStaticDepNoAfdo(t *testing.T) { libBar := result.ModuleForTests("libBar", "android_arm64_armv8-a_static").Module() if !hasDirectDep(result, libTest, libFoo.Module()) { - t.Errorf("libTest missing dependency on afdo variant of libFoo") + t.Errorf("libTest missing dependency on non-afdo variant of libFoo") } if !hasDirectDep(result, libFoo.Module(), libBar) { - t.Errorf("libFoo missing dependency on afdo variant of libBar") + t.Errorf("libFoo missing dependency on non-afdo variant of libBar") } fooVariants := result.ModuleVariantsForTests("foo") @@ -55,7 +55,6 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.BottomUp("test_per_src", TestPerSrcMutator).Parallel() ctx.BottomUp("version", versionMutator).Parallel() ctx.BottomUp("begin", BeginMutator).Parallel() - ctx.BottomUp("fdo_profile", fdoProfileMutator) }) ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { @@ -70,8 +69,7 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.Transition("coverage", &coverageTransitionMutator{}) - ctx.TopDown("afdo_deps", afdoDepsMutator) - ctx.BottomUp("afdo", afdoMutator).Parallel() + ctx.Transition("afdo", &afdoTransitionMutator{}) ctx.Transition("orderfile", &orderfileTransitionMutator{}) @@ -527,7 +525,7 @@ type ModuleContextIntf interface { selectedStl() string baseModuleName() string getVndkExtendsModuleName() string - isAfdoCompile() bool + isAfdoCompile(ctx ModuleContext) bool isOrderfileCompile() bool isCfi() bool isFuzzer() bool @@ -1381,9 +1379,9 @@ func (c *Module) IsVndk() bool { return false } -func (c *Module) isAfdoCompile() bool { +func (c *Module) isAfdoCompile(ctx ModuleContext) bool { if afdo := c.afdo; afdo != nil { - return afdo.Properties.FdoProfilePath != nil + return afdo.isAfdoCompile(ctx) } return false } @@ -1706,8 +1704,8 @@ func (ctx *moduleContextImpl) isVndk() bool { return ctx.mod.IsVndk() } -func (ctx *moduleContextImpl) isAfdoCompile() bool { - return ctx.mod.isAfdoCompile() +func (ctx *moduleContextImpl) isAfdoCompile(mctx ModuleContext) bool { + return ctx.mod.isAfdoCompile(mctx) } func (ctx *moduleContextImpl) isOrderfileCompile() bool { @@ -2351,10 +2349,6 @@ func (c *Module) beginMutator(actx android.BottomUpMutatorContext) { } ctx.ctx = ctx - if !actx.Host() || !ctx.static() || ctx.staticBinary() { - c.afdo.addDep(ctx, actx) - } - c.begin(ctx) } diff --git a/cc/config/darwin_host.go b/cc/config/darwin_host.go index b7895900d..47c61b08f 100644 --- a/cc/config/darwin_host.go +++ b/cc/config/darwin_host.go @@ -29,11 +29,6 @@ var ( "-fPIC", "-funwind-tables", - // Workaround differences in inttypes.h between host and target. - //See bug 12708004. - "-D__STDC_FORMAT_MACROS", - "-D__STDC_CONSTANT_MACROS", - "-isysroot ${macSdkRoot}", "-mmacosx-version-min=${macMinVersion}", "-DMACOSX_DEPLOYMENT_TARGET=${macMinVersion}", diff --git a/cc/config/global.go b/cc/config/global.go index c56261431..d3c265885 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -254,7 +254,7 @@ var ( "-Wno-pointer-to-int-cast", "-Werror=fortify-source", // http://b/315246135 temporarily disabled - "-Wno-error=unused-variable", + "-Wno-unused-variable", // http://b/315250603 temporarily disabled "-Wno-error=format", // Disabled because it produces many false positives. http://b/323050926 diff --git a/cc/config/x86_linux_host.go b/cc/config/x86_linux_host.go index f497bf9c2..9bc54d6b6 100644 --- a/cc/config/x86_linux_host.go +++ b/cc/config/x86_linux_host.go @@ -30,11 +30,6 @@ var ( "-D_FORTIFY_SOURCE=2", "-fstack-protector", - // Workaround differences in inttypes.h between host and target. - //See bug 12708004. - "-D__STDC_FORMAT_MACROS", - "-D__STDC_CONSTANT_MACROS", - "--gcc-toolchain=${LinuxGccRoot}", "-fstack-protector-strong", } diff --git a/cc/config/x86_windows_host.go b/cc/config/x86_windows_host.go index 561c50099..1e61b01d0 100644 --- a/cc/config/x86_windows_host.go +++ b/cc/config/x86_windows_host.go @@ -27,9 +27,8 @@ var ( "-DWIN32_LEAN_AND_MEAN", "-Wno-unused-parameter", - // Workaround differences in inttypes.h between host and target. - //See bug 12708004. - "-D__STDC_FORMAT_MACROS", + // Workaround differences in <stdint.h> between host and target. + // Context: http://b/12708004 "-D__STDC_CONSTANT_MACROS", // Use C99-compliant printf functions (%zd). diff --git a/cc/coverage.go b/cc/coverage.go index 43f5e0762..f6092e447 100644 --- a/cc/coverage.go +++ b/cc/coverage.go @@ -91,7 +91,7 @@ func getClangProfileLibraryName(ctx ModuleContextIntf) string { } func (cov *coverage) deps(ctx DepsContext, deps Deps) Deps { - if cov.Properties.NeedCoverageVariant { + if cov.Properties.NeedCoverageVariant && ctx.Device() { ctx.AddVariationDependencies([]blueprint.Variation{ {Mutator: "link", Variation: "static"}, }, CoverageDepTag, getGcovProfileLibraryName(ctx)) @@ -184,19 +184,22 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags if gcovCoverage { flags.Local.LdFlags = append(flags.Local.LdFlags, "--coverage") - coverage := ctx.GetDirectDepWithTag(getGcovProfileLibraryName(ctx), CoverageDepTag).(*Module) - deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) - - flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--wrap,getenv") + if ctx.Device() { + coverage := ctx.GetDirectDepWithTag(getGcovProfileLibraryName(ctx), CoverageDepTag).(*Module) + deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--wrap,getenv") + } } else if clangCoverage { flags.Local.LdFlags = append(flags.Local.LdFlags, profileInstrFlag) if EnableContinuousCoverage(ctx) { flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-mllvm=-runtime-counter-relocation") } - coverage := ctx.GetDirectDepWithTag(getClangProfileLibraryName(ctx), CoverageDepTag).(*Module) - deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) - flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--wrap,open") + if ctx.Device() { + coverage := ctx.GetDirectDepWithTag(getClangProfileLibraryName(ctx), CoverageDepTag).(*Module) + deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--wrap,open") + } } } @@ -204,7 +207,7 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags } func (cov *coverage) begin(ctx BaseModuleContext) { - if ctx.Host() { + if ctx.Host() && !ctx.Os().Linux() { // TODO(dwillemsen): because of -nodefaultlibs, we must depend on libclang_rt.profile-*.a // Just turn off for now. } else { diff --git a/cc/fdo_profile.go b/cc/fdo_profile.go index 0893da5e0..1a3395773 100644 --- a/cc/fdo_profile.go +++ b/cc/fdo_profile.go @@ -43,23 +43,10 @@ type FdoProfileInfo struct { } // FdoProfileProvider is used to provide path to an fdo profile -var FdoProfileProvider = blueprint.NewMutatorProvider[FdoProfileInfo]("fdo_profile") - -// FdoProfileMutatorInterface is the interface implemented by fdo_profile module type -// module types that can depend on an fdo_profile module -type FdoProfileMutatorInterface interface { - // FdoProfileMutator eithers set or get FdoProfileProvider - fdoProfileMutator(ctx android.BottomUpMutatorContext) -} - -var _ FdoProfileMutatorInterface = (*fdoProfile)(nil) +var FdoProfileProvider = blueprint.NewProvider[FdoProfileInfo]() // GenerateAndroidBuildActions of fdo_profile does not have any build actions -func (fp *fdoProfile) GenerateAndroidBuildActions(ctx android.ModuleContext) {} - -// FdoProfileMutator sets FdoProfileProvider to fdo_profile module -// or sets afdo.Properties.FdoProfilePath to path in FdoProfileProvider of the depended fdo_profile -func (fp *fdoProfile) fdoProfileMutator(ctx android.BottomUpMutatorContext) { +func (fp *fdoProfile) GenerateAndroidBuildActions(ctx android.ModuleContext) { if fp.properties.Profile != nil { path := android.PathForModuleSrc(ctx, *fp.properties.Profile) android.SetProvider(ctx, FdoProfileProvider, FdoProfileInfo{ @@ -68,14 +55,6 @@ func (fp *fdoProfile) fdoProfileMutator(ctx android.BottomUpMutatorContext) { } } -// fdoProfileMutator calls the generic fdoProfileMutator function of fdoProfileMutator -// which is implemented by cc and cc.FdoProfile -func fdoProfileMutator(ctx android.BottomUpMutatorContext) { - if f, ok := ctx.Module().(FdoProfileMutatorInterface); ok { - f.fdoProfileMutator(ctx) - } -} - func FdoProfileFactory() android.Module { m := &fdoProfile{} m.AddProperties(&m.properties) @@ -100,7 +100,7 @@ func (lto *lto) begin(ctx BaseModuleContext) { lto.Properties.LtoEnabled = ltoEnabled } -func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { +func (lto *lto) flags(ctx ModuleContext, flags Flags) Flags { // TODO(b/131771163): CFI and Fuzzer controls LTO flags by themselves. // This has be checked late because these properties can be mutated. if ctx.isCfi() || ctx.isFuzzer() { @@ -139,7 +139,7 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { // Reduce the inlining threshold for a better balance of binary size and // performance. if !ctx.Darwin() { - if ctx.isAfdoCompile() { + if ctx.isAfdoCompile(ctx) { ltoLdFlags = append(ltoLdFlags, "-Wl,-plugin-opt,-import-instr-limit=40") } else { ltoLdFlags = append(ltoLdFlags, "-Wl,-plugin-opt,-import-instr-limit=5") diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index 1aa6f6f39..2c5718063 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -342,7 +342,8 @@ func (oz *OutputZip) writeEntries(entries []string) error { func (oz *OutputZip) getUninitializedPythonPackages(inputZips []InputZip) ([]string, error) { // the runfiles packages needs to be populated with "__init__.py". // the runfiles dirs have been treated as packages. - allPackages := make(map[string]bool) + var allPackages []string // Using a slice to preserve input order. + seenPkgs := make(map[string]bool) initedPackages := make(map[string]bool) getPackage := func(path string) string { ret := filepath.Dir(path) @@ -369,16 +370,17 @@ func (oz *OutputZip) getUninitializedPythonPackages(inputZips []InputZip) ([]str initedPackages[pyPkg] = true } for pyPkg != "" { - if _, found := allPackages[pyPkg]; found { + if _, found := seenPkgs[pyPkg]; found { break } - allPackages[pyPkg] = true + seenPkgs[pyPkg] = true + allPackages = append(allPackages, pyPkg) pyPkg = getPackage(pyPkg) } } } noInitPackages := make([]string, 0) - for pyPkg := range allPackages { + for _, pyPkg := range allPackages { if _, found := initedPackages[pyPkg]; !found { noInitPackages = append(noInitPackages, pyPkg) } diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go index 64b08d018..17228c4e0 100644 --- a/cmd/merge_zips/merge_zips_test.go +++ b/cmd/merge_zips/merge_zips_test.go @@ -103,6 +103,7 @@ func TestMergeZips(t *testing.T) { stripFiles []string stripDirs []string jar bool + par bool sort bool ignoreDuplicates bool stripDirEntries bool @@ -265,6 +266,34 @@ func TestMergeZips(t *testing.T) { }, out: []testZipEntry{withoutTimestamp, a}, }, + { + name: "emulate par", + in: [][]testZipEntry{ + { + testZipEntry{name: "3/main.py"}, + testZipEntry{name: "c/main.py"}, + testZipEntry{name: "a/main.py"}, + testZipEntry{name: "2/main.py"}, + testZipEntry{name: "b/main.py"}, + testZipEntry{name: "1/main.py"}, + }, + }, + out: []testZipEntry{ + testZipEntry{name: "3/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, + testZipEntry{name: "c/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, + testZipEntry{name: "a/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, + testZipEntry{name: "2/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, + testZipEntry{name: "b/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, + testZipEntry{name: "1/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, + testZipEntry{name: "3/main.py", timestamp: jar.DefaultTime}, + testZipEntry{name: "c/main.py", timestamp: jar.DefaultTime}, + testZipEntry{name: "a/main.py", timestamp: jar.DefaultTime}, + testZipEntry{name: "2/main.py", timestamp: jar.DefaultTime}, + testZipEntry{name: "b/main.py", timestamp: jar.DefaultTime}, + testZipEntry{name: "1/main.py", timestamp: jar.DefaultTime}, + }, + par: true, + }, } for _, test := range testCases { @@ -280,7 +309,7 @@ func TestMergeZips(t *testing.T) { writer := zip.NewWriter(out) err := mergeZips(inputZips, writer, "", "", - test.sort, test.jar, false, test.stripDirEntries, test.ignoreDuplicates, + test.sort, test.jar, test.par, test.stripDirEntries, test.ignoreDuplicates, test.stripFiles, test.stripDirs, test.zipsToNotStrip) closeErr := writer.Close() diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index 2f6476c82..6612a6f09 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -50,8 +50,8 @@ type filesystem struct { // Function that builds extra files under the root directory and returns the files buildExtraFiles func(ctx android.ModuleContext, root android.OutputPath) android.OutputPaths - // Function that filters PackagingSpecs returned by PackagingBase.GatherPackagingSpecs() - filterPackagingSpecs func(specs map[string]android.PackagingSpec) + // Function that filters PackagingSpec in PackagingBase.GatherPackagingSpecs() + filterPackagingSpec func(spec android.PackagingSpec) bool output android.OutputPath installDir android.InstallPath @@ -493,10 +493,7 @@ func (f *filesystem) SignedOutputPath() android.Path { // Note that "apex" module installs its contents to "apex"(fake partition) as well // for symbol lookup by imitating "activated" paths. func (f *filesystem) gatherFilteredPackagingSpecs(ctx android.ModuleContext) map[string]android.PackagingSpec { - specs := f.PackagingBase.GatherPackagingSpecs(ctx) - if f.filterPackagingSpecs != nil { - f.filterPackagingSpecs(specs) - } + specs := f.PackagingBase.GatherPackagingSpecsWithFilter(ctx, f.filterPackagingSpec) return specs } diff --git a/filesystem/system_image.go b/filesystem/system_image.go index 75abf702e..34f4ffb6d 100644 --- a/filesystem/system_image.go +++ b/filesystem/system_image.go @@ -37,7 +37,7 @@ func systemImageFactory() android.Module { module := &systemImage{} module.AddProperties(&module.properties) module.filesystem.buildExtraFiles = module.buildExtraFiles - module.filesystem.filterPackagingSpecs = module.filterPackagingSpecs + module.filesystem.filterPackagingSpec = module.filterPackagingSpec initFilesystemModule(&module.filesystem) return module } @@ -73,10 +73,6 @@ func (s *systemImage) buildLinkerConfigFile(ctx android.ModuleContext, root andr // Filter the result of GatherPackagingSpecs to discard items targeting outside "system" partition. // Note that "apex" module installs its contents to "apex"(fake partition) as well // for symbol lookup by imitating "activated" paths. -func (s *systemImage) filterPackagingSpecs(specs map[string]android.PackagingSpec) { - for k, ps := range specs { - if ps.Partition() != "system" { - delete(specs, k) - } - } +func (s *systemImage) filterPackagingSpec(ps android.PackagingSpec) bool { + return ps.Partition() == "system" } diff --git a/java/Android.bp b/java/Android.bp index 2585cd23f..54b36ab60 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -67,6 +67,7 @@ bootstrap_go_package { "plugin.go", "prebuilt_apis.go", "proto.go", + "ravenwood.go", "robolectric.go", "rro.go", "sdk.go", @@ -107,6 +108,7 @@ bootstrap_go_package { "plugin_test.go", "prebuilt_apis_test.go", "proto_test.go", + "ravenwood_test.go", "rro_test.go", "sdk_library_test.go", "sdk_test.go", diff --git a/java/app.go b/java/app.go index 0c56d81fc..9be3f6d50 100755 --- a/java/app.go +++ b/java/app.go @@ -920,15 +920,39 @@ func collectAppDeps(ctx android.ModuleContext, app appDepsInterface, shouldCollectRecursiveNativeDeps bool, checkNativeSdkVersion bool) ([]jniLib, android.Paths, []Certificate) { - var jniLibs []jniLib - var prebuiltJniPackages android.Paths - var certificates []Certificate - seenModulePaths := make(map[string]bool) - if checkNativeSdkVersion { checkNativeSdkVersion = app.SdkVersion(ctx).Specified() && app.SdkVersion(ctx).Kind != android.SdkCorePlatform && !app.RequiresStableAPIs(ctx) } + jniLib, prebuiltJniPackages := collectJniDeps(ctx, shouldCollectRecursiveNativeDeps, + checkNativeSdkVersion, func(dep cc.LinkableInterface) bool { + return !dep.IsNdk(ctx.Config()) && !dep.IsStubs() + }) + + var certificates []Certificate + + ctx.VisitDirectDeps(func(module android.Module) { + otherName := ctx.OtherModuleName(module) + tag := ctx.OtherModuleDependencyTag(module) + + if tag == certificateTag { + if dep, ok := module.(*AndroidAppCertificate); ok { + certificates = append(certificates, dep.Certificate) + } else { + ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", otherName) + } + } + }) + return jniLib, prebuiltJniPackages, certificates +} + +func collectJniDeps(ctx android.ModuleContext, + shouldCollectRecursiveNativeDeps bool, + checkNativeSdkVersion bool, + filter func(cc.LinkableInterface) bool) ([]jniLib, android.Paths) { + var jniLibs []jniLib + var prebuiltJniPackages android.Paths + seenModulePaths := make(map[string]bool) ctx.WalkDeps(func(module android.Module, parent android.Module) bool { otherName := ctx.OtherModuleName(module) @@ -936,7 +960,7 @@ func collectAppDeps(ctx android.ModuleContext, app appDepsInterface, if IsJniDepTag(tag) || cc.IsSharedDepTag(tag) { if dep, ok := module.(cc.LinkableInterface); ok { - if dep.IsNdk(ctx.Config()) || dep.IsStubs() { + if filter != nil && !filter(dep) { return false } @@ -977,18 +1001,10 @@ func collectAppDeps(ctx android.ModuleContext, app appDepsInterface, prebuiltJniPackages = append(prebuiltJniPackages, info.JniPackages...) } - if tag == certificateTag { - if dep, ok := module.(*AndroidAppCertificate); ok { - certificates = append(certificates, dep.Certificate) - } else { - ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", otherName) - } - } - return false }) - return jniLibs, prebuiltJniPackages, certificates + return jniLibs, prebuiltJniPackages } func (a *AndroidApp) WalkPayloadDeps(ctx android.ModuleContext, do android.PayloadDepsCallback) { diff --git a/java/base.go b/java/base.go index 284ec9918..4e2366f7d 100644 --- a/java/base.go +++ b/java/base.go @@ -2441,6 +2441,9 @@ func collectDirectDepsProviders(ctx android.ModuleContext) (result *JarJarProvid // Gather repackage information from deps // If the dep jas a JarJarProvider, it is used. Otherwise, any BaseJarJarProvider is used. ctx.VisitDirectDepsIgnoreBlueprint(func(m android.Module) { + if ctx.OtherModuleDependencyTag(m) == proguardRaiseTag { + return + } merge := func(theirs *JarJarProviderData) { for orig, renamed := range theirs.Rename { if result == nil { diff --git a/java/builder.go b/java/builder.go index 085e7a17f..74a05f281 100644 --- a/java/builder.go +++ b/java/builder.go @@ -277,7 +277,7 @@ var ( gatherReleasedFlaggedApisRule = pctx.AndroidStaticRule("gatherReleasedFlaggedApisRule", blueprint.RuleParams{ - Command: `${aconfig} dump-cache --format='{fully_qualified_name}={state:bool}' ` + + Command: `${aconfig} dump-cache --dedup --format='{fully_qualified_name}={state:bool}' ` + `--out ${out} ` + `${flags_path} ` + `${filter_args} `, diff --git a/java/classpath_fragment.go b/java/classpath_fragment.go index 201780175..0ebab4d8c 100644 --- a/java/classpath_fragment.go +++ b/java/classpath_fragment.go @@ -103,8 +103,8 @@ type classpathJar struct { func gatherPossibleApexModuleNamesAndStems(ctx android.ModuleContext, contents []string, tag blueprint.DependencyTag) []string { set := map[string]struct{}{} for _, name := range contents { - dep := ctx.GetDirectDepWithTag(name, tag) - set[name] = struct{}{} + dep, _ := ctx.GetDirectDepWithTag(name, tag).(android.Module) + set[ModuleStemForDeapexing(dep)] = struct{}{} if m, ok := dep.(ModuleWithStem); ok { set[m.Stem()] = struct{}{} } else { diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 01f60d403..f7e3cb93a 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -726,13 +726,19 @@ func (m *apexNameToApexExportsInfoMap) javaLibraryDexPathOnHost(ctx android.Modu return nil, false } +// Returns the stem of an artifact inside a prebuilt apex +func ModuleStemForDeapexing(m android.Module) string { + bmn, _ := m.(interface{ BaseModuleName() string }) + return bmn.BaseModuleName() +} + // Returns the java libraries exported by the apex for hiddenapi and dexpreopt // This information can come from two mechanisms // 1. New: Direct deps to _selected_ apexes. The apexes return a ApexExportsInfo // 2. Legacy: An edge to java_library or java_import (java_sdk_library) module. For prebuilt apexes, this serves as a hook and is populated by deapexers of prebuilt apxes // TODO: b/308174306 - Once all mainline modules have been flagged, drop (2) func getDexJarForApex(ctx android.ModuleContext, pair apexJarModulePair, apexNameToApexExportsInfoMap apexNameToApexExportsInfoMap) android.Path { - if dex, found := apexNameToApexExportsInfoMap.javaLibraryDexPathOnHost(ctx, pair.apex, android.RemoveOptionalPrebuiltPrefix(pair.jarModule.Name())); found { + if dex, found := apexNameToApexExportsInfoMap.javaLibraryDexPathOnHost(ctx, pair.apex, ModuleStemForDeapexing(pair.jarModule)); found { return dex } // TODO: b/308174306 - Remove the legacy mechanism diff --git a/java/droidstubs.go b/java/droidstubs.go index 56ae427ac..51503f22a 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -744,8 +744,14 @@ func (d *Droidstubs) generateRevertAnnotationArgs(ctx android.ModuleContext, cmd filterArgs = "--filter='state:ENABLED+permission:READ_ONLY' --filter='permission:READ_WRITE'" case Exportable: - filterArgs = "--filter='state:ENABLED+permission:READ_ONLY'" - + // When the build flag RELEASE_EXPORT_RUNTIME_APIS is set to true, apis marked with + // the flagged apis that have read_write permissions are exposed on top of the enabled + // and read_only apis. This is to support local override of flag values at runtime. + if ctx.Config().ReleaseExportRuntimeApis() { + filterArgs = "--filter='state:ENABLED+permission:READ_ONLY' --filter='permission:READ_WRITE'" + } else { + filterArgs = "--filter='state:ENABLED+permission:READ_ONLY'" + } } ctx.Build(pctx, android.BuildParams{ diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go index 52cd1c513..caa834535 100644 --- a/java/droidstubs_test.go +++ b/java/droidstubs_test.go @@ -412,3 +412,48 @@ func TestAconfigDeclarations(t *testing.T) { android.AssertStringDoesContain(t, "foo generates exportable stubs jar", strings.Join(m.AllOutputs(), ""), "exportable/foo-stubs.srcjar") } + +func TestReleaseExportRuntimeApis(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildFlags = map[string]string{ + "RELEASE_HIDDEN_API_EXPORTABLE_STUBS": "true", + "RELEASE_EXPORT_RUNTIME_APIS": "true", + } + }), + android.FixtureMergeMockFs(map[string][]byte{ + "a/A.java": nil, + "a/current.txt": nil, + "a/removed.txt": nil, + }), + ).RunTestWithBp(t, ` + aconfig_declarations { + name: "bar", + package: "com.example.package", + srcs: [ + "bar.aconfig", + ], + } + droidstubs { + name: "foo", + srcs: ["a/A.java"], + api_surface: "public", + check_api: { + current: { + api_file: "a/current.txt", + removed_api_file: "a/removed.txt", + } + }, + aconfig_declarations: [ + "bar", + ], + } + `) + + m := result.ModuleForTests("foo", "android_common") + + rule := m.Output("released-flagged-apis-exportable.txt") + exposeWritableApisFilter := "--filter='state:ENABLED+permission:READ_ONLY' --filter='permission:READ_WRITE'" + android.AssertStringEquals(t, "Filter argument expected to contain READ_WRITE permissions", exposeWritableApisFilter, rule.Args["filter_args"]) +} diff --git a/java/ravenwood.go b/java/ravenwood.go new file mode 100644 index 000000000..908619d5f --- /dev/null +++ b/java/ravenwood.go @@ -0,0 +1,278 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package java + +import ( + "android/soong/android" + "android/soong/tradefed" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +func init() { + RegisterRavenwoodBuildComponents(android.InitRegistrationContext) +} + +func RegisterRavenwoodBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("android_ravenwood_test", ravenwoodTestFactory) + ctx.RegisterModuleType("android_ravenwood_libgroup", ravenwoodLibgroupFactory) +} + +var ravenwoodLibContentTag = dependencyTag{name: "ravenwoodlibcontent"} +var ravenwoodUtilsTag = dependencyTag{name: "ravenwoodutils"} +var ravenwoodRuntimeTag = dependencyTag{name: "ravenwoodruntime"} + +const ravenwoodUtilsName = "ravenwood-utils" +const ravenwoodRuntimeName = "ravenwood-runtime" + +type ravenwoodLibgroupJniDepProviderInfo struct { + // All the jni_libs module names with transient dependencies. + names map[string]bool +} + +var ravenwoodLibgroupJniDepProvider = blueprint.NewProvider[ravenwoodLibgroupJniDepProviderInfo]() + +func getLibPath(archType android.ArchType) string { + if archType.Multilib == "lib64" { + return "lib64" + } + return "lib" +} + +type ravenwoodTestProperties struct { + Jni_libs []string +} + +type ravenwoodTest struct { + Library + + ravenwoodTestProperties ravenwoodTestProperties + + testProperties testProperties + testConfig android.Path + + forceOSType android.OsType + forceArchType android.ArchType +} + +func ravenwoodTestFactory() android.Module { + module := &ravenwoodTest{} + + module.addHostAndDeviceProperties() + module.AddProperties(&module.testProperties, &module.ravenwoodTestProperties) + + module.Module.dexpreopter.isTest = true + module.Module.linter.properties.Lint.Test = proptools.BoolPtr(true) + + module.testProperties.Test_suites = []string{ + "general-tests", + "ravenwood-tests", + } + module.testProperties.Test_options.Unit_test = proptools.BoolPtr(false) + + InitJavaModule(module, android.DeviceSupported) + android.InitDefaultableModule(module) + + return module +} + +func (r *ravenwoodTest) InstallInTestcases() bool { return true } +func (r *ravenwoodTest) InstallForceOS() (*android.OsType, *android.ArchType) { + return &r.forceOSType, &r.forceArchType +} +func (r *ravenwoodTest) TestSuites() []string { + return r.testProperties.Test_suites +} + +func (r *ravenwoodTest) DepsMutator(ctx android.BottomUpMutatorContext) { + r.Library.DepsMutator(ctx) + + // Generically depend on the runtime so that it's installed together with us + ctx.AddVariationDependencies(nil, ravenwoodRuntimeTag, ravenwoodRuntimeName) + + // Directly depend on any utils so that we link against them + utils := ctx.AddVariationDependencies(nil, ravenwoodUtilsTag, ravenwoodUtilsName)[0] + if utils != nil { + for _, lib := range utils.(*ravenwoodLibgroup).ravenwoodLibgroupProperties.Libs { + ctx.AddVariationDependencies(nil, libTag, lib) + } + } + + // Add jni libs + for _, lib := range r.ravenwoodTestProperties.Jni_libs { + ctx.AddVariationDependencies(ctx.Config().BuildOSTarget.Variations(), jniLibTag, lib) + } +} + +func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { + r.forceOSType = ctx.Config().BuildOS + r.forceArchType = ctx.Config().BuildArch + + r.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ + TestConfigProp: r.testProperties.Test_config, + TestConfigTemplateProp: r.testProperties.Test_config_template, + TestSuites: r.testProperties.Test_suites, + AutoGenConfig: r.testProperties.Auto_gen_config, + DeviceTemplate: "${RavenwoodTestConfigTemplate}", + HostTemplate: "${RavenwoodTestConfigTemplate}", + }) + + r.Library.GenerateAndroidBuildActions(ctx) + + // Start by depending on all files installed by dependencies + var installDeps android.InstallPaths + + // All JNI libraries included in the runtime + var runtimeJniModuleNames map[string]bool + + if utils := ctx.GetDirectDepsWithTag(ravenwoodUtilsTag)[0]; utils != nil { + for _, installFile := range utils.FilesToInstall() { + installDeps = append(installDeps, installFile) + } + jniDeps, ok := android.OtherModuleProvider(ctx, utils, ravenwoodLibgroupJniDepProvider) + if ok { + runtimeJniModuleNames = jniDeps.names + } + } + + if runtime := ctx.GetDirectDepsWithTag(ravenwoodRuntimeTag)[0]; runtime != nil { + for _, installFile := range runtime.FilesToInstall() { + installDeps = append(installDeps, installFile) + } + jniDeps, ok := android.OtherModuleProvider(ctx, runtime, ravenwoodLibgroupJniDepProvider) + if ok { + runtimeJniModuleNames = jniDeps.names + } + } + + // Also remember what JNI libs are in the runtime. + + // Also depend on our config + installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) + installConfig := ctx.InstallFile(installPath, ctx.ModuleName()+".config", r.testConfig) + installDeps = append(installDeps, installConfig) + + // Depend on the JNI libraries, but don't install the ones that the runtime already + // contains. + soInstallPath := installPath.Join(ctx, getLibPath(r.forceArchType)) + for _, jniLib := range collectTransitiveJniDeps(ctx) { + if _, ok := runtimeJniModuleNames[jniLib.name]; ok { + continue // Runtime already includes it. + } + installJni := ctx.InstallFile(soInstallPath, jniLib.path.Base(), jniLib.path) + installDeps = append(installDeps, installJni) + } + + // Install our JAR with all dependencies + ctx.InstallFile(installPath, ctx.ModuleName()+".jar", r.outputFile, installDeps...) +} + +func (r *ravenwoodTest) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := r.Library.AndroidMkEntries() + entries := &entriesList[0] + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_UNINSTALLABLE_MODULE", true) + entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", + "general-tests", "ravenwood-tests") + if r.testConfig != nil { + entries.SetPath("LOCAL_FULL_TEST_CONFIG", r.testConfig) + } + }) + return entriesList +} + +type ravenwoodLibgroupProperties struct { + Libs []string + + Jni_libs []string +} + +type ravenwoodLibgroup struct { + android.ModuleBase + + ravenwoodLibgroupProperties ravenwoodLibgroupProperties + + forceOSType android.OsType + forceArchType android.ArchType +} + +func ravenwoodLibgroupFactory() android.Module { + module := &ravenwoodLibgroup{} + module.AddProperties(&module.ravenwoodLibgroupProperties) + + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + return module +} + +func (r *ravenwoodLibgroup) InstallInTestcases() bool { return true } +func (r *ravenwoodLibgroup) InstallForceOS() (*android.OsType, *android.ArchType) { + return &r.forceOSType, &r.forceArchType +} +func (r *ravenwoodLibgroup) TestSuites() []string { + return []string{ + "general-tests", + "ravenwood-tests", + } +} + +func (r *ravenwoodLibgroup) DepsMutator(ctx android.BottomUpMutatorContext) { + // Always depends on our underlying libs + for _, lib := range r.ravenwoodLibgroupProperties.Libs { + ctx.AddVariationDependencies(nil, ravenwoodLibContentTag, lib) + } + for _, lib := range r.ravenwoodLibgroupProperties.Jni_libs { + ctx.AddVariationDependencies(ctx.Config().BuildOSTarget.Variations(), jniLibTag, lib) + } +} + +func (r *ravenwoodLibgroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { + r.forceOSType = ctx.Config().BuildOS + r.forceArchType = ctx.Config().BuildArch + + // Collect the JNI dependencies, including the transitive deps. + jniDepNames := make(map[string]bool) + jniLibs := collectTransitiveJniDeps(ctx) + + for _, jni := range jniLibs { + jniDepNames[jni.name] = true + } + android.SetProvider(ctx, ravenwoodLibgroupJniDepProvider, ravenwoodLibgroupJniDepProviderInfo{ + names: jniDepNames, + }) + + // Install our runtime into expected location for packaging + installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) + for _, lib := range r.ravenwoodLibgroupProperties.Libs { + libModule := ctx.GetDirectDepWithTag(lib, ravenwoodLibContentTag) + libJar := android.OutputFileForModule(ctx, libModule, "") + ctx.InstallFile(installPath, lib+".jar", libJar) + } + soInstallPath := android.PathForModuleInstall(ctx, r.BaseModuleName()).Join(ctx, getLibPath(r.forceArchType)) + + for _, jniLib := range jniLibs { + ctx.InstallFile(soInstallPath, jniLib.path.Base(), jniLib.path) + } + + // Normal build should perform install steps + ctx.Phony(r.BaseModuleName(), android.PathForPhony(ctx, r.BaseModuleName()+"-install")) +} + +// collectTransitiveJniDeps returns all JNI dependencies, including transitive +// ones, including NDK / stub libs. (Because Ravenwood has no "preinstalled" libraries) +func collectTransitiveJniDeps(ctx android.ModuleContext) []jniLib { + libs, _ := collectJniDeps(ctx, true, false, nil) + return libs +} diff --git a/java/ravenwood_test.go b/java/ravenwood_test.go new file mode 100644 index 000000000..59612645c --- /dev/null +++ b/java/ravenwood_test.go @@ -0,0 +1,186 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "runtime" + "testing" + + "android/soong/android" +) + +var prepareRavenwoodRuntime = android.GroupFixturePreparers( + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + RegisterRavenwoodBuildComponents(ctx) + }), + android.FixtureAddTextFile("ravenwood/Android.bp", ` + cc_library_shared { + name: "ravenwood-runtime-jni1", + host_supported: true, + srcs: ["jni.cpp"], + } + cc_library_shared { + name: "ravenwood-runtime-jni2", + host_supported: true, + srcs: ["jni.cpp"], + stem: "libred", + shared_libs: [ + "ravenwood-runtime-jni3", + ], + } + cc_library_shared { + name: "ravenwood-runtime-jni3", + host_supported: true, + srcs: ["jni.cpp"], + } + java_library_static { + name: "framework-minus-apex.ravenwood", + srcs: ["Framework.java"], + } + java_library_static { + name: "framework-services.ravenwood", + srcs: ["Services.java"], + } + java_library_static { + name: "framework-rules.ravenwood", + srcs: ["Rules.java"], + } + android_ravenwood_libgroup { + name: "ravenwood-runtime", + libs: [ + "framework-minus-apex.ravenwood", + "framework-services.ravenwood", + ], + jni_libs: [ + "ravenwood-runtime-jni1", + "ravenwood-runtime-jni2", + ], + } + android_ravenwood_libgroup { + name: "ravenwood-utils", + libs: [ + "framework-rules.ravenwood", + ], + } + `), +) + +var installPathPrefix = "out/soong/host/linux-x86/testcases" + +func TestRavenwoodRuntime(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + prepareRavenwoodRuntime, + ).RunTest(t) + + // Verify that our runtime depends on underlying libs + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-runtime", "android_common", "framework-minus-apex.ravenwood") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-runtime", "android_common", "framework-services.ravenwood") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-runtime", "android_common", "ravenwood-runtime-jni") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-utils", "android_common", "framework-rules.ravenwood") + + // Verify that we've emitted artifacts in expected location + runtime := ctx.ModuleForTests("ravenwood-runtime", "android_common") + runtime.Output(installPathPrefix + "/ravenwood-runtime/framework-minus-apex.ravenwood.jar") + runtime.Output(installPathPrefix + "/ravenwood-runtime/framework-services.ravenwood.jar") + runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/ravenwood-runtime-jni1.so") + runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/libred.so") + runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/ravenwood-runtime-jni3.so") + utils := ctx.ModuleForTests("ravenwood-utils", "android_common") + utils.Output(installPathPrefix + "/ravenwood-utils/framework-rules.ravenwood.jar") +} + +func TestRavenwoodTest(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + + ctx := android.GroupFixturePreparers( + PrepareForIntegrationTestWithJava, + prepareRavenwoodRuntime, + ).RunTestWithBp(t, ` + cc_library_shared { + name: "jni-lib1", + host_supported: true, + srcs: ["jni.cpp"], + } + cc_library_shared { + name: "jni-lib2", + host_supported: true, + srcs: ["jni.cpp"], + stem: "libblue", + shared_libs: [ + "jni-lib3", + ], + } + cc_library_shared { + name: "jni-lib3", + host_supported: true, + srcs: ["jni.cpp"], + stem: "libpink", + } + android_ravenwood_test { + name: "ravenwood-test", + srcs: ["Test.java"], + jni_libs: [ + "jni-lib1", + "jni-lib2", + "ravenwood-runtime-jni2", + ], + sdk_version: "test_current", + } + `) + + // Verify that our test depends on underlying libs + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-test", "android_common", "ravenwood-buildtime") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-test", "android_common", "ravenwood-utils") + CheckModuleHasDependency(t, ctx.TestContext, "ravenwood-test", "android_common", "jni-lib") + + module := ctx.ModuleForTests("ravenwood-test", "android_common") + classpath := module.Rule("javac").Args["classpath"] + + // Verify that we're linking against test_current + android.AssertStringDoesContain(t, "classpath", classpath, "android_test_stubs_current.jar") + // Verify that we're linking against utils + android.AssertStringDoesContain(t, "classpath", classpath, "framework-rules.ravenwood.jar") + // Verify that we're *NOT* linking against runtime + android.AssertStringDoesNotContain(t, "classpath", classpath, "framework-minus-apex.ravenwood.jar") + android.AssertStringDoesNotContain(t, "classpath", classpath, "framework-services.ravenwood.jar") + + // Verify that we've emitted test artifacts in expected location + outputJar := module.Output(installPathPrefix + "/ravenwood-test/ravenwood-test.jar") + module.Output(installPathPrefix + "/ravenwood-test/ravenwood-test.config") + module.Output(installPathPrefix + "/ravenwood-test/lib64/jni-lib1.so") + module.Output(installPathPrefix + "/ravenwood-test/lib64/libblue.so") + module.Output(installPathPrefix + "/ravenwood-test/lib64/libpink.so") + + // ravenwood-runtime*.so are included in the runtime, so it shouldn't be emitted. + for _, o := range module.AllOutputs() { + android.AssertStringDoesNotContain(t, "runtime libs shouldn't be included", o, "/ravenwood-test/lib64/ravenwood-runtime") + } + + // Verify that we're going to install underlying libs + orderOnly := outputJar.OrderOnly.Strings() + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/framework-minus-apex.ravenwood.jar") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/framework-services.ravenwood.jar") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/lib64/ravenwood-runtime-jni1.so") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/lib64/libred.so") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-runtime/lib64/ravenwood-runtime-jni3.so") + android.AssertStringListContains(t, "orderOnly", orderOnly, installPathPrefix+"/ravenwood-utils/framework-rules.ravenwood.jar") +} diff --git a/java/system_modules.go b/java/system_modules.go index f3446483b..92e31cdf9 100644 --- a/java/system_modules.go +++ b/java/system_modules.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" "android/soong/android" ) @@ -210,7 +211,7 @@ func (system *SystemModules) AndroidMk() android.AndroidMkData { // type and the one to use is selected at runtime. func systemModulesImportFactory() android.Module { module := &systemModulesImport{} - module.AddProperties(&module.properties) + module.AddProperties(&module.properties, &module.prebuiltProperties) android.InitPrebuiltModule(module, &module.properties.Libs) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) @@ -219,13 +220,39 @@ func systemModulesImportFactory() android.Module { type systemModulesImport struct { SystemModules - prebuilt android.Prebuilt + prebuilt android.Prebuilt + prebuiltProperties prebuiltSystemModulesProperties +} + +type prebuiltSystemModulesProperties struct { + // Name of the source soong module that gets shadowed by this prebuilt + // If unspecified, follows the naming convention that the source module of + // the prebuilt is Name() without "prebuilt_" prefix + Source_module_name *string } func (system *systemModulesImport) Name() string { return system.prebuilt.Name(system.ModuleBase.Name()) } +// BaseModuleName returns the source module that will get shadowed by this prebuilt +// e.g. +// +// java_system_modules_import { +// name: "my_system_modules.v1", +// source_module_name: "my_system_modules", +// } +// +// java_system_modules_import { +// name: "my_system_modules.v2", +// source_module_name: "my_system_modules", +// } +// +// `BaseModuleName` for both will return `my_system_modules` +func (system *systemModulesImport) BaseModuleName() string { + return proptools.StringDefault(system.prebuiltProperties.Source_module_name, system.ModuleBase.Name()) +} + func (system *systemModulesImport) Prebuilt() *android.Prebuilt { return &system.prebuilt } diff --git a/java/system_modules_test.go b/java/system_modules_test.go index 2ceca5d0b..336dd2134 100644 --- a/java/system_modules_test.go +++ b/java/system_modules_test.go @@ -15,6 +15,7 @@ package java import ( + "fmt" "testing" "android/soong/android" @@ -111,3 +112,85 @@ func TestJavaSystemModulesMixSourceAndPrebuilt(t *testing.T) { expectedPrebuiltPaths := getModuleHeaderJarsAsRelativeToTopPaths(result, "prebuilt_system-module1", "prebuilt_system-module2") android.AssertArrayString(t, "prebuilt system modules inputs", expectedPrebuiltPaths, prebuiltInputs.RelativeToTop().Strings()) } + +func TestMultipleSystemModulesPrebuilts(t *testing.T) { + bp := ` + // an rdep + java_library { + name: "foo", + sdk_version: "none", + system_modules: "my_system_modules", + } + + // multiple variations of java_system_modules + // source + java_system_modules { + name: "my_system_modules", + libs: ["bar"], + } + java_library { + name: "bar", + srcs: ["bar.java"], + } + // prebuilt "v1" + java_system_modules_import { + name: "my_system_modules.v1", + source_module_name: "my_system_modules", + libs: ["bar.v1"], + } + java_import { + name: "bar.v1", + source_module_name: "bar", + jars: ["bar.v1.jar"], + } + // prebuilt "v2" + java_system_modules_import { + name: "my_system_modules.v2", + source_module_name: "my_system_modules", + libs: ["bar.v2"], + } + java_import { + name: "bar.v2", + source_module_name: "bar", + jars: ["bar.v2.jar"], + } + + // selectors + apex_contributions { + name: "myapex_contributions", + contents: ["%v"], + } + ` + testCases := []struct { + desc string + selectedDependencyName string + }{ + { + desc: "Source system_modules is selected using apex_contributions", + selectedDependencyName: "my_system_modules", + }, + { + desc: "Prebuilt system_modules v1 is selected using apex_contributions", + selectedDependencyName: "prebuilt_my_system_modules.v1", + }, + { + desc: "Prebuilt system_modules v2 is selected using apex_contributions", + selectedDependencyName: "prebuilt_my_system_modules.v2", + }, + } + + for _, tc := range testCases { + res := android.GroupFixturePreparers( + prepareForJavaTest, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "myapex_contributions", + } + }), + ).RunTestWithBp(t, fmt.Sprintf(bp, tc.selectedDependencyName)) + + // check that rdep gets the correct variation of system_modules + hasDep := CheckModuleHasDependency(t, res.TestContext, "foo", "android_common", tc.selectedDependencyName) + android.AssertBoolEquals(t, fmt.Sprintf("expected dependency from foo to %s\n", tc.selectedDependencyName), true, hasDep) + } +} diff --git a/python/scripts/precompile_python.py b/python/scripts/precompile_python.py index 80e7c76a1..aa1a5df53 100644 --- a/python/scripts/precompile_python.py +++ b/python/scripts/precompile_python.py @@ -24,7 +24,12 @@ import zipfile # This file needs to support both python 2 and 3. -def process_one_file(info, infile, outzip): +def process_one_file(name, infile, outzip): + # Create a ZipInfo instance with a fixed date to ensure a deterministic output. + # Date was chosen to be the same as + # https://cs.android.com/android/platform/superproject/main/+/main:build/soong/jar/jar.go;l=36;drc=2863e4535eb65e15f955dc8ed48fa99b1d2a1db5 + info = zipfile.ZipInfo(filename=name, date_time=(2008, 1, 1, 0, 0, 0)) + if not info.filename.endswith('.py'): outzip.writestr(info, infile.read()) return @@ -37,17 +42,15 @@ def process_one_file(info, infile, outzip): with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp: out_name = tmp.name try: - # Ensure deterministic pyc by using the hash rather than timestamp. - # This is required to improve caching in accelerated builds. - # Only works on Python 3.7+ (see https://docs.python.org/3/library/py_compile.html#py_compile.PycInvalidationMode) - # which should cover most updated branches and developer machines. + # Ensure a deterministic .pyc output by using the hash rather than the timestamp. + # Only works on Python 3.7+ + # See https://docs.python.org/3/library/py_compile.html#py_compile.PycInvalidationMode if sys.version_info >= (3, 7): py_compile.compile(in_name, out_name, info.filename, doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH) else: py_compile.compile(in_name, out_name, info.filename, doraise=True) with open(out_name, 'rb') as f: info.filename = info.filename + 'c' - # Use ZipInfo rather than str to reuse timestamps for deterministic zip files. outzip.writestr(info, f.read()) finally: os.remove(in_name) @@ -62,9 +65,9 @@ def main(): with open(args.dst_zip, 'wb') as outf, open(args.src_zip, 'rb') as inf: with zipfile.ZipFile(outf, mode='w') as outzip, zipfile.ZipFile(inf, mode='r') as inzip: - for info in inzip.infolist(): - with inzip.open(info.filename, mode='r') as inzipf: - process_one_file(info, inzipf, outzip) + for name in inzip.namelist(): + with inzip.open(name, mode='r') as inzipf: + process_one_file(name, inzipf, outzip) if __name__ == "__main__": diff --git a/python/test.go b/python/test.go index 7eb913620..826f35358 100644 --- a/python/test.go +++ b/python/test.go @@ -158,35 +158,25 @@ func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext } runner := proptools.StringDefault(p.testProperties.Test_options.Runner, "tradefed") - if runner == "tradefed" { - p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ - TestConfigProp: p.testProperties.Test_config, - TestConfigTemplateProp: p.testProperties.Test_config_template, - TestSuites: p.binaryProperties.Test_suites, - OptionsForAutogenerated: configs, - AutoGenConfig: p.binaryProperties.Auto_gen_config, - DeviceTemplate: "${PythonBinaryHostTestConfigTemplate}", - HostTemplate: "${PythonBinaryHostTestConfigTemplate}", - }) - } else if runner == "mobly" { - if p.testProperties.Test_config != nil || p.testProperties.Test_config_template != nil || p.binaryProperties.Auto_gen_config != nil { - panic(fmt.Errorf("cannot set test_config, test_config_template or auto_gen_config for mobly test")) - } - - for _, testSuite := range p.binaryProperties.Test_suites { - if testSuite == "cts" { - configs = append(configs, tradefed.Option{Name: "test-suite-tag", Value: "cts"}) - break - } + template := "${PythonBinaryHostTestConfigTemplate}" + if runner == "mobly" { + // Add tag to enable Atest mobly runner + if !android.InList("mobly", p.testProperties.Test_options.Tags) { + p.testProperties.Test_options.Tags = append(p.testProperties.Test_options.Tags, "mobly") } - p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ - OptionsForAutogenerated: configs, - DeviceTemplate: "${PythonBinaryHostMoblyTestConfigTemplate}", - HostTemplate: "${PythonBinaryHostMoblyTestConfigTemplate}", - }) - } else { + template = "${PythonBinaryHostMoblyTestConfigTemplate}" + } else if runner != "tradefed" { panic(fmt.Errorf("unknown python test runner '%s', should be 'tradefed' or 'mobly'", runner)) } + p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ + TestConfigProp: p.testProperties.Test_config, + TestConfigTemplateProp: p.testProperties.Test_config_template, + TestSuites: p.binaryProperties.Test_suites, + OptionsForAutogenerated: configs, + AutoGenConfig: p.binaryProperties.Auto_gen_config, + DeviceTemplate: template, + HostTemplate: template, + }) for _, dataSrcPath := range android.PathsForModuleSrc(ctx, p.testProperties.Data) { p.data = append(p.data, android.DataPath{SrcPath: dataSrcPath}) @@ -228,6 +218,12 @@ func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries { entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) } + // ATS 2.0 is the test harness for mobly tests and the test config is for ATS 2.0. + // Add "v2" suffix to test config name to distinguish it from the config for TF. + if proptools.String(p.testProperties.Test_options.Runner) == "mobly" { + entries.SetString("LOCAL_TEST_CONFIG_SUFFIX", "v2") + } + entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) android.SetAconfigFileMkEntries(&p.ModuleBase, entries, p.mergedAconfigFiles) diff --git a/rust/afdo.go b/rust/afdo.go index 323ee36a5..6116c5e21 100644 --- a/rust/afdo.go +++ b/rust/afdo.go @@ -44,14 +44,14 @@ func (afdo *afdo) addDep(ctx BaseModuleContext, actx android.BottomUpMutatorCont if err != nil { ctx.ModuleErrorf("%s", err.Error()) } - if fdoProfileName != nil { + if fdoProfileName != "" { actx.AddFarVariationDependencies( []blueprint.Variation{ {Mutator: "arch", Variation: actx.Target().ArchVariation()}, {Mutator: "os", Variation: "android"}, }, cc.FdoProfileTag, - []string{*fdoProfileName}..., + []string{fdoProfileName}..., ) } } diff --git a/rust/config/lints.go b/rust/config/lints.go index 932298132..7770af03e 100644 --- a/rust/config/lints.go +++ b/rust/config/lints.go @@ -44,6 +44,7 @@ var ( // Default Rust lints that applies to Google-authored modules. defaultRustcLints = []string{ "-A deprecated", + "-A unknown_lints", "-D missing-docs", "-D warnings", "-D unsafe_op_in_unsafe_fn", @@ -53,6 +54,7 @@ var ( // deny. defaultClippyLints = []string{ "-A clippy::type-complexity", + "-A clippy::unnecessary_fallible_conversions", "-A clippy::unnecessary-wraps", "-A clippy::unusual-byte-groupings", "-A clippy::upper-case-acronyms", diff --git a/tradefed/config.go b/tradefed/config.go index 326a00666..b0150348b 100644 --- a/tradefed/config.go +++ b/tradefed/config.go @@ -33,6 +33,7 @@ func init() { pctx.SourcePathVariable("NativeTestConfigTemplate", "build/make/core/native_test_config_template.xml") pctx.SourcePathVariable("PythonBinaryHostMoblyTestConfigTemplate", "build/make/core/python_binary_host_mobly_test_config_template.xml") pctx.SourcePathVariable("PythonBinaryHostTestConfigTemplate", "build/make/core/python_binary_host_test_config_template.xml") + pctx.SourcePathVariable("RavenwoodTestConfigTemplate", "build/make/core/ravenwood_test_config_template.xml") pctx.SourcePathVariable("RustDeviceTestConfigTemplate", "build/make/core/rust_device_test_config_template.xml") pctx.SourcePathVariable("RustHostTestConfigTemplate", "build/make/core/rust_host_test_config_template.xml") pctx.SourcePathVariable("RustDeviceBenchmarkConfigTemplate", "build/make/core/rust_device_benchmark_config_template.xml") diff --git a/ui/build/androidmk_denylist.go b/ui/build/androidmk_denylist.go index e004cdcdf..e7896ab29 100644 --- a/ui/build/androidmk_denylist.go +++ b/ui/build/androidmk_denylist.go @@ -16,8 +16,6 @@ package build import ( "strings" - - "android/soong/android" ) var androidmk_denylist []string = []string{ @@ -32,16 +30,17 @@ var androidmk_denylist []string = []string{ "libcore/", "libnativehelper/", "pdk/", - "toolchain/", + // Add back toolchain/ once defensive Android.mk files are removed + //"toolchain/", } -func blockAndroidMks(androidMks []string) []string { - return android.FilterListPred(androidMks, func(s string) bool { +func blockAndroidMks(ctx Context, androidMks []string) { + for _, mkFile := range androidMks { for _, d := range androidmk_denylist { - if strings.HasPrefix(s, d) { - return false + if strings.HasPrefix(mkFile, d) { + ctx.Fatalf("Found blocked Android.mk file: %s. "+ + "Please see androidmk_denylist.go for the blocked directories and contact build system team if the file should not be blocked.", mkFile) } } - return true - }) + } } diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index d364542c5..e17bd5402 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -237,7 +237,6 @@ func runMakeProductConfig(ctx Context, config Config) { "BUILD_BROKEN_SRC_DIR_RW_ALLOWLIST", // Not used, but useful to be in the soong.log - "BOARD_VNDK_VERSION", "TARGET_BUILD_TYPE", "HOST_ARCH", "HOST_2ND_ARCH", diff --git a/ui/build/finder.go b/ui/build/finder.go index a114079ff..573df21d9 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -128,7 +128,7 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { // Stop searching a subdirectory recursively after finding an Android.mk. androidMks := f.FindFirstNamedAt(".", "Android.mk") - androidMks = blockAndroidMks(androidMks) + blockAndroidMks(ctx, androidMks) err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list")) if err != nil { ctx.Fatalf("Could not export module list: %v", err) |