diff options
58 files changed, 1966 insertions, 517 deletions
diff --git a/aconfig/init.go b/aconfig/init.go index 256b213cc..de155ab52 100644 --- a/aconfig/init.go +++ b/aconfig/init.go @@ -47,7 +47,7 @@ var ( // For create-device-config-sysprops: Generate aconfig flag value map text file aconfigTextRule = pctx.AndroidStaticRule("aconfig_text", blueprint.RuleParams{ - Command: `${aconfig} dump-cache --dedup --format='{fully_qualified_name}={state:bool}'` + + Command: `${aconfig} dump-cache --dedup --format='{fully_qualified_name}:{permission}={state:bool}'` + ` --cache ${in}` + ` --out ${out}.tmp` + ` && ( if cmp -s ${out}.tmp ${out} ; then rm ${out}.tmp ; else mv ${out}.tmp ${out} ; fi )`, diff --git a/android/Android.bp b/android/Android.bp index 3c38148b6..774d24a20 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -42,6 +42,7 @@ bootstrap_go_package { "buildinfo_prop.go", "compliance_metadata.go", "config.go", + "container.go", "test_config.go", "configurable_properties.go", "configured_jars.go", diff --git a/android/arch.go b/android/arch.go index e0c6908c1..6d896e5fc 100644 --- a/android/arch.go +++ b/android/arch.go @@ -19,6 +19,7 @@ import ( "fmt" "reflect" "runtime" + "slices" "strings" "github.com/google/blueprint" @@ -587,19 +588,21 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) { } osTargets := mctx.Config().Targets[os] + image := base.commonProperties.ImageVariation // Filter NativeBridge targets unless they are explicitly supported. // Skip creating native bridge variants for non-core modules. if os == Android && !(base.IsNativeBridgeSupported() && image == CoreVariation) { + osTargets = slices.DeleteFunc(slices.Clone(osTargets), func(t Target) bool { + return bool(t.NativeBridge) + }) + } - var targets []Target - for _, t := range osTargets { - if !t.NativeBridge { - targets = append(targets, t) - } - } - - osTargets = targets + // Filter HostCross targets if disabled. + if base.HostSupported() && !base.HostCrossSupported() { + osTargets = slices.DeleteFunc(slices.Clone(osTargets), func(t Target) bool { + return t.HostCross + }) } // only the primary arch in the ramdisk / vendor_ramdisk / recovery partition diff --git a/android/arch_test.go b/android/arch_test.go index f0a58a90b..6134a065f 100644 --- a/android/arch_test.go +++ b/android/arch_test.go @@ -332,6 +332,12 @@ func TestArchMutator(t *testing.T) { } module { + name: "nohostcross", + host_supported: true, + host_cross_supported: false, + } + + module { name: "baz", device_supported: false, } @@ -355,13 +361,14 @@ func TestArchMutator(t *testing.T) { ` testCases := []struct { - name string - preparer FixturePreparer - fooVariants []string - barVariants []string - bazVariants []string - quxVariants []string - firstVariants []string + name string + preparer FixturePreparer + fooVariants []string + barVariants []string + noHostCrossVariants []string + bazVariants []string + quxVariants []string + firstVariants []string multiTargetVariants []string multiTargetVariantsMap map[string][]string @@ -373,6 +380,7 @@ func TestArchMutator(t *testing.T) { preparer: nil, fooVariants: []string{"android_arm64_armv8-a", "android_arm_armv7-a-neon"}, barVariants: append(buildOSVariants, "android_arm64_armv8-a", "android_arm_armv7-a-neon"), + noHostCrossVariants: append(buildOSVariants, "android_arm64_armv8-a", "android_arm_armv7-a-neon"), bazVariants: nil, quxVariants: append(buildOS32Variants, "android_arm_armv7-a-neon"), firstVariants: append(buildOS64Variants, "android_arm64_armv8-a"), @@ -390,6 +398,7 @@ func TestArchMutator(t *testing.T) { }), fooVariants: nil, barVariants: buildOSVariants, + noHostCrossVariants: buildOSVariants, bazVariants: nil, quxVariants: buildOS32Variants, firstVariants: buildOS64Variants, @@ -406,6 +415,7 @@ func TestArchMutator(t *testing.T) { }), fooVariants: []string{"android_arm64_armv8-a", "android_arm_armv7-a-neon"}, barVariants: []string{"linux_musl_x86_64", "linux_musl_arm64", "linux_musl_x86", "android_arm64_armv8-a", "android_arm_armv7-a-neon"}, + noHostCrossVariants: []string{"linux_musl_x86_64", "linux_musl_x86", "android_arm64_armv8-a", "android_arm_armv7-a-neon"}, bazVariants: nil, quxVariants: []string{"linux_musl_x86", "android_arm_armv7-a-neon"}, firstVariants: []string{"linux_musl_x86_64", "linux_musl_arm64", "android_arm64_armv8-a"}, @@ -461,6 +471,10 @@ func TestArchMutator(t *testing.T) { t.Errorf("want bar variants:\n%q\ngot:\n%q\n", w, g) } + if g, w := enabledVariants(ctx, "nohostcross"), tt.noHostCrossVariants; !reflect.DeepEqual(w, g) { + t.Errorf("want nohostcross variants:\n%q\ngot:\n%q\n", w, g) + } + if g, w := enabledVariants(ctx, "baz"), tt.bazVariants; !reflect.DeepEqual(w, g) { t.Errorf("want baz variants:\n%q\ngot:\n%q\n", w, g) } diff --git a/android/container.go b/android/container.go new file mode 100644 index 000000000..c4fdd9c91 --- /dev/null +++ b/android/container.go @@ -0,0 +1,233 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "reflect" + "slices" + + "github.com/google/blueprint" +) + +type StubsAvailableModule interface { + IsStubsModule() bool +} + +// Returns true if the dependency module is a stubs module +var depIsStubsModule = func(_ ModuleContext, _, dep Module) bool { + if stubsModule, ok := dep.(StubsAvailableModule); ok { + return stubsModule.IsStubsModule() + } + return false +} + +// Labels of exception functions, which are used to determine special dependencies that allow +// otherwise restricted inter-container dependencies +type exceptionHandleFuncLabel int + +const ( + checkStubs exceptionHandleFuncLabel = iota +) + +// Functions cannot be used as a value passed in providers, because functions are not +// hashable. As a workaround, the exceptionHandleFunc enum values are passed using providers, +// and the corresponding functions are called from this map. +var exceptionHandleFunctionsTable = map[exceptionHandleFuncLabel]func(ModuleContext, Module, Module) bool{ + checkStubs: depIsStubsModule, +} + +type InstallableModule interface { + EnforceApiContainerChecks() bool +} + +type restriction struct { + // container of the dependency + dependency *container + + // Error message to be emitted to the user when the dependency meets this restriction + errorMessage string + + // List of labels of allowed exception functions that allows bypassing this restriction. + // If any of the functions mapped to each labels returns true, this dependency would be + // considered allowed and an error will not be thrown. + allowedExceptions []exceptionHandleFuncLabel +} +type container struct { + // The name of the container i.e. partition, api domain + name string + + // Map of dependency restricted containers. + restricted []restriction +} + +var ( + VendorContainer = &container{ + name: VendorVariation, + restricted: nil, + } + SystemContainer = &container{ + name: "system", + restricted: []restriction{ + { + dependency: VendorContainer, + errorMessage: "Module belonging to the system partition other than HALs is " + + "not allowed to depend on the vendor partition module, in order to support " + + "independent development/update cycles and to support the Generic System " + + "Image. Try depending on HALs, VNDK or AIDL instead.", + allowedExceptions: []exceptionHandleFuncLabel{}, + }, + }, + } + ProductContainer = &container{ + name: ProductVariation, + restricted: []restriction{ + { + dependency: VendorContainer, + errorMessage: "Module belonging to the product partition is not allowed to " + + "depend on the vendor partition module, as this may lead to security " + + "vulnerabilities. Try depending on the HALs or utilize AIDL instead.", + allowedExceptions: []exceptionHandleFuncLabel{}, + }, + }, + } + ApexContainer = initializeApexContainer() + CtsContainer = &container{ + name: "cts", + restricted: []restriction{ + { + dependency: SystemContainer, + errorMessage: "CTS module should not depend on the modules belonging to the " + + "system partition, including \"framework\". Depending on the system " + + "partition may lead to disclosure of implementation details and regression " + + "due to API changes across platform versions. Try depending on the stubs instead.", + allowedExceptions: []exceptionHandleFuncLabel{checkStubs}, + }, + }, + } +) + +func initializeApexContainer() *container { + apexContainer := &container{ + name: "apex", + restricted: []restriction{ + { + dependency: SystemContainer, + errorMessage: "Module belonging to Apex(es) is not allowed to depend on the " + + "modules belonging to the system partition. Either statically depend on the " + + "module or convert the depending module to java_sdk_library and depend on " + + "the stubs.", + allowedExceptions: []exceptionHandleFuncLabel{checkStubs}, + }, + }, + } + + apexContainer.restricted = append(apexContainer.restricted, restriction{ + dependency: apexContainer, + errorMessage: "Module belonging to Apex(es) is not allowed to depend on the " + + "modules belonging to other Apex(es). Either include the depending " + + "module in the Apex or convert the depending module to java_sdk_library " + + "and depend on its stubs.", + allowedExceptions: []exceptionHandleFuncLabel{checkStubs}, + }) + + return apexContainer +} + +type ContainersInfo struct { + belongingContainers []*container + + belongingApexes []ApexInfo +} + +func (c *ContainersInfo) BelongingContainers() []*container { + return c.belongingContainers +} + +var ContainersInfoProvider = blueprint.NewProvider[ContainersInfo]() + +// Determines if the module can be installed in the system partition or not. +// Logic is identical to that of modulePartition(...) defined in paths.go +func installInSystemPartition(ctx ModuleContext) bool { + module := ctx.Module() + return !module.InstallInTestcases() && + !module.InstallInData() && + !module.InstallInRamdisk() && + !module.InstallInVendorRamdisk() && + !module.InstallInDebugRamdisk() && + !module.InstallInRecovery() && + !module.InstallInVendor() && + !module.InstallInOdm() && + !module.InstallInProduct() && + determineModuleKind(module.base(), ctx.blueprintBaseModuleContext()) == platformModule +} + +func generateContainerInfo(ctx ModuleContext) ContainersInfo { + inSystem := installInSystemPartition(ctx) + inProduct := ctx.Module().InstallInProduct() + inVendor := ctx.Module().InstallInVendor() + inCts := false + inApex := false + + if m, ok := ctx.Module().(ImageInterface); ok { + inProduct = inProduct || m.ProductVariantNeeded(ctx) + inVendor = inVendor || m.VendorVariantNeeded(ctx) + } + + props := ctx.Module().GetProperties() + for _, prop := range props { + val := reflect.ValueOf(prop).Elem() + if val.Kind() == reflect.Struct { + testSuites := val.FieldByName("Test_suites") + if testSuites.IsValid() && testSuites.Kind() == reflect.Slice && slices.Contains(testSuites.Interface().([]string), "cts") { + inCts = true + } + } + } + + var belongingApexes []ApexInfo + if apexInfo, ok := ModuleProvider(ctx, AllApexInfoProvider); ok { + belongingApexes = apexInfo.ApexInfos + inApex = true + } + + containers := []*container{} + if inSystem { + containers = append(containers, SystemContainer) + } + if inProduct { + containers = append(containers, ProductContainer) + } + if inVendor { + containers = append(containers, VendorContainer) + } + if inCts { + containers = append(containers, CtsContainer) + } + if inApex { + containers = append(containers, ApexContainer) + } + + return ContainersInfo{ + belongingContainers: containers, + belongingApexes: belongingApexes, + } +} + +func setContainerInfo(ctx ModuleContext) { + if _, ok := ctx.Module().(InstallableModule); ok { + containersInfo := generateContainerInfo(ctx) + SetProvider(ctx, ContainersInfoProvider, containersInfo) + } +} diff --git a/android/image.go b/android/image.go index c278dcdf9..0f0310701 100644 --- a/android/image.go +++ b/android/image.go @@ -22,7 +22,7 @@ type ImageInterface interface { // VendorVariantNeeded should return true if the module needs a vendor variant (installed on the vendor image). VendorVariantNeeded(ctx BaseModuleContext) bool - // ProductVariantNeeded should return true if the module needs a product variant (unstalled on the product image). + // ProductVariantNeeded should return true if the module needs a product variant (installed on the product image). ProductVariantNeeded(ctx BaseModuleContext) bool // CoreVariantNeeded should return true if the module needs a core variant (installed on the system image). diff --git a/android/module.go b/android/module.go index 9f7cb377d..91f2056d4 100644 --- a/android/module.go +++ b/android/module.go @@ -603,6 +603,11 @@ type hostAndDeviceProperties struct { Device_supported *bool } +type hostCrossProperties struct { + // If set to true, build a variant of the module for the host cross. Defaults to true. + Host_cross_supported *bool +} + type Multilib string const ( @@ -718,6 +723,10 @@ func InitAndroidArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib m.AddProperties(&base.hostAndDeviceProperties) } + if hod&hostCrossSupported != 0 { + m.AddProperties(&base.hostCrossProperties) + } + initArchModule(m) } @@ -803,6 +812,7 @@ type ModuleBase struct { distProperties distProperties variableProperties interface{} hostAndDeviceProperties hostAndDeviceProperties + hostCrossProperties hostCrossProperties // Arch specific versions of structs in GetProperties() prior to // initialization in InitAndroidArchModule, lets call it `generalProperties`. @@ -1207,29 +1217,7 @@ func (m *ModuleBase) GenerateTaggedDistFiles(ctx BaseModuleContext) TaggedDistFi continue } } - - // if the tagged dist file cannot be obtained from OutputFilesProvider, - // fall back to use OutputFileProducer - // TODO: remove this part after OutputFilesProvider fully replaces OutputFileProducer - if outputFileProducer, ok := m.module.(OutputFileProducer); ok { - // Call the OutputFiles(tag) method to get the paths associated with the tag. - distFilesForTag, err := outputFileProducer.OutputFiles(tag) - // If the tag was not supported and is not DefaultDistTag then it is an error. - // Failing to find paths for DefaultDistTag is not an error. It just means - // that the module type requires the legacy behavior. - if err != nil && tag != DefaultDistTag { - ctx.PropertyErrorf("dist.tag", "%s", err.Error()) - } - distFiles = distFiles.addPathsForTag(tag, distFilesForTag...) - } else if tag != DefaultDistTag { - // If the tag was specified then it is an error if the module does not - // implement OutputFileProducer because there is no other way of accessing - // the paths for the specified tag. - ctx.PropertyErrorf("dist.tag", - "tag %s not supported because the module does not implement OutputFileProducer", tag) - } } - return distFiles } @@ -1321,7 +1309,11 @@ func (m *ModuleBase) HostCrossSupported() bool { // hostEnabled is true if the host_supported property is true or the HostOrDeviceSupported // value has the hostDefault bit set. hostEnabled := proptools.BoolDefault(m.hostAndDeviceProperties.Host_supported, hod&hostDefault != 0) - return hod&hostCrossSupported != 0 && hostEnabled + + // Default true for the Host_cross_supported property + hostCrossEnabled := proptools.BoolDefault(m.hostCrossProperties.Host_cross_supported, true) + + return hod&hostCrossSupported != 0 && hostEnabled && hostCrossEnabled } func (m *ModuleBase) Platform() bool { @@ -1779,6 +1771,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) variables: make(map[string]string), } + setContainerInfo(ctx) + m.licenseMetadataFile = PathForModuleOut(ctx, "meta_lic") dependencyInstallFiles, dependencyPackagingSpecs := m.computeInstallDeps(ctx) @@ -2383,7 +2377,7 @@ type sourceOrOutputDependencyTag struct { // The name of the module. moduleName string - // The tag that will be passed to the module's OutputFileProducer.OutputFiles(tag) method. + // The tag that will be used to get the specific output file(s). tag string } @@ -2437,14 +2431,7 @@ type SourceFileProducer interface { Srcs() Paths } -// A module that implements OutputFileProducer can be referenced from any property that is tagged with `android:"path"` -// using the ":module" syntax or ":module{.tag}" syntax and provides a list of output files to be used as if they were -// listed in the property. -type OutputFileProducer interface { - OutputFiles(tag string) (Paths, error) -} - -// OutputFilesForModule returns the paths from an OutputFileProducer with the given tag. On error, including if the +// OutputFilesForModule returns the output file paths with the given tag. On error, including if the // module produced zero paths, it reports errors to the ctx and returns nil. func OutputFilesForModule(ctx PathContext, module blueprint.Module, tag string) Paths { paths, err := outputFilesForModule(ctx, module, tag) @@ -2455,7 +2442,7 @@ func OutputFilesForModule(ctx PathContext, module blueprint.Module, tag string) return paths } -// OutputFileForModule returns the path from an OutputFileProducer with the given tag. On error, including if the +// OutputFileForModule returns the output file paths with the given tag. On error, including if the // module produced zero or multiple paths, it reports errors to the ctx and returns nil. func OutputFileForModule(ctx PathContext, module blueprint.Module, tag string) Path { paths, err := outputFilesForModule(ctx, module, tag) @@ -2531,8 +2518,9 @@ func outputFilesForModuleFromProvider(ctx PathContext, module blueprint.Module, } else if cta, isCta := ctx.(*singletonContextAdaptor); isCta { providerData, _ := cta.moduleProvider(module, OutputFilesProvider) outputFiles, _ = providerData.(OutputFilesInfo) + } else { + return nil, fmt.Errorf("unsupported context %q in method outputFilesForModuleFromProvider", reflect.TypeOf(ctx)) } - // TODO: Add a check for skipped context if outputFiles.isEmpty() { return nil, OutputFilesProviderNotSet diff --git a/android/neverallow.go b/android/neverallow.go index 62c5e595e..ef4b8b8b6 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -237,6 +237,7 @@ func createInitFirstStageRules() []Rule { Without("name", "init_first_stage"). Without("name", "init_first_stage.microdroid"). With("install_in_root", "true"). + NotModuleType("prebuilt_root"). Because("install_in_root is only for init_first_stage."), } } diff --git a/android/packaging.go b/android/packaging.go index ae412e1bb..c247ed2a4 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -17,6 +17,7 @@ package android import ( "fmt" "path/filepath" + "sort" "strings" "github.com/google/blueprint" @@ -377,31 +378,59 @@ func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]Packa // CopySpecsToDir is a helper that will add commands to the rule builder to copy the PackagingSpec // entries into the specified directory. func (p *PackagingBase) CopySpecsToDir(ctx ModuleContext, builder *RuleBuilder, specs map[string]PackagingSpec, dir WritablePath) (entries []string) { - if len(specs) == 0 { + dirsToSpecs := make(map[WritablePath]map[string]PackagingSpec) + dirsToSpecs[dir] = specs + return p.CopySpecsToDirs(ctx, builder, dirsToSpecs) +} + +// CopySpecsToDirs is a helper that will add commands to the rule builder to copy the PackagingSpec +// entries into corresponding directories. +func (p *PackagingBase) CopySpecsToDirs(ctx ModuleContext, builder *RuleBuilder, dirsToSpecs map[WritablePath]map[string]PackagingSpec) (entries []string) { + empty := true + for _, specs := range dirsToSpecs { + if len(specs) > 0 { + empty = false + break + } + } + if empty { return entries } + seenDir := make(map[string]bool) preparerPath := PathForModuleOut(ctx, "preparer.sh") cmd := builder.Command().Tool(preparerPath) var sb strings.Builder sb.WriteString("set -e\n") - for _, k := range SortedKeys(specs) { - ps := specs[k] - destPath := filepath.Join(dir.String(), ps.relPathInPackage) - destDir := filepath.Dir(destPath) - entries = append(entries, ps.relPathInPackage) - if _, ok := seenDir[destDir]; !ok { - seenDir[destDir] = true - sb.WriteString(fmt.Sprintf("mkdir -p %s\n", destDir)) - } - if ps.symlinkTarget == "" { - cmd.Implicit(ps.srcPath) - sb.WriteString(fmt.Sprintf("cp %s %s\n", ps.srcPath, destPath)) - } else { - sb.WriteString(fmt.Sprintf("ln -sf %s %s\n", ps.symlinkTarget, destPath)) - } - if ps.executable { - sb.WriteString(fmt.Sprintf("chmod a+x %s\n", destPath)) + + dirs := make([]WritablePath, 0, len(dirsToSpecs)) + for dir, _ := range dirsToSpecs { + dirs = append(dirs, dir) + } + sort.Slice(dirs, func(i, j int) bool { + return dirs[i].String() < dirs[j].String() + }) + + for _, dir := range dirs { + specs := dirsToSpecs[dir] + for _, k := range SortedKeys(specs) { + ps := specs[k] + destPath := filepath.Join(dir.String(), ps.relPathInPackage) + destDir := filepath.Dir(destPath) + entries = append(entries, ps.relPathInPackage) + if _, ok := seenDir[destDir]; !ok { + seenDir[destDir] = true + sb.WriteString(fmt.Sprintf("mkdir -p %s\n", destDir)) + } + if ps.symlinkTarget == "" { + cmd.Implicit(ps.srcPath) + sb.WriteString(fmt.Sprintf("cp %s %s\n", ps.srcPath, destPath)) + } else { + sb.WriteString(fmt.Sprintf("ln -sf %s %s\n", ps.symlinkTarget, destPath)) + } + if ps.executable { + sb.WriteString(fmt.Sprintf("chmod a+x %s\n", destPath)) + } } } diff --git a/android/paths.go b/android/paths.go index 03772ebcb..dda48dd54 100644 --- a/android/paths.go +++ b/android/paths.go @@ -463,8 +463,8 @@ func ExistentPathsForSources(ctx PathGlobContext, paths []string) Paths { // - glob, relative to the local module directory, resolves as filepath(s), relative to the local // source directory. // - other modules using the ":name{.tag}" syntax. These modules must implement SourceFileProducer -// or OutputFileProducer. These resolve as a filepath to an output filepath or generated source -// filepath. +// or set the OutputFilesProvider. These resolve as a filepath to an output filepath or generated +// source filepath. // // Properties passed as the paths argument must have been annotated with struct tag // `android:"path"` so that dependencies on SourceFileProducer modules will have already been handled by the @@ -491,8 +491,8 @@ type SourceInput struct { // - glob, relative to the local module directory, resolves as filepath(s), relative to the local // source directory. Not valid in excludes. // - other modules using the ":name{.tag}" syntax. These modules must implement SourceFileProducer -// or OutputFileProducer. These resolve as a filepath to an output filepath or generated source -// filepath. +// or set the OutputFilesProvider. These resolve as a filepath to an output filepath or generated +// source filepath. // // excluding the items (similarly resolved // Properties passed as the paths argument must have been annotated with struct tag @@ -620,8 +620,8 @@ func GetModuleFromPathDep(ctx ModuleWithDepsPathContext, moduleName, tag string) // - glob, relative to the local module directory, resolves as filepath(s), relative to the local // source directory. Not valid in excludes. // - other modules using the ":name{.tag}" syntax. These modules must implement SourceFileProducer -// or OutputFileProducer. These resolve as a filepath to an output filepath or generated source -// filepath. +// or set the OutputFilesProvider. These resolve as a filepath to an output filepath or generated +// source filepath. // // and a list of the module names of missing module dependencies are returned as the second return. // Properties passed as the paths argument must have been annotated with struct tag diff --git a/android/rule_builder.go b/android/rule_builder.go index 85e29bd20..8b03124bf 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -58,6 +58,7 @@ type RuleBuilder struct { sboxInputs bool sboxManifestPath WritablePath missingDeps []string + args map[string]string } // NewRuleBuilder returns a newly created RuleBuilder. @@ -78,6 +79,17 @@ func (rb *RuleBuilder) SetSboxOutDirDirAsEmpty() *RuleBuilder { return rb } +// Set the phony_output argument. +// This causes the output files to be ignored. +// If the output isn't created, it's not treated as an error. +// The build rule is run every time whether or not the output is created. +func (rb *RuleBuilder) SetPhonyOutput() { + if rb.args == nil { + rb.args = make(map[string]string) + } + rb.args["phony_output"] = "true" +} + // RuleBuilderInstall is a tuple of install from and to locations. type RuleBuilderInstall struct { From Path @@ -726,6 +738,12 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b commandString = proptools.NinjaEscape(commandString) } + args_vars := make([]string, len(r.args)) + i := 0 + for k, _ := range r.args { + args_vars[i] = k + i++ + } r.ctx.Build(r.pctx, BuildParams{ Rule: r.ctx.Rule(r.pctx, name, blueprint.RuleParams{ Command: commandString, @@ -734,7 +752,7 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b Rspfile: proptools.NinjaEscape(rspFile), RspfileContent: rspFileContent, Pool: pool, - }), + }, args_vars...), Inputs: rspFileInputs, Implicits: inputs, OrderOnly: r.OrderOnlys(), @@ -744,6 +762,7 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b Depfile: depFile, Deps: depFormat, Description: desc, + Args: r.args, }) } diff --git a/android/testing.go b/android/testing.go index 18fd3b31c..e39a1a749 100644 --- a/android/testing.go +++ b/android/testing.go @@ -1018,31 +1018,21 @@ func (m TestingModule) VariablesForTestsRelativeToTop() map[string]string { return normalizeStringMapRelativeToTop(m.config, m.module.VariablesForTests()) } -// OutputFiles first checks if module base outputFiles property has any output +// OutputFiles checks if module base outputFiles property has any output // files can be used to return. -// If not, it calls OutputFileProducer.OutputFiles on the -// encapsulated module, exits the test immediately if there is an error and +// Exits the test immediately if there is an error and // otherwise returns the result of calling Paths.RelativeToTop // on the returned Paths. func (m TestingModule) OutputFiles(t *testing.T, tag string) Paths { - // TODO: remove OutputFileProducer part outputFiles := m.Module().base().outputFiles if tag == "" && outputFiles.DefaultOutputFiles != nil { return outputFiles.DefaultOutputFiles.RelativeToTop() } else if taggedOutputFiles, hasTag := outputFiles.TaggedOutputFiles[tag]; hasTag { - return taggedOutputFiles + return taggedOutputFiles.RelativeToTop() } - producer, ok := m.module.(OutputFileProducer) - if !ok { - t.Fatalf("%q must implement OutputFileProducer\n", m.module.Name()) - } - paths, err := producer.OutputFiles(tag) - if err != nil { - t.Fatal(err) - } - - return paths.RelativeToTop() + t.Fatal(fmt.Errorf("No test output file has been set for tag %q", tag)) + return nil } // TestingSingleton is wrapper around an android.Singleton that provides methods to find information about individual diff --git a/android/util.go b/android/util.go index e21e66b88..3c0af2f38 100644 --- a/android/util.go +++ b/android/util.go @@ -201,6 +201,12 @@ func ListSetDifference[T comparable](l1, l2 []T) (bool, []T, []T) { return listsDiffer, diff1, diff2 } +// Returns true if the two lists have common elements. +func HasIntersection[T comparable](l1, l2 []T) bool { + _, a, b := ListSetDifference(l1, l2) + return len(a)+len(b) < len(setFromList(l1))+len(setFromList(l2)) +} + // Returns true if the given string s is prefixed with any string in the given prefix list. func HasAnyPrefix(s string, prefixList []string) bool { for _, prefix := range prefixList { diff --git a/android/util_test.go b/android/util_test.go index 8e73d835c..6537d69b9 100644 --- a/android/util_test.go +++ b/android/util_test.go @@ -818,3 +818,52 @@ func TestReverseSlice(t *testing.T) { }) } } + +var hasIntersectionTestCases = []struct { + name string + l1 []string + l2 []string + expected bool +}{ + { + name: "empty", + l1: []string{"a", "b", "c"}, + l2: []string{}, + expected: false, + }, + { + name: "both empty", + l1: []string{}, + l2: []string{}, + expected: false, + }, + { + name: "identical", + l1: []string{"a", "b", "c"}, + l2: []string{"a", "b", "c"}, + expected: true, + }, + { + name: "duplicates", + l1: []string{"a", "a", "a"}, + l2: []string{"a", "b", "c"}, + expected: true, + }, + { + name: "duplicates with no intersection", + l1: []string{"d", "d", "d", "d"}, + l2: []string{"a", "b", "c"}, + expected: false, + }, +} + +func TestHasIntersection(t *testing.T) { + for _, testCase := range hasIntersectionTestCases { + t.Run(testCase.name, func(t *testing.T) { + hasIntersection := HasIntersection(testCase.l1, testCase.l2) + if !reflect.DeepEqual(hasIntersection, testCase.expected) { + t.Errorf("expected %#v, got %#v", testCase.expected, hasIntersection) + } + }) + } +} diff --git a/android/variable.go b/android/variable.go index d144f7d43..3b02bc72f 100644 --- a/android/variable.go +++ b/android/variable.go @@ -132,9 +132,11 @@ type variableProperties struct { Keep_symbols *bool Keep_symbols_and_debug_frame *bool } - Static_libs []string - Whole_static_libs []string - Shared_libs []string + Static_libs []string + Exclude_static_libs []string + Whole_static_libs []string + Shared_libs []string + Jni_libs []string Cmdline []string diff --git a/androidmk/parser/parser_test.go b/androidmk/parser/parser_test.go index db3313d27..fb03c2324 100644 --- a/androidmk/parser/parser_test.go +++ b/androidmk/parser/parser_test.go @@ -86,20 +86,19 @@ endif`, }, { name: "Blank line in rule's command", - in: `all: + in: `all: echo first line echo second line`, out: []Node{ &Rule{ - Target: SimpleMakeString("all", NoPos), - RecipePos: NoPos, - Recipe: "echo first line\necho second line", + Target: SimpleMakeString("all", NoPos), + RecipePos: NoPos, + Recipe: "echo first line\necho second line", Prerequisites: SimpleMakeString("", NoPos), }, }, }, - } func TestParse(t *testing.T) { diff --git a/apex/Android.bp b/apex/Android.bp index abae9e261..17fdfc36a 100644 --- a/apex/Android.bp +++ b/apex/Android.bp @@ -37,6 +37,7 @@ bootstrap_go_package { "apex_test.go", "bootclasspath_fragment_test.go", "classpath_element_test.go", + "container_test.go", "dexpreopt_bootjars_test.go", "platform_bootclasspath_test.go", "systemserver_classpath_fragment_test.go", diff --git a/apex/apex.go b/apex/apex.go index 10fe372b7..5a75d3e49 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -703,7 +703,6 @@ func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, nativeM rustLibVariations := append( target.Variations(), []blueprint.Variation{ {Mutator: "rust_libraries", Variation: "dylib"}, - {Mutator: "link", Variation: ""}, }..., ) @@ -2663,12 +2662,20 @@ func (a *apexBundle) checkStaticLinkingToStubLibraries(ctx android.ModuleContext }) } +// TODO (b/221087384): Remove this allowlist +var ( + updatableApexesWithCurrentMinSdkVersionAllowlist = []string{"com.android.profiling"} +) + // checkUpdatable enforces APEX and its transitive dep properties to have desired values for updatable APEXes. func (a *apexBundle) checkUpdatable(ctx android.ModuleContext) { if a.Updatable() { if a.minSdkVersionValue(ctx) == "" { ctx.PropertyErrorf("updatable", "updatable APEXes should set min_sdk_version as well") } + if a.minSdkVersion(ctx).IsCurrent() && !android.InList(ctx.ModuleName(), updatableApexesWithCurrentMinSdkVersionAllowlist) { + ctx.PropertyErrorf("updatable", "updatable APEXes should not set min_sdk_version to current. Please use a finalized API level or a recognized in-development codename") + } if a.UsePlatformApis() { ctx.PropertyErrorf("updatable", "updatable APEXes can't use platform APIs") } diff --git a/apex/apex_test.go b/apex/apex_test.go index 15c713b46..a2dbbfc35 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -11721,3 +11721,20 @@ func TestOverrideApexWithPrebuiltApexPreferred(t *testing.T) { java.CheckModuleHasDependency(t, res.TestContext, "myoverrideapex", "android_common_myoverrideapex_myoverrideapex", "foo") } + +func TestUpdatableApexMinSdkVersionCurrent(t *testing.T) { + testApexError(t, `"myapex" .*: updatable: updatable APEXes should not set min_sdk_version to current. Please use a finalized API level or a recognized in-development codename`, ` + apex { + name: "myapex", + key: "myapex.key", + updatable: true, + min_sdk_version: "current", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) +} diff --git a/apex/container_test.go b/apex/container_test.go new file mode 100644 index 000000000..39311741d --- /dev/null +++ b/apex/container_test.go @@ -0,0 +1,329 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apex + +import ( + "android/soong/android" + "android/soong/java" + "fmt" + "testing" +) + +var checkContainerMatch = func(t *testing.T, name string, container string, expected bool, actual bool) { + errorMessage := fmt.Sprintf("module %s container %s value differ", name, container) + android.AssertBoolEquals(t, errorMessage, expected, actual) +} + +func TestApexDepsContainers(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForApexTest, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithLastReleaseApis("mybootclasspathlib"), + ).RunTestWithBp(t, ` + apex { + name: "myapex", + key: "myapex.key", + bootclasspath_fragments: [ + "mybootclasspathfragment", + ], + updatable: true, + min_sdk_version: "30", + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + bootclasspath_fragment { + name: "mybootclasspathfragment", + contents: [ + "mybootclasspathlib", + ], + apex_available: [ + "myapex", + ], + hidden_api: { + split_packages: ["*"], + }, + } + java_sdk_library { + name: "mybootclasspathlib", + srcs: [ + "mybootclasspathlib.java", + ], + apex_available: [ + "myapex", + ], + compile_dex: true, + static_libs: [ + "foo", + "baz", + ], + libs: [ + "bar", + ], + min_sdk_version: "30", + } + java_library { + name: "foo", + srcs:[ + "A.java", + ], + apex_available: [ + "myapex", + ], + min_sdk_version: "30", + } + java_library { + name: "bar", + srcs:[ + "A.java", + ], + min_sdk_version: "30", + } + java_library { + name: "baz", + srcs:[ + "A.java", + ], + apex_available: [ + "//apex_available:platform", + "myapex", + ], + min_sdk_version: "30", + } + `) + testcases := []struct { + moduleName string + variant string + isSystemContainer bool + isApexContainer bool + }{ + { + moduleName: "mybootclasspathlib", + variant: "android_common_myapex", + isSystemContainer: true, + isApexContainer: true, + }, + { + moduleName: "mybootclasspathlib.impl", + variant: "android_common_apex30", + isSystemContainer: true, + isApexContainer: true, + }, + { + moduleName: "mybootclasspathlib.stubs", + variant: "android_common", + isSystemContainer: true, + isApexContainer: false, + }, + { + moduleName: "foo", + variant: "android_common_apex30", + isSystemContainer: true, + isApexContainer: true, + }, + { + moduleName: "bar", + variant: "android_common", + isSystemContainer: true, + isApexContainer: false, + }, + { + moduleName: "baz", + variant: "android_common_apex30", + isSystemContainer: true, + isApexContainer: true, + }, + } + + for _, c := range testcases { + m := result.ModuleForTests(c.moduleName, c.variant) + containers, _ := android.OtherModuleProvider(result.TestContext.OtherModuleProviderAdaptor(), m.Module(), android.ContainersInfoProvider) + belongingContainers := containers.BelongingContainers() + checkContainerMatch(t, c.moduleName, "system", c.isSystemContainer, android.InList(android.SystemContainer, belongingContainers)) + checkContainerMatch(t, c.moduleName, "apex", c.isApexContainer, android.InList(android.ApexContainer, belongingContainers)) + } +} + +func TestNonUpdatableApexDepsContainers(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForApexTest, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithLastReleaseApis("mybootclasspathlib"), + ).RunTestWithBp(t, ` + apex { + name: "myapex", + key: "myapex.key", + bootclasspath_fragments: [ + "mybootclasspathfragment", + ], + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + bootclasspath_fragment { + name: "mybootclasspathfragment", + contents: [ + "mybootclasspathlib", + ], + apex_available: [ + "myapex", + ], + hidden_api: { + split_packages: ["*"], + }, + } + java_sdk_library { + name: "mybootclasspathlib", + srcs: [ + "mybootclasspathlib.java", + ], + apex_available: [ + "myapex", + ], + compile_dex: true, + static_libs: [ + "foo", + ], + libs: [ + "bar", + ], + } + java_library { + name: "foo", + srcs:[ + "A.java", + ], + apex_available: [ + "myapex", + ], + } + java_library { + name: "bar", + srcs:[ + "A.java", + ], + } + `) + testcases := []struct { + moduleName string + variant string + isSystemContainer bool + isApexContainer bool + }{ + { + moduleName: "mybootclasspathlib", + variant: "android_common_myapex", + isSystemContainer: true, + isApexContainer: true, + }, + { + moduleName: "mybootclasspathlib.impl", + variant: "android_common_apex10000", + isSystemContainer: true, + isApexContainer: true, + }, + { + moduleName: "mybootclasspathlib.stubs", + variant: "android_common", + isSystemContainer: true, + isApexContainer: false, + }, + { + moduleName: "foo", + variant: "android_common_apex10000", + isSystemContainer: true, + isApexContainer: true, + }, + { + moduleName: "bar", + variant: "android_common", + isSystemContainer: true, + isApexContainer: false, + }, + } + + for _, c := range testcases { + m := result.ModuleForTests(c.moduleName, c.variant) + containers, _ := android.OtherModuleProvider(result.TestContext.OtherModuleProviderAdaptor(), m.Module(), android.ContainersInfoProvider) + belongingContainers := containers.BelongingContainers() + checkContainerMatch(t, c.moduleName, "system", c.isSystemContainer, android.InList(android.SystemContainer, belongingContainers)) + checkContainerMatch(t, c.moduleName, "apex", c.isApexContainer, android.InList(android.ApexContainer, belongingContainers)) + } +} + +func TestUpdatableAndNonUpdatableApexesIdenticalMinSdkVersion(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForApexTest, + java.PrepareForTestWithJavaSdkLibraryFiles, + android.FixtureMergeMockFs(android.MockFS{ + "system/sepolicy/apex/myapex_non_updatable-file_contexts": nil, + "system/sepolicy/apex/myapex_updatable-file_contexts": nil, + }), + ).RunTestWithBp(t, ` + apex { + name: "myapex_non_updatable", + key: "myapex_non_updatable.key", + java_libs: [ + "foo", + ], + updatable: false, + min_sdk_version: "30", + } + apex_key { + name: "myapex_non_updatable.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + apex { + name: "myapex_updatable", + key: "myapex_updatable.key", + java_libs: [ + "foo", + ], + updatable: true, + min_sdk_version: "30", + } + apex_key { + name: "myapex_updatable.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_library { + name: "foo", + srcs:[ + "A.java", + ], + apex_available: [ + "myapex_non_updatable", + "myapex_updatable", + ], + min_sdk_version: "30", + sdk_version: "current", + } + `) + + fooApexVariant := result.ModuleForTests("foo", "android_common_apex30") + containers, _ := android.OtherModuleProvider(result.TestContext.OtherModuleProviderAdaptor(), fooApexVariant.Module(), android.ContainersInfoProvider) + belongingContainers := containers.BelongingContainers() + checkContainerMatch(t, "foo", "system", true, android.InList(android.SystemContainer, belongingContainers)) + checkContainerMatch(t, "foo", "apex", true, android.InList(android.ApexContainer, belongingContainers)) +} @@ -48,10 +48,10 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("cc_defaults", defaultsFactory) ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("sdk", sdkMutator).Parallel() + ctx.Transition("sdk", &sdkTransitionMutator{}) ctx.BottomUp("llndk", llndkMutator).Parallel() - ctx.BottomUp("link", LinkageMutator).Parallel() - ctx.BottomUp("version", versionMutator).Parallel() + ctx.Transition("link", &linkageTransitionMutator{}) + ctx.Transition("version", &versionTransitionMutator{}) ctx.BottomUp("begin", BeginMutator).Parallel() }) @@ -1867,8 +1867,10 @@ var ( "libdl": true, "libz": true, // art apex + // TODO(b/234351700): Remove this when com.android.art.debug is gone. "libandroidio": true, "libdexfile": true, + "libdexfiled": true, // com.android.art.debug only "libnativebridge": true, "libnativehelper": true, "libnativeloader": true, diff --git a/cc/ccdeps.go b/cc/ccdeps.go index d30abbab7..469fe31fa 100644 --- a/cc/ccdeps.go +++ b/cc/ccdeps.go @@ -85,9 +85,8 @@ func (c *ccdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCon moduleDeps := ccDeps{} moduleInfos := map[string]ccIdeInfo{} - // Track which projects have already had CMakeLists.txt generated to keep the first - // variant for each project. - seenProjects := map[string]bool{} + // Track if best variant (device arch match) has been found. + bestVariantFound := map[string]bool{} pathToCC, _ := evalVariable(ctx, "${config.ClangBin}/") moduleDeps.C_clang = fmt.Sprintf("%s%s", buildCMakePath(pathToCC), cClang) @@ -96,7 +95,7 @@ func (c *ccdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCon ctx.VisitAllModules(func(module android.Module) { if ccModule, ok := module.(*Module); ok { if compiledModule, ok := ccModule.compiler.(CompiledInterface); ok { - generateCLionProjectData(ctx, compiledModule, ccModule, seenProjects, moduleInfos) + generateCLionProjectData(ctx, compiledModule, ccModule, bestVariantFound, moduleInfos) } } }) @@ -180,26 +179,30 @@ func parseCompilerCCParameters(ctx android.SingletonContext, params []string) cc } func generateCLionProjectData(ctx android.SingletonContext, compiledModule CompiledInterface, - ccModule *Module, seenProjects map[string]bool, moduleInfos map[string]ccIdeInfo) { + ccModule *Module, bestVariantFound map[string]bool, moduleInfos map[string]ccIdeInfo) { + moduleName := ccModule.ModuleBase.Name() srcs := compiledModule.Srcs() - if len(srcs) == 0 { + + // Skip if best variant has already been found. + if bestVariantFound[moduleName] { return } - // Only keep the DeviceArch variant module. - if ctx.DeviceConfig().DeviceArch() != ccModule.ModuleBase.Arch().ArchType.Name { + // Skip if sources are empty. + if len(srcs) == 0 { return } - clionProjectLocation := getCMakeListsForModule(ccModule, ctx) - if seenProjects[clionProjectLocation] { + // Check if device arch matches, in which case this is the best variant and takes precedence. + if ccModule.Device() && ccModule.ModuleBase.Arch().ArchType.Name == ctx.DeviceConfig().DeviceArch() { + bestVariantFound[moduleName] = true + } else if _, ok := moduleInfos[moduleName]; ok { + // Skip because this isn't the best variant and a previous one has already been added. + // Heuristically, ones that appear first are likely to be more relevant. return } - seenProjects[clionProjectLocation] = true - - name := ccModule.ModuleBase.Name() - dpInfo := moduleInfos[name] + dpInfo := ccIdeInfo{} dpInfo.Path = append(dpInfo.Path, path.Dir(ctx.BlueprintFile(ccModule))) dpInfo.Srcs = append(dpInfo.Srcs, srcs.Strings()...) @@ -216,9 +219,9 @@ func generateCLionProjectData(ctx android.SingletonContext, compiledModule Compi dpInfo.Local_Cpp_flags = parseCompilerCCParameters(ctx, ccModule.flags.Local.CppFlags) dpInfo.System_include_flags = parseCompilerCCParameters(ctx, ccModule.flags.SystemIncludeFlags) - dpInfo.Module_name = name + dpInfo.Module_name = moduleName - moduleInfos[name] = dpInfo + moduleInfos[moduleName] = dpInfo } type Deal struct { diff --git a/cc/config/global.go b/cc/config/global.go index 62a4765f4..bf2502fcf 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -397,8 +397,8 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r522817" - ClangDefaultShortVersion = "18" + ClangDefaultVersion = "clang-r530567" + ClangDefaultShortVersion = "19" // Directories with warnings from Android.bp files. WarningAllowedProjects = []string{ diff --git a/cc/library.go b/cc/library.go index 560b1ae40..6dd5b5615 100644 --- a/cc/library.go +++ b/cc/library.go @@ -19,6 +19,7 @@ import ( "io" "path/filepath" "regexp" + "slices" "strconv" "strings" "sync" @@ -711,7 +712,7 @@ type versionedInterface interface { setStubsVersion(string) stubsVersion() string - stubsVersions(ctx android.BaseMutatorContext) []string + stubsVersions(ctx android.BaseModuleContext) []string setAllStubsVersions([]string) allStubsVersions() []string @@ -1903,7 +1904,7 @@ func (library *libraryDecorator) isStubsImplementationRequired() bool { return BoolDefault(library.Properties.Stubs.Implementation_installable, true) } -func (library *libraryDecorator) stubsVersions(ctx android.BaseMutatorContext) []string { +func (library *libraryDecorator) stubsVersions(ctx android.BaseModuleContext) []string { if !library.hasStubsVariants() { return nil } @@ -2064,26 +2065,26 @@ func NewLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) // connects a shared library to a static library in order to reuse its .o files to avoid // compiling source files twice. -func reuseStaticLibrary(mctx android.BottomUpMutatorContext, static, shared *Module) { - if staticCompiler, ok := static.compiler.(*libraryDecorator); ok { - sharedCompiler := shared.compiler.(*libraryDecorator) +func reuseStaticLibrary(ctx android.BottomUpMutatorContext, shared *Module) { + if sharedCompiler, ok := shared.compiler.(*libraryDecorator); ok { // Check libraries in addition to cflags, since libraries may be exporting different // include directories. - if len(staticCompiler.StaticProperties.Static.Cflags.GetOrDefault(mctx, nil)) == 0 && - len(sharedCompiler.SharedProperties.Shared.Cflags.GetOrDefault(mctx, nil)) == 0 && - len(staticCompiler.StaticProperties.Static.Whole_static_libs) == 0 && + if len(sharedCompiler.StaticProperties.Static.Cflags.GetOrDefault(ctx, nil)) == 0 && + len(sharedCompiler.SharedProperties.Shared.Cflags.GetOrDefault(ctx, nil)) == 0 && + len(sharedCompiler.StaticProperties.Static.Whole_static_libs) == 0 && len(sharedCompiler.SharedProperties.Shared.Whole_static_libs) == 0 && - len(staticCompiler.StaticProperties.Static.Static_libs) == 0 && + len(sharedCompiler.StaticProperties.Static.Static_libs) == 0 && len(sharedCompiler.SharedProperties.Shared.Static_libs) == 0 && - len(staticCompiler.StaticProperties.Static.Shared_libs) == 0 && + len(sharedCompiler.StaticProperties.Static.Shared_libs) == 0 && len(sharedCompiler.SharedProperties.Shared.Shared_libs) == 0 && // Compare System_shared_libs properties with nil because empty lists are // semantically significant for them. - staticCompiler.StaticProperties.Static.System_shared_libs == nil && + sharedCompiler.StaticProperties.Static.System_shared_libs == nil && sharedCompiler.SharedProperties.Shared.System_shared_libs == nil { - mctx.AddInterVariantDependency(reuseObjTag, shared, static) + // TODO: namespaces? + ctx.AddVariationDependencies([]blueprint.Variation{{"link", "static"}}, reuseObjTag, ctx.ModuleName()) sharedCompiler.baseCompiler.Properties.OriginalSrcs = sharedCompiler.baseCompiler.Properties.Srcs sharedCompiler.baseCompiler.Properties.Srcs = nil @@ -2091,19 +2092,21 @@ func reuseStaticLibrary(mctx android.BottomUpMutatorContext, static, shared *Mod } // This dep is just to reference static variant from shared variant - mctx.AddInterVariantDependency(staticVariantTag, shared, static) + ctx.AddVariationDependencies([]blueprint.Variation{{"link", "static"}}, staticVariantTag, ctx.ModuleName()) } } -// LinkageMutator adds "static" or "shared" variants for modules depending +// linkageTransitionMutator adds "static" or "shared" variants for modules depending // on whether the module can be built as a static library or a shared library. -func LinkageMutator(mctx android.BottomUpMutatorContext) { +type linkageTransitionMutator struct{} + +func (linkageTransitionMutator) Split(ctx android.BaseModuleContext) []string { ccPrebuilt := false - if m, ok := mctx.Module().(*Module); ok && m.linker != nil { + if m, ok := ctx.Module().(*Module); ok && m.linker != nil { _, ccPrebuilt = m.linker.(prebuiltLibraryInterface) } if ccPrebuilt { - library := mctx.Module().(*Module).linker.(prebuiltLibraryInterface) + library := ctx.Module().(*Module).linker.(prebuiltLibraryInterface) // Differentiate between header only and building an actual static/shared library buildStatic := library.buildStatic() @@ -2112,75 +2115,118 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { // Always create both the static and shared variants for prebuilt libraries, and then disable the one // that is not being used. This allows them to share the name of a cc_library module, which requires that // all the variants of the cc_library also exist on the prebuilt. - modules := mctx.CreateLocalVariations("static", "shared") - static := modules[0].(*Module) - shared := modules[1].(*Module) - - static.linker.(prebuiltLibraryInterface).setStatic() - shared.linker.(prebuiltLibraryInterface).setShared() - - if buildShared { - mctx.AliasVariation("shared") - } else if buildStatic { - mctx.AliasVariation("static") - } - - if !buildStatic { - static.linker.(prebuiltLibraryInterface).disablePrebuilt() - } - if !buildShared { - shared.linker.(prebuiltLibraryInterface).disablePrebuilt() - } + return []string{"static", "shared"} } else { // Header only } - - } else if library, ok := mctx.Module().(LinkableInterface); ok && (library.CcLibraryInterface() || library.RustLibraryInterface()) { + } else if library, ok := ctx.Module().(LinkableInterface); ok && (library.CcLibraryInterface() || library.RustLibraryInterface()) { // Non-cc.Modules may need an empty variant for their mutators. variations := []string{} if library.NonCcVariants() { variations = append(variations, "") } isLLNDK := false - if m, ok := mctx.Module().(*Module); ok { + if m, ok := ctx.Module().(*Module); ok { isLLNDK = m.IsLlndk() } buildStatic := library.BuildStaticVariant() && !isLLNDK buildShared := library.BuildSharedVariant() if buildStatic && buildShared { - variations := append([]string{"static", "shared"}, variations...) - - modules := mctx.CreateLocalVariations(variations...) - static := modules[0].(LinkableInterface) - shared := modules[1].(LinkableInterface) - static.SetStatic() - shared.SetShared() - - if _, ok := library.(*Module); ok { - reuseStaticLibrary(mctx, static.(*Module), shared.(*Module)) - } - mctx.AliasVariation("shared") + variations = append([]string{"static", "shared"}, variations...) + return variations } else if buildStatic { - variations := append([]string{"static"}, variations...) - - modules := mctx.CreateLocalVariations(variations...) - modules[0].(LinkableInterface).SetStatic() - mctx.AliasVariation("static") + variations = append([]string{"static"}, variations...) } else if buildShared { - variations := append([]string{"shared"}, variations...) + variations = append([]string{"shared"}, variations...) + } + + if len(variations) > 0 { + return variations + } + } + return []string{""} +} - modules := mctx.CreateLocalVariations(variations...) - modules[0].(LinkableInterface).SetShared() - mctx.AliasVariation("shared") - } else if len(variations) > 0 { - mctx.CreateLocalVariations(variations...) - mctx.AliasVariation(variations[0]) +func (linkageTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return "" +} + +func (linkageTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + ccPrebuilt := false + if m, ok := ctx.Module().(*Module); ok && m.linker != nil { + _, ccPrebuilt = m.linker.(prebuiltLibraryInterface) + } + if ccPrebuilt { + if incomingVariation != "" { + return incomingVariation } - if library.BuildRlibVariant() && library.IsRustFFI() && !buildStatic { + library := ctx.Module().(*Module).linker.(prebuiltLibraryInterface) + if library.buildShared() { + return "shared" + } else if library.buildStatic() { + return "static" + } + return "" + } else if library, ok := ctx.Module().(LinkableInterface); ok && library.CcLibraryInterface() { + isLLNDK := false + if m, ok := ctx.Module().(*Module); ok { + isLLNDK = m.IsLlndk() + } + buildStatic := library.BuildStaticVariant() && !isLLNDK + buildShared := library.BuildSharedVariant() + if library.BuildRlibVariant() && library.IsRustFFI() && !buildStatic && (incomingVariation == "static" || incomingVariation == "") { // Rust modules do not build static libs, but rlibs are used as if they // were via `static_libs`. Thus we need to alias the BuildRlibVariant // to "static" for Rust FFI libraries. - mctx.CreateAliasVariation("static", "") + return "" + } + if incomingVariation != "" { + return incomingVariation + } + if buildShared { + return "shared" + } else if buildStatic { + return "static" + } + return "" + } + return "" +} + +func (linkageTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + ccPrebuilt := false + if m, ok := ctx.Module().(*Module); ok && m.linker != nil { + _, ccPrebuilt = m.linker.(prebuiltLibraryInterface) + } + if ccPrebuilt { + library := ctx.Module().(*Module).linker.(prebuiltLibraryInterface) + if variation == "static" { + library.setStatic() + if !library.buildStatic() { + library.disablePrebuilt() + } + } else if variation == "shared" { + library.setShared() + if !library.buildShared() { + library.disablePrebuilt() + } + } + } else if library, ok := ctx.Module().(LinkableInterface); ok && library.CcLibraryInterface() { + if variation == "static" { + library.SetStatic() + } else if variation == "shared" { + library.SetShared() + var isLLNDK bool + if m, ok := ctx.Module().(*Module); ok { + isLLNDK = m.IsLlndk() + } + buildStatic := library.BuildStaticVariant() && !isLLNDK + buildShared := library.BuildSharedVariant() + if buildStatic && buildShared { + if _, ok := library.(*Module); ok { + reuseStaticLibrary(ctx, library.(*Module)) + } + } } } } @@ -2204,64 +2250,14 @@ func normalizeVersions(ctx android.BaseModuleContext, versions []string) { } } -func createVersionVariations(mctx android.BottomUpMutatorContext, versions []string) { - // "" is for the non-stubs (implementation) variant for system modules, or the LLNDK variant - // for LLNDK modules. - variants := append(android.CopyOf(versions), "") - - m := mctx.Module().(*Module) - isLLNDK := m.IsLlndk() - isVendorPublicLibrary := m.IsVendorPublicLibrary() - isImportedApiLibrary := m.isImportedApiLibrary() - - modules := mctx.CreateLocalVariations(variants...) - for i, m := range modules { - - if variants[i] != "" || isLLNDK || isVendorPublicLibrary || isImportedApiLibrary { - // A stubs or LLNDK stubs variant. - c := m.(*Module) - if c.sanitize != nil { - c.sanitize.Properties.ForceDisable = true - } - if c.stl != nil { - c.stl.Properties.Stl = StringPtr("none") - } - c.Properties.PreventInstall = true - lib := moduleLibraryInterface(m) - isLatest := i == (len(versions) - 1) - lib.setBuildStubs(isLatest) - - if variants[i] != "" { - // A non-LLNDK stubs module is hidden from make and has a dependency from the - // implementation module to the stubs module. - c.Properties.HideFromMake = true - lib.setStubsVersion(variants[i]) - mctx.AddInterVariantDependency(stubImplDepTag, modules[len(modules)-1], modules[i]) - } - } - } - mctx.AliasVariation("") - latestVersion := "" - if len(versions) > 0 { - latestVersion = versions[len(versions)-1] - } - mctx.CreateAliasVariation("latest", latestVersion) -} - -func createPerApiVersionVariations(mctx android.BottomUpMutatorContext, minSdkVersion string) { +func perApiVersionVariations(mctx android.BaseModuleContext, minSdkVersion string) []string { from, err := nativeApiLevelFromUser(mctx, minSdkVersion) if err != nil { mctx.PropertyErrorf("min_sdk_version", err.Error()) - return + return []string{""} } - versionStrs := ndkLibraryVersions(mctx, from) - modules := mctx.CreateLocalVariations(versionStrs...) - - for i, module := range modules { - module.(*Module).Properties.Sdk_version = StringPtr(versionStrs[i]) - module.(*Module).Properties.Min_sdk_version = StringPtr(versionStrs[i]) - } + return ndkLibraryVersions(mctx, from) } func canBeOrLinkAgainstVersionVariants(module interface { @@ -2291,7 +2287,7 @@ func moduleLibraryInterface(module blueprint.Module) libraryInterface { } // setStubsVersions normalizes the versions in the Stubs.Versions property into MutatedProperties.AllStubsVersions. -func setStubsVersions(mctx android.BottomUpMutatorContext, library libraryInterface, module *Module) { +func setStubsVersions(mctx android.BaseModuleContext, library libraryInterface, module *Module) { if !library.buildShared() || !canBeVersionVariant(module) { return } @@ -2304,25 +2300,98 @@ func setStubsVersions(mctx android.BottomUpMutatorContext, library libraryInterf library.setAllStubsVersions(versions) } -// versionMutator splits a module into the mandatory non-stubs variant +// versionTransitionMutator splits a module into the mandatory non-stubs variant // (which is unnamed) and zero or more stubs variants. -func versionMutator(mctx android.BottomUpMutatorContext) { - if mctx.Os() != android.Android { - return +type versionTransitionMutator struct{} + +func (versionTransitionMutator) Split(ctx android.BaseModuleContext) []string { + if ctx.Os() != android.Android { + return []string{""} + } + + m, ok := ctx.Module().(*Module) + if library := moduleLibraryInterface(ctx.Module()); library != nil && canBeVersionVariant(m) { + setStubsVersions(ctx, library, m) + + return append(slices.Clone(library.allStubsVersions()), "") + } else if ok && m.SplitPerApiLevel() && m.IsSdkVariant() { + return perApiVersionVariations(ctx, m.MinSdkVersion()) + } + + return []string{""} +} + +func (versionTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return "" +} + +func (versionTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if ctx.Os() != android.Android { + return "" + } + m, ok := ctx.Module().(*Module) + if library := moduleLibraryInterface(ctx.Module()); library != nil && canBeVersionVariant(m) { + if incomingVariation == "latest" { + latestVersion := "" + versions := library.allStubsVersions() + if len(versions) > 0 { + latestVersion = versions[len(versions)-1] + } + return latestVersion + } + return incomingVariation + } else if ok && m.SplitPerApiLevel() && m.IsSdkVariant() { + // If this module only has variants with versions and the incoming dependency doesn't specify which one + // is needed then assume the latest version. + if incomingVariation == "" { + return android.FutureApiLevel.String() + } + return incomingVariation } - m, ok := mctx.Module().(*Module) - if library := moduleLibraryInterface(mctx.Module()); library != nil && canBeVersionVariant(m) { - setStubsVersions(mctx, library, m) + return "" +} - createVersionVariations(mctx, library.allStubsVersions()) +func (versionTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + // Optimization: return early if this module can't be affected. + if ctx.Os() != android.Android { return } - if ok { - if m.SplitPerApiLevel() && m.IsSdkVariant() { - createPerApiVersionVariations(mctx, m.MinSdkVersion()) + m, ok := ctx.Module().(*Module) + if library := moduleLibraryInterface(ctx.Module()); library != nil && canBeVersionVariant(m) { + isLLNDK := m.IsLlndk() + isVendorPublicLibrary := m.IsVendorPublicLibrary() + isImportedApiLibrary := m.isImportedApiLibrary() + + if variation != "" || isLLNDK || isVendorPublicLibrary || isImportedApiLibrary { + // A stubs or LLNDK stubs variant. + if m.sanitize != nil { + m.sanitize.Properties.ForceDisable = true + } + if m.stl != nil { + m.stl.Properties.Stl = StringPtr("none") + } + m.Properties.PreventInstall = true + lib := moduleLibraryInterface(m) + allStubsVersions := library.allStubsVersions() + isLatest := len(allStubsVersions) > 0 && variation == allStubsVersions[len(allStubsVersions)-1] + lib.setBuildStubs(isLatest) + } + if variation != "" { + // A non-LLNDK stubs module is hidden from make + library.setStubsVersion(variation) + m.Properties.HideFromMake = true + } else { + // A non-LLNDK implementation module has a dependency to all stubs versions + for _, version := range library.allStubsVersions() { + ctx.AddVariationDependencies([]blueprint.Variation{{"version", version}}, + stubImplDepTag, ctx.ModuleName()) + } } + } else if ok && m.SplitPerApiLevel() && m.IsSdkVariant() { + m.Properties.Sdk_version = StringPtr(variation) + m.Properties.Min_sdk_version = StringPtr(variation) } } diff --git a/cc/library_stub.go b/cc/library_stub.go index 6f06333ac..636782581 100644 --- a/cc/library_stub.go +++ b/cc/library_stub.go @@ -303,7 +303,7 @@ func (d *apiLibraryDecorator) hasStubsVariants() bool { return d.hasApexStubs() } -func (d *apiLibraryDecorator) stubsVersions(ctx android.BaseMutatorContext) []string { +func (d *apiLibraryDecorator) stubsVersions(ctx android.BaseModuleContext) []string { m, ok := ctx.Module().(*Module) if !ok { diff --git a/cc/ndk_library.go b/cc/ndk_library.go index f32606814..b822e5cbb 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -133,7 +133,7 @@ func (stub *stubDecorator) implementationModuleName(name string) string { return strings.TrimSuffix(name, ndkLibrarySuffix) } -func ndkLibraryVersions(ctx android.BaseMutatorContext, from android.ApiLevel) []string { +func ndkLibraryVersions(ctx android.BaseModuleContext, from android.ApiLevel) []string { var versions []android.ApiLevel versionStrs := []string{} for _, version := range ctx.Config().AllSupportedApiLevels() { @@ -147,7 +147,7 @@ func ndkLibraryVersions(ctx android.BaseMutatorContext, from android.ApiLevel) [ return versionStrs } -func (this *stubDecorator) stubsVersions(ctx android.BaseMutatorContext) []string { +func (this *stubDecorator) stubsVersions(ctx android.BaseModuleContext) []string { if !ctx.Module().Enabled(ctx) { return nil } @@ -19,73 +19,133 @@ import ( "android/soong/genrule" ) -// sdkMutator sets a creates a platform and an SDK variant for modules +// sdkTransitionMutator creates a platform and an SDK variant for modules // that set sdk_version, and ignores sdk_version for the platform // variant. The SDK variant will be used for embedding in APKs // that may be installed on older platforms. Apexes use their own // variants that enforce backwards compatibility. -func sdkMutator(ctx android.BottomUpMutatorContext) { +type sdkTransitionMutator struct{} + +func (sdkTransitionMutator) Split(ctx android.BaseModuleContext) []string { if ctx.Os() != android.Android { - return + return []string{""} } switch m := ctx.Module().(type) { case LinkableInterface: - ccModule, isCcModule := ctx.Module().(*Module) if m.AlwaysSdk() { if !m.UseSdk() && !m.SplitPerApiLevel() { ctx.ModuleErrorf("UseSdk() must return true when AlwaysSdk is set, did the factory forget to set Sdk_version?") } - modules := ctx.CreateVariations("sdk") - modules[0].(*Module).Properties.IsSdkVariant = true + return []string{"sdk"} } else if m.UseSdk() || m.SplitPerApiLevel() { - modules := ctx.CreateVariations("", "sdk") - - // Clear the sdk_version property for the platform (non-SDK) variant so later code - // doesn't get confused by it. - modules[0].(*Module).Properties.Sdk_version = nil - - // Mark the SDK variant. - modules[1].(*Module).Properties.IsSdkVariant = true - - if ctx.Config().UnbundledBuildApps() { - // For an unbundled apps build, hide the platform variant from Make - // so that other Make modules don't link against it, but against the - // SDK variant. - modules[0].(*Module).Properties.HideFromMake = true + return []string{"", "sdk"} + } else { + return []string{""} + } + case *genrule.Module: + if p, ok := m.Extra.(*GenruleExtraProperties); ok { + if String(p.Sdk_version) != "" { + return []string{"", "sdk"} } else { - // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when - // exposed to Make. - modules[1].(*Module).Properties.SdkAndPlatformVariantVisibleToMake = true + return []string{""} } - // SDK variant never gets installed because the variant is to be embedded in - // APKs, not to be installed to the platform. - modules[1].(*Module).Properties.PreventInstall = true - ctx.AliasVariation("") + } + case *CcApiVariant: + ccApiVariant, _ := ctx.Module().(*CcApiVariant) + if String(ccApiVariant.properties.Variant) == "ndk" { + return []string{"sdk"} } else { - if isCcModule { - // Clear the sdk_version property for modules that don't have an SDK variant so - // later code doesn't get confused by it. - ccModule.Properties.Sdk_version = nil - } - ctx.CreateVariations("") - ctx.AliasVariation("") + return []string{""} + } + } + + return []string{""} +} + +func (sdkTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return sourceVariation +} + +func (sdkTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if ctx.Os() != android.Android { + return "" + } + switch m := ctx.Module().(type) { + case LinkableInterface: + if m.AlwaysSdk() { + return "sdk" + } else if m.UseSdk() || m.SplitPerApiLevel() { + return incomingVariation } case *genrule.Module: if p, ok := m.Extra.(*GenruleExtraProperties); ok { if String(p.Sdk_version) != "" { - ctx.CreateVariations("", "sdk") - } else { - ctx.CreateVariations("") + return incomingVariation } - ctx.AliasVariation("") } case *CcApiVariant: ccApiVariant, _ := ctx.Module().(*CcApiVariant) if String(ccApiVariant.properties.Variant) == "ndk" { - ctx.CreateVariations("sdk") + return "sdk" + } + } + + if ctx.IsAddingDependency() { + return incomingVariation + } else { + return "" + } +} + +func (sdkTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + if ctx.Os() != android.Android { + return + } + + switch m := ctx.Module().(type) { + case LinkableInterface: + ccModule, isCcModule := ctx.Module().(*Module) + if m.AlwaysSdk() { + if variation != "sdk" { + ctx.ModuleErrorf("tried to create variation %q for module with AlwaysSdk set, expected \"sdk\"", variation) + } + + ccModule.Properties.IsSdkVariant = true + } else if m.UseSdk() || m.SplitPerApiLevel() { + if variation == "" { + // Clear the sdk_version property for the platform (non-SDK) variant so later code + // doesn't get confused by it. + ccModule.Properties.Sdk_version = nil + } else { + // Mark the SDK variant. + ccModule.Properties.IsSdkVariant = true + + // SDK variant never gets installed because the variant is to be embedded in + // APKs, not to be installed to the platform. + ccModule.Properties.PreventInstall = true + } + + if ctx.Config().UnbundledBuildApps() { + if variation == "" { + // For an unbundled apps build, hide the platform variant from Make + // so that other Make modules don't link against it, but against the + // SDK variant. + ccModule.Properties.HideFromMake = true + } + } else { + if variation == "sdk" { + // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when + // exposed to Make. + ccModule.Properties.SdkAndPlatformVariantVisibleToMake = true + } + } } else { - ctx.CreateVariations("") + if isCcModule { + // Clear the sdk_version property for modules that don't have an SDK variant so + // later code doesn't get confused by it. + ccModule.Properties.Sdk_version = nil + } } } } diff --git a/cmd/release_config/release_config_lib/flag_declaration.go b/cmd/release_config/release_config_lib/flag_declaration.go index 97d4d4c76..22001bf09 100644 --- a/cmd/release_config/release_config_lib/flag_declaration.go +++ b/cmd/release_config/release_config_lib/flag_declaration.go @@ -18,10 +18,21 @@ import ( rc_proto "android/soong/cmd/release_config/release_config_proto" ) +var ( + // Allowlist: these flags may have duplicate (identical) declarations + // without generating an error. This will be removed once all such + // declarations have been fixed. + DuplicateDeclarationAllowlist = map[string]bool{} +) + func FlagDeclarationFactory(protoPath string) (fd *rc_proto.FlagDeclaration) { fd = &rc_proto.FlagDeclaration{} if protoPath != "" { LoadMessage(protoPath, fd) } + // If the input didn't specify a value, create one (== UnspecifiedValue). + if fd.Value == nil { + fd.Value = &rc_proto.Value{Val: &rc_proto.Value_UnspecifiedValue{false}} + } return fd } diff --git a/cmd/release_config/release_config_lib/release_configs.go b/cmd/release_config/release_config_lib/release_configs.go index f2e138801..97eb8f156 100644 --- a/cmd/release_config/release_config_lib/release_configs.go +++ b/cmd/release_config/release_config_lib/release_configs.go @@ -42,6 +42,10 @@ type ReleaseConfigMap struct { // Flags declared this directory's flag_declarations/*.textproto FlagDeclarations []rc_proto.FlagDeclaration + + // Potential aconfig and build flag contributions in this map directory. + // This is used to detect errors. + FlagValueDirs map[string][]string } type ReleaseConfigDirMap map[string]int @@ -272,6 +276,20 @@ func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex configs.Aliases[name] = alias.Target } var err error + // Temporarily allowlist duplicate flag declaration files to prevent + // more from entering the tree while we work to clean up the duplicates + // that already exist. + dupFlagFile := filepath.Join(dir, "duplicate_allowlist.txt") + data, err := os.ReadFile(dupFlagFile) + if err == nil { + for _, flag := range strings.Split(string(data), "\n") { + flag = strings.TrimSpace(flag) + if strings.HasPrefix(flag, "//") || strings.HasPrefix(flag, "#") { + continue + } + DuplicateDeclarationAllowlist[flag] = true + } + } err = WalkTextprotoFiles(dir, "flag_declarations", func(path string, d fs.DirEntry, err error) error { flagDeclaration := FlagDeclarationFactory(path) // Container must be specified. @@ -285,14 +303,6 @@ func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex } } - // TODO: once we have namespaces initialized, we can throw an error here. - if flagDeclaration.Namespace == nil { - flagDeclaration.Namespace = proto.String("android_UNKNOWN") - } - // If the input didn't specify a value, create one (== UnspecifiedValue). - if flagDeclaration.Value == nil { - flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_UnspecifiedValue{false}} - } m.FlagDeclarations = append(m.FlagDeclarations, *flagDeclaration) name := *flagDeclaration.Name if name == "RELEASE_ACONFIG_VALUE_SETS" { @@ -300,8 +310,8 @@ func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex } if def, ok := configs.FlagArtifacts[name]; !ok { configs.FlagArtifacts[name] = &FlagArtifact{FlagDeclaration: flagDeclaration, DeclarationIndex: ConfigDirIndex} - } else if !proto.Equal(def.FlagDeclaration, flagDeclaration) { - return fmt.Errorf("Duplicate definition of %s", *flagDeclaration.Name) + } else if !proto.Equal(def.FlagDeclaration, flagDeclaration) || !DuplicateDeclarationAllowlist[name] { + return fmt.Errorf("Duplicate definition of %s in %s", *flagDeclaration.Name, path) } // Set the initial value in the flag artifact. configs.FilesUsedMap[path] = true @@ -317,6 +327,21 @@ func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex return err } + subDirs := func(subdir string) (ret []string) { + if flagVersions, err := os.ReadDir(filepath.Join(dir, subdir)); err == nil { + for _, e := range flagVersions { + if e.IsDir() && validReleaseConfigName(e.Name()) { + ret = append(ret, e.Name()) + } + } + } + return + } + m.FlagValueDirs = map[string][]string{ + "aconfig": subDirs("aconfig"), + "flag_values": subDirs("flag_values"), + } + err = WalkTextprotoFiles(dir, "release_configs", func(path string, d fs.DirEntry, err error) error { releaseConfigContribution := &ReleaseConfigContribution{path: path, DeclarationIndex: ConfigDirIndex} LoadMessage(path, &releaseConfigContribution.proto) @@ -424,6 +449,27 @@ func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) erro } } + // Look for ignored flagging values. Gather the entire list to make it easier to fix them. + errors := []string{} + for _, contrib := range configs.ReleaseConfigMaps { + dirName := filepath.Dir(contrib.path) + for k, names := range contrib.FlagValueDirs { + for _, rcName := range names { + if config, err := configs.GetReleaseConfig(rcName); err == nil { + rcPath := filepath.Join(dirName, "release_configs", fmt.Sprintf("%s.textproto", config.Name)) + if _, err := os.Stat(rcPath); err != nil { + errors = append(errors, fmt.Sprintf("%s exists but %s does not contribute to %s", + filepath.Join(dirName, k, rcName), dirName, config.Name)) + } + } + + } + } + } + if len(errors) > 0 { + return fmt.Errorf("%s", strings.Join(errors, "\n")) + } + releaseConfig, err := configs.GetReleaseConfig(targetRelease) if err != nil { return err diff --git a/cmd/release_config/release_config_lib/util.go b/cmd/release_config/release_config_lib/util.go index 9919c7081..b149293c2 100644 --- a/cmd/release_config/release_config_lib/util.go +++ b/cmd/release_config/release_config_lib/util.go @@ -31,8 +31,9 @@ import ( ) var ( - disableWarnings bool - containerRegexp, _ = regexp.Compile("^[a-z][a-z0-9]*([._][a-z][a-z0-9]*)*$") + disableWarnings bool + containerRegexp, _ = regexp.Compile("^[a-z][a-z0-9]*([._][a-z][a-z0-9]*)*$") + releaseConfigRegexp, _ = regexp.Compile("^[a-z][a-z0-9]*([._][a-z0-9]*)*$") ) type StringList []string @@ -179,6 +180,10 @@ func validContainer(container string) bool { return containerRegexp.MatchString(container) } +func validReleaseConfigName(name string) bool { + return releaseConfigRegexp.MatchString(name) +} + // Returns the default value for release config artifacts. func GetDefaultOutDir() string { outEnv := os.Getenv("OUT_DIR") diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 201515f51..b69a76fc2 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -532,7 +532,7 @@ func OdexOnSystemOtherByName(name string, dexLocation string, global *GlobalConf } for _, f := range global.PatternsOnSystemOther { - if makefileMatch("/" + f, dexLocation) || makefileMatch(filepath.Join(SystemPartition, f), dexLocation) { + if makefileMatch("/"+f, dexLocation) || makefileMatch(filepath.Join(SystemPartition, f), dexLocation) { return true } } diff --git a/docs/resources.md b/docs/resources.md new file mode 100644 index 000000000..c7cb0cf08 --- /dev/null +++ b/docs/resources.md @@ -0,0 +1,89 @@ +## Soong Android Resource Compilation + +The Android build process involves several steps to compile resources into a format that the Android app can use +efficiently in android_library, android_app and android_test modules. See the +[resources documentation](https://developer.android.com/guide/topics/resources/providing-resources) for general +information on resources (with a focus on building with Gradle). + +For all modules, AAPT2 compiles resources provided by directories listed in the resource_dirs directory (which is +implicitly set to `["res"]` if unset, but can be overridden by setting the `resource_dirs` property). + +## android_library with resource processor +For an android_library with resource processor enabled (currently by setting `use_resource_processor: true`, but will be +enabled by default in the future): +- AAPT2 generates the `package-res.apk` file with a resource table that contains all resources from the current +android_library module. `package-res.apk` files from transitive dependencies are passed to AAPT2 with the `-I` flag to +resolve references to resources from dependencies. +- AAPT2 generates an R.txt file that lists all the resources provided by the current android_library module. +- ResourceProcessorBusyBox reads the `R.txt` file for the current android_library and produces an `R.jar` with an +`R.class` in the package listed in the android_library's `AndroidManifest.xml` file that contains java fields for each +resource ID. The resource IDs are non-final, as the final IDs will not be known until the resource table of the final +android_app or android_test module is built. +- The android_library's java and/or kotlin code is compiled with the generated `R.jar` in the classpath, along with the +`R.jar` files from all transitive android_library dependencies. + +## android_app or android_test with resource processor +For an android_app or android_test with resource processor enabled (currently by setting `use_resource_processor: true`, +but will be enabled by default in the future): +- AAPT2 generates the `package-res.apk` file with a resource table that contains all resources from the current +android_app or android_test, as well as all transitive android_library modules referenced via `static_libs`. The +current module is overlaid on dependencies so that resources from the current module replace resources from dependencies +in the case of conflicts. +- AAPT2 generates an R.txt file that lists all the resources provided by the current android_app or android_test, as +well as all transitive android_library modules referenced via `static_libs`. The R.txt file contains the final resource +ID for each resource. +- ResourceProcessorBusyBox reads the `R.txt` file for the current android_app or android_test, as well as all transitive +android_library modules referenced via `static_libs`, and produces an `R.jar` with an `R.class` in the package listed in +the android_app or android_test's `AndroidManifest.xml` file that contains java fields for all local or transitive +resource IDs. In addition, it creates an `R.class` in the package listed in each android_library dependency's +`AndroidManifest.xml` file that contains final resource IDs for the resources that were found in that library. +- The android_app or android_test's java and/or kotlin code is compiled with the current module's `R.jar` in the +classpath, but not the `R.jar` files from transitive android_library dependencies. The `R.jar` file is also merged into +the program classes that are dexed and placed in the final APK. + +## android_app, android_test or android_library without resource processor +For an android_app, android_test or android_library without resource processor enabled (current the default, or +explicitly set with `use_resource_processor: false`): +- AAPT2 generates the `package-res.apk` file with a resource table that contains all resources from the current +android_app, android_test or android_library module, as well as all transitive android_library modules referenced via +`static_libs`. The current module is overlaid on dependencies so that resources from the current module replace +resources from dependencies in the case of conflicts. +- AAPT2 generates an `R.java` file in the package listed in each the current module's `AndroidManifest.xml` file that +contains resource IDs for all resources from the current module as well as all transitive android_library modules +referenced via `static_libs`. The same `R.java` containing all local and transitive resources is also duplicated into +every package listed in an `AndroidManifest.xml` file in any static `android_library` dependency. +- The module's java and/or kotlin code is compiled along with all the generated `R.java` files. + + +## Downsides of legacy resource compilation without resource processor + +Compiling resources without using the resource processor results in a generated R.java source file for every transitive +package that contains every transitive resource. For modules with large transitive dependency trees this can be tens of +thousands of resource IDs duplicated in tens to a hundred java sources. These java sources all have to be compiled in +every successive module in the dependency tree, and then the final R8 step has to drop hundreds of thousands of +unreferenced fields. This results in significant build time and disk usage increases over building with resource +processor. + +## Converting to compilation with resource processor + +### Reference resources using the package name of the module that includes them. +Converting an android_library module to build with resource processor requires fixing any references to resources +provided by android_library dependencies to reference the R classes using the package name found in the +`AndroidManifest.xml` file of the dependency. For example, when referencing an androidx resource: +```java +View.inflate(mContext, R.layout.preference, null)); +``` +must be replaced with: +```java +View.inflate(mContext, androidx.preference.R.layout.preference, null)); +``` + +### Use unique package names for each module in `AndroidManifest.xml` + +Each module will produce an `R.jar` containing an `R.class` in the package specified in it's `AndroidManifest.xml`. +If multiple modules use the same package name they will produce conflicting `R.class` files, which can cause some +resource IDs to appear to be missing. + +If existing code has multiple modules that contribute resources to the same package, one option is to move all the +resources into a single resources-only `android_library` module with no code, and then depend on that from all the other +modules.
\ No newline at end of file diff --git a/elf/Android.bp b/elf/Android.bp index 6450be137..6d3f4f0ed 100644 --- a/elf/Android.bp +++ b/elf/Android.bp @@ -20,6 +20,7 @@ bootstrap_go_package { name: "soong-elf", pkgPath: "android/soong/elf", srcs: [ + "build_id_dir.go", "elf.go", ], testSrcs: [ diff --git a/elf/build_id_dir.go b/elf/build_id_dir.go new file mode 100644 index 000000000..5fb7dda87 --- /dev/null +++ b/elf/build_id_dir.go @@ -0,0 +1,172 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elf + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +func UpdateBuildIdDir(path string) error { + path = filepath.Clean(path) + buildIdPath := path + "/.build-id" + + // Collect the list of files and build-id symlinks. If the symlinks are + // up to date (newer than the symbol files), there is nothing to do. + var buildIdFiles, symbolFiles []string + var buildIdMtime, symbolsMtime time.Time + filepath.WalkDir(path, func(path string, entry fs.DirEntry, err error) error { + if entry == nil || entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + mtime := info.ModTime() + if strings.HasPrefix(path, buildIdPath) { + if buildIdMtime.Compare(mtime) < 0 { + buildIdMtime = mtime + } + buildIdFiles = append(buildIdFiles, path) + } else { + if symbolsMtime.Compare(mtime) < 0 { + symbolsMtime = mtime + } + symbolFiles = append(symbolFiles, path) + } + return nil + }) + if symbolsMtime.Compare(buildIdMtime) < 0 { + return nil + } + + // Collect build-id -> file mapping from ELF files in the symbols directory. + concurrency := 8 + done := make(chan error) + buildIdToFile := make(map[string]string) + var mu sync.Mutex + for i := 0; i != concurrency; i++ { + go func(paths []string) { + for _, path := range paths { + id, err := Identifier(path, true) + if err != nil { + done <- err + return + } + if id == "" { + continue + } + mu.Lock() + oldPath := buildIdToFile[id] + if oldPath == "" || oldPath > path { + buildIdToFile[id] = path + } + mu.Unlock() + } + done <- nil + }(symbolFiles[len(symbolFiles)*i/concurrency : len(symbolFiles)*(i+1)/concurrency]) + } + + // Collect previously generated build-id -> file mapping from the .build-id directory. + // We will use this for incremental updates. If we see anything in the .build-id + // directory that we did not expect, we'll delete it and start over. + prevBuildIdToFile := make(map[string]string) +out: + for _, buildIdFile := range buildIdFiles { + if !strings.HasSuffix(buildIdFile, ".debug") { + prevBuildIdToFile = nil + break + } + buildId := buildIdFile[len(buildIdPath)+1 : len(buildIdFile)-6] + for i, ch := range buildId { + if i == 2 { + if ch != '/' { + prevBuildIdToFile = nil + break out + } + } else { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + prevBuildIdToFile = nil + break out + } + } + } + target, err := os.Readlink(buildIdFile) + if err != nil || !strings.HasPrefix(target, "../../") { + prevBuildIdToFile = nil + break + } + prevBuildIdToFile[buildId[0:2]+buildId[3:]] = path + target[5:] + } + if prevBuildIdToFile == nil { + err := os.RemoveAll(buildIdPath) + if err != nil { + return err + } + prevBuildIdToFile = make(map[string]string) + } + + // Wait for build-id collection from ELF files to finish. + for i := 0; i != concurrency; i++ { + err := <-done + if err != nil { + return err + } + } + + // Delete old symlinks. + for id, _ := range prevBuildIdToFile { + if buildIdToFile[id] == "" { + symlinkDir := buildIdPath + "/" + id[:2] + symlinkPath := symlinkDir + "/" + id[2:] + ".debug" + if err := os.Remove(symlinkPath); err != nil { + return err + } + } + } + + // Add new symlinks and update changed symlinks. + for id, path := range buildIdToFile { + prevPath := prevBuildIdToFile[id] + if prevPath == path { + continue + } + symlinkDir := buildIdPath + "/" + id[:2] + symlinkPath := symlinkDir + "/" + id[2:] + ".debug" + if prevPath == "" { + if err := os.MkdirAll(symlinkDir, 0755); err != nil { + return err + } + } else { + if err := os.Remove(symlinkPath); err != nil { + return err + } + } + + target, err := filepath.Rel(symlinkDir, path) + if err != nil { + return err + } + if err := os.Symlink(target, symlinkPath); err != nil { + return err + } + } + return nil +} diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index d04b2d1f1..fc6d1f74e 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -126,6 +126,15 @@ type prebuiltSubdirProperties struct { Relative_install_path *string `android:"arch_variant"` } +type prebuiltRootProperties struct { + // Install this module to the root directory, without partition subdirs. When this module is + // added to PRODUCT_PACKAGES, this module will be installed to $PRODUCT_OUT/root, which will + // then be copied to the root of system.img. When this module is packaged by other modules like + // android_filesystem, this module will be installed to the root ("/"), unlike normal + // prebuilt_root modules which are installed to the partition subdir (e.g. "/system/"). + Install_in_root *bool +} + type PrebuiltEtcModule interface { android.Module @@ -140,7 +149,12 @@ type PrebuiltEtc struct { android.ModuleBase android.DefaultableModuleBase - properties prebuiltEtcProperties + properties prebuiltEtcProperties + + // rootProperties is used to return the value of the InstallInRoot() method. Currently, only + // prebuilt_avb and prebuilt_root modules use this. + rootProperties prebuiltRootProperties + subdirProperties prebuiltSubdirProperties sourceFilePaths android.Paths @@ -156,9 +170,6 @@ type PrebuiltEtc struct { additionalDependencies *android.Paths usedSrcsProperty bool - // installInRoot is used to return the value of the InstallInRoot() method. The default value is false. - // Currently, only prebuilt_avb can be set to true. - installInRoot bool makeClass string } @@ -246,7 +257,7 @@ func (p *PrebuiltEtc) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) b } func (p *PrebuiltEtc) InstallInRoot() bool { - return p.installInRoot + return proptools.Bool(p.rootProperties.Install_in_root) } func (p *PrebuiltEtc) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { @@ -502,21 +513,20 @@ func (p *PrebuiltEtc) AndroidModuleBase() *android.ModuleBase { func InitPrebuiltEtcModule(p *PrebuiltEtc, dirBase string) { p.installDirBase = dirBase - p.installInRoot = false p.AddProperties(&p.properties) p.AddProperties(&p.subdirProperties) } func InitPrebuiltRootModule(p *PrebuiltEtc) { p.installDirBase = "." - p.installInRoot = false p.AddProperties(&p.properties) + p.AddProperties(&p.rootProperties) } func InitPrebuiltAvbModule(p *PrebuiltEtc) { p.installDirBase = "avb" - p.installInRoot = true p.AddProperties(&p.properties) + p.rootProperties.Install_in_root = proptools.BoolPtr(true) } // prebuilt_etc is for a prebuilt artifact that is installed in diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index 5add95441..5c7ef434f 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -312,6 +312,25 @@ func (f *filesystem) buildNonDepsFiles(ctx android.ModuleContext, builder *andro } } +func (f *filesystem) copyPackagingSpecs(ctx android.ModuleContext, builder *android.RuleBuilder, specs map[string]android.PackagingSpec, rootDir, rebasedDir android.WritablePath) []string { + rootDirSpecs := make(map[string]android.PackagingSpec) + rebasedDirSpecs := make(map[string]android.PackagingSpec) + + for rel, spec := range specs { + if spec.Partition() == "root" { + rootDirSpecs[rel] = spec + } else { + rebasedDirSpecs[rel] = spec + } + } + + dirsToSpecs := make(map[android.WritablePath]map[string]android.PackagingSpec) + dirsToSpecs[rootDir] = rootDirSpecs + dirsToSpecs[rebasedDir] = rebasedDirSpecs + + return f.CopySpecsToDirs(ctx, builder, dirsToSpecs) +} + func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) android.OutputPath { rootDir := android.PathForModuleOut(ctx, "root").OutputPath rebasedDir := rootDir @@ -322,7 +341,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi // Wipe the root dir to get rid of leftover files from prior builds builder.Command().Textf("rm -rf %s && mkdir -p %s", rootDir, rootDir) specs := f.gatherFilteredPackagingSpecs(ctx) - f.entries = f.CopySpecsToDir(ctx, builder, specs, rebasedDir) + f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir) f.buildNonDepsFiles(ctx, builder, rootDir) f.addMakeBuiltFiles(ctx, builder, rootDir) @@ -465,7 +484,7 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) // Wipe the root dir to get rid of leftover files from prior builds builder.Command().Textf("rm -rf %s && mkdir -p %s", rootDir, rootDir) specs := f.gatherFilteredPackagingSpecs(ctx) - f.entries = f.CopySpecsToDir(ctx, builder, specs, rebasedDir) + f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir) f.buildNonDepsFiles(ctx, builder, rootDir) f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) diff --git a/filesystem/system_image.go b/filesystem/system_image.go index 15cacfb4f..69d922df9 100644 --- a/filesystem/system_image.go +++ b/filesystem/system_image.go @@ -94,9 +94,10 @@ func (s *systemImage) buildLinkerConfigFile(ctx android.ModuleContext, root andr return output } -// Filter the result of GatherPackagingSpecs to discard items targeting outside "system" partition. -// Note that "apex" module installs its contents to "apex"(fake partition) as well +// Filter the result of GatherPackagingSpecs to discard items targeting outside "system" / "root" +// partition. Note that "apex" module installs its contents to "apex"(fake partition) as well // for symbol lookup by imitating "activated" paths. func (s *systemImage) filterPackagingSpec(ps android.PackagingSpec) bool { - return s.filesystem.filterInstallablePackagingSpec(ps) && ps.Partition() == "system" + return s.filesystem.filterInstallablePackagingSpec(ps) && + (ps.Partition() == "system" || ps.Partition() == "root") } diff --git a/java/Android.bp b/java/Android.bp index 54b36ab60..a941754db 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -87,6 +87,7 @@ bootstrap_go_package { "app_set_test.go", "app_test.go", "code_metadata_test.go", + "container_test.go", "bootclasspath_fragment_test.go", "device_host_converter_test.go", "dex_test.go", diff --git a/java/aapt2.go b/java/aapt2.go index f704fc6fc..61cf37381 100644 --- a/java/aapt2.go +++ b/java/aapt2.go @@ -69,7 +69,7 @@ var aapt2CompileRule = pctx.AndroidStaticRule("aapt2Compile", // aapt2Compile compiles resources and puts the results in the requested directory. func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths, - flags []string, productToFilter string) android.WritablePaths { + flags []string, productToFilter string, featureFlagsPaths android.Paths) android.WritablePaths { if productToFilter != "" && productToFilter != "default" { // --filter-product leaves only product-specific resources. Product-specific resources only exist // in value resources (values/*.xml), so filter value resource files only. Ignore other types of @@ -85,6 +85,10 @@ func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Pat flags = append([]string{"--filter-product " + productToFilter}, flags...) } + for _, featureFlagsPath := range android.SortedUniquePaths(featureFlagsPaths) { + flags = append(flags, "--feature-flags", "@"+featureFlagsPath.String()) + } + // Shard the input paths so that they can be processed in parallel. If we shard them into too // small chunks, the additional cost of spinning up aapt2 outweighs the performance gain. The // current shard size, 100, seems to be a good balance between the added cost and the gain. @@ -112,6 +116,7 @@ func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Pat ctx.Build(pctx, android.BuildParams{ Rule: aapt2CompileRule, Description: "aapt2 compile " + dir.String() + shardDesc, + Implicits: featureFlagsPaths, Inputs: shard, Outputs: outPaths, Args: map[string]string{ diff --git a/java/aar.go b/java/aar.go index 2f49a959d..b69b7c262 100644 --- a/java/aar.go +++ b/java/aar.go @@ -440,7 +440,8 @@ func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptio var compiledResDirs []android.Paths for _, dir := range resDirs { a.resourceFiles = append(a.resourceFiles, dir.files...) - compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, compileFlags, a.filterProduct()).Paths()) + compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, + compileFlags, a.filterProduct(), opts.aconfigTextFiles).Paths()) } for i, zip := range resZips { @@ -499,7 +500,8 @@ func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptio } for _, dir := range overlayDirs { - compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, compileFlags, a.filterProduct()).Paths()...) + compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, + compileFlags, a.filterProduct(), opts.aconfigTextFiles).Paths()...) } var splitPackages android.WritablePaths diff --git a/java/app_test.go b/java/app_test.go index e878ccf6d..6b7d522f3 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -4364,7 +4364,16 @@ func TestPrivappAllowlistAndroidMk(t *testing.T) { } func TestAppFlagsPackages(t *testing.T) { - ctx := testApp(t, ` + ctx := android.GroupFixturePreparers( + prepareForJavaTest, + android.FixtureMergeMockFs( + map[string][]byte{ + "res/layout/layout.xml": nil, + "res/values/strings.xml": nil, + "res/values-en-rUS/strings.xml": nil, + }, + ), + ).RunTestWithBp(t, ` android_app { name: "foo", srcs: ["a.java"], @@ -4396,10 +4405,10 @@ func TestAppFlagsPackages(t *testing.T) { // android_app module depends on aconfig_declarations listed in flags_packages android.AssertBoolEquals(t, "foo expected to depend on bar", true, - CheckModuleHasDependency(t, ctx, "foo", "android_common", "bar")) + CheckModuleHasDependency(t, ctx.TestContext, "foo", "android_common", "bar")) android.AssertBoolEquals(t, "foo expected to depend on baz", true, - CheckModuleHasDependency(t, ctx, "foo", "android_common", "baz")) + CheckModuleHasDependency(t, ctx.TestContext, "foo", "android_common", "baz")) aapt2LinkRule := foo.Rule("android/soong/java.aapt2Link") linkInFlags := aapt2LinkRule.Args["inFlags"] @@ -4408,6 +4417,14 @@ func TestAppFlagsPackages(t *testing.T) { linkInFlags, "--feature-flags @out/soong/.intermediates/bar/intermediate.txt --feature-flags @out/soong/.intermediates/baz/intermediate.txt", ) + + aapt2CompileRule := foo.Rule("android/soong/java.aapt2Compile") + compileFlags := aapt2CompileRule.Args["cFlags"] + android.AssertStringDoesContain(t, + "aapt2 compile command expected to pass feature flags arguments", + compileFlags, + "--feature-flags @out/soong/.intermediates/bar/intermediate.txt --feature-flags @out/soong/.intermediates/baz/intermediate.txt", + ) } func TestAppFlagsPackagesPropagation(t *testing.T) { diff --git a/java/base.go b/java/base.go index fc68d2018..02dc3e35b 100644 --- a/java/base.go +++ b/java/base.go @@ -552,6 +552,18 @@ type Module struct { aconfigCacheFiles android.Paths } +var _ android.InstallableModule = (*Module)(nil) + +// To satisfy the InstallableModule interface +func (j *Module) EnforceApiContainerChecks() bool { + return true +} + +// Overrides android.ModuleBase.InstallInProduct() +func (j *Module) InstallInProduct() bool { + return j.ProductSpecific() +} + func (j *Module) CheckStableSdkVersion(ctx android.BaseModuleContext) error { sdkVersion := j.SdkVersion(ctx) if sdkVersion.Stable() { diff --git a/java/container_test.go b/java/container_test.go new file mode 100644 index 000000000..344185553 --- /dev/null +++ b/java/container_test.go @@ -0,0 +1,129 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "android/soong/android" + "fmt" + "testing" +) + +var checkContainerMatch = func(t *testing.T, name string, container string, expected bool, actual bool) { + errorMessage := fmt.Sprintf("module %s container %s value differ", name, container) + android.AssertBoolEquals(t, errorMessage, expected, actual) +} + +func TestJavaContainersModuleProperties(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForJavaTest, + ).RunTestWithBp(t, ` + java_library { + name: "foo", + srcs: ["A.java"], + } + java_library { + name: "foo_vendor", + srcs: ["A.java"], + vendor: true, + sdk_version: "current", + } + java_library { + name: "foo_soc_specific", + srcs: ["A.java"], + soc_specific: true, + sdk_version: "current", + } + java_library { + name: "foo_product_specific", + srcs: ["A.java"], + product_specific: true, + sdk_version: "current", + } + java_test { + name: "foo_cts_test", + srcs: ["A.java"], + test_suites: [ + "cts", + ], + } + java_test { + name: "foo_non_cts_test", + srcs: ["A.java"], + test_suites: [ + "general-tests", + ], + } + `) + + testcases := []struct { + moduleName string + isSystemContainer bool + isVendorContainer bool + isProductContainer bool + isCts bool + }{ + { + moduleName: "foo", + isSystemContainer: true, + isVendorContainer: false, + isProductContainer: false, + isCts: false, + }, + { + moduleName: "foo_vendor", + isSystemContainer: false, + isVendorContainer: true, + isProductContainer: false, + isCts: false, + }, + { + moduleName: "foo_soc_specific", + isSystemContainer: false, + isVendorContainer: true, + isProductContainer: false, + isCts: false, + }, + { + moduleName: "foo_product_specific", + isSystemContainer: false, + isVendorContainer: false, + isProductContainer: true, + isCts: false, + }, + { + moduleName: "foo_cts_test", + isSystemContainer: false, + isVendorContainer: false, + isProductContainer: false, + isCts: true, + }, + { + moduleName: "foo_non_cts_test", + isSystemContainer: false, + isVendorContainer: false, + isProductContainer: false, + isCts: false, + }, + } + + for _, c := range testcases { + m := result.ModuleForTests(c.moduleName, "android_common") + containers, _ := android.OtherModuleProvider(result.TestContext.OtherModuleProviderAdaptor(), m.Module(), android.ContainersInfoProvider) + belongingContainers := containers.BelongingContainers() + checkContainerMatch(t, c.moduleName, "system", c.isSystemContainer, android.InList(android.SystemContainer, belongingContainers)) + checkContainerMatch(t, c.moduleName, "vendor", c.isVendorContainer, android.InList(android.VendorContainer, belongingContainers)) + checkContainerMatch(t, c.moduleName, "product", c.isProductContainer, android.InList(android.ProductContainer, belongingContainers)) + } +} diff --git a/java/java.go b/java/java.go index 88b31b586..498b53939 100644 --- a/java/java.go +++ b/java/java.go @@ -2386,10 +2386,35 @@ func (al *ApiLibrary) MinSdkVersion(ctx android.EarlyModuleContext) android.ApiL return android.FutureApiLevel } +func (al *ApiLibrary) IDEInfo(i *android.IdeInfo) { + i.Deps = append(i.Deps, al.ideDeps()...) + i.Libs = append(i.Libs, al.properties.Libs...) + i.Static_libs = append(i.Static_libs, al.properties.Static_libs...) + i.SrcJars = append(i.SrcJars, al.stubsSrcJar.String()) +} + +// deps of java_api_library for module_bp_java_deps.json +func (al *ApiLibrary) ideDeps() []string { + ret := []string{} + ret = append(ret, al.properties.Libs...) + ret = append(ret, al.properties.Static_libs...) + if al.properties.System_modules != nil { + ret = append(ret, proptools.String(al.properties.System_modules)) + } + if al.properties.Full_api_surface_stub != nil { + ret = append(ret, proptools.String(al.properties.Full_api_surface_stub)) + } + // Other non java_library dependencies like java_api_contribution are ignored for now. + return ret +} + // implement the following interfaces for hiddenapi processing var _ hiddenAPIModule = (*ApiLibrary)(nil) var _ UsesLibraryDependency = (*ApiLibrary)(nil) +// implement the following interface for IDE completion. +var _ android.IDEInfo = (*ApiLibrary)(nil) + // // Java prebuilts // diff --git a/python/python.go b/python/python.go index 1ee533fa8..8726f02ef 100644 --- a/python/python.go +++ b/python/python.go @@ -38,7 +38,7 @@ func registerPythonMutators(ctx android.RegistrationContext) { // Exported to support other packages using Python modules in tests. func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("python_version", versionSplitMutator()).Parallel() + ctx.Transition("python_version", &versionSplitTransitionMutator{}) } // the version-specific properties that apply to python modules. @@ -245,7 +245,6 @@ var ( protoExt = ".proto" pyVersion2 = "PY2" pyVersion3 = "PY3" - pyVersion2And3 = "PY2ANDPY3" internalPath = "internal" ) @@ -253,46 +252,68 @@ type basePropertiesProvider interface { getBaseProperties() *BaseProperties } -// versionSplitMutator creates version variants for modules and appends the version-specific -// properties for a given variant to the properties in the variant module -func versionSplitMutator() func(android.BottomUpMutatorContext) { - return func(mctx android.BottomUpMutatorContext) { - if base, ok := mctx.Module().(basePropertiesProvider); ok { - props := base.getBaseProperties() - var versionNames []string - // collect version specific properties, so that we can merge version-specific properties - // into the module's overall properties - var versionProps []VersionProperties - // PY3 is first so that we alias the PY3 variant rather than PY2 if both - // are available - if proptools.BoolDefault(props.Version.Py3.Enabled, true) { - versionNames = append(versionNames, pyVersion3) - versionProps = append(versionProps, props.Version.Py3) - } - if proptools.BoolDefault(props.Version.Py2.Enabled, false) { - if !mctx.DeviceConfig().BuildBrokenUsesSoongPython2Modules() && - mctx.ModuleName() != "py2-cmd" && - mctx.ModuleName() != "py2-stdlib" { - mctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3. This error can be temporarily overridden by setting BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES := true in the product configuration") - } - versionNames = append(versionNames, pyVersion2) - versionProps = append(versionProps, props.Version.Py2) - } - modules := mctx.CreateLocalVariations(versionNames...) - // Alias module to the first variant - if len(versionNames) > 0 { - mctx.AliasVariation(versionNames[0]) - } - for i, v := range versionNames { - // set the actual version for Python module. - newProps := modules[i].(basePropertiesProvider).getBaseProperties() - newProps.Actual_version = v - // append versioned properties for the Python module to the overall properties - err := proptools.AppendMatchingProperties([]interface{}{newProps}, &versionProps[i], nil) - if err != nil { - panic(err) - } +type versionSplitTransitionMutator struct{} + +func (versionSplitTransitionMutator) Split(ctx android.BaseModuleContext) []string { + if base, ok := ctx.Module().(basePropertiesProvider); ok { + props := base.getBaseProperties() + var variants []string + // PY3 is first so that we alias the PY3 variant rather than PY2 if both + // are available + if proptools.BoolDefault(props.Version.Py3.Enabled, true) { + variants = append(variants, pyVersion3) + } + if proptools.BoolDefault(props.Version.Py2.Enabled, false) { + if !ctx.DeviceConfig().BuildBrokenUsesSoongPython2Modules() && + ctx.ModuleName() != "py2-cmd" && + ctx.ModuleName() != "py2-stdlib" { + ctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3. This error can be temporarily overridden by setting BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES := true in the product configuration") } + variants = append(variants, pyVersion2) + } + return variants + } + return []string{""} +} + +func (versionSplitTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return "" +} + +func (versionSplitTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if incomingVariation != "" { + return incomingVariation + } + if base, ok := ctx.Module().(basePropertiesProvider); ok { + props := base.getBaseProperties() + if proptools.BoolDefault(props.Version.Py3.Enabled, true) { + return pyVersion3 + } else { + return pyVersion2 + } + } + + return "" +} + +func (versionSplitTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + if variation == "" { + return + } + if base, ok := ctx.Module().(basePropertiesProvider); ok { + props := base.getBaseProperties() + props.Actual_version = variation + + var versionProps *VersionProperties + if variation == pyVersion3 { + versionProps = &props.Version.Py3 + } else if variation == pyVersion2 { + versionProps = &props.Version.Py2 + } + + err := proptools.AppendMatchingProperties([]interface{}{props}, versionProps, nil) + if err != nil { + panic(err) } } } diff --git a/rust/coverage.go b/rust/coverage.go index e0e919c69..91a78060d 100644 --- a/rust/coverage.go +++ b/rust/coverage.go @@ -47,7 +47,7 @@ func (cov *coverage) deps(ctx DepsContext, deps Deps) Deps { // no_std modules are missing libprofiler_builtins which provides coverage, so we need to add it as a dependency. if rustModule, ok := ctx.Module().(*Module); ok && rustModule.compiler.noStdlibs() { - ctx.AddVariationDependencies([]blueprint.Variation{{Mutator: "rust_libraries", Variation: "rlib"}, {Mutator: "link", Variation: ""}}, rlibDepTag, ProfilerBuiltins) + ctx.AddVariationDependencies([]blueprint.Variation{{Mutator: "rust_libraries", Variation: "rlib"}}, rlibDepTag, ProfilerBuiltins) } } diff --git a/rust/library.go b/rust/library.go index ba73f27fd..50d5a72ae 100644 --- a/rust/library.go +++ b/rust/library.go @@ -20,6 +20,8 @@ import ( "regexp" "strings" + "github.com/google/blueprint" + "android/soong/android" "android/soong/cc" ) @@ -692,31 +694,28 @@ func validateLibraryStem(ctx BaseModuleContext, filename string, crate_name stri } } -// LibraryMutator mutates the libraries into variants according to the -// build{Rlib,Dylib} attributes. -func LibraryMutator(mctx android.BottomUpMutatorContext) { - // Only mutate on Rust libraries. - m, ok := mctx.Module().(*Module) +type libraryTransitionMutator struct{} + +func (libraryTransitionMutator) Split(ctx android.BaseModuleContext) []string { + m, ok := ctx.Module().(*Module) if !ok || m.compiler == nil { - return + return []string{""} } library, ok := m.compiler.(libraryInterface) if !ok { - return + return []string{""} } // Don't produce rlib/dylib/source variants for shared or static variants if library.shared() || library.static() { - return + return []string{""} } var variants []string // The source variant is used for SourceProvider modules. The other variants (i.e. rlib and dylib) // depend on this variant. It must be the first variant to be declared. - sourceVariant := false if m.sourceProvider != nil { - variants = append(variants, "source") - sourceVariant = true + variants = append(variants, sourceVariation) } if library.buildRlib() { variants = append(variants, rlibVariation) @@ -726,92 +725,134 @@ func LibraryMutator(mctx android.BottomUpMutatorContext) { } if len(variants) == 0 { - return + return []string{""} } - modules := mctx.CreateLocalVariations(variants...) - - // The order of the variations (modules) matches the variant names provided. Iterate - // through the new variation modules and set their mutated properties. - var emptyVariant = false - var rlibVariant = false - for i, v := range modules { - switch variants[i] { - case rlibVariation: - v.(*Module).compiler.(libraryInterface).setRlib() - rlibVariant = true - case dylibVariation: - v.(*Module).compiler.(libraryInterface).setDylib() - if v.(*Module).ModuleBase.ImageVariation().Variation == android.VendorRamdiskVariation { - // TODO(b/165791368) - // Disable dylib Vendor Ramdisk variations until we support these. - v.(*Module).Disable() - } - case "source": - v.(*Module).compiler.(libraryInterface).setSource() - // The source variant does not produce any library. - // Disable the compilation steps. - v.(*Module).compiler.SetDisabled() - case "": - emptyVariant = true + return variants +} + +func (libraryTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return "" +} + +func (libraryTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + m, ok := ctx.Module().(*Module) + if !ok || m.compiler == nil { + return "" + } + library, ok := m.compiler.(libraryInterface) + if !ok { + return "" + } + + if incomingVariation == "" { + if m.sourceProvider != nil { + return sourceVariation + } + if library.shared() { + return "" } + if library.buildRlib() { + return rlibVariation + } + if library.buildDylib() { + return dylibVariation + } + } + return incomingVariation +} + +func (libraryTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + m, ok := ctx.Module().(*Module) + if !ok || m.compiler == nil { + return + } + library, ok := m.compiler.(libraryInterface) + if !ok { + return } - if rlibVariant && library.isFFILibrary() { - // If an rlib variant is set and this is an FFI library, make it the - // default variant so CC can link against it appropriately. - mctx.AliasVariation(rlibVariation) - } else if emptyVariant { - // If there's an empty variant, alias it so it is the default variant - mctx.AliasVariation("") + switch variation { + case rlibVariation: + library.setRlib() + case dylibVariation: + library.setDylib() + if m.ModuleBase.ImageVariation().Variation == android.VendorRamdiskVariation { + // TODO(b/165791368) + // Disable dylib Vendor Ramdisk variations until we support these. + m.Disable() + } + + case sourceVariation: + library.setSource() + // The source variant does not produce any library. + // Disable the compilation steps. + m.compiler.SetDisabled() } // If a source variant is created, add an inter-variant dependency // between the other variants and the source variant. - if sourceVariant { - sv := modules[0] - for _, v := range modules[1:] { - if !v.Enabled(mctx) { - continue - } - mctx.AddInterVariantDependency(sourceDepTag, v, sv) - } - // Alias the source variation so it can be named directly in "srcs" properties. - mctx.AliasVariation("source") + if m.sourceProvider != nil && variation != sourceVariation { + ctx.AddVariationDependencies( + []blueprint.Variation{ + {"rust_libraries", sourceVariation}, + }, + sourceDepTag, ctx.ModuleName()) } } -func LibstdMutator(mctx android.BottomUpMutatorContext) { - if m, ok := mctx.Module().(*Module); ok && m.compiler != nil && !m.compiler.Disabled() { - switch library := m.compiler.(type) { - case libraryInterface: - // Only create a variant if a library is actually being built. +type libstdTransitionMutator struct{} + +func (libstdTransitionMutator) Split(ctx android.BaseModuleContext) []string { + if m, ok := ctx.Module().(*Module); ok && m.compiler != nil && !m.compiler.Disabled() { + // Only create a variant if a library is actually being built. + if library, ok := m.compiler.(libraryInterface); ok { if library.rlib() && !library.sysroot() { - // If this is a rust_ffi variant it only needs rlib-std if library.isFFILibrary() { - variants := []string{"rlib-std"} - modules := mctx.CreateLocalVariations(variants...) - rlib := modules[0].(*Module) - rlib.compiler.(libraryInterface).setRlibStd() - rlib.Properties.RustSubName += RlibStdlibSuffix - mctx.AliasVariation("rlib-std") + return []string{"rlib-std"} } else { - variants := []string{"rlib-std", "dylib-std"} - modules := mctx.CreateLocalVariations(variants...) - - rlib := modules[0].(*Module) - dylib := modules[1].(*Module) - rlib.compiler.(libraryInterface).setRlibStd() - dylib.compiler.(libraryInterface).setDylibStd() - if dylib.ModuleBase.ImageVariation().Variation == android.VendorRamdiskVariation { - // TODO(b/165791368) - // Disable rlibs that link against dylib-std on vendor ramdisk variations until those dylib - // variants are properly supported. - dylib.Disable() - } - rlib.Properties.RustSubName += RlibStdlibSuffix + return []string{"rlib-std", "dylib-std"} } } } } + return []string{""} +} + +func (libstdTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return "" +} + +func (libstdTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if m, ok := ctx.Module().(*Module); ok && m.compiler != nil && !m.compiler.Disabled() { + if library, ok := m.compiler.(libraryInterface); ok { + if library.shared() { + return "" + } + if library.rlib() && !library.sysroot() { + if incomingVariation != "" { + return incomingVariation + } + return "rlib-std" + } + } + } + return "" +} + +func (libstdTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + if variation == "rlib-std" { + rlib := ctx.Module().(*Module) + rlib.compiler.(libraryInterface).setRlibStd() + rlib.Properties.RustSubName += RlibStdlibSuffix + } else if variation == "dylib-std" { + dylib := ctx.Module().(*Module) + dylib.compiler.(libraryInterface).setDylibStd() + if dylib.ModuleBase.ImageVariation().Variation == android.VendorRamdiskVariation { + // TODO(b/165791368) + // Disable rlibs that link against dylib-std on vendor ramdisk variations until those dylib + // variants are properly supported. + dylib.Disable() + } + } } diff --git a/rust/rust.go b/rust/rust.go index 9dae75ee5..3402adcc5 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -37,20 +37,24 @@ var pctx = android.NewPackageContext("android/soong/rust") func init() { android.RegisterModuleType("rust_defaults", defaultsFactory) - android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("rust_libraries", LibraryMutator).Parallel() - ctx.BottomUp("rust_stdlinkage", LibstdMutator).Parallel() - ctx.BottomUp("rust_begin", BeginMutator).Parallel() - }) - android.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() - }) + android.PreDepsMutators(registerPreDepsMutators) + android.PostDepsMutators(registerPostDepsMutators) pctx.Import("android/soong/android") pctx.Import("android/soong/rust/config") pctx.ImportAs("cc_config", "android/soong/cc/config") android.InitRegistrationContext.RegisterParallelSingletonType("kythe_rust_extract", kytheExtractRustFactory) } +func registerPreDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.Transition("rust_libraries", &libraryTransitionMutator{}) + ctx.Transition("rust_stdlinkage", &libstdTransitionMutator{}) + ctx.BottomUp("rust_begin", BeginMutator).Parallel() +} + +func registerPostDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() +} + type Flags struct { GlobalRustFlags []string // Flags that apply globally to rust GlobalLinkFlags []string // Flags that apply globally to linker @@ -1128,10 +1132,11 @@ type autoDep struct { } var ( - rlibVariation = "rlib" - dylibVariation = "dylib" - rlibAutoDep = autoDep{variation: rlibVariation, depTag: rlibDepTag} - dylibAutoDep = autoDep{variation: dylibVariation, depTag: dylibDepTag} + sourceVariation = "source" + rlibVariation = "rlib" + dylibVariation = "dylib" + rlibAutoDep = autoDep{variation: rlibVariation, depTag: rlibDepTag} + dylibAutoDep = autoDep{variation: dylibVariation, depTag: dylibDepTag} ) type autoDeppable interface { @@ -1613,7 +1618,6 @@ func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { } rlibDepVariations := commonDepVariations - rlibDepVariations = append(rlibDepVariations, blueprint.Variation{Mutator: "link", Variation: ""}) if lib, ok := mod.compiler.(libraryInterface); !ok || !lib.sysroot() { rlibDepVariations = append(rlibDepVariations, @@ -1629,7 +1633,6 @@ func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { // dylibs dylibDepVariations := append(commonDepVariations, blueprint.Variation{Mutator: "rust_libraries", Variation: dylibVariation}) - dylibDepVariations = append(dylibDepVariations, blueprint.Variation{Mutator: "link", Variation: ""}) for _, lib := range deps.Dylibs { actx.AddVariationDependencies(dylibDepVariations, dylibDepTag, lib) @@ -1650,7 +1653,6 @@ func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { // otherwise select the rlib variant. autoDepVariations := append(commonDepVariations, blueprint.Variation{Mutator: "rust_libraries", Variation: autoDep.variation}) - autoDepVariations = append(autoDepVariations, blueprint.Variation{Mutator: "link", Variation: ""}) if actx.OtherModuleDependencyVariantExists(autoDepVariations, lib) { actx.AddVariationDependencies(autoDepVariations, autoDep.depTag, lib) @@ -1664,8 +1666,7 @@ func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { } else if _, ok := mod.sourceProvider.(*protobufDecorator); ok { for _, lib := range deps.Rustlibs { srcProviderVariations := append(commonDepVariations, - blueprint.Variation{Mutator: "rust_libraries", Variation: "source"}) - srcProviderVariations = append(srcProviderVariations, blueprint.Variation{Mutator: "link", Variation: ""}) + blueprint.Variation{Mutator: "rust_libraries", Variation: sourceVariation}) // Only add rustlib dependencies if they're source providers themselves. // This is used to track which crate names need to be added to the source generated @@ -1681,7 +1682,7 @@ func (mod *Module) DepsMutator(actx android.BottomUpMutatorContext) { if deps.Stdlibs != nil { if mod.compiler.stdLinkage(ctx) == RlibLinkage { for _, lib := range deps.Stdlibs { - actx.AddVariationDependencies(append(commonDepVariations, []blueprint.Variation{{Mutator: "rust_libraries", Variation: "rlib"}, {Mutator: "link", Variation: ""}}...), + actx.AddVariationDependencies(append(commonDepVariations, []blueprint.Variation{{Mutator: "rust_libraries", Variation: "rlib"}}...), rlibDepTag, lib) } } else { diff --git a/rust/rust_test.go b/rust/rust_test.go index 0d005d0a2..eeedf3f57 100644 --- a/rust/rust_test.go +++ b/rust/rust_test.go @@ -60,6 +60,7 @@ var rustMockedFiles = android.MockFS{ // testRust returns a TestContext in which a basic environment has been setup. // This environment contains a few mocked files. See rustMockedFiles for the list of these files. func testRust(t *testing.T, bp string) *android.TestContext { + t.Helper() skipTestIfOsNotSupported(t) result := android.GroupFixturePreparers( prepareForRustTest, diff --git a/rust/test.go b/rust/test.go index 3087d8d94..b7ddd06a3 100644 --- a/rust/test.go +++ b/rust/test.go @@ -116,7 +116,8 @@ func (test *testDecorator) compilerProps() []interface{} { } func (test *testDecorator) install(ctx ModuleContext) { - testInstallBase := "/data/local/tests/unrestricted" + // TODO: (b/167308193) Switch to /data/local/tests/unrestricted as the default install base. + testInstallBase := "/data/local/tmp" if ctx.RustModule().InVendorOrProduct() { testInstallBase = "/data/local/tests/vendor" } diff --git a/rust/testing.go b/rust/testing.go index 6ee49a971..32cc82354 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -200,15 +200,8 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.RegisterModuleType("rust_prebuilt_library", PrebuiltLibraryFactory) ctx.RegisterModuleType("rust_prebuilt_dylib", PrebuiltDylibFactory) ctx.RegisterModuleType("rust_prebuilt_rlib", PrebuiltRlibFactory) - ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - // rust mutators - ctx.BottomUp("rust_libraries", LibraryMutator).Parallel() - ctx.BottomUp("rust_stdlinkage", LibstdMutator).Parallel() - ctx.BottomUp("rust_begin", BeginMutator).Parallel() - }) + ctx.PreDepsMutators(registerPreDepsMutators) ctx.RegisterParallelSingletonType("rust_project_generator", rustProjectGeneratorSingleton) ctx.RegisterParallelSingletonType("kythe_rust_extract", kytheExtractRustFactory) - ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() - }) + ctx.PostDepsMutators(registerPostDepsMutators) } diff --git a/scripts/run-ckati.sh b/scripts/run-ckati.sh index b670c8af8..70f5a7a81 100755 --- a/scripts/run-ckati.sh +++ b/scripts/run-ckati.sh @@ -73,12 +73,12 @@ prebuilts/build-tools/linux-x86/bin/ckati \ --writable out/ \ -f build/make/core/main.mk \ "${tracing[@]}" \ - ANDROID_JAVA_HOME=prebuilts/jdk/jdk17/linux-x86 \ + ANDROID_JAVA_HOME=prebuilts/jdk/jdk21/linux-x86 \ ASAN_SYMBOLIZER_PATH=$PWD/prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer \ BUILD_DATETIME_FILE="$timestamp_file" \ BUILD_HOSTNAME=$(hostname) \ BUILD_USERNAME="$USER" \ - JAVA_HOME=$PWD/prebuilts/jdk/jdk17/linux-x86 \ + JAVA_HOME=$PWD/prebuilts/jdk/jdk21/linux-x86 \ KATI_PACKAGE_MK_DIR="{$out}/target/product/${target_device}/CONFIG/kati_packaging" \ OUT_DIR="$out" \ PATH="$PWD/prebuilts/build-tools/path/linux-x86:$PWD/${out}/.path" \ diff --git a/ui/build/Android.bp b/ui/build/Android.bp index ee286f68a..fcf29c52f 100644 --- a/ui/build/Android.bp +++ b/ui/build/Android.bp @@ -36,6 +36,7 @@ bootstrap_go_package { "blueprint-bootstrap", "blueprint-microfactory", "soong-android", + "soong-elf", "soong-finder", "soong-remoteexec", "soong-shared", diff --git a/ui/build/build.go b/ui/build/build.go index 03d839237..c7319ed33 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -22,6 +22,7 @@ import ( "sync" "text/template" + "android/soong/elf" "android/soong/ui/metrics" ) @@ -344,6 +345,7 @@ func Build(ctx Context, config Config) { installCleanIfNecessary(ctx, config) } runNinjaForBuild(ctx, config) + updateBuildIdDir(ctx, config) } if what&RunDistActions != 0 { @@ -351,6 +353,16 @@ func Build(ctx Context, config Config) { } } +func updateBuildIdDir(ctx Context, config Config) { + ctx.BeginTrace(metrics.RunShutdownTool, "update_build_id_dir") + defer ctx.EndTrace() + + symbolsDir := filepath.Join(config.ProductOut(), "symbols") + if err := elf.UpdateBuildIdDir(symbolsDir); err != nil { + ctx.Printf("failed to update %s/.build-id: %v", symbolsDir, err) + } +} + func evaluateWhatToRun(config Config, verboseln func(v ...interface{})) int { //evaluate what to run what := 0 diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go index 81c678d3f..6c9a1ebb9 100644 --- a/ui/build/paths/config.go +++ b/ui/build/paths/config.go @@ -86,28 +86,28 @@ func GetConfig(name string) PathConfig { // This list specifies whether a particular binary from $PATH is allowed to be // run during the build. For more documentation, see path_interposer.go . var Configuration = map[string]PathConfig{ - "bash": Allowed, - "diff": Allowed, - "dlv": Allowed, - "expr": Allowed, - "fuser": Allowed, - "gcert": Allowed, - "gcertstatus": Allowed, - "gcloud": Allowed, - "git": Allowed, - "hexdump": Allowed, - "jar": Allowed, - "java": Allowed, - "javap": Allowed, - "lsof": Allowed, - "openssl": Allowed, - "pstree": Allowed, - "rsync": Allowed, - "sh": Allowed, - "stubby": Allowed, - "tr": Allowed, - "unzip": Allowed, - "zip": Allowed, + "bash": Allowed, + "diff": Allowed, + "dlv": Allowed, + "expr": Allowed, + "fuser": Allowed, + "gcert": Allowed, + "gcertstatus": Allowed, + "gcloud": Allowed, + "git": Allowed, + "hexdump": Allowed, + "jar": Allowed, + "java": Allowed, + "javap": Allowed, + "lsof": Allowed, + "openssl": Allowed, + "pstree": Allowed, + "rsync": Allowed, + "sh": Allowed, + "stubby": Allowed, + "tr": Allowed, + "unzip": Allowed, + "zip": Allowed, // Host toolchain is removed. In-tree toolchain should be used instead. // GCC also can't find cc1 with this implementation. diff --git a/ui/terminal/format.go b/ui/terminal/format.go index 241a1ddf7..01f8b0d13 100644 --- a/ui/terminal/format.go +++ b/ui/terminal/format.go @@ -23,26 +23,28 @@ import ( ) type formatter struct { - format string - quiet bool - start time.Time + colorize bool + format string + quiet bool + start time.Time } // newFormatter returns a formatter for formatting output to // the terminal in a format similar to Ninja. // format takes nearly all the same options as NINJA_STATUS. // %c is currently unsupported. -func newFormatter(format string, quiet bool) formatter { +func newFormatter(colorize bool, format string, quiet bool) formatter { return formatter{ - format: format, - quiet: quiet, - start: time.Now(), + colorize: colorize, + format: format, + quiet: quiet, + start: time.Now(), } } func (s formatter) message(level status.MsgLevel, message string) string { if level >= status.ErrorLvl { - return fmt.Sprintf("FAILED: %s", message) + return fmt.Sprintf("%s %s", s.failedString(), message) } else if level > status.StatusLvl { return fmt.Sprintf("%s%s", level.Prefix(), message) } else if level == status.StatusLvl { @@ -127,9 +129,9 @@ func (s formatter) result(result status.ActionResult) string { if result.Error != nil { targets := strings.Join(result.Outputs, " ") if s.quiet || result.Command == "" { - ret = fmt.Sprintf("FAILED: %s\n%s", targets, result.Output) + ret = fmt.Sprintf("%s %s\n%s", s.failedString(), targets, result.Output) } else { - ret = fmt.Sprintf("FAILED: %s\n%s\n%s", targets, result.Command, result.Output) + ret = fmt.Sprintf("%s %s\n%s\n%s", s.failedString(), targets, result.Command, result.Output) } } else if result.Output != "" { ret = result.Output @@ -141,3 +143,11 @@ func (s formatter) result(result status.ActionResult) string { return ret } + +func (s formatter) failedString() string { + failed := "FAILED:" + if s.colorize { + failed = ansi.red() + ansi.bold() + failed + ansi.regular() + } + return failed +} diff --git a/ui/terminal/status.go b/ui/terminal/status.go index 2ad174fee..92f299405 100644 --- a/ui/terminal/status.go +++ b/ui/terminal/status.go @@ -27,9 +27,10 @@ import ( // statusFormat takes nearly all the same options as NINJA_STATUS. // %c is currently unsupported. func NewStatusOutput(w io.Writer, statusFormat string, forceSimpleOutput, quietBuild, forceKeepANSI bool) status.StatusOutput { - formatter := newFormatter(statusFormat, quietBuild) + canUseSmartFormatting := !forceSimpleOutput && isSmartTerminal(w) + formatter := newFormatter(canUseSmartFormatting, statusFormat, quietBuild) - if !forceSimpleOutput && isSmartTerminal(w) { + if canUseSmartFormatting { return NewSmartStatusOutput(w, formatter) } else { return NewSimpleStatusOutput(w, formatter, forceKeepANSI) diff --git a/ui/terminal/status_test.go b/ui/terminal/status_test.go index 8dd180967..991eca04e 100644 --- a/ui/terminal/status_test.go +++ b/ui/terminal/status_test.go @@ -58,7 +58,7 @@ func TestStatusOutput(t *testing.T) { { name: "action with error", calls: actionsWithError, - smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\nFAILED: f1 f2\ntouch f1 f2\nerror1\nerror2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", + smart: "\r\x1b[1m[ 0% 0/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action1\x1b[0m\x1b[K\r\x1b[1m[ 33% 1/3] action2\x1b[0m\x1b[K\r\x1b[1m[ 66% 2/3] action2\x1b[0m\x1b[K\n\x1b[31m\x1b[1mFAILED:\x1b[0m f1 f2\ntouch f1 f2\nerror1\nerror2\n\r\x1b[1m[ 66% 2/3] action3\x1b[0m\x1b[K\r\x1b[1m[100% 3/3] action3\x1b[0m\x1b[K\n", simple: "[ 33% 1/3] action1\n[ 66% 2/3] action2\nFAILED: f1 f2\ntouch f1 f2\nerror1\nerror2\n[100% 3/3] action3\n", }, { @@ -70,7 +70,7 @@ func TestStatusOutput(t *testing.T) { { name: "messages", calls: actionsWithMessages, - smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1mstatus\x1b[0m\x1b[K\r\x1b[Kprint\nFAILED: error\n\r\x1b[1m[ 50% 1/2] action2\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", + smart: "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\r\x1b[1mstatus\x1b[0m\x1b[K\r\x1b[Kprint\n\x1b[31m\x1b[1mFAILED:\x1b[0m error\n\r\x1b[1m[ 50% 1/2] action2\x1b[0m\x1b[K\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n", simple: "[ 50% 1/2] action1\nstatus\nprint\nFAILED: error\n[100% 2/2] action2\n", }, { @@ -362,7 +362,7 @@ func TestSmartStatusHideAfterFailure(t *testing.T) { stat.Flush() - w := "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\nFAILED: \nOutput1\n\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\nThere was 1 action that completed after the action that failed. See verbose.log.gz for its output.\n" + w := "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\n\x1b[31m\x1b[1mFAILED:\x1b[0m \nOutput1\n\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\nThere was 1 action that completed after the action that failed. See verbose.log.gz for its output.\n" if g := smart.String(); g != w { t.Errorf("want:\n%q\ngot:\n%q", w, g) @@ -407,7 +407,7 @@ func TestSmartStatusHideAfterFailurePlural(t *testing.T) { stat.Flush() - w := "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action3\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\nFAILED: \nOutput1\n\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\r\x1b[1m[150% 3/2] action3\x1b[0m\x1b[K\nThere were 2 actions that completed after the action that failed. See verbose.log.gz for their output.\n" + w := "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action3\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\n\x1b[31m\x1b[1mFAILED:\x1b[0m \nOutput1\n\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\r\x1b[1m[150% 3/2] action3\x1b[0m\x1b[K\nThere were 2 actions that completed after the action that failed. See verbose.log.gz for their output.\n" if g := smart.String(); g != w { t.Errorf("want:\n%q\ngot:\n%q", w, g) @@ -445,7 +445,7 @@ func TestSmartStatusDontHideErrorAfterFailure(t *testing.T) { stat.Flush() - w := "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\nFAILED: \nOutput1\n\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\nFAILED: \nOutput2\n" + w := "\r\x1b[1m[ 0% 0/2] action1\x1b[0m\x1b[K\r\x1b[1m[ 0% 0/2] action2\x1b[0m\x1b[K\r\x1b[1m[ 50% 1/2] action1\x1b[0m\x1b[K\n\x1b[31m\x1b[1mFAILED:\x1b[0m \nOutput1\n\r\x1b[1m[100% 2/2] action2\x1b[0m\x1b[K\n\x1b[31m\x1b[1mFAILED:\x1b[0m \nOutput2\n" if g := smart.String(); g != w { t.Errorf("want:\n%q\ngot:\n%q", w, g) |