diff options
101 files changed, 1709 insertions, 4486 deletions
@@ -594,19 +594,13 @@ modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced by all of the vendor's other modules using the normal namespace and visibility rules. -`soongConfigTraceMutator` enables modules affected by soong config variables to -write outputs into a hashed directory path. It does this by recording accesses -to soong config variables on each module, and then accumulating records of each -module's all dependencies. `m soong_config_trace` builds information about -hashes to `$OUT_DIR/soong/soong_config_trace.json`. - ## Build logic The build logic is written in Go using the -[blueprint](http://godoc.org/github.com/google/blueprint) framework. Build -logic receives module definitions parsed into Go structures using reflection -and produces build rules. The build rules are collected by blueprint and -written to a [ninja](http://ninja-build.org) build file. +[blueprint](https://android.googlesource.com/platform/build/blueprint) +framework. Build logic receives module definitions parsed into Go structures +using reflection and produces build rules. The build rules are collected by +blueprint and written to a [ninja](http://ninja-build.org) build file. ## Environment Variables Config File diff --git a/aconfig/codegen/Android.bp b/aconfig/codegen/Android.bp index 0c78b946b..5fac0a8d1 100644 --- a/aconfig/codegen/Android.bp +++ b/aconfig/codegen/Android.bp @@ -12,7 +12,6 @@ bootstrap_go_package { "soong", "soong-aconfig", "soong-android", - "soong-bazel", "soong-java", "soong-rust", ], diff --git a/android/Android.bp b/android/Android.bp index ce8c9b0b3..3c38148b6 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -40,6 +40,7 @@ bootstrap_go_package { "base_module_context.go", "build_prop.go", "buildinfo_prop.go", + "compliance_metadata.go", "config.go", "test_config.go", "configurable_properties.go", @@ -112,6 +113,7 @@ bootstrap_go_package { "androidmk_test.go", "apex_test.go", "arch_test.go", + "blueprint_e2e_test.go", "config_test.go", "configured_jars_test.go", "csuite_config_test.go", diff --git a/android/androidmk_test.go b/android/androidmk_test.go index ae2187f48..72b8654a7 100644 --- a/android/androidmk_test.go +++ b/android/androidmk_test.go @@ -36,10 +36,6 @@ type customModule struct { data AndroidMkData distFiles TaggedDistFiles outputFile OptionalPath - - // The paths that will be used as the default dist paths if no tag is - // specified. - defaultDistPaths Paths } const ( @@ -51,6 +47,7 @@ const ( func (m *customModule) GenerateAndroidBuildActions(ctx ModuleContext) { m.base().licenseMetadataFile = PathForOutput(ctx, "meta_lic") + var defaultDistPaths Paths // If the dist_output_file: true then create an output file that is stored in // the OutputFile property of the AndroidMkEntry. @@ -62,7 +59,7 @@ func (m *customModule) GenerateAndroidBuildActions(ctx ModuleContext) { // property in AndroidMkEntry when determining the default dist paths. // Setting this first allows it to be overridden based on the // default_dist_files setting replicating that previous behavior. - m.defaultDistPaths = Paths{path} + defaultDistPaths = Paths{path} } // Based on the setting of the default_dist_files property possibly create a @@ -71,29 +68,40 @@ func (m *customModule) GenerateAndroidBuildActions(ctx ModuleContext) { defaultDistFiles := proptools.StringDefault(m.properties.Default_dist_files, defaultDistFiles_Tagged) switch defaultDistFiles { case defaultDistFiles_None: - // Do nothing + m.setOutputFiles(ctx, defaultDistPaths) case defaultDistFiles_Default: path := PathForTesting("default-dist.out") - m.defaultDistPaths = Paths{path} + defaultDistPaths = Paths{path} + m.setOutputFiles(ctx, defaultDistPaths) m.distFiles = MakeDefaultDistFiles(path) case defaultDistFiles_Tagged: // Module types that set AndroidMkEntry.DistFiles to the result of calling // GenerateTaggedDistFiles(ctx) relied on no tag being treated as "" which - // meant that the default dist paths would be whatever was returned by - // OutputFiles(""). In order to preserve that behavior when treating no tag - // as being equal to DefaultDistTag this ensures that - // OutputFiles(DefaultDistTag) will return the same as OutputFiles(""). - m.defaultDistPaths = PathsForTesting("one.out") + // meant that the default dist paths would be the same as empty-string-tag + // output files. In order to preserve that behavior when treating no tag + // as being equal to DefaultDistTag this ensures that DefaultDistTag output + // will be the same as empty-string-tag output. + defaultDistPaths = PathsForTesting("one.out") + m.setOutputFiles(ctx, defaultDistPaths) // This must be called after setting defaultDistPaths/outputFile as - // GenerateTaggedDistFiles calls into OutputFiles(tag) which may use those - // fields. + // GenerateTaggedDistFiles calls into outputFiles property which may use + // those fields. m.distFiles = m.GenerateTaggedDistFiles(ctx) } } +func (m *customModule) setOutputFiles(ctx ModuleContext, defaultDistPaths Paths) { + ctx.SetOutputFiles(PathsForTesting("one.out"), "") + ctx.SetOutputFiles(PathsForTesting("two.out", "three/four.out"), ".multiple") + ctx.SetOutputFiles(PathsForTesting("another.out"), ".another-tag") + if defaultDistPaths != nil { + ctx.SetOutputFiles(defaultDistPaths, DefaultDistTag) + } +} + func (m *customModule) AndroidMk() AndroidMkData { return AndroidMkData{ Custom: func(w io.Writer, name, prefix, moduleDir string, data AndroidMkData) { @@ -102,25 +110,6 @@ func (m *customModule) AndroidMk() AndroidMkData { } } -func (m *customModule) OutputFiles(tag string) (Paths, error) { - switch tag { - case DefaultDistTag: - if m.defaultDistPaths != nil { - return m.defaultDistPaths, nil - } else { - return nil, fmt.Errorf("default dist tag is not available") - } - case "": - return PathsForTesting("one.out"), nil - case ".multiple": - return PathsForTesting("two.out", "three/four.out"), nil - case ".another-tag": - return PathsForTesting("another.out"), nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (m *customModule) AndroidMkEntries() []AndroidMkEntries { return []AndroidMkEntries{ { diff --git a/android/blueprint_e2e_test.go b/android/blueprint_e2e_test.go new file mode 100644 index 000000000..b27451218 --- /dev/null +++ b/android/blueprint_e2e_test.go @@ -0,0 +1,105 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "testing" +) + +var testCases []struct { + name string + fs MockFS + expectedError string +} = []struct { + name string + fs MockFS + expectedError string +}{ + { + name: "Can't reference variable before assignment", + fs: map[string][]byte{ + "Android.bp": []byte(` +x = foo +foo = "hello" +`), + }, + expectedError: "undefined variable foo", + }, + { + name: "Can't append to variable before assigned to", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo += "world" +foo = "hello" +`), + }, + expectedError: "modified non-existent variable \"foo\" with \\+=", + }, + { + name: "Can't reassign variable", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +foo = "world" +`), + }, + expectedError: "variable already set, previous assignment:", + }, + { + name: "Can't reassign variable in inherited scope", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +`), + "foo/Android.bp": []byte(` +foo = "world" +`), + }, + expectedError: "variable already set in inherited scope, previous assignment:", + }, + { + name: "Can't modify variable in inherited scope", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +`), + "foo/Android.bp": []byte(` +foo += "world" +`), + }, + expectedError: "modified non-local variable \"foo\" with \\+=", + }, + { + name: "Can't modify variable after referencing", + fs: map[string][]byte{ + "Android.bp": []byte(` +foo = "hello" +x = foo +foo += "world" +`), + }, + expectedError: "modified variable \"foo\" with \\+= after referencing", + }, +} + +func TestBlueprintErrors(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fixtures := FixtureMergeMockFs(tc.fs) + fixtures = fixtures.ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(tc.expectedError)) + fixtures.RunTest(t) + }) + } +} diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go index defbff0b0..bba4c0d24 100644 --- a/android/buildinfo_prop.go +++ b/android/buildinfo_prop.go @@ -15,8 +15,6 @@ package android import ( - "fmt" - "github.com/google/blueprint/proptools" ) @@ -41,20 +39,10 @@ type buildinfoPropModule struct { installPath InstallPath } -var _ OutputFileProducer = (*buildinfoPropModule)(nil) - func (p *buildinfoPropModule) installable() bool { return proptools.BoolDefault(p.properties.Installable, true) } -// OutputFileProducer -func (p *buildinfoPropModule) OutputFiles(tag string) (Paths, error) { - if tag != "" { - return nil, fmt.Errorf("unsupported tag %q", tag) - } - return Paths{p.outputFilePath}, nil -} - func shouldAddBuildThumbprint(config Config) bool { knownOemProperties := []string{ "ro.product.brand", @@ -76,6 +64,8 @@ func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { return } p.outputFilePath = PathForModuleOut(ctx, p.Name()).OutputPath + ctx.SetOutputFiles(Paths{p.outputFilePath}, "") + if !ctx.Config().KatiEnabled() { WriteFileRule(ctx, p.outputFilePath, "# no buildinfo.prop if kati is disabled") return diff --git a/android/compliance_metadata.go b/android/compliance_metadata.go new file mode 100644 index 000000000..6ea66541a --- /dev/null +++ b/android/compliance_metadata.go @@ -0,0 +1,314 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "bytes" + "encoding/csv" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/google/blueprint" +) + +var ( + // Constants of property names used in compliance metadata of modules + ComplianceMetadataProp = struct { + NAME string + PACKAGE string + MODULE_TYPE string + OS string + ARCH string + IS_PRIMARY_ARCH string + VARIANT string + IS_STATIC_LIB string + INSTALLED_FILES string + BUILT_FILES string + STATIC_DEPS string + STATIC_DEP_FILES string + WHOLE_STATIC_DEPS string + WHOLE_STATIC_DEP_FILES string + LICENSES string + + // module_type=package + PKG_DEFAULT_APPLICABLE_LICENSES string + + // module_type=license + LIC_LICENSE_KINDS string + LIC_LICENSE_TEXT string + LIC_PACKAGE_NAME string + + // module_type=license_kind + LK_CONDITIONS string + LK_URL string + }{ + "name", + "package", + "module_type", + "os", + "arch", + "is_primary_arch", + "variant", + "is_static_lib", + "installed_files", + "built_files", + "static_deps", + "static_dep_files", + "whole_static_deps", + "whole_static_dep_files", + "licenses", + + "pkg_default_applicable_licenses", + + "lic_license_kinds", + "lic_license_text", + "lic_package_name", + + "lk_conditions", + "lk_url", + } + + // A constant list of all property names in compliance metadata + // Order of properties here is the order of columns in the exported CSV file. + COMPLIANCE_METADATA_PROPS = []string{ + ComplianceMetadataProp.NAME, + ComplianceMetadataProp.PACKAGE, + ComplianceMetadataProp.MODULE_TYPE, + ComplianceMetadataProp.OS, + ComplianceMetadataProp.ARCH, + ComplianceMetadataProp.VARIANT, + ComplianceMetadataProp.IS_STATIC_LIB, + ComplianceMetadataProp.IS_PRIMARY_ARCH, + // Space separated installed files + ComplianceMetadataProp.INSTALLED_FILES, + // Space separated built files + ComplianceMetadataProp.BUILT_FILES, + // Space separated module names of static dependencies + ComplianceMetadataProp.STATIC_DEPS, + // Space separated file paths of static dependencies + ComplianceMetadataProp.STATIC_DEP_FILES, + // Space separated module names of whole static dependencies + ComplianceMetadataProp.WHOLE_STATIC_DEPS, + // Space separated file paths of whole static dependencies + ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, + ComplianceMetadataProp.LICENSES, + // module_type=package + ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES, + // module_type=license + ComplianceMetadataProp.LIC_LICENSE_KINDS, + ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths + ComplianceMetadataProp.LIC_PACKAGE_NAME, + // module_type=license_kind + ComplianceMetadataProp.LK_CONDITIONS, + ComplianceMetadataProp.LK_URL, + } +) + +// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license, +// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility +// methods to get/set properties' values. +type ComplianceMetadataInfo struct { + properties map[string]string +} + +func NewComplianceMetadataInfo() *ComplianceMetadataInfo { + return &ComplianceMetadataInfo{ + properties: map[string]string{}, + } +} + +func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) { + if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { + panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) + } + c.properties[propertyName] = value +} + +func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) { + c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " "))) +} + +func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string { + if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) { + panic(fmt.Errorf("Unknown metadata property: %s.", propertyName)) + } + return c.properties[propertyName] +} + +func (c *ComplianceMetadataInfo) getAllValues() map[string]string { + return c.properties +} + +var ( + ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]() +) + +// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata +// for different module types without accessing their private fields but through android.Module interface +// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider. +func buildComplianceMetadataProvider(ctx ModuleContext, m *ModuleBase) { + complianceMetadataInfo := ctx.ComplianceMetadataInfo() + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType()) + + switch ctx.ModuleType() { + case "license": + licenseModule := m.module.(*licenseModule) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name)) + case "license_kind": + licenseKindModule := m.module.(*licenseKindModule) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url) + default: + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String()) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch())) + complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir()) + if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" { + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings()) + } + + var installed InstallPaths + installed = append(installed, m.module.FilesToInstall()...) + installed = append(installed, m.katiInstalls.InstallPaths()...) + installed = append(installed, m.katiSymlinks.InstallPaths()...) + installed = append(installed, m.katiInitRcInstalls.InstallPaths()...) + installed = append(installed, m.katiVintfInstalls.InstallPaths()...) + complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings())) + } + ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo) +} + +func init() { + RegisterComplianceMetadataSingleton(InitRegistrationContext) +} + +func RegisterComplianceMetadataSingleton(ctx RegistrationContext) { + ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory) +} + +var ( + // sqlite3 command line tool + sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3") + + // Command to import .csv files to sqlite3 database + importCsv = pctx.AndroidStaticRule("importCsv", + blueprint.RuleParams{ + Command: `rm -rf $out && ` + + `${sqlite3} $out ".import --csv $in modules" && ` + + `${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` + + `${sqlite3} $out ".import --csv ${make_modules} make_modules"`, + CommandDeps: []string{"${sqlite3}"}, + }, "make_metadata", "make_modules") +) + +func complianceMetadataSingletonFactory() Singleton { + return &complianceMetadataSingleton{} +} + +type complianceMetadataSingleton struct { +} + +func writerToCsv(csvWriter *csv.Writer, row []string) { + err := csvWriter.Write(row) + if err != nil { + panic(err) + } +} + +// Collect compliance metadata from all Soong modules, write to a CSV file and +// import compliance metadata from Make and Soong to a sqlite3 database. +func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) { + if !ctx.Config().HasDeviceProduct() { + return + } + var buffer bytes.Buffer + csvWriter := csv.NewWriter(&buffer) + + // Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file. + columnNames := []string{"id"} + columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...) + writerToCsv(csvWriter, columnNames) + + rowId := -1 + ctx.VisitAllModules(func(module Module) { + if !module.Enabled(ctx) { + return + } + moduleType := ctx.ModuleType(module) + if moduleType == "package" { + metadataMap := map[string]string{ + ComplianceMetadataProp.NAME: ctx.ModuleName(module), + ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module), + ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "), + } + rowId = rowId + 1 + metadata := []string{strconv.Itoa(rowId)} + for _, propertyName := range COMPLIANCE_METADATA_PROPS { + metadata = append(metadata, metadataMap[propertyName]) + } + writerToCsv(csvWriter, metadata) + return + } + if provider, ok := ctx.moduleProvider(module, ComplianceMetadataProvider); ok { + metadataInfo := provider.(*ComplianceMetadataInfo) + rowId = rowId + 1 + metadata := []string{strconv.Itoa(rowId)} + for _, propertyName := range COMPLIANCE_METADATA_PROPS { + metadata = append(metadata, metadataInfo.getStringValue(propertyName)) + } + writerToCsv(csvWriter, metadata) + return + } + }) + csvWriter.Flush() + + deviceProduct := ctx.Config().DeviceProduct() + modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv") + WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String()) + + // Metadata generated in Make + makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv") + makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv") + + // Import metadata from Make and Soong to sqlite3 database + complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db") + ctx.Build(pctx, BuildParams{ + Rule: importCsv, + Input: modulesCsv, + Implicits: []Path{ + makeMetadataCsv, + makeModulesCsv, + }, + Output: complianceMetadataDb, + Args: map[string]string{ + "make_metadata": makeMetadataCsv.String(), + "make_modules": makeModulesCsv.String(), + }, + }) + + // Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database. + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Inputs: []Path{complianceMetadataDb}, + Output: PathForPhony(ctx, "compliance-metadata.db"), + }) + +} diff --git a/android/image.go b/android/image.go index 9cad05656..c278dcdf9 100644 --- a/android/image.go +++ b/android/image.go @@ -19,6 +19,12 @@ type ImageInterface interface { // ImageMutatorBegin is called before any other method in the ImageInterface. ImageMutatorBegin(ctx BaseModuleContext) + // VendorVariantNeeded should return true if the module needs a vendor variant (installed on the vendor image). + VendorVariantNeeded(ctx BaseModuleContext) bool + + // ProductVariantNeeded should return true if the module needs a product variant (unstalled on the product image). + ProductVariantNeeded(ctx BaseModuleContext) bool + // CoreVariantNeeded should return true if the module needs a core variant (installed on the system image). CoreVariantNeeded(ctx BaseModuleContext) bool @@ -49,6 +55,14 @@ type ImageInterface interface { } const ( + // VendorVariation is the variant name used for /vendor code that does not + // compile against the VNDK. + VendorVariation string = "vendor" + + // ProductVariation is the variant name used for /product code that does not + // compile against the VNDK. + ProductVariation string = "product" + // CoreVariation is the variant used for framework-private libraries, or // SDK libraries. (which framework-private libraries can use), which // will be installed to the system image. @@ -94,6 +108,12 @@ func imageMutator(ctx BottomUpMutatorContext) { if m.RecoveryVariantNeeded(ctx) { variations = append(variations, RecoveryVariation) } + if m.VendorVariantNeeded(ctx) { + variations = append(variations, VendorVariation) + } + if m.ProductVariantNeeded(ctx) { + variations = append(variations, ProductVariation) + } extraVariations := m.ExtraImageVariations(ctx) variations = append(variations, extraVariations...) diff --git a/android/module.go b/android/module.go index b43815063..66f6e1bcd 100644 --- a/android/module.go +++ b/android/module.go @@ -15,9 +15,6 @@ package android import ( - "crypto/md5" - "encoding/hex" - "encoding/json" "fmt" "net/url" "path/filepath" @@ -26,8 +23,6 @@ import ( "sort" "strings" - "android/soong/bazel" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -249,31 +244,6 @@ func SortedUniqueNamedPaths(l NamedPaths) NamedPaths { return l[:k+1] } -// soongConfigTrace holds all references to VendorVars. Uses []string for blueprint:"mutated" -type soongConfigTrace struct { - Bools []string `json:",omitempty"` - Strings []string `json:",omitempty"` - IsSets []string `json:",omitempty"` -} - -func (c *soongConfigTrace) isEmpty() bool { - return len(c.Bools) == 0 && len(c.Strings) == 0 && len(c.IsSets) == 0 -} - -// Returns hash of serialized trace records (empty string if there's no trace recorded) -func (c *soongConfigTrace) hash() string { - // Use MD5 for speed. We don't care collision or preimage attack - if c.isEmpty() { - return "" - } - j, err := json.Marshal(c) - if err != nil { - panic(fmt.Errorf("json marshal of %#v failed: %#v", *c, err)) - } - hash := md5.Sum(j) - return hex.EncodeToString(hash[:]) -} - type nameProperties struct { // The name of the module. Must be unique across all modules. Name *string @@ -525,14 +495,6 @@ type commonProperties struct { // constants in image.go, but can also be set to a custom value by individual module types. ImageVariation string `blueprint:"mutated"` - // SoongConfigTrace records accesses to VendorVars (soong_config). The trace will be hashed - // and used as a subdir of PathForModuleOut. Note that we mainly focus on incremental - // builds among similar products (e.g. aosp_cf_x86_64_phone and aosp_cf_x86_64_foldable), - // and there are variables other than soong_config, which isn't captured by soong config - // trace, but influence modules among products. - SoongConfigTrace soongConfigTrace `blueprint:"mutated"` - SoongConfigTraceHash string `blueprint:"mutated"` - // The team (defined by the owner/vendor) who owns the property. Team *string `android:"path"` } @@ -849,9 +811,6 @@ type ModuleBase struct { // archPropRoot that is filled with arch specific values by the arch mutator. archProperties [][]interface{} - // Properties specific to the Blueprint to BUILD migration. - bazelTargetModuleProperties bazel.BazelTargetModuleProperties - // Information about all the properties on the module that contains visibility rules that need // checking. visibilityPropertyInfo []visibilityProperty @@ -919,6 +878,10 @@ type ModuleBase struct { // outputFiles stores the output of a module by tag and is used to set // the OutputFilesProvider in GenerateBuildActions outputFiles OutputFilesInfo + + // complianceMetadataInfo is for different module types to dump metadata. + // See android.ModuleContext interface. + complianceMetadataInfo *ComplianceMetadataInfo } func (m *ModuleBase) AddJSONData(d *map[string]interface{}) { @@ -1235,17 +1198,28 @@ func (m *ModuleBase) GenerateTaggedDistFiles(ctx BaseModuleContext) TaggedDistFi // the special tag name which represents that. tag := proptools.StringDefault(dist.Tag, DefaultDistTag) + distFileForTagFromProvider, err := outputFilesForModuleFromProvider(ctx, m.module, tag) + if err != OutputFilesProviderNotSet { + if err != nil && tag != DefaultDistTag { + ctx.PropertyErrorf("dist.tag", "%s", err.Error()) + } else { + distFiles = distFiles.addPathsForTag(tag, distFileForTagFromProvider...) + continue + } + } + + // if the tagged dist file cannot be obtained from OutputFilesProvider, + // fall back to use OutputFileProducer + // TODO: remove this part after OutputFilesProvider fully replaces OutputFileProducer if outputFileProducer, ok := m.module.(OutputFileProducer); ok { // Call the OutputFiles(tag) method to get the paths associated with the tag. distFilesForTag, err := outputFileProducer.OutputFiles(tag) - // If the tag was not supported and is not DefaultDistTag then it is an error. // Failing to find paths for DefaultDistTag is not an error. It just means // that the module type requires the legacy behavior. if err != nil && tag != DefaultDistTag { ctx.PropertyErrorf("dist.tag", "%s", err.Error()) } - distFiles = distFiles.addPathsForTag(tag, distFilesForTag...) } else if tag != DefaultDistTag { // If the tag was specified then it is an error if the module does not @@ -2049,6 +2023,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) if m.outputFiles.DefaultOutputFiles != nil || m.outputFiles.TaggedOutputFiles != nil { SetProvider(ctx, OutputFilesProvider, m.outputFiles) } + + buildComplianceMetadataProvider(ctx, m) } func SetJarJarPrefixHandler(handler func(ModuleContext)) { @@ -2253,7 +2229,20 @@ func (e configurationEvalutor) EvaluateConfiguration(condition proptools.Configu variable := condition.Arg(1) if n, ok := ctx.Config().productVariables.VendorVars[namespace]; ok { if v, ok := n[variable]; ok { - return proptools.ConfigurableValueString(v) + ty := "" + if namespaces, ok := ctx.Config().productVariables.VendorVarTypes[namespace]; ok { + ty = namespaces[variable] + } + switch ty { + case "": + // strings are the default, we don't bother writing them to the soong variables json file + return proptools.ConfigurableValueString(v) + case "bool": + return proptools.ConfigurableValueBool(v == "true") + default: + panic("unhandled soong config variable type: " + ty) + } + } } return proptools.ConfigurableValueUndefined() @@ -2499,7 +2488,7 @@ func OutputFileForModule(ctx PathContext, module blueprint.Module, tag string) P func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { outputFilesFromProvider, err := outputFilesForModuleFromProvider(ctx, module, tag) - if outputFilesFromProvider != nil || err != nil { + if outputFilesFromProvider != nil || err != OutputFilesProviderNotSet { return outputFilesFromProvider, err } if outputFileProducer, ok := module.(OutputFileProducer); ok { @@ -2524,34 +2513,45 @@ func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) // *inter-module-communication*. // If mctx module is the same as the param module the output files are obtained // from outputFiles property of module base, to avoid both setting and -// reading OutputFilesProvider before GenerateBuildActions is finished. Also -// only empty-string-tag is supported in this case. +// reading OutputFilesProvider before GenerateBuildActions is finished. // If a module doesn't have the OutputFilesProvider, nil is returned. func outputFilesForModuleFromProvider(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { - // TODO: support OutputFilesProvider for singletons - mctx, ok := ctx.(ModuleContext) - if !ok { - return nil, nil - } - if mctx.Module() != module { - if outputFilesProvider, ok := OtherModuleProvider(mctx, module, OutputFilesProvider); ok { - if tag == "" { - return outputFilesProvider.DefaultOutputFiles, nil - } else if taggedOutputFiles, hasTag := outputFilesProvider.TaggedOutputFiles[tag]; hasTag { - return taggedOutputFiles, nil - } else { - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } + var outputFiles OutputFilesInfo + fromProperty := false + + if mctx, isMctx := ctx.(ModuleContext); isMctx { + if mctx.Module() != module { + outputFiles, _ = OtherModuleProvider(mctx, module, OutputFilesProvider) + } else { + outputFiles = mctx.Module().base().outputFiles + fromProperty = true } + } else if cta, isCta := ctx.(*singletonContextAdaptor); isCta { + providerData, _ := cta.moduleProvider(module, OutputFilesProvider) + outputFiles, _ = providerData.(OutputFilesInfo) + } + // TODO: Add a check for skipped context + + if outputFiles.isEmpty() { + // TODO: Add a check for param module not having OutputFilesProvider set + return nil, OutputFilesProviderNotSet + } + + if tag == "" { + return outputFiles.DefaultOutputFiles, nil + } else if taggedOutputFiles, hasTag := outputFiles.TaggedOutputFiles[tag]; hasTag { + return taggedOutputFiles, nil } else { - if tag == "" { - return mctx.Module().base().outputFiles.DefaultOutputFiles, nil - } else { + if fromProperty { return nil, fmt.Errorf("unsupported tag %q for module getting its own output files", tag) + } else { + return nil, fmt.Errorf("unsupported module reference tag %q", tag) } } - // TODO: Add a check for param module not having OutputFilesProvider set - return nil, nil +} + +func (o OutputFilesInfo) isEmpty() bool { + return o.DefaultOutputFiles == nil && o.TaggedOutputFiles == nil } type OutputFilesInfo struct { @@ -2564,6 +2564,9 @@ type OutputFilesInfo struct { var OutputFilesProvider = blueprint.NewProvider[OutputFilesInfo]() +// This is used to mark the case where OutputFilesProvider is not set on some modules. +var OutputFilesProviderNotSet = fmt.Errorf("No output files from provider") + // Modules can implement HostToolProvider and return a valid OptionalPath from HostToolPath() to // specify that they can be used as a tool by a genrule module. type HostToolProvider interface { @@ -2575,8 +2578,6 @@ type HostToolProvider interface { func init() { RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) - RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc) - FinalDepsMutators(registerSoongConfigTraceMutator) } func BuildTargetSingleton() Singleton { @@ -2738,54 +2739,3 @@ func CheckBlueprintSyntax(ctx BaseModuleContext, filename string, contents strin bpctx := ctx.blueprintBaseModuleContext() return blueprint.CheckBlueprintSyntax(bpctx.ModuleFactories(), filename, contents) } - -func registerSoongConfigTraceMutator(ctx RegisterMutatorsContext) { - ctx.BottomUp("soongconfigtrace", soongConfigTraceMutator).Parallel() -} - -// soongConfigTraceMutator accumulates recorded soong_config trace from children. Also it normalizes -// SoongConfigTrace to make it consistent. -func soongConfigTraceMutator(ctx BottomUpMutatorContext) { - trace := &ctx.Module().base().commonProperties.SoongConfigTrace - ctx.VisitDirectDeps(func(m Module) { - childTrace := &m.base().commonProperties.SoongConfigTrace - trace.Bools = append(trace.Bools, childTrace.Bools...) - trace.Strings = append(trace.Strings, childTrace.Strings...) - trace.IsSets = append(trace.IsSets, childTrace.IsSets...) - }) - trace.Bools = SortedUniqueStrings(trace.Bools) - trace.Strings = SortedUniqueStrings(trace.Strings) - trace.IsSets = SortedUniqueStrings(trace.IsSets) - - ctx.Module().base().commonProperties.SoongConfigTraceHash = trace.hash() -} - -// soongConfigTraceSingleton writes a map from each module's config hash value to trace data. -func soongConfigTraceSingletonFunc() Singleton { - return &soongConfigTraceSingleton{} -} - -type soongConfigTraceSingleton struct { -} - -func (s *soongConfigTraceSingleton) GenerateBuildActions(ctx SingletonContext) { - outFile := PathForOutput(ctx, "soong_config_trace.json") - - traces := make(map[string]*soongConfigTrace) - ctx.VisitAllModules(func(module Module) { - trace := &module.base().commonProperties.SoongConfigTrace - if !trace.isEmpty() { - hash := module.base().commonProperties.SoongConfigTraceHash - traces[hash] = trace - } - }) - - j, err := json.Marshal(traces) - if err != nil { - ctx.Errorf("json marshal to %q failed: %#v", outFile, err) - return - } - - WriteFileRule(ctx, outFile, string(j)) - ctx.Phony("soong_config_trace", outFile) -} diff --git a/android/module_context.go b/android/module_context.go index e2677a4f6..253bebd3b 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -188,7 +188,6 @@ type ModuleContext interface { TargetRequiredModuleNames() []string ModuleSubDir() string - SoongConfigTraceHash() string Variable(pctx PackageContext, name, value string) Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule @@ -216,6 +215,11 @@ type ModuleContext interface { // SetOutputFiles stores the outputFiles to outputFiles property, which is used // to set the OutputFilesProvider later. SetOutputFiles(outputFiles Paths, tag string) + + // ComplianceMetadataInfo returns a ComplianceMetadataInfo instance for different module types to dump metadata, + // which usually happens in GenerateAndroidBuildActions() of a module type. + // See android.ModuleBase.complianceMetadataInfo + ComplianceMetadataInfo() *ComplianceMetadataInfo } type moduleContext struct { @@ -377,10 +381,6 @@ func (m *moduleContext) ModuleSubDir() string { return m.bp.ModuleSubDir() } -func (m *moduleContext) SoongConfigTraceHash() string { - return m.module.base().commonProperties.SoongConfigTraceHash -} - func (m *moduleContext) InstallInData() bool { return m.module.InstallInData() } @@ -729,6 +729,15 @@ func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) { } } +func (m *moduleContext) ComplianceMetadataInfo() *ComplianceMetadataInfo { + if complianceMetadataInfo := m.module.base().complianceMetadataInfo; complianceMetadataInfo != nil { + return complianceMetadataInfo + } + complianceMetadataInfo := NewComplianceMetadataInfo() + m.module.base().complianceMetadataInfo = complianceMetadataInfo + return complianceMetadataInfo +} + // Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must // be tagged with `android:"path" to support automatic source module dependency resolution. // diff --git a/android/paths.go b/android/paths.go index adbee70be..03772ebcb 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1604,11 +1604,10 @@ type ModuleOutPathContext interface { ModuleName() string ModuleDir() string ModuleSubDir() string - SoongConfigTraceHash() string } func pathForModuleOut(ctx ModuleOutPathContext) OutputPath { - return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir(), ctx.SoongConfigTraceHash()) + return PathForOutput(ctx, ".intermediates", ctx.ModuleDir(), ctx.ModuleName(), ctx.ModuleSubDir()) } // PathForModuleOut returns a Path representing the paths... under the module's diff --git a/android/paths_test.go b/android/paths_test.go index 93b9b9a16..941f0ca78 100644 --- a/android/paths_test.go +++ b/android/paths_test.go @@ -1183,9 +1183,6 @@ type pathForModuleSrcOutputFileProviderModule struct { Outs []string Tagged []string } - - outs Paths - tagged Paths } func pathForModuleSrcOutputFileProviderModuleFactory() Module { @@ -1196,24 +1193,17 @@ func pathForModuleSrcOutputFileProviderModuleFactory() Module { } func (p *pathForModuleSrcOutputFileProviderModule) GenerateAndroidBuildActions(ctx ModuleContext) { + var outs, taggedOuts Paths for _, out := range p.props.Outs { - p.outs = append(p.outs, PathForModuleOut(ctx, out)) + outs = append(outs, PathForModuleOut(ctx, out)) } for _, tagged := range p.props.Tagged { - p.tagged = append(p.tagged, PathForModuleOut(ctx, tagged)) + taggedOuts = append(taggedOuts, PathForModuleOut(ctx, tagged)) } -} -func (p *pathForModuleSrcOutputFileProviderModule) OutputFiles(tag string) (Paths, error) { - switch tag { - case "": - return p.outs, nil - case ".tagged": - return p.tagged, nil - default: - return nil, fmt.Errorf("unsupported tag %q", tag) - } + ctx.SetOutputFiles(outs, "") + ctx.SetOutputFiles(taggedOuts, ".tagged") } type pathForModuleSrcTestCase struct { diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index d775ac356..6e4fc0c2f 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -15,7 +15,6 @@ package android import ( - "fmt" "testing" "github.com/google/blueprint" @@ -494,7 +493,6 @@ type prebuiltModule struct { properties struct { Srcs []string `android:"path,arch_variant"` } - src Path } func newPrebuiltModule() Module { @@ -510,24 +508,17 @@ func (p *prebuiltModule) Name() string { } func (p *prebuiltModule) GenerateAndroidBuildActions(ctx ModuleContext) { + var src Path if len(p.properties.Srcs) >= 1 { - p.src = p.prebuilt.SingleSourcePath(ctx) + src = p.prebuilt.SingleSourcePath(ctx) } + ctx.SetOutputFiles(Paths{src}, "") } func (p *prebuiltModule) Prebuilt() *Prebuilt { return &p.prebuilt } -func (p *prebuiltModule) OutputFiles(tag string) (Paths, error) { - switch tag { - case "": - return Paths{p.src}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - type sourceModuleProperties struct { Deps []string `android:"path,arch_variant"` } diff --git a/android/selects_test.go b/android/selects_test.go index 3093deb11..fc020a47d 100644 --- a/android/selects_test.go +++ b/android/selects_test.go @@ -25,12 +25,14 @@ import ( func TestSelects(t *testing.T) { testCases := []struct { - name string - bp string - provider selectsTestProvider - providers map[string]selectsTestProvider - vendorVars map[string]map[string]string - expectedError string + name string + bp string + fs MockFS + provider selectsTestProvider + providers map[string]selectsTestProvider + vendorVars map[string]map[string]string + vendorVarTypes map[string]map[string]string + expectedError string }{ { name: "basic string list", @@ -97,32 +99,38 @@ func TestSelects(t *testing.T) { }, }, { - name: "paths with module references", + name: "Expression in select", bp: ` my_module_type { name: "foo", - my_paths: select(soong_config_variable("my_namespace", "my_variable"), { - "a": [":a"], - "b": [":b"], - default: [":c"], + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + "a": "foo" + "bar", + default: "baz", }), } `, - expectedError: `"foo" depends on undefined module "c"`, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("foobar"), + }, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "a", + }, + }, }, { - name: "Differing types", + name: "paths with module references", bp: ` my_module_type { name: "foo", - my_string: select(soong_config_variable("my_namespace", "my_variable"), { - "a": "a.cpp", - "b": true, - default: "c.cpp", + my_paths: select(soong_config_variable("my_namespace", "my_variable"), { + "a": [":a"], + "b": [":b"], + default: [":c"], }), } `, - expectedError: `Android.bp:8:5: Found select statement with differing types "string" and "bool" in its cases`, + expectedError: `"foo" depends on undefined module "c"`, }, { name: "Select type doesn't match property type", @@ -136,7 +144,7 @@ func TestSelects(t *testing.T) { }), } `, - expectedError: `can't assign bool value to string property "my_string\[0\]"`, + expectedError: `can't assign bool value to string property`, }, { name: "String list non-default", @@ -584,6 +592,31 @@ func TestSelects(t *testing.T) { }, }, { + name: "Select on boolean soong config variable", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + true: "t", + false: "f", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "true", + }, + }, + vendorVarTypes: map[string]map[string]string{ + "my_namespace": { + "my_variable": "bool", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("t"), + }, + }, + { name: "Select on boolean false", bp: ` my_module_type { @@ -799,10 +832,194 @@ func TestSelects(t *testing.T) { my_string_list: &[]string{"a.cpp", "c.cpp", "foo.cpp"}, }, }, + { + name: "Arch variant bool", + bp: ` + my_variable = ["b.cpp"] + my_module_type { + name: "foo", + arch_variant_configurable_bool: false, + target: { + bionic_arm64: { + enabled: true, + }, + }, + } + `, + provider: selectsTestProvider{ + arch_variant_configurable_bool: proptools.BoolPtr(false), + }, + }, + { + name: "Simple string binding", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello " + my_binding, + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("hello world!"), + }, + }, + { + name: "Any branch with binding not taken", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello " + my_binding, + default: "goodbye", + }) + } + `, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("goodbye"), + }, + }, + { + name: "Any branch without binding", + bp: ` + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any: "hello", + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("hello"), + }, + }, + { + name: "Binding conflicts with file-level variable", + bp: ` + my_binding = "asdf" + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello", + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + expectedError: "variable already set in inherited scope, previous assignment", + }, + { + name: "Binding in combination with file-level variable", + bp: ` + my_var = " there " + my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "my_variable"), { + any @ my_binding: "hello" + my_var + my_binding, + default: "goodbye", + }) + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "world!", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("hello there world!"), + }, + }, + { + name: "Bindings in subdirectory inherits variable", + fs: map[string][]byte{ + "Android.bp": []byte(` +my_var = "abcd" +`), + "directoryB/Android.bp": []byte(` +my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "variable_a"), { + any @ my_binding: my_var + my_binding, + default: "", + }), +} +`), + }, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "variable_a": "e", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("abcde"), + }, + }, + { + name: "Cannot modify variable after referenced by select", + bp: ` +my_var = "foo" +my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "variable_a"), { + "a": my_var, + default: "", + }), +} +my_var += "bar" +`, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "variable_a": "b", // notably not the value that causes my_var to be referenced + }, + }, + expectedError: `modified variable "my_var" with \+= after referencing`, + }, + { + name: "Cannot shadow variable with binding", + bp: ` +my_var = "foo" +my_module_type { + name: "foo", + my_string: select(soong_config_variable("my_namespace", "variable_a"), { + any @ my_var: my_var, + default: "", + }), +} +`, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "variable_a": "a", + }, + }, + expectedError: `variable already set in inherited scope, previous assignment:`, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + fs := tc.fs + if fs == nil { + fs = make(MockFS) + } + if tc.bp != "" { + fs["Android.bp"] = []byte(tc.bp) + } fixtures := GroupFixturePreparers( PrepareForTestWithDefaults, PrepareForTestWithArchMutator, @@ -813,12 +1030,14 @@ func TestSelects(t *testing.T) { }), FixtureModifyProductVariables(func(variables FixtureProductVariables) { variables.VendorVars = tc.vendorVars + variables.VendorVarTypes = tc.vendorVarTypes }), + FixtureMergeMockFs(fs), ) if tc.expectedError != "" { fixtures = fixtures.ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(tc.expectedError)) } - result := fixtures.RunTestWithBp(t, tc.bp) + result := fixtures.RunTest(t) if tc.expectedError == "" { if len(tc.providers) == 0 { @@ -846,6 +1065,7 @@ type selectsTestProvider struct { my_string_list *[]string my_paths *[]string replacing_string_list *[]string + arch_variant_configurable_bool *bool my_nonconfigurable_bool *bool my_nonconfigurable_string *string my_nonconfigurable_string_list []string @@ -870,6 +1090,7 @@ func (p *selectsTestProvider) String() string { my_string_list: %s, my_paths: %s, replacing_string_list %s, + arch_variant_configurable_bool %v my_nonconfigurable_bool: %v, my_nonconfigurable_string: %s, my_nonconfigurable_string_list: %s, @@ -879,6 +1100,7 @@ func (p *selectsTestProvider) String() string { p.my_string_list, p.my_paths, p.replacing_string_list, + p.arch_variant_configurable_bool, p.my_nonconfigurable_bool, myNonconfigurableStringStr, p.my_nonconfigurable_string_list, @@ -893,6 +1115,7 @@ type selectsMockModuleProperties struct { My_string_list proptools.Configurable[[]string] My_paths proptools.Configurable[[]string] `android:"path"` Replacing_string_list proptools.Configurable[[]string] `android:"replace_instead_of_append,arch_variant"` + Arch_variant_configurable_bool proptools.Configurable[bool] `android:"replace_instead_of_append,arch_variant"` My_nonconfigurable_bool *bool My_nonconfigurable_string *string My_nonconfigurable_string_list []string @@ -923,6 +1146,7 @@ func (p *selectsMockModule) GenerateAndroidBuildActions(ctx ModuleContext) { my_string_list: optionalToPtr(p.properties.My_string_list.Get(ctx)), my_paths: optionalToPtr(p.properties.My_paths.Get(ctx)), replacing_string_list: optionalToPtr(p.properties.Replacing_string_list.Get(ctx)), + arch_variant_configurable_bool: optionalToPtr(p.properties.Arch_variant_configurable_bool.Get(ctx)), my_nonconfigurable_bool: p.properties.My_nonconfigurable_bool, my_nonconfigurable_string: p.properties.My_nonconfigurable_string, my_nonconfigurable_string_list: p.properties.My_nonconfigurable_string_list, diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 38db92995..e0b1d7cbe 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -463,57 +463,6 @@ func loadSoongConfigModuleTypeDefinition(ctx LoadHookContext, from string) map[s }).(map[string]blueprint.ModuleFactory) } -// tracingConfig is a wrapper to soongconfig.SoongConfig which records all accesses to SoongConfig. -type tracingConfig struct { - config soongconfig.SoongConfig - boolSet map[string]bool - stringSet map[string]string - isSetSet map[string]bool -} - -func (c *tracingConfig) Bool(name string) bool { - c.boolSet[name] = c.config.Bool(name) - return c.boolSet[name] -} - -func (c *tracingConfig) String(name string) string { - c.stringSet[name] = c.config.String(name) - return c.stringSet[name] -} - -func (c *tracingConfig) IsSet(name string) bool { - c.isSetSet[name] = c.config.IsSet(name) - return c.isSetSet[name] -} - -func (c *tracingConfig) getTrace() soongConfigTrace { - ret := soongConfigTrace{} - - for k, v := range c.boolSet { - ret.Bools = append(ret.Bools, fmt.Sprintf("%q:%t", k, v)) - } - for k, v := range c.stringSet { - ret.Strings = append(ret.Strings, fmt.Sprintf("%q:%q", k, v)) - } - for k, v := range c.isSetSet { - ret.IsSets = append(ret.IsSets, fmt.Sprintf("%q:%t", k, v)) - } - - return ret -} - -func newTracingConfig(config soongconfig.SoongConfig) *tracingConfig { - c := tracingConfig{ - config: config, - boolSet: make(map[string]bool), - stringSet: make(map[string]string), - isSetSet: make(map[string]bool), - } - return &c -} - -var _ soongconfig.SoongConfig = (*tracingConfig)(nil) - // configModuleFactory takes an existing soongConfigModuleFactory and a // ModuleType to create a new ModuleFactory that uses a custom loadhook. func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfig.ModuleType) blueprint.ModuleFactory { @@ -561,8 +510,8 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi // conditional on Soong config variables by reading the product // config variables from Make. AddLoadHook(module, func(ctx LoadHookContext) { - tracingConfig := newTracingConfig(ctx.Config().VendorConfig(moduleType.ConfigNamespace)) - newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, tracingConfig) + config := ctx.Config().VendorConfig(moduleType.ConfigNamespace) + newProps, err := soongconfig.PropertiesToApply(moduleType, conditionalProps, config) if err != nil { ctx.ModuleErrorf("%s", err) return @@ -570,8 +519,6 @@ func configModuleFactory(factory blueprint.ModuleFactory, moduleType *soongconfi for _, ps := range newProps { ctx.AppendProperties(ps) } - - module.(Module).base().commonProperties.SoongConfigTrace = tracingConfig.getTrace() }) return module, props } diff --git a/android/soong_config_modules_test.go b/android/soong_config_modules_test.go index a6b2c51c6..04aafdeee 100644 --- a/android/soong_config_modules_test.go +++ b/android/soong_config_modules_test.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "path/filepath" "testing" ) @@ -506,197 +505,3 @@ func TestSoongConfigModuleSingletonModule(t *testing.T) { }) } } - -func TestSoongConfigModuleTrace(t *testing.T) { - bp := ` - soong_config_module_type { - name: "acme_test", - module_type: "test", - config_namespace: "acme", - variables: ["board", "feature1", "FEATURE3", "unused_string_var"], - bool_variables: ["feature2", "unused_feature", "always_true"], - value_variables: ["size", "unused_size"], - properties: ["cflags", "srcs", "defaults"], - } - - soong_config_module_type { - name: "acme_test_defaults", - module_type: "test_defaults", - config_namespace: "acme", - variables: ["board", "feature1", "FEATURE3", "unused_string_var"], - bool_variables: ["feature2", "unused_feature", "always_true"], - value_variables: ["size", "unused_size"], - properties: ["cflags", "srcs", "defaults"], - } - - soong_config_string_variable { - name: "board", - values: ["soc_a", "soc_b", "soc_c"], - } - - soong_config_string_variable { - name: "unused_string_var", - values: ["a", "b"], - } - - soong_config_bool_variable { - name: "feature1", - } - - soong_config_bool_variable { - name: "FEATURE3", - } - - test_defaults { - name: "test_defaults", - cflags: ["DEFAULT"], - } - - test { - name: "normal", - defaults: ["test_defaults"], - } - - acme_test { - name: "board_1", - defaults: ["test_defaults"], - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - }, - } - - acme_test { - name: "board_2", - defaults: ["test_defaults"], - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - }, - } - - acme_test { - name: "size", - defaults: ["test_defaults"], - soong_config_variables: { - size: { - cflags: ["-DSIZE=%s"], - }, - }, - } - - acme_test { - name: "board_and_size", - defaults: ["test_defaults"], - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - size: { - cflags: ["-DSIZE=%s"], - }, - }, - } - - acme_test_defaults { - name: "board_defaults", - soong_config_variables: { - board: { - soc_a: { - cflags: ["-DSOC_A"], - }, - }, - }, - } - - acme_test_defaults { - name: "size_defaults", - soong_config_variables: { - size: { - cflags: ["-DSIZE=%s"], - }, - }, - } - - test { - name: "board_and_size_with_defaults", - defaults: ["board_defaults", "size_defaults"], - } - ` - - fixtureForVendorVars := func(vars map[string]map[string]string) FixturePreparer { - return FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = vars - }) - } - - preparer := fixtureForVendorVars(map[string]map[string]string{ - "acme": { - "board": "soc_a", - "size": "42", - "feature1": "true", - "feature2": "false", - // FEATURE3 unset - "unused_feature": "true", // unused - "unused_size": "1", // unused - "unused_string_var": "a", // unused - "always_true": "true", - }, - }) - - t.Run("soong config trace hash", func(t *testing.T) { - result := GroupFixturePreparers( - preparer, - PrepareForTestWithDefaults, - PrepareForTestWithSoongConfigModuleBuildComponents, - prepareForSoongConfigTestModule, - FixtureRegisterWithContext(func(ctx RegistrationContext) { - ctx.FinalDepsMutators(registerSoongConfigTraceMutator) - }), - FixtureWithRootAndroidBp(bp), - ).RunTest(t) - - // Hashes of modules not using soong config should be empty - normal := result.ModuleForTests("normal", "").Module().(*soongConfigTestModule) - AssertDeepEquals(t, "normal hash", normal.base().commonProperties.SoongConfigTraceHash, "") - AssertDeepEquals(t, "normal hash out", normal.outputPath.RelativeToTop().String(), "out/soong/.intermediates/normal/test") - - board1 := result.ModuleForTests("board_1", "").Module().(*soongConfigTestModule) - board2 := result.ModuleForTests("board_2", "").Module().(*soongConfigTestModule) - size := result.ModuleForTests("size", "").Module().(*soongConfigTestModule) - - // Trace mutator sets soong config trace hash correctly - board1Hash := board1.base().commonProperties.SoongConfigTrace.hash() - board1Output := board1.outputPath.RelativeToTop().String() - AssertDeepEquals(t, "board hash calc", board1Hash, board1.base().commonProperties.SoongConfigTraceHash) - AssertDeepEquals(t, "board hash path", board1Output, filepath.Join("out/soong/.intermediates/board_1", board1Hash, "test")) - - sizeHash := size.base().commonProperties.SoongConfigTrace.hash() - sizeOutput := size.outputPath.RelativeToTop().String() - AssertDeepEquals(t, "size hash calc", sizeHash, size.base().commonProperties.SoongConfigTraceHash) - AssertDeepEquals(t, "size hash path", sizeOutput, filepath.Join("out/soong/.intermediates/size", sizeHash, "test")) - - // Trace should be identical for modules using the same set of variables - AssertDeepEquals(t, "board trace", board1.base().commonProperties.SoongConfigTrace, board2.base().commonProperties.SoongConfigTrace) - AssertDeepEquals(t, "board hash", board1.base().commonProperties.SoongConfigTraceHash, board2.base().commonProperties.SoongConfigTraceHash) - - // Trace hash should be different for different sets of soong variables - AssertBoolEquals(t, "board hash not equal to size hash", board1.base().commonProperties.SoongConfigTraceHash == size.commonProperties.SoongConfigTraceHash, false) - - boardSize := result.ModuleForTests("board_and_size", "").Module().(*soongConfigTestModule) - boardSizeDefaults := result.ModuleForTests("board_and_size_with_defaults", "").Module() - - // Trace should propagate - AssertDeepEquals(t, "board_size hash calc", boardSize.base().commonProperties.SoongConfigTrace.hash(), boardSize.base().commonProperties.SoongConfigTraceHash) - AssertDeepEquals(t, "board_size trace", boardSize.base().commonProperties.SoongConfigTrace, boardSizeDefaults.base().commonProperties.SoongConfigTrace) - AssertDeepEquals(t, "board_size hash", boardSize.base().commonProperties.SoongConfigTraceHash, boardSizeDefaults.base().commonProperties.SoongConfigTraceHash) - }) -} diff --git a/android/soongconfig/Android.bp b/android/soongconfig/Android.bp index 8fe1ff1eb..5a6df2684 100644 --- a/android/soongconfig/Android.bp +++ b/android/soongconfig/Android.bp @@ -9,7 +9,6 @@ bootstrap_go_package { "blueprint", "blueprint-parser", "blueprint-proptools", - "soong-bazel", "soong-starlark-format", ], srcs: [ diff --git a/android/testing.go b/android/testing.go index 8dd467dce..18fd3b31c 100644 --- a/android/testing.go +++ b/android/testing.go @@ -1025,9 +1025,12 @@ func (m TestingModule) VariablesForTestsRelativeToTop() map[string]string { // otherwise returns the result of calling Paths.RelativeToTop // on the returned Paths. func (m TestingModule) OutputFiles(t *testing.T, tag string) Paths { - // TODO: add non-empty-string tag case and remove OutputFileProducer part - if tag == "" && m.module.base().outputFiles.DefaultOutputFiles != nil { - return m.module.base().outputFiles.DefaultOutputFiles.RelativeToTop() + // TODO: remove OutputFileProducer part + outputFiles := m.Module().base().outputFiles + if tag == "" && outputFiles.DefaultOutputFiles != nil { + return outputFiles.DefaultOutputFiles.RelativeToTop() + } else if taggedOutputFiles, hasTag := outputFiles.TaggedOutputFiles[tag]; hasTag { + return taggedOutputFiles } producer, ok := m.module.(OutputFileProducer) diff --git a/android/variable.go b/android/variable.go index 2cdcd5355..a331439f6 100644 --- a/android/variable.go +++ b/android/variable.go @@ -399,7 +399,8 @@ type ProductVariables struct { PlatformSepolicyCompatVersions []string `json:",omitempty"` - VendorVars map[string]map[string]string `json:",omitempty"` + VendorVars map[string]map[string]string `json:",omitempty"` + VendorVarTypes map[string]map[string]string `json:",omitempty"` Ndk_abis *bool `json:",omitempty"` diff --git a/androidmk/androidmk/android.go b/androidmk/androidmk/android.go index 8a8bb2eaa..570f36c16 100644 --- a/androidmk/androidmk/android.go +++ b/androidmk/androidmk/android.go @@ -341,9 +341,6 @@ func classifyLocalOrGlobalPath(value bpparser.Expression) (string, bpparser.Expr firstOperand := v.Args[0] secondOperand := v.Args[1] - if firstOperand.Type() != bpparser.StringType { - return "global", value, nil - } if _, ok := firstOperand.(*bpparser.Operator); ok { return "global", value, nil diff --git a/androidmk/androidmk/androidmk.go b/androidmk/androidmk/androidmk.go index 2e8810fe8..6fb20dcc1 100644 --- a/androidmk/androidmk/androidmk.go +++ b/androidmk/androidmk/androidmk.go @@ -493,7 +493,6 @@ func setVariable(file *bpFile, plusequals bool, prefix, name string, value bppar Name: name, NamePos: pos, Value: value, - OrigValue: value, EqualsPos: pos, Assigner: "+=", } @@ -506,7 +505,6 @@ func setVariable(file *bpFile, plusequals bool, prefix, name string, value bppar Name: name, NamePos: pos, Value: value, - OrigValue: value, EqualsPos: pos, Assigner: "=", } diff --git a/androidmk/androidmk/values.go b/androidmk/androidmk/values.go index 9618142fa..701c708e2 100644 --- a/androidmk/androidmk/values.go +++ b/androidmk/androidmk/values.go @@ -81,7 +81,7 @@ func makeToStringExpression(ms *mkparser.MakeString, file *bpFile) (bpparser.Exp } tmp := &bpparser.Variable{ Name: name, - Value: &bpparser.String{}, + Type_: bpparser.StringType, } if tmp.Name == "TOP" { @@ -150,7 +150,7 @@ func makeToListExpression(ms *mkparser.MakeString, file *bpFile) (bpparser.Expre } listOfListValues = append(listOfListValues, &bpparser.Variable{ Name: name, - Value: &bpparser.List{}, + Type_: bpparser.ListType, }) listValue = &bpparser.List{} } @@ -215,7 +215,7 @@ func makeToBoolExpression(ms *mkparser.MakeString, file *bpFile) (bpparser.Expre } return &bpparser.Variable{ Name: name, - Value: &bpparser.Bool{}, + Type_: bpparser.BoolType, }, nil } else { return nil, fmt.Errorf("non-const bool expression %s", ms.Dump()) diff --git a/apex/apex.go b/apex/apex.go index 754f8981e..a2c1f2c39 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -157,8 +157,7 @@ type apexBundleProperties struct { // Default: true. Installable *bool - // If set true, VNDK libs are considered as stable libs and are not included in this APEX. - // Should be only used in non-system apexes (e.g. vendor: true). Default is false. + // Deprecated. Do not use. TODO(b/350644693) remove this after removing all usage Use_vndk_as_stable *bool // The type of filesystem to use. Either 'ext4', 'f2fs' or 'erofs'. Default 'ext4'. @@ -748,9 +747,9 @@ func (a *apexBundle) getImageVariationPair() (string, string) { prefix := android.CoreVariation if a.SocSpecific() || a.DeviceSpecific() { - prefix = cc.VendorVariation + prefix = android.VendorVariation } else if a.ProductSpecific() { - prefix = cc.ProductVariation + prefix = android.ProductVariation } return prefix, "" @@ -950,24 +949,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { return } - // Special casing for APEXes on non-system (e.g., vendor, odm, etc.) partitions. They are - // provided with a property named use_vndk_as_stable, which when set to true doesn't collect - // VNDK libraries as transitive dependencies. This option is useful for reducing the size of - // the non-system APEXes because the VNDK libraries won't be included (and duped) in the - // APEX, but shared across APEXes via the VNDK APEX. - useVndk := a.SocSpecific() || a.DeviceSpecific() || (a.ProductSpecific() && mctx.Config().EnforceProductPartitionInterface()) - if proptools.Bool(a.properties.Use_vndk_as_stable) { - if !useVndk { - mctx.PropertyErrorf("use_vndk_as_stable", "not supported for system/system_ext APEXes") - } - if a.minSdkVersionValue(mctx) != "" { - mctx.PropertyErrorf("use_vndk_as_stable", "not supported when min_sdk_version is set") - } - if mctx.Failed() { - return - } - } - continueApexDepsWalk := func(child, parent android.Module) bool { am, ok := child.(android.ApexModule) if !ok || !am.CanHaveApexVariants() { @@ -985,10 +966,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { return false } - if useVndk && child.Name() == "libbinder" { - mctx.ModuleErrorf("Module %s in the vendor APEX %s should not use libbinder. Use libbinder_ndk instead.", parent.Name(), a.Name()) - } - // By default, all the transitive dependencies are collected, unless filtered out // above. return true @@ -1172,6 +1149,7 @@ var ( "test_com.android.os.statsd", "test_com.android.permission", "test_com.android.wifi", + "test_imgdiag_com.android.art", "test_jitzygote_com.android.art", // go/keep-sorted end } @@ -1370,25 +1348,6 @@ func (a *apexBundle) DepIsInSameApex(_ android.BaseModuleContext, _ android.Modu return true } -var _ android.OutputFileProducer = (*apexBundle)(nil) - -// Implements android.OutputFileProducer -func (a *apexBundle) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "", android.DefaultDistTag: - // This is the default dist path. - return android.Paths{a.outputFile}, nil - case imageApexSuffix: - // uncompressed one - if a.outputApexFile != nil { - return android.Paths{a.outputApexFile}, nil - } - fallthrough - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - var _ multitree.Exportable = (*apexBundle)(nil) func (a *apexBundle) Exportable() bool { @@ -2432,6 +2391,8 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.providePrebuiltInfo(ctx) a.required = a.RequiredModuleNames(ctx) + + a.setOutputFiles(ctx) } // Set prebuiltInfoProvider. This will be used by `apex_prebuiltinfo_singleton` to print out a metadata file @@ -2460,6 +2421,18 @@ func (a *apexBundle) provideApexExportsInfo(ctx android.ModuleContext) { }) } +// Set output files to outputFiles property, which is later used to set the +// OutputFilesProvider +func (a *apexBundle) setOutputFiles(ctx android.ModuleContext) { + // default dist path + ctx.SetOutputFiles(android.Paths{a.outputFile}, "") + ctx.SetOutputFiles(android.Paths{a.outputFile}, android.DefaultDistTag) + // uncompressed one + if a.outputApexFile != nil { + ctx.SetOutputFiles(android.Paths{a.outputApexFile}, imageApexSuffix) + } +} + // apexBootclasspathFragmentFiles returns the list of apexFile structures defining the files that // the bootclasspath_fragment contributes to the apex. func apexBootclasspathFragmentFiles(ctx android.ModuleContext, module blueprint.Module) []apexFile { @@ -2722,9 +2695,6 @@ func (a *apexBundle) checkUpdatable(ctx android.ModuleContext) { if a.UsePlatformApis() { ctx.PropertyErrorf("updatable", "updatable APEXes can't use platform APIs") } - if proptools.Bool(a.properties.Use_vndk_as_stable) { - ctx.PropertyErrorf("use_vndk_as_stable", "updatable APEXes can't use external VNDK libs") - } if a.FutureUpdatable() { ctx.PropertyErrorf("future_updatable", "Already updatable. Remove `future_updatable: true:`") } diff --git a/apex/apex_test.go b/apex/apex_test.go index dfc4bb372..df9db74e4 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -7125,6 +7125,46 @@ func TestJavaSDKLibrary(t *testing.T) { ensureMatches(t, contents, "<library\\n\\s+name=\\\"foo\\\"\\n\\s+file=\\\"/apex/myapex/javalib/foo.jar\\\"") } +func TestJavaSDKLibraryOverrideApexes(t *testing.T) { + ctx := testApex(t, ` + override_apex { + name: "mycompanyapex", + base: "myapex", + } + apex { + name: "myapex", + key: "myapex.key", + java_libs: ["foo"], + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + java_sdk_library { + name: "foo", + srcs: ["a.java"], + api_packages: ["foo"], + apex_available: [ "myapex" ], + } + + prebuilt_apis { + name: "sdk", + api_dirs: ["100"], + } + `, withFiles(filesForSdkLibrary)) + + // Permission XML should point to the activated path of impl jar of java_sdk_library. + // Since override variants (com.mycompany.android.foo) are installed in the same package as the overridden variant + // (com.android.foo), the filepath should not contain override apex name. + sdkLibrary := ctx.ModuleForTests("foo.xml", "android_common_mycompanyapex").Output("foo.xml") + contents := android.ContentFromFileRuleForTests(t, ctx, sdkLibrary) + ensureMatches(t, contents, "<library\\n\\s+name=\\\"foo\\\"\\n\\s+file=\\\"/apex/myapex/javalib/foo.jar\\\"") +} + func TestJavaSDKLibrary_WithinApex(t *testing.T) { ctx := testApex(t, ` apex { @@ -8238,60 +8278,6 @@ func TestUpdatableDefault_should_set_min_sdk_version(t *testing.T) { `) } -func Test_use_vndk_as_stable_shouldnt_be_used_for_updatable_vendor_apexes(t *testing.T) { - testApexError(t, `"myapex" .*: use_vndk_as_stable: updatable APEXes can't use external VNDK libs`, ` - apex { - name: "myapex", - key: "myapex.key", - updatable: true, - use_vndk_as_stable: true, - soc_specific: true, - } - - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - } - `) -} - -func Test_use_vndk_as_stable_shouldnt_be_used_with_min_sdk_version(t *testing.T) { - testApexError(t, `"myapex" .*: use_vndk_as_stable: not supported when min_sdk_version is set`, ` - apex { - name: "myapex", - key: "myapex.key", - updatable: false, - min_sdk_version: "29", - use_vndk_as_stable: true, - vendor: true, - } - - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - } - `) -} - -func Test_use_vndk_as_stable_shouldnt_be_used_for_non_vendor_apexes(t *testing.T) { - testApexError(t, `"myapex" .*: use_vndk_as_stable: not supported for system/system_ext APEXes`, ` - apex { - name: "myapex", - key: "myapex.key", - updatable: false, - use_vndk_as_stable: true, - } - - apex_key { - name: "myapex.key", - public_key: "testkey.avbpubkey", - private_key: "testkey.pem", - } - `) -} - func TestUpdatable_should_not_set_generate_classpaths_proto(t *testing.T) { testApexError(t, `"mysystemserverclasspathfragment" .* it must not set generate_classpaths_proto to false`, ` apex { @@ -9997,6 +9983,56 @@ func TestApexStrictUpdtabilityLintBcpFragmentDeps(t *testing.T) { } } +func TestApexLintBcpFragmentSdkLibDeps(t *testing.T) { + bp := ` + apex { + name: "myapex", + key: "myapex.key", + bootclasspath_fragments: ["mybootclasspathfragment"], + min_sdk_version: "29", + } + apex_key { + name: "myapex.key", + } + bootclasspath_fragment { + name: "mybootclasspathfragment", + contents: ["foo"], + apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, + } + java_sdk_library { + name: "foo", + srcs: ["MyClass.java"], + apex_available: [ "myapex" ], + sdk_version: "current", + min_sdk_version: "29", + compile_dex: true, + } + ` + fs := android.MockFS{ + "lint-baseline.xml": nil, + } + + result := android.GroupFixturePreparers( + prepareForApexTest, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.PrepareForTestWithJacocoInstrumentation, + java.FixtureWithLastReleaseApis("foo"), + android.FixtureModifyConfig(func(config android.Config) { + config.SetApiLibraries([]string{"foo"}) + }), + android.FixtureMergeMockFs(fs), + ).RunTestWithBp(t, bp) + + myapex := result.ModuleForTests("myapex", "android_common_myapex") + lintReportInputs := strings.Join(myapex.Output("lint-report-xml.zip").Inputs.Strings(), " ") + android.AssertStringDoesContain(t, + "myapex lint report expected to contain that of the sdk library impl lib as an input", + lintReportInputs, "foo.impl") +} + // updatable apexes should propagate updatable=true to its apps func TestUpdatableApexEnforcesAppUpdatability(t *testing.T) { bp := ` diff --git a/bazel/Android.bp b/bazel/Android.bp index 4709f5cd3..f8273a847 100644 --- a/bazel/Android.bp +++ b/bazel/Android.bp @@ -6,22 +6,17 @@ bootstrap_go_package { name: "soong-bazel", pkgPath: "android/soong/bazel", srcs: [ - "aquery.go", - "bazel_proxy.go", "configurability.go", - "constants.go", "properties.go", "testing.go", ], testSrcs: [ - "aquery_test.go", "properties_test.go", ], pluginFor: [ "soong_build", ], deps: [ - "bazel_analysis_v2_proto", "blueprint", ], } diff --git a/bazel/aquery.go b/bazel/aquery.go deleted file mode 100644 index 35942bc32..000000000 --- a/bazel/aquery.go +++ /dev/null @@ -1,768 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "path/filepath" - "reflect" - "sort" - "strings" - "sync" - - analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" - - "github.com/google/blueprint/metrics" - "github.com/google/blueprint/proptools" - "google.golang.org/protobuf/proto" -) - -type artifactId int -type depsetId int -type pathFragmentId int - -// KeyValuePair represents Bazel's aquery proto, KeyValuePair. -type KeyValuePair struct { - Key string - Value string -} - -// AqueryDepset is a depset definition from Bazel's aquery response. This is -// akin to the `depSetOfFiles` in the response proto, except: -// - direct artifacts are enumerated by full path instead of by ID -// - it has a hash of the depset contents, instead of an int ID (for determinism) -// -// A depset is a data structure for efficient transitive handling of artifact -// paths. A single depset consists of one or more artifact paths and one or -// more "child" depsets. -type AqueryDepset struct { - ContentHash string - DirectArtifacts []string - TransitiveDepSetHashes []string -} - -// BuildStatement contains information to register a build statement corresponding (one to one) -// with a Bazel action from Bazel's action graph. -type BuildStatement struct { - Command string - Depfile *string - OutputPaths []string - SymlinkPaths []string - Env []*analysis_v2_proto.KeyValuePair - Mnemonic string - - // Inputs of this build statement, either as unexpanded depsets or expanded - // input paths. There should be no overlap between these fields; an input - // path should either be included as part of an unexpanded depset or a raw - // input path string, but not both. - InputDepsetHashes []string - InputPaths []string - FileContents string - // If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment - // and run the mixed build action there - ShouldRunInSbox bool - // A list of files to add as implicit deps to the outputs of this BuildStatement. - // Unlike most properties in BuildStatement, these paths must be relative to the root of - // the whole out/ folder, instead of relative to ctx.Config().BazelContext.OutputBase() - ImplicitDeps []string - IsExecutable bool -} - -// A helper type for aquery processing which facilitates retrieval of path IDs from their -// less readable Bazel structures (depset and path fragment). -type aqueryArtifactHandler struct { - // Maps depset id to AqueryDepset, a representation of depset which is - // post-processed for middleman artifact handling, unhandled artifact - // dropping, content hashing, etc. - depsetIdToAqueryDepset map[depsetId]AqueryDepset - emptyDepsetIds map[depsetId]struct{} - // Maps content hash to AqueryDepset. - depsetHashToAqueryDepset map[string]AqueryDepset - - // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening - // may be an expensive operation. - depsetHashToArtifactPathsCache sync.Map - // Maps artifact ids to fully expanded paths. - artifactIdToPath map[artifactId]string -} - -// The tokens should be substituted with the value specified here, instead of the -// one returned in 'substitutions' of TemplateExpand action. -var templateActionOverriddenTokens = map[string]string{ - // Uses "python3" for %python_binary% instead of the value returned by aquery - // which is "py3wrapper.sh". See removePy3wrapperScript. - "%python_binary%": "python3", -} - -const ( - middlemanMnemonic = "Middleman" - // The file name of py3wrapper.sh, which is used by py_binary targets. - py3wrapperFileName = "/py3wrapper.sh" -) - -func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V { - m := map[K]V{} - for _, v := range values { - m[keyFn(v)] = v - } - return m -} - -func newAqueryHandler(aqueryResult *analysis_v2_proto.ActionGraphContainer) (*aqueryArtifactHandler, error) { - pathFragments := indexBy(aqueryResult.PathFragments, func(pf *analysis_v2_proto.PathFragment) pathFragmentId { - return pathFragmentId(pf.Id) - }) - - artifactIdToPath := make(map[artifactId]string, len(aqueryResult.Artifacts)) - for _, artifact := range aqueryResult.Artifacts { - artifactPath, err := expandPathFragment(pathFragmentId(artifact.PathFragmentId), pathFragments) - if err != nil { - return nil, err - } - if artifact.IsTreeArtifact && - !strings.HasPrefix(artifactPath, "bazel-out/io_bazel_rules_go/") && - !strings.HasPrefix(artifactPath, "bazel-out/rules_java_builtin/") { - // Since we're using ninja as an executor, we can't use tree artifacts. Ninja only - // considers a file/directory "dirty" when it's mtime changes. Directories' mtimes will - // only change when a file in the directory is added/removed, but not when files in - // the directory are changed, or when files in subdirectories are changed/added/removed. - // Bazel handles this by walking the directory and generating a hash for it after the - // action runs, which we would have to do as well if we wanted to support these - // artifacts in mixed builds. - // - // However, there are some bazel built-in rules that use tree artifacts. Allow those, - // but keep in mind that they'll have incrementality issues. - return nil, fmt.Errorf("tree artifacts are currently not supported in mixed builds: " + artifactPath) - } - artifactIdToPath[artifactId(artifact.Id)] = artifactPath - } - - // Map middleman artifact ContentHash to input artifact depset ID. - // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, - // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then, - // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for - // that action instead. - middlemanIdToDepsetIds := map[artifactId][]uint32{} - for _, actionEntry := range aqueryResult.Actions { - if actionEntry.Mnemonic == middlemanMnemonic { - for _, outputId := range actionEntry.OutputIds { - middlemanIdToDepsetIds[artifactId(outputId)] = actionEntry.InputDepSetIds - } - } - } - - depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d *analysis_v2_proto.DepSetOfFiles) depsetId { - return depsetId(d.Id) - }) - - aqueryHandler := aqueryArtifactHandler{ - depsetIdToAqueryDepset: map[depsetId]AqueryDepset{}, - depsetHashToAqueryDepset: map[string]AqueryDepset{}, - depsetHashToArtifactPathsCache: sync.Map{}, - emptyDepsetIds: make(map[depsetId]struct{}, 0), - artifactIdToPath: artifactIdToPath, - } - - // Validate and adjust aqueryResult.DepSetOfFiles values. - for _, depset := range aqueryResult.DepSetOfFiles { - _, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset) - if err != nil { - return nil, err - } - } - - return &aqueryHandler, nil -} - -// Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given -// depset. -func (a *aqueryArtifactHandler) populateDepsetMaps(depset *analysis_v2_proto.DepSetOfFiles, middlemanIdToDepsetIds map[artifactId][]uint32, depsetIdToDepset map[depsetId]*analysis_v2_proto.DepSetOfFiles) (*AqueryDepset, error) { - if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depsetId(depset.Id)]; containsDepset { - return &aqueryDepset, nil - } - transitiveDepsetIds := depset.TransitiveDepSetIds - directArtifactPaths := make([]string, 0, len(depset.DirectArtifactIds)) - for _, id := range depset.DirectArtifactIds { - aId := artifactId(id) - path, pathExists := a.artifactIdToPath[aId] - if !pathExists { - return nil, fmt.Errorf("undefined input artifactId %d", aId) - } - // Filter out any inputs which are universally dropped, and swap middleman - // artifacts with their corresponding depsets. - if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[aId]; isMiddleman { - // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts. - transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...) - } else if strings.HasSuffix(path, py3wrapperFileName) || - strings.HasPrefix(path, "../bazel_tools") { - continue - // Drop these artifacts. - // See go/python-binary-host-mixed-build for more details. - // 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the - // TemplateExpandAction handles everything necessary to launch a Pythin application. - // 2) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the - // containing depset to always be considered newer than their outputs. - } else { - directArtifactPaths = append(directArtifactPaths, path) - } - } - - childDepsetHashes := make([]string, 0, len(transitiveDepsetIds)) - for _, id := range transitiveDepsetIds { - childDepsetId := depsetId(id) - childDepset, exists := depsetIdToDepset[childDepsetId] - if !exists { - if _, empty := a.emptyDepsetIds[childDepsetId]; empty { - continue - } else { - return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id) - } - } - if childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset); err != nil { - return nil, err - } else if childAqueryDepset == nil { - continue - } else { - childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash) - } - } - if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 { - a.emptyDepsetIds[depsetId(depset.Id)] = struct{}{} - return nil, nil - } - aqueryDepset := AqueryDepset{ - ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes), - DirectArtifacts: directArtifactPaths, - TransitiveDepSetHashes: childDepsetHashes, - } - a.depsetIdToAqueryDepset[depsetId(depset.Id)] = aqueryDepset - a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset - return &aqueryDepset, nil -} - -// getInputPaths flattens the depsets of the given IDs and returns all transitive -// input paths contained in these depsets. -// This is a potentially expensive operation, and should not be invoked except -// for actions which need specialized input handling. -func (a *aqueryArtifactHandler) getInputPaths(depsetIds []uint32) ([]string, error) { - var inputPaths []string - - for _, id := range depsetIds { - inputDepSetId := depsetId(id) - depset := a.depsetIdToAqueryDepset[inputDepSetId] - inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash) - if err != nil { - return nil, err - } - for _, inputPath := range inputArtifacts { - inputPaths = append(inputPaths, inputPath) - } - } - - return inputPaths, nil -} - -func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) { - if result, exists := a.depsetHashToArtifactPathsCache.Load(depsetHash); exists { - return result.([]string), nil - } - if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists { - result := depset.DirectArtifacts - for _, childHash := range depset.TransitiveDepSetHashes { - childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash) - if err != nil { - return nil, err - } - result = append(result, childArtifactIds...) - } - a.depsetHashToArtifactPathsCache.Store(depsetHash, result) - return result, nil - } else { - return nil, fmt.Errorf("undefined input depset hash %s", depsetHash) - } -} - -// AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset -// which should be registered (and output to a ninja file) to correspond with Bazel's -// action graph, as described by the given action graph json proto. -// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets -// are one-to-one with Bazel's depSetOfFiles objects. -func AqueryBuildStatements(aqueryJsonProto []byte, eventHandler *metrics.EventHandler) ([]*BuildStatement, []AqueryDepset, error) { - aqueryProto := &analysis_v2_proto.ActionGraphContainer{} - err := proto.Unmarshal(aqueryJsonProto, aqueryProto) - if err != nil { - return nil, nil, err - } - - var aqueryHandler *aqueryArtifactHandler - { - eventHandler.Begin("init_handler") - defer eventHandler.End("init_handler") - aqueryHandler, err = newAqueryHandler(aqueryProto) - if err != nil { - return nil, nil, err - } - } - - // allocate both length and capacity so each goroutine can write to an index independently without - // any need for synchronization for slice access. - buildStatements := make([]*BuildStatement, len(aqueryProto.Actions)) - { - eventHandler.Begin("build_statements") - defer eventHandler.End("build_statements") - wg := sync.WaitGroup{} - var errOnce sync.Once - id2targets := make(map[uint32]string, len(aqueryProto.Targets)) - for _, t := range aqueryProto.Targets { - id2targets[t.GetId()] = t.GetLabel() - } - for i, actionEntry := range aqueryProto.Actions { - wg.Add(1) - go func(i int, actionEntry *analysis_v2_proto.Action) { - if strings.HasPrefix(id2targets[actionEntry.TargetId], "@bazel_tools//") { - // bazel_tools are removed depsets in `populateDepsetMaps()` so skipping - // conversion to build statements as well - buildStatements[i] = nil - } else if buildStatement, aErr := aqueryHandler.actionToBuildStatement(actionEntry); aErr != nil { - errOnce.Do(func() { - aErr = fmt.Errorf("%s: [%s] [%s]", aErr.Error(), actionEntry.GetMnemonic(), id2targets[actionEntry.TargetId]) - err = aErr - }) - } else { - // set build statement at an index rather than appending such that each goroutine does not - // impact other goroutines - buildStatements[i] = buildStatement - } - wg.Done() - }(i, actionEntry) - } - wg.Wait() - } - if err != nil { - return nil, nil, err - } - - depsetsByHash := map[string]AqueryDepset{} - depsets := make([]AqueryDepset, 0, len(aqueryHandler.depsetIdToAqueryDepset)) - { - eventHandler.Begin("depsets") - defer eventHandler.End("depsets") - for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset { - if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey { - // Two depsets collide on hash. Ensure that their contents are identical. - if !reflect.DeepEqual(aqueryDepset, prevEntry) { - return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset) - } - } else { - depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset - depsets = append(depsets, aqueryDepset) - } - } - } - - eventHandler.Do("build_statement_sort", func() { - // Build Statements and depsets must be sorted by their content hash to - // preserve determinism between builds (this will result in consistent ninja file - // output). Note they are not sorted by their original IDs nor their Bazel ordering, - // as Bazel gives nondeterministic ordering / identifiers in aquery responses. - sort.Slice(buildStatements, func(i, j int) bool { - // Sort all nil statements to the end of the slice - if buildStatements[i] == nil { - return false - } else if buildStatements[j] == nil { - return true - } - //For build statements, compare output lists. In Bazel, each output file - // may only have one action which generates it, so this will provide - // a deterministic ordering. - outputs_i := buildStatements[i].OutputPaths - outputs_j := buildStatements[j].OutputPaths - if len(outputs_i) != len(outputs_j) { - return len(outputs_i) < len(outputs_j) - } - if len(outputs_i) == 0 { - // No outputs for these actions, so compare commands. - return buildStatements[i].Command < buildStatements[j].Command - } - // There may be multiple outputs, but the output ordering is deterministic. - return outputs_i[0] < outputs_j[0] - }) - }) - eventHandler.Do("depset_sort", func() { - sort.Slice(depsets, func(i, j int) bool { - return depsets[i].ContentHash < depsets[j].ContentHash - }) - }) - return buildStatements, depsets, nil -} - -// depsetContentHash computes and returns a SHA256 checksum of the contents of -// the given depset. This content hash may serve as the depset's identifier. -// Using a content hash for an identifier is superior for determinism. (For example, -// using an integer identifier which depends on the order in which the depsets are -// created would result in nondeterministic depset IDs.) -func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string { - h := sha256.New() - // Use newline as delimiter, as paths cannot contain newline. - h.Write([]byte(strings.Join(directPaths, "\n"))) - h.Write([]byte(strings.Join(transitiveDepsetHashes, ""))) - fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) - return fullHash -} - -func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []uint32) ([]string, error) { - var hashes []string - for _, id := range inputDepsetIds { - dId := depsetId(id) - if aqueryDepset, exists := a.depsetIdToAqueryDepset[dId]; !exists { - if _, empty := a.emptyDepsetIds[dId]; !empty { - return nil, fmt.Errorf("undefined (not even empty) input depsetId %d", dId) - } - } else { - hashes = append(hashes, aqueryDepset.ContentHash) - } - } - return hashes, nil -} - -// escapes the args received from aquery and creates a command string -func commandString(actionEntry *analysis_v2_proto.Action) string { - argsEscaped := make([]string, len(actionEntry.Arguments)) - for i, arg := range actionEntry.Arguments { - if arg == "" { - // If this is an empty string, add '' - // And not - // 1. (literal empty) - // 2. `''\'''\'''` (escaped version of '') - // - // If we had used (1), then this would appear as a whitespace when we strings.Join - argsEscaped[i] = "''" - } else { - argsEscaped[i] = proptools.ShellEscapeIncludingSpaces(arg) - } - } - return strings.Join(argsEscaped, " ") -} - -func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - command := commandString(actionEntry) - inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputDepsetHashes: inputDepsetHashes, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } - if buildStatement.Mnemonic == "GoToolchainBinaryBuild" { - // Unlike b's execution root, mixed build execution root contains a symlink to prebuilts/go - // This causes issues for `GOCACHE=$(mktemp -d) go build ...` - // To prevent this, sandbox this action in mixed builds as well - buildStatement.ShouldRunInSbox = true - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - if len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) - } - expandedTemplateContent := expandTemplateContent(actionEntry) - // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, - // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might - // change \n to space and mess up the format of Python programs. - // sed is used to convert \\n back to \n before saving to output file. - // See go/python-binary-host-mixed-build for more details. - command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, - escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) - inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputDepsetHashes: inputDepsetHashes, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, _, err := a.getOutputPaths(actionEntry) - var depsetHashes []string - if err == nil { - depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds) - } - if err != nil { - return nil, err - } - return &BuildStatement{ - Depfile: nil, - OutputPaths: outputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - InputDepsetHashes: depsetHashes, - FileContents: actionEntry.FileContents, - IsExecutable: actionEntry.IsExecutable, - }, nil -} - -func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, _, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - if len(inputPaths) != 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) - } - // The actual command is generated in bazelSingleton.GenerateBuildActions - return &BuildStatement{ - Depfile: nil, - OutputPaths: outputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - InputPaths: inputPaths, - }, nil -} - -type bazelSandwichJson struct { - Target string `json:"target"` - DependOnTarget *bool `json:"depend_on_target,omitempty"` - ImplicitDeps []string `json:"implicit_deps"` -} - -func (a *aqueryArtifactHandler) unresolvedSymlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - if len(actionEntry.InputDepSetIds) != 0 || len(outputPaths) != 1 { - return nil, fmt.Errorf("expected 0 inputs and 1 output to symlink action, got: input %q, output %q", actionEntry.InputDepSetIds, outputPaths) - } - target := actionEntry.UnresolvedSymlinkTarget - if target == "" { - return nil, fmt.Errorf("expected an unresolved_symlink_target, but didn't get one") - } - if filepath.Clean(target) != target { - return nil, fmt.Errorf("expected %q, got %q", filepath.Clean(target), target) - } - if strings.HasPrefix(target, "/") { - return nil, fmt.Errorf("no absolute symlinks allowed: %s", target) - } - - out := outputPaths[0] - outDir := filepath.Dir(out) - var implicitDeps []string - if strings.HasPrefix(target, "bazel_sandwich:") { - j := bazelSandwichJson{} - err := json.Unmarshal([]byte(target[len("bazel_sandwich:"):]), &j) - if err != nil { - return nil, err - } - if proptools.BoolDefault(j.DependOnTarget, true) { - implicitDeps = append(implicitDeps, j.Target) - } - implicitDeps = append(implicitDeps, j.ImplicitDeps...) - dotDotsToReachCwd := "" - if outDir != "." { - dotDotsToReachCwd = strings.Repeat("../", strings.Count(outDir, "/")+1) - } - target = proptools.ShellEscapeIncludingSpaces(j.Target) - target = "{DOTDOTS_TO_OUTPUT_ROOT}" + dotDotsToReachCwd + target - } else { - target = proptools.ShellEscapeIncludingSpaces(target) - } - - outDir = proptools.ShellEscapeIncludingSpaces(outDir) - out = proptools.ShellEscapeIncludingSpaces(out) - // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). - command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, target) - symlinkPaths := outputPaths[:] - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - SymlinkPaths: symlinkPaths, - ImplicitDeps: implicitDeps, - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return nil, err - } - - inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - if len(inputPaths) != 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) - } - out := outputPaths[0] - outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) - out = proptools.ShellEscapeIncludingSpaces(out) - in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) - // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). - command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) - symlinkPaths := outputPaths[:] - - buildStatement := &BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputPaths: inputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - SymlinkPaths: symlinkPaths, - } - return buildStatement, nil -} - -func (a *aqueryArtifactHandler) getOutputPaths(actionEntry *analysis_v2_proto.Action) (outputPaths []string, depfile *string, err error) { - for _, outputId := range actionEntry.OutputIds { - outputPath, exists := a.artifactIdToPath[artifactId(outputId)] - if !exists { - err = fmt.Errorf("undefined outputId %d", outputId) - return - } - ext := filepath.Ext(outputPath) - if ext == ".d" { - if depfile != nil { - err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) - return - } else { - depfile = &outputPath - } - } else { - outputPaths = append(outputPaths, outputPath) - } - } - return -} - -// expandTemplateContent substitutes the tokens in a template. -func expandTemplateContent(actionEntry *analysis_v2_proto.Action) string { - replacerString := make([]string, len(actionEntry.Substitutions)*2) - for i, pair := range actionEntry.Substitutions { - value := pair.Value - if val, ok := templateActionOverriddenTokens[pair.Key]; ok { - value = val - } - replacerString[i*2] = pair.Key - replacerString[i*2+1] = value - } - replacer := strings.NewReplacer(replacerString...) - return replacer.Replace(actionEntry.TemplateContent) -} - -// \->\\, $->\$, `->\`, "->\", \n->\\n, '->'"'"' -var commandLineArgumentReplacer = strings.NewReplacer( - `\`, `\\`, - `$`, `\$`, - "`", "\\`", - `"`, `\"`, - "\n", "\\n", - `'`, `'"'"'`, -) - -func escapeCommandlineArgument(str string) string { - return commandLineArgumentReplacer.Replace(str) -} - -func (a *aqueryArtifactHandler) actionToBuildStatement(actionEntry *analysis_v2_proto.Action) (*BuildStatement, error) { - switch actionEntry.Mnemonic { - // Middleman actions are not handled like other actions; they are handled separately as a - // preparatory step so that their inputs may be relayed to actions depending on middleman - // artifacts. - case middlemanMnemonic: - return nil, nil - // PythonZipper is bogus action returned by aquery, ignore it (b/236198693) - case "PythonZipper": - return nil, nil - // Skip "Fail" actions, which are placeholder actions designed to always fail. - case "Fail": - return nil, nil - case "BaselineCoverage": - return nil, nil - case "Symlink", "SolibSymlink", "ExecutableSymlink": - return a.symlinkActionBuildStatement(actionEntry) - case "TemplateExpand": - if len(actionEntry.Arguments) < 1 { - return a.templateExpandActionBuildStatement(actionEntry) - } - case "FileWrite", "SourceSymlinkManifest", "RepoMappingManifest": - return a.fileWriteActionBuildStatement(actionEntry) - case "SymlinkTree": - return a.symlinkTreeActionBuildStatement(actionEntry) - case "UnresolvedSymlink": - return a.unresolvedSymlinkActionBuildStatement(actionEntry) - } - - if len(actionEntry.Arguments) < 1 { - return nil, errors.New("received action with no command") - } - return a.normalActionBuildStatement(actionEntry) - -} - -func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]*analysis_v2_proto.PathFragment) (string, error) { - var labels []string - currId := id - // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node. - for currId > 0 { - currFragment, ok := pathFragmentsMap[currId] - if !ok { - return "", fmt.Errorf("undefined path fragment id %d", currId) - } - labels = append([]string{currFragment.Label}, labels...) - parentId := pathFragmentId(currFragment.ParentId) - if currId == parentId { - return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment) - } - currId = parentId - } - return filepath.Join(labels...), nil -} diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go deleted file mode 100644 index cbd27919c..000000000 --- a/bazel/aquery_test.go +++ /dev/null @@ -1,1411 +0,0 @@ -// Copyright 2020 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "encoding/json" - "fmt" - "reflect" - "sort" - "testing" - - analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2" - - "github.com/google/blueprint/metrics" - "google.golang.org/protobuf/proto" -) - -func TestAqueryMultiArchGenrule(t *testing.T) { - // This input string is retrieved from a real build of bionic-related genrules. - const inputString = ` -{ - "Artifacts": [ - { "Id": 1, "path_fragment_id": 1 }, - { "Id": 2, "path_fragment_id": 6 }, - { "Id": 3, "path_fragment_id": 8 }, - { "Id": 4, "path_fragment_id": 12 }, - { "Id": 5, "path_fragment_id": 19 }, - { "Id": 6, "path_fragment_id": 20 }, - { "Id": 7, "path_fragment_id": 21 }], - "Actions": [{ - "target_id": 1, - "action_key": "ab53f6ecbdc2ee8cb8812613b63205464f1f5083f6dca87081a0a398c0f1ecf7", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [1], - "output_ids": [4], - "primary_output_id": 4 - }, { - "target_id": 2, - "action_key": "9f4309ce165dac458498cb92811c18b0b7919782cc37b82a42d2141b8cc90826", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [2], - "output_ids": [5], - "primary_output_id": 5 - }, { - "target_id": 3, - "action_key": "50d6c586103ebeed3a218195540bcc30d329464eae36377eb82f8ce7c36ac342", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py x86_64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-x86_64.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [3], - "output_ids": [6], - "primary_output_id": 6 - }, { - "target_id": 4, - "action_key": "f30cbe442f5216f4223cf16a39112cad4ec56f31f49290d85cff587e48647ffa", - "Mnemonic": "Genrule", - "configuration_id": 1, - "Arguments": ["/bin/bash", "-c", "source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py arm64 ../sourceroot/bionic/libc/SYSCALLS.TXT \u003e bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-arm64.S"], - "environment_variables": [{ - "Key": "PATH", - "Value": "/bin:/usr/bin:/usr/local/bin" - }], - "input_dep_set_ids": [4], - "output_ids": [7], - "primary_output_id": 7 - }], - "Targets": [ - { "Id": 1, "Label": "@sourceroot//bionic/libc:syscalls-arm", "rule_class_id": 1 }, - { "Id": 2, "Label": "@sourceroot//bionic/libc:syscalls-x86", "rule_class_id": 1 }, - { "Id": 3, "Label": "@sourceroot//bionic/libc:syscalls-x86_64", "rule_class_id": 1 }, - { "Id": 4, "Label": "@sourceroot//bionic/libc:syscalls-arm64", "rule_class_id": 1 }], - "dep_set_of_files": [ - { "Id": 1, "direct_artifact_ids": [1, 2, 3] }, - { "Id": 2, "direct_artifact_ids": [1, 2, 3] }, - { "Id": 3, "direct_artifact_ids": [1, 2, 3] }, - { "Id": 4, "direct_artifact_ids": [1, 2, 3] }], - "Configuration": [{ - "Id": 1, - "Mnemonic": "k8-fastbuild", - "platform_name": "k8", - "Checksum": "485c362832c178e367d972177f68e69e0981e51e67ef1c160944473db53fe046" - }], - "rule_classes": [{ "Id": 1, "Name": "genrule"}], - "path_fragments": [ - { "Id": 5, "Label": ".." }, - { "Id": 4, "Label": "sourceroot", "parent_id": 5 }, - { "Id": 3, "Label": "bionic", "parent_id": 4 }, - { "Id": 2, "Label": "libc", "parent_id": 3 }, - { "Id": 1, "Label": "SYSCALLS.TXT", "parent_id": 2 }, - { "Id": 7, "Label": "tools", "parent_id": 2 }, - { "Id": 6, "Label": "gensyscalls.py", "parent_id": 7 }, - { "Id": 11, "Label": "bazel_tools", "parent_id": 5 }, - { "Id": 10, "Label": "tools", "parent_id": 11 }, - { "Id": 9, "Label": "genrule", "parent_id": 10 }, - { "Id": 8, "Label": "genrule-setup.sh", "parent_id": 9 }, - { "Id": 18, "Label": "bazel-out" }, - { "Id": 17, "Label": "sourceroot", "parent_id": 18 }, - { "Id": 16, "Label": "k8-fastbuild", "parent_id": 17 }, - { "Id": 15, "Label": "bin", "parent_id": 16 }, - { "Id": 14, "Label": "bionic", "parent_id": 15 }, - { "Id": 13, "Label": "libc", "parent_id": 14 }, - { "Id": 12, "Label": "syscalls-arm.S", "parent_id": 13 }, - { "Id": 19, "Label": "syscalls-x86.S", "parent_id": 13 }, - { "Id": 20, "Label": "syscalls-x86_64.S", "parent_id": 13 }, - { "Id": 21, "Label": "syscalls-arm64.S", "parent_id": 13 }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - var expectedBuildStatements []*BuildStatement - for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} { - expectedBuildStatements = append(expectedBuildStatements, - &BuildStatement{ - Command: fmt.Sprintf( - "/bin/bash -c 'source ../bazel_tools/tools/genrule/genrule-setup.sh; ../sourceroot/bionic/libc/tools/gensyscalls.py %s ../sourceroot/bionic/libc/SYSCALLS.TXT > bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S'", - arch, arch), - OutputPaths: []string{ - fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch), - }, - Env: []*analysis_v2_proto.KeyValuePair{ - {Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, - }, - Mnemonic: "Genrule", - }) - } - assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) - - expectedFlattenedInputs := []string{ - "../sourceroot/bionic/libc/SYSCALLS.TXT", - "../sourceroot/bionic/libc/tools/gensyscalls.py", - } - // In this example, each depset should have the same expected inputs. - for _, actualDepset := range actualDepsets { - actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) - if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { - t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) - } - } -} - -func TestInvalidOutputId(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "X", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [3], - "primary_output_id": 3 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined outputId 3: [X] []") -} - -func TestInvalidInputDepsetIdFromAction(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "X", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [2], - "output_ids": [1], - "primary_output_id": 1 - }], - "targets": [{ - "id": 1, - "label": "target_x" - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined (not even empty) input depsetId 2: [X] [target_x]") -} - -func TestInvalidInputDepsetIdFromDepset(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2], "transitive_dep_set_ids": [42] }], - "path_fragments": [ - { "id": 1, "label": "one"}, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)") -} - -func TestInvalidInputArtifactId(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 3] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined input artifactId 3") -} - -func TestInvalidPathFragmentId(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [1], - "primary_output_id": 1 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two", "parent_id": 3 }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined path fragment id 3") -} - -func TestDepfiles(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }], - "actions": [{ - "target_Id": 1, - "action_Key": "x", - "mnemonic": "x", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [2, 3], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_Artifact_Ids": [1, 2, 3] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }, - { "id": 3, "label": "two.d" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - if expected := 1; len(actual) != expected { - t.Fatalf("Expected %d build statements, got %d", expected, len(actual)) - return - } - - bs := actual[0] - expectedDepfile := "two.d" - if bs.Depfile == nil { - t.Errorf("Expected depfile %q, but there was none found", expectedDepfile) - } else if *bs.Depfile != expectedDepfile { - t.Errorf("Expected depfile %q, but got %q", expectedDepfile, *bs.Depfile) - } -} - -func TestMultipleDepfiles(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }, - { "id": 4, "path_fragment_id": 4 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "X", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [2,3,4], - "primary_output_id": 2 - }], - "dep_set_of_files": [{ - "id": 1, - "direct_artifact_ids": [1, 2, 3, 4] - }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "two" }, - { "id": 3, "label": "two.d" }, - { "id": 4, "label": "other.d" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, `found multiple potential depfiles "two.d", "other.d": [X] []`) -} - -func TestTransitiveInputDepsets(t *testing.T) { - // The input aquery for this test comes from a proof-of-concept starlark rule which registers - // a single action with many inputs given via a deep depset. - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 7 }, - { "id": 3, "path_fragment_id": 8 }, - { "id": 4, "path_fragment_id": 9 }, - { "id": 5, "path_fragment_id": 10 }, - { "id": 6, "path_fragment_id": 11 }, - { "id": 7, "path_fragment_id": 12 }, - { "id": 8, "path_fragment_id": 13 }, - { "id": 9, "path_fragment_id": 14 }, - { "id": 10, "path_fragment_id": 15 }, - { "id": 11, "path_fragment_id": 16 }, - { "id": 12, "path_fragment_id": 17 }, - { "id": 13, "path_fragment_id": 18 }, - { "id": 14, "path_fragment_id": 19 }, - { "id": 15, "path_fragment_id": 20 }, - { "id": 16, "path_fragment_id": 21 }, - { "id": 17, "path_fragment_id": 22 }, - { "id": 18, "path_fragment_id": 23 }, - { "id": 19, "path_fragment_id": 24 }, - { "id": 20, "path_fragment_id": 25 }, - { "id": 21, "path_fragment_id": 26 }], - "actions": [{ - "target_id": 1, - "action_key": "3b826d17fadbbbcd8313e456b90ec47c078c438088891dd45b4adbcd8889dc50", - "mnemonic": "Action", - "configuration_id": 1, - "arguments": ["/bin/bash", "-c", "touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"], - "input_dep_set_ids": [1], - "output_ids": [21], - "primary_output_id": 21 - }], - "dep_set_of_files": [ - { "id": 3, "direct_artifact_ids": [1, 2, 3, 4, 5] }, - { "id": 4, "direct_artifact_ids": [6, 7, 8, 9, 10] }, - { "id": 2, "transitive_dep_set_ids": [3, 4], "direct_artifact_ids": [11, 12, 13, 14, 15] }, - { "id": 5, "direct_artifact_ids": [16, 17, 18, 19] }, - { "id": 1, "transitive_dep_set_ids": [2, 5], "direct_artifact_ids": [20] }], - "path_fragments": [ - { "id": 6, "label": "bazel-out" }, - { "id": 5, "label": "sourceroot", "parent_id": 6 }, - { "id": 4, "label": "k8-fastbuild", "parent_id": 5 }, - { "id": 3, "label": "bin", "parent_id": 4 }, - { "id": 2, "label": "testpkg", "parent_id": 3 }, - { "id": 1, "label": "test_1", "parent_id": 2 }, - { "id": 7, "label": "test_2", "parent_id": 2 }, - { "id": 8, "label": "test_3", "parent_id": 2 }, - { "id": 9, "label": "test_4", "parent_id": 2 }, - { "id": 10, "label": "test_5", "parent_id": 2 }, - { "id": 11, "label": "test_6", "parent_id": 2 }, - { "id": 12, "label": "test_7", "parent_id": 2 }, - { "id": 13, "label": "test_8", "parent_id": 2 }, - { "id": 14, "label": "test_9", "parent_id": 2 }, - { "id": 15, "label": "test_10", "parent_id": 2 }, - { "id": 16, "label": "test_11", "parent_id": 2 }, - { "id": 17, "label": "test_12", "parent_id": 2 }, - { "id": 18, "label": "test_13", "parent_id": 2 }, - { "id": 19, "label": "test_14", "parent_id": 2 }, - { "id": 20, "label": "test_15", "parent_id": 2 }, - { "id": 21, "label": "test_16", "parent_id": 2 }, - { "id": 22, "label": "test_17", "parent_id": 2 }, - { "id": 23, "label": "test_18", "parent_id": 2 }, - { "id": 24, "label": "test_19", "parent_id": 2 }, - { "id": 25, "label": "test_root", "parent_id": 2 }, - { "id": 26,"label": "test_out", "parent_id": 2 }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualbuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'", - OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, - Mnemonic: "Action", - SymlinkPaths: []string{}, - }, - } - assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) - - // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs - // are given via a deep depset, but the depset is flattened when returned as a - // BuildStatement slice. - var expectedFlattenedInputs []string - for i := 1; i < 20; i++ { - expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) - } - expectedFlattenedInputs = append(expectedFlattenedInputs, "bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root") - - actualDepsetHashes := actualbuildStatements[0].InputDepsetHashes - actualFlattenedInputs := flattenDepsets(actualDepsetHashes, actualDepsets) - if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { - t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) - } -} - -func TestSymlinkTree(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "SymlinkTree", - "configuration_id": 1, - "input_dep_set_ids": [1], - "output_ids": [2], - "primary_output_id": 2, - "execution_platform": "//build/bazel/platforms:linux_x86_64" - }], - "path_fragments": [ - { "id": 1, "label": "foo.manifest" }, - { "id": 2, "label": "foo.runfiles/MANIFEST" }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{ - &BuildStatement{ - Command: "", - OutputPaths: []string{"foo.runfiles/MANIFEST"}, - Mnemonic: "SymlinkTree", - InputPaths: []string{"foo.manifest"}, - SymlinkPaths: []string{}, - }, - }, actual) -} - -func TestBazelToolsRemovalFromInputDepsets(t *testing.T) { - const inputString = `{ - "artifacts": [ - { "id": 1, "path_fragment_id": 10 }, - { "id": 2, "path_fragment_id": 20 }, - { "id": 3, "path_fragment_id": 30 }, - { "id": 4, "path_fragment_id": 40 }], - "dep_set_of_files": [{ - "id": 1111, - "direct_artifact_ids": [3 , 4] - }, { - "id": 2222, - "direct_artifact_ids": [3] - }], - "actions": [{ - "target_id": 100, - "action_key": "x", - "input_dep_set_ids": [1111, 2222], - "mnemonic": "x", - "arguments": ["bogus", "command"], - "output_ids": [2], - "primary_output_id": 1 - }], - "path_fragments": [ - { "id": 10, "label": "input" }, - { "id": 20, "label": "output" }, - { "id": 30, "label": "dep1", "parent_id": 50 }, - { "id": 40, "label": "dep2", "parent_id": 60 }, - { "id": 50, "label": "bazel_tools", "parent_id": 60 }, - { "id": 60, "label": ".."} - ] -}` - /* depsets - 1111 2222 - / \ | - ../dep2 ../bazel_tools/dep1 - */ - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - if len(actualDepsets) != 1 { - t.Errorf("expected 1 depset but found %#v", actualDepsets) - return - } - dep2Found := false - for _, dep := range flattenDepsets([]string{actualDepsets[0].ContentHash}, actualDepsets) { - if dep == "../bazel_tools/dep1" { - t.Errorf("dependency %s expected to be removed but still exists", dep) - } else if dep == "../dep2" { - dep2Found = true - } - } - if !dep2Found { - t.Errorf("dependency ../dep2 expected but not found") - } - - expectedBuildStatement := &BuildStatement{ - Command: "bogus command", - OutputPaths: []string{"output"}, - Mnemonic: "x", - SymlinkPaths: []string{}, - } - buildStatementFound := false - for _, actualBuildStatement := range actualBuildStatements { - if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { - buildStatementFound = true - break - } - } - if !buildStatementFound { - t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements) - return - } -} - -func TestBazelToolsRemovalFromTargets(t *testing.T) { - const inputString = `{ - "artifacts": [{ "id": 1, "path_fragment_id": 10 }], - "targets": [ - { "id": 100, "label": "targetX" }, - { "id": 200, "label": "@bazel_tools//tool_y" } -], - "actions": [{ - "target_id": 100, - "action_key": "actionX", - "arguments": ["bogus", "command"], - "mnemonic" : "x", - "output_ids": [1] - }, { - "target_id": 200, - "action_key": "y" - }], - "path_fragments": [{ "id": 10, "label": "outputX"}] -}` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - if len(actualDepsets) != 0 { - t.Errorf("expected 0 depset but found %#v", actualDepsets) - return - } - expectedBuildStatement := &BuildStatement{ - Command: "bogus command", - OutputPaths: []string{"outputX"}, - Mnemonic: "x", - SymlinkPaths: []string{}, - } - buildStatementFound := false - for _, actualBuildStatement := range actualBuildStatements { - if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { - buildStatementFound = true - break - } - } - if !buildStatementFound { - t.Errorf("expected but missing %#v in %#v build statements", expectedBuildStatement, len(actualBuildStatements)) - return - } -} - -func TestBazelToolsRemovalFromTransitiveInputDepsets(t *testing.T) { - const inputString = `{ - "artifacts": [ - { "id": 1, "path_fragment_id": 10 }, - { "id": 2, "path_fragment_id": 20 }, - { "id": 3, "path_fragment_id": 30 }], - "dep_set_of_files": [{ - "id": 1111, - "transitive_dep_set_ids": [2222] - }, { - "id": 2222, - "direct_artifact_ids": [3] - }, { - "id": 3333, - "direct_artifact_ids": [3] - }, { - "id": 4444, - "transitive_dep_set_ids": [3333] - }], - "actions": [{ - "target_id": 100, - "action_key": "x", - "input_dep_set_ids": [1111, 4444], - "mnemonic": "x", - "arguments": ["bogus", "command"], - "output_ids": [2], - "primary_output_id": 1 - }], - "path_fragments": [ - { "id": 10, "label": "input" }, - { "id": 20, "label": "output" }, - { "id": 30, "label": "dep", "parent_id": 50 }, - { "id": 50, "label": "bazel_tools", "parent_id": 60 }, - { "id": 60, "label": ".."} - ] -}` - /* depsets - 1111 4444 - || || - 2222 3333 - | | - ../bazel_tools/dep - Note: in dep_set_of_files: - 1111 appears BEFORE its dependency,2222 while - 4444 appears AFTER its dependency 3333 - and this test shows that that order doesn't affect empty depset pruning - */ - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data, &metrics.EventHandler{}) - if len(actualDepsets) != 0 { - t.Errorf("expected 0 depsets but found %#v", actualDepsets) - return - } - - expectedBuildStatement := &BuildStatement{ - Command: "bogus command", - OutputPaths: []string{"output"}, - Mnemonic: "x", - } - buildStatementFound := false - for _, actualBuildStatement := range actualBuildStatements { - if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { - buildStatementFound = true - break - } - } - if !buildStatementFound { - t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements) - return - } -} - -func TestMiddlemenAction(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }, - { "id": 4, "path_fragment_id": 4 }, - { "id": 5, "path_fragment_id": 5 }, - { "id": 6, "path_fragment_id": 6 }], - "path_fragments": [ - { "id": 1, "label": "middleinput_one" }, - { "id": 2, "label": "middleinput_two" }, - { "id": 3, "label": "middleman_artifact" }, - { "id": 4, "label": "maininput_one" }, - { "id": 5, "label": "maininput_two" }, - { "id": 6, "label": "output" }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1, 2] }, - { "id": 2, "direct_artifact_ids": [3, 4, 5] }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "Middleman", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [1], - "output_ids": [3], - "primary_output_id": 3 - }, { - "target_id": 2, - "action_key": "y", - "mnemonic": "Main action", - "arguments": ["touch", "foo"], - "input_dep_set_ids": [2], - "output_ids": [6], - "primary_output_id": 6 - }] -}` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actualBuildStatements, actualDepsets, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - if expected := 2; len(actualBuildStatements) != expected { - t.Fatalf("Expected %d build statements, got %d %#v", expected, len(actualBuildStatements), actualBuildStatements) - return - } - - expectedDepsetFiles := [][]string{ - {"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"}, - {"middleinput_one", "middleinput_two"}, - } - assertFlattenedDepsets(t, actualDepsets, expectedDepsetFiles) - - bs := actualBuildStatements[0] - if len(bs.InputPaths) > 0 { - t.Errorf("Expected main action raw inputs to be empty, but got %q", bs.InputPaths) - } - - expectedOutputs := []string{"output"} - if !reflect.DeepEqual(bs.OutputPaths, expectedOutputs) { - t.Errorf("Expected main action outputs %q, but got %q", expectedOutputs, bs.OutputPaths) - } - - expectedFlattenedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"} - actualFlattenedInputs := flattenDepsets(bs.InputDepsetHashes, actualDepsets) - - if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { - t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) - } - - bs = actualBuildStatements[1] - if bs != nil { - t.Errorf("Expected nil action for skipped") - } -} - -// Returns the contents of given depsets in concatenated post order. -func flattenDepsets(depsetHashesToFlatten []string, allDepsets []AqueryDepset) []string { - depsetsByHash := map[string]AqueryDepset{} - for _, depset := range allDepsets { - depsetsByHash[depset.ContentHash] = depset - } - var result []string - for _, depsetId := range depsetHashesToFlatten { - result = append(result, flattenDepset(depsetId, depsetsByHash)...) - } - return result -} - -// Returns the contents of a given depset in post order. -func flattenDepset(depsetHashToFlatten string, allDepsets map[string]AqueryDepset) []string { - depset := allDepsets[depsetHashToFlatten] - var result []string - for _, depsetId := range depset.TransitiveDepSetHashes { - result = append(result, flattenDepset(depsetId, allDepsets)...) - } - result = append(result, depset.DirectArtifacts...) - return result -} - -func assertFlattenedDepsets(t *testing.T, actualDepsets []AqueryDepset, expectedDepsetFiles [][]string) { - t.Helper() - if len(actualDepsets) != len(expectedDepsetFiles) { - t.Errorf("Expected %d depsets, but got %d depsets", len(expectedDepsetFiles), len(actualDepsets)) - } - for i, actualDepset := range actualDepsets { - actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) - if !reflect.DeepEqual(actualFlattenedInputs, expectedDepsetFiles[i]) { - t.Errorf("Expected depset files: %v, but got %v", expectedDepsetFiles[i], actualFlattenedInputs) - } - } -} - -func TestSimpleSymlink(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 3 }, - { "id": 2, "path_fragment_id": 5 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "Symlink", - "input_dep_set_ids": [1], - "output_ids": [2], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "file_subdir", "parent_id": 1 }, - { "id": 3, "label": "file", "parent_id": 2 }, - { "id": 4, "label": "symlink_subdir", "parent_id": 1 }, - { "id": 5, "label": "symlink", "parent_id": 4 }] -}` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "mkdir -p one/symlink_subdir && " + - "rm -f one/symlink_subdir/symlink && " + - "ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink", - InputPaths: []string{"one/file_subdir/file"}, - OutputPaths: []string{"one/symlink_subdir/symlink"}, - SymlinkPaths: []string{"one/symlink_subdir/symlink"}, - Mnemonic: "Symlink", - }, - } - assertBuildStatements(t, actual, expectedBuildStatements) -} - -func TestSymlinkQuotesPaths(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 3 }, - { "id": 2, "path_fragment_id": 5 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "SolibSymlink", - "input_dep_set_ids": [1], - "output_ids": [2], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }], - "path_fragments": [ - { "id": 1, "label": "one" }, - { "id": 2, "label": "file subdir", "parent_id": 1 }, - { "id": 3, "label": "file", "parent_id": 2 }, - { "id": 4, "label": "symlink subdir", "parent_id": 1 }, - { "id": 5, "label": "symlink", "parent_id": 4 }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "mkdir -p 'one/symlink subdir' && " + - "rm -f 'one/symlink subdir/symlink' && " + - "ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'", - InputPaths: []string{"one/file subdir/file"}, - OutputPaths: []string{"one/symlink subdir/symlink"}, - SymlinkPaths: []string{"one/symlink subdir/symlink"}, - Mnemonic: "SolibSymlink", - }, - } - assertBuildStatements(t, expectedBuildStatements, actual) -} - -func TestSymlinkMultipleInputs(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 2, "path_fragment_id": 2 }, - { "id": 3, "path_fragment_id": 3 }], - "actions": [{ - "target_id": 1, - "action_key": "action_x", - "mnemonic": "Symlink", - "input_dep_set_ids": [1], - "output_ids": [3], - "primary_output_id": 3 - }], - "dep_set_of_files": [{ "id": 1, "direct_artifact_ids": [1,2] }], - "path_fragments": [ - { "id": 1, "label": "file" }, - { "id": 2, "label": "other_file" }, - { "id": 3, "label": "symlink" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]: [Symlink] []`) -} - -func TestSymlinkMultipleOutputs(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }, - { "id": 3, "path_fragment_id": 3 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "Symlink", - "input_dep_set_ids": [1], - "output_ids": [2,3], - "primary_output_id": 2 - }], - "dep_set_of_files": [ - { "id": 1, "direct_artifact_ids": [1] }], - "path_fragments": [ - { "id": 1, "label": "file" }, - { "id": 2, "label": "symlink" }, - { "id": 3, "label": "other_symlink" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, "undefined outputId 2: [Symlink] []") -} - -func TestTemplateExpandActionSubstitutions(t *testing.T) { - const inputString = ` -{ - "artifacts": [{ - "id": 1, - "path_fragment_id": 1 - }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "TemplateExpand", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "template_content": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [ - { "key": "%token1%", "value": "abcd" }, - { "key": "%python_binary%", "value": "python3" }] - }], - "path_fragments": [ - { "id": 1, "label": "template_file" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - - expectedBuildStatements := []*BuildStatement{ - &BuildStatement{ - Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " + - "chmod a+x template_file'", - OutputPaths: []string{"template_file"}, - Mnemonic: "TemplateExpand", - SymlinkPaths: []string{}, - }, - } - assertBuildStatements(t, expectedBuildStatements, actual) -} - -func TestTemplateExpandActionNoOutput(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "TemplateExpand", - "configuration_id": 1, - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "templateContent": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [ - { "key": "%token1%", "value": "abcd" }, - { "key": "%python_binary%", "value": "python3" }] - }], - "path_fragments": [ - { "id": 1, "label": "template_file" }] -}` - - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - _, _, err = AqueryBuildStatements(data, &metrics.EventHandler{}) - assertError(t, err, `Expect 1 output to template expand action, got: output []: [TemplateExpand] []`) -} - -func TestFileWrite(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "FileWrite", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "file_contents": "file data\n" - }], - "path_fragments": [ - { "id": 1, "label": "foo.manifest" }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{ - &BuildStatement{ - OutputPaths: []string{"foo.manifest"}, - Mnemonic: "FileWrite", - FileContents: "file data\n", - SymlinkPaths: []string{}, - }, - }, actual) -} - -func TestSourceSymlinkManifest(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 }], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "SourceSymlinkManifest", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "file_contents": "symlink target\n" - }], - "path_fragments": [ - { "id": 1, "label": "foo.manifest" }] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{ - &BuildStatement{ - OutputPaths: []string{"foo.manifest"}, - Mnemonic: "SourceSymlinkManifest", - SymlinkPaths: []string{}, - }, - }, actual) -} - -func TestUnresolvedSymlink(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 } - ], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "UnresolvedSymlink", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "unresolved_symlink_target": "symlink/target" - }], - "path_fragments": [ - { "id": 1, "label": "path/to/symlink" } - ] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{{ - Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf symlink/target path/to/symlink", - OutputPaths: []string{"path/to/symlink"}, - Mnemonic: "UnresolvedSymlink", - SymlinkPaths: []string{"path/to/symlink"}, - }}, actual) -} - -func TestUnresolvedSymlinkBazelSandwich(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 } - ], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "UnresolvedSymlink", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "unresolved_symlink_target": "bazel_sandwich:{\"target\":\"target/product/emulator_x86_64/system\"}" - }], - "path_fragments": [ - { "id": 1, "label": "path/to/symlink" } - ] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{{ - Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink", - OutputPaths: []string{"path/to/symlink"}, - Mnemonic: "UnresolvedSymlink", - SymlinkPaths: []string{"path/to/symlink"}, - ImplicitDeps: []string{"target/product/emulator_x86_64/system"}, - }}, actual) -} - -func TestUnresolvedSymlinkBazelSandwichWithAlternativeDeps(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "path_fragment_id": 1 } - ], - "actions": [{ - "target_id": 1, - "action_key": "x", - "mnemonic": "UnresolvedSymlink", - "configuration_id": 1, - "output_ids": [1], - "primary_output_id": 1, - "execution_platform": "//build/bazel/platforms:linux_x86_64", - "unresolved_symlink_target": "bazel_sandwich:{\"depend_on_target\":false,\"implicit_deps\":[\"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp\"],\"target\":\"target/product/emulator_x86_64/system\"}" - }], - "path_fragments": [ - { "id": 1, "label": "path/to/symlink" } - ] -} -` - data, err := JsonToActionGraphContainer(inputString) - if err != nil { - t.Error(err) - return - } - actual, _, err := AqueryBuildStatements(data, &metrics.EventHandler{}) - if err != nil { - t.Errorf("Unexpected error %q", err) - return - } - assertBuildStatements(t, []*BuildStatement{{ - Command: "mkdir -p path/to && rm -f path/to/symlink && ln -sf {DOTDOTS_TO_OUTPUT_ROOT}../../target/product/emulator_x86_64/system path/to/symlink", - OutputPaths: []string{"path/to/symlink"}, - Mnemonic: "UnresolvedSymlink", - SymlinkPaths: []string{"path/to/symlink"}, - // Note that the target of the symlink, target/product/emulator_x86_64/system, is not listed here - ImplicitDeps: []string{"target/product/emulator_x86_64/obj/PACKAGING/systemimage_intermediates/staging_dir.stamp"}, - }}, actual) -} - -func assertError(t *testing.T, err error, expected string) { - t.Helper() - if err == nil { - t.Errorf("expected error '%s', but got no error", expected) - } else if err.Error() != expected { - t.Errorf("expected error:\n\t'%s', but got:\n\t'%s'", expected, err.Error()) - } -} - -// Asserts that the given actual build statements match the given expected build statements. -// Build statement equivalence is determined using buildStatementEquals. -func assertBuildStatements(t *testing.T, expected []*BuildStatement, actual []*BuildStatement) { - t.Helper() - if len(expected) != len(actual) { - t.Errorf("expected %d build statements, but got %d,\n expected: %#v,\n actual: %#v", - len(expected), len(actual), expected, actual) - return - } - type compareFn = func(i int, j int) bool - byCommand := func(slice []*BuildStatement) compareFn { - return func(i int, j int) bool { - if slice[i] == nil { - return false - } else if slice[j] == nil { - return false - } - return slice[i].Command < slice[j].Command - } - } - sort.SliceStable(expected, byCommand(expected)) - sort.SliceStable(actual, byCommand(actual)) - for i, actualStatement := range actual { - expectedStatement := expected[i] - if differingField := buildStatementEquals(actualStatement, expectedStatement); differingField != "" { - t.Errorf("%s differs\nunexpected build statement %#v.\nexpected: %#v", - differingField, actualStatement, expectedStatement) - return - } - } -} - -func buildStatementEquals(first *BuildStatement, second *BuildStatement) string { - if (first == nil) != (second == nil) { - return "Nil" - } - if first.Mnemonic != second.Mnemonic { - return "Mnemonic" - } - if first.Command != second.Command { - return "Command" - } - // Ordering is significant for environment variables. - if !reflect.DeepEqual(first.Env, second.Env) { - return "Env" - } - // Ordering is irrelevant for input and output paths, so compare sets. - if !reflect.DeepEqual(sortedStrings(first.InputPaths), sortedStrings(second.InputPaths)) { - return "InputPaths" - } - if !reflect.DeepEqual(sortedStrings(first.OutputPaths), sortedStrings(second.OutputPaths)) { - return "OutputPaths" - } - if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) { - return "SymlinkPaths" - } - if !reflect.DeepEqual(sortedStrings(first.ImplicitDeps), sortedStrings(second.ImplicitDeps)) { - return "ImplicitDeps" - } - if first.Depfile != second.Depfile { - return "Depfile" - } - return "" -} - -func sortedStrings(stringSlice []string) []string { - sorted := make([]string, len(stringSlice)) - copy(sorted, stringSlice) - sort.Strings(sorted) - return sorted -} - -// Transform the json format to ActionGraphContainer -func JsonToActionGraphContainer(inputString string) ([]byte, error) { - var aqueryProtoResult analysis_v2_proto.ActionGraphContainer - err := json.Unmarshal([]byte(inputString), &aqueryProtoResult) - if err != nil { - return []byte(""), err - } - data, _ := proto.Marshal(&aqueryProtoResult) - return data, err -} diff --git a/bazel/bazel_proxy.go b/bazel/bazel_proxy.go deleted file mode 100644 index 229818da0..000000000 --- a/bazel/bazel_proxy.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2023 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "bytes" - "encoding/gob" - "fmt" - "net" - os_lib "os" - "os/exec" - "path/filepath" - "strings" - "time" -) - -// Logs events of ProxyServer. -type ServerLogger interface { - Fatal(v ...interface{}) - Fatalf(format string, v ...interface{}) - Println(v ...interface{}) -} - -// CmdRequest is a request to the Bazel Proxy server. -type CmdRequest struct { - // Args to the Bazel command. - Argv []string - // Environment variables to pass to the Bazel invocation. Strings should be of - // the form "KEY=VALUE". - Env []string -} - -// CmdResponse is a response from the Bazel Proxy server. -type CmdResponse struct { - Stdout string - Stderr string - ErrorString string -} - -// ProxyClient is a client which can issue Bazel commands to the Bazel -// proxy server. Requests are issued (and responses received) via a unix socket. -// See ProxyServer for more details. -type ProxyClient struct { - outDir string -} - -// ProxyServer is a server which runs as a background goroutine. Each -// request to the server describes a Bazel command which the server should run. -// The server then issues the Bazel command, and returns a response describing -// the stdout/stderr of the command. -// Client-server communication is done via a unix socket under the output -// directory. -// The server is intended to circumvent sandboxing for subprocesses of the -// build. The build orchestrator (soong_ui) can launch a server to exist outside -// of sandboxing, and sandboxed processes (such as soong_build) can issue -// bazel commands through this socket tunnel. This allows a sandboxed process -// to issue bazel requests to a bazel that resides outside of sandbox. This -// is particularly useful to maintain a persistent Bazel server which lives -// past the duration of a single build. -// The ProxyServer will only live as long as soong_ui does; the -// underlying Bazel server will live past the duration of the build. -type ProxyServer struct { - logger ServerLogger - outDir string - workspaceDir string - bazeliskVersion string - // The server goroutine will listen on this channel and stop handling requests - // once it is written to. - done chan struct{} -} - -// NewProxyClient is a constructor for a ProxyClient. -func NewProxyClient(outDir string) *ProxyClient { - return &ProxyClient{ - outDir: outDir, - } -} - -func unixSocketPath(outDir string) string { - return filepath.Join(outDir, "bazelsocket.sock") -} - -// IssueCommand issues a request to the Bazel Proxy Server to issue a Bazel -// request. Returns a response describing the output from the Bazel process -// (if the Bazel process had an error, then the response will include an error). -// Returns an error if there was an issue with the connection to the Bazel Proxy -// server. -func (b *ProxyClient) IssueCommand(req CmdRequest) (CmdResponse, error) { - var resp CmdResponse - var err error - // Check for connections every 1 second. This is chosen to be a relatively - // short timeout, because the proxy server should accept requests quite - // quickly. - d := net.Dialer{Timeout: 1 * time.Second} - var conn net.Conn - conn, err = d.Dial("unix", unixSocketPath(b.outDir)) - if err != nil { - return resp, err - } - defer conn.Close() - - enc := gob.NewEncoder(conn) - if err = enc.Encode(req); err != nil { - return resp, err - } - dec := gob.NewDecoder(conn) - err = dec.Decode(&resp) - return resp, err -} - -// NewProxyServer is a constructor for a ProxyServer. -func NewProxyServer(logger ServerLogger, outDir string, workspaceDir string, bazeliskVersion string) *ProxyServer { - if len(bazeliskVersion) > 0 { - logger.Println("** Using Bazelisk for this build, due to env var USE_BAZEL_VERSION=" + bazeliskVersion + " **") - } - - return &ProxyServer{ - logger: logger, - outDir: outDir, - workspaceDir: workspaceDir, - done: make(chan struct{}), - bazeliskVersion: bazeliskVersion, - } -} - -func ExecBazel(bazelPath string, workspaceDir string, request CmdRequest) (stdout []byte, stderr []byte, cmdErr error) { - bazelCmd := exec.Command(bazelPath, request.Argv...) - bazelCmd.Dir = workspaceDir - bazelCmd.Env = request.Env - - stderrBuffer := &bytes.Buffer{} - bazelCmd.Stderr = stderrBuffer - - if output, err := bazelCmd.Output(); err != nil { - cmdErr = fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---", - err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderrBuffer) - } else { - stdout = output - } - stderr = stderrBuffer.Bytes() - return -} - -func (b *ProxyServer) handleRequest(conn net.Conn) error { - defer conn.Close() - - dec := gob.NewDecoder(conn) - var req CmdRequest - if err := dec.Decode(&req); err != nil { - return fmt.Errorf("Error decoding request: %s", err) - } - - if len(b.bazeliskVersion) > 0 { - req.Env = append(req.Env, "USE_BAZEL_VERSION="+b.bazeliskVersion) - } - stdout, stderr, cmdErr := ExecBazel("./build/bazel/bin/bazel", b.workspaceDir, req) - errorString := "" - if cmdErr != nil { - errorString = cmdErr.Error() - } - - resp := CmdResponse{string(stdout), string(stderr), errorString} - enc := gob.NewEncoder(conn) - if err := enc.Encode(&resp); err != nil { - return fmt.Errorf("Error encoding response: %s", err) - } - return nil -} - -func (b *ProxyServer) listenUntilClosed(listener net.Listener) error { - for { - // Check for connections every 1 second. This is a blocking operation, so - // if the server is closed, the goroutine will not fully close until this - // deadline is reached. Thus, this deadline is short (but not too short - // so that the routine churns). - listener.(*net.UnixListener).SetDeadline(time.Now().Add(time.Second)) - conn, err := listener.Accept() - - select { - case <-b.done: - return nil - default: - } - - if err != nil { - if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() { - // Timeout is normal and expected while waiting for client to establish - // a connection. - continue - } else { - b.logger.Fatalf("Listener error: %s", err) - } - } - - err = b.handleRequest(conn) - if err != nil { - b.logger.Fatal(err) - } - } -} - -// Start initializes the server unix socket and (in a separate goroutine) -// handles requests on the socket until the server is closed. Returns an error -// if a failure occurs during initialization. Will log any post-initialization -// errors to the server's logger. -func (b *ProxyServer) Start() error { - unixSocketAddr := unixSocketPath(b.outDir) - if err := os_lib.RemoveAll(unixSocketAddr); err != nil { - return fmt.Errorf("couldn't remove socket '%s': %s", unixSocketAddr, err) - } - listener, err := net.Listen("unix", unixSocketAddr) - - if err != nil { - return fmt.Errorf("error listening on socket '%s': %s", unixSocketAddr, err) - } - - go b.listenUntilClosed(listener) - return nil -} - -// Close shuts down the server. This will stop the server from listening for -// additional requests. -func (b *ProxyServer) Close() { - b.done <- struct{}{} -} diff --git a/bazel/constants.go b/bazel/constants.go deleted file mode 100644 index b10f256f0..000000000 --- a/bazel/constants.go +++ /dev/null @@ -1,30 +0,0 @@ -package bazel - -type RunName string - -// Below is a list bazel execution run names used through out the -// Platform Build systems. Each run name represents an unique key -// to query the bazel metrics. -const ( - // Perform a bazel build of the phony root to generate symlink forests - // for dependencies of the bazel build. - BazelBuildPhonyRootRunName = RunName("bazel-build-phony-root") - - // Perform aquery of the bazel build root to retrieve action information. - AqueryBuildRootRunName = RunName("aquery-buildroot") - - // Perform cquery of the Bazel build root and its dependencies. - CqueryBuildRootRunName = RunName("cquery-buildroot") - - // Run bazel as a ninja executer - BazelNinjaExecRunName = RunName("bazel-ninja-exec") - - SoongInjectionDirName = "soong_injection" - - GeneratedBazelFileWarning = "# GENERATED FOR BAZEL FROM SOONG. DO NOT EDIT." -) - -// String returns the name of the run. -func (c RunName) String() string { - return string(c) -} diff --git a/bazel/cquery/Android.bp b/bazel/cquery/Android.bp deleted file mode 100644 index 74f772184..000000000 --- a/bazel/cquery/Android.bp +++ /dev/null @@ -1,17 +0,0 @@ -package { - default_applicable_licenses: ["Android-Apache-2.0"], -} - -bootstrap_go_package { - name: "soong-cquery", - pkgPath: "android/soong/bazel/cquery", - srcs: [ - "request_type.go", - ], - pluginFor: [ - "soong_build", - ], - testSrcs: [ - "request_type_test.go", - ], -} diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go deleted file mode 100644 index 791c6bc23..000000000 --- a/bazel/cquery/request_type.go +++ /dev/null @@ -1,426 +0,0 @@ -package cquery - -import ( - "encoding/json" - "fmt" - "strings" -) - -var ( - GetOutputFiles = &getOutputFilesRequestType{} - GetCcInfo = &getCcInfoType{} - GetApexInfo = &getApexInfoType{} - GetCcUnstrippedInfo = &getCcUnstrippedInfoType{} - GetPrebuiltFileInfo = &getPrebuiltFileInfo{} -) - -type CcAndroidMkInfo struct { - LocalStaticLibs []string - LocalWholeStaticLibs []string - LocalSharedLibs []string -} - -type CcInfo struct { - CcAndroidMkInfo - OutputFiles []string - CcObjectFiles []string - CcSharedLibraryFiles []string - CcStaticLibraryFiles []string - Includes []string - SystemIncludes []string - Headers []string - // Archives owned by the current target (not by its dependencies). These will - // be a subset of OutputFiles. (or static libraries, this will be equal to OutputFiles, - // but general cc_library will also have dynamic libraries in output files). - RootStaticArchives []string - // Dynamic libraries (.so files) created by the current target. These will - // be a subset of OutputFiles. (or shared libraries, this will be equal to OutputFiles, - // but general cc_library will also have dynamic libraries in output files). - RootDynamicLibraries []string - TidyFiles []string - TocFile string - UnstrippedOutput string - AbiDiffFiles []string -} - -type getOutputFilesRequestType struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getOutputFilesRequestType) Name() string { - return "getOutputFiles" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getOutputFilesRequestType) StarlarkFunctionBody() string { - return "return ', '.join([f.path for f in target.files.to_list()])" -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getOutputFilesRequestType) ParseResult(rawString string) []string { - return splitOrEmpty(rawString, ", ") -} - -type getCcInfoType struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getCcInfoType) Name() string { - return "getCcInfo" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getCcInfoType) StarlarkFunctionBody() string { - return ` -outputFiles = [f.path for f in target.files.to_list()] -p = providers(target) -cc_info = p.get("CcInfo") -if not cc_info: - fail("%s did not provide CcInfo" % id_string) - -includes = cc_info.compilation_context.includes.to_list() -system_includes = cc_info.compilation_context.system_includes.to_list() -headers = [f.path for f in cc_info.compilation_context.headers.to_list()] - -ccObjectFiles = [] -staticLibraries = [] -rootStaticArchives = [] -linker_inputs = cc_info.linking_context.linker_inputs.to_list() - -static_info_tag = "//build/bazel/rules/cc:cc_library_static.bzl%CcStaticLibraryInfo" -if static_info_tag in p: - static_info = p[static_info_tag] - ccObjectFiles = [f.path for f in static_info.objects] - rootStaticArchives = [static_info.root_static_archive.path] -else: - for linker_input in linker_inputs: - for library in linker_input.libraries: - for object in library.objects: - ccObjectFiles += [object.path] - if library.static_library: - staticLibraries.append(library.static_library.path) - if linker_input.owner == target.label: - rootStaticArchives.append(library.static_library.path) - -sharedLibraries = [] -rootSharedLibraries = [] - -shared_info_tag = "//build/bazel/rules/cc:cc_library_shared.bzl%CcSharedLibraryOutputInfo" -stubs_tag = "//build/bazel/rules/cc:cc_stub_library.bzl%CcStubInfo" -unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo" -unstripped = "" - -if shared_info_tag in p: - shared_info = p[shared_info_tag] - path = shared_info.output_file.path - sharedLibraries.append(path) - rootSharedLibraries += [path] - unstripped = path - if unstripped_tag in p: - unstripped = p[unstripped_tag].unstripped.path -elif stubs_tag in p: - rootSharedLibraries.extend([f.path for f in target.files.to_list()]) -else: - for linker_input in linker_inputs: - for library in linker_input.libraries: - if library.dynamic_library: - path = library.dynamic_library.path - sharedLibraries.append(path) - if linker_input.owner == target.label: - rootSharedLibraries.append(path) - -toc_file = "" -toc_file_tag = "//build/bazel/rules/cc:generate_toc.bzl%CcTocInfo" -if toc_file_tag in p: - toc_file = p[toc_file_tag].toc.path -else: - # NOTE: It's OK if there's no ToC, as Soong just uses it for optimization - pass - -tidy_files = [] -clang_tidy_info = p.get("//build/bazel/rules/cc:clang_tidy.bzl%ClangTidyInfo") -if clang_tidy_info: - tidy_files = [v.path for v in clang_tidy_info.transitive_tidy_files.to_list()] - -abi_diff_files = [] -abi_diff_info = p.get("//build/bazel/rules/abi:abi_dump.bzl%AbiDiffInfo") -if abi_diff_info: - abi_diff_files = [f.path for f in abi_diff_info.diff_files.to_list()] - -local_static_libs = [] -local_whole_static_libs = [] -local_shared_libs = [] -androidmk_tag = "//build/bazel/rules/cc:cc_library_common.bzl%CcAndroidMkInfo" -if androidmk_tag in p: - androidmk_info = p[androidmk_tag] - local_static_libs = androidmk_info.local_static_libs - local_whole_static_libs = androidmk_info.local_whole_static_libs - local_shared_libs = androidmk_info.local_shared_libs - -return json.encode({ - "OutputFiles": outputFiles, - "CcObjectFiles": ccObjectFiles, - "CcSharedLibraryFiles": sharedLibraries, - "CcStaticLibraryFiles": staticLibraries, - "Includes": includes, - "SystemIncludes": system_includes, - "Headers": headers, - "RootStaticArchives": rootStaticArchives, - "RootDynamicLibraries": rootSharedLibraries, - "TidyFiles": [t for t in tidy_files], - "TocFile": toc_file, - "UnstrippedOutput": unstripped, - "AbiDiffFiles": abi_diff_files, - "LocalStaticLibs": [l for l in local_static_libs], - "LocalWholeStaticLibs": [l for l in local_whole_static_libs], - "LocalSharedLibs": [l for l in local_shared_libs], -})` - -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getCcInfoType) ParseResult(rawString string) (CcInfo, error) { - var ccInfo CcInfo - if err := parseJson(rawString, &ccInfo); err != nil { - return ccInfo, err - } - return ccInfo, nil -} - -// Query Bazel for the artifacts generated by the apex modules. -type getApexInfoType struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getApexInfoType) Name() string { - return "getApexInfo" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getApexInfoType) StarlarkFunctionBody() string { - return ` -info = providers(target).get("//build/bazel/rules/apex:apex_info.bzl%ApexInfo") -if not info: - fail("%s did not provide ApexInfo" % id_string) -bundle_key_info = info.bundle_key_info -container_key_info = info.container_key_info - -signed_compressed_output = "" # no .capex if the apex is not compressible, cannot be None as it needs to be json encoded. -if info.signed_compressed_output: - signed_compressed_output = info.signed_compressed_output.path - -mk_info = providers(target).get("//build/bazel/rules/apex:apex_info.bzl%ApexMkInfo") -if not mk_info: - fail("%s did not provide ApexMkInfo" % id_string) - -tidy_files = [] -clang_tidy_info = providers(target).get("//build/bazel/rules/cc:clang_tidy.bzl%ClangTidyInfo") -if clang_tidy_info: - tidy_files = [v.path for v in clang_tidy_info.transitive_tidy_files.to_list()] - -return json.encode({ - "signed_output": info.signed_output.path, - "signed_compressed_output": signed_compressed_output, - "unsigned_output": info.unsigned_output.path, - "provides_native_libs": [str(lib) for lib in info.provides_native_libs], - "requires_native_libs": [str(lib) for lib in info.requires_native_libs], - "bundle_key_info": [bundle_key_info.public_key.path, bundle_key_info.private_key.path], - "container_key_info": [container_key_info.pem.path, container_key_info.pk8.path, container_key_info.key_name], - "package_name": info.package_name, - "symbols_used_by_apex": info.symbols_used_by_apex.path, - "java_symbols_used_by_apex": info.java_symbols_used_by_apex.path, - "backing_libs": info.backing_libs.path, - "bundle_file": info.base_with_config_zip.path, - "installed_files": info.installed_files.path, - "make_modules_to_install": mk_info.make_modules_to_install, - "files_info": mk_info.files_info, - "tidy_files": [t for t in tidy_files], -})` -} - -type ApexInfo struct { - // From the ApexInfo provider - SignedOutput string `json:"signed_output"` - SignedCompressedOutput string `json:"signed_compressed_output"` - UnsignedOutput string `json:"unsigned_output"` - ProvidesLibs []string `json:"provides_native_libs"` - RequiresLibs []string `json:"requires_native_libs"` - BundleKeyInfo []string `json:"bundle_key_info"` - ContainerKeyInfo []string `json:"container_key_info"` - PackageName string `json:"package_name"` - SymbolsUsedByApex string `json:"symbols_used_by_apex"` - JavaSymbolsUsedByApex string `json:"java_symbols_used_by_apex"` - BackingLibs string `json:"backing_libs"` - BundleFile string `json:"bundle_file"` - InstalledFiles string `json:"installed_files"` - TidyFiles []string `json:"tidy_files"` - - // From the ApexMkInfo provider - MakeModulesToInstall []string `json:"make_modules_to_install"` - PayloadFilesInfo []map[string]string `json:"files_info"` -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getApexInfoType) ParseResult(rawString string) (ApexInfo, error) { - var info ApexInfo - err := parseJson(rawString, &info) - return info, err -} - -// getCcUnstrippedInfoType implements cqueryRequest interface. It handles the -// interaction with `bazel cquery` to retrieve CcUnstrippedInfo provided -// by the` cc_binary` and `cc_shared_library` rules. -type getCcUnstrippedInfoType struct{} - -func (g getCcUnstrippedInfoType) Name() string { - return "getCcUnstrippedInfo" -} - -func (g getCcUnstrippedInfoType) StarlarkFunctionBody() string { - return ` -p = providers(target) -output_path = target.files.to_list()[0].path - -unstripped = output_path -unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo" -if unstripped_tag in p: - unstripped_info = p[unstripped_tag] - unstripped = unstripped_info.unstripped[0].files.to_list()[0].path - -local_static_libs = [] -local_whole_static_libs = [] -local_shared_libs = [] -androidmk_tag = "//build/bazel/rules/cc:cc_library_common.bzl%CcAndroidMkInfo" -if androidmk_tag in p: - androidmk_info = p[androidmk_tag] - local_static_libs = androidmk_info.local_static_libs - local_whole_static_libs = androidmk_info.local_whole_static_libs - local_shared_libs = androidmk_info.local_shared_libs - -tidy_files = [] -clang_tidy_info = p.get("//build/bazel/rules/cc:clang_tidy.bzl%ClangTidyInfo") -if clang_tidy_info: - tidy_files = [v.path for v in clang_tidy_info.transitive_tidy_files.to_list()] - -return json.encode({ - "OutputFile": output_path, - "UnstrippedOutput": unstripped, - "LocalStaticLibs": [l for l in local_static_libs], - "LocalWholeStaticLibs": [l for l in local_whole_static_libs], - "LocalSharedLibs": [l for l in local_shared_libs], - "TidyFiles": [t for t in tidy_files], -}) -` -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getCcUnstrippedInfoType) ParseResult(rawString string) (CcUnstrippedInfo, error) { - var info CcUnstrippedInfo - err := parseJson(rawString, &info) - return info, err -} - -type CcUnstrippedInfo struct { - CcAndroidMkInfo - OutputFile string - UnstrippedOutput string - TidyFiles []string -} - -// splitOrEmpty is a modification of strings.Split() that returns an empty list -// if the given string is empty. -func splitOrEmpty(s string, sep string) []string { - if len(s) < 1 { - return []string{} - } else { - return strings.Split(s, sep) - } -} - -// parseJson decodes json string into the fields of the receiver. -// Unknown attribute name causes panic. -func parseJson(jsonString string, info interface{}) error { - decoder := json.NewDecoder(strings.NewReader(jsonString)) - decoder.DisallowUnknownFields() //useful to detect typos, e.g. in unit tests - err := decoder.Decode(info) - if err != nil { - return fmt.Errorf("cannot parse cquery result '%s': %s", jsonString, err) - } - return nil -} - -type getPrebuiltFileInfo struct{} - -// Name returns a string name for this request type. Such request type names must be unique, -// and must only consist of alphanumeric characters. -func (g getPrebuiltFileInfo) Name() string { - return "getPrebuiltFileInfo" -} - -// StarlarkFunctionBody returns a starlark function body to process this request type. -// The returned string is the body of a Starlark function which obtains -// all request-relevant information about a target and returns a string containing -// this information. -// The function should have the following properties: -// - The arguments are `target` (a configured target) and `id_string` (the label + configuration). -// - The return value must be a string. -// - The function body should not be indented outside of its own scope. -func (g getPrebuiltFileInfo) StarlarkFunctionBody() string { - return ` -p = providers(target) -prebuilt_file_info = p.get("//build/bazel/rules:prebuilt_file.bzl%PrebuiltFileInfo") -if not prebuilt_file_info: - fail("%s did not provide PrebuiltFileInfo" % id_string) - -return json.encode({ - "Src": prebuilt_file_info.src.path, - "Dir": prebuilt_file_info.dir, - "Filename": prebuilt_file_info.filename, - "Installable": prebuilt_file_info.installable, -})` -} - -type PrebuiltFileInfo struct { - // TODO: b/207489266 - Fully support all properties in prebuilt_file - Src string - Dir string - Filename string - Installable bool -} - -// ParseResult returns a value obtained by parsing the result of the request's Starlark function. -// The given rawString must correspond to the string output which was created by evaluating the -// Starlark given in StarlarkFunctionBody. -func (g getPrebuiltFileInfo) ParseResult(rawString string) (PrebuiltFileInfo, error) { - var info PrebuiltFileInfo - err := parseJson(rawString, &info) - return info, err -} diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go deleted file mode 100644 index e772bb7d6..000000000 --- a/bazel/cquery/request_type_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package cquery - -import ( - "encoding/json" - "reflect" - "strings" - "testing" -) - -func TestGetOutputFilesParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput []string - }{ - { - description: "no result", - input: "", - expectedOutput: []string{}, - }, - { - description: "one result", - input: "test", - expectedOutput: []string{"test"}, - }, - { - description: "splits on comma with space", - input: "foo, bar", - expectedOutput: []string{"foo", "bar"}, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput := GetOutputFiles.ParseResult(tc.input) - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetCcInfoParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - inputCcInfo CcInfo - expectedOutput CcInfo - }{ - { - description: "no result", - inputCcInfo: CcInfo{}, - expectedOutput: CcInfo{}, - }, - { - description: "all items set", - inputCcInfo: CcInfo{ - OutputFiles: []string{"out1", "out2"}, - CcObjectFiles: []string{"object1", "object2"}, - CcSharedLibraryFiles: []string{"shared_lib1", "shared_lib2"}, - CcStaticLibraryFiles: []string{"static_lib1", "static_lib2"}, - Includes: []string{".", "dir/subdir"}, - SystemIncludes: []string{"system/dir", "system/other/dir"}, - Headers: []string{"dir/subdir/hdr.h"}, - RootStaticArchives: []string{"rootstaticarchive1"}, - RootDynamicLibraries: []string{"rootdynamiclibrary1"}, - TocFile: "lib.so.toc", - }, - expectedOutput: CcInfo{ - OutputFiles: []string{"out1", "out2"}, - CcObjectFiles: []string{"object1", "object2"}, - CcSharedLibraryFiles: []string{"shared_lib1", "shared_lib2"}, - CcStaticLibraryFiles: []string{"static_lib1", "static_lib2"}, - Includes: []string{".", "dir/subdir"}, - SystemIncludes: []string{"system/dir", "system/other/dir"}, - Headers: []string{"dir/subdir/hdr.h"}, - RootStaticArchives: []string{"rootstaticarchive1"}, - RootDynamicLibraries: []string{"rootdynamiclibrary1"}, - TocFile: "lib.so.toc", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - jsonInput, _ := json.Marshal(tc.inputCcInfo) - actualOutput, err := GetCcInfo.ParseResult(string(jsonInput)) - if err != nil { - t.Errorf("error parsing result: %q", err) - } else if err == nil && !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v\n!= actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetCcInfoParseResultsError(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedError string - }{ - { - description: "not json", - input: ``, - expectedError: `cannot parse cquery result '': EOF`, - }, - { - description: "invalid field", - input: `{ - "toc_file": "dir/file.so.toc" -}`, - expectedError: `json: unknown field "toc_file"`, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - _, err := GetCcInfo.ParseResult(tc.input) - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) - } - }) - } -} - -func TestGetApexInfoParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput ApexInfo - }{ - { - description: "no result", - input: "{}", - expectedOutput: ApexInfo{}, - }, - { - description: "one result", - input: `{ - "signed_output":"my.apex", - "unsigned_output":"my.apex.unsigned", - "requires_native_libs":["//bionic/libc:libc","//bionic/libdl:libdl"], - "bundle_key_info":["foo.pem", "foo.privkey"], - "container_key_info":["foo.x509.pem", "foo.pk8", "foo"], - "package_name":"package.name", - "symbols_used_by_apex": "path/to/my.apex_using.txt", - "backing_libs":"path/to/backing.txt", - "bundle_file": "dir/bundlefile.zip", - "installed_files":"path/to/installed-files.txt", - "provides_native_libs":[], - "make_modules_to_install": ["foo","bar"] -}`, - expectedOutput: ApexInfo{ - // ApexInfo - SignedOutput: "my.apex", - UnsignedOutput: "my.apex.unsigned", - RequiresLibs: []string{"//bionic/libc:libc", "//bionic/libdl:libdl"}, - ProvidesLibs: []string{}, - BundleKeyInfo: []string{"foo.pem", "foo.privkey"}, - ContainerKeyInfo: []string{"foo.x509.pem", "foo.pk8", "foo"}, - PackageName: "package.name", - SymbolsUsedByApex: "path/to/my.apex_using.txt", - BackingLibs: "path/to/backing.txt", - BundleFile: "dir/bundlefile.zip", - InstalledFiles: "path/to/installed-files.txt", - - // ApexMkInfo - MakeModulesToInstall: []string{"foo", "bar"}, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput, err := GetApexInfo.ParseResult(tc.input) - if err != nil { - t.Errorf("Unexpected error %q", err) - } - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetApexInfoParseResultsError(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedError string - }{ - { - description: "not json", - input: ``, - expectedError: `cannot parse cquery result '': EOF`, - }, - { - description: "invalid field", - input: `{ - "fake_field": "path/to/file" -}`, - expectedError: `json: unknown field "fake_field"`, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - _, err := GetApexInfo.ParseResult(tc.input) - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) - } - }) - } -} - -func TestGetCcUnstrippedParseResults(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedOutput CcUnstrippedInfo - }{ - { - description: "no result", - input: "{}", - expectedOutput: CcUnstrippedInfo{}, - }, - { - description: "one result", - input: `{"OutputFile":"myapp", "UnstrippedOutput":"myapp_unstripped"}`, - expectedOutput: CcUnstrippedInfo{ - OutputFile: "myapp", - UnstrippedOutput: "myapp_unstripped", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - actualOutput, err := GetCcUnstrippedInfo.ParseResult(tc.input) - if err != nil { - t.Errorf("Unexpected error %q", err) - } - if !reflect.DeepEqual(tc.expectedOutput, actualOutput) { - t.Errorf("expected %#v != actual %#v", tc.expectedOutput, actualOutput) - } - }) - } -} - -func TestGetCcUnstrippedParseResultsErrors(t *testing.T) { - t.Parallel() - testCases := []struct { - description string - input string - expectedError string - }{ - { - description: "not json", - input: ``, - expectedError: `cannot parse cquery result '': EOF`, - }, - { - description: "invalid field", - input: `{ - "fake_field": "path/to/file" -}`, - expectedError: `json: unknown field "fake_field"`, - }, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - _, err := GetCcUnstrippedInfo.ParseResult(tc.input) - if !strings.Contains(err.Error(), tc.expectedError) { - t.Errorf("expected string %q in error message, got %q", tc.expectedError, err) - } - }) - } -} diff --git a/bin/aninja b/bin/aninja index cceb79489..5acb9681c 100755 --- a/bin/aninja +++ b/bin/aninja @@ -20,6 +20,19 @@ source $(cd $(dirname $BASH_SOURCE) &> /dev/null && pwd)/../../make/shell_utils. require_top require_lunch +case $(uname -s) in + Darwin) + host_arch=darwin-x86 + ;; + Linux) + host_arch=linux-x86 + ;; + *) + >&2 echo Unknown host $(uname -s) + exit 1 + ;; +esac + cd $(gettop) -prebuilts/build-tools/linux-x86/bin/ninja -f out/combined-${TARGET_PRODUCT}.ninja "$@" +prebuilts/build-tools/${host_arch}/bin/ninja -f out/combined-${TARGET_PRODUCT}.ninja "$@" diff --git a/bin/soongdbg b/bin/soongdbg index bfdbbde70..a73bdf90d 100755 --- a/bin/soongdbg +++ b/bin/soongdbg @@ -32,11 +32,13 @@ class Graph: dep.rdeps.add(node) node.dep_tags.setdefault(dep, list()).append(d) - def find_paths(self, id1, id2): + def find_paths(self, id1, id2, tag_filter): # Throws KeyError if one of the names isn't found def recurse(node1, node2, visited): result = set() for dep in node1.rdeps: + if not matches_tag(dep, node1, tag_filter): + continue if dep == node2: result.add(node2) if dep not in visited: @@ -214,6 +216,8 @@ def print_args(parser): help="jq query for each module metadata") parser.add_argument("--deptags", action="store_true", help="show dependency tags (makes the graph much more complex)") + parser.add_argument("--tag", action="append", + help="Limit output to these dependency tags.") group = parser.add_argument_group("output formats", "If no format is provided, a dot file will be written to" @@ -259,13 +263,21 @@ def print_nodes(args, nodes, module_formatter): sys.stdout.write(text) -def get_deps(nodes, root, maxdepth, reverse): +def matches_tag(node, dep, tag_filter): + if not tag_filter: + return True + return not tag_filter.isdisjoint([t.tag_type for t in node.dep_tags[dep]]) + + +def get_deps(nodes, root, maxdepth, reverse, tag_filter): if root in nodes: return nodes.add(root) if maxdepth != 0: for dep in (root.rdeps if reverse else root.deps): - get_deps(nodes, dep, maxdepth-1, reverse) + if not matches_tag(root, dep, tag_filter): + continue + get_deps(nodes, dep, maxdepth-1, reverse, tag_filter) def new_module_formatter(args): @@ -302,7 +314,7 @@ class BetweenCommand: def run(self, args): graph = load_graph() - print_nodes(args, graph.find_paths(args.module[0], args.module[1]), + print_nodes(args, graph.find_paths(args.module[0], args.module[1], set(args.tag)), new_module_formatter(args)) @@ -328,7 +340,7 @@ class DepsCommand: sys.stderr.write(f"error: Can't find root: {id}\n") err = True continue - get_deps(nodes, root, args.depth, args.reverse) + get_deps(nodes, root, args.depth, args.reverse, set(args.tag)) if err: sys.exit(1) print_nodes(args, nodes, new_module_formatter(args)) diff --git a/bpf/bpf.go b/bpf/bpf.go index ce00b5b24..09262e507 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -104,6 +104,14 @@ var _ android.ImageInterface = (*bpf)(nil) func (bpf *bpf) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (bpf *bpf) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return proptools.Bool(bpf.properties.Vendor) +} + +func (bpf *bpf) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + func (bpf *bpf) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !proptools.Bool(bpf.properties.Vendor) } @@ -125,9 +133,6 @@ func (bpf *bpf) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { } func (bpf *bpf) ExtraImageVariations(ctx android.BaseModuleContext) []string { - if proptools.Bool(bpf.properties.Vendor) { - return []string{"vendor"} - } return nil } diff --git a/bpfix/bpfix/bpfix.go b/bpfix/bpfix/bpfix.go index ddaa98aa3..9163ab759 100644 --- a/bpfix/bpfix/bpfix.go +++ b/bpfix/bpfix/bpfix.go @@ -286,7 +286,7 @@ func (f *Fixer) reparse() ([]byte, error) { } func parse(name string, r io.Reader) (*parser.File, error) { - tree, errs := parser.Parse(name, r, parser.NewScope(nil)) + tree, errs := parser.Parse(name, r) if errs != nil { s := "parse error: " for _, err := range errs { diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index b5b49b1ab..f487d3c7c 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -46,7 +46,7 @@ func buildTree(local_include_dirs []string, export_include_dirs []string) (file } `, printListOfStrings(local_include_dirs), printListOfStrings(export_include_dirs)) - tree, errs := parser.Parse("", strings.NewReader(input), parser.NewScope(nil)) + tree, errs := parser.Parse("", strings.NewReader(input)) if len(errs) > 0 { errs = append([]error{fmt.Errorf("failed to parse:\n%s", input)}, errs...) } @@ -167,7 +167,7 @@ func preProcessIn(in string) (fixer *Fixer, err error) { return fixer, err } - tree, errs := parser.Parse("<testcase>", bytes.NewBufferString(in), parser.NewScope(nil)) + tree, errs := parser.Parse("<testcase>", bytes.NewBufferString(in)) if errs != nil { return fixer, err } diff --git a/bpfix/cmd_lib/bpfix.go b/bpfix/cmd_lib/bpfix.go index 1106d4af7..41430f8e4 100644 --- a/bpfix/cmd_lib/bpfix.go +++ b/bpfix/cmd_lib/bpfix.go @@ -66,7 +66,7 @@ func processFile(filename string, in io.Reader, out io.Writer, fixRequest bpfix. return err } r := bytes.NewBuffer(append([]byte(nil), src...)) - file, errs := parser.Parse(filename, r, parser.NewScope(nil)) + file, errs := parser.Parse(filename, r) if len(errs) > 0 { for _, err := range errs { fmt.Fprintln(os.Stderr, err) diff --git a/cc/afdo.go b/cc/afdo.go index 00b22456f..6921edfba 100644 --- a/cc/afdo.go +++ b/cc/afdo.go @@ -176,6 +176,9 @@ func (a *afdoTransitionMutator) IncomingTransition(ctx android.IncomingTransitio func (a *afdoTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { if m, ok := ctx.Module().(*Module); ok && m.afdo != nil { + if !m.Enabled(ctx) { + return + } if variation == "" { // The empty variation is either a module that has enabled AFDO for itself, or the non-AFDO // variant of a dependency. diff --git a/cc/binary.go b/cc/binary.go index 3ff35de56..2ac9a45bc 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -451,7 +451,7 @@ func (binary *binaryDecorator) unstrippedOutputFilePath() android.Path { } func (binary *binaryDecorator) strippedAllOutputFilePath() android.Path { - panic("Not implemented.") + return nil } func (binary *binaryDecorator) setSymlinkList(ctx ModuleContext) { diff --git a/cc/builder.go b/cc/builder.go index 8719d4f87..367bda380 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -798,9 +798,12 @@ func transformObjToStaticLib(ctx android.ModuleContext, // Generate a Rust staticlib from a list of rlibDeps. Returns nil if TransformRlibstoStaticlib is nil or rlibDeps is empty. func generateRustStaticlib(ctx android.ModuleContext, rlibDeps []RustRlibDep) android.Path { if TransformRlibstoStaticlib == nil && len(rlibDeps) > 0 { - // This should only be reachable if a module defines static_rlibs and + // This should only be reachable if a module defines Rust deps in static_libs and // soong-rust hasn't been loaded alongside soong-cc (e.g. in soong-cc tests). - panic(fmt.Errorf("TransformRlibstoStaticlib is not set and static_rlibs is defined in %s", ctx.ModuleName())) + panic(fmt.Errorf( + "TransformRlibstoStaticlib is not set and rust deps are defined in static_libs for %s", + ctx.ModuleName())) + } else if len(rlibDeps) == 0 { return nil } @@ -829,6 +832,7 @@ func generateRustStaticlib(ctx android.ModuleContext, rlibDeps []RustRlibDep) an func genRustStaticlibSrcFile(crateNames []string) string { lines := []string{ "// @Soong generated Source", + "#![no_std]", // pre-emptively set no_std to support both std and no_std. } for _, crate := range crateNames { lines = append(lines, fmt.Sprintf("extern crate %s;", crate)) @@ -99,7 +99,6 @@ type Deps struct { StaticLibs, LateStaticLibs, WholeStaticLibs []string HeaderLibs []string RuntimeLibs []string - Rlibs []string // UnexportedStaticLibs are static libraries that are also passed to -Wl,--exclude-libs= to // prevent automatically exporting symbols. @@ -361,6 +360,8 @@ type BaseProperties struct { Recovery_available *bool // Used by imageMutator, set by ImageMutatorBegin() + VendorVariantNeeded bool `blueprint:"mutated"` + ProductVariantNeeded bool `blueprint:"mutated"` CoreVariantNeeded bool `blueprint:"mutated"` RamdiskVariantNeeded bool `blueprint:"mutated"` VendorRamdiskVariantNeeded bool `blueprint:"mutated"` @@ -744,11 +745,6 @@ func (d libraryDependencyTag) static() bool { return d.Kind == staticLibraryDependency } -// rlib returns true if the libraryDependencyTag is tagging an rlib dependency. -func (d libraryDependencyTag) rlib() bool { - return d.Kind == rlibLibraryDependency -} - func (d libraryDependencyTag) LicenseAnnotations() []android.LicenseAnnotation { if d.shared() { return []android.LicenseAnnotation{android.LicenseAnnotationSharedDependency} @@ -915,6 +911,8 @@ type Module struct { hideApexVariantFromMake bool logtagsPaths android.Paths + + WholeRustStaticlib bool } func (c *Module) AddJSONData(d *map[string]interface{}) { @@ -1190,6 +1188,16 @@ func (c *Module) BuildSharedVariant() bool { panic(fmt.Errorf("BuildSharedVariant called on non-library module: %q", c.BaseModuleName())) } +func (c *Module) BuildRlibVariant() bool { + // cc modules can never build rlib variants + return false +} + +func (c *Module) IsRustFFI() bool { + // cc modules are not Rust modules + return false +} + func (c *Module) Module() android.Module { return c } @@ -2117,10 +2125,60 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake { moduleInfoJSON.Uninstallable = true } + } + + buildComplianceMetadataInfo(ctx, c, deps) + + c.setOutputFiles(ctx) +} +func (c *Module) setOutputFiles(ctx ModuleContext) { + if c.outputFile.Valid() { + ctx.SetOutputFiles(android.Paths{c.outputFile.Path()}, "") + } else { + ctx.SetOutputFiles(android.Paths{}, "") + } + if c.linker != nil { + ctx.SetOutputFiles(android.PathsIfNonNil(c.linker.unstrippedOutputFilePath()), "unstripped") + ctx.SetOutputFiles(android.PathsIfNonNil(c.linker.strippedAllOutputFilePath()), "stripped_all") } } +func buildComplianceMetadataInfo(ctx ModuleContext, c *Module, deps PathDeps) { + // Dump metadata that can not be done in android/compliance-metadata.go + complianceMetadataInfo := ctx.ComplianceMetadataInfo() + complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(ctx.static())) + complianceMetadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, c.outputFile.String()) + + // Static deps + staticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(false)) + staticDepNames := make([]string, 0, len(staticDeps)) + for _, dep := range staticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + + staticDepPaths := make([]string, 0, len(deps.StaticLibs)) + for _, dep := range deps.StaticLibs { + staticDepPaths = append(staticDepPaths, dep.String()) + } + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths)) + + // Whole static deps + wholeStaticDeps := ctx.GetDirectDepsWithTag(StaticDepTag(true)) + wholeStaticDepNames := make([]string, 0, len(wholeStaticDeps)) + for _, dep := range wholeStaticDeps { + wholeStaticDepNames = append(wholeStaticDepNames, dep.Name()) + } + + wholeStaticDepPaths := make([]string, 0, len(deps.WholeStaticLibs)) + for _, dep := range deps.WholeStaticLibs { + wholeStaticDepPaths = append(wholeStaticDepPaths, dep.String()) + } + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEPS, android.FirstUniqueStrings(wholeStaticDepNames)) + complianceMetadataInfo.SetListValue(android.ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES, android.FirstUniqueStrings(wholeStaticDepPaths)) +} + func (c *Module) maybeUnhideFromMake() { // If a lib is directly included in any of the APEXes or is not available to the // platform (which is often the case when the stub is provided as a prebuilt), @@ -2228,7 +2286,6 @@ func (c *Module) deps(ctx DepsContext) Deps { deps.WholeStaticLibs = android.LastUniqueStrings(deps.WholeStaticLibs) deps.StaticLibs = android.LastUniqueStrings(deps.StaticLibs) - deps.Rlibs = android.LastUniqueStrings(deps.Rlibs) deps.LateStaticLibs = android.LastUniqueStrings(deps.LateStaticLibs) deps.SharedLibs = android.LastUniqueStrings(deps.SharedLibs) deps.LateSharedLibs = android.LastUniqueStrings(deps.LateSharedLibs) @@ -2509,7 +2566,7 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { if c.ImageVariation().Variation == android.CoreVariation && c.Device() && c.Target().NativeBridge == android.NativeBridgeDisabled { actx.AddVariationDependencies( - []blueprint.Variation{{Mutator: "image", Variation: VendorVariation}}, + []blueprint.Variation{{Mutator: "image", Variation: android.VendorVariation}}, llndkHeaderLibTag, deps.LlndkHeaderLibs...) } @@ -2523,28 +2580,20 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { } for _, lib := range deps.StaticLibs { + // Some dependencies listed in static_libs might actually be rust_ffi rlib variants. depTag := libraryDependencyTag{Kind: staticLibraryDependency} + if inList(lib, deps.ReexportStaticLibHeaders) { depTag.reexportFlags = true } if inList(lib, deps.ExcludeLibsForApex) { depTag.excludeInApex = true } - actx.AddVariationDependencies([]blueprint.Variation{ {Mutator: "link", Variation: "static"}, }, depTag, lib) } - for _, lib := range deps.Rlibs { - depTag := libraryDependencyTag{Kind: rlibLibraryDependency} - actx.AddVariationDependencies([]blueprint.Variation{ - {Mutator: "link", Variation: ""}, - {Mutator: "rust_libraries", Variation: "rlib"}, - {Mutator: "rust_stdlinkage", Variation: "rlib-std"}, - }, depTag, lib) - } - // staticUnwinderDep is treated as staticDep for Q apexes // so that native libraries/binaries are linked with static unwinder // because Q libc doesn't have unwinder APIs @@ -3132,78 +3181,86 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) } - case libDepTag.rlib(): - rlibDep := RustRlibDep{LibPath: linkFile.Path(), CrateName: ccDep.CrateName(), LinkDirs: ccDep.ExportedCrateLinkDirs()} - depPaths.ReexportedRustRlibDeps = append(depPaths.ReexportedRustRlibDeps, rlibDep) - depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, rlibDep) - depPaths.IncludeDirs = append(depPaths.IncludeDirs, depExporterInfo.IncludeDirs...) - depPaths.ReexportedDirs = append(depPaths.ReexportedDirs, depExporterInfo.IncludeDirs...) - case libDepTag.static(): - staticLibraryInfo, isStaticLib := android.OtherModuleProvider(ctx, dep, StaticLibraryInfoProvider) - if !isStaticLib { - if !ctx.Config().AllowMissingDependencies() { - ctx.ModuleErrorf("module %q is not a static library", depName) - } else { - ctx.AddMissingDependencies([]string{depName}) + if ccDep.RustLibraryInterface() { + rlibDep := RustRlibDep{LibPath: linkFile.Path(), CrateName: ccDep.CrateName(), LinkDirs: ccDep.ExportedCrateLinkDirs()} + depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, rlibDep) + depPaths.IncludeDirs = append(depPaths.IncludeDirs, depExporterInfo.IncludeDirs...) + if libDepTag.wholeStatic { + depPaths.ReexportedDirs = append(depPaths.ReexportedDirs, depExporterInfo.IncludeDirs...) + depPaths.ReexportedRustRlibDeps = append(depPaths.ReexportedRustRlibDeps, rlibDep) + + // If whole_static, track this as we want to make sure that in a final linkage for a shared library, + // exported functions from the rust generated staticlib still exported. + if c.CcLibrary() && c.Shared() { + c.WholeRustStaticlib = true + } } - return - } - // Stubs lib doesn't link to the static lib dependencies. Don't set - // linkFile, depFile, and ptr. - if c.IsStubs() { - break - } + } else { + staticLibraryInfo, isStaticLib := android.OtherModuleProvider(ctx, dep, StaticLibraryInfoProvider) + if !isStaticLib { + if !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf("module %q is not a static library", depName) + } else { + ctx.AddMissingDependencies([]string{depName}) + } + return + } - linkFile = android.OptionalPathForPath(staticLibraryInfo.StaticLibrary) - if libDepTag.wholeStatic { - ptr = &depPaths.WholeStaticLibs - if len(staticLibraryInfo.Objects.objFiles) > 0 { - depPaths.WholeStaticLibObjs = depPaths.WholeStaticLibObjs.Append(staticLibraryInfo.Objects) - } else { - // This case normally catches prebuilt static - // libraries, but it can also occur when - // AllowMissingDependencies is on and the - // dependencies has no sources of its own - // but has a whole_static_libs dependency - // on a missing library. We want to depend - // on the .a file so that there is something - // in the dependency tree that contains the - // error rule for the missing transitive - // dependency. - depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, linkFile.Path()) + // Stubs lib doesn't link to the static lib dependencies. Don't set + // linkFile, depFile, and ptr. + if c.IsStubs() { + break } - depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, - staticLibraryInfo.WholeStaticLibsFromPrebuilts...) - } else { - switch libDepTag.Order { - case earlyLibraryDependency: - panic(fmt.Errorf("early static libs not suppported")) - case normalLibraryDependency: - // static dependencies will be handled separately so they can be ordered - // using transitive dependencies. - ptr = nil - directStaticDeps = append(directStaticDeps, staticLibraryInfo) - case lateLibraryDependency: - ptr = &depPaths.LateStaticLibs - default: - panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) + + linkFile = android.OptionalPathForPath(staticLibraryInfo.StaticLibrary) + if libDepTag.wholeStatic { + ptr = &depPaths.WholeStaticLibs + if len(staticLibraryInfo.Objects.objFiles) > 0 { + depPaths.WholeStaticLibObjs = depPaths.WholeStaticLibObjs.Append(staticLibraryInfo.Objects) + } else { + // This case normally catches prebuilt static + // libraries, but it can also occur when + // AllowMissingDependencies is on and the + // dependencies has no sources of its own + // but has a whole_static_libs dependency + // on a missing library. We want to depend + // on the .a file so that there is something + // in the dependency tree that contains the + // error rule for the missing transitive + // dependency. + depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, linkFile.Path()) + } + depPaths.WholeStaticLibsFromPrebuilts = append(depPaths.WholeStaticLibsFromPrebuilts, + staticLibraryInfo.WholeStaticLibsFromPrebuilts...) + } else { + switch libDepTag.Order { + case earlyLibraryDependency: + panic(fmt.Errorf("early static libs not supported")) + case normalLibraryDependency: + // static dependencies will be handled separately so they can be ordered + // using transitive dependencies. + ptr = nil + directStaticDeps = append(directStaticDeps, staticLibraryInfo) + case lateLibraryDependency: + ptr = &depPaths.LateStaticLibs + default: + panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) + } } - } - // We re-export the Rust static_rlibs so rlib dependencies don't need to be redeclared by cc_library_static dependents. - // E.g. libfoo (cc_library_static) depends on libfoo.ffi (a rust_ffi rlib), libbar depending on libfoo shouldn't have to also add libfoo.ffi to static_rlibs. - depPaths.ReexportedRustRlibDeps = append(depPaths.ReexportedRustRlibDeps, depExporterInfo.RustRlibDeps...) - depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, depExporterInfo.RustRlibDeps...) + // Collect any exported Rust rlib deps from static libraries which have been included as whole_static_libs + depPaths.RustRlibDeps = append(depPaths.RustRlibDeps, depExporterInfo.RustRlibDeps...) - if libDepTag.unexportedSymbols { - depPaths.LdFlags = append(depPaths.LdFlags, - "-Wl,--exclude-libs="+staticLibraryInfo.StaticLibrary.Base()) + if libDepTag.unexportedSymbols { + depPaths.LdFlags = append(depPaths.LdFlags, + "-Wl,--exclude-libs="+staticLibraryInfo.StaticLibrary.Base()) + } } } - if libDepTag.static() && !libDepTag.wholeStatic { + if libDepTag.static() && !libDepTag.wholeStatic && !ccDep.RustLibraryInterface() { if !ccDep.CcLibraryInterface() || !ccDep.Static() { ctx.ModuleErrorf("module %q not a static library", depName) return @@ -3290,12 +3347,14 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { c.Properties.AndroidMkSharedLibs = append( c.Properties.AndroidMkSharedLibs, makeLibName) case libDepTag.static(): - if libDepTag.wholeStatic { - c.Properties.AndroidMkWholeStaticLibs = append( - c.Properties.AndroidMkWholeStaticLibs, makeLibName) - } else { - c.Properties.AndroidMkStaticLibs = append( - c.Properties.AndroidMkStaticLibs, makeLibName) + if !ccDep.RustLibraryInterface() { + if libDepTag.wholeStatic { + c.Properties.AndroidMkWholeStaticLibs = append( + c.Properties.AndroidMkWholeStaticLibs, makeLibName) + } else { + c.Properties.AndroidMkStaticLibs = append( + c.Properties.AndroidMkStaticLibs, makeLibName) + } } } } else if !c.IsStubs() { @@ -3576,28 +3635,6 @@ func (c *Module) IntermPathForModuleOut() android.OptionalPath { return c.outputFile } -func (c *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if c.outputFile.Valid() { - return android.Paths{c.outputFile.Path()}, nil - } - return android.Paths{}, nil - case "unstripped": - if c.linker != nil { - return android.PathsIfNonNil(c.linker.unstrippedOutputFilePath()), nil - } - return nil, nil - case "stripped_all": - if c.linker != nil { - return android.PathsIfNonNil(c.linker.strippedAllOutputFilePath()), nil - } - return nil, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (c *Module) static() bool { if static, ok := c.linker.(interface { static() bool diff --git a/cc/cc_test.go b/cc/cc_test.go index c2bb25ad8..ccdaae58f 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -300,13 +300,9 @@ func TestDataLibs(t *testing.T) { config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module() - testBinary := module.(*Module).linker.(*testBinary) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Errorf("Expected cc_test to produce output files, error: %s", err) - return - } + testingModule := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon") + testBinary := testingModule.Module().(*Module).linker.(*testBinary) + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Errorf("expected exactly one output file. output files: [%s]", outputFiles) return @@ -356,12 +352,10 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module() + testingModule := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon") + module := testingModule.Module() testBinary := module.(*Module).linker.(*testBinary) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected cc_test to produce output files, error: %s", err) - } + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } @@ -1407,12 +1401,10 @@ func TestDataLibsPrebuiltSharedTestLibrary(t *testing.T) { config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module() + testingModule := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon") + module := testingModule.Module() testBinary := module.(*Module).linker.(*testBinary) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected cc_test to produce output files, error: %s", err) - } + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Errorf("expected exactly one output file. output files: [%s]", outputFiles) } @@ -2908,8 +2900,6 @@ func TestIncludeDirectoryOrdering(t *testing.T) { PrepareForIntegrationTestWithCc, android.FixtureAddTextFile("external/foo/Android.bp", bp), ).RunTest(t) - // Use the arm variant instead of the arm64 variant so that it gets headers from - // ndk_libandroid_support to test LateStaticLibs. cflags := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_sdk_static").Output("obj/external/foo/foo.o").Args["cFlags"] var includes []string @@ -3120,12 +3110,8 @@ func TestStrippedAllOutputFile(t *testing.T) { ` config := TestConfig(t.TempDir(), android.Android, nil, bp, nil) ctx := testCcWithConfig(t, config) - module := ctx.ModuleForTests("test_lib", "android_arm_armv7-a-neon_shared").Module() - outputFile, err := module.(android.OutputFileProducer).OutputFiles("stripped_all") - if err != nil { - t.Errorf("Expected cc_library to produce output files, error: %s", err) - return - } + testingModule := ctx.ModuleForTests("test_lib", "android_arm_armv7-a-neon_shared") + outputFile := testingModule.OutputFiles(t, "stripped_all") if !strings.HasSuffix(outputFile.Strings()[0], "/stripped_all/test_lib.so") { t.Errorf("Unexpected output file: %s", outputFile.Strings()[0]) return diff --git a/cc/cmake_ext_add_aidl_library.txt b/cc/cmake_ext_add_aidl_library.txt index af5bdf6c0..d5c134e83 100644 --- a/cc/cmake_ext_add_aidl_library.txt +++ b/cc/cmake_ext_add_aidl_library.txt @@ -1,3 +1,12 @@ +if ("${CMAKE_HOST_SYSTEM_PROCESSOR}" MATCHES "^(arm|aarch)") + set(PREBUILTS_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prebuilts/host/linux_musl-arm64/bin") +else() + set(PREBUILTS_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prebuilts/host/linux-x86/bin") +endif() +if (NOT AIDL_BIN) + find_program(AIDL_BIN aidl REQUIRED HINTS "${PREBUILTS_BIN_DIR}") +endif() + function(add_aidl_library NAME LANG AIDLROOT SOURCES AIDLFLAGS) if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20") cmake_policy(SET CMP0116 NEW) @@ -25,7 +34,7 @@ function(add_aidl_library NAME LANG AIDLROOT SOURCES AIDLFLAGS) endif() set(DEPFILE_ARG) - if (NOT ${CMAKE_GENERATOR} MATCHES "Unix Makefiles") + if (NOT ${CMAKE_GENERATOR} STREQUAL "Unix Makefiles") set(DEPFILE_ARG DEPFILE "${GEN_SOURCE}.d") endif() @@ -57,7 +66,7 @@ function(add_aidl_library NAME LANG AIDLROOT SOURCES AIDLFLAGS) "${GEN_DIR}/include" ) - if (${LANG} MATCHES "ndk") + if (${LANG} STREQUAL "ndk") set(BINDER_LIB_NAME "libbinder_ndk_sdk") else() set(BINDER_LIB_NAME "libbinder_sdk") diff --git a/cc/cmake_main.txt b/cc/cmake_main.txt index e9177d6e2..eeabf53e1 100644 --- a/cc/cmake_main.txt +++ b/cc/cmake_main.txt @@ -6,16 +6,12 @@ enable_testing() set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(AddAidlLibrary) include(AppendCxxFlagsIfSupported) +include(FindThreads) if (NOT ANDROID_BUILD_TOP) set(ANDROID_BUILD_TOP "${CMAKE_CURRENT_SOURCE_DIR}") endif() -set(PREBUILTS_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prebuilts/host/linux-x86/bin") -if (NOT AIDL_BIN) - find_program(AIDL_BIN aidl REQUIRED HINTS "${PREBUILTS_BIN_DIR}") -endif() - <<cflagsList .M.Name "_CFLAGS" .M.Properties.Cflags .M.Properties.Unportable_flags .M.Properties.Cflags_ignored>> <<range .Pprop.SystemPackages ->> @@ -25,6 +21,7 @@ find_package(<<.>> REQUIRED) add_subdirectory("${ANDROID_BUILD_TOP}/<<.>>" "<<.>>/build" EXCLUDE_FROM_ALL) <<end>> add_compile_options(${<<.M.Name>>_CFLAGS}) +link_libraries(${CMAKE_THREAD_LIBS_INIT}) <<range $moduleDir, $value := .ModuleDirs ->> add_subdirectory(<<$moduleDir>>) <<end>> diff --git a/cc/cmake_snapshot.go b/cc/cmake_snapshot.go index b4d1268b6..b79776453 100644 --- a/cc/cmake_snapshot.go +++ b/cc/cmake_snapshot.go @@ -15,7 +15,6 @@ package cc import ( - "android/soong/android" "bytes" _ "embed" "fmt" @@ -25,6 +24,8 @@ import ( "strings" "text/template" + "android/soong/android" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -50,8 +51,10 @@ var cmakeExtAddAidlLibrary string var cmakeExtAppendFlags string var defaultUnportableFlags []string = []string{ + "-Wno-c99-designator", "-Wno-class-memaccess", "-Wno-exit-time-destructors", + "-Winconsistent-missing-override", "-Wno-inconsistent-missing-override", "-Wreorder-init-list", "-Wno-reorder-init-list", @@ -61,8 +64,18 @@ var defaultUnportableFlags []string = []string{ } var ignoredSystemLibs []string = []string{ + "crtbegin_dynamic", + "crtend_android", + "libc", "libc++", "libc++_static", + "libc_musl", + "libc_musl_crtbegin_so", + "libc_musl_crtbegin_static", + "libc_musl_crtend", + "libc_musl_crtend_so", + "libdl", + "libm", "prebuilt_libclang_rt.builtins", "prebuilt_libclang_rt.ubsan_minimal", } @@ -83,9 +96,18 @@ type LibraryMappingProperty struct { } type CmakeSnapshotProperties struct { - // Modules to add to the snapshot package. Their dependencies are pulled in automatically. + // TODO: remove Modules []string + // Host modules to add to the snapshot package. Their dependencies are pulled in automatically. + Modules_host []string + + // System modules to add to the snapshot package. Their dependencies are pulled in automatically. + Modules_system []string + + // Vendor modules to add to the snapshot package. Their dependencies are pulled in automatically. + Modules_vendor []string + // Host prebuilts to bundle with the snapshot. These are tools needed to build outside Android. Prebuilts []string @@ -141,11 +163,7 @@ func parseTemplate(templateContents string) *template.Template { return list.String() }, "toStrings": func(files android.Paths) []string { - strings := make([]string, len(files)) - for idx, file := range files { - strings[idx] = file.String() - } - return strings + return files.Strings() }, "concat5": func(list1 []string, list2 []string, list3 []string, list4 []string, list5 []string) []string { return append(append(append(append(list1, list2...), list3...), list4...), list5...) @@ -266,12 +284,20 @@ func executeTemplate(templ *template.Template, buffer *bytes.Buffer, data any) s } func (m *CmakeSnapshot) DepsMutator(ctx android.BottomUpMutatorContext) { - variations := []blueprint.Variation{ - {"os", "linux_glibc"}, - {"arch", "x86_64"}, + deviceVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() + deviceSystemVariations := append(deviceVariations, blueprint.Variation{"image", ""}) + deviceVendorVariations := append(deviceVariations, blueprint.Variation{"image", "vendor"}) + hostVariations := ctx.Config().BuildOSTarget.Variations() + + ctx.AddVariationDependencies(hostVariations, cmakeSnapshotModuleTag, m.Properties.Modules...) + ctx.AddVariationDependencies(hostVariations, cmakeSnapshotModuleTag, m.Properties.Modules_host...) + ctx.AddVariationDependencies(deviceSystemVariations, cmakeSnapshotModuleTag, m.Properties.Modules_system...) + ctx.AddVariationDependencies(deviceVendorVariations, cmakeSnapshotModuleTag, m.Properties.Modules_vendor...) + + if len(m.Properties.Prebuilts) > 0 { + prebuilts := append(m.Properties.Prebuilts, "libc++") + ctx.AddVariationDependencies(hostVariations, cmakeSnapshotPrebuiltTag, prebuilts...) } - ctx.AddVariationDependencies(variations, cmakeSnapshotModuleTag, m.Properties.Modules...) - ctx.AddVariationDependencies(variations, cmakeSnapshotPrebuiltTag, m.Properties.Prebuilts...) } func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -389,7 +415,8 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Merging CMakeLists.txt contents for every module directory var makefilesList android.Paths - for moduleDir, fragments := range moduleDirs { + for _, moduleDir := range android.SortedKeys(moduleDirs) { + fragments := moduleDirs[moduleDir] moduleCmakePath := android.PathForModuleGen(ctx, moduleDir, "CMakeLists.txt") makefilesList = append(makefilesList, moduleCmakePath) sort.Strings(fragments) @@ -429,8 +456,9 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Packaging all sources into the zip file if m.Properties.Include_sources { var sourcesList android.Paths - for _, file := range sourceFiles { - sourcesList = append(sourcesList, file) + for _, file := range android.SortedKeys(sourceFiles) { + path := sourceFiles[file] + sourcesList = append(sourcesList, path) } sourcesRspFile := android.PathForModuleObj(ctx, ctx.ModuleName()+"_sources.rsp") @@ -462,15 +490,8 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Finish generating the final zip file zipRule.Build(m.zipPath.String(), "archiving "+ctx.ModuleName()) -} -func (m *CmakeSnapshot) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{m.zipPath}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } + ctx.SetOutputFiles(android.Paths{m.zipPath}, "") } func (m *CmakeSnapshot) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/cc/compiler.go b/cc/compiler.go index d8446fb84..03f9899d8 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -539,12 +539,10 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.Global.CommonFlags = append(flags.Global.CommonFlags, "${config.ExternalCflags}") } - if tc.Bionic() { - if Bool(compiler.Properties.Rtti) { - flags.Local.CppFlags = append(flags.Local.CppFlags, "-frtti") - } else { - flags.Local.CppFlags = append(flags.Local.CppFlags, "-fno-rtti") - } + if Bool(compiler.Properties.Rtti) { + flags.Local.CppFlags = append(flags.Local.CppFlags, "-frtti") + } else { + flags.Local.CppFlags = append(flags.Local.CppFlags, "-fno-rtti") } flags.Global.AsFlags = append(flags.Global.AsFlags, "${config.CommonGlobalAsflags}") @@ -796,9 +794,6 @@ type RustBindgenClangProperties struct { // be added to the include path using -I Local_include_dirs []string `android:"arch_variant,variant_prepend"` - // list of Rust static libraries. - Static_rlibs []string `android:"arch_variant,variant_prepend"` - // list of static libraries that provide headers for this binding. Static_libs []string `android:"arch_variant,variant_prepend"` diff --git a/cc/config/riscv64_device.go b/cc/config/riscv64_device.go index 724676ae6..6a5293f4c 100644 --- a/cc/config/riscv64_device.go +++ b/cc/config/riscv64_device.go @@ -29,14 +29,10 @@ var ( // This is already the driver's Android default, but duplicated here (and // below) for ease of experimentation with additional extensions. "-march=rv64gcv_zba_zbb_zbs", - // TODO: move to driver (https://github.com/google/android-riscv64/issues/111) - "-mno-strict-align", // TODO: remove when qemu V works (https://gitlab.com/qemu-project/qemu/-/issues/1976) // (Note that we'll probably want to wait for berberis to be good enough // that most people don't care about qemu's V performance either!) "-mno-implicit-float", - // TODO: remove when clang default changed (https://github.com/google/android-riscv64/issues/124) - "-mllvm -jump-is-expensive=false", } riscv64ArchVariantCflags = map[string][]string{} diff --git a/cc/genrule.go b/cc/genrule.go index cabf7875b..fe3b127ea 100644 --- a/cc/genrule.go +++ b/cc/genrule.go @@ -79,6 +79,14 @@ var _ android.ImageInterface = (*GenruleExtraProperties)(nil) func (g *GenruleExtraProperties) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (g *GenruleExtraProperties) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return Bool(g.Vendor_available) || Bool(g.Odm_available) || ctx.SocSpecific() || ctx.DeviceSpecific() +} + +func (g *GenruleExtraProperties) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return Bool(g.Product_available) || ctx.ProductSpecific() +} + func (g *GenruleExtraProperties) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !(ctx.SocSpecific() || ctx.DeviceSpecific() || ctx.ProductSpecific()) } @@ -102,18 +110,7 @@ func (g *GenruleExtraProperties) RecoveryVariantNeeded(ctx android.BaseModuleCon } func (g *GenruleExtraProperties) ExtraImageVariations(ctx android.BaseModuleContext) []string { - var variants []string - vendorVariantRequired := Bool(g.Vendor_available) || Bool(g.Odm_available) || ctx.SocSpecific() || ctx.DeviceSpecific() - productVariantRequired := Bool(g.Product_available) || ctx.ProductSpecific() - - if vendorVariantRequired { - variants = append(variants, VendorVariation) - } - if productVariantRequired { - variants = append(variants, ProductVariation) - } - - return variants + return nil } func (g *GenruleExtraProperties) SetImageVariation(ctx android.BaseModuleContext, variation string) { diff --git a/cc/image.go b/cc/image.go index 48a9174e3..2e52ccc63 100644 --- a/cc/image.go +++ b/cc/image.go @@ -39,18 +39,10 @@ const ( ) const ( - // VendorVariation is the variant name used for /vendor code that does not - // compile against the VNDK. - VendorVariation = "vendor" - // VendorVariationPrefix is the variant prefix used for /vendor code that compiles // against the VNDK. VendorVariationPrefix = "vendor." - // ProductVariation is the variant name used for /product code that does not - // compile against the VNDK. - ProductVariation = "product" - // ProductVariationPrefix is the variant prefix used for /product code that compiles // against the VNDK. ProductVariationPrefix = "product." @@ -117,12 +109,12 @@ func (c *Module) HasNonSystemVariants() bool { // Returns true if the module is "product" variant. Usually these modules are installed in /product func (c *Module) InProduct() bool { - return c.Properties.ImageVariation == ProductVariation + return c.Properties.ImageVariation == android.ProductVariation } // Returns true if the module is "vendor" variant. Usually these modules are installed in /vendor func (c *Module) InVendor() bool { - return c.Properties.ImageVariation == VendorVariation + return c.Properties.ImageVariation == android.VendorVariation } // Returns true if the module is "vendor" or "product" variant. This replaces previous UseVndk usages @@ -207,6 +199,12 @@ type ImageMutatableModule interface { // SetCoreVariantNeeded sets whether the Core Variant is needed. SetCoreVariantNeeded(b bool) + + // SetProductVariantNeeded sets whether the Product Variant is needed. + SetProductVariantNeeded(b bool) + + // SetVendorVariantNeeded sets whether the Vendor Variant is needed. + SetVendorVariantNeeded(b bool) } var _ ImageMutatableModule = (*Module)(nil) @@ -267,6 +265,14 @@ func (m *Module) SetCoreVariantNeeded(b bool) { m.Properties.CoreVariantNeeded = b } +func (m *Module) SetProductVariantNeeded(b bool) { + m.Properties.ProductVariantNeeded = b +} + +func (m *Module) SetVendorVariantNeeded(b bool) { + m.Properties.VendorVariantNeeded = b +} + func (m *Module) SnapshotVersion(mctx android.BaseModuleContext) string { if snapshot, ok := m.linker.(SnapshotInterface); ok { return snapshot.Version() @@ -319,41 +325,34 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { } } + var vendorVariantNeeded bool = false + var productVariantNeeded bool = false var coreVariantNeeded bool = false var ramdiskVariantNeeded bool = false var vendorRamdiskVariantNeeded bool = false var recoveryVariantNeeded bool = false - var vendorVariants []string - var productVariants []string - - needVndkVersionVendorVariantForLlndk := false - if m.NeedsLlndkVariants() { // This is an LLNDK library. The implementation of the library will be on /system, // and vendor and product variants will be created with LLNDK stubs. // The LLNDK libraries need vendor variants even if there is no VNDK. coreVariantNeeded = true - vendorVariants = append(vendorVariants, "") - productVariants = append(productVariants, "") - // Generate vendor variants for boardVndkVersion only if the VNDK snapshot does not - // provide the LLNDK stub libraries. - if needVndkVersionVendorVariantForLlndk { - vendorVariants = append(vendorVariants, "") - } + vendorVariantNeeded = true + productVariantNeeded = true + } else if m.NeedsVendorPublicLibraryVariants() { // A vendor public library has the implementation on /vendor, with stub variants // for system and product. coreVariantNeeded = true - vendorVariants = append(vendorVariants, "") - productVariants = append(productVariants, "") + vendorVariantNeeded = true + productVariantNeeded = true } else if m.IsSnapshotPrebuilt() { // Make vendor variants only for the versions in BOARD_VNDK_VERSION and // PRODUCT_EXTRA_VNDK_VERSIONS. if m.InstallInRecovery() { recoveryVariantNeeded = true } else { - vendorVariants = append(vendorVariants, m.SnapshotVersion(mctx)) + m.AppendExtraVariant(VendorVariationPrefix + m.SnapshotVersion(mctx)) } } else if m.HasNonSystemVariants() { // This will be available to /system unless it is product_specific @@ -364,16 +363,16 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { // BOARD_VNDK_VERSION. The other modules are regarded as AOSP, or // PLATFORM_VNDK_VERSION. if m.HasVendorVariant() { - vendorVariants = append(vendorVariants, "") + vendorVariantNeeded = true } // product_available modules are available to /product. if m.HasProductVariant() { - productVariants = append(productVariants, "") + productVariantNeeded = true } } else if vendorSpecific && m.SdkVersion() == "" { // This will be available in /vendor (or /odm) only - vendorVariants = append(vendorVariants, "") + vendorVariantNeeded = true } else { // This is either in /system (or similar: /data), or is a // module built with the NDK. Modules built with the NDK @@ -384,7 +383,7 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { if coreVariantNeeded && productSpecific && m.SdkVersion() == "" { // The module has "product_specific: true" that does not create core variant. coreVariantNeeded = false - productVariants = append(productVariants, "") + productVariantNeeded = true } if m.RamdiskAvailable() { @@ -414,36 +413,32 @@ func MutateImage(mctx android.BaseModuleContext, m ImageMutatableModule) { coreVariantNeeded = false } - for _, variant := range android.FirstUniqueStrings(vendorVariants) { - if variant == "" { - m.AppendExtraVariant(VendorVariation) - } else { - m.AppendExtraVariant(VendorVariationPrefix + variant) - } - } - - for _, variant := range android.FirstUniqueStrings(productVariants) { - if variant == "" { - m.AppendExtraVariant(ProductVariation) - } else { - m.AppendExtraVariant(ProductVariationPrefix + variant) - } - } - m.SetRamdiskVariantNeeded(ramdiskVariantNeeded) m.SetVendorRamdiskVariantNeeded(vendorRamdiskVariantNeeded) m.SetRecoveryVariantNeeded(recoveryVariantNeeded) m.SetCoreVariantNeeded(coreVariantNeeded) + m.SetProductVariantNeeded(productVariantNeeded) + m.SetVendorVariantNeeded(vendorVariantNeeded) // Disable the module if no variants are needed. if !ramdiskVariantNeeded && !recoveryVariantNeeded && !coreVariantNeeded && + !productVariantNeeded && + !vendorVariantNeeded && len(m.ExtraVariants()) == 0 { m.Disable() } } +func (c *Module) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.VendorVariantNeeded +} + +func (c *Module) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return c.Properties.ProductVariantNeeded +} + func (c *Module) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return c.Properties.CoreVariantNeeded } @@ -537,15 +532,15 @@ func (c *Module) SetImageVariation(ctx android.BaseModuleContext, variant string } else if variant == android.RecoveryVariation { c.MakeAsPlatform() squashRecoverySrcs(c) - } else if strings.HasPrefix(variant, VendorVariation) { - c.Properties.ImageVariation = VendorVariation + } else if strings.HasPrefix(variant, android.VendorVariation) { + c.Properties.ImageVariation = android.VendorVariation if strings.HasPrefix(variant, VendorVariationPrefix) { c.Properties.VndkVersion = strings.TrimPrefix(variant, VendorVariationPrefix) } squashVendorSrcs(c) - } else if strings.HasPrefix(variant, ProductVariation) { - c.Properties.ImageVariation = ProductVariation + } else if strings.HasPrefix(variant, android.ProductVariation) { + c.Properties.ImageVariation = android.ProductVariation if strings.HasPrefix(variant, ProductVariationPrefix) { c.Properties.VndkVersion = strings.TrimPrefix(variant, ProductVariationPrefix) } diff --git a/cc/library.go b/cc/library.go index e49f50cc0..4373b46b4 100644 --- a/cc/library.go +++ b/cc/library.go @@ -1135,8 +1135,12 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, linkerDeps = append(linkerDeps, deps.SharedLibsDeps...) linkerDeps = append(linkerDeps, deps.LateSharedLibsDeps...) - if generatedLib := generateRustStaticlib(ctx, deps.RustRlibDeps); generatedLib != nil { - deps.StaticLibs = append(deps.StaticLibs, generatedLib) + if generatedLib := generateRustStaticlib(ctx, deps.RustRlibDeps); generatedLib != nil && !library.buildStubs() { + if ctx.Module().(*Module).WholeRustStaticlib { + deps.WholeStaticLibs = append(deps.WholeStaticLibs, generatedLib) + } else { + deps.StaticLibs = append(deps.StaticLibs, generatedLib) + } } transformObjToDynamicBinary(ctx, objs.objFiles, sharedLibs, @@ -2149,7 +2153,6 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { modules := mctx.CreateLocalVariations(variations...) static := modules[0].(LinkableInterface) shared := modules[1].(LinkableInterface) - static.SetStatic() shared.SetShared() @@ -2173,6 +2176,12 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { mctx.CreateLocalVariations(variations...) mctx.AliasVariation(variations[0]) } + if library.BuildRlibVariant() && library.IsRustFFI() && !buildStatic { + // Rust modules do not build static libs, but rlibs are used as if they + // were via `static_libs`. Thus we need to alias the BuildRlibVariant + // to "static" for Rust FFI libraries. + mctx.CreateAliasVariation("static", "") + } } } diff --git a/cc/library_stub.go b/cc/library_stub.go index 9643ec2a1..6f06333ac 100644 --- a/cc/library_stub.go +++ b/cc/library_stub.go @@ -494,6 +494,12 @@ func BuildApiVariantName(baseName string, variant string, version string) string // Implement ImageInterface to generate image variants func (v *CcApiVariant) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (v *CcApiVariant) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return String(v.properties.Variant) == "llndk" +} +func (v *CcApiVariant) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return String(v.properties.Variant) == "llndk" +} func (v *CcApiVariant) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return inList(String(v.properties.Variant), []string{"ndk", "apex"}) } @@ -501,15 +507,6 @@ func (v *CcApiVariant) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool func (v *CcApiVariant) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { return false } func (v *CcApiVariant) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { return false } func (v *CcApiVariant) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { return false } -func (v *CcApiVariant) ExtraImageVariations(ctx android.BaseModuleContext) []string { - var variations []string - - if String(v.properties.Variant) == "llndk" { - variations = append(variations, VendorVariation) - variations = append(variations, ProductVariation) - } - - return variations -} +func (v *CcApiVariant) ExtraImageVariations(ctx android.BaseModuleContext) []string { return nil } func (v *CcApiVariant) SetImageVariation(ctx android.BaseModuleContext, variation string) { } diff --git a/cc/linkable.go b/cc/linkable.go index 2309fe800..1672366a7 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -94,12 +94,16 @@ type LinkableInterface interface { SelectedStl() string BuildStaticVariant() bool + BuildRlibVariant() bool BuildSharedVariant() bool SetStatic() SetShared() IsPrebuilt() bool Toc() android.OptionalPath + // IsRustFFI returns true if this is a Rust FFI library. + IsRustFFI() bool + // IsFuzzModule returns true if this a *_fuzz module. IsFuzzModule() bool diff --git a/cc/linker.go b/cc/linker.go index 1675df698..d2974c204 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -39,9 +39,6 @@ type BaseLinkerProperties struct { // the dependency's .a file will be linked into this module using -Wl,--whole-archive. Whole_static_libs []string `android:"arch_variant,variant_prepend"` - // list of Rust libs that should be statically linked into this module. - Static_rlibs []string `android:"arch_variant"` - // list of modules that should be statically linked into this module. Static_libs []string `android:"arch_variant,variant_prepend"` @@ -127,10 +124,6 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Header_libs []string - // list of Rust libs that should be statically linked to build vendor or product - // variant. - Static_rlibs []string - // list of shared libs that should not be used to build vendor or // product variant of the C/C++ module. Exclude_shared_libs []string @@ -159,10 +152,6 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Static_libs []string - // list of Rust libs that should be statically linked to build the recovery - // variant. - Static_rlibs []string - // list of shared libs that should not be used to build // the recovery variant of the C/C++ module. Exclude_shared_libs []string @@ -184,10 +173,6 @@ type BaseLinkerProperties struct { // variant of the C/C++ module. Static_libs []string - // list of Rust libs that should be statically linked to build the ramdisk - // variant. - Static_rlibs []string - // list of shared libs that should not be used to build // the ramdisk variant of the C/C++ module. Exclude_shared_libs []string @@ -205,10 +190,6 @@ type BaseLinkerProperties struct { // the vendor ramdisk variant of the C/C++ module. Exclude_shared_libs []string - // list of Rust libs that should be statically linked to build the vendor ramdisk - // variant. - Static_rlibs []string - // list of static libs that should not be used to build // the vendor ramdisk variant of the C/C++ module. Exclude_static_libs []string @@ -224,10 +205,6 @@ type BaseLinkerProperties struct { // variants. Shared_libs []string - // list of Rust libs that should be statically linked to build the vendor ramdisk - // variant. - Static_rlibs []string - // list of ehader libs that only should be used to build platform variant of // the C/C++ module. Header_libs []string @@ -322,7 +299,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.WholeStaticLibs = append(deps.WholeStaticLibs, linker.Properties.Whole_static_libs...) deps.HeaderLibs = append(deps.HeaderLibs, linker.Properties.Header_libs...) deps.StaticLibs = append(deps.StaticLibs, linker.Properties.Static_libs...) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Static_rlibs...) deps.SharedLibs = append(deps.SharedLibs, linker.Properties.Shared_libs...) deps.RuntimeLibs = append(deps.RuntimeLibs, linker.Properties.Runtime_libs...) @@ -366,7 +342,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Vendor.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Vendor.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Vendor.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Vendor.Static_rlibs...) } if ctx.inProduct() { @@ -380,7 +355,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Product.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Product.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Product.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Product.Static_rlibs...) } if ctx.inRecovery() { @@ -394,7 +368,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Recovery.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Recovery.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Recovery.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Recovery.Static_rlibs...) } if ctx.inRamdisk() { @@ -405,7 +378,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Ramdisk.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Ramdisk.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Ramdisk.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Ramdisk.Static_rlibs...) } if ctx.inVendorRamdisk() { @@ -415,7 +387,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.ReexportStaticLibHeaders = removeListFromList(deps.ReexportStaticLibHeaders, linker.Properties.Target.Vendor_ramdisk.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Target.Vendor_ramdisk.Exclude_static_libs) deps.RuntimeLibs = removeListFromList(deps.RuntimeLibs, linker.Properties.Target.Vendor_ramdisk.Exclude_runtime_libs) - deps.Rlibs = append(deps.Rlibs, linker.Properties.Target.Vendor_ramdisk.Static_rlibs...) } if !ctx.useSdk() { diff --git a/cc/object.go b/cc/object.go index 6c0391f3b..8b23295c7 100644 --- a/cc/object.go +++ b/cc/object.go @@ -220,7 +220,7 @@ func (object *objectLinker) unstrippedOutputFilePath() android.Path { } func (object *objectLinker) strippedAllOutputFilePath() android.Path { - panic("Not implemented.") + return nil } func (object *objectLinker) nativeCoverage() bool { diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 93351f1fc..201515f51 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -527,12 +527,12 @@ func OdexOnSystemOtherByName(name string, dexLocation string, global *GlobalConf return false } - if contains(global.SpeedApps, name) || contains(global.SystemServerApps, name) { + if contains(global.SystemServerApps, name) { return false } for _, f := range global.PatternsOnSystemOther { - if makefileMatch(filepath.Join(SystemPartition, f), dexLocation) { + if makefileMatch("/" + f, dexLocation) || makefileMatch(filepath.Join(SystemPartition, f), dexLocation) { return true } } diff --git a/dexpreopt/dexpreopt_test.go b/dexpreopt/dexpreopt_test.go index eff2416e5..6f7d3bb67 100644 --- a/dexpreopt/dexpreopt_test.go +++ b/dexpreopt/dexpreopt_test.go @@ -153,7 +153,7 @@ func TestDexPreoptSystemOther(t *testing.T) { moduleTests: []moduleTest{ {module: systemModule, expectedPartition: "system_other/system"}, {module: systemProductModule, expectedPartition: "system_other/system/product"}, - {module: productModule, expectedPartition: "product"}, + {module: productModule, expectedPartition: "system_other/product"}, }, }, } diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index 207548894..5a4818f4f 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -216,6 +216,14 @@ var _ android.ImageInterface = (*PrebuiltEtc)(nil) func (p *PrebuiltEtc) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (p *PrebuiltEtc) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + +func (p *PrebuiltEtc) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return false +} + func (p *PrebuiltEtc) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !p.ModuleBase.InstallInRecovery() && !p.ModuleBase.InstallInRamdisk() && !p.ModuleBase.InstallInVendorRamdisk() && !p.ModuleBase.InstallInDebugRamdisk() diff --git a/filesystem/aconfig_files.go b/filesystem/aconfig_files.go index 8daee850e..5c047bc83 100644 --- a/filesystem/aconfig_files.go +++ b/filesystem/aconfig_files.go @@ -16,7 +16,6 @@ package filesystem import ( "android/soong/android" - "path/filepath" "strings" "github.com/google/blueprint/proptools" @@ -56,6 +55,7 @@ func (f *filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder * sb.WriteString(" \\\n") sb.WriteString(sbCaches.String()) cmd.ImplicitOutput(installAconfigFlagsPath) + f.appendToEntry(ctx, installAconfigFlagsPath) installAconfigStorageDir := dir.Join(ctx, "etc", "aconfig") sb.WriteString("mkdir -p ") @@ -63,16 +63,18 @@ func (f *filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder * sb.WriteRune('\n') generatePartitionAconfigStorageFile := func(fileType, fileName string) { + outputPath := installAconfigStorageDir.Join(ctx, fileName) sb.WriteString(aconfigToolPath.String()) sb.WriteString(" create-storage --container ") sb.WriteString(f.PartitionType()) sb.WriteString(" --file ") sb.WriteString(fileType) sb.WriteString(" --out ") - sb.WriteString(filepath.Join(installAconfigStorageDir.String(), fileName)) + sb.WriteString(outputPath.String()) sb.WriteString(" \\\n") sb.WriteString(sbCaches.String()) - cmd.ImplicitOutput(installAconfigStorageDir.Join(ctx, fileName)) + cmd.ImplicitOutput(outputPath) + f.appendToEntry(ctx, outputPath) } generatePartitionAconfigStorageFile("package_map", "package.map") generatePartitionAconfigStorageFile("flag_map", "flag.map") diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index c889dd61c..ffa30a013 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -60,7 +60,9 @@ type filesystem struct { output android.OutputPath installDir android.InstallPath - // For testing. Keeps the result of CopySpecsToDir() + fileListFile android.OutputPath + + // Keeps the entries installed from this filesystem entries []string } @@ -221,8 +223,26 @@ func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { f.installDir = android.PathForModuleInstall(ctx, "etc") ctx.InstallFile(f.installDir, f.installFileName(), f.output) - ctx.SetOutputFiles([]android.Path{f.output}, "") + + f.fileListFile = android.PathForModuleOut(ctx, "fileList").OutputPath + android.WriteFileRule(ctx, f.fileListFile, f.installedFilesList()) +} + +func (f *filesystem) appendToEntry(ctx android.ModuleContext, installedFile android.OutputPath) { + partitionBaseDir := android.PathForModuleOut(ctx, "root", f.partitionName()).String() + "/" + + relPath, inTargetPartition := strings.CutPrefix(installedFile.String(), partitionBaseDir) + if inTargetPartition { + f.entries = append(f.entries, relPath) + } +} + +func (f *filesystem) installedFilesList() string { + installedFilePaths := android.FirstUniqueStrings(f.entries) + slices.Sort(installedFilePaths) + + return strings.Join(installedFilePaths, "\n") } func validatePartitionType(ctx android.ModuleContext, p partition) { @@ -269,17 +289,19 @@ func (f *filesystem) buildNonDepsFiles(ctx android.ModuleContext, builder *andro builder.Command().Textf("(! [ -e %s -o -L %s ] || (echo \"%s already exists from an earlier stage of the build\" && exit 1))", dst, dst, dst) builder.Command().Text("mkdir -p").Text(filepath.Dir(dst.String())) builder.Command().Text("ln -sf").Text(proptools.ShellEscape(target)).Text(dst.String()) + f.appendToEntry(ctx, dst) } // create extra files if there's any if f.buildExtraFiles != nil { rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath extraFiles := f.buildExtraFiles(ctx, rootForExtraFiles) - for _, f := range extraFiles { - rel, err := filepath.Rel(rootForExtraFiles.String(), f.String()) + for _, extraFile := range extraFiles { + rel, err := filepath.Rel(rootForExtraFiles.String(), extraFile.String()) if err != nil || strings.HasPrefix(rel, "..") { - ctx.ModuleErrorf("can't make %q relative to %q", f, rootForExtraFiles) + ctx.ModuleErrorf("can't make %q relative to %q", extraFile, rootForExtraFiles) } + f.appendToEntry(ctx, rootDir.Join(ctx, rel)) } if len(extraFiles) > 0 { builder.Command().BuiltTool("merge_directories"). @@ -535,6 +557,8 @@ func (f *filesystem) buildEventLogtagsFile(ctx android.ModuleContext, builder *a for _, path := range android.SortedKeys(logtagsFilePaths) { cmd.Text(path) } + + f.appendToEntry(ctx, eventLogtagsPath) } type partition interface { @@ -558,6 +582,7 @@ func (f *filesystem) AndroidMkEntries() []android.AndroidMkEntries { func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.SetString("LOCAL_MODULE_PATH", f.installDir.String()) entries.SetString("LOCAL_INSTALLED_MODULE_STEM", f.installFileName()) + entries.SetString("LOCAL_FILESYSTEM_FILELIST", f.fileListFile.String()) }, }, }} diff --git a/filesystem/fsverity_metadata.go b/filesystem/fsverity_metadata.go index 3e50ff752..d7bb654b9 100644 --- a/filesystem/fsverity_metadata.go +++ b/filesystem/fsverity_metadata.go @@ -87,6 +87,7 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build sb.WriteRune(' ') sb.WriteString(srcPath.String()) sb.WriteRune('\n') + f.appendToEntry(ctx, destPath) } // STEP 2: generate signed BuildManifest.apk @@ -108,6 +109,7 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build sb.WriteString(" --output ") sb.WriteString(manifestPbPath.String()) sb.WriteRune(' ') + f.appendToEntry(ctx, manifestPbPath) manifestGeneratorListPath := android.PathForModuleOut(ctx, "fsverity_manifest.list") f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath.OutputPath, matchedSpecs, rebasedDir) @@ -115,15 +117,18 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build sb.WriteString(manifestGeneratorListPath.String()) sb.WriteRune('\n') cmd.Implicit(manifestGeneratorListPath) + f.appendToEntry(ctx, manifestGeneratorListPath.OutputPath) // STEP 2-2: generate BuildManifest.apk (unsigned) aapt2Path := ctx.Config().HostToolPath(ctx, "aapt2") apkPath := rebasedDir.Join(ctx, "etc", "security", "fsverity", "BuildManifest.apk") + idsigPath := rebasedDir.Join(ctx, "etc", "security", "fsverity", "BuildManifest.apk.idsig") manifestTemplatePath := android.PathForSource(ctx, "system/security/fsverity/AndroidManifest.xml") libs := android.PathsForModuleSrc(ctx, f.properties.Fsverity.Libs) cmd.Implicit(aapt2Path) cmd.Implicit(manifestTemplatePath) cmd.Implicits(libs) + cmd.ImplicitOutput(apkPath) sb.WriteString(aapt2Path.String()) sb.WriteString(" link -o ") @@ -150,12 +155,15 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build sb.WriteString(f.partitionName()) sb.WriteRune('\n') + f.appendToEntry(ctx, apkPath) + // STEP 2-3: sign BuildManifest.apk apksignerPath := ctx.Config().HostToolPath(ctx, "apksigner") pemPath, keyPath := ctx.Config().DefaultAppCertificate(ctx) cmd.Implicit(apksignerPath) cmd.Implicit(pemPath) cmd.Implicit(keyPath) + cmd.ImplicitOutput(idsigPath) sb.WriteString(apksignerPath.String()) sb.WriteString(" sign --in ") sb.WriteString(apkPath.String()) @@ -165,5 +173,7 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build sb.WriteString(keyPath.String()) sb.WriteRune('\n') + f.appendToEntry(ctx, idsigPath) + android.WriteExecutableFileRuleVerbatim(ctx, fsverityBuilderPath, sb.String()) } diff --git a/genrule/genrule.go b/genrule/genrule.go index b23530369..c0942d3f1 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -643,6 +643,8 @@ func generatorFactory(taskGenerator taskFunc, props ...interface{}) *Module { type noopImageInterface struct{} func (x noopImageInterface) ImageMutatorBegin(android.BaseModuleContext) {} +func (x noopImageInterface) VendorVariantNeeded(android.BaseModuleContext) bool { return false } +func (x noopImageInterface) ProductVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) CoreVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) RamdiskVariantNeeded(android.BaseModuleContext) bool { return false } func (x noopImageInterface) VendorRamdiskVariantNeeded(android.BaseModuleContext) bool { return false } @@ -5,6 +5,5 @@ go 1.22 require ( github.com/google/blueprint v0.0.0 google.golang.org/protobuf v0.0.0 - prebuilts/bazel/common/proto/analysis_v2 v0.0.0 go.starlark.net v0.0.0 ) @@ -5,8 +5,6 @@ use ( ../../external/go-cmp ../../external/golang-protobuf ../../external/starlark-go - ../../prebuilts/bazel/common/proto/analysis_v2 - ../../prebuilts/bazel/common/proto/build ../blueprint ) @@ -15,7 +13,5 @@ replace ( github.com/google/blueprint v0.0.0 => ../blueprint github.com/google/go-cmp v0.0.0 => ../../external/go-cmp google.golang.org/protobuf v0.0.0 => ../../external/golang-protobuf - prebuilts/bazel/common/proto/analysis_v2 v0.0.0 => ../../prebuilts/bazel/common/proto/analysis_v2 - prebuilts/bazel/common/proto/build v0.0.0 => ../../prebuilts/bazel/common/proto/build go.starlark.net v0.0.0 => ../../external/starlark-go ) diff --git a/java/android_manifest.go b/java/android_manifest.go index 859900376..0c77968e6 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -71,12 +71,15 @@ func shouldReturnFinalOrFutureInt(ctx android.ModuleContext, targetSdkVersionLev return targetSdkVersionLevel.IsPreview() && (ctx.Config().UnbundledBuildApps() || includedInMts(ctx.Module())) } -// Helper function that casts android.Module to java.androidTestApp -// If this type conversion is possible, it queries whether the test app is included in an MTS suite +// Helper function that returns true if android_test, android_test_helper_app, java_test are in an MTS suite. func includedInMts(module android.Module) bool { if test, ok := module.(androidTestApp); ok { return test.includedInTestSuite("mts") } + // java_test + if test, ok := module.(*Test); ok { + return android.PrefixInList(test.testProperties.Test_suites, "mts") + } return false } diff --git a/java/app.go b/java/app.go index 739ef1a50..f35e4c3d4 100644 --- a/java/app.go +++ b/java/app.go @@ -345,7 +345,35 @@ func (a *AndroidApp) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato } } +// TODO(b/156476221): Remove this allowlist +var ( + missingMinSdkVersionMtsAllowlist = []string{ + "CellBroadcastReceiverGoogleUnitTests", + "CellBroadcastReceiverUnitTests", + "CtsBatterySavingTestCases", + "CtsDeviceAndProfileOwnerApp23", + "CtsDeviceAndProfileOwnerApp30", + "CtsIntentSenderApp", + "CtsJobSchedulerTestCases", + "CtsMimeMapTestCases", + "CtsTareTestCases", + "LibStatsPullTests", + "MediaProviderClientTests", + "TeleServiceTests", + "TestExternalImsServiceApp", + "TestSmsRetrieverApp", + "TetheringPrivilegedTests", + } +) + +func checkMinSdkVersionMts(ctx android.ModuleContext, minSdkVersion android.ApiLevel) { + if includedInMts(ctx.Module()) && !minSdkVersion.Specified() && !android.InList(ctx.ModuleName(), missingMinSdkVersionMtsAllowlist) { + ctx.PropertyErrorf("min_sdk_version", "min_sdk_version is a required property for tests included in MTS") + } +} + func (a *AndroidTestHelperApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { + checkMinSdkVersionMts(ctx, a.MinSdkVersion(ctx)) applicationId := a.appTestHelperAppProperties.Manifest_values.ApplicationId if applicationId != nil { if a.overridableAppProperties.Package_name != nil { @@ -1366,6 +1394,7 @@ func (a *AndroidTestHelperApp) includedInTestSuite(searchPrefix string) bool { } func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { + checkMinSdkVersionMts(ctx, a.MinSdkVersion(ctx)) var configs []tradefed.Config if a.appTestProperties.Instrumentation_target_package != nil { a.additionalAaptFlags = append(a.additionalAaptFlags, diff --git a/java/app_test.go b/java/app_test.go index 1a862fa54..9e2d19ee8 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -4146,6 +4146,7 @@ func TestTargetSdkVersionMtsTests(t *testing.T) { bpTemplate := ` %v { name: "mytest", + min_sdk_version: "34", target_sdk_version: "%v", test_suites: ["othersuite", "%v"], } diff --git a/java/dex.go b/java/dex.go index 32546d985..c75e7749b 100644 --- a/java/dex.go +++ b/java/dex.go @@ -180,7 +180,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", "$r8Template": &remoteexec.REParams{ Labels: map[string]string{"type": "compile", "compiler": "r8"}, Inputs: []string{"$implicits", "${config.R8Jar}"}, - OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}", "${resourcesOutput}"}, + OutputFiles: []string{"${outUsage}", "${outConfig}", "${outDict}", "${resourcesOutput}", "${outR8ArtProfile}"}, ExecStrategy: "${config.RER8ExecStrategy}", ToolchainInputs: []string{"${config.JavaCmd}"}, Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, @@ -200,7 +200,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, }, }, []string{"outDir", "outDict", "outConfig", "outUsage", "outUsageZip", "outUsageDir", - "r8Flags", "zipFlags", "mergeZipsFlags", "resourcesOutput"}, []string{"implicits"}) + "r8Flags", "zipFlags", "mergeZipsFlags", "resourcesOutput", "outR8ArtProfile"}, []string{"implicits"}) func (d *dexer) dexCommonFlags(ctx android.ModuleContext, dexParams *compileDexParams) (flags []string, deps android.Paths) { @@ -463,13 +463,6 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam proguardConfiguration, } r8Flags, r8Deps, r8ArtProfileOutputPath := d.r8Flags(ctx, dexParams) - if r8ArtProfileOutputPath != nil { - artProfileOutputPath = r8ArtProfileOutputPath - implicitOutputs = append( - implicitOutputs, - artProfileOutputPath, - ) - } rule := r8 args := map[string]string{ "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), @@ -482,6 +475,17 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam "outDir": outDir.String(), "mergeZipsFlags": mergeZipsFlags, } + if r8ArtProfileOutputPath != nil { + artProfileOutputPath = r8ArtProfileOutputPath + implicitOutputs = append( + implicitOutputs, + artProfileOutputPath, + ) + // Add the implicit r8 Art profile output to args so that r8RE knows + // about this implicit output + args["outR8ArtProfile"] = artProfileOutputPath.String() + } + if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_R8") { rule = r8RE args["implicits"] = strings.Join(r8Deps.Strings(), ",") diff --git a/java/droiddoc.go b/java/droiddoc.go index 176779eb4..730f23696 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -223,17 +223,6 @@ type Javadoc struct { exportableStubsSrcJar android.WritablePath } -func (j *Javadoc) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{j.stubsSrcJar}, nil - case ".docs.zip": - return android.Paths{j.docZip}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - // javadoc converts .java source files to documentation using javadoc. func JavadocFactory() android.Module { module := &Javadoc{} @@ -254,8 +243,6 @@ func JavadocHostFactory() android.Module { return module } -var _ android.OutputFileProducer = (*Javadoc)(nil) - func (j *Javadoc) SdkVersion(ctx android.EarlyModuleContext) android.SdkSpec { return android.SdkSpecFrom(ctx, String(j.properties.Sdk_version)) } @@ -585,6 +572,9 @@ func (j *Javadoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { zipSyncCleanupCmd(rule, srcJarDir) rule.Build("javadoc", "javadoc") + + ctx.SetOutputFiles(android.Paths{j.stubsSrcJar}, "") + ctx.SetOutputFiles(android.Paths{j.docZip}, ".docs.zip") } // Droiddoc @@ -616,15 +606,6 @@ func DroiddocHostFactory() android.Module { return module } -func (d *Droiddoc) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "", ".docs.zip": - return android.Paths{d.Javadoc.docZip}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (d *Droiddoc) DepsMutator(ctx android.BottomUpMutatorContext) { d.Javadoc.addDeps(ctx) @@ -876,6 +857,9 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { zipSyncCleanupCmd(rule, srcJarDir) rule.Build("javadoc", desc) + + ctx.SetOutputFiles(android.Paths{d.Javadoc.docZip}, "") + ctx.SetOutputFiles(android.Paths{d.Javadoc.docZip}, ".docs.zip") } // Exported Droiddoc Directory diff --git a/java/droidstubs.go b/java/droidstubs.go index b32b754d3..01571858c 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -1453,17 +1453,6 @@ type PrebuiltStubsSources struct { stubsSrcJar android.Path } -func (p *PrebuiltStubsSources) OutputFiles(tag string) (android.Paths, error) { - switch tag { - // prebuilt droidstubs does not output "exportable" stubs. - // Output the "everything" stubs srcjar file if the tag is ".exportable". - case "", ".exportable": - return android.Paths{p.stubsSrcJar}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (d *PrebuiltStubsSources) StubsSrcJar(_ StubsType) (android.Path, error) { return d.stubsSrcJar, nil } @@ -1502,6 +1491,11 @@ func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleCon rule.Build("zip src", "Create srcjar from prebuilt source") p.stubsSrcJar = outPath } + + ctx.SetOutputFiles(android.Paths{p.stubsSrcJar}, "") + // prebuilt droidstubs does not output "exportable" stubs. + // Output the "everything" stubs srcjar file if the tag is ".exportable". + ctx.SetOutputFiles(android.Paths{p.stubsSrcJar}, ".exportable") } func (p *PrebuiltStubsSources) Prebuilt() *android.Prebuilt { diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index cab5402e9..4144de82b 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -1255,8 +1255,9 @@ func buildRuleToGenerateRemovedDexSignatures(ctx android.ModuleContext, suffix s rule := android.NewRuleBuilder(pctx, ctx) rule.Command(). BuiltTool("metalava"). + Text("signature-to-dex"). Inputs(removedTxtFiles). - FlagWithOutput("--dex-api ", output) + FlagWithOutput("--out ", output) rule.Build("modular-hiddenapi-removed-dex-signatures"+suffix, "modular hiddenapi removed dex signatures"+suffix) return android.OptionalPathForPath(output) } diff --git a/java/java.go b/java/java.go index 6fee7ce9a..a2fc5fbd1 100644 --- a/java/java.go +++ b/java/java.go @@ -1510,6 +1510,7 @@ func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { + checkMinSdkVersionMts(ctx, j.MinSdkVersion(ctx)) j.generateAndroidBuildActionsWithConfig(ctx, nil) android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 8d4cf6823..38553a61b 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -15,8 +15,6 @@ package java import ( - "fmt" - "android/soong/android" "android/soong/dexpreopt" ) @@ -57,9 +55,6 @@ type platformBootclasspathModule struct { // Path to the monolithic hiddenapi-unsupported.csv file. hiddenAPIMetadataCSV android.OutputPath - - // Path to a srcjar containing all the transitive sources of the bootclasspath. - srcjar android.OutputPath } type platformBootclasspathProperties struct { @@ -76,8 +71,6 @@ func platformBootclasspathFactory() android.SingletonModule { return m } -var _ android.OutputFileProducer = (*platformBootclasspathModule)(nil) - func (b *platformBootclasspathModule) AndroidMkEntries() (entries []android.AndroidMkEntries) { entries = append(entries, android.AndroidMkEntries{ Class: "FAKE", @@ -89,22 +82,6 @@ func (b *platformBootclasspathModule) AndroidMkEntries() (entries []android.Andr return } -// Make the hidden API files available from the platform-bootclasspath module. -func (b *platformBootclasspathModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "hiddenapi-flags.csv": - return android.Paths{b.hiddenAPIFlagsCSV}, nil - case "hiddenapi-index.csv": - return android.Paths{b.hiddenAPIIndexCSV}, nil - case "hiddenapi-metadata.csv": - return android.Paths{b.hiddenAPIMetadataCSV}, nil - case ".srcjar": - return android.Paths{b.srcjar}, nil - } - - return nil, fmt.Errorf("unknown tag %s", tag) -} - func (b *platformBootclasspathModule) DepsMutator(ctx android.BottomUpMutatorContext) { // Create a dependency on all_apex_contributions to determine the selected mainline module ctx.AddDependency(ctx.Module(), apexContributionsMetadataDepTag, "all_apex_contributions") @@ -198,8 +175,8 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo } jarArgs := resourcePathsToJarArgs(transitiveSrcFiles) jarArgs = append(jarArgs, "-srcjar") // Move srcfiles to the right package - b.srcjar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-transitive.srcjar").OutputPath - TransformResourcesToJar(ctx, b.srcjar, jarArgs, transitiveSrcFiles) + srcjar := android.PathForModuleOut(ctx, ctx.ModuleName()+"-transitive.srcjar").OutputPath + TransformResourcesToJar(ctx, srcjar, jarArgs, transitiveSrcFiles) // Gather all the fragments dependencies. b.fragments = gatherApexModulePairDepsWithTag(ctx, bootclasspathFragmentDepTag) @@ -213,6 +190,11 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo bootDexJarByModule := b.generateHiddenAPIBuildActions(ctx, b.configuredModules, b.fragments) buildRuleForBootJarsPackageCheck(ctx, bootDexJarByModule) + + ctx.SetOutputFiles(android.Paths{b.hiddenAPIFlagsCSV}, "hiddenapi-flags.csv") + ctx.SetOutputFiles(android.Paths{b.hiddenAPIIndexCSV}, "hiddenapi-index.csv") + ctx.SetOutputFiles(android.Paths{b.hiddenAPIMetadataCSV}, "hiddenapi-metadata.csv") + ctx.SetOutputFiles(android.Paths{srcjar}, ".srcjar") } // Generate classpaths.proto config diff --git a/java/platform_compat_config.go b/java/platform_compat_config.go index 45b994498..67ed84e1d 100644 --- a/java/platform_compat_config.go +++ b/java/platform_compat_config.go @@ -15,7 +15,6 @@ package java import ( - "fmt" "path/filepath" "android/soong/android" @@ -110,23 +109,13 @@ func (p *platformCompatConfig) GenerateAndroidBuildActions(ctx android.ModuleCon p.installDirPath = android.PathForModuleInstall(ctx, "etc", "compatconfig") p.installConfigFile = android.PathForModuleInstall(ctx, "etc", "compatconfig", p.configFile.Base()) rule.Build(configFileName, "Extract compat/compat_config.xml and install it") -} - -func (p *platformCompatConfig) FilesToInstall() android.InstallPaths { - return android.InstallPaths{p.installConfigFile} + ctx.InstallFile(p.installDirPath, p.configFile.Base(), p.configFile) } func (p *platformCompatConfig) AndroidMkEntries() []android.AndroidMkEntries { return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "ETC", OutputFile: android.OptionalPathForPath(p.configFile), - Include: "$(BUILD_PREBUILT)", - ExtraEntries: []android.AndroidMkExtraEntriesFunc{ - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.SetString("LOCAL_MODULE_PATH", p.installDirPath.String()) - entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.configFile.Base()) - }, - }, }} } @@ -290,32 +279,23 @@ type globalCompatConfig struct { android.ModuleBase properties globalCompatConfigProperties - - outputFilePath android.OutputPath } func (c *globalCompatConfig) GenerateAndroidBuildActions(ctx android.ModuleContext) { filename := String(c.properties.Filename) inputPath := platformCompatConfigPath(ctx) - c.outputFilePath = android.PathForModuleOut(ctx, filename).OutputPath + outputFilePath := android.PathForModuleOut(ctx, filename).OutputPath // This ensures that outputFilePath has the correct name for others to // use, as the source file may have a different name. ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, - Output: c.outputFilePath, + Output: outputFilePath, Input: inputPath, }) -} -func (h *globalCompatConfig) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{h.outputFilePath}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } + ctx.SetOutputFiles(android.Paths{outputFilePath}, "") } // global_compat_config provides access to the merged compat config xml file generated by the build. diff --git a/java/robolectric.go b/java/robolectric.go index 18386c90c..cb22fa0db 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -116,7 +116,7 @@ func (r *robolectricTest) DepsMutator(ctx android.BottomUpMutatorContext) { if v := String(r.robolectricProperties.Robolectric_prebuilt_version); v != "" { ctx.AddVariationDependencies(nil, libTag, fmt.Sprintf(robolectricPrebuiltLibPattern, v)) - } else if !proptools.Bool(r.robolectricProperties.Strict_mode) { + } else if !proptools.BoolDefault(r.robolectricProperties.Strict_mode, true) { if proptools.Bool(r.robolectricProperties.Upstream) { ctx.AddVariationDependencies(nil, libTag, robolectricCurrentLib+"_upstream") } else { @@ -124,8 +124,11 @@ func (r *robolectricTest) DepsMutator(ctx android.BottomUpMutatorContext) { } } - if proptools.Bool(r.robolectricProperties.Strict_mode) { + if proptools.BoolDefault(r.robolectricProperties.Strict_mode, true) { ctx.AddVariationDependencies(nil, roboRuntimeOnlyTag, robolectricCurrentLib+"_upstream") + } else { + // opting out from strict mode, robolectric_non_strict_mode_permission lib should be added + ctx.AddVariationDependencies(nil, libTag, "robolectric_non_strict_mode_permission") } ctx.AddVariationDependencies(nil, libTag, robolectricDefaultLibs...) diff --git a/java/sdk_library.go b/java/sdk_library.go index c19b07bb6..e6cb6c49b 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -118,9 +118,6 @@ type apiScope struct { // The tag to use to depend on the stubs source module (if separate from the API module). stubsSourceTag scopeDependencyTag - // The tag to use to depend on the API file generating module (if separate from the stubs source module). - apiFileTag scopeDependencyTag - // The tag to use to depend on the stubs source and API module. stubsSourceAndApiTag scopeDependencyTag @@ -195,11 +192,6 @@ func initApiScope(scope *apiScope) *apiScope { apiScope: scope, depInfoExtractor: (*scopePaths).extractStubsSourceInfoFromDep, } - scope.apiFileTag = scopeDependencyTag{ - name: name + "-api", - apiScope: scope, - depInfoExtractor: (*scopePaths).extractApiInfoFromDep, - } scope.stubsSourceAndApiTag = scopeDependencyTag{ name: name + "-stubs-source-and-api", apiScope: scope, @@ -804,12 +796,6 @@ func (paths *scopePaths) extractApiInfoFromApiStubsProvider(provider ApiStubsPro return combinedError } -func (paths *scopePaths) extractApiInfoFromDep(ctx android.ModuleContext, dep android.Module) error { - return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) error { - return paths.extractApiInfoFromApiStubsProvider(provider, Everything) - }) -} - func (paths *scopePaths) extractStubsSourceInfoFromApiStubsProviders(provider ApiStubsSrcProvider, stubsType StubsType) error { stubsSrcJar, err := provider.StubsSrcJar(stubsType) if err == nil { @@ -819,22 +805,23 @@ func (paths *scopePaths) extractStubsSourceInfoFromApiStubsProviders(provider Ap } func (paths *scopePaths) extractStubsSourceInfoFromDep(ctx android.ModuleContext, dep android.Module) error { + stubsType := Everything + if ctx.Config().ReleaseHiddenApiExportableStubs() { + stubsType = Exportable + } return paths.treatDepAsApiStubsSrcProvider(dep, func(provider ApiStubsSrcProvider) error { - return paths.extractStubsSourceInfoFromApiStubsProviders(provider, Everything) + return paths.extractStubsSourceInfoFromApiStubsProviders(provider, stubsType) }) } func (paths *scopePaths) extractStubsSourceAndApiInfoFromApiStubsProvider(ctx android.ModuleContext, dep android.Module) error { + stubsType := Everything if ctx.Config().ReleaseHiddenApiExportableStubs() { - return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) error { - extractApiInfoErr := paths.extractApiInfoFromApiStubsProvider(provider, Exportable) - extractStubsSourceInfoErr := paths.extractStubsSourceInfoFromApiStubsProviders(provider, Exportable) - return errors.Join(extractApiInfoErr, extractStubsSourceInfoErr) - }) + stubsType = Exportable } return paths.treatDepAsApiStubsProvider(dep, func(provider ApiStubsProvider) error { - extractApiInfoErr := paths.extractApiInfoFromApiStubsProvider(provider, Everything) - extractStubsSourceInfoErr := paths.extractStubsSourceInfoFromApiStubsProviders(provider, Everything) + extractApiInfoErr := paths.extractApiInfoFromApiStubsProvider(provider, stubsType) + extractStubsSourceInfoErr := paths.extractStubsSourceInfoFromApiStubsProviders(provider, stubsType) return errors.Join(extractApiInfoErr, extractStubsSourceInfoErr) }) } @@ -1679,6 +1666,7 @@ func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) module.dexer.proguardDictionary = module.implLibraryModule.dexer.proguardDictionary module.dexer.proguardUsageZip = module.implLibraryModule.dexer.proguardUsageZip module.linter.reports = module.implLibraryModule.linter.reports + module.linter.outputs.depSets = module.implLibraryModule.LintDepSets() if !module.Host() { module.hostdexInstallFile = module.implLibraryModule.hostdexInstallFile diff --git a/multitree/api_surface.go b/multitree/api_surface.go index f739a2430..0f605d84a 100644 --- a/multitree/api_surface.go +++ b/multitree/api_surface.go @@ -16,8 +16,6 @@ package multitree import ( "android/soong/android" - "fmt" - "github.com/google/blueprint" ) @@ -40,7 +38,6 @@ type ApiSurface struct { ExportableModuleBase properties apiSurfaceProperties - allOutputs android.Paths taggedOutputs map[string]android.Paths } @@ -86,15 +83,9 @@ func (surface *ApiSurface) GenerateAndroidBuildActions(ctx android.ModuleContext Inputs: allOutputs, }) - surface.allOutputs = allOutputs surface.taggedOutputs = contributionFiles -} -func (surface *ApiSurface) OutputFiles(tag string) (android.Paths, error) { - if tag != "" { - return nil, fmt.Errorf("unknown tag: %q", tag) - } - return surface.allOutputs, nil + ctx.SetOutputFiles(allOutputs, "") } func (surface *ApiSurface) TaggedOutputs() map[string]android.Paths { @@ -105,7 +96,6 @@ func (surface *ApiSurface) Exportable() bool { return true } -var _ android.OutputFileProducer = (*ApiSurface)(nil) var _ Exportable = (*ApiSurface)(nil) type ApiContribution interface { diff --git a/multitree/export.go b/multitree/export.go index aecade58d..8be8f7058 100644 --- a/multitree/export.go +++ b/multitree/export.go @@ -50,7 +50,6 @@ type Exportable interface { type ExportableModule interface { android.Module - android.OutputFileProducer Exportable } diff --git a/python/binary.go b/python/binary.go index b935aba45..5f60761be 100644 --- a/python/binary.go +++ b/python/binary.go @@ -103,6 +103,7 @@ func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleConte p.buildBinary(ctx) p.installedDest = ctx.InstallFile(installDir(ctx, "bin", "", ""), p.installSource.Base(), p.installSource) + ctx.SetOutputFiles(android.Paths{p.installSource}, "") } func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { @@ -187,16 +188,6 @@ func (p *PythonBinaryModule) HostToolPath() android.OptionalPath { return android.OptionalPathForPath(p.installedDest) } -// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. -func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{p.installSource}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool { return BoolDefault(p.properties.Embedded_launcher, true) } diff --git a/rust/binary.go b/rust/binary.go index 996951366..cba29a023 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -134,6 +134,9 @@ func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps Path ret := buildOutput{outputFile: outputFile} crateRootPath := crateRootPath(ctx, binary) + // Ensure link dirs are not duplicated + deps.linkDirs = android.FirstUniqueStrings(deps.linkDirs) + flags.RustFlags = append(flags.RustFlags, deps.depFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.linkObjects...) diff --git a/rust/builder_test.go b/rust/builder_test.go index c093ac4df..ae5ccde27 100644 --- a/rust/builder_test.go +++ b/rust/builder_test.go @@ -65,6 +65,11 @@ func TestCompilationOutputFiles(t *testing.T) { crate_name: "rust_ffi", srcs: ["lib.rs"], } + rust_ffi_static { + name: "librust_ffi_static", + crate_name: "rust_ffi", + srcs: ["lib.rs"], + } `) testcases := []struct { testName string @@ -118,14 +123,14 @@ func TestCompilationOutputFiles(t *testing.T) { }, }, { - testName: "rust_ffi static", - moduleName: "librust_ffi", - variant: "android_arm64_armv8-a_static", + testName: "rust_ffi_static rlib", + moduleName: "librust_ffi_static", + variant: "android_arm64_armv8-a_rlib_rlib-std", expectedFiles: []string{ - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/librust_ffi.a", - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/librust_ffi.a.clippy", - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/meta_lic", - "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_static/rustdoc.timestamp", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/librust_ffi_static.rlib", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/librust_ffi_static.rlib.clippy", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/meta_lic", + "out/soong/.intermediates/librust_ffi_static/android_arm64_armv8-a_rlib_rlib-std/rustdoc.timestamp", }, }, { @@ -148,6 +153,7 @@ func TestCompilationOutputFiles(t *testing.T) { "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/unstripped/librust_ffi.so", "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/unstripped/librust_ffi.so.toc", "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/meta_lic", + "out/soong/.intermediates/librust_ffi/android_arm64_armv8-a_shared/rustdoc.timestamp", "out/soong/target/product/test_device/system/lib64/librust_ffi.so", }, }, diff --git a/rust/fuzz_test.go b/rust/fuzz_test.go index 0d4622a7e..6cb8b9319 100644 --- a/rust/fuzz_test.go +++ b/rust/fuzz_test.go @@ -114,17 +114,23 @@ func TestCCFuzzDepBundling(t *testing.T) { srcs: ["foo.rs"], shared_libs: ["libcc_transitive_dep"], } + rust_ffi_static { + name: "libtest_fuzzing_static", + crate_name: "test_fuzzing", + srcs: ["foo.rs"], + shared_libs: ["libcc_transitive_dep"], + } cc_fuzz { name: "fuzz_shared_libtest", shared_libs: ["libtest_fuzzing"], } cc_fuzz { name: "fuzz_static_libtest", - static_rlibs: ["libtest_fuzzing"], + static_libs: ["libtest_fuzzing"], } cc_fuzz { name: "fuzz_staticffi_libtest", - static_libs: ["libtest_fuzzing"], + static_libs: ["libtest_fuzzing_static"], } `) diff --git a/rust/image.go b/rust/image.go index fec6d92d8..26929b1ac 100644 --- a/rust/image.go +++ b/rust/image.go @@ -77,6 +77,14 @@ func (mod *Module) SetCoreVariantNeeded(b bool) { mod.Properties.CoreVariantNeeded = b } +func (mod *Module) SetProductVariantNeeded(b bool) { + mod.Properties.ProductVariantNeeded = b +} + +func (mod *Module) SetVendorVariantNeeded(b bool) { + mod.Properties.VendorVariantNeeded = b +} + func (mod *Module) SnapshotVersion(mctx android.BaseModuleContext) string { if snapshot, ok := mod.compiler.(cc.SnapshotInterface); ok { return snapshot.Version() @@ -86,6 +94,14 @@ func (mod *Module) SnapshotVersion(mctx android.BaseModuleContext) string { } } +func (mod *Module) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return mod.Properties.VendorVariantNeeded +} + +func (mod *Module) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return mod.Properties.ProductVariantNeeded +} + func (mod *Module) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { return mod.Properties.VendorRamdiskVariantNeeded } @@ -184,12 +200,12 @@ func (mod *Module) HasNonSystemVariants() bool { } func (mod *Module) InProduct() bool { - return mod.Properties.ImageVariation == cc.ProductVariation + return mod.Properties.ImageVariation == android.ProductVariation } // Returns true if the module is "vendor" variant. Usually these modules are installed in /vendor func (mod *Module) InVendor() bool { - return mod.Properties.ImageVariation == cc.VendorVariation + return mod.Properties.ImageVariation == android.VendorVariation } // Returns true if the module is "vendor" or "product" variant. @@ -202,13 +218,13 @@ func (mod *Module) SetImageVariation(ctx android.BaseModuleContext, variant stri mod.MakeAsPlatform() } else if variant == android.RecoveryVariation { mod.MakeAsPlatform() - } else if strings.HasPrefix(variant, cc.VendorVariation) { - mod.Properties.ImageVariation = cc.VendorVariation + } else if strings.HasPrefix(variant, android.VendorVariation) { + mod.Properties.ImageVariation = android.VendorVariation if strings.HasPrefix(variant, cc.VendorVariationPrefix) { mod.Properties.VndkVersion = strings.TrimPrefix(variant, cc.VendorVariationPrefix) } - } else if strings.HasPrefix(variant, cc.ProductVariation) { - mod.Properties.ImageVariation = cc.ProductVariation + } else if strings.HasPrefix(variant, android.ProductVariation) { + mod.Properties.ImageVariation = android.ProductVariation if strings.HasPrefix(variant, cc.ProductVariationPrefix) { mod.Properties.VndkVersion = strings.TrimPrefix(variant, cc.ProductVariationPrefix) } diff --git a/rust/image_test.go b/rust/image_test.go index 71e271c89..d84eb10c5 100644 --- a/rust/image_test.go +++ b/rust/image_test.go @@ -22,18 +22,20 @@ import ( "android/soong/cc" ) -// Test that cc modules can link against vendor_available rust_ffi_rlib/rust_ffi_static libraries. +// Test that cc modules can depend on vendor_available rust_ffi_rlib/rust_ffi_static libraries. func TestVendorLinkage(t *testing.T) { ctx := testRust(t, ` cc_binary { name: "fizz_vendor_available", - static_libs: ["libfoo_vendor_static"], - static_rlibs: ["libfoo_vendor"], + static_libs: [ + "libfoo_vendor", + "libfoo_vendor_static" + ], vendor_available: true, } cc_binary { name: "fizz_soc_specific", - static_rlibs: ["libfoo_vendor"], + static_libs: ["libfoo_vendor"], soc_specific: true, } rust_ffi_rlib { @@ -52,8 +54,8 @@ func TestVendorLinkage(t *testing.T) { vendorBinary := ctx.ModuleForTests("fizz_vendor_available", "android_vendor_arm64_armv8-a").Module().(*cc.Module) - if !android.InList("libfoo_vendor_static.vendor", vendorBinary.Properties.AndroidMkStaticLibs) { - t.Errorf("vendorBinary should have a dependency on libfoo_vendor_static.vendor: %#v", vendorBinary.Properties.AndroidMkStaticLibs) + if android.InList("libfoo_vendor_static.vendor", vendorBinary.Properties.AndroidMkStaticLibs) { + t.Errorf("vendorBinary should not have a staticlib dependency on libfoo_vendor_static.vendor: %#v", vendorBinary.Properties.AndroidMkStaticLibs) } } @@ -110,8 +112,10 @@ func TestVendorRamdiskLinkage(t *testing.T) { ctx := testRust(t, ` cc_library_shared { name: "libcc_vendor_ramdisk", - static_rlibs: ["libfoo_vendor_ramdisk"], - static_libs: ["libfoo_static_vendor_ramdisk"], + static_libs: [ + "libfoo_vendor_ramdisk", + "libfoo_static_vendor_ramdisk" + ], system_shared_libs: [], vendor_ramdisk_available: true, } @@ -131,8 +135,8 @@ func TestVendorRamdiskLinkage(t *testing.T) { vendorRamdiskLibrary := ctx.ModuleForTests("libcc_vendor_ramdisk", "android_vendor_ramdisk_arm64_armv8-a_shared").Module().(*cc.Module) - if !android.InList("libfoo_static_vendor_ramdisk.vendor_ramdisk", vendorRamdiskLibrary.Properties.AndroidMkStaticLibs) { - t.Errorf("libcc_vendor_ramdisk should have a dependency on libfoo_static_vendor_ramdisk") + if android.InList("libfoo_static_vendor_ramdisk.vendor_ramdisk", vendorRamdiskLibrary.Properties.AndroidMkStaticLibs) { + t.Errorf("libcc_vendor_ramdisk should not have a dependency on the libfoo_static_vendor_ramdisk static library") } } diff --git a/rust/library.go b/rust/library.go index 2a21263bd..96c02c832 100644 --- a/rust/library.go +++ b/rust/library.go @@ -43,9 +43,9 @@ func init() { android.RegisterModuleType("rust_ffi_host_rlib", RustFFIRlibHostFactory) // TODO: Remove when all instances of rust_ffi_static have been switched to rust_ffi_rlib - // Alias rust_ffi_static to the combined rust_ffi_rlib factory - android.RegisterModuleType("rust_ffi_static", RustFFIStaticRlibFactory) - android.RegisterModuleType("rust_ffi_host_static", RustFFIStaticRlibHostFactory) + // Alias rust_ffi_static to the rust_ffi_rlib factory + android.RegisterModuleType("rust_ffi_static", RustFFIRlibFactory) + android.RegisterModuleType("rust_ffi_host_static", RustFFIRlibHostFactory) } type VariantLibraryProperties struct { @@ -353,7 +353,7 @@ func RustFFISharedHostFactory() android.Module { // type "rlib"). func RustFFIRlibHostFactory() android.Module { module, library := NewRustLibrary(android.HostSupported) - library.BuildOnlyRlibStatic() + library.BuildOnlyRlib() library.isFFI = true return module.Init() @@ -368,30 +368,12 @@ func RustFFIRlibFactory() android.Module { return module.Init() } -// rust_ffi_static produces a staticlib and an rlib variant -func RustFFIStaticRlibFactory() android.Module { - module, library := NewRustLibrary(android.HostAndDeviceSupported) - library.BuildOnlyRlibStatic() - - library.isFFI = true - return module.Init() -} - -// rust_ffi_static_host produces a staticlib and an rlib variant for the host -func RustFFIStaticRlibHostFactory() android.Module { - module, library := NewRustLibrary(android.HostSupported) - library.BuildOnlyRlibStatic() - - library.isFFI = true - return module.Init() -} - func (library *libraryDecorator) BuildOnlyFFI() { library.MutatedProperties.BuildDylib = false // we build rlibs for later static ffi linkage. library.MutatedProperties.BuildRlib = true library.MutatedProperties.BuildShared = true - library.MutatedProperties.BuildStatic = true + library.MutatedProperties.BuildStatic = false library.isFFI = true } @@ -417,14 +399,6 @@ func (library *libraryDecorator) BuildOnlyRlib() { library.MutatedProperties.BuildStatic = false } -func (library *libraryDecorator) BuildOnlyRlibStatic() { - library.MutatedProperties.BuildDylib = false - library.MutatedProperties.BuildRlib = true - library.MutatedProperties.BuildShared = false - library.MutatedProperties.BuildStatic = true - library.isFFI = true -} - func (library *libraryDecorator) BuildOnlyStatic() { library.MutatedProperties.BuildRlib = false library.MutatedProperties.BuildDylib = false @@ -766,10 +740,13 @@ func LibraryMutator(mctx android.BottomUpMutatorContext) { // The order of the variations (modules) matches the variant names provided. Iterate // through the new variation modules and set their mutated properties. + var emptyVariant = false + var rlibVariant = false for i, v := range modules { switch variants[i] { case rlibVariation: v.(*Module).compiler.(libraryInterface).setRlib() + rlibVariant = true case dylibVariation: v.(*Module).compiler.(libraryInterface).setDylib() if v.(*Module).ModuleBase.ImageVariation().Variation == android.VendorRamdiskVariation { @@ -784,11 +761,19 @@ func LibraryMutator(mctx android.BottomUpMutatorContext) { // Disable the compilation steps. v.(*Module).compiler.SetDisabled() case "": - // if there's an empty variant, alias it so it is the default variant - mctx.AliasVariation("") + emptyVariant = true } } + if rlibVariant && library.isFFILibrary() { + // If an rlib variant is set and this is an FFI library, make it the + // default variant so CC can link against it appropriately. + mctx.AliasVariation(rlibVariation) + } else if emptyVariant { + // If there's an empty variant, alias it so it is the default variant + mctx.AliasVariation("") + } + // If a source variant is created, add an inter-variant dependency // between the other variants and the source variant. if sourceVariant { @@ -817,6 +802,7 @@ func LibstdMutator(mctx android.BottomUpMutatorContext) { rlib := modules[0].(*Module) rlib.compiler.(libraryInterface).setRlibStd() rlib.Properties.RustSubName += RlibStdlibSuffix + mctx.AliasVariation("rlib-std") } else { variants := []string{"rlib-std", "dylib-std"} modules := mctx.CreateLocalVariations(variants...) diff --git a/rust/library_test.go b/rust/library_test.go index 1133c28ed..35a420cd5 100644 --- a/rust/library_test.go +++ b/rust/library_test.go @@ -34,10 +34,14 @@ func TestLibraryVariants(t *testing.T) { name: "libfoo.ffi", srcs: ["foo.rs"], crate_name: "foo" + } + rust_ffi_host_static { + name: "libfoo.ffi_static", + srcs: ["foo.rs"], + crate_name: "foo" }`) // Test all variants are being built. - libfooStatic := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_static").Rule("rustc") libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib_rlib-std").Rule("rustc") libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib").Rule("rustc") libfooFFIRlib := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_rlib_rlib-std").Rule("rustc") @@ -46,7 +50,6 @@ func TestLibraryVariants(t *testing.T) { rlibCrateType := "rlib" dylibCrateType := "dylib" sharedCrateType := "cdylib" - staticCrateType := "staticlib" // Test crate type for rlib is correct. if !strings.Contains(libfooRlib.Args["rustcFlags"], "crate-type="+rlibCrateType) { @@ -58,11 +61,6 @@ func TestLibraryVariants(t *testing.T) { t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", dylibCrateType, libfooDylib.Args["rustcFlags"]) } - // Test crate type for C static libraries is correct. - if !strings.Contains(libfooStatic.Args["rustcFlags"], "crate-type="+staticCrateType) { - t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", staticCrateType, libfooStatic.Args["rustcFlags"]) - } - // Test crate type for FFI rlibs is correct if !strings.Contains(libfooFFIRlib.Args["rustcFlags"], "crate-type="+rlibCrateType) { t.Errorf("missing crate-type for static variant, expecting %#v, rustcFlags: %#v", rlibCrateType, libfooFFIRlib.Args["rustcFlags"]) @@ -188,23 +186,18 @@ func TestSharedLibraryToc(t *testing.T) { func TestStaticLibraryLinkage(t *testing.T) { ctx := testRust(t, ` - rust_ffi { + rust_ffi_static { name: "libfoo", srcs: ["foo.rs"], crate_name: "foo", }`) libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_rlib_rlib-std") - libfooStatic := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_static") if !android.InList("libstd", libfoo.Module().(*Module).Properties.AndroidMkRlibs) { t.Errorf("Static libstd rlib expected to be a dependency of Rust rlib libraries. Rlib deps are: %#v", libfoo.Module().(*Module).Properties.AndroidMkDylibs) } - if !android.InList("libstd", libfooStatic.Module().(*Module).Properties.AndroidMkRlibs) { - t.Errorf("Static libstd rlib expected to be a dependency of Rust static libraries. Rlib deps are: %#v", - libfoo.Module().(*Module).Properties.AndroidMkDylibs) - } } func TestNativeDependencyOfRlib(t *testing.T) { @@ -215,39 +208,31 @@ func TestNativeDependencyOfRlib(t *testing.T) { rlibs: ["librust_rlib"], srcs: ["foo.rs"], } - rust_ffi_static { - name: "libffi_static", - crate_name: "ffi_static", - rlibs: ["librust_rlib"], - srcs: ["foo.rs"], - } rust_library_rlib { name: "librust_rlib", crate_name: "rust_rlib", srcs: ["foo.rs"], - shared_libs: ["shared_cc_dep"], - static_libs: ["static_cc_dep"], + shared_libs: ["libshared_cc_dep"], + static_libs: ["libstatic_cc_dep"], } cc_library_shared { - name: "shared_cc_dep", + name: "libshared_cc_dep", srcs: ["foo.cpp"], } cc_library_static { - name: "static_cc_dep", + name: "libstatic_cc_dep", srcs: ["foo.cpp"], } `) rustRlibRlibStd := ctx.ModuleForTests("librust_rlib", "android_arm64_armv8-a_rlib_rlib-std") rustRlibDylibStd := ctx.ModuleForTests("librust_rlib", "android_arm64_armv8-a_rlib_dylib-std") - ffiStatic := ctx.ModuleForTests("libffi_static", "android_arm64_armv8-a_static") ffiRlib := ctx.ModuleForTests("libffi_rlib", "android_arm64_armv8-a_rlib_rlib-std") modules := []android.TestingModule{ rustRlibRlibStd, rustRlibDylibStd, ffiRlib, - ffiStatic, } // librust_rlib specifies -L flag to cc deps output directory on rustc command @@ -258,17 +243,17 @@ func TestNativeDependencyOfRlib(t *testing.T) { // TODO: We could consider removing these flags for _, module := range modules { if !strings.Contains(module.Rule("rustc").Args["libFlags"], - "-L out/soong/.intermediates/shared_cc_dep/android_arm64_armv8-a_shared/") { + "-L out/soong/.intermediates/libshared_cc_dep/android_arm64_armv8-a_shared/") { t.Errorf( - "missing -L flag for shared_cc_dep, rustcFlags: %#v", - rustRlibRlibStd.Rule("rustc").Args["libFlags"], + "missing -L flag for libshared_cc_dep of %s, rustcFlags: %#v", + module.Module().Name(), rustRlibRlibStd.Rule("rustc").Args["libFlags"], ) } if !strings.Contains(module.Rule("rustc").Args["libFlags"], - "-L out/soong/.intermediates/static_cc_dep/android_arm64_armv8-a_static/") { + "-L out/soong/.intermediates/libstatic_cc_dep/android_arm64_armv8-a_static/") { t.Errorf( - "missing -L flag for static_cc_dep, rustcFlags: %#v", - rustRlibRlibStd.Rule("rustc").Args["libFlags"], + "missing -L flag for libstatic_cc_dep of %s, rustcFlags: %#v", + module.Module().Name(), rustRlibRlibStd.Rule("rustc").Args["libFlags"], ) } } @@ -305,15 +290,23 @@ func TestAutoDeps(t *testing.T) { "libbar", "librlib_only", ], + } + rust_ffi_host_static { + name: "libfoo.ffi.static", + srcs: ["foo.rs"], + crate_name: "foo", + rustlibs: [ + "libbar", + "librlib_only", + ], }`) libfooRlib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_rlib_rlib-std") libfooDylib := ctx.ModuleForTests("libfoo", "linux_glibc_x86_64_dylib") libfooFFIRlib := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_rlib_rlib-std") - libfooStatic := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_static") libfooShared := ctx.ModuleForTests("libfoo.ffi", "linux_glibc_x86_64_shared") - for _, static := range []android.TestingModule{libfooRlib, libfooStatic, libfooFFIRlib} { + for _, static := range []android.TestingModule{libfooRlib, libfooFFIRlib} { if !android.InList("libbar.rlib-std", static.Module().(*Module).Properties.AndroidMkRlibs) { t.Errorf("libbar not present as rlib dependency in static lib: %s", static.Module().Name()) } @@ -381,6 +374,12 @@ func TestLibstdLinkage(t *testing.T) { crate_name: "bar", rustlibs: ["libfoo"], } + rust_ffi_static { + name: "libbar_static", + srcs: ["foo.rs"], + crate_name: "bar", + rustlibs: ["libfoo"], + } rust_ffi { name: "libbar.prefer_rlib", srcs: ["foo.rs"], @@ -394,7 +393,6 @@ func TestLibstdLinkage(t *testing.T) { libfooRlibDynamic := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_rlib_dylib-std").Module().(*Module) libbarShared := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_shared").Module().(*Module) - libbarStatic := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_static").Module().(*Module) libbarFFIRlib := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_rlib_rlib-std").Module().(*Module) // prefer_rlib works the same for both rust_library and rust_ffi, so a single check is sufficient here. @@ -413,12 +411,6 @@ func TestLibstdLinkage(t *testing.T) { if !android.InList("libstd", libbarShared.Properties.AndroidMkDylibs) { t.Errorf("Device rust_ffi_shared does not link libstd as an dylib") } - if !android.InList("libstd", libbarStatic.Properties.AndroidMkRlibs) { - t.Errorf("Device rust_ffi_static does not link libstd as an rlib") - } - if !android.InList("libfoo.rlib-std", libbarStatic.Properties.AndroidMkRlibs) { - t.Errorf("Device rust_ffi_static does not link dependent rustlib rlib-std variant") - } if !android.InList("libstd", libbarFFIRlib.Properties.AndroidMkRlibs) { t.Errorf("Device rust_ffi_rlib does not link libstd as an rlib") } diff --git a/rust/rust.go b/rust/rust.go index 7cd9df4c7..6b7b2fbae 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -16,6 +16,7 @@ package rust import ( "fmt" + "strconv" "strings" "android/soong/bloaty" @@ -80,6 +81,8 @@ type BaseProperties struct { RustSubName string `blueprint:"mutated"` // Set by imageMutator + ProductVariantNeeded bool `blueprint:"mutated"` + VendorVariantNeeded bool `blueprint:"mutated"` CoreVariantNeeded bool `blueprint:"mutated"` VendorRamdiskVariantNeeded bool `blueprint:"mutated"` RamdiskVariantNeeded bool `blueprint:"mutated"` @@ -206,27 +209,6 @@ func (mod *Module) IsPrebuilt() bool { return false } -func (mod *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if mod.sourceProvider != nil && (mod.compiler == nil || mod.compiler.Disabled()) { - return mod.sourceProvider.Srcs(), nil - } else { - if mod.OutputFile().Valid() { - return android.Paths{mod.OutputFile().Path()}, nil - } - return android.Paths{}, nil - } - case "unstripped": - if mod.compiler != nil { - return android.PathsIfNonNil(mod.compiler.unstrippedOutputFilePath()), nil - } - return nil, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (mod *Module) SelectedStl() string { return "" } @@ -438,7 +420,7 @@ type PathDeps struct { depFlags []string depLinkFlags []string - // linkDirs are link paths passed via -L to rustc. linkObjects are objects passed directly to the linker. + // linkDirs are link paths passed via -L to rustc. linkObjects are objects passed directly to the linker // Both of these are exported and propagate to dependencies. linkDirs []string linkObjects []string @@ -460,9 +442,6 @@ type PathDeps struct { // Paths to generated source files SrcDeps android.Paths srcProviderFiles android.Paths - - // Used by Generated Libraries - depExportedRlibs []cc.RustRlibDep } type RustLibraries []RustLibrary @@ -483,6 +462,7 @@ type xref interface { type flagExporter struct { linkDirs []string + ccLinkDirs []string linkObjects []string } @@ -676,6 +656,24 @@ func (mod *Module) BuildStaticVariant() bool { panic(fmt.Errorf("BuildStaticVariant called on non-library module: %q", mod.BaseModuleName())) } +func (mod *Module) BuildRlibVariant() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.buildRlib() + } + } + panic(fmt.Errorf("BuildRlibVariant called on non-library module: %q", mod.BaseModuleName())) +} + +func (mod *Module) IsRustFFI() bool { + if mod.compiler != nil { + if library, ok := mod.compiler.(libraryInterface); ok { + return library.isFFILibrary() + } + } + return false +} + func (mod *Module) BuildSharedVariant() bool { if mod.compiler != nil { if library, ok := mod.compiler.(libraryInterface); ok { @@ -986,6 +984,61 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if mod.testModule { android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } + + mod.setOutputFiles(ctx) + + buildComplianceMetadataInfo(ctx, mod, deps) +} + +func (mod *Module) setOutputFiles(ctx ModuleContext) { + if mod.sourceProvider != nil && (mod.compiler == nil || mod.compiler.Disabled()) { + ctx.SetOutputFiles(mod.sourceProvider.Srcs(), "") + } else if mod.OutputFile().Valid() { + ctx.SetOutputFiles(android.Paths{mod.OutputFile().Path()}, "") + } else { + ctx.SetOutputFiles(android.Paths{}, "") + } + if mod.compiler != nil { + ctx.SetOutputFiles(android.PathsIfNonNil(mod.compiler.unstrippedOutputFilePath()), "unstripped") + } +} + +func buildComplianceMetadataInfo(ctx *moduleContext, mod *Module, deps PathDeps) { + // Dump metadata that can not be done in android/compliance-metadata.go + metadataInfo := ctx.ComplianceMetadataInfo() + metadataInfo.SetStringValue(android.ComplianceMetadataProp.IS_STATIC_LIB, strconv.FormatBool(mod.Static())) + metadataInfo.SetStringValue(android.ComplianceMetadataProp.BUILT_FILES, mod.outputFile.String()) + + // Static libs + staticDeps := ctx.GetDirectDepsWithTag(rlibDepTag) + staticDepNames := make([]string, 0, len(staticDeps)) + for _, dep := range staticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + ccStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(false)) + for _, dep := range ccStaticDeps { + staticDepNames = append(staticDepNames, dep.Name()) + } + + staticDepPaths := make([]string, 0, len(deps.StaticLibs)+len(deps.RLibs)) + // C static libraries + for _, dep := range deps.StaticLibs { + staticDepPaths = append(staticDepPaths, dep.String()) + } + // Rust static libraries + for _, dep := range deps.RLibs { + staticDepPaths = append(staticDepPaths, dep.Path.String()) + } + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEP_FILES, android.FirstUniqueStrings(staticDepPaths)) + + // C Whole static libs + ccWholeStaticDeps := ctx.GetDirectDepsWithTag(cc.StaticDepTag(true)) + wholeStaticDepNames := make([]string, 0, len(ccWholeStaticDeps)) + for _, dep := range ccStaticDeps { + wholeStaticDepNames = append(wholeStaticDepNames, dep.Name()) + } + metadataInfo.SetListValue(android.ComplianceMetadataProp.STATIC_DEPS, android.FirstUniqueStrings(staticDepNames)) } func (mod *Module) deps(ctx DepsContext) Deps { @@ -1200,7 +1253,6 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { ctx.VisitDirectDeps(func(dep android.Module) { depName := ctx.OtherModuleName(dep) depTag := ctx.OtherModuleDependencyTag(dep) - if _, exists := skipModuleList[depName]; exists { return } @@ -1213,8 +1265,8 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { //Handle Rust Modules makeLibName := rustMakeLibName(ctx, mod, rustDep, depName+rustDep.Properties.RustSubName) - switch depTag { - case dylibDepTag: + switch { + case depTag == dylibDepTag: dylib, ok := rustDep.compiler.(libraryInterface) if !ok || !dylib.dylib() { ctx.ModuleErrorf("mod %q not an dylib library", depName) @@ -1224,8 +1276,7 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { mod.Properties.AndroidMkDylibs = append(mod.Properties.AndroidMkDylibs, makeLibName) mod.Properties.SnapshotDylibs = append(mod.Properties.SnapshotDylibs, cc.BaseLibName(depName)) - case rlibDepTag: - + case depTag == rlibDepTag: rlib, ok := rustDep.compiler.(libraryInterface) if !ok || !rlib.rlib() { ctx.ModuleErrorf("mod %q not an rlib library", makeLibName) @@ -1240,16 +1291,25 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { depPaths.depIncludePaths = append(depPaths.depIncludePaths, exportedInfo.IncludeDirs...) depPaths.exportedLinkDirs = append(depPaths.exportedLinkDirs, linkPathFromFilePath(rustDep.OutputFile().Path())) - case procMacroDepTag: + case depTag == procMacroDepTag: directProcMacroDeps = append(directProcMacroDeps, rustDep) mod.Properties.AndroidMkProcMacroLibs = append(mod.Properties.AndroidMkProcMacroLibs, makeLibName) // proc_macro link dirs need to be exported, so collect those here. depPaths.exportedLinkDirs = append(depPaths.exportedLinkDirs, linkPathFromFilePath(rustDep.OutputFile().Path())) - case sourceDepTag: + case depTag == sourceDepTag: if _, ok := mod.sourceProvider.(*protobufDecorator); ok { collectIncludedProtos(mod, rustDep) } + case cc.IsStaticDepTag(depTag): + // Rust FFI rlibs should not be declared in a Rust modules + // "static_libs" list as we can't handle them properly at the + // moment (for example, they only produce an rlib-std variant). + // Instead, a normal rust_library variant should be used. + ctx.PropertyErrorf("static_libs", + "found '%s' in static_libs; use a rust_library module in rustlibs instead of a rust_ffi module in static_libs", + depName) + } transitiveAndroidMkSharedLibs = append(transitiveAndroidMkSharedLibs, rustDep.transitiveAndroidMkSharedLibs) @@ -1469,7 +1529,7 @@ func (mod *Module) depsToPaths(ctx android.ModuleContext) PathDeps { var srcProviderDepFiles android.Paths for _, dep := range directSrcProvidersDeps { - srcs, _ := dep.OutputFiles("") + srcs := android.OutputFilesForModule(ctx, dep, "") srcProviderDepFiles = append(srcProviderDepFiles, srcs...) } for _, dep := range directSrcDeps { @@ -1856,5 +1916,3 @@ var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String var StringPtr = proptools.StringPtr - -var _ android.OutputFileProducer = (*Module)(nil) diff --git a/rust/rust_test.go b/rust/rust_test.go index 8b96df8b3..0d005d0a2 100644 --- a/rust/rust_test.go +++ b/rust/rust_test.go @@ -447,23 +447,30 @@ func TestRustRlibs(t *testing.T) { export_include_dirs: ["foo_includes"] } + rust_ffi_rlib { + name: "libbuzz", + crate_name: "buzz", + srcs: ["src/lib.rs"], + export_include_dirs: ["buzz_includes"] + } + cc_library_shared { name: "libcc_shared", srcs:["foo.c"], - static_rlibs: ["libbar"], + static_libs: ["libbar"], } cc_library_static { name: "libcc_static", srcs:["foo.c"], - static_rlibs: ["libfoo"], + static_libs: ["libbuzz"], + whole_static_libs: ["libfoo"], } cc_binary { name: "ccBin", srcs:["foo.c"], - static_rlibs: ["libbar"], - static_libs: ["libcc_static"], + static_libs: ["libcc_static", "libbar"], } `) @@ -514,10 +521,13 @@ func TestRustRlibs(t *testing.T) { "-Ibar_includes", ccbin_cc.Args) } - // Make sure that direct dependencies and indirect dependencies are + // Make sure that direct dependencies and indirect whole static dependencies are // propagating correctly to the generated rlib. if !strings.Contains(ccbin_rustc.Args["libFlags"], "--extern foo=") { - t.Errorf("Missing indirect dependency libfoo when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) + t.Errorf("Missing indirect whole_static_lib dependency libfoo when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) + } + if strings.Contains(ccbin_rustc.Args["libFlags"], "--extern buzz=") { + t.Errorf("Indirect static_lib dependency libbuzz found when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) } if !strings.Contains(ccbin_rustc.Args["libFlags"], "--extern bar=") { t.Errorf("Missing direct dependency libbar when writing generated Rust staticlib: %#v", ccbin_rustc.Args["libFlags"]) diff --git a/rust/test_test.go b/rust/test_test.go index 6d0ebcf28..dc796c8a4 100644 --- a/rust/test_test.go +++ b/rust/test_test.go @@ -106,12 +106,9 @@ func TestDataLibs(t *testing.T) { ctx := testRust(t, bp) - module := ctx.ModuleForTests("main_test", "android_arm64_armv8-a").Module() - testBinary := module.(*Module).compiler.(*testDecorator) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected rust_test to produce output files, error: %s", err) - } + testingModule := ctx.ModuleForTests("main_test", "android_arm64_armv8-a") + testBinary := testingModule.Module().(*Module).compiler.(*testDecorator) + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } @@ -168,12 +165,10 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { ` ctx := testRust(t, bp) - module := ctx.ModuleForTests("main_test", "android_arm64_armv8-a").Module() + testingModule := ctx.ModuleForTests("main_test", "android_arm64_armv8-a") + module := testingModule.Module() testBinary := module.(*Module).compiler.(*testDecorator) - outputFiles, err := module.(android.OutputFileProducer).OutputFiles("") - if err != nil { - t.Fatalf("Expected rust_test to produce output files, error: %s", err) - } + outputFiles := testingModule.OutputFiles(t, "") if len(outputFiles) != 1 { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } diff --git a/rust/testing.go b/rust/testing.go index f31c59188..6ee49a971 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -189,11 +189,11 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.RegisterModuleType("rust_ffi", RustFFIFactory) ctx.RegisterModuleType("rust_ffi_shared", RustFFISharedFactory) ctx.RegisterModuleType("rust_ffi_rlib", RustFFIRlibFactory) - ctx.RegisterModuleType("rust_ffi_static", RustFFIStaticRlibFactory) + ctx.RegisterModuleType("rust_ffi_static", RustFFIRlibFactory) ctx.RegisterModuleType("rust_ffi_host", RustFFIHostFactory) ctx.RegisterModuleType("rust_ffi_host_shared", RustFFISharedHostFactory) ctx.RegisterModuleType("rust_ffi_host_rlib", RustFFIRlibHostFactory) - ctx.RegisterModuleType("rust_ffi_host_static", RustFFIStaticRlibHostFactory) + ctx.RegisterModuleType("rust_ffi_host_static", RustFFIRlibHostFactory) ctx.RegisterModuleType("rust_proc_macro", ProcMacroFactory) ctx.RegisterModuleType("rust_protobuf", RustProtobufFactory) ctx.RegisterModuleType("rust_protobuf_host", RustProtobufHostFactory) diff --git a/scripts/buildinfo.py b/scripts/buildinfo.py index 3383bf76c..db9920918 100755 --- a/scripts/buildinfo.py +++ b/scripts/buildinfo.py @@ -68,6 +68,7 @@ def parse_args(): option.build_id = product_config["BuildId"] option.build_type = product_config["BuildType"] option.build_variant = get_build_variant(product_config) + option.build_version_tags = product_config["BuildVersionTags"] option.cpu_abis = product_config["DeviceAbi"] option.default_locale = None if len(product_config.get("ProductLocales", [])) > 0: @@ -96,9 +97,11 @@ def main(): build_hostname = option.build_hostname_file.read().strip() build_number = option.build_number_file.read().strip() - build_version_tags = option.build_keys + build_version_tags_list = option.build_version_tags if option.build_type == "debug": - build_version_tags = "debug," + build_version_tags + build_version_tags_list.append("debug") + build_version_tags_list.append(option.build_keys) + build_version_tags = ",".join(sorted(set(build_version_tags_list))) raw_date = option.date_file.read().strip() date = subprocess.check_output(["date", "-d", f"@{raw_date}"], text=True).strip() diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt index 47eae0769..bb88ccebf 100644 --- a/scripts/check_boot_jars/package_allowed_list.txt +++ b/scripts/check_boot_jars/package_allowed_list.txt @@ -3,52 +3,7 @@ ################################################### # core-libart.jar & core-oj.jar -java\.awt\.font -java\.beans -java\.io -java\.lang -java\.lang\.annotation -java\.lang\.constant -java\.lang\.invoke -java\.lang\.ref -java\.lang\.reflect -java\.lang\.runtime -java\.math -java\.net -java\.nio -java\.nio\.file -java\.nio\.file\.spi -java\.nio\.file\.attribute -java\.nio\.channels -java\.nio\.channels\.spi -java\.nio\.charset -java\.nio\.charset\.spi -java\.security -java\.security\.acl -java\.security\.cert -java\.security\.interfaces -java\.security\.spec -java\.sql -java\.text -java\.text\.spi -java\.time -java\.time\.chrono -java\.time\.format -java\.time\.temporal -java\.time\.zone -java\.util -java\.util\.concurrent -java\.util\.concurrent\.atomic -java\.util\.concurrent\.locks -java\.util\.function -java\.util\.jar -java\.util\.logging -java\.util\.prefs -java\.util\.random -java\.util\.regex -java\.util\.spi -java\.util\.stream -java\.util\.zip +java(\..*)? # TODO: Remove javax.annotation.processing if possible, see http://b/132338110: javax\.annotation\.processing javax\.crypto @@ -72,18 +27,7 @@ javax\.xml\.transform\.sax javax\.xml\.transform\.stream javax\.xml\.validation javax\.xml\.xpath -jdk\.internal -jdk\.internal\.access -jdk\.internal\.math -jdk\.internal\.misc -jdk\.internal\.ref -jdk\.internal\.reflect -jdk\.internal\.util -jdk\.internal\.util\.jar -jdk\.internal\.util\.random -jdk\.internal\.vm\.annotation -jdk\.net -jdk\.random +jdk\..* org\.w3c\.dom org\.w3c\.dom\.ls org\.w3c\.dom\.traversal diff --git a/scripts/gen_build_prop.py b/scripts/gen_build_prop.py index 6c029061d..799e00b19 100644 --- a/scripts/gen_build_prop.py +++ b/scripts/gen_build_prop.py @@ -19,9 +19,12 @@ import argparse import contextlib import json +import os import subprocess import sys +TEST_KEY_DIR = "build/make/target/product/security" + def get_build_variant(product_config): if product_config["Eng"]: return "eng" @@ -73,9 +76,13 @@ def parse_args(): config["BuildHostname"] = args.build_hostname_file.read().strip() config["BuildNumber"] = args.build_number_file.read().strip() config["BuildUsername"] = args.build_username - config["BuildVersionTags"] = config["BuildKeys"] + + build_version_tags_list = config["BuildVersionTags"] if config["BuildType"] == "debug": - config["BuildVersionTags"] = "debug," + config["BuildVersionTags"] + build_version_tags_list.append("debug") + build_version_tags_list.append(config["BuildKeys"]) + build_version_tags = ",".join(sorted(set(build_version_tags_list))) + config["BuildVersionTags"] = build_version_tags raw_date = args.date_file.read().strip() config["Date"] = subprocess.check_output(["date", "-d", f"@{raw_date}"], text=True).strip() diff --git a/sdk/sdk.go b/sdk/sdk.go index fd16ab63f..5b644fd85 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -193,6 +193,10 @@ func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Generate the snapshot from the member info. s.buildSnapshot(ctx, sdkVariants) } + + if s.snapshotFile.Valid() { + ctx.SetOutputFiles([]android.Path{s.snapshotFile.Path()}, "") + } } func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { @@ -222,18 +226,6 @@ func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { }} } -func (s *sdk) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if s.snapshotFile.Valid() { - return []android.Path{s.snapshotFile.Path()}, nil - } - return nil, fmt.Errorf("snapshot file not defined. This is most likely because this isn't the common_os variant of this module") - default: - return nil, fmt.Errorf("unknown tag %q", tag) - } -} - // gatherTraits gathers the traits from the dynamically generated trait specific properties. // // Returns a map from member name to the set of required traits. diff --git a/sh/sh_binary.go b/sh/sh_binary.go index 48a442de4..2e48d83e6 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -212,6 +212,14 @@ var _ android.ImageInterface = (*ShBinary)(nil) func (s *ShBinary) ImageMutatorBegin(ctx android.BaseModuleContext) {} +func (s *ShBinary) VendorVariantNeeded(ctx android.BaseModuleContext) bool { + return s.InstallInVendor() +} + +func (s *ShBinary) ProductVariantNeeded(ctx android.BaseModuleContext) bool { + return s.InstallInProduct() +} + func (s *ShBinary) CoreVariantNeeded(ctx android.BaseModuleContext) bool { return !s.InstallInRecovery() && !s.InstallInRamdisk() && !s.InstallInVendorRamdisk() && !s.ModuleBase.InstallInVendor() } @@ -233,14 +241,7 @@ func (s *ShBinary) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool { } func (s *ShBinary) ExtraImageVariations(ctx android.BaseModuleContext) []string { - extraVariations := []string{} - if s.InstallInProduct() { - extraVariations = append(extraVariations, cc.ProductVariation) - } - if s.InstallInVendor() { - extraVariations = append(extraVariations, cc.VendorVariation) - } - return extraVariations + return nil } func (s *ShBinary) SetImageVariation(ctx android.BaseModuleContext, variation string) { @@ -306,7 +307,7 @@ func (s *ShBinary) generateAndroidBuildActions(ctx android.ModuleContext) { func (s *ShBinary) GetSubname(ctx android.ModuleContext) string { ret := "" if s.properties.ImageVariation != "" { - if s.properties.ImageVariation != cc.VendorVariation { + if s.properties.ImageVariation != android.VendorVariation { ret = "." + s.properties.ImageVariation } } diff --git a/shared/Android.bp b/shared/Android.bp index 3c84f5532..d5e86146f 100644 --- a/shared/Android.bp +++ b/shared/Android.bp @@ -15,7 +15,6 @@ bootstrap_go_package { "paths_test.go", ], deps: [ - "soong-bazel", "golang-protobuf-proto", ], } diff --git a/shared/paths.go b/shared/paths.go index fca8b4c15..1ee66d581 100644 --- a/shared/paths.go +++ b/shared/paths.go @@ -18,8 +18,6 @@ package shared import ( "path/filepath" - - "android/soong/bazel" ) // A SharedPaths represents a list of paths that are shared between @@ -49,11 +47,3 @@ func JoinPath(base string, rest ...string) string { func TempDirForOutDir(outDir string) (tempPath string) { return filepath.Join(outDir, ".temp") } - -// BazelMetricsFilename returns the bazel profile filename based -// on the action name. This is to help to store a set of bazel -// profiles since bazel may execute multiple times during a single -// build. -func BazelMetricsFilename(s SharedPaths, actionName bazel.RunName) string { - return filepath.Join(s.BazelMetricsDir(), actionName.String()+"_bazel_profile.gz") -} diff --git a/ui/build/config.go b/ui/build/config.go index 8dddea584..1d5269c7c 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -99,9 +99,10 @@ type configImpl struct { // Autodetected totalRAM uint64 - brokenDupRules bool - brokenUsesNetwork bool - brokenNinjaEnvVars []string + brokenDupRules bool + brokenUsesNetwork bool + brokenNinjaEnvVars []string + brokenMissingOutputs bool pathReplaced bool @@ -1246,6 +1247,11 @@ func (c *configImpl) StartGoma() bool { } func (c *configImpl) canSupportRBE() bool { + // Only supported on linux + if runtime.GOOS != "linux" { + return false + } + // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since // its unlikely that we will be able to obtain necessary creds without stubby. authType, _ := c.rbeAuth() @@ -1603,6 +1609,14 @@ func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { return c.brokenNinjaEnvVars } +func (c *configImpl) SetBuildBrokenMissingOutputs(val bool) { + c.brokenMissingOutputs = val +} + +func (c *configImpl) BuildBrokenMissingOutputs() bool { + return c.brokenMissingOutputs +} + func (c *configImpl) SetTargetDeviceDir(dir string) { c.targetDeviceDir = dir } diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index eba86a0fc..e77df44a5 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -235,6 +235,11 @@ func runMakeProductConfig(ctx Context, config Config) { "BUILD_BROKEN_SRC_DIR_IS_WRITABLE", "BUILD_BROKEN_SRC_DIR_RW_ALLOWLIST", + // Whether missing outputs should be treated as warnings + // instead of errors. + // `true` will relegate missing outputs to warnings. + "BUILD_BROKEN_MISSING_OUTPUTS", + // Not used, but useful to be in the soong.log "TARGET_BUILD_TYPE", "HOST_ARCH", @@ -301,4 +306,5 @@ func runMakeProductConfig(ctx Context, config Config) { config.SetBuildBrokenUsesNetwork(makeVars["BUILD_BROKEN_USES_NETWORK"] == "true") config.SetBuildBrokenNinjaUsesEnvVars(strings.Fields(makeVars["BUILD_BROKEN_NINJA_USES_ENV_VARS"])) config.SetSourceRootDirs(strings.Fields(makeVars["PRODUCT_SOURCE_ROOT_DIRS"])) + config.SetBuildBrokenMissingOutputs(makeVars["BUILD_BROKEN_MISSING_OUTPUTS"] == "true") } diff --git a/ui/build/ninja.go b/ui/build/ninja.go index 551b8ab41..ae27330a9 100644 --- a/ui/build/ninja.go +++ b/ui/build/ninja.go @@ -77,6 +77,14 @@ func runNinjaForBuild(ctx Context, config Config) { "-w", "dupbuild=err", "-w", "missingdepfile=err") + if !config.BuildBrokenMissingOutputs() { + // Missing outputs will be treated as errors. + // BUILD_BROKEN_MISSING_OUTPUTS can be used to bypass this check. + args = append(args, + "-w", "missingoutfile=err", + ) + } + cmd := Command(ctx, config, "ninja", executable, args...) // Set up the nsjail sandbox Ninja runs in. diff --git a/ui/build/soong.go b/ui/build/soong.go index 9a4583cdd..77fee0ac6 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -27,7 +27,6 @@ import ( "android/soong/ui/tracer" - "android/soong/bazel" "android/soong/ui/metrics" "android/soong/ui/metrics/metrics_proto" "android/soong/ui/status" @@ -603,10 +602,6 @@ func runSoong(ctx Context, config Config) { checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag)) - // Remove bazel files in the event that bazel is disabled for the build. - // These files may have been left over from a previous bazel-enabled build. - cleanBazelFiles(config) - if config.JsonModuleGraph() { checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag)) } @@ -758,18 +753,6 @@ func loadSoongBuildMetrics(ctx Context, config Config, oldTimestamp time.Time) { } } -func cleanBazelFiles(config Config) { - files := []string{ - shared.JoinPath(config.SoongOutDir(), "bp2build"), - shared.JoinPath(config.SoongOutDir(), "workspace"), - shared.JoinPath(config.SoongOutDir(), bazel.SoongInjectionDirName), - shared.JoinPath(config.OutDir(), "bazel")} - - for _, f := range files { - os.RemoveAll(f) - } -} - func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) { ctx.BeginTrace(metrics.RunSoong, name) defer ctx.EndTrace() |