diff options
46 files changed, 1209 insertions, 254 deletions
diff --git a/Android.bp b/Android.bp index 1219c626a..434ee9f96 100644 --- a/Android.bp +++ b/Android.bp @@ -172,6 +172,9 @@ build_prop { name: "system-build.prop", stem: "build.prop", product_config: ":product_config", + footer_files: [ + ":applied_backported_fixes", + ], // Currently, only microdroid, Ravenwood, and cf system image can refer to system-build.prop visibility: [ "//build/make/target/product/generic", diff --git a/android/apex.go b/android/apex.go index 9277ff31b..1bfcf04d4 100644 --- a/android/apex.go +++ b/android/apex.go @@ -224,11 +224,8 @@ type ApexModule interface { // run. DirectlyInAnyApex() bool - // NotInPlatform tells whether or not this module is included in an APEX and therefore - // shouldn't be exposed to the platform (i.e. outside of the APEX) directly. A module is - // considered to be included in an APEX either when there actually is an APEX that - // explicitly has the module as its dependency or the module is not available to the - // platform, which indicates that the module belongs to at least one or more other APEXes. + // NotInPlatform returns true if the module is not available to the platform due to + // apex_available being set and not containing "//apex_available:platform". NotInPlatform() bool // Tests if this module could have APEX variants. Even when a module type implements @@ -291,14 +288,6 @@ type ApexProperties struct { // See ApexModule.DirectlyInAnyApex() DirectlyInAnyApex bool `blueprint:"mutated"` - // AnyVariantDirectlyInAnyApex is true in the primary variant of a module if _any_ variant - // of the module is directly in any apex. This includes host, arch, asan, etc. variants. It - // is unused in any variant that is not the primary variant. Ideally this wouldn't be used, - // as it incorrectly mixes arch variants if only one arch is in an apex, but a few places - // depend on it, for example when an ASAN variant is created before the apexMutator. Call - // this after apex.apexMutator is run. - AnyVariantDirectlyInAnyApex bool `blueprint:"mutated"` - // See ApexModule.NotAvailableForPlatform() NotAvailableForPlatform bool `blueprint:"mutated"` @@ -428,7 +417,7 @@ func (m *ApexModuleBase) DirectlyInAnyApex() bool { // Implements ApexModule func (m *ApexModuleBase) NotInPlatform() bool { - return m.ApexProperties.AnyVariantDirectlyInAnyApex || !m.AvailableFor(AvailableToPlatform) + return !m.AvailableFor(AvailableToPlatform) } // Implements ApexModule @@ -802,22 +791,6 @@ func UpdateDirectlyInAnyApex(mctx BottomUpMutatorContext, am ApexModule) { } return false }) - - if base.ApexProperties.DirectlyInAnyApex { - // Variants of a module are always visited sequentially in order, so it is safe to - // write to another variant of this module. For a BottomUpMutator the - // PrimaryModule() is visited first and FinalModule() is visited last. - mctx.FinalModule().(ApexModule).apexModuleBase().ApexProperties.AnyVariantDirectlyInAnyApex = true - } - - // If this is the FinalModule (last visited module) copy - // AnyVariantDirectlyInAnyApex to all the other variants - if mctx.IsFinalModule(am) { - mctx.VisitAllModuleVariants(func(variant Module) { - variant.(ApexModule).apexModuleBase().ApexProperties.AnyVariantDirectlyInAnyApex = - base.ApexProperties.AnyVariantDirectlyInAnyApex - }) - } } // ApexMembership tells how a module became part of an APEX. diff --git a/android/build_prop.go b/android/build_prop.go index cda56f1e0..2f71bc03f 100644 --- a/android/build_prop.go +++ b/android/build_prop.go @@ -15,6 +15,8 @@ package android import ( + "fmt" + "github.com/google/blueprint/proptools" ) @@ -173,7 +175,16 @@ func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { postProcessCmd.Text(outputFilePath.String()) postProcessCmd.Flags(p.properties.Block_list) - rule.Command().Text("echo").Text(proptools.NinjaAndShellEscape("# end of file")).FlagWithArg(">> ", outputFilePath.String()) + for _, footer := range p.properties.Footer_files { + path := PathForModuleSrc(ctx, footer) + rule.appendText(outputFilePath, "####################################") + rule.appendTextf(outputFilePath, "# Adding footer from %v", footer) + rule.appendTextf(outputFilePath, "# with path %v", path) + rule.appendText(outputFilePath, "####################################") + rule.Command().Text("cat").FlagWithInput("", path).FlagWithArg(">> ", outputFilePath.String()) + } + + rule.appendText(outputFilePath, "# end of file") rule.Build(ctx.ModuleName(), "generating build.prop") @@ -184,6 +195,14 @@ func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { p.outputFilePath = outputFilePath } +func (r *RuleBuilder) appendText(path ModuleOutPath, text string) { + r.Command().Text("echo").Text(proptools.NinjaAndShellEscape(text)).FlagWithArg(">> ", path.String()) +} + +func (r *RuleBuilder) appendTextf(path ModuleOutPath, format string, a ...any) { + r.appendText(path, fmt.Sprintf(format, a...)) +} + func (p *buildPropModule) AndroidMkEntries() []AndroidMkEntries { return []AndroidMkEntries{{ Class: "ETC", diff --git a/android/module_context.go b/android/module_context.go index 20149074e..ae7b54f66 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -831,6 +831,11 @@ func (m *moduleContext) ModuleInfoJSON() *ModuleInfoJSON { } func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) { + for _, outputFile := range outputFiles { + if outputFile == nil { + panic("outputfiles cannot be nil") + } + } if tag == "" { if len(m.outputFiles.DefaultOutputFiles) > 0 { m.ModuleErrorf("Module %s default OutputFiles cannot be overwritten", m.ModuleName()) diff --git a/android/packaging.go b/android/packaging.go index e71d983eb..dcd8844f0 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -410,9 +410,9 @@ func (PackagingItemAlwaysDepTag) IsPackagingItem() bool { return true } -// highPriorityDepTag provides default implementation of HighPriorityPackagingItem interface. type highPriorityDepTag struct { - blueprint.DependencyTag + blueprint.BaseDependencyTag + PackagingItemAlwaysDepTag } // See PackageModule.AddDeps @@ -433,7 +433,7 @@ func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext, depTag blueprint.Dep } depTagToUse := depTag if highPriority { - depTagToUse = highPriorityDepTag{depTag} + depTagToUse = highPriorityDepTag{} } ctx.AddFarVariationDependencies(targetVariation, depTagToUse, dep) diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index 5e4af0ba5..b90ef3b1c 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -508,11 +508,10 @@ func (p *prebuiltModule) Name() string { } func (p *prebuiltModule) GenerateAndroidBuildActions(ctx ModuleContext) { - var src Path if len(p.properties.Srcs) >= 1 { - src = p.prebuilt.SingleSourcePath(ctx) + src := p.prebuilt.SingleSourcePath(ctx) + ctx.SetOutputFiles(Paths{src}, "") } - ctx.SetOutputFiles(Paths{src}, "") } func (p *prebuiltModule) Prebuilt() *Prebuilt { diff --git a/android/variable.go b/android/variable.go index 2d43c6da4..88cf5a54b 100644 --- a/android/variable.go +++ b/android/variable.go @@ -610,6 +610,7 @@ type PartitionVariables struct { CopyImagesForTargetFilesZip bool `json:",omitempty"` // Boot image stuff + BuildingRamdiskImage bool `json:",omitempty"` ProductBuildBootImage bool `json:",omitempty"` ProductBuildInitBootImage bool `json:",omitempty"` BoardUsesRecoveryAsBoot bool `json:",omitempty"` @@ -618,6 +619,7 @@ type PartitionVariables struct { BoardBootimagePartitionSize string `json:",omitempty"` BoardInitBootimagePartitionSize string `json:",omitempty"` BoardBootHeaderVersion string `json:",omitempty"` + TargetKernelPath string `json:",omitempty"` // Avb (android verified boot) stuff BoardAvbEnable bool `json:",omitempty"` diff --git a/cc/binary.go b/cc/binary.go index 2ac9a45bc..4b77bea64 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -505,7 +505,7 @@ func (binary *binaryDecorator) install(ctx ModuleContext, file android.Path) { // The original path becomes a symlink to the corresponding file in the // runtime APEX. translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled - if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !ctx.Host() && ctx.directlyInAnyApex() && + if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !ctx.Host() && !ctx.isSdkVariant() && !translatedArch && ctx.apexVariationName() == "" && !ctx.inRamdisk() && !ctx.inRecovery() && !ctx.inVendorRamdisk() { @@ -548,7 +548,6 @@ type ModuleContextIntf interface { apexSdkVersion() android.ApiLevel bootstrap() bool nativeCoverage() bool - directlyInAnyApex() bool isPreventInstall() bool isCfiAssemblySupportEnabled() bool getSharedFlags() *SharedFlags @@ -1692,10 +1691,6 @@ func (ctx *moduleContextImpl) nativeCoverage() bool { return ctx.mod.nativeCoverage() } -func (ctx *moduleContextImpl) directlyInAnyApex() bool { - return ctx.mod.DirectlyInAnyApex() -} - func (ctx *moduleContextImpl) isPreventInstall() bool { return ctx.mod.Properties.PreventInstall } diff --git a/cc/config/global.go b/cc/config/global.go index 36690d6be..6984ea412 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -290,8 +290,6 @@ var ( "-Wno-error=deprecated", // in external/googletest/googletest // Disabling until the warning is fixed in libc++abi header files b/366180429 "-Wno-deprecated-dynamic-exception-spec", - // New warnings to be fixed after clang-r475365 - "-Wno-error=enum-constexpr-conversion", // http://b/243964282 // New warnings to be fixed after clang-r522817 "-Wno-error=invalid-offsetof", "-Wno-error=thread-safety-reference-return", diff --git a/cc/image.go b/cc/image.go index ee4048390..9766af3df 100644 --- a/cc/image.go +++ b/cc/image.go @@ -179,9 +179,6 @@ type ImageMutatableModule interface { // SnapshotVersion returns the snapshot version for this module. SnapshotVersion(mctx android.ImageInterfaceContext) string - // SdkVersion returns the SDK version for this module. - SdkVersion() string - // ExtraVariants returns the list of extra variants this module requires. ExtraVariants() []string @@ -370,7 +367,7 @@ func MutateImage(mctx android.ImageInterfaceContext, m ImageMutatableModule) { if m.HasProductVariant() { productVariantNeeded = true } - } else if vendorSpecific && m.SdkVersion() == "" { + } else if vendorSpecific { // This will be available in /vendor (or /odm) only vendorVariantNeeded = true } else { @@ -380,7 +377,7 @@ func MutateImage(mctx android.ImageInterfaceContext, m ImageMutatableModule) { coreVariantNeeded = true } - if coreVariantNeeded && productSpecific && m.SdkVersion() == "" { + if coreVariantNeeded && productSpecific { // The module has "product_specific: true" that does not create core variant. coreVariantNeeded = false productVariantNeeded = true diff --git a/cc/library.go b/cc/library.go index 4ce506e90..c97dbf922 100644 --- a/cc/library.go +++ b/cc/library.go @@ -656,7 +656,7 @@ func (library *libraryDecorator) compileModuleLibApiStubs(ctx ModuleContext, fla // However, having this distinction helps guard accidental // promotion or demotion of API and also helps the API review process b/191371676 var flag string - if ctx.Module().(android.ApexModule).NotInPlatform() { + if ctx.notInPlatform() { flag = "--apex" } else { flag = "--systemapi" @@ -1750,21 +1750,17 @@ func (library *libraryDecorator) installSymlinkToRuntimeApex(ctx ModuleContext, func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) { if library.shared() { - if library.hasStubsVariants() && !ctx.Host() && ctx.directlyInAnyApex() { + translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled + if library.hasStubsVariants() && !ctx.Host() && !ctx.isSdkVariant() && + InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && + !translatedArch && !ctx.inRamdisk() && !ctx.inVendorRamdisk() && !ctx.inRecovery() { // Bionic libraries (e.g. libc.so) is installed to the bootstrap subdirectory. // The original path becomes a symlink to the corresponding file in the // runtime APEX. - translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled - if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && - !translatedArch && !ctx.inRamdisk() && !ctx.inVendorRamdisk() && !ctx.inRecovery() { - if ctx.Device() { - library.installSymlinkToRuntimeApex(ctx, file) - } - library.baseInstaller.subDir = "bootstrap" + if ctx.Device() { + library.installSymlinkToRuntimeApex(ctx, file) } - } else if ctx.directlyInAnyApex() && ctx.IsLlndk() && !isBionic(ctx.baseModuleName()) { - // Skip installing LLNDK (non-bionic) libraries moved to APEX. - ctx.Module().HideFromMake() + library.baseInstaller.subDir = "bootstrap" } library.baseInstaller.install(ctx, file) diff --git a/cc/sanitize.go b/cc/sanitize.go index f0d734376..d8d8c7aef 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -1504,9 +1504,6 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { if Bool(sanProps.Memtag_globals) { sanitizers = append(sanitizers, "memtag-globals") - // TODO(mitchp): For now, enable memtag-heap with memtag-globals because the linker - // isn't new enough (https://reviews.llvm.org/differential/changeset/?ref=4243566). - sanitizers = append(sanitizers, "memtag-heap") } if Bool(sanProps.Fuzzer) { diff --git a/cmd/find_input_delta/find_input_delta/main.go b/cmd/find_input_delta/find_input_delta/main.go index 6b657ea57..a8645843c 100644 --- a/cmd/find_input_delta/find_input_delta/main.go +++ b/cmd/find_input_delta/find_input_delta/main.go @@ -85,4 +85,11 @@ func main() { if err = file_list.Format(os.Stdout, template); err != nil { panic(err) } + + metrics_file := os.Getenv("SOONG_METRICS_AGGREGATION_FILE") + if metrics_file != "" { + if err = file_list.SendMetrics(metrics_file); err != nil { + panic(err) + } + } } diff --git a/cmd/find_input_delta/find_input_delta_lib/Android.bp b/cmd/find_input_delta/find_input_delta_lib/Android.bp index 95bdba81d..ef9c65b1d 100644 --- a/cmd/find_input_delta/find_input_delta_lib/Android.bp +++ b/cmd/find_input_delta/find_input_delta_lib/Android.bp @@ -25,6 +25,7 @@ bootstrap_go_package { "golang-protobuf-runtime-protoimpl", "soong-cmd-find_input_delta-proto", "soong-cmd-find_input_delta-proto_internal", + "android-archive-zip", "blueprint-pathtools", ], srcs: [ diff --git a/cmd/find_input_delta/find_input_delta_lib/file_list.go b/cmd/find_input_delta/find_input_delta_lib/file_list.go index 23337adc3..01242a0e7 100644 --- a/cmd/find_input_delta/find_input_delta_lib/file_list.go +++ b/cmd/find_input_delta/find_input_delta_lib/file_list.go @@ -15,10 +15,15 @@ package find_input_delta_lib import ( + "fmt" "io" + "os" + "path/filepath" + "slices" "text/template" fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" ) @@ -47,28 +52,148 @@ type FileList struct { // The modified files Changes []FileList + + // Map of file_extension:counts + ExtCountMap map[string]*FileCounts + + // Total number of added/changed/deleted files. + TotalDelta uint32 +} + +// The maximum number of files that will be recorded by name. +var MaxFilesRecorded uint32 = 50 + +type FileCounts struct { + Additions uint32 + Deletions uint32 + Changes uint32 +} + +func FileListFactory(name string) *FileList { + return &FileList{ + Name: name, + ExtCountMap: make(map[string]*FileCounts), + } +} + +func (fl *FileList) addFile(name string) { + fl.Additions = append(fl.Additions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Additions += 1 +} + +func (fl *FileList) deleteFile(name string) { + fl.Deletions = append(fl.Deletions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Deletions += 1 +} + +func (fl *FileList) changeFile(name string, ch *FileList) { + fl.Changes = append(fl.Changes, *ch) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Changes += 1 +} + +func (fl FileList) ToProto() (*fid_exp.FileList, error) { + var count uint32 + return fl.toProto(&count) } -func (fl FileList) Marshal() (*fid_exp.FileList, error) { +func (fl FileList) toProto(count *uint32) (*fid_exp.FileList, error) { ret := &fid_exp.FileList{ Name: proto.String(fl.Name), } - if len(fl.Additions) > 0 { - ret.Additions = fl.Additions + for _, a := range fl.Additions { + if *count >= MaxFilesRecorded { + break + } + ret.Additions = append(ret.Additions, a) + *count += 1 } for _, ch := range fl.Changes { - change, err := ch.Marshal() - if err != nil { - return nil, err + if *count >= MaxFilesRecorded { + break + } else { + // Pre-increment to limit what the call adds. + *count += 1 + change, err := ch.toProto(count) + if err != nil { + return nil, err + } + ret.Changes = append(ret.Changes, change) + } + } + for _, d := range fl.Deletions { + if *count >= MaxFilesRecorded { + break } - ret.Changes = append(ret.Changes, change) + ret.Deletions = append(ret.Deletions, d) } - if len(fl.Deletions) > 0 { - ret.Deletions = fl.Deletions + ret.TotalDelta = proto.Uint32(*count) + exts := []string{} + for k := range fl.ExtCountMap { + exts = append(exts, k) + } + slices.Sort(exts) + for _, k := range exts { + v := fl.ExtCountMap[k] + ret.Counts = append(ret.Counts, &fid_exp.FileCount{ + Extension: proto.String(k), + Additions: proto.Uint32(v.Additions), + Deletions: proto.Uint32(v.Deletions), + Modifications: proto.Uint32(v.Changes), + }) } return ret, nil } +func (fl FileList) SendMetrics(path string) error { + if path == "" { + return fmt.Errorf("No path given") + } + message, err := fl.ToProto() + if err != nil { + return err + } + + // Marshal the message wrapped in SoongCombinedMetrics. + data := protowire.AppendVarint( + []byte{}, + protowire.EncodeTag( + protowire.Number(fid_exp.FieldNumbers_FIELD_NUMBERS_FILE_LIST), + protowire.BytesType)) + size := uint64(proto.Size(message)) + data = protowire.AppendVarint(data, size) + data, err = proto.MarshalOptions{UseCachedSize: true}.MarshalAppend(data, message) + if err != nil { + return err + } + + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to close %s: %v\n", path, err) + } + }() + _, err = out.Write(data) + return err +} + func (fl FileList) Format(wr io.Writer, format string) error { tmpl, err := template.New("filelist").Parse(format) if err != nil { diff --git a/cmd/find_input_delta/find_input_delta_lib/fs.go b/cmd/find_input_delta/find_input_delta_lib/fs.go index 4a83ed7ff..09a8aa6cb 100644 --- a/cmd/find_input_delta/find_input_delta_lib/fs.go +++ b/cmd/find_input_delta/find_input_delta_lib/fs.go @@ -15,7 +15,6 @@ package find_input_delta_lib import ( - "io" "io/fs" "os" ) @@ -30,14 +29,6 @@ type fileSystem interface { ReadFile(path string) ([]byte, error) } -type file interface { - io.Closer - io.Reader - io.ReaderAt - io.Seeker - Stat() (os.FileInfo, error) -} - // osFS implements fileSystem using the local disk. type osFS struct{} diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state.go b/cmd/find_input_delta/find_input_delta_lib/internal_state.go index b2ff8c704..2b8c39527 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state.go @@ -18,9 +18,11 @@ import ( "errors" "fmt" "io/fs" + "regexp" "slices" fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal" + "android/soong/third_party/zip" "github.com/google/blueprint/pathtools" "google.golang.org/protobuf/proto" ) @@ -57,6 +59,7 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* // If we ever have an easy hash, assign it here. } if inspect_contents { + // NOTE: When we find it useful, we can parallelize the file inspection for speed. contents, err := InspectFileContents(input) if err != nil { return ret, err @@ -70,14 +73,36 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* return ret, nil } +// We ignore any suffix digit caused by sharding. +var InspectExtsZipRegexp = regexp.MustCompile("\\.(jar|apex|apk)[0-9]*$") + // Inspect the file and extract the state of the elements in the archive. // If this is not an archive of some sort, nil is returned. func InspectFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { - // TODO: Actually inspect the contents. - fmt.Printf("inspecting contents for %s\n", name) + if InspectExtsZipRegexp.Match([]byte(name)) { + return inspectZipFileContents(name) + } return nil, nil } +func inspectZipFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { + rc, err := zip.OpenReader(name) + if err != nil { + return nil, err + } + ret := []*fid_proto.PartialCompileInput{} + for _, v := range rc.File { + pci := &fid_proto.PartialCompileInput{ + Name: proto.String(v.Name), + MtimeNsec: proto.Int64(v.ModTime().UnixNano()), + Hash: proto.String(fmt.Sprintf("%08x", v.CRC32)), + } + ret = append(ret, pci) + // We do not support nested inspection. + } + return ret, nil +} + func WriteState(s *fid_proto.PartialCompileInputs, path string) error { data, err := proto.Marshal(s) if err != nil { @@ -91,9 +116,7 @@ func CompareInternalState(prior, other *fid_proto.PartialCompileInputs, target s } func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name string) *FileList { - fl := &FileList{ - Name: name, - } + fl := FileListFactory(name) PriorMap := make(map[string]*fid_proto.PartialCompileInput, len(prior)) // We know that the lists are properly sorted, so we can simply compare them. for _, v := range prior { @@ -105,17 +128,17 @@ func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name strin otherMap[name] = v if _, ok := PriorMap[name]; !ok { // Added file - fl.Additions = append(fl.Additions, name) + fl.addFile(name) } else if !proto.Equal(PriorMap[name], v) { // Changed file - fl.Changes = append(fl.Changes, *CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) + fl.changeFile(name, CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) } } for _, v := range prior { name := v.GetName() if _, ok := otherMap[name]; !ok { // Deleted file - fl.Deletions = append(fl.Deletions, name) + fl.deleteFile(name) } } return fl diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go index 20b8efaa8..c168d5a6b 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go @@ -199,7 +199,7 @@ func TestCompareInternalState(t *testing.T) { }, }, { - Name: "one of each", + Name: "one each add modify delete", Target: "foo", Prior: &fid_proto.PartialCompileInputs{ InputFiles: []*fid_proto.PartialCompileInput{ @@ -222,11 +222,62 @@ func TestCompareInternalState(t *testing.T) { Deletions: []string{"file2"}, }, }, + { + Name: "interior one each add modify delete", + Target: "bar", + Prior: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 405, "", []*fid_proto.PartialCompileInput{ + protoFile("innerC", 400, "crc32:11111111", nil), + protoFile("innerD", 400, "crc32:44444444", nil), + }), + }, + }, + New: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 505, "", []*fid_proto.PartialCompileInput{ + protoFile("innerA", 400, "crc32:55555555", nil), + protoFile("innerC", 500, "crc32:66666666", nil), + }), + }, + }, + Expected: &FileList{ + Name: "bar", + Changes: []FileList{FileList{ + Name: "file1", + Additions: []string{"innerA"}, + Changes: []FileList{FileList{Name: "innerC"}}, + Deletions: []string{"innerD"}, + }}, + }, + }, } for _, tc := range testCases { actual := CompareInternalState(tc.Prior, tc.New, tc.Target) if !tc.Expected.Equal(actual) { - t.Errorf("%s: expected %q, actual %q", tc.Name, tc.Expected, actual) + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) + } + } +} + +func TestCompareInspectExtsZipRegexp(t *testing.T) { + testCases := []struct { + Name string + Expected bool + }{ + {Name: ".jar", Expected: true}, + {Name: ".jar5", Expected: true}, + {Name: ".apex", Expected: true}, + {Name: ".apex9", Expected: true}, + {Name: ".apexx", Expected: false}, + {Name: ".apk", Expected: true}, + {Name: ".apk3", Expected: true}, + {Name: ".go", Expected: false}, + } + for _, tc := range testCases { + actual := InspectExtsZipRegexp.Match([]byte(tc.Name)) + if tc.Expected != actual { + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) } } } diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go index 648ef224a..745de2db2 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: file_list.proto -package proto +package find_input_delta_proto import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -35,6 +35,62 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_file_list_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_file_list_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{0} +} + type FileList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -46,10 +102,14 @@ type FileList struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The added files. Additions []string `protobuf:"bytes,2,rep,name=additions" json:"additions,omitempty"` - // The deleted files. - Deletions []string `protobuf:"bytes,3,rep,name=deletions" json:"deletions,omitempty"` // The changed files. - Changes []*FileList `protobuf:"bytes,4,rep,name=changes" json:"changes,omitempty"` + Changes []*FileList `protobuf:"bytes,3,rep,name=changes" json:"changes,omitempty"` + // The deleted files. + Deletions []string `protobuf:"bytes,4,rep,name=deletions" json:"deletions,omitempty"` + // Count of files added/changed/deleted. + TotalDelta *uint32 `protobuf:"varint,5,opt,name=total_delta,json=totalDelta" json:"total_delta,omitempty"` + // Counts by extension. + Counts []*FileCount `protobuf:"bytes,6,rep,name=counts" json:"counts,omitempty"` } func (x *FileList) Reset() { @@ -98,6 +158,13 @@ func (x *FileList) GetAdditions() []string { return nil } +func (x *FileList) GetChanges() []*FileList { + if x != nil { + return x.Changes + } + return nil +} + func (x *FileList) GetDeletions() []string { if x != nil { return x.Deletions @@ -105,32 +172,135 @@ func (x *FileList) GetDeletions() []string { return nil } -func (x *FileList) GetChanges() []*FileList { +func (x *FileList) GetTotalDelta() uint32 { + if x != nil && x.TotalDelta != nil { + return *x.TotalDelta + } + return 0 +} + +func (x *FileList) GetCounts() []*FileCount { if x != nil { - return x.Changes + return x.Counts } return nil } +type FileCount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The file extension + Extension *string `protobuf:"bytes,1,opt,name=extension" json:"extension,omitempty"` + // Number of added files with this extension. + Additions *uint32 `protobuf:"varint,2,opt,name=additions" json:"additions,omitempty"` + // Number of modified files with this extension. + Modifications *uint32 `protobuf:"varint,3,opt,name=modifications" json:"modifications,omitempty"` + // Number of deleted files with this extension. + Deletions *uint32 `protobuf:"varint,4,opt,name=deletions" json:"deletions,omitempty"` +} + +func (x *FileCount) Reset() { + *x = FileCount{} + if protoimpl.UnsafeEnabled { + mi := &file_file_list_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCount) ProtoMessage() {} + +func (x *FileCount) ProtoReflect() protoreflect.Message { + mi := &file_file_list_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCount.ProtoReflect.Descriptor instead. +func (*FileCount) Descriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{1} +} + +func (x *FileCount) GetExtension() string { + if x != nil && x.Extension != nil { + return *x.Extension + } + return "" +} + +func (x *FileCount) GetAdditions() uint32 { + if x != nil && x.Additions != nil { + return *x.Additions + } + return 0 +} + +func (x *FileCount) GetModifications() uint32 { + if x != nil && x.Modifications != nil { + return *x.Modifications + } + return 0 +} + +func (x *FileCount) GetDeletions() uint32 { + if x != nil && x.Deletions != nil { + return *x.Deletions + } + return 0 +} + var File_file_list_proto protoreflect.FileDescriptor var file_file_list_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x9e, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, + 0x6f, 0x22, 0x82, 0x02, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x42, - 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, - 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, - 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x42, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, + 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x65, + 0x6c, 0x74, 0x61, 0x12, 0x41, 0x0a, 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, + 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x06, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, + 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, + 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x01, + 0x42, 0x3b, 0x5a, 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -145,17 +315,21 @@ func file_file_list_proto_rawDescGZIP() []byte { return file_file_list_proto_rawDescData } -var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_file_list_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_file_list_proto_goTypes = []interface{}{ - (*FileList)(nil), // 0: android.find_input_delta_proto.FileList + (FieldNumbers)(0), // 0: android.find_input_delta_proto.FieldNumbers + (*FileList)(nil), // 1: android.find_input_delta_proto.FileList + (*FileCount)(nil), // 2: android.find_input_delta_proto.FileCount } var file_file_list_proto_depIdxs = []int32{ - 0, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 1, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList + 2, // 1: android.find_input_delta_proto.FileList.counts:type_name -> android.find_input_delta_proto.FileCount + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_file_list_proto_init() } @@ -176,19 +350,32 @@ func file_file_list_proto_init() { return nil } } + file_file_list_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileCount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_file_list_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, + NumEnums: 1, + NumMessages: 2, NumExtensions: 0, NumServices: 0, }, GoTypes: file_file_list_proto_goTypes, DependencyIndexes: file_file_list_proto_depIdxs, + EnumInfos: file_file_list_proto_enumTypes, MessageInfos: file_file_list_proto_msgTypes, }.Build() File_file_list_proto = out.File diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.proto b/cmd/find_input_delta/find_input_delta_proto/file_list.proto index d7faca9c8..7180358c8 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.proto +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.proto @@ -15,7 +15,12 @@ syntax = "proto2"; package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +option go_package = "android/soong/cmd/find_input_delta/find_input_delta_proto"; + +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} message FileList { // The name of the file. @@ -26,9 +31,29 @@ message FileList { // The added files. repeated string additions = 2; + // The changed files. + repeated FileList changes = 3; + // The deleted files. - repeated string deletions = 3; + repeated string deletions = 4; - // The changed files. - repeated FileList changes = 4; + // Count of files added/changed/deleted. + optional uint32 total_delta = 5; + + // Counts by extension. + repeated FileCount counts = 6; +} + +message FileCount { + // The file extension + optional string extension = 1; + + // Number of added files with this extension. + optional uint32 additions = 2; + + // Number of modified files with this extension. + optional uint32 modifications = 3; + + // Number of deleted files with this extension. + optional uint32 deletions = 4; } diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go index 2229a324d..c5b048bb8 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: internal_state.proto -package proto +package find_input_delta_proto_internal import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -94,7 +94,7 @@ type PartialCompileInput struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The timestamp of the file in (Unix) nanoseconds. MtimeNsec *int64 `protobuf:"varint,2,opt,name=mtime_nsec,json=mtimeNsec" json:"mtime_nsec,omitempty"` - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. Hash *string `protobuf:"bytes,3,opt,name=hash" json:"hash,omitempty"` // Contents of the file, if the file was inspected (such as jar files, etc). Contents []*PartialCompileInput `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty"` @@ -164,29 +164,33 @@ var File_internal_state_proto protoreflect.FileDescriptor var file_internal_state_proto_rawDesc = []byte{ 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x27, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6c, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, - 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x54, - 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, - 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, - 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, - 0x69, 0x6c, 0x65, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x12, 0x4f, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, - 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, - 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, - 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x22, + 0x75, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x5d, 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, + 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, + 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, + 0x63, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x58, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, + 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x40, 0x5a, 0x3e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, + 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, } var ( @@ -203,12 +207,12 @@ func file_internal_state_proto_rawDescGZIP() []byte { var file_internal_state_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_internal_state_proto_goTypes = []interface{}{ - (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto.PartialCompileInputs - (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto.PartialCompileInput + (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto_internal.PartialCompileInputs + (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto_internal.PartialCompileInput } var file_internal_state_proto_depIdxs = []int32{ - 1, // 0: android.find_input_delta_proto.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto.PartialCompileInput - 1, // 1: android.find_input_delta_proto.PartialCompileInput.contents:type_name -> android.find_input_delta_proto.PartialCompileInput + 1, // 0: android.find_input_delta_proto_internal.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto_internal.PartialCompileInput + 1, // 1: android.find_input_delta_proto_internal.PartialCompileInput.contents:type_name -> android.find_input_delta_proto_internal.PartialCompileInput 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto index 113fc6434..54c90cc72 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto @@ -14,8 +14,8 @@ // limitations under the License. syntax = "proto2"; -package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +package android.find_input_delta_proto_internal; +option go_package = "android/soong/find_input_delta/find_input_delta_proto_internal"; // The state of all inputs. message PartialCompileInputs { @@ -31,7 +31,7 @@ message PartialCompileInput { // The timestamp of the file in (Unix) nanoseconds. optional int64 mtime_nsec = 2; - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. optional string hash = 3; // Contents of the file, if the file was inspected (such as jar files, etc). diff --git a/cmd/release_config/build_flag/main.go b/cmd/release_config/build_flag/main.go index 46efce7f7..5d103cc25 100644 --- a/cmd/release_config/build_flag/main.go +++ b/cmd/release_config/build_flag/main.go @@ -90,6 +90,9 @@ func MarshalFlagValue(config *rc_lib.ReleaseConfig, name string) (ret string, er if !ok { return "", fmt.Errorf("%s not found in %s", name, config.Name) } + if fa.Redacted { + return "==REDACTED==", nil + } return rc_lib.MarshalValue(fa.Value), nil } diff --git a/cmd/release_config/release_config_lib/flag_artifact.go b/cmd/release_config/release_config_lib/flag_artifact.go index bb321da87..cb13fdce4 100644 --- a/cmd/release_config/release_config_lib/flag_artifact.go +++ b/cmd/release_config/release_config_lib/flag_artifact.go @@ -84,8 +84,10 @@ func FlagArtifactsFactory(artifactsPath string) *FlagArtifacts { func (fas *FlagArtifacts) SortedFlagNames() []string { var names []string - for k, _ := range *fas { - names = append(names, k) + for k, v := range *fas { + if !v.Redacted { + names = append(names, k) + } } slices.Sort(names) return names @@ -183,15 +185,20 @@ func (src FlagArtifacts) Clone() (dst FlagArtifacts) { // error: any error encountered func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { name := *flagValue.proto.Name - fa.Traces = append(fa.Traces, &rc_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) - if flagValue.proto.GetRedacted() { - fa.Redacted = true + redacted := flagValue.proto.GetRedacted() + if redacted { + fa.Redact() + flagValue.proto.Value = fa.Value fmt.Printf("Redacting flag %s in %s\n", name, flagValue.path) - return nil + } else { + // If we are assigning a value, then the flag is no longer redacted. + fa.Redacted = false } + fa.Traces = append(fa.Traces, &rc_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) if fa.Value.GetObsolete() { return fmt.Errorf("Attempting to set obsolete flag %s. Trace=%v", name, fa.Traces) } + var newValue *rc_proto.Value switch val := flagValue.proto.Value.Val.(type) { case *rc_proto.Value_StringValue: @@ -213,6 +220,11 @@ func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { return nil } +func (fa *FlagArtifact) Redact() { + fa.Redacted = true + fa.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{StringValue: "*REDACTED*"}} +} + // Marshal the FlagArtifact into a flag_artifact message. func (fa *FlagArtifact) Marshal() (*rc_proto.FlagArtifact, error) { if fa.Redacted { diff --git a/cmd/release_config/release_config_lib/release_config.go b/cmd/release_config/release_config_lib/release_config.go index ee71336c5..3c834cc1d 100644 --- a/cmd/release_config/release_config_lib/release_config.go +++ b/cmd/release_config/release_config_lib/release_config.go @@ -21,7 +21,6 @@ import ( "path/filepath" "regexp" "slices" - "sort" "strings" rc_proto "android/soong/cmd/release_config/release_config_proto" @@ -107,6 +106,9 @@ func (config *ReleaseConfig) InheritConfig(iConfig *ReleaseConfig) error { if !ok { return fmt.Errorf("Could not inherit flag %s from %s", name, iConfig.Name) } + if fa.Redacted { + myFa.Redact() + } if name == "RELEASE_ACONFIG_VALUE_SETS" { // If there is a value assigned, add the trace. if len(fa.Value.GetStringValue()) > 0 { @@ -166,6 +168,13 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro } myInherits = append(myInherits, inherit) myInheritsSet[inherit] = true + // TODO: there are some configs that rely on vgsbr being + // present on branches where it isn't. Once the broken configs + // are fixed, we can be more strict. In the meantime, they + // will wind up inheriting `trunk_stable` instead of the + // non-existent (alias) that they reference today. Once fixed, + // this becomes: + // iConfig, err := configs.GetReleaseConfigStrict(inherit) iConfig, err := configs.GetReleaseConfig(inherit) if err != nil { return err @@ -261,9 +270,6 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro if err := fa.UpdateValue(*value); err != nil { return err } - if fa.Redacted { - delete(config.FlagArtifacts, name) - } } } // Now remove any duplicates from the actual value of RELEASE_ACONFIG_VALUE_SETS @@ -313,6 +319,10 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro if err != nil { return err } + // Redacted flags return nil when rendered. + if artifact == nil { + continue + } for _, container := range v.FlagDeclaration.Containers { if _, ok := config.PartitionBuildFlags[container]; !ok { config.PartitionBuildFlags[container] = &rc_proto.FlagArtifacts{} @@ -325,12 +335,7 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro OtherNames: config.OtherNames, Flags: func() []*rc_proto.FlagArtifact { ret := []*rc_proto.FlagArtifact{} - flagNames := []string{} - for k := range config.FlagArtifacts { - flagNames = append(flagNames, k) - } - sort.Strings(flagNames) - for _, flagName := range flagNames { + for _, flagName := range config.FlagArtifacts.SortedFlagNames() { flag := config.FlagArtifacts[flagName] ret = append(ret, &rc_proto.FlagArtifact{ FlagDeclaration: flag.FlagDeclaration, @@ -365,7 +370,7 @@ func (config *ReleaseConfig) WriteMakefile(outFile, targetRelease string, config } } for _, rcName := range extraAconfigReleaseConfigs { - rc, err := configs.GetReleaseConfig(rcName) + rc, err := configs.GetReleaseConfigStrict(rcName) if err != nil { return err } diff --git a/cmd/release_config/release_config_lib/release_configs.go b/cmd/release_config/release_config_lib/release_configs.go index 831ec02f2..f947b7f1f 100644 --- a/cmd/release_config/release_config_lib/release_configs.go +++ b/cmd/release_config/release_config_lib/release_configs.go @@ -410,6 +410,14 @@ func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex } func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) { + return configs.getReleaseConfig(name, configs.allowMissing) +} + +func (configs *ReleaseConfigs) GetReleaseConfigStrict(name string) (*ReleaseConfig, error) { + return configs.getReleaseConfig(name, false) +} + +func (configs *ReleaseConfigs) getReleaseConfig(name string, allow_missing bool) (*ReleaseConfig, error) { trace := []string{name} for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] { name = *target @@ -418,7 +426,7 @@ func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, er if config, ok := configs.ReleaseConfigs[name]; ok { return config, nil } - if configs.allowMissing { + if allow_missing { if config, ok := configs.ReleaseConfigs["trunk_staging"]; ok { return config, nil } @@ -467,7 +475,7 @@ func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) erro dirName := filepath.Dir(contrib.path) for k, names := range contrib.FlagValueDirs { for _, rcName := range names { - if config, err := configs.GetReleaseConfig(rcName); err == nil { + if config, err := configs.getReleaseConfig(rcName, false); err == nil { rcPath := filepath.Join(dirName, "release_configs", fmt.Sprintf("%s.textproto", config.Name)) if _, err := os.Stat(rcPath); err != nil { errors = append(errors, fmt.Sprintf("%s exists but %s does not contribute to %s", diff --git a/filesystem/bootimg.go b/filesystem/bootimg.go index c9bd61788..226d95cbf 100644 --- a/filesystem/bootimg.go +++ b/filesystem/bootimg.go @@ -26,19 +26,19 @@ import ( ) func init() { - android.RegisterModuleType("bootimg", bootimgFactory) + android.RegisterModuleType("bootimg", BootimgFactory) } type bootimg struct { android.ModuleBase - properties bootimgProperties + properties BootimgProperties output android.Path installDir android.InstallPath } -type bootimgProperties struct { +type BootimgProperties struct { // Set the name of the output. Defaults to <module_name>.img. Stem *string @@ -82,7 +82,7 @@ type bootimgProperties struct { } // bootimg is the image for the boot partition. It consists of header, kernel, ramdisk, and dtb. -func bootimgFactory() android.Module { +func BootimgFactory() android.Module { module := &bootimg{} module.AddProperties(&module.properties) android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index f325d96ef..7c342cc8c 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -666,7 +666,9 @@ func TestUseSharedVariationOfNativeLib(t *testing.T) { partition := result.ModuleForTests("myfilesystem", "android_common") fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) - android.AssertDeepEquals(t, "cc_library listed in deps", "lib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo.so\nlib64/libm.so\n", fileList) + android.AssertDeepEquals(t, "cc_library listed in deps", + "lib64/bootstrap/libc.so\nlib64/bootstrap/libdl.so\nlib64/bootstrap/libm.so\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo.so\nlib64/libm.so\n", + fileList) } // binfoo1 overrides binbar. transitive deps of binbar should not be installed. @@ -701,7 +703,9 @@ cc_library { partition := result.ModuleForTests("myfilesystem", "android_common") fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) - android.AssertDeepEquals(t, "Shared library dep of overridden binary should not be installed", fileList, "bin/binfoo1\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo2.so\nlib64/libm.so\n") + android.AssertDeepEquals(t, "Shared library dep of overridden binary should not be installed", + "bin/binfoo1\nlib64/bootstrap/libc.so\nlib64/bootstrap/libdl.so\nlib64/bootstrap/libm.so\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo2.so\nlib64/libm.so\n", + fileList) } func TestInstallLinkerConfigFile(t *testing.T) { diff --git a/fsgen/Android.bp b/fsgen/Android.bp index 8cd7518cb..a0225811e 100644 --- a/fsgen/Android.bp +++ b/fsgen/Android.bp @@ -13,6 +13,7 @@ bootstrap_go_package { "soong-kernel", ], srcs: [ + "boot_imgs.go", "filesystem_creator.go", "fsgen_mutators.go", "prebuilt_etc_modules_gen.go", diff --git a/fsgen/boot_imgs.go b/fsgen/boot_imgs.go new file mode 100644 index 000000000..66d910750 --- /dev/null +++ b/fsgen/boot_imgs.go @@ -0,0 +1,78 @@ +package fsgen + +import ( + "android/soong/android" + "android/soong/filesystem" + "path/filepath" + + "github.com/google/blueprint/proptools" +) + +func createBootImage(ctx android.LoadHookContext) bool { + partitionVariables := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + + if partitionVariables.TargetKernelPath == "" { + // There are potentially code paths that don't set TARGET_KERNEL_PATH + return false + } + + kernelDir := filepath.Dir(partitionVariables.TargetKernelPath) + kernelBase := filepath.Base(partitionVariables.TargetKernelPath) + kernelFilegroupName := generatedModuleName(ctx.Config(), "kernel") + + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + kernelDir, + &struct { + Name *string + Srcs []string + Visibility []string + }{ + Name: proptools.StringPtr(kernelFilegroupName), + Srcs: []string{kernelBase}, + Visibility: []string{"//visibility:public"}, + }, + ) + + bootImageName := generatedModuleNameForPartition(ctx.Config(), "boot") + + ctx.CreateModule( + filesystem.BootimgFactory, + &filesystem.BootimgProperties{ + Kernel_prebuilt: proptools.StringPtr(":" + kernelFilegroupName), + Ramdisk_module: proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "ramdisk")), + Header_version: proptools.StringPtr(partitionVariables.BoardBootHeaderVersion), + }, + &struct { + Name *string + }{ + Name: proptools.StringPtr(bootImageName), + }, + ) + return true +} + +// Returns the equivalent of the BUILDING_BOOT_IMAGE variable in make. Derived from this logic: +// https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/board_config.mk;l=458;drc=5b55f926830963c02ab1d2d91e46442f04ba3af0 +func buildingBootImage(partitionVars android.PartitionVariables) bool { + if partitionVars.BoardUsesRecoveryAsBoot { + return false + } + + if partitionVars.ProductBuildBootImage { + return true + } + + if len(partitionVars.BoardPrebuiltBootimage) > 0 { + return false + } + + if len(partitionVars.BoardBootimagePartitionSize) > 0 { + return true + } + + // TODO: return true if BOARD_KERNEL_BINARIES is set and has a *_BOOTIMAGE_PARTITION_SIZE + // variable. However, I don't think BOARD_KERNEL_BINARIES is ever set in practice. + + return false +} diff --git a/fsgen/filesystem_creator.go b/fsgen/filesystem_creator.go index c0c18d84b..7daefcb6d 100644 --- a/fsgen/filesystem_creator.go +++ b/fsgen/filesystem_creator.go @@ -47,6 +47,8 @@ type filesystemCreatorProps struct { Vbmeta_module_names []string `blueprint:"mutated"` Vbmeta_partition_names []string `blueprint:"mutated"` + + Boot_image string `blueprint:"mutated" android:"path_device_first"` } type filesystemCreator struct { @@ -64,12 +66,45 @@ func filesystemCreatorFactory() android.Module { generatedPrebuiltEtcModuleNames := createPrebuiltEtcModules(ctx) avbpubkeyGenerated := createAvbpubkeyModule(ctx) createFsGenState(ctx, generatedPrebuiltEtcModuleNames, avbpubkeyGenerated) + module.createAvbKeyFilegroups(ctx) module.createInternalModules(ctx) }) return module } +func generatedPartitions(ctx android.LoadHookContext) []string { + generatedPartitions := []string{"system"} + if ctx.DeviceConfig().SystemExtPath() == "system_ext" { + generatedPartitions = append(generatedPartitions, "system_ext") + } + if ctx.DeviceConfig().BuildingVendorImage() && ctx.DeviceConfig().VendorPath() == "vendor" { + generatedPartitions = append(generatedPartitions, "vendor") + } + if ctx.DeviceConfig().BuildingProductImage() && ctx.DeviceConfig().ProductPath() == "product" { + generatedPartitions = append(generatedPartitions, "product") + } + if ctx.DeviceConfig().BuildingOdmImage() && ctx.DeviceConfig().OdmPath() == "odm" { + generatedPartitions = append(generatedPartitions, "odm") + } + if ctx.DeviceConfig().BuildingUserdataImage() && ctx.DeviceConfig().UserdataPath() == "data" { + generatedPartitions = append(generatedPartitions, "userdata") + } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingSystemDlkmImage { + generatedPartitions = append(generatedPartitions, "system_dlkm") + } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingVendorDlkmImage { + generatedPartitions = append(generatedPartitions, "vendor_dlkm") + } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingOdmDlkmImage { + generatedPartitions = append(generatedPartitions, "odm_dlkm") + } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingRamdiskImage { + generatedPartitions = append(generatedPartitions, "ramdisk") + } + return generatedPartitions +} + func (f *filesystemCreator) createInternalModules(ctx android.LoadHookContext) { soongGeneratedPartitions := generatedPartitions(ctx) finalSoongGeneratedPartitions := make([]string, 0, len(soongGeneratedPartitions)) @@ -82,6 +117,14 @@ func (f *filesystemCreator) createInternalModules(ctx android.LoadHookContext) { } } + if buildingBootImage(ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse) { + if createBootImage(ctx) { + f.properties.Boot_image = ":" + generatedModuleNameForPartition(ctx.Config(), "boot") + } else { + f.properties.Unsupported_partition_types = append(f.properties.Unsupported_partition_types, "boot") + } + } + for _, x := range createVbmetaPartitions(ctx, finalSoongGeneratedPartitions) { f.properties.Vbmeta_module_names = append(f.properties.Vbmeta_module_names, x.moduleName) f.properties.Vbmeta_partition_names = append(f.properties.Vbmeta_partition_names, x.partitionName) @@ -251,6 +294,51 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti return true } +// Creates filegroups for the files specified in BOARD_(partition_)AVB_KEY_PATH +func (f *filesystemCreator) createAvbKeyFilegroups(ctx android.LoadHookContext) { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + var files []string + + if len(partitionVars.BoardAvbKeyPath) > 0 { + files = append(files, partitionVars.BoardAvbKeyPath) + } + for _, partition := range android.SortedKeys(partitionVars.PartitionQualifiedVariables) { + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partition] + if len(specificPartitionVars.BoardAvbKeyPath) > 0 { + files = append(files, specificPartitionVars.BoardAvbKeyPath) + } + } + + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + for _, file := range files { + if _, ok := fsGenState.avbKeyFilegroups[file]; ok { + continue + } + if file == "external/avb/test/data/testkey_rsa4096.pem" { + // There already exists a checked-in filegroup for this commonly-used key, just use that + fsGenState.avbKeyFilegroups[file] = "avb_testkey_rsa4096" + continue + } + dir := filepath.Dir(file) + base := filepath.Base(file) + name := fmt.Sprintf("avb_key_%x", strings.ReplaceAll(file, "/", "_")) + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + dir, + &struct { + Name *string + Srcs []string + Visibility []string + }{ + Name: proptools.StringPtr(name), + Srcs: []string{base}, + Visibility: []string{"//visibility:public"}, + }, + ) + fsGenState.avbKeyFilegroups[file] = name + } +} + // createPrebuiltKernelModules creates `prebuilt_kernel_modules`. These modules will be added to deps of the // autogenerated *_dlkm filsystem modules. Each _dlkm partition should have a single prebuilt_kernel_modules dependency. // This ensures that the depmod artifacts (modules.* installed in /lib/modules/) are generated with a complete view. @@ -416,21 +504,41 @@ func generateBaseProps(namePtr *string) *filesystemBaseProperty { } func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*filesystem.FilesystemProperties, bool) { + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) fsProps := &filesystem.FilesystemProperties{} partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse - var specificPartitionVars android.PartitionQualifiedVariablesType var boardAvbEnable bool + var boardAvbKeyPath string + var boardAvbAlgorithm string + var boardAvbRollbackIndex string var fsType string if strings.Contains(partitionType, "ramdisk") { fsType = "compressed_cpio" } else { - specificPartitionVars = partitionVars.PartitionQualifiedVariables[partitionType] - boardAvbEnable = partitionVars.BoardAvbEnable + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partitionType] fsType = specificPartitionVars.BoardFileSystemType + boardAvbEnable = partitionVars.BoardAvbEnable + boardAvbKeyPath = specificPartitionVars.BoardAvbKeyPath + boardAvbAlgorithm = specificPartitionVars.BoardAvbAlgorithm + boardAvbRollbackIndex = specificPartitionVars.BoardAvbRollbackIndex + if boardAvbEnable { + if boardAvbKeyPath == "" { + boardAvbKeyPath = partitionVars.BoardAvbKeyPath + } + if boardAvbAlgorithm == "" { + boardAvbAlgorithm = partitionVars.BoardAvbAlgorithm + } + if boardAvbRollbackIndex == "" { + boardAvbRollbackIndex = partitionVars.BoardAvbRollbackIndex + } + } + if fsType == "" { + fsType = "ext4" //default + } } - if fsType == "" { - fsType = "ext4" //default + if boardAvbKeyPath != "" { + boardAvbKeyPath = ":" + fsGenState.avbKeyFilegroups[boardAvbKeyPath] } fsProps.Type = proptools.StringPtr(fsType) @@ -446,11 +554,11 @@ func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*fil // BOARD_AVB_ENABLE fsProps.Use_avb = proptools.BoolPtr(boardAvbEnable) // BOARD_AVB_KEY_PATH - fsProps.Avb_private_key = proptools.StringPtr(specificPartitionVars.BoardAvbKeyPath) + fsProps.Avb_private_key = proptools.StringPtr(boardAvbKeyPath) // BOARD_AVB_ALGORITHM - fsProps.Avb_algorithm = proptools.StringPtr(specificPartitionVars.BoardAvbAlgorithm) + fsProps.Avb_algorithm = proptools.StringPtr(boardAvbAlgorithm) // BOARD_AVB_SYSTEM_ROLLBACK_INDEX - if rollbackIndex, err := strconv.ParseInt(specificPartitionVars.BoardAvbRollbackIndex, 10, 64); err == nil { + if rollbackIndex, err := strconv.ParseInt(boardAvbRollbackIndex, 10, 64); err == nil { fsProps.Rollback_index = proptools.Int64Ptr(rollbackIndex) } @@ -477,7 +585,7 @@ func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*fil return fsProps, true } -func (f *filesystemCreator) createDiffTest(ctx android.ModuleContext, partitionType string) android.Path { +func (f *filesystemCreator) createFileListDiffTest(ctx android.ModuleContext, partitionType string) android.Path { partitionModuleName := generatedModuleNameForPartition(ctx.Config(), partitionType) systemImage := ctx.GetDirectDepWithTag(partitionModuleName, generatedFilesystemDepTag) filesystemInfo, ok := android.OtherModuleProvider(ctx, systemImage, filesystem.FilesystemProvider) @@ -522,13 +630,17 @@ func createVbmetaDiff(ctx android.ModuleContext, vbmetaModuleName string, vbmeta makeVbmetaFile := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/%s.img", ctx.Config().DeviceName(), vbmetaPartitionName)) diffTestResultFile := android.PathForModuleOut(ctx, fmt.Sprintf("diff_test_%s.txt", vbmetaModuleName)) + createDiffTest(ctx, diffTestResultFile, soongVbMetaFile, makeVbmetaFile) + return diffTestResultFile +} + +func createDiffTest(ctx android.ModuleContext, diffTestResultFile android.WritablePath, file1 android.Path, file2 android.Path) { builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("diff"). - Input(soongVbMetaFile). - Input(makeVbmetaFile) + Input(file1). + Input(file2) builder.Command().Text("touch").Output(diffTestResultFile) - builder.Build(vbmetaModuleName+" diff test", vbmetaModuleName+" diff test") - return diffTestResultFile + builder.Build("diff test "+diffTestResultFile.String(), "diff test") } type systemImageDepTagType struct { @@ -565,7 +677,7 @@ func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContex var diffTestFiles []android.Path for _, partitionType := range f.properties.Generated_partition_types { - diffTestFile := f.createDiffTest(ctx, partitionType) + diffTestFile := f.createFileListDiffTest(ctx, partitionType) diffTestFiles = append(diffTestFiles, diffTestFile) ctx.Phony(fmt.Sprintf("soong_generated_%s_filesystem_test", partitionType), diffTestFile) } @@ -579,6 +691,14 @@ func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContex diffTestFiles = append(diffTestFiles, diffTestFile) ctx.Phony(fmt.Sprintf("soong_generated_%s_filesystem_test", f.properties.Vbmeta_partition_names[i]), diffTestFile) } + if f.properties.Boot_image != "" { + diffTestFile := android.PathForModuleOut(ctx, "boot_diff_test.txt") + soongBootImg := android.PathForModuleSrc(ctx, f.properties.Boot_image) + makeBootImage := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/boot.img", ctx.Config().DeviceName())) + createDiffTest(ctx, diffTestFile, soongBootImg, makeBootImage) + diffTestFiles = append(diffTestFiles, diffTestFile) + ctx.Phony("soong_generated_boot_filesystem_test", diffTestFile) + } ctx.Phony("soong_generated_filesystem_tests", diffTestFiles...) } diff --git a/fsgen/filesystem_creator_test.go b/fsgen/filesystem_creator_test.go index 6111a4117..fe4a40325 100644 --- a/fsgen/filesystem_creator_test.go +++ b/fsgen/filesystem_creator_test.go @@ -47,6 +47,12 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) { }), android.FixtureMergeMockFs(android.MockFS{ "external/avb/test/data/testkey_rsa4096.pem": nil, + "external/avb/test/Android.bp": []byte(` + filegroup { + name: "avb_testkey_rsa4096", + srcs: ["data/testkey_rsa4096.pem"], + } + `), "build/soong/fsgen/Android.bp": []byte(` soong_filesystem_creator { name: "foo", @@ -66,8 +72,8 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) { ) android.AssertStringEquals( t, - "Property expected to match the product variable 'BOARD_AVB_KEY_PATH'", - "external/avb/test/data/testkey_rsa4096.pem", + "Property the avb_private_key property to be set to the existing filegroup", + ":avb_testkey_rsa4096", proptools.String(fooSystem.FsProps().Avb_private_key), ) android.AssertStringEquals( diff --git a/fsgen/fsgen_mutators.go b/fsgen/fsgen_mutators.go index e0e103a98..0d18660d5 100644 --- a/fsgen/fsgen_mutators.go +++ b/fsgen/fsgen_mutators.go @@ -68,6 +68,8 @@ type FsGenState struct { moduleToInstallationProps map[string]installationProperties // List of prebuilt_* modules that are autogenerated. generatedPrebuiltEtcModuleNames []string + // Mapping from a path to an avb key to the name of a filegroup module that contains it + avbKeyFilegroups map[string]string } type installationProperties struct { @@ -82,36 +84,6 @@ func defaultDepCandidateProps(config android.Config) *depCandidateProps { } } -func generatedPartitions(ctx android.LoadHookContext) []string { - generatedPartitions := []string{"system", "ramdisk"} - if ctx.DeviceConfig().SystemExtPath() == "system_ext" { - generatedPartitions = append(generatedPartitions, "system_ext") - } - if ctx.DeviceConfig().BuildingVendorImage() && ctx.DeviceConfig().VendorPath() == "vendor" { - generatedPartitions = append(generatedPartitions, "vendor") - } - if ctx.DeviceConfig().BuildingProductImage() && ctx.DeviceConfig().ProductPath() == "product" { - generatedPartitions = append(generatedPartitions, "product") - } - if ctx.DeviceConfig().BuildingOdmImage() && ctx.DeviceConfig().OdmPath() == "odm" { - generatedPartitions = append(generatedPartitions, "odm") - } - if ctx.DeviceConfig().BuildingUserdataImage() && ctx.DeviceConfig().UserdataPath() == "data" { - generatedPartitions = append(generatedPartitions, "userdata") - } - if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingSystemDlkmImage { - generatedPartitions = append(generatedPartitions, "system_dlkm") - } - if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingVendorDlkmImage { - generatedPartitions = append(generatedPartitions, "vendor_dlkm") - } - if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingOdmDlkmImage { - generatedPartitions = append(generatedPartitions, "odm_dlkm") - } - - return generatedPartitions -} - func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNames []string, avbpubkeyGenerated bool) *FsGenState { return ctx.Config().Once(fsGenStateOnceKey, func() interface{} { partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse @@ -179,6 +151,7 @@ func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNam fsDepsMutex: sync.Mutex{}, moduleToInstallationProps: map[string]installationProperties{}, generatedPrebuiltEtcModuleNames: generatedPrebuiltEtcModuleNames, + avbKeyFilegroups: map[string]string{}, } if avbpubkeyGenerated { @@ -386,6 +359,7 @@ func generateDepStruct(deps map[string]*depCandidateProps, highPriorityDeps []st depsStruct.Multilib.Prefer32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Prefer32.Deps) depsStruct.Multilib.Both.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Both.Deps) depsStruct.Multilib.Common.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Common.Deps) + depsStruct.High_priority_deps = android.SortedUniqueStrings(depsStruct.High_priority_deps) return &depsStruct } diff --git a/fsgen/vbmeta_partitions.go b/fsgen/vbmeta_partitions.go index b7fff687d..11c57590c 100644 --- a/fsgen/vbmeta_partitions.go +++ b/fsgen/vbmeta_partitions.go @@ -19,6 +19,7 @@ import ( "android/soong/filesystem" "slices" "strconv" + "strings" "github.com/google/blueprint/proptools" ) @@ -153,8 +154,10 @@ func createVbmetaPartitions(ctx android.LoadHookContext, generatedPartitionTypes // Already handled by a chained vbmeta partition continue } - if partitionType == "ramdisk" { + if strings.Contains(partitionType, "ramdisk") || strings.Contains(partitionType, "boot") { // ramdisk is never signed with avb information + // boot partitions just have the avb footer, and don't have a corresponding vbmeta + // partition. continue } partitionModules = append(partitionModules, generatedModuleNameForPartition(ctx.Config(), partitionType)) diff --git a/java/base.go b/java/base.go index 8dad2d9ad..427c28c52 100644 --- a/java/base.go +++ b/java/base.go @@ -714,10 +714,10 @@ func (j *Module) provideHiddenAPIPropertyInfo(ctx android.ModuleContext) { // helper method for java modules to set OutputFilesProvider func setOutputFiles(ctx android.ModuleContext, m Module) { - ctx.SetOutputFiles(append(android.Paths{m.outputFile}, m.extraOutputFiles...), "") - ctx.SetOutputFiles(android.Paths{m.outputFile}, android.DefaultDistTag) - ctx.SetOutputFiles(android.Paths{m.implementationAndResourcesJar}, ".jar") - ctx.SetOutputFiles(android.Paths{m.headerJarFile}, ".hjar") + ctx.SetOutputFiles(append(android.PathsIfNonNil(m.outputFile), m.extraOutputFiles...), "") + ctx.SetOutputFiles(android.PathsIfNonNil(m.outputFile), android.DefaultDistTag) + ctx.SetOutputFiles(android.PathsIfNonNil(m.implementationAndResourcesJar), ".jar") + ctx.SetOutputFiles(android.PathsIfNonNil(m.headerJarFile), ".hjar") if m.dexer.proguardDictionary.Valid() { ctx.SetOutputFiles(android.Paths{m.dexer.proguardDictionary.Path()}, ".proguard_map") } @@ -1735,7 +1735,22 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath completeStaticLibsImplementationJarsToCombine := completeStaticLibsImplementationJars - if j.shouldInstrument(ctx) { + // Enable dex compilation for the APEX variants, unless it is disabled explicitly + compileDex := Bool(j.dexProperties.Compile_dex) + apexInfo, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) + if j.DirectlyInAnyApex() && !apexInfo.IsForPlatform() { + if j.dexProperties.Compile_dex == nil { + compileDex = true + } + if j.deviceProperties.Hostdex == nil { + j.deviceProperties.Hostdex = proptools.BoolPtr(true) + } + } + if Bool(j.properties.Installable) { + compileDex = true + } + + if j.shouldInstrument(ctx) && (!ctx.Device() || compileDex) { instrumentedOutputFile := j.instrument(ctx, flags, outputFile, jarName, specs) completeStaticLibsImplementationJarsToCombine = depset.New(depset.PREORDER, android.Paths{instrumentedOutputFile}, nil) outputFile = instrumentedOutputFile @@ -1764,19 +1779,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath j.implementationAndResourcesJar = outputFile - // Enable dex compilation for the APEX variants, unless it is disabled explicitly - compileDex := j.dexProperties.Compile_dex - apexInfo, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) - if j.DirectlyInAnyApex() && !apexInfo.IsForPlatform() { - if compileDex == nil { - compileDex = proptools.BoolPtr(true) - } - if j.deviceProperties.Hostdex == nil { - j.deviceProperties.Hostdex = proptools.BoolPtr(true) - } - } - - if ctx.Device() && (Bool(j.properties.Installable) || Bool(compileDex)) { + if ctx.Device() && compileDex { if j.hasCode(ctx) { if j.shouldInstrumentStatic(ctx) { j.dexer.extraProguardFlagsFiles = append(j.dexer.extraProguardFlagsFiles, diff --git a/java/dex.go b/java/dex.go index dea71f538..983377ead 100644 --- a/java/dex.go +++ b/java/dex.go @@ -295,7 +295,7 @@ func (d *dexer) d8Flags(ctx android.ModuleContext, dexParams *compileDexParams) return d8Flags, d8Deps, artProfileOutput } -func (d *dexer) r8Flags(ctx android.ModuleContext, dexParams *compileDexParams) (r8Flags []string, r8Deps android.Paths, artProfileOutput *android.OutputPath) { +func (d *dexer) r8Flags(ctx android.ModuleContext, dexParams *compileDexParams, debugMode bool) (r8Flags []string, r8Deps android.Paths, artProfileOutput *android.OutputPath) { flags := dexParams.flags opt := d.dexProperties.Optimize @@ -363,7 +363,9 @@ func (d *dexer) r8Flags(ctx android.ModuleContext, dexParams *compileDexParams) r8Flags = append(r8Flags, "--force-proguard-compatibility") } - if Bool(opt.Optimize) || Bool(opt.Obfuscate) { + // Avoid unnecessary stack frame noise by only injecting source map ids for non-debug + // optimized or obfuscated targets. + if (Bool(opt.Optimize) || Bool(opt.Obfuscate)) && !debugMode { // TODO(b/213833843): Allow configuration of the prefix via a build variable. var sourceFilePrefix = "go/retraceme " var sourceFileTemplate = "\"" + sourceFilePrefix + "%MAP_ID\"" @@ -483,7 +485,8 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam proguardUsageZip, proguardConfiguration, } - r8Flags, r8Deps, r8ArtProfileOutputPath := d.r8Flags(ctx, dexParams) + debugMode := android.InList("--debug", commonFlags) + r8Flags, r8Deps, r8ArtProfileOutputPath := d.r8Flags(ctx, dexParams, debugMode) rule := r8 args := map[string]string{ "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), diff --git a/java/droiddoc.go b/java/droiddoc.go index 82713920d..2dda72b0e 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -578,7 +578,6 @@ func (j *Javadoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { rule.Build("javadoc", "javadoc") - ctx.SetOutputFiles(android.Paths{j.stubsSrcJar}, "") ctx.SetOutputFiles(android.Paths{j.docZip}, ".docs.zip") } diff --git a/java/droidstubs.go b/java/droidstubs.go index cf3e21925..bc2652797 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -1011,7 +1011,7 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro cmd.FlagWithOutput("--update-baseline:api-lint ", updatedBaselineOutput) msg += fmt.Sprintf(``+ - `3. FOR LSC ONLY: You can update the baseline by executing\n` + + `3. FOR LSC ONLY: You can update the baseline by executing\n`+ ` the following command:\n`+ ` (cd $ANDROID_BUILD_TOP && cp \\\n`+ ` "%s" \\\n`+ @@ -1374,7 +1374,7 @@ func (d *Droidstubs) setOutputFiles(ctx android.ModuleContext) { for _, stubType := range android.SortedKeys(stubsTypeToPrefix) { tagWithPrefix := stubsTypeToPrefix[stubType] + tag outputFile, err := tagToOutputFileFunc[tag](stubType) - if err == nil { + if err == nil && outputFile != nil { ctx.SetOutputFiles(android.Paths{outputFile}, tagWithPrefix) } } diff --git a/java/java_test.go b/java/java_test.go index 54eb3e14e..d415679bd 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -3050,6 +3050,7 @@ func TestCoverage(t *testing.T) { java_library { name: "android.car", srcs: ["android.car.java"], + installable: true, } `) diff --git a/ui/build/soong.go b/ui/build/soong.go index e6d01dd1d..0963f76b7 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -433,13 +433,13 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) } } -func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { +func updateSymlinks(ctx Context, dir, prevCWD, cwd string, updateSemaphore chan struct{}) error { defer symlinkWg.Done() visit := func(path string, d fs.DirEntry, err error) error { if d.IsDir() && path != dir { symlinkWg.Add(1) - go updateSymlinks(ctx, path, prevCWD, cwd) + go updateSymlinks(ctx, path, prevCWD, cwd, updateSemaphore) return filepath.SkipDir } f, err := d.Info() @@ -470,12 +470,27 @@ func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { return nil } + <-updateSemaphore + defer func() { updateSemaphore <- struct{}{} }() if err := filepath.WalkDir(dir, visit); err != nil { return err } return nil } +// b/376466642: If the concurrency of updateSymlinks is unbounded, Go's runtime spawns a +// theoretically unbounded number of threads to handle blocking syscalls. This causes the runtime to +// panic due to hitting thread limits in rare cases. Limiting to GOMAXPROCS concurrent symlink +// updates should make this a non-issue. +func newUpdateSemaphore() chan struct{} { + numPermits := runtime.GOMAXPROCS(0) + c := make(chan struct{}, numPermits) + for i := 0; i < numPermits; i++ { + c <- struct{}{} + } + return c +} + func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { cwd, err := os.Getwd() if err != nil { @@ -508,7 +523,7 @@ func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { } symlinkWg.Add(1) - if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil { + if err := updateSymlinks(ctx, outDir, prevCWD, cwd, newUpdateSemaphore()); err != nil { return err } symlinkWg.Wait() diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 77871fc09..591e3cca6 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -26,6 +26,7 @@ bootstrap_go_package { "soong-ui-metrics_proto", "soong-ui-mk_metrics_proto", "soong-shared", + "soong-ui-metrics_combined_proto", ], srcs: [ "hostinfo.go", @@ -63,6 +64,19 @@ bootstrap_go_package { } bootstrap_go_package { + name: "soong-ui-metrics_combined_proto", + pkgPath: "android/soong/ui/metrics/combined_metrics_proto", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-find_input_delta-proto", + ], + srcs: [ + "metrics_proto/metrics.pb.go", + ], +} + +bootstrap_go_package { name: "soong-ui-metrics_upload_proto", pkgPath: "android/soong/ui/metrics/upload_proto", deps: [ diff --git a/ui/metrics/metrics_proto/combined_metrics.pb.go b/ui/metrics/metrics_proto/combined_metrics.pb.go new file mode 100644 index 000000000..f49d64d54 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics.pb.go @@ -0,0 +1,239 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: combined_metrics.proto + +package metrics_proto + +import ( + find_input_delta_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// These field numbers are also found in the inner message declarations. +// We verify that the values are the same, and that every enum value is checked +// in combined_metrics_test.go. +// Do not change this enum without also updating: +// - the submessage's .proto file +// - combined_metrics_test.go +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_combined_metrics_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_combined_metrics_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_combined_metrics_proto_rawDescGZIP(), []int{0} +} + +type SoongCombinedMetrics struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // cmd/find_input_delta/find_input_delta_proto.FileList + FileList *find_input_delta_proto.FileList `protobuf:"bytes,1,opt,name=file_list,json=fileList" json:"file_list,omitempty"` +} + +func (x *SoongCombinedMetrics) Reset() { + *x = SoongCombinedMetrics{} + if protoimpl.UnsafeEnabled { + mi := &file_combined_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SoongCombinedMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SoongCombinedMetrics) ProtoMessage() {} + +func (x *SoongCombinedMetrics) ProtoReflect() protoreflect.Message { + mi := &file_combined_metrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SoongCombinedMetrics.ProtoReflect.Descriptor instead. +func (*SoongCombinedMetrics) Descriptor() ([]byte, []int) { + return file_combined_metrics_proto_rawDescGZIP(), []int{0} +} + +func (x *SoongCombinedMetrics) GetFileList() *find_input_delta_proto.FileList { + if x != nil { + return x.FileList + } + return nil +} + +var File_combined_metrics_proto protoreflect.FileDescriptor + +var file_combined_metrics_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x3b, 0x63, + 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, + 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, + 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5d, 0x0a, 0x14, 0x53, 0x6f, + 0x6f, 0x6e, 0x67, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x12, 0x45, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, + 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, + 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, + 0x49, 0x53, 0x54, 0x10, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, +} + +var ( + file_combined_metrics_proto_rawDescOnce sync.Once + file_combined_metrics_proto_rawDescData = file_combined_metrics_proto_rawDesc +) + +func file_combined_metrics_proto_rawDescGZIP() []byte { + file_combined_metrics_proto_rawDescOnce.Do(func() { + file_combined_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_combined_metrics_proto_rawDescData) + }) + return file_combined_metrics_proto_rawDescData +} + +var file_combined_metrics_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_combined_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_combined_metrics_proto_goTypes = []interface{}{ + (FieldNumbers)(0), // 0: soong_build_metrics.FieldNumbers + (*SoongCombinedMetrics)(nil), // 1: soong_build_metrics.SoongCombinedMetrics + (*find_input_delta_proto.FileList)(nil), // 2: android.find_input_delta_proto.FileList +} +var file_combined_metrics_proto_depIdxs = []int32{ + 2, // 0: soong_build_metrics.SoongCombinedMetrics.file_list:type_name -> android.find_input_delta_proto.FileList + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_combined_metrics_proto_init() } +func file_combined_metrics_proto_init() { + if File_combined_metrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_combined_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SoongCombinedMetrics); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_combined_metrics_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_combined_metrics_proto_goTypes, + DependencyIndexes: file_combined_metrics_proto_depIdxs, + EnumInfos: file_combined_metrics_proto_enumTypes, + MessageInfos: file_combined_metrics_proto_msgTypes, + }.Build() + File_combined_metrics_proto = out.File + file_combined_metrics_proto_rawDesc = nil + file_combined_metrics_proto_goTypes = nil + file_combined_metrics_proto_depIdxs = nil +} diff --git a/ui/metrics/metrics_proto/combined_metrics.proto b/ui/metrics/metrics_proto/combined_metrics.proto new file mode 100644 index 000000000..3cd9a5313 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics.proto @@ -0,0 +1,36 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_build_metrics; +option go_package = "android/soong/ui/metrics/metrics_proto"; + +import "cmd/find_input_delta/find_input_delta_proto/file_list.proto"; + +// These field numbers are also found in the inner message declarations. +// We verify that the values are the same, and that every enum value is checked +// in combined_metrics_test.go. +// Do not change this enum without also updating: +// - the submessage's .proto file +// - combined_metrics_test.go +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} + +message SoongCombinedMetrics { + // cmd/find_input_delta/find_input_delta_proto.FileList + optional android.find_input_delta_proto.FileList file_list = 1; +} diff --git a/ui/metrics/metrics_proto/combined_metrics_test.go b/ui/metrics/metrics_proto/combined_metrics_test.go new file mode 100644 index 000000000..eedb12a34 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics_test.go @@ -0,0 +1,33 @@ +package metrics_proto + +import ( + "testing" + + find_input_delta_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" +) + +func TestCombinedMetricsMessageNums(t *testing.T) { + testCases := []struct { + Name string + FieldNumbers map[string]int32 + }{ + { + Name: "find_input_delta_proto", + FieldNumbers: find_input_delta_proto.FieldNumbers_value, + }, + } + verifiedMap := make(map[string]bool) + for _, tc := range testCases { + for k, v := range tc.FieldNumbers { + if FieldNumbers_value[k] != v { + t.Errorf("%s: Expected FieldNumbers.%s == %v, found %v", tc.Name, k, FieldNumbers_value[k], v) + } + verifiedMap[k] = true + } + } + for k, v := range FieldNumbers_value { + if !verifiedMap[k] { + t.Errorf("No test case verifies FieldNumbers.%s=%v", k, v) + } + } +} diff --git a/ui/metrics/metrics_proto/regen.sh b/ui/metrics/metrics_proto/regen.sh index 8eb2d747b..5e5f9b83f 100755 --- a/ui/metrics/metrics_proto/regen.sh +++ b/ui/metrics/metrics_proto/regen.sh @@ -12,6 +12,6 @@ if ! hash aprotoc &>/dev/null; then die "could not find aprotoc. ${error_msg}" fi -if ! aprotoc --go_out=paths=source_relative:. metrics.proto; then +if ! aprotoc --go_out=paths=source_relative:. -I .:../../.. metrics.proto combined_metrics.proto; then die "build failed. ${error_msg}" fi |