diff options
author | 2024-11-14 16:33:25 +0000 | |
---|---|---|
committer | 2024-11-14 16:33:25 +0000 | |
commit | 1caa78e983d741bfb7768579579fcaf613f2f957 (patch) | |
tree | 1dbbc864beaeebcd3ddce25b60eec3c4a9073965 /cmd | |
parent | 56498834da528ec766c739d7f0d8122292f1ae72 (diff) | |
parent | 6890a004d2c0ca4e358e25e501f82b0437d637e3 (diff) |
Merge "find_input_delta: Add jar inspection, metrics generation" into main
Diffstat (limited to 'cmd')
10 files changed, 481 insertions, 90 deletions
diff --git a/cmd/find_input_delta/find_input_delta/main.go b/cmd/find_input_delta/find_input_delta/main.go index 6b657ea57..a8645843c 100644 --- a/cmd/find_input_delta/find_input_delta/main.go +++ b/cmd/find_input_delta/find_input_delta/main.go @@ -85,4 +85,11 @@ func main() { if err = file_list.Format(os.Stdout, template); err != nil { panic(err) } + + metrics_file := os.Getenv("SOONG_METRICS_AGGREGATION_FILE") + if metrics_file != "" { + if err = file_list.SendMetrics(metrics_file); err != nil { + panic(err) + } + } } diff --git a/cmd/find_input_delta/find_input_delta_lib/Android.bp b/cmd/find_input_delta/find_input_delta_lib/Android.bp index 95bdba81d..ef9c65b1d 100644 --- a/cmd/find_input_delta/find_input_delta_lib/Android.bp +++ b/cmd/find_input_delta/find_input_delta_lib/Android.bp @@ -25,6 +25,7 @@ bootstrap_go_package { "golang-protobuf-runtime-protoimpl", "soong-cmd-find_input_delta-proto", "soong-cmd-find_input_delta-proto_internal", + "android-archive-zip", "blueprint-pathtools", ], srcs: [ diff --git a/cmd/find_input_delta/find_input_delta_lib/file_list.go b/cmd/find_input_delta/find_input_delta_lib/file_list.go index 23337adc3..01242a0e7 100644 --- a/cmd/find_input_delta/find_input_delta_lib/file_list.go +++ b/cmd/find_input_delta/find_input_delta_lib/file_list.go @@ -15,10 +15,15 @@ package find_input_delta_lib import ( + "fmt" "io" + "os" + "path/filepath" + "slices" "text/template" fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" ) @@ -47,28 +52,148 @@ type FileList struct { // The modified files Changes []FileList + + // Map of file_extension:counts + ExtCountMap map[string]*FileCounts + + // Total number of added/changed/deleted files. + TotalDelta uint32 +} + +// The maximum number of files that will be recorded by name. +var MaxFilesRecorded uint32 = 50 + +type FileCounts struct { + Additions uint32 + Deletions uint32 + Changes uint32 +} + +func FileListFactory(name string) *FileList { + return &FileList{ + Name: name, + ExtCountMap: make(map[string]*FileCounts), + } +} + +func (fl *FileList) addFile(name string) { + fl.Additions = append(fl.Additions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Additions += 1 +} + +func (fl *FileList) deleteFile(name string) { + fl.Deletions = append(fl.Deletions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Deletions += 1 +} + +func (fl *FileList) changeFile(name string, ch *FileList) { + fl.Changes = append(fl.Changes, *ch) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Changes += 1 +} + +func (fl FileList) ToProto() (*fid_exp.FileList, error) { + var count uint32 + return fl.toProto(&count) } -func (fl FileList) Marshal() (*fid_exp.FileList, error) { +func (fl FileList) toProto(count *uint32) (*fid_exp.FileList, error) { ret := &fid_exp.FileList{ Name: proto.String(fl.Name), } - if len(fl.Additions) > 0 { - ret.Additions = fl.Additions + for _, a := range fl.Additions { + if *count >= MaxFilesRecorded { + break + } + ret.Additions = append(ret.Additions, a) + *count += 1 } for _, ch := range fl.Changes { - change, err := ch.Marshal() - if err != nil { - return nil, err + if *count >= MaxFilesRecorded { + break + } else { + // Pre-increment to limit what the call adds. + *count += 1 + change, err := ch.toProto(count) + if err != nil { + return nil, err + } + ret.Changes = append(ret.Changes, change) + } + } + for _, d := range fl.Deletions { + if *count >= MaxFilesRecorded { + break } - ret.Changes = append(ret.Changes, change) + ret.Deletions = append(ret.Deletions, d) } - if len(fl.Deletions) > 0 { - ret.Deletions = fl.Deletions + ret.TotalDelta = proto.Uint32(*count) + exts := []string{} + for k := range fl.ExtCountMap { + exts = append(exts, k) + } + slices.Sort(exts) + for _, k := range exts { + v := fl.ExtCountMap[k] + ret.Counts = append(ret.Counts, &fid_exp.FileCount{ + Extension: proto.String(k), + Additions: proto.Uint32(v.Additions), + Deletions: proto.Uint32(v.Deletions), + Modifications: proto.Uint32(v.Changes), + }) } return ret, nil } +func (fl FileList) SendMetrics(path string) error { + if path == "" { + return fmt.Errorf("No path given") + } + message, err := fl.ToProto() + if err != nil { + return err + } + + // Marshal the message wrapped in SoongCombinedMetrics. + data := protowire.AppendVarint( + []byte{}, + protowire.EncodeTag( + protowire.Number(fid_exp.FieldNumbers_FIELD_NUMBERS_FILE_LIST), + protowire.BytesType)) + size := uint64(proto.Size(message)) + data = protowire.AppendVarint(data, size) + data, err = proto.MarshalOptions{UseCachedSize: true}.MarshalAppend(data, message) + if err != nil { + return err + } + + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to close %s: %v\n", path, err) + } + }() + _, err = out.Write(data) + return err +} + func (fl FileList) Format(wr io.Writer, format string) error { tmpl, err := template.New("filelist").Parse(format) if err != nil { diff --git a/cmd/find_input_delta/find_input_delta_lib/fs.go b/cmd/find_input_delta/find_input_delta_lib/fs.go index 4a83ed7ff..09a8aa6cb 100644 --- a/cmd/find_input_delta/find_input_delta_lib/fs.go +++ b/cmd/find_input_delta/find_input_delta_lib/fs.go @@ -15,7 +15,6 @@ package find_input_delta_lib import ( - "io" "io/fs" "os" ) @@ -30,14 +29,6 @@ type fileSystem interface { ReadFile(path string) ([]byte, error) } -type file interface { - io.Closer - io.Reader - io.ReaderAt - io.Seeker - Stat() (os.FileInfo, error) -} - // osFS implements fileSystem using the local disk. type osFS struct{} diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state.go b/cmd/find_input_delta/find_input_delta_lib/internal_state.go index b2ff8c704..f0242b724 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state.go @@ -18,9 +18,11 @@ import ( "errors" "fmt" "io/fs" + "path/filepath" "slices" fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal" + "android/soong/third_party/zip" "github.com/google/blueprint/pathtools" "google.golang.org/protobuf/proto" ) @@ -57,6 +59,7 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* // If we ever have an easy hash, assign it here. } if inspect_contents { + // NOTE: When we find it useful, we can parallelize the file inspection for speed. contents, err := InspectFileContents(input) if err != nil { return ret, err @@ -73,9 +76,30 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* // Inspect the file and extract the state of the elements in the archive. // If this is not an archive of some sort, nil is returned. func InspectFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { - // TODO: Actually inspect the contents. - fmt.Printf("inspecting contents for %s\n", name) - return nil, nil + switch filepath.Ext(name) { + case ".jar", ".apex", ".apk": + return inspectZipFileContents(name) + default: + return nil, nil + } +} + +func inspectZipFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { + rc, err := zip.OpenReader(name) + if err != nil { + return nil, err + } + ret := []*fid_proto.PartialCompileInput{} + for _, v := range rc.File { + pci := &fid_proto.PartialCompileInput{ + Name: proto.String(v.Name), + MtimeNsec: proto.Int64(v.ModTime().UnixNano()), + Hash: proto.String(fmt.Sprintf("%08x", v.CRC32)), + } + ret = append(ret, pci) + // We do not support nested inspection. + } + return ret, nil } func WriteState(s *fid_proto.PartialCompileInputs, path string) error { @@ -91,9 +115,7 @@ func CompareInternalState(prior, other *fid_proto.PartialCompileInputs, target s } func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name string) *FileList { - fl := &FileList{ - Name: name, - } + fl := FileListFactory(name) PriorMap := make(map[string]*fid_proto.PartialCompileInput, len(prior)) // We know that the lists are properly sorted, so we can simply compare them. for _, v := range prior { @@ -105,17 +127,17 @@ func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name strin otherMap[name] = v if _, ok := PriorMap[name]; !ok { // Added file - fl.Additions = append(fl.Additions, name) + fl.addFile(name) } else if !proto.Equal(PriorMap[name], v) { // Changed file - fl.Changes = append(fl.Changes, *CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) + fl.changeFile(name, CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) } } for _, v := range prior { name := v.GetName() if _, ok := otherMap[name]; !ok { // Deleted file - fl.Deletions = append(fl.Deletions, name) + fl.deleteFile(name) } } return fl diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go index 20b8efaa8..e69424c54 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go @@ -199,7 +199,7 @@ func TestCompareInternalState(t *testing.T) { }, }, { - Name: "one of each", + Name: "one each add modify delete", Target: "foo", Prior: &fid_proto.PartialCompileInputs{ InputFiles: []*fid_proto.PartialCompileInput{ @@ -222,11 +222,40 @@ func TestCompareInternalState(t *testing.T) { Deletions: []string{"file2"}, }, }, + { + Name: "interior one each add modify delete", + Target: "bar", + Prior: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 405, "", []*fid_proto.PartialCompileInput{ + protoFile("innerC", 400, "crc32:11111111", nil), + protoFile("innerD", 400, "crc32:44444444", nil), + }), + }, + }, + New: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 505, "", []*fid_proto.PartialCompileInput{ + protoFile("innerA", 400, "crc32:55555555", nil), + protoFile("innerC", 500, "crc32:66666666", nil), + }), + }, + }, + Expected: &FileList{ + Name: "bar", + Changes: []FileList{FileList{ + Name: "file1", + Additions: []string{"innerA"}, + Changes: []FileList{FileList{Name: "innerC"}}, + Deletions: []string{"innerD"}, + }}, + }, + }, } for _, tc := range testCases { actual := CompareInternalState(tc.Prior, tc.New, tc.Target) if !tc.Expected.Equal(actual) { - t.Errorf("%s: expected %q, actual %q", tc.Name, tc.Expected, actual) + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) } } } diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go index 648ef224a..745de2db2 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: file_list.proto -package proto +package find_input_delta_proto import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -35,6 +35,62 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_file_list_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_file_list_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{0} +} + type FileList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -46,10 +102,14 @@ type FileList struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The added files. Additions []string `protobuf:"bytes,2,rep,name=additions" json:"additions,omitempty"` - // The deleted files. - Deletions []string `protobuf:"bytes,3,rep,name=deletions" json:"deletions,omitempty"` // The changed files. - Changes []*FileList `protobuf:"bytes,4,rep,name=changes" json:"changes,omitempty"` + Changes []*FileList `protobuf:"bytes,3,rep,name=changes" json:"changes,omitempty"` + // The deleted files. + Deletions []string `protobuf:"bytes,4,rep,name=deletions" json:"deletions,omitempty"` + // Count of files added/changed/deleted. + TotalDelta *uint32 `protobuf:"varint,5,opt,name=total_delta,json=totalDelta" json:"total_delta,omitempty"` + // Counts by extension. + Counts []*FileCount `protobuf:"bytes,6,rep,name=counts" json:"counts,omitempty"` } func (x *FileList) Reset() { @@ -98,6 +158,13 @@ func (x *FileList) GetAdditions() []string { return nil } +func (x *FileList) GetChanges() []*FileList { + if x != nil { + return x.Changes + } + return nil +} + func (x *FileList) GetDeletions() []string { if x != nil { return x.Deletions @@ -105,32 +172,135 @@ func (x *FileList) GetDeletions() []string { return nil } -func (x *FileList) GetChanges() []*FileList { +func (x *FileList) GetTotalDelta() uint32 { + if x != nil && x.TotalDelta != nil { + return *x.TotalDelta + } + return 0 +} + +func (x *FileList) GetCounts() []*FileCount { if x != nil { - return x.Changes + return x.Counts } return nil } +type FileCount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The file extension + Extension *string `protobuf:"bytes,1,opt,name=extension" json:"extension,omitempty"` + // Number of added files with this extension. + Additions *uint32 `protobuf:"varint,2,opt,name=additions" json:"additions,omitempty"` + // Number of modified files with this extension. + Modifications *uint32 `protobuf:"varint,3,opt,name=modifications" json:"modifications,omitempty"` + // Number of deleted files with this extension. + Deletions *uint32 `protobuf:"varint,4,opt,name=deletions" json:"deletions,omitempty"` +} + +func (x *FileCount) Reset() { + *x = FileCount{} + if protoimpl.UnsafeEnabled { + mi := &file_file_list_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCount) ProtoMessage() {} + +func (x *FileCount) ProtoReflect() protoreflect.Message { + mi := &file_file_list_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCount.ProtoReflect.Descriptor instead. +func (*FileCount) Descriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{1} +} + +func (x *FileCount) GetExtension() string { + if x != nil && x.Extension != nil { + return *x.Extension + } + return "" +} + +func (x *FileCount) GetAdditions() uint32 { + if x != nil && x.Additions != nil { + return *x.Additions + } + return 0 +} + +func (x *FileCount) GetModifications() uint32 { + if x != nil && x.Modifications != nil { + return *x.Modifications + } + return 0 +} + +func (x *FileCount) GetDeletions() uint32 { + if x != nil && x.Deletions != nil { + return *x.Deletions + } + return 0 +} + var File_file_list_proto protoreflect.FileDescriptor var file_file_list_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x9e, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, + 0x6f, 0x22, 0x82, 0x02, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x42, - 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, - 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, - 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x42, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, + 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x65, + 0x6c, 0x74, 0x61, 0x12, 0x41, 0x0a, 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, + 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x06, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, + 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, + 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x01, + 0x42, 0x3b, 0x5a, 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -145,17 +315,21 @@ func file_file_list_proto_rawDescGZIP() []byte { return file_file_list_proto_rawDescData } -var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_file_list_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_file_list_proto_goTypes = []interface{}{ - (*FileList)(nil), // 0: android.find_input_delta_proto.FileList + (FieldNumbers)(0), // 0: android.find_input_delta_proto.FieldNumbers + (*FileList)(nil), // 1: android.find_input_delta_proto.FileList + (*FileCount)(nil), // 2: android.find_input_delta_proto.FileCount } var file_file_list_proto_depIdxs = []int32{ - 0, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 1, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList + 2, // 1: android.find_input_delta_proto.FileList.counts:type_name -> android.find_input_delta_proto.FileCount + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_file_list_proto_init() } @@ -176,19 +350,32 @@ func file_file_list_proto_init() { return nil } } + file_file_list_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileCount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_file_list_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, + NumEnums: 1, + NumMessages: 2, NumExtensions: 0, NumServices: 0, }, GoTypes: file_file_list_proto_goTypes, DependencyIndexes: file_file_list_proto_depIdxs, + EnumInfos: file_file_list_proto_enumTypes, MessageInfos: file_file_list_proto_msgTypes, }.Build() File_file_list_proto = out.File diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.proto b/cmd/find_input_delta/find_input_delta_proto/file_list.proto index d7faca9c8..7180358c8 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.proto +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.proto @@ -15,7 +15,12 @@ syntax = "proto2"; package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +option go_package = "android/soong/cmd/find_input_delta/find_input_delta_proto"; + +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} message FileList { // The name of the file. @@ -26,9 +31,29 @@ message FileList { // The added files. repeated string additions = 2; + // The changed files. + repeated FileList changes = 3; + // The deleted files. - repeated string deletions = 3; + repeated string deletions = 4; - // The changed files. - repeated FileList changes = 4; + // Count of files added/changed/deleted. + optional uint32 total_delta = 5; + + // Counts by extension. + repeated FileCount counts = 6; +} + +message FileCount { + // The file extension + optional string extension = 1; + + // Number of added files with this extension. + optional uint32 additions = 2; + + // Number of modified files with this extension. + optional uint32 modifications = 3; + + // Number of deleted files with this extension. + optional uint32 deletions = 4; } diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go index 2229a324d..c5b048bb8 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: internal_state.proto -package proto +package find_input_delta_proto_internal import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -94,7 +94,7 @@ type PartialCompileInput struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The timestamp of the file in (Unix) nanoseconds. MtimeNsec *int64 `protobuf:"varint,2,opt,name=mtime_nsec,json=mtimeNsec" json:"mtime_nsec,omitempty"` - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. Hash *string `protobuf:"bytes,3,opt,name=hash" json:"hash,omitempty"` // Contents of the file, if the file was inspected (such as jar files, etc). Contents []*PartialCompileInput `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty"` @@ -164,29 +164,33 @@ var File_internal_state_proto protoreflect.FileDescriptor var file_internal_state_proto_rawDesc = []byte{ 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x27, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6c, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, - 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x54, - 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, - 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, - 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, - 0x69, 0x6c, 0x65, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x12, 0x4f, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, - 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, - 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, - 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x22, + 0x75, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x5d, 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, + 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, + 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, + 0x63, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x58, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, + 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x40, 0x5a, 0x3e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, + 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, } var ( @@ -203,12 +207,12 @@ func file_internal_state_proto_rawDescGZIP() []byte { var file_internal_state_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_internal_state_proto_goTypes = []interface{}{ - (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto.PartialCompileInputs - (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto.PartialCompileInput + (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto_internal.PartialCompileInputs + (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto_internal.PartialCompileInput } var file_internal_state_proto_depIdxs = []int32{ - 1, // 0: android.find_input_delta_proto.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto.PartialCompileInput - 1, // 1: android.find_input_delta_proto.PartialCompileInput.contents:type_name -> android.find_input_delta_proto.PartialCompileInput + 1, // 0: android.find_input_delta_proto_internal.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto_internal.PartialCompileInput + 1, // 1: android.find_input_delta_proto_internal.PartialCompileInput.contents:type_name -> android.find_input_delta_proto_internal.PartialCompileInput 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto index 113fc6434..54c90cc72 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto @@ -14,8 +14,8 @@ // limitations under the License. syntax = "proto2"; -package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +package android.find_input_delta_proto_internal; +option go_package = "android/soong/find_input_delta/find_input_delta_proto_internal"; // The state of all inputs. message PartialCompileInputs { @@ -31,7 +31,7 @@ message PartialCompileInput { // The timestamp of the file in (Unix) nanoseconds. optional int64 mtime_nsec = 2; - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. optional string hash = 3; // Contents of the file, if the file was inspected (such as jar files, etc). |