| // Copyright 2018 Google Inc. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package main |
| |
| import ( |
| "bytes" |
| "fmt" |
| "hash/crc32" |
| "os" |
| "strconv" |
| "strings" |
| "testing" |
| "time" |
| |
| "android/soong/jar" |
| "android/soong/third_party/zip" |
| ) |
| |
| type testZipEntry struct { |
| name string |
| mode os.FileMode |
| data []byte |
| method uint16 |
| timestamp time.Time |
| } |
| |
| var ( |
| A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime} |
| a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime} |
| a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate, jar.DefaultTime} |
| a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate, jar.DefaultTime} |
| bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime} |
| bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime} |
| bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate, jar.DefaultTime} |
| ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime} |
| bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate, jar.DefaultTime} |
| bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate, jar.DefaultTime} |
| be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate, jar.DefaultTime} |
| |
| withTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime.Add(time.Hour)} |
| withoutTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime} |
| |
| service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store, jar.DefaultTime} |
| service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate, jar.DefaultTime} |
| service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store, jar.DefaultTime} |
| service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate, jar.DefaultTime} |
| |
| metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime} |
| manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate, jar.DefaultTime} |
| manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate, jar.DefaultTime} |
| moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate, jar.DefaultTime} |
| ) |
| |
| type testInputZip struct { |
| name string |
| entries []testZipEntry |
| reader *zip.Reader |
| } |
| |
| func (tiz *testInputZip) Name() string { |
| return tiz.name |
| } |
| |
| func (tiz *testInputZip) Open() error { |
| if tiz.reader == nil { |
| tiz.reader = testZipEntriesToZipReader(tiz.entries) |
| } |
| return nil |
| } |
| |
| func (tiz *testInputZip) Close() error { |
| tiz.reader = nil |
| return nil |
| } |
| |
| func (tiz *testInputZip) Entries() []*zip.File { |
| if tiz.reader == nil { |
| panic(fmt.Errorf("%s: should be open to get entries", tiz.Name())) |
| } |
| return tiz.reader.File |
| } |
| |
| func (tiz *testInputZip) IsOpen() bool { |
| return tiz.reader != nil |
| } |
| |
| func TestMergeZips(t *testing.T) { |
| testCases := []struct { |
| name string |
| in [][]testZipEntry |
| stripFiles []string |
| stripDirs []string |
| jar bool |
| par bool |
| sort bool |
| ignoreDuplicates bool |
| stripDirEntries bool |
| zipsToNotStrip map[string]bool |
| |
| out []testZipEntry |
| err string |
| }{ |
| { |
| name: "duplicates error", |
| in: [][]testZipEntry{ |
| {a}, |
| {a2}, |
| {a3}, |
| }, |
| out: []testZipEntry{a}, |
| err: "duplicate", |
| }, |
| { |
| name: "duplicates take first", |
| in: [][]testZipEntry{ |
| {a}, |
| {a2}, |
| {a3}, |
| }, |
| out: []testZipEntry{a}, |
| |
| ignoreDuplicates: true, |
| }, |
| { |
| name: "duplicates identical", |
| in: [][]testZipEntry{ |
| {a}, |
| {a}, |
| }, |
| out: []testZipEntry{a}, |
| }, |
| { |
| name: "sort", |
| in: [][]testZipEntry{ |
| {be, bc, bDir, bbDir, bbb, A, metainfDir, manifestFile}, |
| }, |
| out: []testZipEntry{A, metainfDir, manifestFile, bDir, bbDir, bbb, bc, be}, |
| |
| sort: true, |
| }, |
| { |
| name: "jar sort", |
| in: [][]testZipEntry{ |
| {be, bc, bDir, A, metainfDir, manifestFile}, |
| }, |
| out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be}, |
| |
| jar: true, |
| }, |
| { |
| name: "jar merge", |
| in: [][]testZipEntry{ |
| {metainfDir, manifestFile, bDir, be}, |
| {metainfDir, manifestFile2, bDir, bc}, |
| {metainfDir, manifestFile2, A}, |
| }, |
| out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be}, |
| |
| jar: true, |
| }, |
| { |
| name: "merge", |
| in: [][]testZipEntry{ |
| {bDir, be}, |
| {bDir, bc}, |
| {A}, |
| }, |
| out: []testZipEntry{bDir, be, bc, A}, |
| }, |
| { |
| name: "strip dir entries", |
| in: [][]testZipEntry{ |
| {a, bDir, bbDir, bbb, bc, bd, be}, |
| }, |
| out: []testZipEntry{a, bbb, bc, bd, be}, |
| |
| stripDirEntries: true, |
| }, |
| { |
| name: "strip files", |
| in: [][]testZipEntry{ |
| {a, bDir, bbDir, bbb, bc, bd, be}, |
| }, |
| out: []testZipEntry{a, bDir, bbDir, bbb, bc}, |
| |
| stripFiles: []string{"b/d", "b/e"}, |
| }, |
| { |
| // merge_zips used to treat -stripFile a as stripping any file named a, it now only strips a in the |
| // root of the zip. |
| name: "strip file name", |
| in: [][]testZipEntry{ |
| {a, bDir, ba}, |
| }, |
| out: []testZipEntry{bDir, ba}, |
| |
| stripFiles: []string{"a"}, |
| }, |
| { |
| name: "strip files glob", |
| in: [][]testZipEntry{ |
| {a, bDir, ba}, |
| }, |
| out: []testZipEntry{bDir}, |
| |
| stripFiles: []string{"**/a"}, |
| }, |
| { |
| name: "strip dirs", |
| in: [][]testZipEntry{ |
| {a, bDir, bbDir, bbb, bc, bd, be}, |
| }, |
| out: []testZipEntry{a}, |
| |
| stripDirs: []string{"b"}, |
| }, |
| { |
| name: "strip dirs glob", |
| in: [][]testZipEntry{ |
| {a, bDir, bbDir, bbb, bc, bd, be}, |
| }, |
| out: []testZipEntry{a, bDir, bc, bd, be}, |
| |
| stripDirs: []string{"b/*"}, |
| }, |
| { |
| name: "zips to not strip", |
| in: [][]testZipEntry{ |
| {a, bDir, bc}, |
| {bDir, bd}, |
| {bDir, be}, |
| }, |
| out: []testZipEntry{a, bDir, bd}, |
| |
| stripDirs: []string{"b"}, |
| zipsToNotStrip: map[string]bool{ |
| "in1": true, |
| }, |
| }, |
| { |
| name: "services", |
| in: [][]testZipEntry{ |
| {service1a, service2}, |
| {service1b}, |
| }, |
| jar: true, |
| out: []testZipEntry{service1combined, service2}, |
| }, |
| { |
| name: "strip timestamps", |
| in: [][]testZipEntry{ |
| {withTimestamp}, |
| {a}, |
| }, |
| out: []testZipEntry{withoutTimestamp, a}, |
| }, |
| { |
| name: "emulate par", |
| in: [][]testZipEntry{ |
| { |
| testZipEntry{name: "3/main.py"}, |
| testZipEntry{name: "c/main.py"}, |
| testZipEntry{name: "a/main.py"}, |
| testZipEntry{name: "2/main.py"}, |
| testZipEntry{name: "b/main.py"}, |
| testZipEntry{name: "1/main.py"}, |
| }, |
| }, |
| out: []testZipEntry{ |
| testZipEntry{name: "3/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, |
| testZipEntry{name: "c/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, |
| testZipEntry{name: "a/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, |
| testZipEntry{name: "2/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, |
| testZipEntry{name: "b/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, |
| testZipEntry{name: "1/__init__.py", mode: 0700, timestamp: jar.DefaultTime}, |
| testZipEntry{name: "3/main.py", timestamp: jar.DefaultTime}, |
| testZipEntry{name: "c/main.py", timestamp: jar.DefaultTime}, |
| testZipEntry{name: "a/main.py", timestamp: jar.DefaultTime}, |
| testZipEntry{name: "2/main.py", timestamp: jar.DefaultTime}, |
| testZipEntry{name: "b/main.py", timestamp: jar.DefaultTime}, |
| testZipEntry{name: "1/main.py", timestamp: jar.DefaultTime}, |
| }, |
| par: true, |
| }, |
| } |
| |
| for _, test := range testCases { |
| t.Run(test.name, func(t *testing.T) { |
| inputZips := make([]InputZip, len(test.in)) |
| for i, in := range test.in { |
| inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in} |
| } |
| |
| want := testZipEntriesToBuf(test.out) |
| |
| out := &bytes.Buffer{} |
| writer := zip.NewWriter(out) |
| |
| err := mergeZips(inputZips, writer, "", "", |
| test.sort, test.jar, test.par, test.stripDirEntries, test.ignoreDuplicates, |
| test.stripFiles, test.stripDirs, test.zipsToNotStrip) |
| |
| closeErr := writer.Close() |
| if closeErr != nil { |
| t.Fatal(closeErr) |
| } |
| |
| if test.err != "" { |
| if err == nil { |
| t.Fatal("missing err, expected: ", test.err) |
| } else if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.err)) { |
| t.Fatal("incorrect err, want:", test.err, "got:", err) |
| } |
| return |
| } else if err != nil { |
| t.Fatal("unexpected err: ", err) |
| } |
| |
| if !bytes.Equal(want, out.Bytes()) { |
| t.Error("incorrect zip output") |
| t.Errorf("want:\n%s", dumpZip(want)) |
| t.Errorf("got:\n%s", dumpZip(out.Bytes())) |
| os.WriteFile("/tmp/got.zip", out.Bytes(), 0755) |
| os.WriteFile("/tmp/want.zip", want, 0755) |
| } |
| }) |
| } |
| } |
| |
| func testZipEntriesToBuf(entries []testZipEntry) []byte { |
| b := &bytes.Buffer{} |
| zw := zip.NewWriter(b) |
| |
| for _, e := range entries { |
| fh := zip.FileHeader{ |
| Name: e.name, |
| } |
| fh.SetMode(e.mode) |
| fh.Method = e.method |
| fh.SetModTime(e.timestamp) |
| fh.UncompressedSize64 = uint64(len(e.data)) |
| fh.CRC32 = crc32.ChecksumIEEE(e.data) |
| if fh.Method == zip.Store { |
| fh.CompressedSize64 = fh.UncompressedSize64 |
| } |
| |
| w, err := zw.CreateHeaderAndroid(&fh) |
| if err != nil { |
| panic(err) |
| } |
| |
| _, err = w.Write(e.data) |
| if err != nil { |
| panic(err) |
| } |
| } |
| |
| err := zw.Close() |
| if err != nil { |
| panic(err) |
| } |
| |
| return b.Bytes() |
| } |
| |
| func testZipEntriesToZipReader(entries []testZipEntry) *zip.Reader { |
| b := testZipEntriesToBuf(entries) |
| r := bytes.NewReader(b) |
| |
| zr, err := zip.NewReader(r, int64(len(b))) |
| if err != nil { |
| panic(err) |
| } |
| |
| return zr |
| } |
| |
| func dumpZip(buf []byte) string { |
| r := bytes.NewReader(buf) |
| zr, err := zip.NewReader(r, int64(len(buf))) |
| if err != nil { |
| panic(err) |
| } |
| |
| var ret string |
| |
| for _, f := range zr.File { |
| ret += fmt.Sprintf("%v: %v %v %08x %s\n", f.Name, f.Mode(), f.UncompressedSize64, f.CRC32, f.ModTime()) |
| } |
| |
| return ret |
| } |
| |
| type DummyInpuZip struct { |
| isOpen bool |
| } |
| |
| func (diz *DummyInpuZip) Name() string { |
| return "dummy" |
| } |
| |
| func (diz *DummyInpuZip) Open() error { |
| diz.isOpen = true |
| return nil |
| } |
| |
| func (diz *DummyInpuZip) Close() error { |
| diz.isOpen = false |
| return nil |
| } |
| |
| func (DummyInpuZip) Entries() []*zip.File { |
| panic("implement me") |
| } |
| |
| func (diz *DummyInpuZip) IsOpen() bool { |
| return diz.isOpen |
| } |
| |
| func TestInputZipsManager(t *testing.T) { |
| const nInputZips = 20 |
| const nMaxOpenZips = 10 |
| izm := NewInputZipsManager(20, 10) |
| managedZips := make([]InputZip, nInputZips) |
| for i := 0; i < nInputZips; i++ { |
| managedZips[i] = izm.Manage(&DummyInpuZip{}) |
| } |
| |
| t.Run("InputZipsManager", func(t *testing.T) { |
| for i, iz := range managedZips { |
| if err := iz.Open(); err != nil { |
| t.Fatalf("Step %d: open failed: %s", i, err) |
| return |
| } |
| if izm.nOpenZips > nMaxOpenZips { |
| t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips) |
| } |
| } |
| if !managedZips[nInputZips-1].IsOpen() { |
| t.Error("The last input should stay open") |
| } |
| for _, iz := range managedZips { |
| iz.Close() |
| } |
| if izm.nOpenZips > 0 { |
| t.Error("Some input zips are still open") |
| } |
| }) |
| } |