diff options
| -rw-r--r-- | Android.bp | 1 | ||||
| -rw-r--r-- | androidmk/cmd/androidmk/android.go | 9 | ||||
| -rw-r--r-- | androidmk/cmd/androidmk/androidmk_test.go | 19 | ||||
| -rw-r--r-- | androidmk/parser/make_strings.go | 11 | ||||
| -rw-r--r-- | cc/cc.go | 18 | ||||
| -rw-r--r-- | cc/config/arm64_device.go | 2 | ||||
| -rw-r--r-- | cc/config/arm_device.go | 1 | ||||
| -rw-r--r-- | cc/config/global.go | 6 | ||||
| -rw-r--r-- | cc/lto.go | 117 | ||||
| -rw-r--r-- | finder/cmd/finder.go | 15 | ||||
| -rw-r--r-- | finder/finder.go | 125 | ||||
| -rw-r--r-- | finder/finder_test.go | 97 | ||||
| -rw-r--r-- | fs/fs.go | 68 | ||||
| -rw-r--r-- | java/androidmk.go | 23 | ||||
| -rw-r--r-- | java/builder.go | 84 | ||||
| -rw-r--r-- | java/config/config.go | 3 | ||||
| -rw-r--r-- | java/java.go | 8 | ||||
| -rw-r--r-- | java/resources.go | 2 | ||||
| -rwxr-xr-x | scripts/jar-args.sh | 64 |
19 files changed, 574 insertions, 99 deletions
diff --git a/Android.bp b/Android.bp index 0c79bf539..d1b8f0022 100644 --- a/Android.bp +++ b/Android.bp @@ -115,6 +115,7 @@ bootstrap_go_package { "cc/check.go", "cc/coverage.go", "cc/gen.go", + "cc/lto.go", "cc/makevars.go", "cc/prebuilt.go", "cc/proto.go", diff --git a/androidmk/cmd/androidmk/android.go b/androidmk/cmd/androidmk/android.go index 194b2c93e..4afb3b420 100644 --- a/androidmk/cmd/androidmk/android.go +++ b/androidmk/cmd/androidmk/android.go @@ -35,6 +35,7 @@ var rewriteProperties = map[string](func(variableAssignmentContext) error){ "LOCAL_SRC_FILES": srcFiles, "LOCAL_SANITIZE": sanitize(""), "LOCAL_SANITIZE_DIAG": sanitize("diag."), + "LOCAL_CFLAGS": cflags, // composite functions "LOCAL_MODULE_TAGS": includeVariableIf(bpVariable{"tags", bpparser.ListType}, not(valueDumpEquals("optional"))), @@ -83,7 +84,6 @@ func init() { "LOCAL_SYSTEM_SHARED_LIBRARIES": "system_shared_libs", "LOCAL_ASFLAGS": "asflags", "LOCAL_CLANG_ASFLAGS": "clang_asflags", - "LOCAL_CFLAGS": "cflags", "LOCAL_CONLYFLAGS": "conlyflags", "LOCAL_CPPFLAGS": "cppflags", "LOCAL_REQUIRED_MODULES": "required", @@ -532,6 +532,13 @@ func ldflags(ctx variableAssignmentContext) error { return nil } +func cflags(ctx variableAssignmentContext) error { + // The Soong replacement for CFLAGS doesn't need the same extra escaped quotes that were present in Make + ctx.mkvalue = ctx.mkvalue.Clone() + ctx.mkvalue.ReplaceLiteral(`\"`, `"`) + return includeVariableNow(bpVariable{"cflags", bpparser.ListType}, ctx) +} + // given a conditional, returns a function that will insert a variable assignment or not, based on the conditional func includeVariableIf(bpVar bpVariable, conditional func(ctx variableAssignmentContext) bool) func(ctx variableAssignmentContext) error { return func(ctx variableAssignmentContext) error { diff --git a/androidmk/cmd/androidmk/androidmk_test.go b/androidmk/cmd/androidmk/androidmk_test.go index 5fbc951f0..07d1c109c 100644 --- a/androidmk/cmd/androidmk/androidmk_test.go +++ b/androidmk/cmd/androidmk/androidmk_test.go @@ -374,6 +374,25 @@ cc_library_shared { } `, }, + + { + desc: "Input containing escaped quotes", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE:= libsensorservice +LOCAL_CFLAGS:= -DLOG_TAG=\"-DDontEscapeMe\" +LOCAL_SRC_FILES := \"EscapeMe.cc\" +include $(BUILD_SHARED_LIBRARY) +`, + + expected: ` +cc_library_shared { + name: "libsensorservice", + cflags: ["-DLOG_TAG=\"-DDontEscapeMe\""], + srcs: ["\\\"EscapeMe.cc\\\""], +} +`, + }, } func reformatBlueprint(input string) string { diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go index 00d331b32..142dc71c4 100644 --- a/androidmk/parser/make_strings.go +++ b/androidmk/parser/make_strings.go @@ -29,6 +29,11 @@ func SimpleMakeString(s string, pos Pos) *MakeString { } } +func (ms *MakeString) Clone() (result *MakeString) { + clone := *ms + return &clone +} + func (ms *MakeString) Pos() Pos { return ms.StringPos } @@ -164,6 +169,12 @@ func (ms *MakeString) EndsWith(ch rune) bool { return s[len(s)-1] == uint8(ch) } +func (ms *MakeString) ReplaceLiteral(input string, output string) { + for i := range ms.Strings { + ms.Strings[i] = strings.Replace(ms.Strings[i], input, output, -1) + } +} + func splitAnyN(s, sep string, n int) []string { ret := []string{} for n == -1 || n > 1 { @@ -51,6 +51,9 @@ func init() { ctx.BottomUp("coverage", coverageLinkingMutator).Parallel() ctx.TopDown("vndk_deps", sabiDepsMutator) + + ctx.TopDown("lto_deps", ltoDepsMutator) + ctx.BottomUp("lto", ltoMutator).Parallel() }) pctx.Import("android/soong/cc/config") @@ -295,6 +298,7 @@ type Module struct { coverage *coverage sabi *sabi vndkdep *vndkdep + lto *lto androidMkSharedLibDeps []string @@ -334,6 +338,9 @@ func (c *Module) Init() android.Module { if c.vndkdep != nil { c.AddProperties(c.vndkdep.props()...) } + if c.lto != nil { + c.AddProperties(c.lto.props()...) + } for _, feature := range c.features { c.AddProperties(feature.props()...) } @@ -489,6 +496,7 @@ func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Mo module.coverage = &coverage{} module.sabi = &sabi{} module.vndkdep = &vndkdep{} + module.lto = <o{} return module } @@ -537,6 +545,9 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if c.coverage != nil { flags = c.coverage.flags(ctx, flags) } + if c.lto != nil { + flags = c.lto.flags(ctx, flags) + } for _, feature := range c.features { flags = feature.flags(ctx, flags) } @@ -620,6 +631,9 @@ func (c *Module) begin(ctx BaseModuleContext) { if c.vndkdep != nil { c.vndkdep.begin(ctx) } + if c.lto != nil { + c.lto.begin(ctx) + } for _, feature := range c.features { feature.begin(ctx) } @@ -656,6 +670,9 @@ func (c *Module) deps(ctx DepsContext) Deps { if c.vndkdep != nil { deps = c.vndkdep.deps(ctx, deps) } + if c.lto != nil { + deps = c.lto.deps(ctx, deps) + } for _, feature := range c.features { deps = feature.deps(ctx, deps) } @@ -1191,6 +1208,7 @@ func DefaultsFactory(props ...interface{}) android.Module { &CoverageProperties{}, &SAbiProperties{}, &VndkProperties{}, + <OProperties{}, ) android.InitDefaultsModule(module) diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index 025d3a5f6..0d22ed498 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -60,7 +60,7 @@ var ( "-Wl,--build-id=md5", "-Wl,--warn-shared-textrel", "-Wl,--fatal-warnings", - "-Wl,-maarch64linux", + "-Wl,-m,aarch64_elf64_le_vec", "-Wl,--hash-style=gnu", "-Wl,--fix-cortex-a53-843419", "-fuse-ld=gold", diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index 38816aabe..d50de2b4f 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -67,6 +67,7 @@ var ( "-Wl,--icf=safe", "-Wl,--hash-style=gnu", "-Wl,--no-undefined-version", + "-Wl,-m,armelf", } armArmCflags = []string{ diff --git a/cc/config/global.go b/cc/config/global.go index 56de35139..82a44e615 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -16,6 +16,7 @@ package config import ( "fmt" + "runtime" "strings" "android/soong/android" @@ -149,6 +150,11 @@ func init() { return ClangDefaultShortVersion, nil }) pctx.StaticVariable("ClangAsanLibDir", "${ClangPath}/lib64/clang/${ClangShortVersion}/lib/linux") + if runtime.GOOS == "darwin" { + pctx.StaticVariable("LLVMGoldPlugin", "${ClangPath}/lib64/LLVMgold.dylib") + } else { + pctx.StaticVariable("LLVMGoldPlugin", "${ClangPath}/lib64/LLVMgold.so") + } // These are tied to the version of LLVM directly in external/llvm, so they might trail the host prebuilts // being used for the rest of the build process. diff --git a/cc/lto.go b/cc/lto.go new file mode 100644 index 000000000..f49677244 --- /dev/null +++ b/cc/lto.go @@ -0,0 +1,117 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "github.com/google/blueprint" + + "android/soong/android" +) + +// LTO (link-time optimization) allows the compiler to optimize and generate +// code for the entire module at link time, rather than per-compilation +// unit. LTO is required for Clang CFI and other whole-program optimization +// techniques. LTO also allows cross-compilation unit optimizations that should +// result in faster and smaller code, at the expense of additional compilation +// time. +// +// To properly build a module with LTO, the module and all recursive static +// dependencies should be compiled with -flto which directs the compiler to emit +// bitcode rather than native object files. These bitcode files are then passed +// by the linker to the LLVM plugin for compilation at link time. Static +// dependencies not built as bitcode will still function correctly but cannot be +// optimized at link time and may not be compatible with features that require +// LTO, such as CFI. +// +// This file adds support to soong to automatically propogate LTO options to a +// new variant of all static dependencies for each module with LTO enabled. + +type LTOProperties struct { + // Lto must violate capitialization style for acronyms so that it can be + // referred to in blueprint files as "lto" + Lto *bool `android:"arch_variant"` + LTODep bool `blueprint:"mutated"` +} + +type lto struct { + Properties LTOProperties +} + +func (lto *lto) props() []interface{} { + return []interface{}{<o.Properties} +} + +func (lto *lto) begin(ctx BaseModuleContext) { +} + +func (lto *lto) deps(ctx BaseModuleContext, deps Deps) Deps { + return deps +} + +func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { + if Bool(lto.Properties.Lto) { + flags.CFlags = append(flags.CFlags, "-flto") + flags.LdFlags = append(flags.LdFlags, "-flto") + if ctx.Device() { + // Work around bug in Clang that doesn't pass correct emulated + // TLS option to target + flags.LdFlags = append(flags.LdFlags, "-Wl,-plugin-opt,-emulated-tls") + } + flags.ArFlags = append(flags.ArFlags, " --plugin ${config.LLVMGoldPlugin}") + } + return flags +} + +// Can be called with a null receiver +func (lto *lto) LTO() bool { + if lto == nil { + return false + } + + return Bool(lto.Properties.Lto) +} + +// Propagate lto requirements down from binaries +func ltoDepsMutator(mctx android.TopDownMutatorContext) { + if c, ok := mctx.Module().(*Module); ok && c.lto.LTO() { + mctx.VisitDepsDepthFirst(func(m blueprint.Module) { + tag := mctx.OtherModuleDependencyTag(m) + switch tag { + case staticDepTag, staticExportDepTag, lateStaticDepTag, wholeStaticDepTag, objDepTag, reuseObjTag: + if cc, ok := m.(*Module); ok && cc.lto != nil { + cc.lto.Properties.LTODep = true + } + } + }) + } +} + +// Create lto variants for modules that need them +func ltoMutator(mctx android.BottomUpMutatorContext) { + if c, ok := mctx.Module().(*Module); ok && c.lto != nil { + if c.lto.LTO() { + mctx.SetDependencyVariation("lto") + } else if c.lto.Properties.LTODep { + modules := mctx.CreateVariations("", "lto") + modules[0].(*Module).lto.Properties.Lto = boolPtr(false) + modules[1].(*Module).lto.Properties.Lto = boolPtr(true) + modules[0].(*Module).lto.Properties.LTODep = false + modules[1].(*Module).lto.Properties.LTODep = false + modules[1].(*Module).Properties.PreventInstall = true + modules[1].(*Module).Properties.HideFromMake = true + } + c.lto.Properties.LTODep = false + } +} diff --git a/finder/cmd/finder.go b/finder/cmd/finder.go index 9da166026..70c1dc4a6 100644 --- a/finder/cmd/finder.go +++ b/finder/cmd/finder.go @@ -127,9 +127,13 @@ func run() error { usage() return errors.New("Param 'db' must be nonempty") } + matches := []string{} for i := 0; i < numIterations; i++ { - matches = runFind(params, logger) + matches, err = runFind(params, logger) + if err != nil { + return err + } } findDuration := time.Since(startTime) logger.Printf("Found these %v inodes in %v :\n", len(matches), findDuration) @@ -142,8 +146,11 @@ func run() error { return nil } -func runFind(params finder.CacheParams, logger *log.Logger) (paths []string) { - service := finder.New(params, fs.OsFs, logger, dbPath) +func runFind(params finder.CacheParams, logger *log.Logger) (paths []string, err error) { + service, err := finder.New(params, fs.OsFs, logger, dbPath) + if err != nil { + return []string{}, err + } defer service.Shutdown() - return service.FindAll() + return service.FindAll(), nil } diff --git a/finder/finder.go b/finder/finder.go index ad85ee9af..f15c8c13a 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -150,6 +150,8 @@ type Finder struct { // temporary state threadPool *threadPool mutex sync.Mutex + fsErrs []fsErr + errlock sync.Mutex // non-temporary state modifiedFlag int32 @@ -158,7 +160,7 @@ type Finder struct { // New creates a new Finder for use func New(cacheParams CacheParams, filesystem fs.FileSystem, - logger Logger, dbPath string) *Finder { + logger Logger, dbPath string) (f *Finder, err error) { numThreads := runtime.NumCPU() * 2 numDbLoadingThreads := numThreads @@ -172,7 +174,7 @@ func New(cacheParams CacheParams, filesystem fs.FileSystem, }, } - finder := &Finder{ + f = &Finder{ numDbLoadingThreads: numDbLoadingThreads, numSearchingThreads: numSearchingThreads, cacheMetadata: metadata, @@ -183,10 +185,23 @@ func New(cacheParams CacheParams, filesystem fs.FileSystem, DbPath: dbPath, } - finder.loadFromFilesystem() + f.loadFromFilesystem() - finder.verbosef("Done parsing db\n") - return finder + // check for any filesystem errors + err = f.getErr() + if err != nil { + return nil, err + } + + // confirm that every path mentioned in the CacheConfig exists + for _, path := range cacheParams.RootDirs { + node := f.nodes.GetNode(filepath.Clean(path), false) + if node == nil || node.ModTime == 0 { + return nil, fmt.Errorf("%v does not exist\n", path) + } + } + + return f, nil } // FindNamed searches for every cached file @@ -338,10 +353,6 @@ func (f *Finder) loadFromFilesystem() { f.startWithoutExternalCache() } - startTime := time.Now() - f.verbosef("Waiting for pending requests to complete\n") - f.threadPool.Wait() - f.verbosef("Is idle after %v\n", time.Now().Sub(startTime)) f.threadPool = nil } @@ -598,6 +609,15 @@ func (p *threadPool) Wait() { p.receivedRequests.Wait() } +type fsErr struct { + path string + err error +} + +func (e fsErr) String() string { + return e.path + ": " + e.err.Error() +} + func (f *Finder) serializeCacheEntry(dirInfos []dirFullInfo) ([]byte, error) { // group each dirFullInfo by its Device, to avoid having to repeat it in the output dirsByDevice := map[uint64][]PersistedDirInfo{} @@ -943,13 +963,17 @@ func (f *Finder) startFromExternalCache() (err error) { for i := range nodesToWalk { f.listDirsAsync(nodesToWalk[i]) } - f.verbosef("Loaded db and statted its contents in %v\n", time.Since(startTime)) + f.verbosef("Loaded db and statted known dirs in %v\n", time.Since(startTime)) + f.threadPool.Wait() + f.verbosef("Loaded db and statted all dirs in %v\n", time.Now().Sub(startTime)) + return err } // startWithoutExternalCache starts scanning the filesystem according to the cache config // startWithoutExternalCache should be called if startFromExternalCache is not applicable func (f *Finder) startWithoutExternalCache() { + startTime := time.Now() configDirs := f.cacheMetadata.Config.RootDirs // clean paths @@ -977,6 +1001,10 @@ func (f *Finder) startWithoutExternalCache() { f.verbosef("Starting find of %v\n", path) f.startFind(path) } + + f.threadPool.Wait() + + f.verbosef("Scanned filesystem (not using cache) in %v\n", time.Now().Sub(startTime)) } // isInfoUpToDate tells whether <new> can confirm that results computed at <old> are still valid @@ -1114,6 +1142,79 @@ func (f *Finder) dumpDb() error { f.verbosef("Wrote db in %v\n", time.Now().Sub(serializeDate)) return nil + +} + +// canIgnoreFsErr checks for certain classes of filesystem errors that are safe to ignore +func (f *Finder) canIgnoreFsErr(err error) bool { + pathErr, isPathErr := err.(*os.PathError) + if !isPathErr { + // Don't recognize this error + return false + } + if pathErr.Err == os.ErrPermission { + // Permission errors are ignored: + // https://issuetracker.google.com/37553659 + // https://github.com/google/kati/pull/116 + return true + } + if pathErr.Err == os.ErrNotExist { + // If a directory doesn't exist, that generally means the cache is out-of-date + return true + } + // Don't recognize this error + return false +} + +// onFsError should be called whenever a potentially fatal error is returned from a filesystem call +func (f *Finder) onFsError(path string, err error) { + if !f.canIgnoreFsErr(err) { + // We could send the errors through a channel instead, although that would cause this call + // to block unless we preallocated a sufficient buffer or spawned a reader thread. + // Although it wouldn't be too complicated to spawn a reader thread, it's still slightly + // more convenient to use a lock. Only in an unusual situation should this code be + // invoked anyway. + f.errlock.Lock() + f.fsErrs = append(f.fsErrs, fsErr{path: path, err: err}) + f.errlock.Unlock() + } +} + +// discardErrsForPrunedPaths removes any errors for paths that are no longer included in the cache +func (f *Finder) discardErrsForPrunedPaths() { + // This function could be somewhat inefficient due to being single-threaded, + // but the length of f.fsErrs should be approximately 0, so it shouldn't take long anyway. + relevantErrs := make([]fsErr, 0, len(f.fsErrs)) + for _, fsErr := range f.fsErrs { + path := fsErr.path + node := f.nodes.GetNode(path, false) + if node != nil { + // The path in question wasn't pruned due to a failure to process a parent directory. + // So, the failure to process this path is important + relevantErrs = append(relevantErrs, fsErr) + } + } + f.fsErrs = relevantErrs +} + +// getErr returns an error based on previous calls to onFsErr, if any +func (f *Finder) getErr() error { + f.discardErrsForPrunedPaths() + + numErrs := len(f.fsErrs) + if numErrs < 1 { + return nil + } + + maxNumErrsToInclude := 10 + message := "" + if numErrs > maxNumErrsToInclude { + message = fmt.Sprintf("finder encountered %v errors: %v...", numErrs, f.fsErrs[:maxNumErrsToInclude]) + } else { + message = fmt.Sprintf("finder encountered %v errors: %v", numErrs, f.fsErrs) + } + + return errors.New(message) } func (f *Finder) statDirAsync(dir *pathMap) { @@ -1145,6 +1246,8 @@ func (f *Finder) statDirSync(path string) statResponse { var stats statResponse if err != nil { + // possibly record this error + f.onFsError(path, err) // in case of a failure to stat the directory, treat the directory as missing (modTime = 0) return stats } @@ -1248,6 +1351,8 @@ func (f *Finder) listDirSync(dir *pathMap) { children, err := f.filesystem.ReadDir(path) if err != nil { + // possibly record this error + f.onFsError(path, err) // if listing the contents of the directory fails (presumably due to // permission denied), then treat the directory as empty children = []os.FileInfo{} diff --git a/finder/finder_test.go b/finder/finder_test.go index 60e5eb281..15c3728b1 100644 --- a/finder/finder_test.go +++ b/finder/finder_test.go @@ -16,18 +16,17 @@ package finder import ( "fmt" + "io/ioutil" "log" + "os" "path/filepath" "reflect" - "testing" - "sort" - - "io/ioutil" + "testing" + "time" "android/soong/fs" "runtime/debug" - "time" ) // some utils for tests to use @@ -36,6 +35,14 @@ func newFs() *fs.MockFs { } func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Finder { + f, err := newFinderAndErr(t, filesystem, cacheParams) + if err != nil { + fatal(t, err.Error()) + } + return f +} + +func newFinderAndErr(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) (*Finder, error) { cachePath := "/finder/finder-db" cacheDir := filepath.Dir(cachePath) filesystem.MkDirs(cacheDir) @@ -44,16 +51,25 @@ func newFinder(t *testing.T, filesystem *fs.MockFs, cacheParams CacheParams) *Fi } logger := log.New(ioutil.Discard, "", 0) - finder := New(cacheParams, filesystem, logger, cachePath) - return finder + f, err := New(cacheParams, filesystem, logger, cachePath) + return f, err } func finderWithSameParams(t *testing.T, original *Finder) *Finder { - return New( + f, err := finderAndErrorWithSameParams(t, original) + if err != nil { + fatal(t, err.Error()) + } + return f +} + +func finderAndErrorWithSameParams(t *testing.T, original *Finder) (*Finder, error) { + f, err := New( original.cacheMetadata.Config.CacheParams, original.filesystem, original.logger, original.DbPath) + return f, err } func write(t *testing.T, path string, content string, filesystem *fs.MockFs) { @@ -61,7 +77,7 @@ func write(t *testing.T, path string, content string, filesystem *fs.MockFs) { filesystem.MkDirs(parent) err := filesystem.WriteFile(path, []byte(content), 0777) if err != nil { - t.Fatal(err.Error()) + fatal(t, err.Error()) } } @@ -72,21 +88,21 @@ func create(t *testing.T, path string, filesystem *fs.MockFs) { func delete(t *testing.T, path string, filesystem *fs.MockFs) { err := filesystem.Remove(path) if err != nil { - t.Fatal(err.Error()) + fatal(t, err.Error()) } } func removeAll(t *testing.T, path string, filesystem *fs.MockFs) { err := filesystem.RemoveAll(path) if err != nil { - t.Fatal(err.Error()) + fatal(t, err.Error()) } } func move(t *testing.T, oldPath string, newPath string, filesystem *fs.MockFs) { err := filesystem.Rename(oldPath, newPath) if err != nil { - t.Fatal(err.Error()) + fatal(t, err.Error()) } } @@ -98,7 +114,7 @@ func link(t *testing.T, newPath string, oldPath string, filesystem *fs.MockFs) { } err = filesystem.Symlink(oldPath, newPath) if err != nil { - t.Fatal(err.Error()) + fatal(t, err.Error()) } } func read(t *testing.T, path string, filesystem *fs.MockFs) string { @@ -125,11 +141,20 @@ func setReadable(t *testing.T, path string, readable bool, filesystem *fs.MockFs t.Fatal(err.Error()) } } + +func setReadErr(t *testing.T, path string, readErr error, filesystem *fs.MockFs) { + err := filesystem.SetReadErr(path, readErr) + if err != nil { + t.Fatal(err.Error()) + } +} + func fatal(t *testing.T, message string) { t.Error(message) debug.PrintStack() t.FailNow() } + func assertSameResponse(t *testing.T, actual []string, expected []string) { sort.Strings(actual) sort.Strings(expected) @@ -280,11 +305,11 @@ func TestFilesystemRoot(t *testing.T) { assertSameResponse(t, foundPaths, []string{createdPath}) } -func TestNonexistentPath(t *testing.T) { +func TestNonexistentDir(t *testing.T) { filesystem := newFs() create(t, "/tmp/findme.txt", filesystem) - finder := newFinder( + _, err := newFinderAndErr( t, filesystem, CacheParams{ @@ -292,11 +317,9 @@ func TestNonexistentPath(t *testing.T) { IncludeFiles: []string{"findme.txt", "skipme.txt"}, }, ) - defer finder.Shutdown() - - foundPaths := finder.FindNamedAt("/tmp/IAlsoDontExist", "findme.txt") - - assertSameResponse(t, foundPaths, []string{}) + if err == nil { + fatal(t, "Did not fail when given a nonexistent root directory") + } } func TestExcludeDirs(t *testing.T) { @@ -392,7 +415,7 @@ func TestUncachedDir(t *testing.T) { t, filesystem, CacheParams{ - RootDirs: []string{"/IDoNotExist"}, + RootDirs: []string{"/tmp/b"}, IncludeFiles: []string{"findme.txt"}, }, ) @@ -483,7 +506,7 @@ func TestRootDirsContainedInOtherRootDirs(t *testing.T) { t, filesystem, CacheParams{ - RootDirs: []string{"/", "/a/b/c", "/a/b/c/d/e/f", "/a/b/c/d/e/f/g/h/i"}, + RootDirs: []string{"/", "/tmp/a/b/c", "/tmp/a/b/c/d/e/f", "/tmp/a/b/c/d/e/f/g/h/i"}, IncludeFiles: []string{"findme.txt"}, }, ) @@ -1571,3 +1594,33 @@ func TestFileNotPermitted(t *testing.T) { // check results assertSameResponse(t, foundPaths, []string{"/tmp/hi.txt"}) } + +func TestCacheEntryPathUnexpectedError(t *testing.T) { + // setup filesystem + filesystem := newFs() + create(t, "/tmp/a/hi.txt", filesystem) + + // run the first finder + finder := newFinder( + t, + filesystem, + CacheParams{ + RootDirs: []string{"/tmp"}, + IncludeFiles: []string{"hi.txt"}, + }, + ) + foundPaths := finder.FindAll() + filesystem.Clock.Tick() + finder.Shutdown() + // check results + assertSameResponse(t, foundPaths, []string{"/tmp/a/hi.txt"}) + + // make the directory not readable + setReadErr(t, "/tmp/a", os.ErrInvalid, filesystem) + + // run the second finder + _, err := finderAndErrorWithSameParams(t, finder) + if err == nil { + fatal(t, "Failed to detect unexpected filesystem error") + } +} @@ -159,7 +159,7 @@ type mockInode struct { permTime time.Time sys interface{} inodeNumber uint64 - readable bool + readErr error } func (m mockInode) ModTime() time.Time { @@ -221,11 +221,11 @@ func (m *MockFs) followLinks(path string, followLastLink bool, count int) (canon if err != nil { return "", err } - if !parentNode.readable { + if parentNode.readErr != nil { return "", &os.PathError{ Op: "read", Path: path, - Err: os.ErrPermission, + Err: parentNode.readErr, } } @@ -240,11 +240,11 @@ func (m *MockFs) followLinks(path string, followLastLink bool, count int) (canon } } - if !link.readable { + if link.readErr != nil { return "", &os.PathError{ Op: "read", Path: path, - Err: os.ErrPermission, + Err: link.readErr, } } @@ -277,11 +277,11 @@ func (m *MockFs) getFile(parentDir *mockDir, fileName string) (file *mockFile, e Err: os.ErrNotExist, } } - if !file.readable { + if file.readErr != nil { return nil, &os.PathError{ Op: "open", Path: fileName, - Err: os.ErrPermission, + Err: file.readErr, } } return file, nil @@ -491,11 +491,11 @@ func (m *MockFs) ReadDir(path string) (contents []os.FileInfo, err error) { if err != nil { return nil, err } - if !dir.readable { + if dir.readErr != nil { return nil, &os.PathError{ Op: "read", Path: path, - Err: os.ErrPermission, + Err: dir.readErr, } } // describe its contents @@ -532,11 +532,11 @@ func (m *MockFs) Rename(sourcePath string, destPath string) error { Err: os.ErrNotExist, } } - if !sourceParentDir.readable { + if sourceParentDir.readErr != nil { return &os.PathError{ Op: "move", Path: sourcePath, - Err: os.ErrPermission, + Err: sourceParentDir.readErr, } } @@ -554,11 +554,11 @@ func (m *MockFs) Rename(sourcePath string, destPath string) error { Err: os.ErrNotExist, } } - if !destParentDir.readable { + if destParentDir.readErr != nil { return &os.PathError{ Op: "move", Path: destParentPath, - Err: os.ErrPermission, + Err: destParentDir.readErr, } } // check the source and dest themselves @@ -648,11 +648,11 @@ func (m *MockFs) WriteFile(filePath string, data []byte, perm os.FileMode) error Err: os.ErrNotExist, } } - if !parentDir.readable { + if parentDir.readErr != nil { return &os.PathError{ Op: "write", Path: parentPath, - Err: os.ErrPermission, + Err: parentDir.readErr, } } @@ -662,11 +662,12 @@ func (m *MockFs) WriteFile(filePath string, data []byte, perm os.FileMode) error parentDir.modTime = m.Clock.Time() parentDir.files[baseName] = m.newFile() } else { - if !parentDir.files[baseName].readable { + readErr := parentDir.files[baseName].readErr + if readErr != nil { return &os.PathError{ Op: "write", Path: filePath, - Err: os.ErrPermission, + Err: readErr, } } } @@ -681,7 +682,6 @@ func (m *MockFs) newFile() *mockFile { newFile.inodeNumber = m.newInodeNumber() newFile.modTime = m.Clock.Time() newFile.permTime = newFile.modTime - newFile.readable = true return newFile } @@ -694,7 +694,6 @@ func (m *MockFs) newDir() *mockDir { newDir.inodeNumber = m.newInodeNumber() newDir.modTime = m.Clock.Time() newDir.permTime = newDir.modTime - newDir.readable = true return newDir } @@ -705,7 +704,6 @@ func (m *MockFs) newLink(target string) *mockLink { newLink.inodeNumber = m.newInodeNumber() newLink.modTime = m.Clock.Time() newLink.permTime = newLink.modTime - newLink.readable = true return newLink } @@ -729,11 +727,11 @@ func (m *MockFs) getDir(path string, createIfMissing bool) (dir *mockDir, err er if err != nil { return nil, err } - if !parent.readable { + if parent.readErr != nil { return nil, &os.PathError{ Op: "stat", Path: path, - Err: os.ErrPermission, + Err: parent.readErr, } } childDir, dirExists := parent.subdirs[leaf] @@ -781,11 +779,11 @@ func (m *MockFs) Remove(path string) (err error) { Err: os.ErrNotExist, } } - if !parentDir.readable { + if parentDir.readErr != nil { return &os.PathError{ Op: "remove", Path: path, - Err: os.ErrPermission, + Err: parentDir.readErr, } } _, isDir := parentDir.subdirs[leaf] @@ -822,11 +820,11 @@ func (m *MockFs) Symlink(oldPath string, newPath string) (err error) { newParentPath, leaf := pathSplit(newPath) newParentDir, err := m.getDir(newParentPath, false) - if !newParentDir.readable { + if newParentDir.readErr != nil { return &os.PathError{ Op: "link", Path: newPath, - Err: os.ErrPermission, + Err: newParentDir.readErr, } } if err != nil { @@ -856,11 +854,11 @@ func (m *MockFs) RemoveAll(path string) (err error) { Err: os.ErrNotExist, } } - if !parentDir.readable { + if parentDir.readErr != nil { return &os.PathError{ Op: "removeAll", Path: path, - Err: os.ErrPermission, + Err: parentDir.readErr, } } @@ -886,6 +884,14 @@ func (m *MockFs) RemoveAll(path string) (err error) { } func (m *MockFs) SetReadable(path string, readable bool) error { + var readErr error + if !readable { + readErr = os.ErrPermission + } + return m.SetReadErr(path, readErr) +} + +func (m *MockFs) SetReadErr(path string, readErr error) error { path, err := m.resolve(path, false) if err != nil { return err @@ -895,11 +901,11 @@ func (m *MockFs) SetReadable(path string, readable bool) error { if err != nil { return err } - if !parentDir.readable { + if parentDir.readErr != nil { return &os.PathError{ Op: "chmod", Path: parentPath, - Err: os.ErrPermission, + Err: parentDir.readErr, } } @@ -907,7 +913,7 @@ func (m *MockFs) SetReadable(path string, readable bool) error { if err != nil { return err } - inode.readable = readable + inode.readErr = readErr inode.permTime = m.Clock.Time() return nil } diff --git a/java/androidmk.go b/java/androidmk.go index 193992496..12643cf72 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -17,6 +17,7 @@ package java import ( "fmt" "io" + "strings" "android/soong/android" ) @@ -44,3 +45,25 @@ func (prebuilt *Import) AndroidMk() android.AndroidMkData { }, } } + +func (binary *Binary) AndroidMk() android.AndroidMkData { + return android.AndroidMkData{ + Class: "JAVA_LIBRARIES", + OutputFile: android.OptionalPathForPath(binary.outputFile), + SubName: ".jar", + Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + android.WriteAndroidMkData(w, data) + + fmt.Fprintln(w, "include $(CLEAR_VARS)") + fmt.Fprintln(w, "LOCAL_MODULE := "+name) + fmt.Fprintln(w, "LOCAL_MODULE_CLASS := EXECUTABLES") + if strings.Contains(prefix, "HOST_") { + fmt.Fprintln(w, "LOCAL_IS_HOST_MODULE := true") + } + fmt.Fprintln(w, "LOCAL_STRIP_MODULE := false") + fmt.Fprintln(w, "LOCAL_REQUIRED_MODULES := "+name+".jar") + fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE := "+binary.wrapperFile.String()) + fmt.Fprintln(w, "include $(BUILD_PREBUILT)") + }, + } +} diff --git a/java/builder.go b/java/builder.go index 017f64f8b..efe0a6bbe 100644 --- a/java/builder.go +++ b/java/builder.go @@ -41,8 +41,9 @@ var ( `${config.JavacWrapper}${config.JavacCmd} ${config.CommonJdkFlags} ` + `$javacFlags $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + - `-d $outDir -s $annoDir @$out.rsp || ( rm -rf "$outDir"; exit 41 ) && ` + - `find $outDir -name "*.class" > $out`, + `-d $outDir -s $annoDir @$out.rsp && ` + + `find $outDir -type f | sort | ${config.JarArgsCmd} $outDir > $out`, + CommandDeps: []string{"${config.JavacCmd}", "${config.JarArgsCmd}"}, Rspfile: "$out.rsp", RspfileContent: "$in", }, @@ -50,17 +51,17 @@ var ( jar = pctx.AndroidStaticRule("jar", blueprint.RuleParams{ - Command: `${config.SoongZipCmd} -o $out -d $jarArgs`, - CommandDeps: []string{"${config.SoongZipCmd}"}, + Command: `${config.JarCmd} $operation ${out}.tmp $manifest $jarArgs && ${config.Zip2ZipCmd} -t -i ${out}.tmp -o ${out} && rm ${out}.tmp`, + CommandDeps: []string{"${config.JarCmd}"}, }, - "jarCmd", "jarArgs") + "operation", "manifest", "jarArgs") dx = pctx.AndroidStaticRule("dx", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `${config.DxCmd} --dex --output=$outDir $dxFlags $in || ( rm -rf "$outDir"; exit 41 ) && ` + - `find "$outDir" -name "classes*.dex" | sort > $out`, - CommandDeps: []string{"${config.DxCmd}"}, + `${config.DxCmd} --dex --output=$outDir $dxFlags $in && ` + + `find "$outDir" -name "classes*.dex" | sort | ${config.JarArgsCmd} ${outDir} > $out`, + CommandDeps: []string{"${config.DxCmd}", "${config.JarArgsCmd}"}, }, "outDir", "dxFlags") @@ -74,11 +75,18 @@ var ( extractPrebuilt = pctx.AndroidStaticRule("extractPrebuilt", blueprint.RuleParams{ Command: `rm -rf $outDir && unzip -qo $in -d $outDir && ` + - `find $outDir -name "*.class" > $classFile && ` + - `find $outDir -type f -a \! -name "*.class" -a \! -name "MANIFEST.MF" > $resourceFile || ` + - `(rm -rf $outDir; exit 42)`, + `find $outDir -name "*.class" | sort | ${config.JarArgsCmd} ${outDir} > $classFile && ` + + `find $outDir -type f -a \! -name "*.class" -a \! -name "MANIFEST.MF" | sort | ${config.JarArgsCmd} ${outDir} > $resourceFile`, + CommandDeps: []string{"${config.JarArgsCmd}"}, }, "outDir", "classFile", "resourceFile") + + fileListToJarArgs = pctx.AndroidStaticRule("fileListToJarArgs", + blueprint.RuleParams{ + Command: `${config.JarArgsCmd} -f $in -p ${outDir} -o $out`, + CommandDeps: []string{"${config.JarjarCmd}"}, + }, + "outDir") ) func init() { @@ -95,11 +103,15 @@ type javaBuilderFlags struct { } type jarSpec struct { - fileList, dir android.Path + android.ModuleOutPath } -func (j jarSpec) soongJarArgs() string { - return "-C " + j.dir.String() + " -l " + j.fileList.String() +func (j jarSpec) jarArgs() string { + return "@" + j.String() +} + +func (j jarSpec) path() android.Path { + return j.ModuleOutPath } func TransformJavaToClasses(ctx android.ModuleContext, srcFiles android.Paths, srcFileLists android.Paths, @@ -129,7 +141,7 @@ func TransformJavaToClasses(ctx android.ModuleContext, srcFiles android.Paths, s }, }) - return jarSpec{classFileList, classDir} + return jarSpec{classFileList} } func TransformClassesToJar(ctx android.ModuleContext, classes []jarSpec, @@ -141,13 +153,14 @@ func TransformClassesToJar(ctx android.ModuleContext, classes []jarSpec, jarArgs := []string{} for _, j := range classes { - deps = append(deps, j.fileList) - jarArgs = append(jarArgs, j.soongJarArgs()) + deps = append(deps, j.path()) + jarArgs = append(jarArgs, j.jarArgs()) } + operation := "cf" if manifest.Valid() { + operation = "cfm" deps = append(deps, manifest.Path()) - jarArgs = append(jarArgs, "-m "+manifest.String()) } ctx.ModuleBuild(pctx, android.ModuleBuildParams{ @@ -156,7 +169,9 @@ func TransformClassesToJar(ctx android.ModuleContext, classes []jarSpec, Output: outputFile, Implicits: deps, Args: map[string]string{ - "jarArgs": strings.Join(jarArgs, " "), + "jarArgs": strings.Join(jarArgs, " "), + "operation": operation, + "manifest": manifest.String(), }, }) @@ -180,7 +195,7 @@ func TransformClassesJarToDex(ctx android.ModuleContext, classesJar android.Path }, }) - return jarSpec{outputFile, outDir} + return jarSpec{outputFile} } func TransformDexToJavaLib(ctx android.ModuleContext, resources []jarSpec, @@ -191,12 +206,12 @@ func TransformDexToJavaLib(ctx android.ModuleContext, resources []jarSpec, var jarArgs []string for _, j := range resources { - deps = append(deps, j.fileList) - jarArgs = append(jarArgs, j.soongJarArgs()) + deps = append(deps, j.path()) + jarArgs = append(jarArgs, j.jarArgs()) } - deps = append(deps, dexJarSpec.fileList) - jarArgs = append(jarArgs, dexJarSpec.soongJarArgs()) + deps = append(deps, dexJarSpec.path()) + jarArgs = append(jarArgs, dexJarSpec.jarArgs()) ctx.ModuleBuild(pctx, android.ModuleBuildParams{ Rule: jar, @@ -204,7 +219,8 @@ func TransformDexToJavaLib(ctx android.ModuleContext, resources []jarSpec, Output: outputFile, Implicits: deps, Args: map[string]string{ - "jarArgs": strings.Join(jarArgs, " "), + "operation": "cf", + "jarArgs": strings.Join(jarArgs, " "), }, }) @@ -246,5 +262,21 @@ func TransformPrebuiltJarToClasses(ctx android.ModuleContext, }, }) - return jarSpec{classFileList, classDir}, jarSpec{resourceFileList, classDir} + return jarSpec{classFileList}, jarSpec{resourceFileList} +} + +func TransformFileListToJarSpec(ctx android.ModuleContext, dir, fileListFile android.Path) jarSpec { + outputFile := android.PathForModuleOut(ctx, fileListFile.Base()+".jarArgs") + + ctx.ModuleBuild(pctx, android.ModuleBuildParams{ + Rule: fileListToJarArgs, + Description: "file list to jar args", + Output: outputFile, + Input: fileListFile, + Args: map[string]string{ + "outDir": dir.String(), + }, + }) + + return jarSpec{outputFile} } diff --git a/java/config/config.go b/java/config/config.go index 44651cb4c..90d0fb51c 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -53,7 +53,8 @@ func init() { pctx.SourcePathVariable("JarCmd", "${JavaToolchain}/jar") pctx.SourcePathVariable("JavadocCmd", "${JavaToolchain}/javadoc") - pctx.StaticVariable("SoongZipCmd", filepath.Join("${bootstrap.ToolDir}", "soong_zip")) + pctx.StaticVariable("Zip2ZipCmd", filepath.Join("${bootstrap.ToolDir}", "zip2zip")) + pctx.SourcePathVariable("JarArgsCmd", "build/soong/scripts/jar-args.sh") pctx.HostBinToolVariable("DxCmd", "dx") pctx.HostJavaToolVariable("JarjarCmd", "jarjar.jar") diff --git a/java/java.go b/java/java.go index 4c614e5df..ac88020cc 100644 --- a/java/java.go +++ b/java/java.go @@ -486,6 +486,9 @@ type Binary struct { Library binaryProperties binaryProperties + + wrapperFile android.ModuleSrcPath + binaryFile android.OutputPath } func (j *Binary) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -493,8 +496,9 @@ func (j *Binary) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Depend on the installed jar (j.installFile) so that the wrapper doesn't get executed by // another build rule before the jar has been installed. - ctx.InstallFile(android.PathForModuleInstall(ctx, "bin"), android.PathForModuleSrc(ctx, j.binaryProperties.Wrapper), - j.installFile) + j.wrapperFile = android.PathForModuleSrc(ctx, j.binaryProperties.Wrapper) + j.binaryFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "bin"), + j.wrapperFile, j.installFile) } func (j *Binary) DepsMutator(ctx android.BottomUpMutatorContext) { diff --git a/java/resources.go b/java/resources.go index 60dc9349f..f1c9d0659 100644 --- a/java/resources.go +++ b/java/resources.go @@ -63,7 +63,7 @@ func ResourceDirsToJarSpecs(ctx android.ModuleContext, resourceDirs, excludeDirs pattern := filepath.Join(dir.String(), "**/*") bootstrap.GlobFile(ctx, pattern, excludes, fileListFile.String(), depFile) - jarSpecs = append(jarSpecs, jarSpec{fileListFile, dir}) + jarSpecs = append(jarSpecs, TransformFileListToJarSpec(ctx, dir, fileListFile)) } } diff --git a/scripts/jar-args.sh b/scripts/jar-args.sh new file mode 100755 index 000000000..9f05394d3 --- /dev/null +++ b/scripts/jar-args.sh @@ -0,0 +1,64 @@ +#!/bin/bash -e + +# Script that takes a list of files on stdin and converts it to arguments to jar on stdout +# Usage: +# find $dir -type f | sort | jar-args.sh $dir > jar_args.txt +# jar cf out.jar @jar_args.txt + +case $(uname) in + Linux) + extended_re=-r + ;; + Darwin) + extended_re=-E + ;; + *) echo "unknown OS:" $(uname) >&2 && exit 1;; +esac + +if [ "$1" == "--test" ]; then + in=$(mktemp) + expected=$(mktemp) + out=$(mktemp) + cat > $in <<EOF +a +a/b +a/b/' +a/b/" +a/b/\\ +a/b/# +a/b/a +EOF + cat > $expected <<EOF + +-C 'a' 'b' +-C 'a' 'b/\\'' +-C 'a' 'b/"' +-C 'a' 'b/\\\\' +-C 'a' 'b/#' +-C 'a' 'b/a' +EOF + cat $in | $0 a > $out + + if cmp $out $expected; then + status=0 + echo "PASS" + else + status=1 + echo "FAIL" + echo "got:" + cat $out + echo "expected:" + cat $expected + fi + rm -f $in $expected $out + exit $status +fi + +# In order, the regexps: +# - Strip $1/ from the beginning of each line, and everything from lines that just have $1 +# - Escape single and double quotes, '#', ' ', and '\' +# - Prefix each non-blank line with -C $1 +sed ${extended_re} \ + -e"s,^$1(/|\$),," \ + -e"s,(['\\]),\\\\\1,g" \ + -e"s,^(.+),-C '$1' '\1'," |