diff options
Diffstat (limited to 'android')
| -rw-r--r-- | android/Android.bp | 1 | ||||
| -rw-r--r-- | android/androidmk.go | 1 | ||||
| -rw-r--r-- | android/config.go | 6 | ||||
| -rw-r--r-- | android/defs.go | 108 | ||||
| -rw-r--r-- | android/packaging.go | 15 | ||||
| -rw-r--r-- | android/raw_files.go | 279 | ||||
| -rw-r--r-- | android/register.go | 3 | ||||
| -rw-r--r-- | android/rule_builder_test.go | 13 | ||||
| -rw-r--r-- | android/util.go | 30 |
9 files changed, 338 insertions, 118 deletions
diff --git a/android/Android.bp b/android/Android.bp index b359df9dd..f71f34f18 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -81,6 +81,7 @@ bootstrap_go_package { "prebuilt_build_tool.go", "proto.go", "provider.go", + "raw_files.go", "register.go", "rule_builder.go", "sandbox.go", diff --git a/android/androidmk.go b/android/androidmk.go index a0ed1e449..004ae9edf 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -852,6 +852,7 @@ func translateAndroidModule(ctx SingletonContext, w io.Writer, mod blueprint.Mod case "*java.SystemModules": // doesn't go through base_rules case "*java.systemModulesImport": // doesn't go through base_rules case "*phony.phony": // license properties written + case "*phony.PhonyRule": // writes phony deps and acts like `.PHONY` case "*selinux.selinuxContextsModule": // license properties written case "*sysprop.syspropLibrary": // license properties written default: diff --git a/android/config.go b/android/config.go index 312a5da49..24b9b8a62 100644 --- a/android/config.go +++ b/android/config.go @@ -18,6 +18,7 @@ package android // product variables necessary for soong_build's operation. import ( + "android/soong/shared" "encoding/json" "fmt" "os" @@ -118,6 +119,11 @@ func (c Config) SoongOutDir() string { return c.soongOutDir } +// tempDir returns the path to out/soong/.temp, which is cleared at the beginning of every build. +func (c Config) tempDir() string { + return shared.TempDirForOutDir(c.soongOutDir) +} + func (c Config) OutDir() string { return c.outDir } diff --git a/android/defs.go b/android/defs.go index 03968c10d..a9889648b 100644 --- a/android/defs.go +++ b/android/defs.go @@ -15,13 +15,8 @@ package android import ( - "fmt" - "strings" - "testing" - "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" - "github.com/google/blueprint/proptools" ) var ( @@ -72,8 +67,7 @@ var ( Command: "if ! cmp -s $in $out; then cp $in $out; fi", Description: "cp if changed $out", Restat: true, - }, - "cpFlags") + }) CpExecutable = pctx.AndroidStaticRule("CpExecutable", blueprint.RuleParams{ @@ -146,106 +140,6 @@ func BazelCcToolchainVars(config Config) string { return BazelToolchainVars(config, exportedVars) } -var ( - // echoEscaper escapes a string such that passing it to "echo -e" will produce the input value. - echoEscaper = strings.NewReplacer( - `\`, `\\`, // First escape existing backslashes so they aren't interpreted by `echo -e`. - "\n", `\n`, // Then replace newlines with \n - ) - - // echoEscaper reverses echoEscaper. - echoUnescaper = strings.NewReplacer( - `\n`, "\n", - `\\`, `\`, - ) - - // shellUnescaper reverses the replacer in proptools.ShellEscape - shellUnescaper = strings.NewReplacer(`'\''`, `'`) -) - -func buildWriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { - content = echoEscaper.Replace(content) - content = proptools.NinjaEscape(proptools.ShellEscapeIncludingSpaces(content)) - if content == "" { - content = "''" - } - ctx.Build(pctx, BuildParams{ - Rule: writeFile, - Output: outputFile, - Description: "write " + outputFile.Base(), - Args: map[string]string{ - "content": content, - }, - }) -} - -// WriteFileRule creates a ninja rule to write contents to a file. The contents will be escaped -// so that the file contains exactly the contents passed to the function, plus a trailing newline. -func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { - WriteFileRuleVerbatim(ctx, outputFile, content+"\n") -} - -// WriteFileRuleVerbatim creates a ninja rule to write contents to a file. The contents will be -// escaped so that the file contains exactly the contents passed to the function. -func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { - // This is MAX_ARG_STRLEN subtracted with some safety to account for shell escapes - const SHARD_SIZE = 131072 - 10000 - - if len(content) > SHARD_SIZE { - var chunks WritablePaths - for i, c := range ShardString(content, SHARD_SIZE) { - tempPath := outputFile.ReplaceExtension(ctx, fmt.Sprintf("%s.%d", outputFile.Ext(), i)) - buildWriteFileRule(ctx, tempPath, c) - chunks = append(chunks, tempPath) - } - ctx.Build(pctx, BuildParams{ - Rule: Cat, - Inputs: chunks.Paths(), - Output: outputFile, - Description: "Merging to " + outputFile.Base(), - }) - return - } - buildWriteFileRule(ctx, outputFile, content) -} - -// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result -func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { - intermediate := PathForIntermediates(ctx, "write_executable_file_intermediates").Join(ctx, outputFile.String()) - WriteFileRuleVerbatim(ctx, intermediate, content) - ctx.Build(pctx, BuildParams{ - Rule: CpExecutable, - Output: outputFile, - Input: intermediate, - }) -} - -// shellUnescape reverses proptools.ShellEscape -func shellUnescape(s string) string { - // Remove leading and trailing quotes if present - if len(s) >= 2 && s[0] == '\'' { - s = s[1 : len(s)-1] - } - s = shellUnescaper.Replace(s) - return s -} - -// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use -// in tests. -func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string { - t.Helper() - if g, w := params.Rule, writeFile; g != w { - t.Errorf("expected params.Rule to be %q, was %q", w, g) - return "" - } - - content := params.Args["content"] - content = shellUnescape(content) - content = echoUnescaper.Replace(content) - - return content -} - // GlobToListFileRule creates a rule that writes a list of files matching a pattern to a file. func GlobToListFileRule(ctx ModuleContext, pattern string, excludes []string, file WritablePath) { bootstrap.GlobFile(ctx.blueprintModuleContext(), pattern, excludes, file.String()) diff --git a/android/packaging.go b/android/packaging.go index 503bb97e0..2506378d7 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -17,6 +17,7 @@ package android import ( "fmt" "path/filepath" + "strings" "github.com/google/blueprint" ) @@ -240,6 +241,9 @@ func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]Packa // entries into the specified directory. func (p *PackagingBase) CopySpecsToDir(ctx ModuleContext, builder *RuleBuilder, specs map[string]PackagingSpec, dir WritablePath) (entries []string) { seenDir := make(map[string]bool) + preparerPath := PathForModuleOut(ctx, "preparer.sh") + cmd := builder.Command().Tool(preparerPath) + var sb strings.Builder for _, k := range SortedKeys(specs) { ps := specs[k] destPath := filepath.Join(dir.String(), ps.relPathInPackage) @@ -247,18 +251,21 @@ func (p *PackagingBase) CopySpecsToDir(ctx ModuleContext, builder *RuleBuilder, entries = append(entries, ps.relPathInPackage) if _, ok := seenDir[destDir]; !ok { seenDir[destDir] = true - builder.Command().Text("mkdir").Flag("-p").Text(destDir) + sb.WriteString(fmt.Sprintf("mkdir -p %s\n", destDir)) } if ps.symlinkTarget == "" { - builder.Command().Text("cp").Input(ps.srcPath).Text(destPath) + cmd.Implicit(ps.srcPath) + sb.WriteString(fmt.Sprintf("cp %s %s\n", ps.srcPath, destPath)) } else { - builder.Command().Text("ln").Flag("-sf").Text(ps.symlinkTarget).Text(destPath) + sb.WriteString(fmt.Sprintf("ln -sf %s %s\n", ps.symlinkTarget, destPath)) } if ps.executable { - builder.Command().Text("chmod").Flag("a+x").Text(destPath) + sb.WriteString(fmt.Sprintf("chmod a+x %s\n", destPath)) } } + WriteExecutableFileRuleVerbatim(ctx, preparerPath, sb.String()) + return entries } diff --git a/android/raw_files.go b/android/raw_files.go new file mode 100644 index 000000000..9d7f5e82d --- /dev/null +++ b/android/raw_files.go @@ -0,0 +1,279 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "github.com/google/blueprint" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/blueprint/proptools" +) + +// WriteFileRule creates a ninja rule to write contents to a file by immediately writing the +// contents, plus a trailing newline, to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating +// a ninja rule to copy the file into place. +func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { + writeFileRule(ctx, outputFile, content, true, false) +} + +// WriteFileRuleVerbatim creates a ninja rule to write contents to a file by immediately writing the +// contents to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating a ninja rule to copy the file into place. +func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { + writeFileRule(ctx, outputFile, content, false, false) +} + +// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result +func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { + writeFileRule(ctx, outputFile, content, false, true) +} + +// tempFile provides a testable wrapper around a file in out/soong/.temp. It writes to a temporary file when +// not in tests, but writes to a buffer in memory when used in tests. +type tempFile struct { + // tempFile contains wraps an io.Writer, which will be file if testMode is false, or testBuf if it is true. + io.Writer + + file *os.File + testBuf *strings.Builder +} + +func newTempFile(ctx BuilderContext, pattern string, testMode bool) *tempFile { + if testMode { + testBuf := &strings.Builder{} + return &tempFile{ + Writer: testBuf, + testBuf: testBuf, + } + } else { + f, err := os.CreateTemp(absolutePath(ctx.Config().tempDir()), pattern) + if err != nil { + panic(fmt.Errorf("failed to open temporary raw file: %w", err)) + } + return &tempFile{ + Writer: f, + file: f, + } + } +} + +func (t *tempFile) close() error { + if t.file != nil { + return t.file.Close() + } + return nil +} + +func (t *tempFile) name() string { + if t.file != nil { + return t.file.Name() + } + return "temp_file_in_test" +} + +func (t *tempFile) rename(to string) { + if t.file != nil { + os.MkdirAll(filepath.Dir(to), 0777) + err := os.Rename(t.file.Name(), to) + if err != nil { + panic(fmt.Errorf("failed to rename %s to %s: %w", t.file.Name(), to, err)) + } + } +} + +func (t *tempFile) remove() error { + if t.file != nil { + return os.Remove(t.file.Name()) + } + return nil +} + +func writeContentToTempFileAndHash(ctx BuilderContext, content string, newline bool) (*tempFile, string) { + tempFile := newTempFile(ctx, "raw", ctx.Config().captureBuild) + defer tempFile.close() + + hash := sha1.New() + w := io.MultiWriter(tempFile, hash) + + _, err := io.WriteString(w, content) + if err == nil && newline { + _, err = io.WriteString(w, "\n") + } + if err != nil { + panic(fmt.Errorf("failed to write to temporary raw file %s: %w", tempFile.name(), err)) + } + return tempFile, hex.EncodeToString(hash.Sum(nil)) +} + +func writeFileRule(ctx BuilderContext, outputFile WritablePath, content string, newline bool, executable bool) { + // Write the contents to a temporary file while computing its hash. + tempFile, hash := writeContentToTempFileAndHash(ctx, content, newline) + + // Shard the final location of the raw file into a subdirectory based on the first two characters of the + // hash to avoid making the raw directory too large and slowing down accesses. + relPath := filepath.Join(hash[0:2], hash) + + // These files are written during soong_build. If something outside the build deleted them there would be no + // trigger to rerun soong_build, and the build would break with dependencies on missing files. Writing them + // to their final locations would risk having them deleted when cleaning a module, and would also pollute the + // output directory with files for modules that have never been built. + // Instead, the files are written to a separate "raw" directory next to the build.ninja file, and a ninja + // rule is created to copy the files into their final location as needed. + // Obsolete files written by previous runs of soong_build must be cleaned up to avoid continually growing + // disk usage as the hashes of the files change over time. The cleanup must not remove files that were + // created by previous runs of soong_build for other products, as the build.ninja files for those products + // may still exist and still reference those files. The raw files from different products are kept + // separate by appending the Make_suffix to the directory name. + rawPath := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix), relPath) + + rawFileInfo := rawFileInfo{ + relPath: relPath, + } + + if ctx.Config().captureBuild { + // When running tests tempFile won't write to disk, instead store the contents for later retrieval by + // ContentFromFileRuleForTests. + rawFileInfo.contentForTests = tempFile.testBuf.String() + } + + rawFileSet := getRawFileSet(ctx.Config()) + if _, exists := rawFileSet.LoadOrStore(hash, rawFileInfo); exists { + // If a raw file with this hash has already been created delete the temporary file. + tempFile.remove() + } else { + // If this is the first time this hash has been seen then move it from the temporary directory + // to the raw directory. If the file already exists in the raw directory assume it has the correct + // contents. + absRawPath := absolutePath(rawPath.String()) + _, err := os.Stat(absRawPath) + if os.IsNotExist(err) { + tempFile.rename(absRawPath) + } else if err != nil { + panic(fmt.Errorf("failed to stat %q: %w", absRawPath, err)) + } else { + tempFile.remove() + } + } + + // Emit a rule to copy the file from raw directory to the final requested location in the output tree. + // Restat is used to ensure that two different products that produce identical files copied from their + // own raw directories they don't cause everything downstream to rebuild. + rule := rawFileCopy + if executable { + rule = rawFileCopyExecutable + } + ctx.Build(pctx, BuildParams{ + Rule: rule, + Input: rawPath, + Output: outputFile, + Description: "raw " + outputFile.Base(), + }) +} + +var ( + rawFileCopy = pctx.AndroidStaticRule("rawFileCopy", + blueprint.RuleParams{ + Command: "if ! cmp -s $in $out; then cp $in $out; fi", + Description: "copy raw file $out", + Restat: true, + }) + rawFileCopyExecutable = pctx.AndroidStaticRule("rawFileCopyExecutable", + blueprint.RuleParams{ + Command: "if ! cmp -s $in $out; then cp $in $out; fi && chmod +x $out", + Description: "copy raw exectuable file $out", + Restat: true, + }) +) + +type rawFileInfo struct { + relPath string + contentForTests string +} + +var rawFileSetKey OnceKey = NewOnceKey("raw file set") + +func getRawFileSet(config Config) *SyncMap[string, rawFileInfo] { + return config.Once(rawFileSetKey, func() any { + return &SyncMap[string, rawFileInfo]{} + }).(*SyncMap[string, rawFileInfo]) +} + +// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use +// in tests. +func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string { + t.Helper() + if params.Rule != rawFileCopy && params.Rule != rawFileCopyExecutable { + t.Errorf("expected params.Rule to be rawFileCopy or rawFileCopyExecutable, was %q", params.Rule) + return "" + } + + key := filepath.Base(params.Input.String()) + rawFileSet := getRawFileSet(ctx.Config()) + rawFileInfo, _ := rawFileSet.Load(key) + + return rawFileInfo.contentForTests +} + +func rawFilesSingletonFactory() Singleton { + return &rawFilesSingleton{} +} + +type rawFilesSingleton struct{} + +func (rawFilesSingleton) GenerateBuildActions(ctx SingletonContext) { + if ctx.Config().captureBuild { + // Nothing to do when running in tests, no temporary files were created. + return + } + rawFileSet := getRawFileSet(ctx.Config()) + rawFilesDir := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix)).String() + absRawFilesDir := absolutePath(rawFilesDir) + err := filepath.WalkDir(absRawFilesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Ignore obsolete directories for now. + return nil + } + + // Assume the basename of the file is a hash + key := filepath.Base(path) + relPath, err := filepath.Rel(absRawFilesDir, path) + if err != nil { + return err + } + + // Check if a file with the same hash was written by this run of soong_build. If the file was not written, + // or if a file with the same hash was written but to a different path in the raw directory, then delete it. + // Checking that the path matches allows changing the structure of the raw directory, for example to increase + // the sharding. + rawFileInfo, written := rawFileSet.Load(key) + if !written || rawFileInfo.relPath != relPath { + os.Remove(path) + } + return nil + }) + if err != nil { + panic(fmt.Errorf("failed to clean %q: %w", rawFilesDir, err)) + } +} diff --git a/android/register.go b/android/register.go index cd968cd02..d00c15fd0 100644 --- a/android/register.go +++ b/android/register.go @@ -191,8 +191,9 @@ func collateGloballyRegisteredSingletons() sortableComponents { // Register makevars after other singletons so they can export values through makevars singleton{parallel: false, name: "makevars", factory: makeVarsSingletonFunc}, - // Register env and ninjadeps last so that they can track all used environment variables and + // Register rawfiles and ninjadeps last so that they can track all used environment variables and // Ninja file dependencies stored in the config. + singleton{parallel: false, name: "rawfiles", factory: rawFilesSingletonFactory}, singleton{parallel: false, name: "ninjadeps", factory: ninjaDepsSingletonFactory}, ) diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index d659dcca7..9e5f12dff 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -816,13 +816,13 @@ func TestRuleBuilderHashInputs(t *testing.T) { func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) { bp := ` rule_builder_test { - name: "foo_sbox_escaped_ninja", + name: "foo_sbox_escaped", flags: ["${cmdFlags}"], sbox: true, sbox_inputs: true, } rule_builder_test { - name: "foo_sbox", + name: "foo_sbox_unescaped", flags: ["${cmdFlags}"], sbox: true, sbox_inputs: true, @@ -834,15 +834,16 @@ func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) { FixtureWithRootAndroidBp(bp), ).RunTest(t) - escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped_ninja", "").Rule("writeFile") + escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped", "").Output("sbox.textproto") + AssertStringEquals(t, "expected rule", "android/soong/android.rawFileCopy", escapedNinjaMod.Rule.String()) AssertStringDoesContain( t, "", - escapedNinjaMod.BuildParams.Args["content"], - "$${cmdFlags}", + ContentFromFileRuleForTests(t, result.TestContext, escapedNinjaMod), + "${cmdFlags}", ) - unescapedNinjaMod := result.ModuleForTests("foo_sbox", "").Rule("unescapedWriteFile") + unescapedNinjaMod := result.ModuleForTests("foo_sbox_unescaped", "").Rule("unescapedWriteFile") AssertStringDoesContain( t, "", diff --git a/android/util.go b/android/util.go index ae1c65756..51313ceec 100644 --- a/android/util.go +++ b/android/util.go @@ -22,6 +22,7 @@ import ( "runtime" "sort" "strings" + "sync" ) // CopyOf returns a new slice that has the same contents as s. @@ -597,3 +598,32 @@ func AddToStringSet(set map[string]bool, items []string) { set[item] = true } } + +// SyncMap is a wrapper around sync.Map that provides type safety via generics. +type SyncMap[K comparable, V any] struct { + sync.Map +} + +// Load returns the value stored in the map for a key, or the zero value if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *SyncMap[K, V]) Load(key K) (value V, ok bool) { + v, ok := m.Map.Load(key) + if !ok { + return *new(V), false + } + return v.(V), true +} + +// Store sets the value for a key. +func (m *SyncMap[K, V]) Store(key K, value V) { + m.Map.Store(key, value) +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *SyncMap[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { + v, loaded := m.Map.LoadOrStore(key, value) + return v.(V), loaded +} |