diff options
| -rw-r--r-- | android/bazel_handler.go | 229 | ||||
| -rw-r--r-- | apex/builder.go | 10 | ||||
| -rw-r--r-- | bazel/aquery.go | 184 | ||||
| -rw-r--r-- | bazel/aquery_test.go | 183 | ||||
| -rw-r--r-- | java/java.go | 14 | ||||
| -rw-r--r-- | java/kotlin.go | 5 |
6 files changed, 281 insertions, 344 deletions
diff --git a/android/bazel_handler.go b/android/bazel_handler.go index f1ec55eb9..3be9805be 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -34,6 +34,21 @@ import ( "android/soong/bazel" ) +var ( + writeBazelFile = pctx.AndroidStaticRule("bazelWriteFileRule", blueprint.RuleParams{ + Command: `sed "s/\\\\n/\n/g" ${out}.rsp >${out}`, + Rspfile: "${out}.rsp", + RspfileContent: "${content}", + }, "content") + _ = pctx.HostBinToolVariable("bazelBuildRunfilesTool", "build-runfiles") + buildRunfilesRule = pctx.AndroidStaticRule("bazelBuildRunfiles", blueprint.RuleParams{ + Command: "${bazelBuildRunfilesTool} ${in} ${outDir}", + Depfile: "", + Description: "", + CommandDeps: []string{"${bazelBuildRunfilesTool}"}, + }, "outDir") +) + func init() { RegisterMixedBuildsMutator(InitRegistrationContext) } @@ -173,26 +188,26 @@ type MockBazelContext struct { LabelToPythonBinary map[string]string } -func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { +func (m MockBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) { panic("unimplemented") } -func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { +func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) { result, _ := m.LabelToOutputFiles[label] return result, nil } -func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { +func (m MockBazelContext) GetCcInfo(label string, _ configKey) (cquery.CcInfo, error) { result, _ := m.LabelToCcInfo[label] return result, nil } -func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { +func (m MockBazelContext) GetPythonBinary(label string, _ configKey) (string, error) { result, _ := m.LabelToPythonBinary[label] return result, nil } -func (m MockBazelContext) InvokeBazel(config Config) error { +func (m MockBazelContext) InvokeBazel(_ Config) error { panic("unimplemented") } @@ -246,23 +261,23 @@ func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (s return "", fmt.Errorf("no bazel response found for %v", key) } -func (n noopBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { +func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) { panic("unimplemented") } -func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { +func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) { panic("unimplemented") } -func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { +func (n noopBazelContext) GetCcInfo(_ string, _ configKey) (cquery.CcInfo, error) { panic("unimplemented") } -func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { +func (n noopBazelContext) GetPythonBinary(_ string, _ configKey) (string, error) { panic("unimplemented") } -func (n noopBazelContext) InvokeBazel(config Config) error { +func (n noopBazelContext) InvokeBazel(_ Config) error { panic("unimplemented") } @@ -304,7 +319,7 @@ func bazelPathsFromConfig(c *config) (*bazelPaths, error) { p := bazelPaths{ soongOutDir: c.soongOutDir, } - missingEnvVars := []string{} + var missingEnvVars []string if len(c.Getenv("BAZEL_HOME")) > 1 { p.homeDir = c.Getenv("BAZEL_HOME") } else { @@ -365,10 +380,8 @@ type mockBazelRunner struct { extraFlags []string } -func (r *mockBazelRunner) issueBazelCommand(paths *bazelPaths, - runName bazel.RunName, - command bazelCommand, - extraFlags ...string) (string, string, error) { +func (r *mockBazelRunner) issueBazelCommand(_ *bazelPaths, _ bazel.RunName, + command bazelCommand, extraFlags ...string) (string, string, error) { r.commands = append(r.commands, command) r.extraFlags = append(r.extraFlags, strings.Join(extraFlags, " ")) if ret, ok := r.bazelCommandResults[command]; ok { @@ -396,26 +409,30 @@ func (r *builtinBazelRunner) issueBazelCommand(paths *bazelPaths, runName bazel. command.command, } cmdFlags = append(cmdFlags, command.expression) - cmdFlags = append(cmdFlags, "--profile="+shared.BazelMetricsFilename(paths, runName)) - - // Set default platforms to canonicalized values for mixed builds requests. - // If these are set in the bazelrc, they will have values that are - // non-canonicalized to @sourceroot labels, and thus be invalid when - // referenced from the buildroot. - // - // The actual platform values here may be overridden by configuration - // transitions from the buildroot. - cmdFlags = append(cmdFlags, - fmt.Sprintf("--platforms=%s", "//build/bazel/platforms:android_target")) cmdFlags = append(cmdFlags, - fmt.Sprintf("--extra_toolchains=%s", "//prebuilts/clang/host/linux-x86:all")) - // This should be parameterized on the host OS, but let's restrict to linux - // to keep things simple for now. - cmdFlags = append(cmdFlags, - fmt.Sprintf("--host_platform=%s", "//build/bazel/platforms:linux_x86_64")) - - // Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network. - cmdFlags = append(cmdFlags, "--experimental_repository_disable_download") + // TODO(asmundak): is it needed in every build? + "--profile="+shared.BazelMetricsFilename(paths, runName), + + // Set default platforms to canonicalized values for mixed builds requests. + // If these are set in the bazelrc, they will have values that are + // non-canonicalized to @sourceroot labels, and thus be invalid when + // referenced from the buildroot. + // + // The actual platform values here may be overridden by configuration + // transitions from the buildroot. + fmt.Sprintf("--platforms=%s", "//build/bazel/platforms:android_target"), + fmt.Sprintf("--extra_toolchains=%s", "//prebuilts/clang/host/linux-x86:all"), + + // This should be parameterized on the host OS, but let's restrict to linux + // to keep things simple for now. + fmt.Sprintf("--host_platform=%s", "//build/bazel/platforms:linux_x86_64"), + + // Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network. + "--experimental_repository_disable_download", + + // Suppress noise + "--ui_event_filters=-INFO", + "--noshow_progress") cmdFlags = append(cmdFlags, extraFlags...) bazelCmd := exec.Command(paths.bazelPath, cmdFlags...) @@ -682,8 +699,6 @@ func (p *bazelPaths) outDir() string { func (context *bazelContext) InvokeBazel(config Config) error { context.results = make(map[cqueryKey]string) - var cqueryOutput string - var cqueryErr string var err error soongInjectionPath := absolutePath(context.paths.injectedFilesDir()) @@ -700,45 +715,27 @@ func (context *bazelContext) InvokeBazel(config Config) error { return err } } - err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666) - if err != nil { + if err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "WORKSPACE.bazel"), []byte{}, 0666); err != nil { return err } - - err = ioutil.WriteFile( - filepath.Join(mixedBuildsPath, "main.bzl"), - context.mainBzlFileContents(), 0666) - if err != nil { + if err = ioutil.WriteFile(filepath.Join(mixedBuildsPath, "main.bzl"), context.mainBzlFileContents(), 0666); err != nil { return err } - - err = ioutil.WriteFile( - filepath.Join(mixedBuildsPath, "BUILD.bazel"), - context.mainBuildFileContents(), 0666) - if err != nil { + if err = ioutil.WriteFile(filepath.Join(mixedBuildsPath, "BUILD.bazel"), context.mainBuildFileContents(), 0666); err != nil { return err } cqueryFileRelpath := filepath.Join(context.paths.injectedFilesDir(), "buildroot.cquery") - err = ioutil.WriteFile( - absolutePath(cqueryFileRelpath), - context.cqueryStarlarkFileContents(), 0666) - if err != nil { + if err = ioutil.WriteFile(absolutePath(cqueryFileRelpath), context.cqueryStarlarkFileContents(), 0666); err != nil { return err } - buildrootLabel := "@soong_injection//mixed_builds:buildroot" - cqueryOutput, cqueryErr, err = context.issueBazelCommand( - context.paths, - bazel.CqueryBuildRootRunName, - bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)}, - "--output=starlark", - "--starlark:file="+absolutePath(cqueryFileRelpath)) - err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), - []byte(cqueryOutput), 0666) + const buildrootLabel = "@soong_injection//mixed_builds:buildroot" + cqueryCmd := bazelCommand{"cquery", fmt.Sprintf("deps(%s, 2)", buildrootLabel)} + cqueryOutput, cqueryErr, err := context.issueBazelCommand(context.paths, bazel.CqueryBuildRootRunName, cqueryCmd, + "--output=starlark", "--starlark:file="+absolutePath(cqueryFileRelpath)) if err != nil { - return err + err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryOutput), 0666) } - if err != nil { return err } @@ -750,7 +747,6 @@ func (context *bazelContext) InvokeBazel(config Config) error { cqueryResults[splitLine[0]] = splitLine[1] } } - for val := range context.requests { if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok { context.results[val] = cqueryResult @@ -762,37 +758,27 @@ func (context *bazelContext) InvokeBazel(config Config) error { // Issue an aquery command to retrieve action information about the bazel build tree. // - var aqueryOutput string - var coverageFlags []string + // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's + // proto sources, which would add a number of unnecessary dependencies. + extraFlags := []string{"--output=jsonproto", "--include_file_write_contents"} if Bool(config.productVariables.ClangCoverage) { - coverageFlags = append(coverageFlags, "--collect_code_coverage") - if len(config.productVariables.NativeCoveragePaths) > 0 || - len(config.productVariables.NativeCoverageExcludePaths) > 0 { - includePaths := JoinWithPrefixAndSeparator(config.productVariables.NativeCoveragePaths, "+", ",") - excludePaths := JoinWithPrefixAndSeparator(config.productVariables.NativeCoverageExcludePaths, "-", ",") - if len(includePaths) > 0 && len(excludePaths) > 0 { - includePaths += "," - } - coverageFlags = append(coverageFlags, fmt.Sprintf(`--instrumentation_filter=%s`, - includePaths+excludePaths)) + extraFlags = append(extraFlags, "--collect_code_coverage") + paths := make([]string, 0, 2) + if p := config.productVariables.NativeCoveragePaths; len(p) > 0 { + paths = append(paths, JoinWithPrefixAndSeparator(p, "+", ",")) + } + if p := config.productVariables.NativeCoverageExcludePaths; len(p) > 0 { + paths = append(paths, JoinWithPrefixAndSeparator(p, "-", ",")) + } + if len(paths) > 0 { + extraFlags = append(extraFlags, "--instrumentation_filter="+strings.Join(paths, ",")) } } - - extraFlags := append([]string{"--output=jsonproto"}, coverageFlags...) - - aqueryOutput, _, err = context.issueBazelCommand( - context.paths, - bazel.AqueryBuildRootRunName, - bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)}, - // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's - // proto sources, which would add a number of unnecessary dependencies. - extraFlags...) - - if err != nil { - return err + aqueryCmd := bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)} + if aqueryOutput, _, err := context.issueBazelCommand(context.paths, bazel.AqueryBuildRootRunName, aqueryCmd, + extraFlags...); err == nil { + context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) } - - context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) if err != nil { return err } @@ -800,12 +786,8 @@ func (context *bazelContext) InvokeBazel(config Config) error { // Issue a build command of the phony root to generate symlink forests for dependencies of the // Bazel build. This is necessary because aquery invocations do not generate this symlink forest, // but some of symlinks may be required to resolve source dependencies of the build. - _, _, err = context.issueBazelCommand( - context.paths, - bazel.BazelBuildPhonyRootRunName, - bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"}) - - if err != nil { + buildCmd := bazelCommand{"build", "@soong_injection//mixed_builds:phonyroot"} + if _, _, err = context.issueBazelCommand(context.paths, bazel.BazelBuildPhonyRootRunName, buildCmd); err != nil { return err } @@ -874,13 +856,56 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__") bazelOutDir := path.Join(executionRoot, "bazel-out") for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { - if len(buildStatement.Command) < 1 { + if len(buildStatement.Command) > 0 { + rule := NewRuleBuilder(pctx, ctx) + createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx) + desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) + rule.Build(fmt.Sprintf("bazel %d", index), desc) + continue + } + // Certain actions returned by aquery (for instance FileWrite) do not contain a command + // and thus require special treatment. If BuildStatement were an interface implementing + // buildRule(ctx) function, the code here would just call it. + // Unfortunately, the BuildStatement is defined in + // the 'bazel' package, which cannot depend on 'android' package where ctx is defined, + // because this would cause circular dependency. So, until we move aquery processing + // to the 'android' package, we need to handle special cases here. + if buildStatement.Mnemonic == "FileWrite" || buildStatement.Mnemonic == "SourceSymlinkManifest" { + // Pass file contents as the value of the rule's "content" argument. + // Escape newlines and $ in the contents (the action "writeBazelFile" restores "\\n" + // back to the newline, and Ninja reads $$ as $. + escaped := strings.ReplaceAll(strings.ReplaceAll(buildStatement.FileContents, "\n", "\\n"), + "$", "$$") + ctx.Build(pctx, BuildParams{ + Rule: writeBazelFile, + Output: PathForBazelOut(ctx, buildStatement.OutputPaths[0]), + Description: fmt.Sprintf("%s %s", buildStatement.Mnemonic, buildStatement.OutputPaths[0]), + Args: map[string]string{ + "content": escaped, + }, + }) + } else if buildStatement.Mnemonic == "SymlinkTree" { + // build-runfiles arguments are the manifest file and the target directory + // where it creates the symlink tree according to this manifest (and then + // writes the MANIFEST file to it). + outManifest := PathForBazelOut(ctx, buildStatement.OutputPaths[0]) + outManifestPath := outManifest.String() + if !strings.HasSuffix(outManifestPath, "MANIFEST") { + panic("the base name of the symlink tree action should be MANIFEST, got " + outManifestPath) + } + outDir := filepath.Dir(outManifestPath) + ctx.Build(pctx, BuildParams{ + Rule: buildRunfilesRule, + Output: outManifest, + Inputs: []Path{PathForBazelOut(ctx, buildStatement.InputPaths[0])}, + Description: "symlink tree for " + outDir, + Args: map[string]string{ + "outDir": outDir, + }, + }) + } else { panic(fmt.Sprintf("unhandled build statement: %v", buildStatement)) } - rule := NewRuleBuilder(pctx, ctx) - createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx) - desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) - rule.Build(fmt.Sprintf("bazel %d", index), desc) } } diff --git a/apex/builder.go b/apex/builder.go index 7e2b924c7..e3c447632 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -112,6 +112,7 @@ var ( `--canned_fs_config ${canned_fs_config} ` + `--include_build_info ` + `--payload_type image ` + + `--apex_version ${apex_version} ` + `--key ${key} ${opt_flags} ${image_dir} ${out} `, CommandDeps: []string{"${apexer}", "${avbtool}", "${e2fsdroid}", "${merge_zips}", "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}", "${sload_f2fs}", "${make_erofs}", @@ -119,7 +120,7 @@ var ( Rspfile: "${out}.copy_commands", RspfileContent: "${copy_commands}", Description: "APEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type") + }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type", "apex_version") zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{ Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + @@ -127,12 +128,13 @@ var ( `APEXER_TOOL_PATH=${tool_path} ` + `${apexer} --force --manifest ${manifest} ` + `--payload_type zip ` + + `--apex_version ${apex_version} ` + `${image_dir} ${out} `, CommandDeps: []string{"${apexer}", "${merge_zips}", "${soong_zip}", "${zipalign}", "${aapt2}"}, Rspfile: "${out}.copy_commands", RspfileContent: "${copy_commands}", Description: "ZipAPEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "manifest") + }, "tool_path", "image_dir", "copy_commands", "manifest", "apex_version") apexProtoConvertRule = pctx.AndroidStaticRule("apexProtoConvertRule", blueprint.RuleParams{ @@ -667,8 +669,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { optFlags = append(optFlags, "--manifest_json "+a.manifestJsonOut.String()) } - optFlags = append(optFlags, "--apex_version "+defaultManifestVersion) - optFlags = append(optFlags, "--payload_fs_type "+a.payloadFsType.string()) ctx.Build(pctx, android.BuildParams{ @@ -684,6 +684,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { "file_contexts": fileContexts.String(), "canned_fs_config": cannedFsConfig.String(), "key": a.privateKeyFile.String(), + "apex_version": defaultManifestVersion, "opt_flags": strings.Join(optFlags, " "), }, }) @@ -780,6 +781,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { "image_dir": imageDir.String(), "copy_commands": strings.Join(copyCommands, " && "), "manifest": a.manifestPbOut.String(), + "apex_version": defaultManifestVersion, }, }) } diff --git a/bazel/aquery.go b/bazel/aquery.go index 1d1f49cdf..ae2b107e1 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -21,7 +21,6 @@ import ( "fmt" "path/filepath" "reflect" - "regexp" "sort" "strings" @@ -83,6 +82,7 @@ type action struct { OutputIds []artifactId TemplateContent string Substitutions []KeyValuePair + FileContents string } // actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer. @@ -110,6 +110,7 @@ type BuildStatement struct { // input path string, but not both. InputDepsetHashes []string InputPaths []string + FileContents string } // A helper type for aquery processing which facilitates retrieval of path IDs from their @@ -139,9 +140,6 @@ var templateActionOverriddenTokens = map[string]string{ "%python_binary%": "python3", } -// This pattern matches the MANIFEST file created for a py_binary target. -var manifestFilePattern = regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$") - // The file name of py3wrapper.sh, which is used by py_binary targets. const py3wrapperFileName = "/py3wrapper.sh" @@ -225,20 +223,12 @@ func (a *aqueryArtifactHandler) populateDepsetMaps(depset depSetOfFiles, middlem // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts. transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...) } else if strings.HasSuffix(path, py3wrapperFileName) || - manifestFilePattern.MatchString(path) || strings.HasPrefix(path, "../bazel_tools") { // Drop these artifacts. // See go/python-binary-host-mixed-build for more details. - // 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of - // Bazel py_binary targets, so there is no Ninja build statements generated for creating it. - // 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, - // but it doesn't contain sufficient information so no Ninja build statements are generated - // for creating it. - // So in mixed build mode, when these two are used as input of some Ninja build statement, - // since there is no build statement to create them, they should be removed from input paths. - // TODO(b/197135294): Clean up this custom runfiles handling logic when - // SourceSymlinkManifest and SymlinkTree actions are supported. - // 3) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the + // 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the + // TemplateExpandAction handles everything necessary to launch a Pythin application. + // 2) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the // containing depset to always be considered newer than their outputs. } else { directArtifactPaths = append(directArtifactPaths, path) @@ -346,12 +336,14 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDe } var buildStatement BuildStatement - if isSymlinkAction(actionEntry) { + if actionEntry.isSymlinkAction() { buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry) - } else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 { + } else if actionEntry.isTemplateExpandAction() && len(actionEntry.Arguments) < 1 { buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry) - } else if isPythonZipperAction(actionEntry) { - buildStatement, err = aqueryHandler.pythonZipperActionBuildStatement(actionEntry, buildStatements) + } else if actionEntry.isFileWriteAction() { + buildStatement, err = aqueryHandler.fileWriteActionBuildStatement(actionEntry) + } else if actionEntry.isSymlinkTreeAction() { + buildStatement, err = aqueryHandler.symlinkTreeActionBuildStatement(actionEntry) } else if len(actionEntry.Arguments) < 1 { return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic) } else { @@ -452,54 +444,6 @@ func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry action) ( return buildStatement, nil } -func (a *aqueryArtifactHandler) pythonZipperActionBuildStatement(actionEntry action, prevBuildStatements []BuildStatement) (BuildStatement, error) { - inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return BuildStatement{}, err - } - outputPaths, depfile, err := a.getOutputPaths(actionEntry) - if err != nil { - return BuildStatement{}, err - } - - if len(inputPaths) < 1 || len(outputPaths) != 1 { - return BuildStatement{}, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths) - } - command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ") - inputPaths, command = removePy3wrapperScript(inputPaths, command) - command = addCommandForPyBinaryRunfilesDir(command, outputPaths[0]) - // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements. - // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input, - // which is not sufficient without the python zip file from which runfiles directory is created for py_binary. - // - // The following logic relies on that Bazel aquery output returns actions in the order that - // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions - // in that order, the following logic might not find the build statement generated for Python binary - // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output. - // See go/python-binary-host-mixed-build for more details. - pythonZipFilePath := outputPaths[0] - pyBinaryFound := false - for i := range prevBuildStatements { - if len(prevBuildStatements[i].OutputPaths) == 1 && prevBuildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath { - prevBuildStatements[i].InputPaths = append(prevBuildStatements[i].InputPaths, pythonZipFilePath) - pyBinaryFound = true - } - } - if !pyBinaryFound { - return BuildStatement{}, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths) - } - - buildStatement := BuildStatement{ - Command: command, - Depfile: depfile, - OutputPaths: outputPaths, - InputPaths: inputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } - return buildStatement, nil -} - func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry action) (BuildStatement, error) { outputPaths, depfile, err := a.getOutputPaths(actionEntry) if err != nil { @@ -532,6 +476,47 @@ func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry a return buildStatement, nil } +func (a *aqueryArtifactHandler) fileWriteActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, _, err := a.getOutputPaths(actionEntry) + var depsetHashes []string + if err == nil { + depsetHashes, err = a.depsetContentHashes(actionEntry.InputDepSetIds) + } + if err != nil { + return BuildStatement{}, err + } + return BuildStatement{ + Depfile: nil, + OutputPaths: outputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + InputDepsetHashes: depsetHashes, + FileContents: actionEntry.FileContents, + }, nil +} + +func (a *aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, _, err := a.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + if len(inputPaths) != 1 || len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) + } + // The actual command is generated in bazelSingleton.GenerateBuildActions + return BuildStatement{ + Depfile: nil, + OutputPaths: outputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + InputPaths: inputPaths, + }, nil +} + func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) { outputPaths, depfile, err := a.getOutputPaths(actionEntry) if err != nil { @@ -614,76 +599,35 @@ func escapeCommandlineArgument(str string) string { return replacer.Replace(str) } -// removePy3wrapperScript removes py3wrapper.sh from the input paths and command of the action of -// creating python zip file in mixed build mode. py3wrapper.sh is returned as input by aquery but -// there is no action returned by aquery for creating it. So in mixed build "python3" is used -// as the PYTHON_BINARY in python binary stub script, and py3wrapper.sh is not needed and should be -// removed from input paths and command of creating python zip file. -// See go/python-binary-host-mixed-build for more details. -// TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode. -func removePy3wrapperScript(inputPaths []string, command string) (newInputPaths []string, newCommand string) { - // Remove from inputs - var filteredInputPaths []string - for _, path := range inputPaths { - if !strings.HasSuffix(path, py3wrapperFileName) { - filteredInputPaths = append(filteredInputPaths, path) - } - } - newInputPaths = filteredInputPaths - - // Remove from command line - var re = regexp.MustCompile(`\S*` + py3wrapperFileName) - newCommand = re.ReplaceAllString(command, "") - return -} - -// addCommandForPyBinaryRunfilesDir adds commands creating python binary runfiles directory. -// runfiles directory is created by using MANIFEST file and MANIFEST file is the output of -// SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, -// but since SourceSymlinkManifest doesn't contain sufficient information -// so MANIFEST file could not be created, which also blocks the creation of runfiles directory. -// See go/python-binary-host-mixed-build for more details. -// TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action. -func addCommandForPyBinaryRunfilesDir(oldCommand string, zipFilePath string) string { - // Unzip the zip file, zipFilePath looks like <python_binary>.zip - runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles" - command := fmt.Sprintf("%s x %s -d %s", "../bazel_tools/tools/zip/zipper/zipper", zipFilePath, runfilesDirName) - // Create a symbolic link in <python_binary>.runfiles/, which is the expected structure - // when running the python binary stub script. - command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName) - return oldCommand + " && " + command -} - -func isSymlinkAction(a action) bool { +func (a action) isSymlinkAction() bool { return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" || a.Mnemonic == "ExecutableSymlink" } -func isTemplateExpandAction(a action) bool { +func (a action) isTemplateExpandAction() bool { return a.Mnemonic == "TemplateExpand" } -func isPythonZipperAction(a action) bool { - return a.Mnemonic == "PythonZipper" +func (a action) isFileWriteAction() bool { + return a.Mnemonic == "FileWrite" || a.Mnemonic == "SourceSymlinkManifest" +} + +func (a action) isSymlinkTreeAction() bool { + return a.Mnemonic == "SymlinkTree" } func shouldSkipAction(a action) bool { - // TODO(b/180945121): Handle complex symlink actions. - if a.Mnemonic == "SymlinkTree" || a.Mnemonic == "SourceSymlinkManifest" { - return true - } // Middleman actions are not handled like other actions; they are handled separately as a // preparatory step so that their inputs may be relayed to actions depending on middleman // artifacts. if a.Mnemonic == "Middleman" { return true } - // Skip "Fail" actions, which are placeholder actions designed to always fail. - if a.Mnemonic == "Fail" { + // PythonZipper is bogus action returned by aquery, ignore it (b/236198693) + if a.Mnemonic == "PythonZipper" { return true } - // TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information - // about the contents that are written. - if a.Mnemonic == "FileWrite" { + // Skip "Fail" actions, which are placeholder actions designed to always fail. + if a.Mnemonic == "Fail" { return true } if a.Mnemonic == "BaselineCoverage" { diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index c759d56ee..3a2bf0f38 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -460,6 +460,43 @@ func TestTransitiveInputDepsets(t *testing.T) { } } +func TestSymlinkTree(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "SymlinkTree", + "configurationId": 1, + "inputDepSetIds": [1], + "outputIds": [2], + "primaryOutputId": 2, + "executionPlatform": "//build/bazel/platforms:linux_x86_64" + }], + "pathFragments": [ + { "id": 1, "label": "foo.manifest" }, + { "id": 2, "label": "foo.runfiles/MANIFEST" }], + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1] }] +} +` + actual, _, err := AqueryBuildStatements([]byte(inputString)) + if err != nil { + t.Errorf("Unexpected error %q", err) + } + assertBuildStatements(t, []BuildStatement{ + { + Command: "", + OutputPaths: []string{"foo.runfiles/MANIFEST"}, + Mnemonic: "SymlinkTree", + InputPaths: []string{"foo.manifest"}, + }, + }, actual) +} + func TestBazelOutRemovalFromInputDepsets(t *testing.T) { const inputString = `{ "artifacts": [{ @@ -861,151 +898,67 @@ func TestTemplateExpandActionNoOutput(t *testing.T) { assertError(t, err, `Expect 1 output to template expand action, got: output []`) } -func TestPythonZipperActionSuccess(t *testing.T) { +func TestFileWrite(t *testing.T) { const inputString = ` { "artifacts": [ - { "id": 1, "pathFragmentId": 1 }, - { "id": 2, "pathFragmentId": 2 }, - { "id": 3, "pathFragmentId": 3 }, - { "id": 4, "pathFragmentId": 4 }, - { "id": 5, "pathFragmentId": 10 }, - { "id": 10, "pathFragmentId": 20 }], + { "id": 1, "pathFragmentId": 1 }], "actions": [{ "targetId": 1, "actionKey": "x", - "mnemonic": "TemplateExpand", + "mnemonic": "FileWrite", "configurationId": 1, "outputIds": [1], "primaryOutputId": 1, "executionPlatform": "//build/bazel/platforms:linux_x86_64", - "templateContent": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [{ - "key": "%token1%", - "value": "abcd" - },{ - "key": "%python_binary%", - "value": "python3" - }] - },{ - "targetId": 1, - "actionKey": "x", - "mnemonic": "PythonZipper", - "configurationId": 1, - "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], - "outputIds": [2], - "inputDepSetIds": [1], - "primaryOutputId": 2 + "fileContents": "file data\n" }], - "depSetOfFiles": [ - { "id": 1, "directArtifactIds": [4, 3, 5] }], "pathFragments": [ - { "id": 1, "label": "python_binary" }, - { "id": 2, "label": "python_binary.zip" }, - { "id": 3, "label": "python_binary.py" }, - { "id": 9, "label": ".." }, - { "id": 8, "label": "bazel_tools", "parentId": 9 }, - { "id": 7, "label": "tools", "parentId": 8 }, - { "id": 6, "label": "zip", "parentId": 7 }, - { "id": 5, "label": "zipper", "parentId": 6 }, - { "id": 4, "label": "zipper", "parentId": 5 }, - { "id": 16, "label": "bazel-out" }, - { "id": 15, "label": "bazel_tools", "parentId": 16 }, - { "id": 14, "label": "k8-fastbuild", "parentId": 15 }, - { "id": 13, "label": "bin", "parentId": 14 }, - { "id": 12, "label": "tools", "parentId": 13 }, - { "id": 11, "label": "python", "parentId": 12 }, - { "id": 10, "label": "py3wrapper.sh", "parentId": 11 }, - { "id": 20, "label": "python_binary" }] -}` + { "id": 1, "label": "foo.manifest" }] +} +` actual, _, err := AqueryBuildStatements([]byte(inputString)) - if err != nil { t.Errorf("Unexpected error %q", err) } - - expectedBuildStatements := []BuildStatement{ - { - Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > python_binary && " + - "chmod a+x python_binary'", - InputPaths: []string{"python_binary.zip"}, - OutputPaths: []string{"python_binary"}, - Mnemonic: "TemplateExpand", - }, + assertBuildStatements(t, []BuildStatement{ { - Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " + - "__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py && " + - "../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles", - InputPaths: []string{"python_binary.py"}, - OutputPaths: []string{"python_binary.zip"}, - Mnemonic: "PythonZipper", + OutputPaths: []string{"foo.manifest"}, + Mnemonic: "FileWrite", + FileContents: "file data\n", }, - } - assertBuildStatements(t, expectedBuildStatements, actual) + }, actual) } -func TestPythonZipperActionNoInput(t *testing.T) { +func TestSourceSymlinkManifest(t *testing.T) { const inputString = ` { "artifacts": [ - { "id": 1, "pathFragmentId": 1 }, - { "id": 2, "pathFragmentId": 2 }], + { "id": 1, "pathFragmentId": 1 }], "actions": [{ "targetId": 1, "actionKey": "x", - "mnemonic": "PythonZipper", + "mnemonic": "SourceSymlinkManifest", "configurationId": 1, - "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], - "outputIds": [2], - "primaryOutputId": 2 + "outputIds": [1], + "primaryOutputId": 1, + "executionPlatform": "//build/bazel/platforms:linux_x86_64", + "fileContents": "symlink target\n" }], "pathFragments": [ - { "id": 1, "label": "python_binary" }, - { "id": 2, "label": "python_binary.zip" }] -}` - _, _, err := AqueryBuildStatements([]byte(inputString)) - assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`) + { "id": 1, "label": "foo.manifest" }] } - -func TestPythonZipperActionNoOutput(t *testing.T) { - const inputString = ` -{ - "artifacts": [ - { "id": 1, "pathFragmentId": 1 }, - { "id": 2, "pathFragmentId": 2 }, - { "id": 3, "pathFragmentId": 3 }, - { "id": 4, "pathFragmentId": 4 }, - { "id": 5, "pathFragmentId": 10 }], - "actions": [{ - "targetId": 1, - "actionKey": "x", - "mnemonic": "PythonZipper", - "configurationId": 1, - "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], - "inputDepSetIds": [1] - }], - "depSetOfFiles": [ - { "id": 1, "directArtifactIds": [4, 3, 5]}], - "pathFragments": [ - { "id": 1, "label": "python_binary" }, - { "id": 2, "label": "python_binary.zip" }, - { "id": 3, "label": "python_binary.py" }, - { "id": 9, "label": ".." }, - { "id": 8, "label": "bazel_tools", "parentId": 9 }, - { "id": 7, "label": "tools", "parentId": 8 }, - { "id": 6, "label": "zip", "parentId": 7 }, - { "id": 5, "label": "zipper", "parentId": 6 }, - { "id": 4, "label": "zipper", "parentId": 5 }, - { "id": 16, "label": "bazel-out" }, - { "id": 15, "label": "bazel_tools", "parentId": 16 }, - { "id": 14, "label": "k8-fastbuild", "parentId": 15 }, - { "id": 13, "label": "bin", "parentId": 14 }, - { "id": 12, "label": "tools", "parentId": 13 }, - { "id": 11, "label": "python", "parentId": 12 }, - { "id": 10, "label": "py3wrapper.sh", "parentId": 11 }] -}` - _, _, err := AqueryBuildStatements([]byte(inputString)) - assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["python_binary.py"], output []`) +` + actual, _, err := AqueryBuildStatements([]byte(inputString)) + if err != nil { + t.Errorf("Unexpected error %q", err) + } + assertBuildStatements(t, []BuildStatement{ + { + OutputPaths: []string{"foo.manifest"}, + Mnemonic: "SourceSymlinkManifest", + }, + }, actual) } func assertError(t *testing.T, err error, expected string) { diff --git a/java/java.go b/java/java.go index cbdc2bd28..6e0515964 100644 --- a/java/java.go +++ b/java/java.go @@ -513,6 +513,20 @@ func (v javaVersion) String() string { } } +func (v javaVersion) StringForKotlinc() string { + // $ ./external/kotlinc/bin/kotlinc -jvm-target foo + // error: unknown JVM target version: foo + // Supported versions: 1.6, 1.8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + switch v { + case JAVA_VERSION_7: + return "1.6" + case JAVA_VERSION_9: + return "9" + default: + return v.String() + } +} + // Returns true if javac targeting this version uses system modules instead of a bootclasspath. func (v javaVersion) usesJavaModules() bool { return v >= 9 diff --git a/java/kotlin.go b/java/kotlin.go index 903c6249b..9bff5ea01 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -119,9 +119,8 @@ func kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile andro "srcJarDir": android.PathForModuleOut(ctx, "kotlinc", "srcJars").String(), "kotlinBuildFile": android.PathForModuleOut(ctx, "kotlinc-build.xml").String(), "emptyDir": android.PathForModuleOut(ctx, "kotlinc", "empty").String(), - // http://b/69160377 kotlinc only supports -jvm-target 1.6 and 1.8 - "kotlinJvmTarget": "1.8", - "name": kotlinName, + "kotlinJvmTarget": flags.javaVersion.StringForKotlinc(), + "name": kotlinName, }, }) } |