Create .kzip files for kotlin translation units

This CL creates the build rules to invoke the standalone kotlin
extractor and create .kzip files for each kotlin translation unit. These
will be indexed by the internal g3 kotlin indexers

Implementation details
- Create a `kotlinKytheExtract` static rule. Its inputs will be source
  .kt/.srcjar files and other verbatim args passed to kotlinc. The
extrator will serialize this information into a .kzip file. The .kzip
file should contain the necessary information to "replay" the
compilation
- Create a xref_kotlin phony rule to build all .kzip files corresponding
  to kotlin translation units.

This implementation has one limitation. Since the kotlin indexers
"replay" the compilation using its own version of kotlinc-jvm, there
might be indexing issues caused by kotlinc version skew between android
builds and indexing builds. `-kotlin-home` can likely solve that, but
this CL defers that for now.

Bug: 265428637
Test: Built the phony `xref_kotlin`, and ran the indexer locally using go/kythe-git-local-run#indexing-to-local-xrefs-server

Change-Id: Ifb370f2df8aa46f71df3fe1ae7d7aa5da773cc8b

4 files changed, 63 insertions, 3 deletions

diff --git a/java/base.go b/java/base.go
index a63533127..30485d0ca 100644
--- a/java/base.go
+++ b/java/base.go
@@ -538,7 +538,8 @@ type Module struct {
 	linter
 
 	// list of the xref extraction files
-	kytheFiles android.Paths
+	kytheFiles       android.Paths
+	kytheKotlinFiles android.Paths
 
 	hideApexVariantFromMake bool
 
@@ -1372,7 +1373,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath
 
 		kotlinJar := android.PathForModuleOut(ctx, "kotlin", jarName)
 		kotlinHeaderJar := android.PathForModuleOut(ctx, "kotlin_headers", jarName)
-		kotlinCompile(ctx, kotlinJar, kotlinHeaderJar, uniqueSrcFiles, kotlinCommonSrcFiles, srcJars, flags)
+		j.kotlinCompile(ctx, kotlinJar, kotlinHeaderJar, uniqueSrcFiles, kotlinCommonSrcFiles, srcJars, flags)
 		if ctx.Failed() {
 			return
 		}
diff --git a/java/config/config.go b/java/config/config.go
index a50c1b42d..ca712827f 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -144,6 +144,7 @@ func init() {
 	pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod")
 	pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar")
 	pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar")
+	pctx.SourcePathVariable("KotlinKytheExtractor", "prebuilts/build-tools/${hostPrebuiltTag}/bin/kotlinc_extractor")
 	pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime")
 
 	pctx.SourcePathVariable("ResourceProcessorBusyBox", "prebuilts/bazel/common/android_tools/android_tools/all_android_tools_deploy.jar")
diff --git a/java/java.go b/java/java.go
index 797a90dfc..99f44374f 100644
--- a/java/java.go
+++ b/java/java.go
@@ -356,12 +356,17 @@ type UsesLibraryDependency interface {
 // TODO(jungjw): Move this to kythe.go once it's created.
 type xref interface {
 	XrefJavaFiles() android.Paths
+	XrefKotlinFiles() android.Paths
 }
 
 func (j *Module) XrefJavaFiles() android.Paths {
 	return j.kytheFiles
 }
 
+func (j *Module) XrefKotlinFiles() android.Paths {
+	return j.kytheKotlinFiles
+}
+
 func (d dependencyTag) PropagateAconfigValidation() bool {
 	return d.static
 }
@@ -3304,15 +3309,20 @@ type kytheExtractJavaSingleton struct {
 
 func (ks *kytheExtractJavaSingleton) GenerateBuildActions(ctx android.SingletonContext) {
 	var xrefTargets android.Paths
+	var xrefKotlinTargets android.Paths
 	ctx.VisitAllModules(func(module android.Module) {
 		if javaModule, ok := module.(xref); ok {
 			xrefTargets = append(xrefTargets, javaModule.XrefJavaFiles()...)
+			xrefKotlinTargets = append(xrefKotlinTargets, javaModule.XrefKotlinFiles()...)
 		}
 	})
 	// TODO(asmundak): perhaps emit a rule to output a warning if there were no xrefTargets
 	if len(xrefTargets) > 0 {
 		ctx.Phony("xref_java", xrefTargets...)
 	}
+	if len(xrefKotlinTargets) > 0 {
+		ctx.Phony("xref_kotlin", xrefKotlinTargets...)
+	}
 }
 
 var Bool = proptools.Bool
diff --git a/java/kotlin.go b/java/kotlin.go
index c28bc3f54..79781a06d 100644
--- a/java/kotlin.go
+++ b/java/kotlin.go
@@ -64,6 +64,29 @@ var kotlinc = pctx.AndroidRemoteStaticRule("kotlinc", android.RemoteRuleSupports
 	"kotlincFlags", "classpath", "srcJars", "commonSrcFilesArg", "srcJarDir", "classesDir",
 	"headerClassesDir", "headerJar", "kotlinJvmTarget", "kotlinBuildFile", "emptyDir", "name")
 
+var kotlinKytheExtract = pctx.AndroidStaticRule("kotlinKythe",
+	// TODO (b/265428637): To prevent kotlinc version skew between android builds and internal kotlin indexers (g3), consider embedding the kotlinc used by android into the kzip file.
+	// This has an impact on .kzip sizes, so defer that for now.
+	blueprint.RuleParams{
+		Command: `rm -rf "$srcJarDir" && mkdir -p "$srcJarDir" && ` +
+			`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" -f "*.kt" $srcJars && ` +
+			`${config.KotlinKytheExtractor} -corpus ${kytheCorpus} --srcs @$out.rsp --srcs @"$srcJarDir/list" $commonSrcFilesList --cp @$classpath -o $out --kotlin_out $outJar ` +
+			// wrap the additional kotlin args.
+			// Skip Xbuild file, pass the cp explicitly.
+			// Skip header jars, those should not have an effect on kythe results.
+			` --args '${config.KotlincGlobalFlags} ` +
+			` ${config.KotlincSuppressJDK9Warnings} ${config.JavacHeapFlags} ` +
+			` $kotlincFlags -jvm-target $kotlinJvmTarget'`,
+		CommandDeps: []string{
+			"${config.KotlinKytheExtractor}",
+			"${config.ZipSyncCmd}",
+		},
+		Rspfile:        "$out.rsp",
+		RspfileContent: "$in",
+	},
+	"classpath", "kotlincFlags", "commonSrcFilesList", "kotlinJvmTarget", "outJar", "srcJars", "srcJarDir",
+)
+
 func kotlinCommonSrcsList(ctx android.ModuleContext, commonSrcFiles android.Paths) android.OptionalPath {
 	if len(commonSrcFiles) > 0 {
 		// The list of common_srcs may be too long to put on the command line, but
@@ -81,7 +104,7 @@ func kotlinCommonSrcsList(ctx android.ModuleContext, commonSrcFiles android.Path
 }
 
 // kotlinCompile takes .java and .kt sources and srcJars, and compiles the .kt sources into a classes jar in outputFile.
-func kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile android.WritablePath,
+func (j *Module) kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile android.WritablePath,
 	srcFiles, commonSrcFiles, srcJars android.Paths,
 	flags javaBuilderFlags) {
 
@@ -127,6 +150,31 @@ func kotlinCompile(ctx android.ModuleContext, outputFile, headerOutputFile andro
 			"name":              kotlinName,
 		},
 	})
+
+	// Emit kythe xref rule
+	if (ctx.Config().EmitXrefRules()) && ctx.Module() == ctx.PrimaryModule() {
+		extractionFile := outputFile.ReplaceExtension(ctx, "kzip")
+		args := map[string]string{
+			"classpath":       classpathRspFile.String(),
+			"kotlincFlags":    flags.kotlincFlags,
+			"kotlinJvmTarget": flags.javaVersion.StringForKotlinc(),
+			"outJar":          outputFile.String(),
+			"srcJars":         strings.Join(srcJars.Strings(), " "),
+			"srcJarDir":       android.PathForModuleOut(ctx, "kotlinc", "srcJars.xref").String(),
+		}
+		if commonSrcsList.Valid() {
+			args["commonSrcFilesList"] = "--srcs @" + commonSrcsList.String()
+		}
+		ctx.Build(pctx, android.BuildParams{
+			Rule:        kotlinKytheExtract,
+			Description: "kotlinKythe",
+			Output:      extractionFile,
+			Inputs:      srcFiles,
+			Implicits:   deps,
+			Args:        args,
+		})
+		j.kytheKotlinFiles = append(j.kytheKotlinFiles, extractionFile)
+	}
 }
 
 var kaptStubs = pctx.AndroidRemoteStaticRule("kaptStubs", android.RemoteRuleSupports{Goma: true},