Support Javac sharding in Soong.
Test: m clean && m -j java and java_test.go
Change-Id: I110a0ff029448d3319aed2788d25d90a6c1a7fa0
diff --git a/java/builder.go b/java/builder.go
index 8b6eb9f..45e59a4 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -19,6 +19,8 @@
// functions.
import (
+ "path/filepath"
+ "strconv"
"strings"
"github.com/google/blueprint"
@@ -211,12 +213,16 @@
})
}
-func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
- srcFiles, srcJars android.Paths,
- flags javaBuilderFlags, deps android.Paths) {
+func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, shardIdx int,
+ srcFiles, srcJars android.Paths, flags javaBuilderFlags, deps android.Paths) {
- transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, deps,
- "javac", "javac", javac)
+ // Compile java sources into .class files
+ desc := "javac"
+ if shardIdx >= 0 {
+ desc += strconv.Itoa(shardIdx)
+ }
+
+ transformJavaToClasses(ctx, outputFile, shardIdx, srcFiles, srcJars, flags, deps, "javac", desc, javac)
}
func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath,
@@ -226,7 +232,7 @@
ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?")
}
- transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, nil,
+ transformJavaToClasses(ctx, outputFile, -1, srcFiles, srcJars, flags, nil,
"errorprone", "errorprone", errorprone)
}
@@ -275,7 +281,7 @@
// suffix will be appended to various intermediate files and directories to avoid collisions when
// this function is called twice in the same module directory.
func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath,
- srcFiles, srcJars android.Paths,
+ shardIdx int, srcFiles, srcJars android.Paths,
flags javaBuilderFlags, deps android.Paths,
intermediatesDir, desc string, rule blueprint.Rule) {
@@ -298,6 +304,15 @@
deps = append(deps, flags.classpath...)
+ srcJarDir := "srcjars"
+ outDir := "classes"
+ annoDir := "anno"
+ if shardIdx >= 0 {
+ shardDir := "shard" + strconv.Itoa(shardIdx)
+ srcJarDir = filepath.Join(shardDir, srcJarDir)
+ outDir = filepath.Join(shardDir, outDir)
+ annoDir = filepath.Join(shardDir, annoDir)
+ }
ctx.Build(pctx, android.BuildParams{
Rule: rule,
Description: desc,
@@ -309,9 +324,9 @@
"bootClasspath": bootClasspath,
"classpath": flags.classpath.FormJavaClassPath("-classpath"),
"srcJars": strings.Join(srcJars.Strings(), " "),
- "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, "srcjars").String(),
- "outDir": android.PathForModuleOut(ctx, intermediatesDir, "classes").String(),
- "annoDir": android.PathForModuleOut(ctx, intermediatesDir, "anno").String(),
+ "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, srcJarDir).String(),
+ "outDir": android.PathForModuleOut(ctx, intermediatesDir, outDir).String(),
+ "annoDir": android.PathForModuleOut(ctx, intermediatesDir, annoDir).String(),
"javaVersion": flags.javaVersion,
},
})
diff --git a/java/java.go b/java/java.go
index f9a4c04..bb6e556 100644
--- a/java/java.go
+++ b/java/java.go
@@ -117,6 +117,9 @@
// List of classes to pass to javac to use as annotation processors
Annotation_processor_classes []string
+ // The number of Java source entries each Javac instance can process
+ Javac_shard_size *int64
+
Openjdk9 struct {
// List of source files that should only be used when passing -source 1.9
Srcs []string
@@ -355,6 +358,18 @@
return false
}
+func shardPaths(paths android.Paths, shardSize int) []android.Paths {
+ ret := make([]android.Paths, 0, (len(paths)+shardSize-1)/shardSize)
+ for len(paths) > shardSize {
+ ret = append(ret, paths[0:shardSize])
+ paths = paths[shardSize:]
+ }
+ if len(paths) > 0 {
+ ret = append(ret, paths)
+ }
+ return ret
+}
+
func (j *Module) hasSrcExt(ext string) bool {
return hasSrcExt(j.properties.Srcs, ext)
}
@@ -571,7 +586,17 @@
}
}
+ enable_sharding := false
if ctx.Device() && !ctx.AConfig().IsEnvFalse("TURBINE_ENABLED") {
+ if j.properties.Javac_shard_size != nil && *(j.properties.Javac_shard_size) > 0 {
+ enable_sharding = true
+ if len(j.properties.Annotation_processors) != 0 ||
+ len(j.properties.Annotation_processor_classes) != 0 {
+ ctx.PropertyErrorf("javac_shard_size",
+ "%q cannot be set when annotation processors are enabled.",
+ j.properties.Javac_shard_size)
+ }
+ }
// If sdk jar is java module, then directly return classesJar as header.jar
if j.Name() != "android_stubs_current" && j.Name() != "android_system_stubs_current" &&
j.Name() != "android_test_stubs_current" {
@@ -590,18 +615,35 @@
// TODO(ccross): Once we always compile with javac9 we may be able to conditionally
// enable error-prone without affecting the output class files.
errorprone := android.PathForModuleOut(ctx, "errorprone", jarName)
- RunErrorProne(ctx, errorprone, javaSrcFiles, srcJars, flags)
+ RunErrorProne(ctx, errorprone, uniqueSrcFiles, srcJars, flags)
extraJarDeps = append(extraJarDeps, errorprone)
}
- // Compile java sources into .class files
- classes := android.PathForModuleOut(ctx, "javac", jarName)
- TransformJavaToClasses(ctx, classes, javaSrcFiles, srcJars, flags, extraJarDeps)
+ if enable_sharding {
+ flags.classpath.AddPaths([]android.Path{j.headerJarFile})
+ shardSize := int(*(j.properties.Javac_shard_size))
+ var shardSrcs []android.Paths
+ if len(uniqueSrcFiles) > 0 {
+ shardSrcs = shardPaths(uniqueSrcFiles, shardSize)
+ for idx, shardSrc := range shardSrcs {
+ classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx))
+ TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps)
+ jars = append(jars, classes)
+ }
+ }
+ if len(srcJars) > 0 {
+ classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(len(shardSrcs)))
+ TransformJavaToClasses(ctx, classes, len(shardSrcs), nil, srcJars, flags, extraJarDeps)
+ jars = append(jars, classes)
+ }
+ } else {
+ classes := android.PathForModuleOut(ctx, "javac", jarName)
+ TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps)
+ jars = append(jars, classes)
+ }
if ctx.Failed() {
return
}
-
- jars = append(jars, classes)
}
dirArgs, dirDeps := ResourceDirsToJarArgs(ctx, j.properties.Java_resource_dirs, j.properties.Exclude_java_resource_dirs)
diff --git a/java/java_test.go b/java/java_test.go
index b819447..82eff1e 100644
--- a/java/java_test.go
+++ b/java/java_test.go
@@ -22,6 +22,7 @@
"os"
"path/filepath"
"reflect"
+ "strconv"
"strings"
"testing"
)
@@ -669,6 +670,71 @@
}
}
+func TestTurbine(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "foo",
+ srcs: ["a.java"],
+ }
+
+ java_library {
+ name: "bar",
+ srcs: ["b.java"],
+ static_libs: ["foo"],
+ }
+
+ java_library {
+ name: "baz",
+ srcs: ["c.java"],
+ libs: ["bar"],
+ sdk_version: "14",
+ }
+ `)
+
+ fooTurbine := ctx.ModuleForTests("foo", "android_common").Rule("turbine")
+ barTurbine := ctx.ModuleForTests("bar", "android_common").Rule("turbine")
+ barJavac := ctx.ModuleForTests("bar", "android_common").Rule("javac")
+ barTurbineCombined := ctx.ModuleForTests("bar", "android_common").Description("for turbine")
+ bazJavac := ctx.ModuleForTests("baz", "android_common").Rule("javac")
+
+ if len(fooTurbine.Inputs) != 1 || fooTurbine.Inputs[0].String() != "a.java" {
+ t.Errorf(`foo inputs %v != ["a.java"]`, fooTurbine.Inputs)
+ }
+
+ fooHeaderJar := filepath.Join(buildDir, ".intermediates", "foo", "android_common", "turbine-combined", "foo.jar")
+ if !strings.Contains(barTurbine.Args["classpath"], fooHeaderJar) {
+ t.Errorf("bar turbine classpath %v does not contain %q", barTurbine.Args["classpath"], fooHeaderJar)
+ }
+ if !strings.Contains(barJavac.Args["classpath"], fooHeaderJar) {
+ t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], fooHeaderJar)
+ }
+ if len(barTurbineCombined.Inputs) != 2 || barTurbineCombined.Inputs[1].String() != fooHeaderJar {
+ t.Errorf("bar turbine combineJar inputs %v does not contain %q", barTurbineCombined.Inputs, fooHeaderJar)
+ }
+ if !strings.Contains(bazJavac.Args["classpath"], "prebuilts/sdk/14/android.jar") {
+ t.Errorf("baz javac classpath %v does not contain %q", bazJavac.Args["classpath"],
+ "prebuilts/sdk/14/android.jar")
+ }
+}
+
+func TestSharding(t *testing.T) {
+ ctx := testJava(t, `
+ java_library {
+ name: "bar",
+ srcs: ["a.java","b.java","c.java"],
+ javac_shard_size: 1
+ }
+ `)
+
+ barHeaderJar := filepath.Join(buildDir, ".intermediates", "bar", "android_common", "turbine-combined", "bar.jar")
+ for i := 0; i < 3; i++ {
+ barJavac := ctx.ModuleForTests("bar", "android_common").Description("javac" + strconv.Itoa(i))
+ if !strings.Contains(barJavac.Args["classpath"], barHeaderJar) {
+ t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], barHeaderJar)
+ }
+ }
+}
+
func fail(t *testing.T, errs []error) {
if len(errs) > 0 {
for _, err := range errs {