Run test build: Refactor path handling
Refactor the script to use python Path object rather than strings.
Keep all paths absolute to make the script independent of working dir.
This allows us to use threads rather than forks to compile the tests.
Test: All build artifacts are identical.
Change-Id: Id38881f6457c697d8b5f7ce509beab6e56bd3277
diff --git a/test/run_test_build.py b/test/run_test_build.py
index f2656f0..90dcd35 100755
--- a/test/run_test_build.py
+++ b/test/run_test_build.py
@@ -23,26 +23,26 @@
import functools
import glob
import os
+import pathlib
import shlex
import shutil
import subprocess
import sys
-import tempfile
import zipfile
from argparse import ArgumentParser
from fcntl import lockf, LOCK_EX, LOCK_NB
from importlib.machinery import SourceFileLoader
-from multiprocessing import Pool
-from multiprocessing.pool import ApplyResult
-from os import environ, getcwd, chdir, cpu_count, remove, path
-from os.path import join, basename
+from concurrent.futures import ThreadPoolExecutor
+from os import environ, getcwd, chdir, cpu_count, chmod
+from os.path import relpath
from pathlib import Path
from pprint import pprint
from re import match
from shutil import copytree, rmtree
from subprocess import run
-from typing import Dict, List, Union
+from tempfile import TemporaryDirectory, NamedTemporaryFile
+from typing import Dict, List, Union, Set
USE_RBE_FOR_JAVAC = 100 # Percentage of tests that can use RBE (between 0 and 100)
USE_RBE_FOR_D8 = 100 # Percentage of tests that can use RBE (between 0 and 100)
@@ -51,7 +51,7 @@
class BuildTestContext:
- def __init__(self, args, test_dir):
+ def __init__(self, args, android_build_top, test_dir):
self.test_name = test_dir.name
self.test_dir = test_dir.absolute()
self.mode = args.mode
@@ -60,10 +60,9 @@
self.target = (self.mode == "target")
assert self.jvm or self.host or self.target
- self.android_build_top = Path(getcwd())
- self.tmp_dir = args.out.parent.absolute()
+ self.android_build_top = android_build_top.absolute()
- self.java_home = Path(os.environ.get("JAVA_HOME"))
+ self.java_home = Path(os.environ.get("JAVA_HOME")).absolute()
self.java = self.java_home / "bin/java"
self.javac = self.java_home / "bin/javac"
self.javac_args = "-g -Xlint:-options -source 1.8 -target 1.8"
@@ -72,11 +71,12 @@
self.d8 = args.d8.absolute()
self.hiddenapi = args.hiddenapi.absolute()
self.jasmin = args.jasmin.absolute()
- self.need_dex = (self.host or self.target)
self.smali = args.smali.absolute()
self.soong_zip = args.soong_zip.absolute()
self.zipalign = args.zipalign.absolute()
+ self.need_dex = (self.host or self.target)
+
# Minimal environment needed for bash commands that we execute.
self.bash_env = {
"ANDROID_BUILD_TOP": self.android_build_top,
@@ -93,7 +93,11 @@
}
def bash(self, cmd):
- return subprocess.run(cmd, shell=True, env=self.bash_env, check=True)
+ return subprocess.run(cmd,
+ shell=True,
+ cwd=self.test_dir,
+ env=self.bash_env,
+ check=True)
def default_build(self, **kwargs):
globals()['default_build'](self, **kwargs)
@@ -122,30 +126,35 @@
has_jasmin=None,
):
+ # Wrap "pathlib.Path" with our own version that ensures all paths are absolute.
+ # Plain filenames are assumed to be relative to ctx.test_dir and made absolute.
+ class Path(pathlib.Path):
+ def __new__(cls, filename: str):
+ path = pathlib.Path(filename)
+ return path if path.is_absolute() else (ctx.test_dir / path)
+
ANDROID_BUILD_TOP = ctx.android_build_top
- CWD = os.getcwd()
TEST_NAME = ctx.test_name
- ART_TEST_RUN_TEST_BOOTCLASSPATH = path.relpath(ctx.bootclasspath, CWD)
NEED_DEX = ctx.need_dex if need_dex is None else need_dex
RBE_exec_root = os.environ.get("RBE_exec_root")
RBE_rewrapper = ctx.android_build_top / "prebuilts/remoteexecution-client/live/rewrapper"
# Set default values for directories.
- HAS_SMALI = path.exists("smali") if has_smali is None else has_smali
- HAS_JASMIN = path.exists("jasmin") if has_jasmin is None else has_jasmin
- HAS_SRC = path.exists("src")
- HAS_SRC_ART = path.exists("src-art")
- HAS_SRC2 = path.exists("src2")
- HAS_SRC_MULTIDEX = path.exists("src-multidex")
- HAS_SMALI_MULTIDEX = path.exists("smali-multidex")
- HAS_JASMIN_MULTIDEX = path.exists("jasmin-multidex")
- HAS_SMALI_EX = path.exists("smali-ex")
- HAS_SRC_EX = path.exists("src-ex")
- HAS_SRC_EX2 = path.exists("src-ex2")
- HAS_SRC_AOTEX = path.exists("src-aotex")
- HAS_SRC_BCPEX = path.exists("src-bcpex")
- HAS_HIDDENAPI_SPEC = path.exists("hiddenapi-flags.csv")
+ HAS_SMALI = Path("smali").exists() if has_smali is None else has_smali
+ HAS_JASMIN = Path("jasmin").exists() if has_jasmin is None else has_jasmin
+ HAS_SRC = Path("src").exists()
+ HAS_SRC_ART = Path("src-art").exists()
+ HAS_SRC2 = Path("src2").exists()
+ HAS_SRC_MULTIDEX = Path("src-multidex").exists()
+ HAS_SMALI_MULTIDEX = Path("smali-multidex").exists()
+ HAS_JASMIN_MULTIDEX = Path("jasmin-multidex").exists()
+ HAS_SMALI_EX = Path("smali-ex").exists()
+ HAS_SRC_EX = Path("src-ex").exists()
+ HAS_SRC_EX2 = Path("src-ex2").exists()
+ HAS_SRC_AOTEX = Path("src-aotex").exists()
+ HAS_SRC_BCPEX = Path("src-bcpex").exists()
+ HAS_HIDDENAPI_SPEC = Path("hiddenapi-flags.csv").exists()
JAVAC_ARGS = shlex.split(ctx.javac_args) + javac_args
SMALI_ARGS = smali_args.copy()
@@ -177,17 +186,22 @@
SMALI_ARGS.extend(["--api", str(api_level)])
D8_FLAGS.extend(["--min-api", str(api_level)])
- def run(executable: Path, args: List[str]):
- assert isinstance(executable, Path), executable
- cmd: List[Union[Path, str]] = []
+ def run(executable: pathlib.Path, args: List[str]):
+ assert isinstance(executable, pathlib.Path), executable
+ cmd: List[Union[pathlib.Path, str]] = []
if executable.suffix == ".sh":
cmd += ["/bin/bash"]
cmd += [executable]
cmd += args
env = ctx.bash_env
env.update({k: v for k, v in os.environ.items() if k.startswith("RBE_")})
+ for i, arg in enumerate(cmd):
+ if isinstance(arg, pathlib.Path):
+ assert arg.absolute(), arg
+ cmd[i] = relpath(arg, ctx.test_dir)
p = subprocess.run(cmd,
encoding=sys.stdout.encoding,
+ cwd=ctx.test_dir,
env=ctx.bash_env,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
@@ -211,12 +225,16 @@
assert version, "Could not parse RBE version"
assert tuple(map(int, version.groups())) >= (0, 76, 0), "Please update " + RBE_rewrapper
- def rbe_wrap(args, inputs=None):
- inputs = inputs or set()
- with tempfile.NamedTemporaryFile(mode="w+t", dir=ctx.tmp_dir) as input_list:
- for arg in args:
- inputs.update(filter(path.exists, arg.split(":")))
- input_list.writelines([path.relpath(i, RBE_exec_root)+"\n" for i in inputs])
+ def rbe_wrap(args, inputs: Set[pathlib.Path]=None):
+ with NamedTemporaryFile(mode="w+t") as input_list:
+ inputs = inputs or set()
+ inputs.update(arg for arg in args if isinstance(arg, pathlib.Path))
+ for i, arg in enumerate(args):
+ if isinstance(arg, str):
+ for part in arg.split(":"):
+ if (ctx.test_dir / part).exists():
+ inputs.add(Path(ctx.test_dir / part))
+ input_list.writelines([relpath(i, RBE_exec_root)+"\n" for i in inputs])
input_list.flush()
return run(RBE_rewrapper, [
"--platform=" + os.environ["RBE_platform"],
@@ -225,32 +243,25 @@
if USE_RBE_FOR_JAVAC > (hash(TEST_NAME) % 100): # Use for given percentage of tests.
def javac(args):
- output = path.relpath(path.join(CWD, args[args.index("-d") + 1]), RBE_exec_root)
- return rbe_wrap([
- "--output_directories", output,
- os.path.relpath(ctx.javac, CWD),
- ] + args)
+ output = relpath(Path(args[args.index("-d") + 1]), RBE_exec_root)
+ return rbe_wrap(["--output_directories", output, ctx.javac] + args)
if USE_RBE_FOR_D8 > (hash(TEST_NAME) % 100): # Use for given percentage of tests.
def d8(args):
inputs = set([ctx.d8.parent.parent / "framework/d8.jar"])
- output = path.relpath(path.join(CWD, args[args.index("--output") + 1]), RBE_exec_root)
+ output = relpath(Path(args[args.index("--output") + 1]), RBE_exec_root)
return rbe_wrap([
"--output_files" if output.endswith(".jar") else "--output_directories", output,
"--toolchain_inputs=prebuilts/jdk/jdk11/linux-x86/bin/java",
- os.path.relpath(ctx.d8, CWD)] + args, inputs)
+ ctx.d8] + args, inputs)
# If wrapper script exists, use it instead of the default javac.
- javac_wrapper = ctx.test_dir / "javac_wrapper.sh"
+ javac_wrapper = Path("javac_wrapper.sh")
if javac_wrapper.exists():
javac = functools.partial(run, javac_wrapper)
- def find(root, name):
- return sorted(glob.glob(path.join(root, "**", name), recursive=True))
-
-
- def zip(zip_target, *files):
- zip_args = ["-o", zip_target]
+ def zip(zip_target: Path, *files: Path):
+ zip_args = ["-o", zip_target, "-C", zip_target.parent]
if zip_compression_method == "store":
zip_args.extend(["-L", "0"])
for f in files:
@@ -259,15 +270,15 @@
if zip_align_bytes:
# zipalign does not operate in-place, so write results to a temp file.
- with tempfile.TemporaryDirectory(dir=ctx.tmp_dir) as tmp_dir:
- tmp_file = path.join(tmp_dir, "aligned.zip")
+ with TemporaryDirectory() as tmp_dir:
+ tmp_file = Path(tmp_dir) / "aligned.zip"
zipalign(["-f", str(zip_align_bytes), zip_target, tmp_file])
# replace original zip target with our temp file.
- os.rename(tmp_file, zip_target)
+ tmp_file.rename(zip_target)
- def make_jasmin(out_directory, jasmin_sources):
- os.makedirs(out_directory, exist_ok=True)
+ def make_jasmin(out_directory: Path, jasmin_sources: List[Path]):
+ out_directory.mkdir()
jasmin(["-d", out_directory] + sorted(jasmin_sources))
@@ -275,18 +286,18 @@
def javac_with_bootclasspath(args):
flags = JAVAC_ARGS + ["-encoding", "utf8"]
if BUILD_MODE != "jvm":
- flags.extend(["-bootclasspath", ART_TEST_RUN_TEST_BOOTCLASSPATH])
+ flags.extend(["-bootclasspath", ctx.bootclasspath])
javac(flags + args)
# Make a "dex" file given a directory of classes. This will be
# packaged in a jar file.
- def make_dex(name):
- d8_inputs = find(name, "*.class")
- d8_output = name + ".jar"
- dex_output = name + ".dex"
+ def make_dex(name: str):
+ d8_inputs = sorted(Path(name).glob("**/*.class"))
+ d8_output = Path(name + ".jar")
+ dex_output = Path(name + ".dex")
if use_desugar:
- flags = ["--lib", ART_TEST_RUN_TEST_BOOTCLASSPATH]
+ flags = ["--lib", ctx.bootclasspath]
else:
flags = ["--no-desugaring"]
assert d8_inputs
@@ -295,48 +306,47 @@
# D8 outputs to JAR files today rather than DEX files as DX used
# to. To compensate, we extract the DEX from d8's output to meet the
# expectations of make_dex callers.
- with tempfile.TemporaryDirectory(dir=ctx.tmp_dir) as tmp_dir:
+ with TemporaryDirectory() as tmp_dir:
zipfile.ZipFile(d8_output, "r").extractall(tmp_dir)
- os.rename(path.join(tmp_dir, "classes.dex"), dex_output)
-
+ (Path(tmp_dir) / "classes.dex").rename(dex_output)
# Merge all the dex files.
# Skip non-existing files, but at least 1 file must exist.
- def make_dexmerge(*dex_files_to_merge):
+ def make_dexmerge(*dex_files_to_merge: Path):
# Dex file that acts as the destination.
dst_file = dex_files_to_merge[0]
# Skip any non-existing files.
- dex_files_to_merge = list(filter(path.exists, dex_files_to_merge))
+ src_files = [f for f in dex_files_to_merge if f.exists()]
# NB: We merge even if there is just single input.
# It is useful to normalize non-deterministic smali output.
- tmp_dir = "dexmerge"
- os.makedirs(tmp_dir)
- d8(["--min-api", api_level, "--output", tmp_dir] + dex_files_to_merge)
- assert not path.exists(path.join(tmp_dir, "classes2.dex"))
- for input_dex in dex_files_to_merge:
- os.remove(input_dex)
- os.rename(path.join(tmp_dir, "classes.dex"), dst_file)
- os.rmdir(tmp_dir)
+ tmp_dir = ctx.test_dir / "dexmerge"
+ tmp_dir.mkdir()
+ d8(["--min-api", api_level, "--output", tmp_dir] + src_files)
+ assert not (tmp_dir / "classes2.dex").exists()
+ for src_file in src_files:
+ src_file.unlink()
+ (tmp_dir / "classes.dex").rename(dst_file)
+ tmp_dir.rmdir()
- def make_hiddenapi(*dex_files):
- args = ["encode"]
+ def make_hiddenapi(*dex_files: Path):
+ args: List[Union[str, Path]] = ["encode"]
for dex_file in dex_files:
- args.extend(["--input-dex=" + dex_file, "--output-dex=" + dex_file])
+ args.extend(["--input-dex=" + str(dex_file), "--output-dex=" + str(dex_file)])
args.append("--api-flags=hiddenapi-flags.csv")
args.append("--no-force-assign-all")
hiddenapi(args)
- if path.exists("classes.dex"):
- zip(TEST_NAME + ".jar", "classes.dex")
+ if Path("classes.dex").exists():
+ zip(Path(TEST_NAME + ".jar"), Path("classes.dex"))
return
- if path.exists("classes.dm"):
- zip(TEST_NAME + ".jar", "classes.dm")
+ if Path("classes.dm").exists():
+ zip(Path(TEST_NAME + ".jar"), Path("classes.dm"))
return
@@ -354,11 +364,13 @@
src_tmp_all = []
if HAS_JASMIN:
- make_jasmin("jasmin_classes", find("jasmin", "*.j"))
+ make_jasmin(Path("jasmin_classes"),
+ sorted(Path("jasmin").glob("**/*.j")))
src_tmp_all = add_to_cp_args(src_tmp_all, "jasmin_classes")
if HAS_JASMIN_MULTIDEX:
- make_jasmin("jasmin_classes2", find("jasmin-multidex", "*.j"))
+ make_jasmin(Path("jasmin_classes2"),
+ sorted(Path("jasmin-multidex").glob("**/*.j")))
src_tmp_all = add_to_cp_args(src_tmp_all, "jasmin_classes2")
if HAS_SRC and (HAS_SRC_MULTIDEX or HAS_SRC_AOTEX or HAS_SRC_BCPEX or
@@ -368,65 +380,67 @@
# Replacement sources in src-art/, src2/ and src-ex2/ can replace symbols
# used by the other src-* sources we compile here but everything needed to
# compile the other src-* sources should be present in src/ (and jasmin*/).
- os.makedirs("classes-tmp-all")
+ Path("classes-tmp-all").mkdir()
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes-tmp-all"] +
- find("src", "*.java") +
- find("src-multidex", "*.java") +
- find("src-aotex", "*.java") +
- find("src-bcpex", "*.java") +
- find("src-ex", "*.java"))
+ ["-d", Path("classes-tmp-all")] +
+ sorted(Path("src").glob("**/*.java")) +
+ sorted(Path("src-multidex").glob("**/*.java")) +
+ sorted(Path("src-aotex").glob("**/*.java")) +
+ sorted(Path("src-bcpex").glob("**/*.java")) +
+ sorted(Path("src-ex").glob("**/*.java")))
src_tmp_all = add_to_cp_args(src_tmp_all, "classes-tmp-all")
if HAS_SRC_AOTEX:
- os.makedirs("classes-aotex")
+ Path("classes-aotex").mkdir()
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes-aotex"] +
- find("src-aotex", "*.java"))
+ ["-d", Path("classes-aotex")] +
+ sorted(Path("src-aotex").glob("**/*.java")))
if NEED_DEX:
make_dex("classes-aotex")
# rename it so it shows up as "classes.dex" in the zip file.
- os.rename("classes-aotex.dex", "classes.dex")
- zip(TEST_NAME + "-aotex.jar", "classes.dex")
+ Path("classes-aotex.dex").rename(Path("classes.dex"))
+ zip(Path(TEST_NAME + "-aotex.jar"), Path("classes.dex"))
if HAS_SRC_BCPEX:
- os.makedirs("classes-bcpex")
+ Path("classes-bcpex").mkdir()
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes-bcpex"] +
- find("src-bcpex", "*.java"))
+ ["-d", Path("classes-bcpex")] +
+ sorted(Path("src-bcpex").glob("**/*.java")))
if NEED_DEX:
make_dex("classes-bcpex")
# rename it so it shows up as "classes.dex" in the zip file.
- os.rename("classes-bcpex.dex", "classes.dex")
- zip(TEST_NAME + "-bcpex.jar", "classes.dex")
+ Path("classes-bcpex.dex").rename(Path("classes.dex"))
+ zip(Path(TEST_NAME + "-bcpex.jar"), Path("classes.dex"))
if HAS_SRC:
- os.makedirs("classes", exist_ok=True)
+ Path("classes").mkdir(exist_ok=True)
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes"] + find("src", "*.java"))
+ ["-d", Path("classes")] +
+ sorted(Path("src").glob("**/*.java")))
if HAS_SRC_ART:
- os.makedirs("classes", exist_ok=True)
+ Path("classes").mkdir(exist_ok=True)
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes"] + find("src-art", "*.java"))
+ ["-d", Path("classes")] +
+ sorted(Path("src-art").glob("**/*.java")))
if HAS_SRC_MULTIDEX:
- os.makedirs("classes2")
+ Path("classes2").mkdir()
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes2"] +
- find("src-multidex", "*.java"))
+ ["-d", Path("classes2")] +
+ sorted(Path("src-multidex").glob("**/*.java")))
if NEED_DEX:
make_dex("classes2")
if HAS_SRC2:
- os.makedirs("classes", exist_ok=True)
+ Path("classes").mkdir(exist_ok=True)
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes"] +
- find("src2", "*.java"))
+ ["-d", Path("classes")] +
+ sorted(Path("src2").glob("**/*.java")))
# If the classes directory is not-empty, package classes in a DEX file.
# NB: some tests provide classes rather than java files.
- if find("classes", "*"):
+ if any(Path("classes").glob("*")):
if NEED_DEX:
make_dex("classes")
@@ -434,103 +448,97 @@
# Compile Jasmin classes as if they were part of the classes.dex file.
if NEED_DEX:
make_dex("jasmin_classes")
- make_dexmerge("classes.dex", "jasmin_classes.dex")
+ make_dexmerge(Path("classes.dex"), Path("jasmin_classes.dex"))
else:
# Move jasmin classes into classes directory so that they are picked up
# with -cp classes.
- os.makedirs("classes", exist_ok=True)
- shutil.copytree("jasmin_classes", "classes", dirs_exist_ok=True)
+ Path("classes").mkdir(exist_ok=True)
+ copytree(Path("jasmin_classes"), Path("classes"), dirs_exist_ok=True)
if HAS_SMALI and NEED_DEX:
# Compile Smali classes
+ smali_output = Path("smali_classes.dex")
smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
- ["--output", "smali_classes.dex"] + find("smali", "*.smali"))
- assert path.exists("smali_classes.dex")
+ ["--output", smali_output] + sorted(Path("smali").glob("**/*.smali")))
+ assert smali_output.exists()
# Merge smali files into classes.dex,
# this takes priority over any jasmin files.
- make_dexmerge("classes.dex", "smali_classes.dex")
+ make_dexmerge(Path("classes.dex"), Path("smali_classes.dex"))
# Compile Jasmin classes in jasmin-multidex as if they were part of
# the classes2.jar
if HAS_JASMIN_MULTIDEX:
if NEED_DEX:
make_dex("jasmin_classes2")
- make_dexmerge("classes2.dex", "jasmin_classes2.dex")
+ make_dexmerge(Path("classes2.dex"), Path("jasmin_classes2.dex"))
else:
# Move jasmin classes into classes2 directory so that
# they are picked up with -cp classes2.
- os.makedirs("classes2", exist_ok=True)
- shutil.copytree("jasmin_classes2", "classes2", dirs_exist_ok=True)
- shutil.rmtree("jasmin_classes2")
+ Path("classes2").mkdir()
+ copytree(Path("jasmin_classes2"), Path("classes2"), dirs_exist_ok=True)
+ rmtree(Path("jasmin_classes2"))
if HAS_SMALI_MULTIDEX and NEED_DEX:
# Compile Smali classes
smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
- ["--output", "smali_classes2.dex"] + find("smali-multidex", "*.smali"))
+ ["--output", "smali_classes2.dex"] + sorted(Path("smali-multidex").glob("**/*.smali")))
# Merge smali_classes2.dex into classes2.dex
- make_dexmerge("classes2.dex", "smali_classes2.dex")
+ make_dexmerge(Path("classes2.dex"), Path("smali_classes2.dex"))
if HAS_SRC_EX:
- os.makedirs("classes-ex", exist_ok=True)
+ Path("classes-ex").mkdir()
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes-ex"] + find("src-ex", "*.java"))
+ ["-d", Path("classes-ex")] +
+ sorted(Path("src-ex").glob("**/*.java")))
if HAS_SRC_EX2:
- os.makedirs("classes-ex", exist_ok=True)
+ Path("classes-ex").mkdir(exist_ok=True)
javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
- ["-d", "classes-ex"] + find("src-ex2", "*.java"))
+ ["-d", Path("classes-ex")] +
+ sorted(Path("src-ex2").glob("**/*.java")))
- if path.exists("classes-ex") and NEED_DEX:
+ if Path("classes-ex").exists() and NEED_DEX:
make_dex("classes-ex")
if HAS_SMALI_EX and NEED_DEX:
# Compile Smali classes
smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
- ["--output", "smali_classes-ex.dex"] + find("smali-ex", "*.smali"))
- assert path.exists("smali_classes-ex.dex")
+ ["--output", "smali_classes-ex.dex"] + sorted(Path("smali-ex").glob("**/*.smali")))
+ assert Path("smali_classes-ex.dex").exists()
# Merge smali files into classes-ex.dex.
- make_dexmerge("classes-ex.dex", "smali_classes-ex.dex")
+ make_dexmerge(Path("classes-ex.dex"), Path("smali_classes-ex.dex"))
- if path.exists("classes-ex.dex"):
+ if Path("classes-ex.dex").exists():
# Apply hiddenapi on the dex files if the test has API list file(s).
if use_hiddenapi and HAS_HIDDENAPI_SPEC:
- make_hiddenapi("classes-ex.dex")
+ make_hiddenapi(Path("classes-ex.dex"))
# quick shuffle so that the stored name is "classes.dex"
- os.rename("classes.dex", "classes-1.dex")
- os.rename("classes-ex.dex", "classes.dex")
- zip(TEST_NAME + "-ex.jar", "classes.dex")
- os.rename("classes.dex", "classes-ex.dex")
- os.rename("classes-1.dex", "classes.dex")
+ Path("classes.dex").rename(Path("classes-1.dex"))
+ Path("classes-ex.dex").rename(Path("classes.dex"))
+ zip(Path(TEST_NAME + "-ex.jar"), Path("classes.dex"))
+ Path("classes.dex").rename(Path("classes-ex.dex"))
+ Path("classes-1.dex").rename(Path("classes.dex"))
# Apply hiddenapi on the dex files if the test has API list file(s).
if NEED_DEX and use_hiddenapi and HAS_HIDDENAPI_SPEC:
if has_multidex():
- make_hiddenapi("classes.dex", "classes2.dex")
+ make_hiddenapi(Path("classes.dex"), Path("classes2.dex"))
else:
- make_hiddenapi("classes.dex")
+ make_hiddenapi(Path("classes.dex"))
# Create a single dex jar with two dex files for multidex.
if NEED_DEX:
- if path.exists("classes2.dex"):
- zip(TEST_NAME + ".jar", "classes.dex", "classes2.dex")
+ if Path("classes2.dex").exists():
+ zip(Path(TEST_NAME + ".jar"), Path("classes.dex"), Path("classes2.dex"))
else:
- zip(TEST_NAME + ".jar", "classes.dex")
-
-
-def copy_sources(args, ziproot: Path, mode: str, srcdir: Path) -> Path:
- """Copy test files from Android tree into the build sandbox and return its path."""
-
- dstdir = ziproot / mode / srcdir.name
- copytree(srcdir, dstdir)
- return dstdir
+ zip(Path(TEST_NAME + ".jar"), Path("classes.dex"))
def build_test(ctx: BuildTestContext) -> None:
"""Run the build script for single run-test"""
- chdir(ctx.test_dir)
script = ctx.test_dir / "build.py"
if script.exists():
module = SourceFileLoader("build_" + ctx.test_name,
@@ -546,7 +554,7 @@
# file to allow only one shard to use multiprocessing at the same time.
def use_multiprocessing(mode: str) -> bool:
global lock_file
- lock_path = join(environ["TMPDIR"], "art-test-run-test-build-py-" + mode)
+ lock_path = Path(environ["TMPDIR"]) / ("art-test-run-test-build-py-" + mode)
lock_file = open(lock_path, "w")
try:
lockf(lock_file, LOCK_EX | LOCK_NB)
@@ -569,24 +577,36 @@
parser.add_argument("srcs", nargs="+", type=Path)
args = parser.parse_args()
- ziproot = Path(args.out).parent / "zip"
- srcdirs = set(s.parents[-4] for s in args.srcs)
- dstdirs = [copy_sources(args, ziproot, args.mode, s) for s in srcdirs]
+ android_build_top = Path(getcwd()).absolute()
+ ziproot = args.out.absolute().parent / "zip"
+ srcdirs = set(s.parents[-4].absolute() for s in args.srcs)
- # Use multiprocessing (i.e. forking) since tests modify their current working directory.
- with Pool(cpu_count() if use_multiprocessing(args.mode) else 1) as pool:
- jobs: Dict[Path, ApplyResult] = {}
- for dstdir in dstdirs:
- ctx = BuildTestContext(args, dstdir)
- jobs[dstdir] = pool.apply_async(build_test, (ctx,))
- for dstdir, job in jobs.items():
- try:
- job.get()
- except Exception as e:
- raise Exception("Failed to build " + dstdir.name) from e.__cause__
+ # Initialize the test objects.
+ # We need to do this before we change the working directory below.
+ tests: List[BuildTestContext] = []
+ for srcdir in srcdirs:
+ dstdir = ziproot / args.mode / srcdir.name
+ copytree(srcdir, dstdir)
+ tests.append(BuildTestContext(args, android_build_top, dstdir))
+
+ # We can not change the working directory per each thread since they all run in parallel.
+ # Create invalid read-only directory to catch accidental use of current working directory.
+ with TemporaryDirectory("-do-not-use-cwd") as invalid_tmpdir:
+ os.chdir(invalid_tmpdir)
+ os.chmod(invalid_tmpdir, 0)
+ with ThreadPoolExecutor(cpu_count() if use_multiprocessing(args.mode) else 1) as pool:
+ jobs = {}
+ for ctx in tests:
+ jobs[ctx.test_name] = pool.submit(build_test, ctx)
+ for test_name, job in jobs.items():
+ try:
+ job.result()
+ except Exception as e:
+ raise Exception("Failed to build " + test_name) from e
# Create the final zip file which contains the content of the temporary directory.
- proc = run([args.soong_zip, "-o", args.out, "-C", ziproot, "-D", ziproot], check=True)
+ proc = run([android_build_top / args.soong_zip, "-o", android_build_top / args.out,
+ "-C", ziproot, "-D", ziproot], check=True)
if __name__ == "__main__":