blob: 75cd64f0614b09c43291087bc96e4053379a8caa [file] [log] [blame]
David Srbecky37274d52022-11-03 11:04:16 +00001#!/usr/bin/env python3
David Srbecky8106b382022-04-20 13:37:15 +01002#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
David Srbecky37274d52022-11-03 11:04:16 +000017"""
18This scripts compiles Java files which are needed to execute run-tests.
19It is intended to be used only from soong genrule.
David Srbecky8106b382022-04-20 13:37:15 +010020"""
21
22import argparse
23import functools
24import glob
25import os
David Srbecky36b520b2022-10-27 14:43:12 +010026import pathlib
David Srbecky8106b382022-04-20 13:37:15 +010027import shlex
28import shutil
29import subprocess
David Srbecky6bd1cd12022-11-05 18:54:19 +000030import sys
David Srbecky8106b382022-04-20 13:37:15 +010031import zipfile
David Srbecky6bd1cd12022-11-05 18:54:19 +000032
David Srbecky37274d52022-11-03 11:04:16 +000033from argparse import ArgumentParser
34from fcntl import lockf, LOCK_EX, LOCK_NB
35from importlib.machinery import SourceFileLoader
David Srbecky36b520b2022-10-27 14:43:12 +010036from concurrent.futures import ThreadPoolExecutor
37from os import environ, getcwd, chdir, cpu_count, chmod
38from os.path import relpath
David Srbecky37274d52022-11-03 11:04:16 +000039from pathlib import Path
David Srbecky6bd1cd12022-11-05 18:54:19 +000040from pprint import pprint
David Srbeckye0c3cd82022-08-24 14:56:24 +010041from re import match
David Srbecky37274d52022-11-03 11:04:16 +000042from shutil import copytree, rmtree
43from subprocess import run
David Srbecky36b520b2022-10-27 14:43:12 +010044from tempfile import TemporaryDirectory, NamedTemporaryFile
David Srbecky100fe672022-11-14 11:32:27 +000045from typing import Dict, List, Union, Set, Optional
David Srbecky8106b382022-04-20 13:37:15 +010046
David Srbeckya42a2e32022-11-17 13:43:22 +000047USE_RBE = 100 # Percentage of tests that can use RBE (between 0 and 100)
David Srbecky37274d52022-11-03 11:04:16 +000048
49lock_file = None # Keep alive as long as this process is alive.
50
David Srbeckyf32b2582022-11-17 15:28:35 +000051RBE_D8_DISABLED_FOR = {
David Srbeckyf32b2582022-11-17 15:28:35 +000052 "952-invoke-custom", # b/228312861: RBE uses wrong inputs.
53 "979-const-method-handle", # b/228312861: RBE uses wrong inputs.
54}
David Srbecky89010a32022-07-14 16:47:30 +000055
David Srbecky51dec052022-10-27 14:43:12 +010056class BuildTestContext:
David Srbecky36b520b2022-10-27 14:43:12 +010057 def __init__(self, args, android_build_top, test_dir):
David Srbeckyfeed4af2022-11-17 10:44:39 +000058 self.android_build_top = android_build_top.absolute()
59 self.bootclasspath = args.bootclasspath.absolute()
David Srbecky6bd1cd12022-11-05 18:54:19 +000060 self.test_name = test_dir.name
61 self.test_dir = test_dir.absolute()
David Srbecky51dec052022-10-27 14:43:12 +010062 self.mode = args.mode
63 self.jvm = (self.mode == "jvm")
64 self.host = (self.mode == "host")
65 self.target = (self.mode == "target")
66 assert self.jvm or self.host or self.target
67
David Srbecky36b520b2022-10-27 14:43:12 +010068 self.java_home = Path(os.environ.get("JAVA_HOME")).absolute()
David Srbeckyfeed4af2022-11-17 10:44:39 +000069 self.java_path = self.java_home / "bin/java"
70 self.javac_path = self.java_home / "bin/javac"
David Srbecky51dec052022-10-27 14:43:12 +010071 self.javac_args = "-g -Xlint:-options -source 1.8 -target 1.8"
David Srbecky6bd1cd12022-11-05 18:54:19 +000072
David Srbeckyfeed4af2022-11-17 10:44:39 +000073 # Helper functions to execute tools.
74 self.d8 = functools.partial(self.run, args.d8.absolute())
75 self.jasmin = functools.partial(self.run, args.jasmin.absolute())
76 self.javac = functools.partial(self.run, self.javac_path)
77 self.smali = functools.partial(self.run, args.smali.absolute())
78 self.soong_zip = functools.partial(self.run, args.soong_zip.absolute())
79 self.zipalign = functools.partial(self.run, args.zipalign.absolute())
80 if args.hiddenapi:
81 self.hiddenapi = functools.partial(self.run, args.hiddenapi.absolute())
82
83 # RBE wrapper for some of the tools.
Hans Boehm6183e752023-01-11 00:45:46 +000084 if "RBE_server_address" in os.environ and USE_RBE > (hash(self.test_name) % 100):
David Srbeckyfeed4af2022-11-17 10:44:39 +000085 self.rbe_exec_root = os.environ.get("RBE_exec_root")
86 self.rbe_rewrapper = self.android_build_top / "prebuilts/remoteexecution-client/live/rewrapper"
David Srbeckyf32b2582022-11-17 15:28:35 +000087 if self.test_name not in RBE_D8_DISABLED_FOR:
88 self.d8 = functools.partial(self.rbe_d8, args.d8.absolute())
David Srbeckya42a2e32022-11-17 13:43:22 +000089 self.javac = functools.partial(self.rbe_javac, self.javac_path)
90 self.smali = functools.partial(self.rbe_smali, args.smali.absolute())
David Srbecky51dec052022-10-27 14:43:12 +010091
92 # Minimal environment needed for bash commands that we execute.
93 self.bash_env = {
94 "ANDROID_BUILD_TOP": self.android_build_top,
David Srbeckyfeed4af2022-11-17 10:44:39 +000095 "D8": args.d8.absolute(),
96 "JAVA": self.java_path,
97 "JAVAC": self.javac_path,
David Srbecky51dec052022-10-27 14:43:12 +010098 "JAVAC_ARGS": self.javac_args,
David Srbecky6bd1cd12022-11-05 18:54:19 +000099 "JAVA_HOME": self.java_home,
David Srbecky51dec052022-10-27 14:43:12 +0100100 "PATH": os.environ["PATH"],
101 "PYTHONDONTWRITEBYTECODE": "1",
David Srbeckyfeed4af2022-11-17 10:44:39 +0000102 "SMALI": args.smali.absolute(),
103 "SOONG_ZIP": args.soong_zip.absolute(),
David Srbecky51dec052022-10-27 14:43:12 +0100104 "TEST_NAME": self.test_name,
105 }
106
107 def bash(self, cmd):
David Srbecky36b520b2022-10-27 14:43:12 +0100108 return subprocess.run(cmd,
109 shell=True,
110 cwd=self.test_dir,
111 env=self.bash_env,
112 check=True)
David Srbecky51dec052022-10-27 14:43:12 +0100113
David Srbeckyf32b2582022-11-17 15:28:35 +0000114 def run(self, executable: pathlib.Path, args: List[Union[pathlib.Path, str]]):
David Srbeckyfeed4af2022-11-17 10:44:39 +0000115 assert isinstance(executable, pathlib.Path), executable
116 cmd: List[Union[pathlib.Path, str]] = []
117 if executable.suffix == ".sh":
118 cmd += ["/bin/bash"]
119 cmd += [executable]
120 cmd += args
121 env = self.bash_env
122 env.update({k: v for k, v in os.environ.items() if k.startswith("RBE_")})
123 # Make paths relative as otherwise we could create too long command line.
124 for i, arg in enumerate(cmd):
125 if isinstance(arg, pathlib.Path):
126 assert arg.absolute(), arg
127 cmd[i] = relpath(arg, self.test_dir)
128 elif isinstance(arg, list):
129 assert all(p.absolute() for p in arg), arg
130 cmd[i] = ":".join(relpath(p, self.test_dir) for p in arg)
131 else:
132 assert isinstance(arg, str), arg
133 p = subprocess.run(cmd,
134 encoding=sys.stdout.encoding,
135 cwd=self.test_dir,
136 env=self.bash_env,
137 stderr=subprocess.STDOUT,
138 stdout=subprocess.PIPE)
139 if p.returncode != 0:
140 raise Exception("Command failed with exit code {}\n$ {}\n{}".format(
141 p.returncode, " ".join(map(str, cmd)), p.stdout))
142 return p
143
144 def rbe_wrap(self, args, inputs: Set[pathlib.Path]=None):
145 with NamedTemporaryFile(mode="w+t") as input_list:
146 inputs = inputs or set()
147 for i, arg in enumerate(args):
148 if isinstance(arg, pathlib.Path):
149 assert arg.absolute(), arg
150 inputs.add(arg)
151 elif isinstance(arg, list):
152 assert all(p.absolute() for p in arg), arg
153 inputs.update(arg)
154 input_list.writelines([relpath(i, self.rbe_exec_root)+"\n" for i in inputs])
155 input_list.flush()
156 return self.run(self.rbe_rewrapper, [
157 "--platform=" + os.environ["RBE_platform"],
158 "--input_list_paths=" + input_list.name,
159 ] + args)
160
David Srbeckya42a2e32022-11-17 13:43:22 +0000161 def rbe_javac(self, javac_path:Path, args):
David Srbeckyfeed4af2022-11-17 10:44:39 +0000162 output = relpath(Path(args[args.index("-d") + 1]), self.rbe_exec_root)
David Srbeckya42a2e32022-11-17 13:43:22 +0000163 return self.rbe_wrap(["--output_directories", output, javac_path] + args)
David Srbeckyfeed4af2022-11-17 10:44:39 +0000164
David Srbeckya42a2e32022-11-17 13:43:22 +0000165 def rbe_d8(self, d8_path:Path, args):
166 inputs = set([d8_path.parent.parent / "framework/d8.jar"])
David Srbeckyfeed4af2022-11-17 10:44:39 +0000167 output = relpath(Path(args[args.index("--output") + 1]), self.rbe_exec_root)
168 return self.rbe_wrap([
169 "--output_files" if output.endswith(".jar") else "--output_directories", output,
Sorin Bascae1f8e3c2022-12-21 15:58:58 +0000170 "--toolchain_inputs=prebuilts/jdk/jdk17/linux-x86/bin/java",
David Srbeckya42a2e32022-11-17 13:43:22 +0000171 d8_path] + args, inputs)
172
173 def rbe_smali(self, smali_path:Path, args):
174 inputs = set([smali_path.parent.parent / "framework/smali.jar"])
175 output = relpath(Path(args[args.index("--output") + 1]), self.rbe_exec_root)
176 return self.rbe_wrap([
177 "--output_files", output,
Sorin Bascae1f8e3c2022-12-21 15:58:58 +0000178 "--toolchain_inputs=prebuilts/jdk/jdk17/linux-x86/bin/java",
David Srbeckya42a2e32022-11-17 13:43:22 +0000179 smali_path] + args, inputs)
David Srbeckyfeed4af2022-11-17 10:44:39 +0000180
David Srbeckyd272f2d2022-11-15 19:13:34 +0000181 def build(self) -> None:
182 script = self.test_dir / "build.py"
183 if script.exists():
184 module = SourceFileLoader("build_" + self.test_name,
185 str(script)).load_module()
186 module.build(self)
187 else:
188 self.default_build()
189
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000190 def default_build(
David Srbeckyd272f2d2022-11-15 19:13:34 +0000191 self,
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000192 use_desugar=True,
193 use_hiddenapi=True,
194 need_dex=None,
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000195 zip_compression_method="deflate",
196 zip_align_bytes=None,
David Srbeckye82c87d2022-11-16 20:31:11 +0000197 api_level:Union[int, str]=26, # Can also be named alias (string).
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000198 javac_args=[],
David Srbeckyf32b2582022-11-17 15:28:35 +0000199 javac_classpath: List[Path]=[],
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000200 d8_flags=[],
201 smali_args=[],
202 use_smali=True,
203 use_jasmin=True,
204 ):
David Srbeckyf32b2582022-11-17 15:28:35 +0000205 javac_classpath = javac_classpath.copy() # Do not modify default value.
David Srbecky51dec052022-10-27 14:43:12 +0100206
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000207 # Wrap "pathlib.Path" with our own version that ensures all paths are absolute.
David Srbeckyd272f2d2022-11-15 19:13:34 +0000208 # Plain filenames are assumed to be relative to self.test_dir and made absolute.
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000209 class Path(pathlib.Path):
210 def __new__(cls, filename: str):
211 path = pathlib.Path(filename)
David Srbeckyd272f2d2022-11-15 19:13:34 +0000212 return path if path.is_absolute() else (self.test_dir / path)
David Srbecky8106b382022-04-20 13:37:15 +0100213
David Srbeckyd272f2d2022-11-15 19:13:34 +0000214 need_dex = (self.host or self.target) if need_dex is None else need_dex
David Srbecky8106b382022-04-20 13:37:15 +0100215
David Srbeckyd272f2d2022-11-15 19:13:34 +0000216 if self.jvm:
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000217 # No desugaring on jvm because it supports the latest functionality.
218 use_desugar = False
David Srbecky8106b382022-04-20 13:37:15 +0100219
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000220 # Set API level for smali and d8.
David Srbeckye82c87d2022-11-16 20:31:11 +0000221 if isinstance(api_level, str):
222 API_LEVEL = {
David Srbeckyd272f2d2022-11-15 19:13:34 +0000223 "default-methods": 24,
224 "parameter-annotations": 25,
225 "agents": 26,
226 "method-handles": 26,
227 "var-handles": 28,
228 }
David Srbeckye82c87d2022-11-16 20:31:11 +0000229 api_level = API_LEVEL[api_level]
230 assert isinstance(api_level, int), api_level
David Srbecky8106b382022-04-20 13:37:15 +0100231
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000232 def zip(zip_target: Path, *files: Path):
233 zip_args = ["-o", zip_target, "-C", zip_target.parent]
234 if zip_compression_method == "store":
235 zip_args.extend(["-L", "0"])
236 for f in files:
237 zip_args.extend(["-f", f])
David Srbeckyfeed4af2022-11-17 10:44:39 +0000238 self.soong_zip(zip_args)
David Srbecky8106b382022-04-20 13:37:15 +0100239
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000240 if zip_align_bytes:
241 # zipalign does not operate in-place, so write results to a temp file.
242 with TemporaryDirectory() as tmp_dir:
243 tmp_file = Path(tmp_dir) / "aligned.zip"
David Srbeckyfeed4af2022-11-17 10:44:39 +0000244 self.zipalign(["-f", str(zip_align_bytes), zip_target, tmp_file])
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000245 # replace original zip target with our temp file.
246 tmp_file.rename(zip_target)
247
248
249 def make_jasmin(dst_dir: Path, src_dir: Path) -> Optional[Path]:
250 if not use_jasmin or not src_dir.exists():
251 return None # No sources to compile.
252 dst_dir.mkdir()
David Srbeckyfeed4af2022-11-17 10:44:39 +0000253 self.jasmin(["-d", dst_dir] + sorted(src_dir.glob("**/*.j")))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000254 return dst_dir
255
256 def make_smali(dst_dex: Path, src_dir: Path) -> Optional[Path]:
257 if not use_smali or not src_dir.exists():
258 return None # No sources to compile.
David Srbeckyfeed4af2022-11-17 10:44:39 +0000259 self.smali(["-JXmx512m", "assemble"] + smali_args + ["--api", str(api_level)] +
260 ["--output", dst_dex] + sorted(src_dir.glob("**/*.smali")))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000261 return dst_dex
262
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000263 def make_java(dst_dir: Path, *src_dirs: Path) -> Optional[Path]:
264 if not any(src_dir.exists() for src_dir in src_dirs):
265 return None # No sources to compile.
266 dst_dir.mkdir(exist_ok=True)
David Srbeckyd272f2d2022-11-15 19:13:34 +0000267 args = self.javac_args.split(" ") + javac_args
268 args += ["-implicit:none", "-encoding", "utf8", "-d", dst_dir]
269 if not self.jvm:
270 args += ["-bootclasspath", self.bootclasspath]
David Srbeckyf32b2582022-11-17 15:28:35 +0000271 if javac_classpath:
272 args += ["-classpath", javac_classpath]
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000273 for src_dir in src_dirs:
274 args += sorted(src_dir.glob("**/*.java"))
David Srbeckyfeed4af2022-11-17 10:44:39 +0000275 self.javac(args)
David Srbeckyf32b2582022-11-17 15:28:35 +0000276 javac_post = Path("javac_post.sh")
277 if javac_post.exists():
278 self.run(javac_post, [dst_dir])
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000279 return dst_dir
280
281
282 # Make a "dex" file given a directory of classes. This will be
283 # packaged in a jar file.
284 def make_dex(src_dir: Path):
285 dst_jar = Path(src_dir.name + ".jar")
David Srbeckyd272f2d2022-11-15 19:13:34 +0000286 args = d8_flags + ["--min-api", str(api_level), "--output", dst_jar]
287 args += ["--lib", self.bootclasspath] if use_desugar else ["--no-desugaring"]
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000288 args += sorted(src_dir.glob("**/*.class"))
David Srbeckyfeed4af2022-11-17 10:44:39 +0000289 self.d8(args)
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000290
291 # D8 outputs to JAR files today rather than DEX files as DX used
292 # to. To compensate, we extract the DEX from d8's output to meet the
293 # expectations of make_dex callers.
294 dst_dex = Path(src_dir.name + ".dex")
David Srbecky36b520b2022-10-27 14:43:12 +0100295 with TemporaryDirectory() as tmp_dir:
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000296 zipfile.ZipFile(dst_jar, "r").extractall(tmp_dir)
297 (Path(tmp_dir) / "classes.dex").rename(dst_dex)
298
299 # Merge all the dex files.
300 # Skip non-existing files, but at least 1 file must exist.
301 def make_dexmerge(dst_dex: Path, *src_dexs: Path):
302 # Include destination. Skip any non-existing files.
303 srcs = [f for f in [dst_dex] + list(src_dexs) if f.exists()]
304
305 # NB: We merge even if there is just single input.
306 # It is useful to normalize non-deterministic smali output.
David Srbeckyd272f2d2022-11-15 19:13:34 +0000307 tmp_dir = self.test_dir / "dexmerge"
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000308 tmp_dir.mkdir()
David Srbeckyfeed4af2022-11-17 10:44:39 +0000309 self.d8(["--min-api", str(api_level), "--output", tmp_dir] + srcs)
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000310 assert not (tmp_dir / "classes2.dex").exists()
311 for src_file in srcs:
312 src_file.unlink()
313 (tmp_dir / "classes.dex").rename(dst_dex)
314 tmp_dir.rmdir()
David Srbecky8106b382022-04-20 13:37:15 +0100315
316
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000317 def make_hiddenapi(*dex_files: Path):
David Srbeckyd272f2d2022-11-15 19:13:34 +0000318 if not use_hiddenapi or not Path("hiddenapi-flags.csv").exists():
319 return # Nothing to do.
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000320 args: List[Union[str, Path]] = ["encode"]
321 for dex_file in dex_files:
322 args.extend(["--input-dex=" + str(dex_file), "--output-dex=" + str(dex_file)])
323 args.append("--api-flags=hiddenapi-flags.csv")
324 args.append("--no-force-assign-all")
David Srbeckyfeed4af2022-11-17 10:44:39 +0000325 self.hiddenapi(args)
David Srbecky8106b382022-04-20 13:37:15 +0100326
327
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000328 if Path("classes.dex").exists():
David Srbeckyd272f2d2022-11-15 19:13:34 +0000329 zip(Path(self.test_name + ".jar"), Path("classes.dex"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000330 return
331
332 if Path("classes.dm").exists():
David Srbeckyd272f2d2022-11-15 19:13:34 +0000333 zip(Path(self.test_name + ".jar"), Path("classes.dm"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000334 return
David Srbecky37274d52022-11-03 11:04:16 +0000335
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000336 if make_jasmin(Path("jasmin_classes"), Path("jasmin")):
David Srbeckyf32b2582022-11-17 15:28:35 +0000337 javac_classpath.append(Path("jasmin_classes"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000338
339 if make_jasmin(Path("jasmin_classes2"), Path("jasmin-multidex")):
David Srbeckyf32b2582022-11-17 15:28:35 +0000340 javac_classpath.append(Path("jasmin_classes2"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000341
David Srbeckyd272f2d2022-11-15 19:13:34 +0000342 # To allow circular references, compile src/, src-multidex/, src-aotex/,
343 # src-bcpex/, src-ex/ together and pass the output as class path argument.
344 # Replacement sources in src-art/, src2/ and src-ex2/ can replace symbols
345 # used by the other src-* sources we compile here but everything needed to
346 # compile the other src-* sources should be present in src/ (and jasmin*/).
347 extra_srcs = ["src-multidex", "src-aotex", "src-bcpex", "src-ex"]
348 replacement_srcs = ["src2", "src-ex2"] + ([] if self.jvm else ["src-art"])
349 if (Path("src").exists() and
350 any(Path(p).exists() for p in extra_srcs + replacement_srcs)):
351 make_java(Path("classes-tmp-all"), Path("src"), *map(Path, extra_srcs))
David Srbeckyf32b2582022-11-17 15:28:35 +0000352 javac_classpath.append(Path("classes-tmp-all"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000353
354 if make_java(Path("classes-aotex"), Path("src-aotex")) and need_dex:
355 make_dex(Path("classes-aotex"))
356 # rename it so it shows up as "classes.dex" in the zip file.
357 Path("classes-aotex.dex").rename(Path("classes.dex"))
David Srbeckyd272f2d2022-11-15 19:13:34 +0000358 zip(Path(self.test_name + "-aotex.jar"), Path("classes.dex"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000359
360 if make_java(Path("classes-bcpex"), Path("src-bcpex")) and need_dex:
361 make_dex(Path("classes-bcpex"))
362 # rename it so it shows up as "classes.dex" in the zip file.
363 Path("classes-bcpex.dex").rename(Path("classes.dex"))
David Srbeckyd272f2d2022-11-15 19:13:34 +0000364 zip(Path(self.test_name + "-bcpex.jar"), Path("classes.dex"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000365
366 make_java(Path("classes"), Path("src"))
367
David Srbeckyd272f2d2022-11-15 19:13:34 +0000368 if not self.jvm:
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000369 # Do not attempt to build src-art directories on jvm,
370 # since it would fail without libcore.
371 make_java(Path("classes"), Path("src-art"))
372
373 if make_java(Path("classes2"), Path("src-multidex")) and need_dex:
374 make_dex(Path("classes2"))
375
376 make_java(Path("classes"), Path("src2"))
377
378 # If the classes directory is not-empty, package classes in a DEX file.
379 # NB: some tests provide classes rather than java files.
380 if any(Path("classes").glob("*")) and need_dex:
381 make_dex(Path("classes"))
382
383 if Path("jasmin_classes").exists():
384 # Compile Jasmin classes as if they were part of the classes.dex file.
385 if need_dex:
386 make_dex(Path("jasmin_classes"))
387 make_dexmerge(Path("classes.dex"), Path("jasmin_classes.dex"))
388 else:
389 # Move jasmin classes into classes directory so that they are picked up
390 # with -cp classes.
391 Path("classes").mkdir(exist_ok=True)
392 copytree(Path("jasmin_classes"), Path("classes"), dirs_exist_ok=True)
393
394 if need_dex and make_smali(Path("smali_classes.dex"), Path("smali")):
395 # Merge smali files into classes.dex,
396 # this takes priority over any jasmin files.
397 make_dexmerge(Path("classes.dex"), Path("smali_classes.dex"))
398
399 # Compile Jasmin classes in jasmin-multidex as if they were part of
400 # the classes2.jar
David Srbeckyd272f2d2022-11-15 19:13:34 +0000401 if Path("jasmin-multidex").exists():
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000402 if need_dex:
403 make_dex(Path("jasmin_classes2"))
404 make_dexmerge(Path("classes2.dex"), Path("jasmin_classes2.dex"))
405 else:
406 # Move jasmin classes into classes2 directory so that
407 # they are picked up with -cp classes2.
408 Path("classes2").mkdir()
409 copytree(Path("jasmin_classes2"), Path("classes2"), dirs_exist_ok=True)
410 rmtree(Path("jasmin_classes2"))
411
412 if need_dex and make_smali(Path("smali_classes2.dex"), Path("smali-multidex")):
413 # Merge smali_classes2.dex into classes2.dex
414 make_dexmerge(Path("classes2.dex"), Path("smali_classes2.dex"))
415
416 make_java(Path("classes-ex"), Path("src-ex"))
417
418 make_java(Path("classes-ex"), Path("src-ex2"))
419
420 if Path("classes-ex").exists() and need_dex:
421 make_dex(Path("classes-ex"))
422
423 if need_dex and make_smali(Path("smali_classes-ex.dex"), Path("smali-ex")):
424 # Merge smali files into classes-ex.dex.
425 make_dexmerge(Path("classes-ex.dex"), Path("smali_classes-ex.dex"))
426
427 if Path("classes-ex.dex").exists():
428 # Apply hiddenapi on the dex files if the test has API list file(s).
David Srbeckyd272f2d2022-11-15 19:13:34 +0000429 make_hiddenapi(Path("classes-ex.dex"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000430
431 # quick shuffle so that the stored name is "classes.dex"
432 Path("classes.dex").rename(Path("classes-1.dex"))
433 Path("classes-ex.dex").rename(Path("classes.dex"))
David Srbeckyd272f2d2022-11-15 19:13:34 +0000434 zip(Path(self.test_name + "-ex.jar"), Path("classes.dex"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000435 Path("classes.dex").rename(Path("classes-ex.dex"))
436 Path("classes-1.dex").rename(Path("classes.dex"))
437
438 # Apply hiddenapi on the dex files if the test has API list file(s).
David Srbeckyd272f2d2022-11-15 19:13:34 +0000439 if need_dex:
440 if any(Path(".").glob("*-multidex")):
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000441 make_hiddenapi(Path("classes.dex"), Path("classes2.dex"))
442 else:
443 make_hiddenapi(Path("classes.dex"))
444
445 # Create a single dex jar with two dex files for multidex.
446 if need_dex:
447 if Path("classes2.dex").exists():
David Srbeckyd272f2d2022-11-15 19:13:34 +0000448 zip(Path(self.test_name + ".jar"), Path("classes.dex"), Path("classes2.dex"))
David Srbeckyaf0ad5a2022-11-15 18:28:35 +0000449 else:
David Srbeckyd272f2d2022-11-15 19:13:34 +0000450 zip(Path(self.test_name + ".jar"), Path("classes.dex"))
David Srbecky37274d52022-11-03 11:04:16 +0000451
452
453# If we build just individual shard, we want to split the work among all the cores,
454# but if the build system builds all shards, we don't want to overload the machine.
455# We don't know which situation we are in, so as simple work-around, we use a lock
456# file to allow only one shard to use multiprocessing at the same time.
457def use_multiprocessing(mode: str) -> bool:
David Srbeckyd26f9072022-12-09 16:40:40 +0000458 if "RBE_server_address" in os.environ:
459 return True
David Srbecky37274d52022-11-03 11:04:16 +0000460 global lock_file
David Srbecky36b520b2022-10-27 14:43:12 +0100461 lock_path = Path(environ["TMPDIR"]) / ("art-test-run-test-build-py-" + mode)
David Srbecky37274d52022-11-03 11:04:16 +0000462 lock_file = open(lock_path, "w")
463 try:
464 lockf(lock_file, LOCK_EX | LOCK_NB)
465 return True # We are the only instance of this script in the build system.
466 except BlockingIOError:
467 return False # Some other instance is already running.
468
469
470def main() -> None:
471 parser = ArgumentParser(description=__doc__)
David Srbecky6bd1cd12022-11-05 18:54:19 +0000472 parser.add_argument("--out", type=Path, help="Final zip file")
David Srbecky37274d52022-11-03 11:04:16 +0000473 parser.add_argument("--mode", choices=["host", "jvm", "target"])
David Srbecky6bd1cd12022-11-05 18:54:19 +0000474 parser.add_argument("--bootclasspath", type=Path)
475 parser.add_argument("--d8", type=Path)
476 parser.add_argument("--hiddenapi", type=Path)
477 parser.add_argument("--jasmin", type=Path)
478 parser.add_argument("--smali", type=Path)
479 parser.add_argument("--soong_zip", type=Path)
480 parser.add_argument("--zipalign", type=Path)
481 parser.add_argument("srcs", nargs="+", type=Path)
David Srbecky37274d52022-11-03 11:04:16 +0000482 args = parser.parse_args()
483
David Srbecky36b520b2022-10-27 14:43:12 +0100484 android_build_top = Path(getcwd()).absolute()
485 ziproot = args.out.absolute().parent / "zip"
486 srcdirs = set(s.parents[-4].absolute() for s in args.srcs)
David Srbecky37274d52022-11-03 11:04:16 +0000487
David Srbeckyc282bef2022-11-16 21:52:33 +0000488 # Special hidden-api shard: If the --hiddenapi flag is provided, build only
489 # hiddenapi tests. Otherwise exclude all hiddenapi tests from normal shards.
490 def filter_by_hiddenapi(srcdir: Path) -> bool:
491 return (args.hiddenapi != None) == ("hiddenapi" in srcdir.name)
492
David Srbecky36b520b2022-10-27 14:43:12 +0100493 # Initialize the test objects.
494 # We need to do this before we change the working directory below.
495 tests: List[BuildTestContext] = []
David Srbeckyc282bef2022-11-16 21:52:33 +0000496 for srcdir in filter(filter_by_hiddenapi, srcdirs):
David Srbecky36b520b2022-10-27 14:43:12 +0100497 dstdir = ziproot / args.mode / srcdir.name
498 copytree(srcdir, dstdir)
499 tests.append(BuildTestContext(args, android_build_top, dstdir))
500
501 # We can not change the working directory per each thread since they all run in parallel.
502 # Create invalid read-only directory to catch accidental use of current working directory.
503 with TemporaryDirectory("-do-not-use-cwd") as invalid_tmpdir:
504 os.chdir(invalid_tmpdir)
505 os.chmod(invalid_tmpdir, 0)
506 with ThreadPoolExecutor(cpu_count() if use_multiprocessing(args.mode) else 1) as pool:
507 jobs = {}
508 for ctx in tests:
David Srbeckyd272f2d2022-11-15 19:13:34 +0000509 jobs[ctx.test_name] = pool.submit(ctx.build)
David Srbecky36b520b2022-10-27 14:43:12 +0100510 for test_name, job in jobs.items():
511 try:
512 job.result()
513 except Exception as e:
514 raise Exception("Failed to build " + test_name) from e
David Srbecky37274d52022-11-03 11:04:16 +0000515
516 # Create the final zip file which contains the content of the temporary directory.
David Srbecky36b520b2022-10-27 14:43:12 +0100517 proc = run([android_build_top / args.soong_zip, "-o", android_build_top / args.out,
518 "-C", ziproot, "-D", ziproot], check=True)
David Srbecky37274d52022-11-03 11:04:16 +0000519
520
521if __name__ == "__main__":
522 main()