Convert run-test bash build scripts to python.
Tests use the build script for two distinct purposes:
* Some tests generate their source code before completion.
Keep this code in bash for now and rename it "generate-sources".
* To customize arguments passed to the default compilation script.
Since the default compilation script is in python, make this
a normal python call, and add any customization as arguments.
This speeds up the run-test compilation. There was small cost to
pay due to the switching from python to bash to python.
This added up for the thousands of run-test compilations that we do.
It also removes the need for argument parser in the default script.
We could also remove the data passing via environment (to-do later).
It moves the definition to more build-system-like look and feel.
Test: The outputs are bit-for-bit identical as before.
Change-Id: Idca4181a4676036f06aae0a8f6eea3a3c30d390e
diff --git a/test/art_build_rules.py b/test/art_build_rules.py
new file mode 100644
index 0000000..4e4a959
--- /dev/null
+++ b/test/art_build_rules.py
@@ -0,0 +1,403 @@
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This is the default build script for run-tests.
+
+It can be overwrite by specific run-tests if needed.
+It is used from soong build and not intended to be called directly.
+"""
+
+import argparse
+import functools
+import glob
+import os
+from os import path
+import shlex
+import shutil
+import subprocess
+import tempfile
+import zipfile
+from shutil import rmtree
+from os import remove
+
+def rm(*patterns):
+ for pattern in patterns:
+ for path in glob.glob(pattern):
+ if os.path.isdir(path):
+ shutil.rmtree(path)
+ else:
+ os.remove(path)
+
+def build_run_test(
+ use_desugar=True,
+ use_hiddenapi=True,
+ need_dex=None,
+ experimental="no-experiment",
+ zip_compression_method="deflate",
+ zip_align_bytes=None,
+ api_level=None,
+ javac_args=[],
+ d8_flags=[],
+ smali_args=[],
+ has_smali=None,
+ has_jasmin=None,
+ ):
+
+ def parse_bool(text):
+ return {"true": True, "false": False}[text.lower()]
+
+ TEST_NAME = os.environ["TEST_NAME"]
+ ART_TEST_RUN_TEST_BOOTCLASSPATH = os.environ["ART_TEST_RUN_TEST_BOOTCLASSPATH"]
+ NEED_DEX = parse_bool(os.environ["NEED_DEX"]) if need_dex is None else need_dex
+
+ # Set default values for directories.
+ HAS_SMALI = path.exists("smali") if has_smali is None else has_smali
+ HAS_JASMIN = path.exists("jasmin") if has_jasmin is None else has_jasmin
+ HAS_SRC = path.exists("src")
+ HAS_SRC_ART = path.exists("src-art")
+ HAS_SRC2 = path.exists("src2")
+ HAS_SRC_MULTIDEX = path.exists("src-multidex")
+ HAS_SMALI_MULTIDEX = path.exists("smali-multidex")
+ HAS_JASMIN_MULTIDEX = path.exists("jasmin-multidex")
+ HAS_SMALI_EX = path.exists("smali-ex")
+ HAS_SRC_EX = path.exists("src-ex")
+ HAS_SRC_EX2 = path.exists("src-ex2")
+ HAS_SRC_AOTEX = path.exists("src-aotex")
+ HAS_SRC_BCPEX = path.exists("src-bcpex")
+ HAS_HIDDENAPI_SPEC = path.exists("hiddenapi-flags.csv")
+
+ JAVAC_ARGS = shlex.split(os.environ.get("JAVAC_ARGS", "")) + javac_args
+ SMALI_ARGS = shlex.split(os.environ.get("SMALI_ARGS", "")) + smali_args
+ D8_FLAGS = shlex.split(os.environ.get("D8_FLAGS", "")) + d8_flags
+
+ BUILD_MODE = os.environ["BUILD_MODE"]
+
+ # Setup experimental API level mappings in a bash associative array.
+ EXPERIMENTAL_API_LEVEL = {}
+ EXPERIMENTAL_API_LEVEL["no-experiment"] = "26"
+ EXPERIMENTAL_API_LEVEL["default-methods"] = "24"
+ EXPERIMENTAL_API_LEVEL["parameter-annotations"] = "25"
+ EXPERIMENTAL_API_LEVEL["agents"] = "26"
+ EXPERIMENTAL_API_LEVEL["method-handles"] = "26"
+ EXPERIMENTAL_API_LEVEL["var-handles"] = "28"
+
+ if BUILD_MODE == "jvm":
+ # No desugaring on jvm because it supports the latest functionality.
+ use_desugar = False
+ # Do not attempt to build src-art directories on jvm,
+ # since it would fail without libcore.
+ HAS_SRC_ART = False
+
+ # Set API level for smali and d8.
+ if not api_level:
+ api_level = EXPERIMENTAL_API_LEVEL[experimental]
+
+ # Add API level arguments to smali and dx
+ SMALI_ARGS.extend(["--api", str(api_level)])
+ D8_FLAGS.extend(["--min-api", str(api_level)])
+
+
+ def run(executable, args):
+ cmd = shlex.split(executable) + args
+ if executable.endswith(".sh"):
+ cmd = ["/bin/bash"] + cmd
+ p = subprocess.run(cmd,
+ encoding=os.sys.stdout.encoding,
+ stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE)
+ if p.returncode != 0:
+ raise Exception("Command failed with exit code {}\n$ {}\n{}".format(
+ p.returncode, " ".join(cmd), p.stdout))
+
+
+ # Helper functions to execute tools.
+ soong_zip = functools.partial(run, os.environ["SOONG_ZIP"])
+ zipalign = functools.partial(run, os.environ["ZIPALIGN"])
+ javac = functools.partial(run, os.environ["JAVAC"])
+ jasmin = functools.partial(run, os.environ["JASMIN"])
+ smali = functools.partial(run, os.environ["SMALI"])
+ d8 = functools.partial(run, os.environ["D8"])
+ hiddenapi = functools.partial(run, os.environ["HIDDENAPI"])
+
+ # If wrapper script exists, use it instead of the default javac.
+ if os.path.exists("javac_wrapper.sh"):
+ javac = functools.partial(run, "javac_wrapper.sh")
+
+ def find(root, name):
+ return sorted(glob.glob(path.join(root, "**", name), recursive=True))
+
+
+ def zip(zip_target, *files):
+ zip_args = ["-o", zip_target]
+ if zip_compression_method == "store":
+ zip_args.extend(["-L", "0"])
+ for f in files:
+ zip_args.extend(["-f", f])
+ soong_zip(zip_args)
+
+ if zip_align_bytes:
+ # zipalign does not operate in-place, so write results to a temp file.
+ with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
+ tmp_file = path.join(tmp_dir, "aligned.zip")
+ zipalign(["-f", str(zip_align_bytes), zip_target, tmp_file])
+ # replace original zip target with our temp file.
+ os.rename(tmp_file, zip_target)
+
+
+ def make_jasmin(out_directory, jasmin_sources):
+ os.makedirs(out_directory, exist_ok=True)
+ jasmin(["-d", out_directory] + sorted(jasmin_sources))
+
+
+ # Like regular javac but may include libcore on the bootclasspath.
+ def javac_with_bootclasspath(args):
+ flags = JAVAC_ARGS + ["-encoding", "utf8"]
+ if BUILD_MODE != "jvm":
+ flags.extend(["-bootclasspath", ART_TEST_RUN_TEST_BOOTCLASSPATH])
+ javac(flags + args)
+
+
+ # Make a "dex" file given a directory of classes. This will be
+ # packaged in a jar file.
+ def make_dex(name):
+ d8_inputs = find(name, "*.class")
+ d8_output = name + ".jar"
+ dex_output = name + ".dex"
+ if use_desugar:
+ flags = ["--lib", ART_TEST_RUN_TEST_BOOTCLASSPATH]
+ else:
+ flags = ["--no-desugaring"]
+ assert d8_inputs
+ d8(D8_FLAGS + flags + ["--output", d8_output] + d8_inputs)
+
+ # D8 outputs to JAR files today rather than DEX files as DX used
+ # to. To compensate, we extract the DEX from d8's output to meet the
+ # expectations of make_dex callers.
+ with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
+ zipfile.ZipFile(d8_output, "r").extractall(tmp_dir)
+ os.rename(path.join(tmp_dir, "classes.dex"), dex_output)
+
+
+ # Merge all the dex files.
+ # Skip non-existing files, but at least 1 file must exist.
+ def make_dexmerge(*dex_files_to_merge):
+ # Dex file that acts as the destination.
+ dst_file = dex_files_to_merge[0]
+
+ # Skip any non-existing files.
+ dex_files_to_merge = list(filter(path.exists, dex_files_to_merge))
+
+ # NB: We merge even if there is just single input.
+ # It is useful to normalize non-deterministic smali output.
+
+ with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
+ d8(["--min-api", str(api_level), "--output", tmp_dir] + dex_files_to_merge)
+ assert not path.exists(path.join(tmp_dir, "classes2.dex"))
+ for input_dex in dex_files_to_merge:
+ os.remove(input_dex)
+ os.rename(path.join(tmp_dir, "classes.dex"), dst_file)
+
+
+ def make_hiddenapi(*dex_files):
+ args = ["encode"]
+ for dex_file in dex_files:
+ args.extend(["--input-dex=" + dex_file, "--output-dex=" + dex_file])
+ args.append("--api-flags=hiddenapi-flags.csv")
+ args.append("--no-force-assign-all")
+ hiddenapi(args)
+
+
+ if path.exists("classes.dex"):
+ zip(TEST_NAME + ".jar", "classes.dex")
+ return
+
+
+ def has_multidex():
+ return HAS_SRC_MULTIDEX or HAS_JASMIN_MULTIDEX or HAS_SMALI_MULTIDEX
+
+
+ def add_to_cp_args(old_cp_args, path):
+ if len(old_cp_args) == 0:
+ return ["-cp", path]
+ else:
+ return ["-cp", old_cp_args[1] + ":" + path]
+
+
+ src_tmp_all = []
+
+ if HAS_JASMIN:
+ make_jasmin("jasmin_classes", find("jasmin", "*.j"))
+ src_tmp_all = add_to_cp_args(src_tmp_all, "jasmin_classes")
+
+ if HAS_JASMIN_MULTIDEX:
+ make_jasmin("jasmin_classes2", find("jasmin-multidex", "*.j"))
+ src_tmp_all = add_to_cp_args(src_tmp_all, "jasmin_classes2")
+
+ if HAS_SRC and (HAS_SRC_MULTIDEX or HAS_SRC_AOTEX or HAS_SRC_BCPEX or
+ HAS_SRC_EX or HAS_SRC_ART or HAS_SRC2 or HAS_SRC_EX2):
+ # To allow circular references, compile src/, src-multidex/, src-aotex/,
+ # src-bcpex/, src-ex/ together and pass the output as class path argument.
+ # Replacement sources in src-art/, src2/ and src-ex2/ can replace symbols
+ # used by the other src-* sources we compile here but everything needed to
+ # compile the other src-* sources should be present in src/ (and jasmin*/).
+ os.makedirs("classes-tmp-all")
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes-tmp-all"] +
+ find("src", "*.java") +
+ find("src-multidex", "*.java") +
+ find("src-aotex", "*.java") +
+ find("src-bcpex", "*.java") +
+ find("src-ex", "*.java"))
+ src_tmp_all = add_to_cp_args(src_tmp_all, "classes-tmp-all")
+
+ if HAS_SRC_AOTEX:
+ os.makedirs("classes-aotex")
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes-aotex"] +
+ find("src-aotex", "*.java"))
+ if NEED_DEX:
+ make_dex("classes-aotex")
+ # rename it so it shows up as "classes.dex" in the zip file.
+ os.rename("classes-aotex.dex", "classes.dex")
+ zip(TEST_NAME + "-aotex.jar", "classes.dex")
+
+ if HAS_SRC_BCPEX:
+ os.makedirs("classes-bcpex")
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes-bcpex"] +
+ find("src-bcpex", "*.java"))
+ if NEED_DEX:
+ make_dex("classes-bcpex")
+ # rename it so it shows up as "classes.dex" in the zip file.
+ os.rename("classes-bcpex.dex", "classes.dex")
+ zip(TEST_NAME + "-bcpex.jar", "classes.dex")
+
+ if HAS_SRC:
+ os.makedirs("classes", exist_ok=True)
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes"] + find("src", "*.java"))
+
+ if HAS_SRC_ART:
+ os.makedirs("classes", exist_ok=True)
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes"] + find("src-art", "*.java"))
+
+ if HAS_SRC_MULTIDEX:
+ os.makedirs("classes2")
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes2"] +
+ find("src-multidex", "*.java"))
+ if NEED_DEX:
+ make_dex("classes2")
+
+ if HAS_SRC2:
+ os.makedirs("classes", exist_ok=True)
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes"] +
+ find("src2", "*.java"))
+
+ # If the classes directory is not-empty, package classes in a DEX file.
+ # NB: some tests provide classes rather than java files.
+ if find("classes", "*"):
+ if NEED_DEX:
+ make_dex("classes")
+
+ if HAS_JASMIN:
+ # Compile Jasmin classes as if they were part of the classes.dex file.
+ if NEED_DEX:
+ make_dex("jasmin_classes")
+ make_dexmerge("classes.dex", "jasmin_classes.dex")
+ else:
+ # Move jasmin classes into classes directory so that they are picked up
+ # with -cp classes.
+ os.makedirs("classes", exist_ok=True)
+ shutil.copytree("jasmin_classes", "classes", dirs_exist_ok=True)
+
+ if HAS_SMALI and NEED_DEX:
+ # Compile Smali classes
+ smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
+ ["--output", "smali_classes.dex"] + find("smali", "*.smali"))
+ assert path.exists("smali_classes.dex")
+ # Merge smali files into classes.dex,
+ # this takes priority over any jasmin files.
+ make_dexmerge("classes.dex", "smali_classes.dex")
+
+ # Compile Jasmin classes in jasmin-multidex as if they were part of
+ # the classes2.jar
+ if HAS_JASMIN_MULTIDEX:
+ if NEED_DEX:
+ make_dex("jasmin_classes2")
+ make_dexmerge("classes2.dex", "jasmin_classes2.dex")
+ else:
+ # Move jasmin classes into classes2 directory so that
+ # they are picked up with -cp classes2.
+ os.makedirs("classes2", exist_ok=True)
+ shutil.copytree("jasmin_classes2", "classes2", dirs_exist_ok=True)
+ shutil.rmtree("jasmin_classes2")
+
+ if HAS_SMALI_MULTIDEX and NEED_DEX:
+ # Compile Smali classes
+ smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
+ ["--output", "smali_classes2.dex"] + find("smali-multidex", "*.smali"))
+
+ # Merge smali_classes2.dex into classes2.dex
+ make_dexmerge("classes2.dex", "smali_classes2.dex")
+
+ if HAS_SRC_EX:
+ os.makedirs("classes-ex", exist_ok=True)
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes-ex"] + find("src-ex", "*.java"))
+
+ if HAS_SRC_EX2:
+ os.makedirs("classes-ex", exist_ok=True)
+ javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
+ ["-d", "classes-ex"] + find("src-ex2", "*.java"))
+
+ if path.exists("classes-ex") and NEED_DEX:
+ make_dex("classes-ex")
+
+ if HAS_SMALI_EX and NEED_DEX:
+ # Compile Smali classes
+ smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
+ ["--output", "smali_classes-ex.dex"] + find("smali-ex", "*.smali"))
+ assert path.exists("smali_classes-ex.dex")
+ # Merge smali files into classes-ex.dex.
+ make_dexmerge("classes-ex.dex", "smali_classes-ex.dex")
+
+ if path.exists("classes-ex.dex"):
+ # Apply hiddenapi on the dex files if the test has API list file(s).
+ if use_hiddenapi and HAS_HIDDENAPI_SPEC:
+ make_hiddenapi("classes-ex.dex")
+
+ # quick shuffle so that the stored name is "classes.dex"
+ os.rename("classes.dex", "classes-1.dex")
+ os.rename("classes-ex.dex", "classes.dex")
+ zip(TEST_NAME + "-ex.jar", "classes.dex")
+ os.rename("classes.dex", "classes-ex.dex")
+ os.rename("classes-1.dex", "classes.dex")
+
+ # Apply hiddenapi on the dex files if the test has API list file(s).
+ if NEED_DEX and use_hiddenapi and HAS_HIDDENAPI_SPEC:
+ if has_multidex():
+ make_hiddenapi("classes.dex", "classes2.dex")
+ else:
+ make_hiddenapi("classes.dex")
+
+ # Create a single dex jar with two dex files for multidex.
+ if NEED_DEX:
+ if path.exists("classes2.dex"):
+ zip(TEST_NAME + ".jar", "classes.dex", "classes2.dex")
+ else:
+ zip(TEST_NAME + ".jar", "classes.dex")