test/art_build_rules.py

#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This is the default build script for run-tests.

It can be overwrite by specific run-tests if needed.
It is used from soong build and not intended to be called directly.
"""

import argparse
import functools
import glob
import os
from os import path
import shlex
import shutil
import subprocess
import tempfile
import zipfile
from shutil import rmtree
from os import remove
from re import match

USE_RBE_FOR_JAVAC = 100    # Percentage of tests that can use RBE (between 0 and 100)
USE_RBE_FOR_D8 = 100       # Percentage of tests that can use RBE (between 0 and 100)

def rm(*patterns):
  for pattern in patterns:
    for path in glob.glob(pattern):
      if os.path.isdir(path):
        shutil.rmtree(path)
      else:
        os.remove(path)

def build_run_test(
    use_desugar=True,
    use_hiddenapi=True,
    need_dex=None,
    experimental="no-experiment",
    zip_compression_method="deflate",
    zip_align_bytes=None,
    api_level=None,
    javac_args=[],
    d8_flags=[],
    smali_args=[],
    has_smali=None,
    has_jasmin=None,
  ):

  def parse_bool(text):
    return {"true": True, "false": False}[text.lower()]

  ANDROID_BUILD_TOP = os.environ["ANDROID_BUILD_TOP"]
  SBOX_PATH = os.environ["SBOX_PATH"]
  CWD = os.getcwd()
  TEST_NAME = os.environ["TEST_NAME"]
  ART_TEST_RUN_TEST_BOOTCLASSPATH = path.relpath(os.environ["ART_TEST_RUN_TEST_BOOTCLASSPATH"], CWD)
  NEED_DEX = parse_bool(os.environ["NEED_DEX"]) if need_dex is None else need_dex

  RBE_exec_root = os.environ.get("RBE_exec_root")
  RBE_rewrapper = path.join(ANDROID_BUILD_TOP, "prebuilts/remoteexecution-client/live/rewrapper")

  # Set default values for directories.
  HAS_SMALI = path.exists("smali") if has_smali is None else has_smali
  HAS_JASMIN = path.exists("jasmin") if has_jasmin is None else has_jasmin
  HAS_SRC = path.exists("src")
  HAS_SRC_ART = path.exists("src-art")
  HAS_SRC2 = path.exists("src2")
  HAS_SRC_MULTIDEX = path.exists("src-multidex")
  HAS_SMALI_MULTIDEX = path.exists("smali-multidex")
  HAS_JASMIN_MULTIDEX = path.exists("jasmin-multidex")
  HAS_SMALI_EX = path.exists("smali-ex")
  HAS_SRC_EX = path.exists("src-ex")
  HAS_SRC_EX2 = path.exists("src-ex2")
  HAS_SRC_AOTEX = path.exists("src-aotex")
  HAS_SRC_BCPEX = path.exists("src-bcpex")
  HAS_HIDDENAPI_SPEC = path.exists("hiddenapi-flags.csv")

  JAVAC_ARGS = shlex.split(os.environ.get("JAVAC_ARGS", "")) + javac_args
  SMALI_ARGS = shlex.split(os.environ.get("SMALI_ARGS", "")) + smali_args
  D8_FLAGS = shlex.split(os.environ.get("D8_FLAGS", "")) + d8_flags

  BUILD_MODE = os.environ["BUILD_MODE"]

  # Setup experimental API level mappings in a bash associative array.
  EXPERIMENTAL_API_LEVEL = {}
  EXPERIMENTAL_API_LEVEL["no-experiment"] = "26"
  EXPERIMENTAL_API_LEVEL["default-methods"] = "24"
  EXPERIMENTAL_API_LEVEL["parameter-annotations"] = "25"
  EXPERIMENTAL_API_LEVEL["agents"] = "26"
  EXPERIMENTAL_API_LEVEL["method-handles"] = "26"
  EXPERIMENTAL_API_LEVEL["var-handles"] = "28"

  if BUILD_MODE == "jvm":
    # No desugaring on jvm because it supports the latest functionality.
    use_desugar = False
    # Do not attempt to build src-art directories on jvm,
    # since it would fail without libcore.
    HAS_SRC_ART = False

  # Set API level for smali and d8.
  if not api_level:
    api_level = EXPERIMENTAL_API_LEVEL[experimental]

  # Add API level arguments to smali and dx
  SMALI_ARGS.extend(["--api", str(api_level)])
  D8_FLAGS.extend(["--min-api", str(api_level)])


  def run(executable, args):
    cmd = shlex.split(executable) + args
    if executable.endswith(".sh"):
      cmd = ["/bin/bash"] + cmd
    p = subprocess.run(cmd,
                       encoding=os.sys.stdout.encoding,
                       stderr=subprocess.STDOUT,
                       stdout=subprocess.PIPE)
    if p.returncode != 0:
      raise Exception("Command failed with exit code {}\n$ {}\n{}".format(
                      p.returncode, " ".join(cmd), p.stdout))
    return p


  # Helper functions to execute tools.
  soong_zip = functools.partial(run, os.environ["SOONG_ZIP"])
  zipalign = functools.partial(run, os.environ["ZIPALIGN"])
  javac = functools.partial(run, os.environ["JAVAC"])
  jasmin = functools.partial(run, os.environ["JASMIN"])
  smali = functools.partial(run, os.environ["SMALI"])
  d8 = functools.partial(run, os.environ["D8"])
  hiddenapi = functools.partial(run, os.environ["HIDDENAPI"])

  if "RBE_server_address" in os.environ:
    version = match(r"Version: (\d*)\.(\d*)\.(\d*)", run(RBE_rewrapper, ["--version"]).stdout)
    assert version, "Could not parse RBE version"
    assert tuple(map(int, version.groups())) >= (0, 76, 0), "Please update " + RBE_rewrapper

    def rbe_wrap(args, inputs=set()):
      with tempfile.NamedTemporaryFile(mode="w+t", dir=RBE_exec_root) as input_list:
        for arg in args:
          inputs.update(filter(path.exists, arg.split(":")))
        input_list.writelines([path.relpath(i, RBE_exec_root)+"\n" for i in inputs])
        input_list.flush()
        return run(RBE_rewrapper, [
          "--platform=" + os.environ["RBE_platform"],
          "--input_list_paths=" + input_list.name,
        ] + args)

    if USE_RBE_FOR_JAVAC > (hash(TEST_NAME) % 100):  # Use for given percentage of tests.
      def javac(args):
        output = path.relpath(path.join(CWD, args[args.index("-d") + 1]), RBE_exec_root)
        return rbe_wrap([
          "--output_directories", output,
          os.path.relpath(os.environ["JAVAC"], CWD),
        ] + args)

    if USE_RBE_FOR_D8 > (hash(TEST_NAME) % 100):  # Use for given percentage of tests.
      def d8(args):
        inputs = set([path.join(SBOX_PATH, "tools/out/framework/d8.jar")])
        output = path.relpath(path.join(CWD, args[args.index("--output") + 1]), RBE_exec_root)
        return rbe_wrap([
          "--output_files" if output.endswith(".jar") else "--output_directories", output,
          "--toolchain_inputs=prebuilts/jdk/jdk11/linux-x86/bin/java",
          os.path.relpath(os.environ["D8"], CWD)] + args, inputs)

  # If wrapper script exists, use it instead of the default javac.
  if os.path.exists("javac_wrapper.sh"):
    javac = functools.partial(run, "javac_wrapper.sh")

  def find(root, name):
    return sorted(glob.glob(path.join(root, "**", name), recursive=True))


  def zip(zip_target, *files):
    zip_args = ["-o", zip_target]
    if zip_compression_method == "store":
      zip_args.extend(["-L", "0"])
    for f in files:
      zip_args.extend(["-f", f])
    soong_zip(zip_args)

    if zip_align_bytes:
      # zipalign does not operate in-place, so write results to a temp file.
      with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
        tmp_file = path.join(tmp_dir, "aligned.zip")
        zipalign(["-f", str(zip_align_bytes), zip_target, tmp_file])
        # replace original zip target with our temp file.
        os.rename(tmp_file, zip_target)


  def make_jasmin(out_directory, jasmin_sources):
    os.makedirs(out_directory, exist_ok=True)
    jasmin(["-d", out_directory] + sorted(jasmin_sources))


  # Like regular javac but may include libcore on the bootclasspath.
  def javac_with_bootclasspath(args):
    flags = JAVAC_ARGS + ["-encoding", "utf8"]
    if BUILD_MODE != "jvm":
      flags.extend(["-bootclasspath", ART_TEST_RUN_TEST_BOOTCLASSPATH])
    javac(flags + args)


  # Make a "dex" file given a directory of classes. This will be
  # packaged in a jar file.
  def make_dex(name):
    d8_inputs = find(name, "*.class")
    d8_output = name + ".jar"
    dex_output = name + ".dex"
    if use_desugar:
      flags = ["--lib", ART_TEST_RUN_TEST_BOOTCLASSPATH]
    else:
      flags = ["--no-desugaring"]
    assert d8_inputs
    d8(D8_FLAGS + flags + ["--output", d8_output] + d8_inputs)

    # D8 outputs to JAR files today rather than DEX files as DX used
    # to. To compensate, we extract the DEX from d8's output to meet the
    # expectations of make_dex callers.
    with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
      zipfile.ZipFile(d8_output, "r").extractall(tmp_dir)
      os.rename(path.join(tmp_dir, "classes.dex"), dex_output)


  # Merge all the dex files.
  # Skip non-existing files, but at least 1 file must exist.
  def make_dexmerge(*dex_files_to_merge):
    # Dex file that acts as the destination.
    dst_file = dex_files_to_merge[0]

    # Skip any non-existing files.
    dex_files_to_merge = list(filter(path.exists, dex_files_to_merge))

    # NB: We merge even if there is just single input.
    # It is useful to normalize non-deterministic smali output.

    tmp_dir = "dexmerge"
    os.makedirs(tmp_dir)
    d8(["--min-api", api_level, "--output", tmp_dir] + dex_files_to_merge)
    assert not path.exists(path.join(tmp_dir, "classes2.dex"))
    for input_dex in dex_files_to_merge:
      os.remove(input_dex)
    os.rename(path.join(tmp_dir, "classes.dex"), dst_file)
    os.rmdir(tmp_dir)


  def make_hiddenapi(*dex_files):
    args = ["encode"]
    for dex_file in dex_files:
      args.extend(["--input-dex=" + dex_file, "--output-dex=" + dex_file])
    args.append("--api-flags=hiddenapi-flags.csv")
    args.append("--no-force-assign-all")
    hiddenapi(args)


  if path.exists("classes.dex"):
    zip(TEST_NAME + ".jar", "classes.dex")
    return


  def has_multidex():
    return HAS_SRC_MULTIDEX or HAS_JASMIN_MULTIDEX or HAS_SMALI_MULTIDEX


  def add_to_cp_args(old_cp_args, path):
    if len(old_cp_args) == 0:
      return ["-cp", path]
    else:
      return ["-cp", old_cp_args[1] + ":" + path]


  src_tmp_all = []

  if HAS_JASMIN:
    make_jasmin("jasmin_classes", find("jasmin", "*.j"))
    src_tmp_all = add_to_cp_args(src_tmp_all, "jasmin_classes")

  if HAS_JASMIN_MULTIDEX:
    make_jasmin("jasmin_classes2", find("jasmin-multidex", "*.j"))
    src_tmp_all = add_to_cp_args(src_tmp_all, "jasmin_classes2")

  if HAS_SRC and (HAS_SRC_MULTIDEX or HAS_SRC_AOTEX or HAS_SRC_BCPEX or
                  HAS_SRC_EX or HAS_SRC_ART or HAS_SRC2 or HAS_SRC_EX2):
    # To allow circular references, compile src/, src-multidex/, src-aotex/,
    # src-bcpex/, src-ex/ together and pass the output as class path argument.
    # Replacement sources in src-art/, src2/ and src-ex2/ can replace symbols
    # used by the other src-* sources we compile here but everything needed to
    # compile the other src-* sources should be present in src/ (and jasmin*/).
    os.makedirs("classes-tmp-all")
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes-tmp-all"] +
                             find("src", "*.java") +
                             find("src-multidex", "*.java") +
                             find("src-aotex", "*.java") +
                             find("src-bcpex", "*.java") +
                             find("src-ex", "*.java"))
    src_tmp_all = add_to_cp_args(src_tmp_all, "classes-tmp-all")

  if HAS_SRC_AOTEX:
    os.makedirs("classes-aotex")
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes-aotex"] +
                             find("src-aotex", "*.java"))
    if NEED_DEX:
      make_dex("classes-aotex")
      # rename it so it shows up as "classes.dex" in the zip file.
      os.rename("classes-aotex.dex", "classes.dex")
      zip(TEST_NAME + "-aotex.jar", "classes.dex")

  if HAS_SRC_BCPEX:
    os.makedirs("classes-bcpex")
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes-bcpex"] +
                             find("src-bcpex", "*.java"))
    if NEED_DEX:
      make_dex("classes-bcpex")
      # rename it so it shows up as "classes.dex" in the zip file.
      os.rename("classes-bcpex.dex", "classes.dex")
      zip(TEST_NAME + "-bcpex.jar", "classes.dex")

  if HAS_SRC:
    os.makedirs("classes", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes"] + find("src", "*.java"))

  if HAS_SRC_ART:
    os.makedirs("classes", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes"] + find("src-art", "*.java"))

  if HAS_SRC_MULTIDEX:
    os.makedirs("classes2")
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes2"] +
                             find("src-multidex", "*.java"))
    if NEED_DEX:
      make_dex("classes2")

  if HAS_SRC2:
    os.makedirs("classes", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes"] +
                             find("src2", "*.java"))

  # If the classes directory is not-empty, package classes in a DEX file.
  # NB: some tests provide classes rather than java files.
  if find("classes", "*"):
    if NEED_DEX:
      make_dex("classes")

  if HAS_JASMIN:
    # Compile Jasmin classes as if they were part of the classes.dex file.
    if NEED_DEX:
      make_dex("jasmin_classes")
      make_dexmerge("classes.dex", "jasmin_classes.dex")
    else:
      # Move jasmin classes into classes directory so that they are picked up
      # with -cp classes.
      os.makedirs("classes", exist_ok=True)
      shutil.copytree("jasmin_classes", "classes", dirs_exist_ok=True)

  if HAS_SMALI and NEED_DEX:
    # Compile Smali classes
    smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
          ["--output", "smali_classes.dex"] + find("smali", "*.smali"))
    assert path.exists("smali_classes.dex")
    # Merge smali files into classes.dex,
    # this takes priority over any jasmin files.
    make_dexmerge("classes.dex", "smali_classes.dex")

  # Compile Jasmin classes in jasmin-multidex as if they were part of
  # the classes2.jar
  if HAS_JASMIN_MULTIDEX:
    if NEED_DEX:
      make_dex("jasmin_classes2")
      make_dexmerge("classes2.dex", "jasmin_classes2.dex")
    else:
      # Move jasmin classes into classes2 directory so that
      # they are picked up with -cp classes2.
      os.makedirs("classes2", exist_ok=True)
      shutil.copytree("jasmin_classes2", "classes2", dirs_exist_ok=True)
      shutil.rmtree("jasmin_classes2")

  if HAS_SMALI_MULTIDEX and NEED_DEX:
    # Compile Smali classes
    smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
          ["--output", "smali_classes2.dex"] + find("smali-multidex", "*.smali"))

    # Merge smali_classes2.dex into classes2.dex
    make_dexmerge("classes2.dex", "smali_classes2.dex")

  if HAS_SRC_EX:
    os.makedirs("classes-ex", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes-ex"] + find("src-ex", "*.java"))

  if HAS_SRC_EX2:
    os.makedirs("classes-ex", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes-ex"] + find("src-ex2", "*.java"))

  if path.exists("classes-ex") and NEED_DEX:
    make_dex("classes-ex")

  if HAS_SMALI_EX and NEED_DEX:
    # Compile Smali classes
    smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
          ["--output", "smali_classes-ex.dex"] + find("smali-ex", "*.smali"))
    assert path.exists("smali_classes-ex.dex")
    # Merge smali files into classes-ex.dex.
    make_dexmerge("classes-ex.dex", "smali_classes-ex.dex")

  if path.exists("classes-ex.dex"):
    # Apply hiddenapi on the dex files if the test has API list file(s).
    if use_hiddenapi and HAS_HIDDENAPI_SPEC:
      make_hiddenapi("classes-ex.dex")

    # quick shuffle so that the stored name is "classes.dex"
    os.rename("classes.dex", "classes-1.dex")
    os.rename("classes-ex.dex", "classes.dex")
    zip(TEST_NAME + "-ex.jar", "classes.dex")
    os.rename("classes.dex", "classes-ex.dex")
    os.rename("classes-1.dex", "classes.dex")

  # Apply hiddenapi on the dex files if the test has API list file(s).
  if NEED_DEX and use_hiddenapi and HAS_HIDDENAPI_SPEC:
    if has_multidex():
      make_hiddenapi("classes.dex", "classes2.dex")
    else:
      make_hiddenapi("classes.dex")

  # Create a single dex jar with two dex files for multidex.
  if NEED_DEX:
    if path.exists("classes2.dex"):
      zip(TEST_NAME + ".jar", "classes.dex", "classes2.dex")
    else:
      zip(TEST_NAME + ".jar", "classes.dex")