Misc tools improvements

Rewrote compile-jar.sh in python to add ability to compile against
host BCP, compile multiple apks/dexs, launch with debugger and utilize
profiles more easily.  Changed compile-jar.sh to be a wrapper around
this script.

```
compile-jar.py [-h] [--dex2oat DEX2OAT] [--debug] [--profman PROFMAN]
               [--debug-profman]
               [--profile-file PROFILE | --profile-line PROFILE_LINE]
               [--arch {arm,arm64,x86,x86_64,host64,host32}]
               [--odex-file ODEX_FILE] [--save-profile SAVE_PROFILE]
               DEX [DEX ...]
```

Added pylibdexfile.py a python wrapper around libdexfile_external for
scripting use. It can be used by 'include'ing it or interactively with
`python3 -i art/tools/pylibdexfile.py`. Currently it has support for
opening individual dexfiles (either from buffers or files) and opening
jars. It is also able to get a list of all methods referenced in the
dexfile. As libdexfile_external is expanded we could add to this if
desired.

Test: Manual
Test: ./art/tools/compile-jar.py /tmp/benchmarks.dex /tmp/ritz.jar --debug --odex-file /tmp/ritz.odex --dump-stats --profile-line 'HSLbenchmarks/common/java/BenchmarkBase;->measure()D' --profile-line 'HSLbenchmarks/common/java/SelfTimingBenchmarkBase;->measureFor(ZJ)D' -j1 --compiler-filter=speed-profile
Change-Id: I7bbbce6e68aef5a5fa1173960b0055bf5f76cd7f

3 files changed, 453 insertions, 9 deletions

diff --git a/tools/compile-jar.py b/tools/compile-jar.py
new file mode 100755
index 0000000000..1211e748a9
--- /dev/null
+++ b/tools/compile-jar.py
@@ -0,0 +1,217 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script runs dex2oat on the host to compile a provided JAR or APK.
+#
+
+import argparse
+import shlex
+import subprocess
+import os
+import os.path
+
+
+def run_print(lst):
+  return " ".join(map(shlex.quote, lst))
+
+
+def parse_args():
+  parser = argparse.ArgumentParser(
+      description="compile dex or jar files",
+      epilog="Unrecognized options are passed on to dex2oat unmodified.")
+  parser.add_argument(
+      "--dex2oat",
+      action="store",
+      default=os.path.expandvars("$ANDROID_HOST_OUT/bin/dex2oatd64"),
+      help="selects the dex2oat to use.")
+  parser.add_argument(
+      "--debug",
+      action="store_true",
+      default=False,
+      help="launches dex2oatd with lldb-server g :5039. Connect using vscode or remote lldb"
+  )
+  parser.add_argument(
+      "--profman",
+      action="store",
+      default=os.path.expandvars("$ANDROID_HOST_OUT/bin/profmand"),
+      help="selects the profman to use.")
+  parser.add_argument(
+      "--debug-profman",
+      action="store_true",
+      default=False,
+      help="launches profman with lldb-server g :5039. Connect using vscode or remote lldb"
+  )
+  profs = parser.add_mutually_exclusive_group()
+  profs.add_argument(
+      "--profile-file",
+      action="store",
+      help="Use this profile file. Probably want to pass --compiler-filter=speed-profile with this."
+  )
+  profs.add_argument(
+      "--profile-line",
+      action="append",
+      default=[],
+      help="functions to add to a profile. Probably want to pass --compiler-filter=speed-profile with this. All functions are marked as 'hot'. Use --profile-file for more control."
+  )
+  parser.add_argument(
+      "--arch",
+      action="store",
+      choices=["arm", "arm64", "x86", "x86_64", "host64", "host32"],
+      default="host64",
+      help="architecture to compile for. Defaults to host64")
+  parser.add_argument(
+      "--odex-file",
+      action="store",
+      help="odex file to write. File discarded if not set",
+      default=None)
+  parser.add_argument(
+      "--save-profile",
+      action="store",
+      type=argparse.FileType("w"),
+      default=None,
+      help="File path to store the profile to")
+  parser.add_argument(
+      "dex_files", help="dex/jar files", nargs="+", metavar="DEX")
+  return parser.parse_known_args()
+
+
+def get_bcp_runtime_args(image, arch):
+  if arch != "host32" and arch != "host64":
+    args = [
+        "art/tools/host_bcp.sh",
+        os.path.expandvars(
+            "${{OUT}}/system/framework/oat/{}/services.odex".format(arch)),
+        "--use-first-dir"
+    ]
+    print("Running: {}".format(run_print(args)))
+    print("=START=======================================")
+    res = subprocess.run(args, capture_output=True, text=True)
+    print("=END=========================================")
+    res.check_returncode()
+    return res.stdout.split()
+  else:
+    # Host we just use the bcp locations for both.
+    res = open(
+        os.path.expandvars(
+            "$ANDROID_HOST_OUT/apex/art_boot_images/javalib/{}/boot.oat".format(
+                "x86" if arch == "host32" else "x86_64")), "rb").read()
+    bcp_tag = b"bootclasspath\0"
+    bcp_start = res.find(bcp_tag) + len(bcp_tag)
+    bcp = res[bcp_start:bcp_start + res[bcp_start:].find(b"\0")]
+    str_bcp = bcp.decode()
+    return [
+        "--runtime-arg", "-Xbootclasspath:{}".format(str_bcp), "--runtime-arg",
+        "-Xbootclasspath-locations:{}".format(str_bcp)
+    ]
+
+
+def fdfile(fd):
+  return "/proc/{}/fd/{}".format(os.getpid(), fd)
+
+
+def get_profile_args(args, location_base):
+  """Handle all the profile file options."""
+  if args.profile_file is None and len(args.profile_line) == 0:
+    return []
+  if args.debug_profman:
+    profman_args = ["lldb-server", "g", ":5039", "--", args.profman]
+  else:
+    profman_args = [args.profman]
+  if args.save_profile:
+    prof_out_fd = args.save_profile.fileno()
+    os.set_inheritable(prof_out_fd, True)
+  else:
+    prof_out_fd = os.memfd_create("reference_prof", flags=0)
+  if args.debug_profman:
+    profman_args.append("--reference-profile-file={}".format(
+        fdfile(prof_out_fd)))
+  else:
+    profman_args.append("--reference-profile-file-fd={}".format(prof_out_fd))
+  if args.profile_file:
+    profman_args.append("--create-profile-from={}".format(args.profile_file))
+  else:
+    prof_in_fd = os.memfd_create("input_prof", flags=0)
+    # Why on earth does fdopen take control of the fd and not mention it in the docs.
+    with os.fdopen(os.dup(prof_in_fd), "w") as prof_in:
+      for l in args.profile_line:
+        print(l, file=prof_in)
+    profman_args.append("--create-profile-from={}".format(fdfile(prof_in_fd)))
+  for f in args.dex_files:
+    profman_args.append("--apk={}".format(f))
+    profman_args.append("--dex-location={}".format(
+        os.path.join(location_base, os.path.basename(f))))
+  print("Running: {}".format(run_print(profman_args)))
+  print("=START=======================================")
+  subprocess.run(profman_args, close_fds=False).check_returncode()
+  print("=END=========================================")
+  if args.debug:
+    return ["--profile-file={}".format(fdfile(prof_out_fd))]
+  else:
+    return ["--profile-file={}".format(fdfile(prof_out_fd))]
+
+
+def main():
+  args, extra = parse_args()
+  if args.arch == "host32" or args.arch == "host64":
+    location_base = os.path.expandvars("${ANDROID_HOST_OUT}/framework/")
+    real_arch = "x86" if args.arch == "host32" else "x86_64"
+    boot_image = os.path.expandvars(
+        "$ANDROID_HOST_OUT/apex/art_boot_images/javalib/boot.art")
+    android_root = os.path.expandvars("$ANDROID_HOST_OUT")
+    for f in args.dex_files:
+      extra.append("--dex-location={}".format(
+          os.path.join(location_base, os.path.basename(f))))
+      extra.append("--dex-file={}".format(f))
+  else:
+    location_base = "/system/framework"
+    real_arch = args.arch
+    boot_image = os.path.expandvars(":".join([
+        "${OUT}/apex/com.android.art.debug/javalib/boot.art",
+        "${OUT}/system/framework/boot-framework.art"
+    ]))
+    android_root = os.path.expandvars("$OUT/system")
+    for f in args.dex_files:
+      extra.append("--dex-location={}".format(
+          os.path.join(location_base, os.path.basename(f))))
+      extra.append("--dex-file={}".format(f))
+  extra += get_bcp_runtime_args(boot_image, args.arch)
+  extra += get_profile_args(args, location_base)
+  extra.append("--instruction-set={}".format(real_arch))
+  extra.append("--boot-image={}".format(boot_image))
+  extra.append("--android-root={}".format(android_root))
+  extra += ["--runtime-arg", "-Xms64m", "--runtime-arg", "-Xmx512m"]
+  if args.odex_file is not None:
+    extra.append("--oat-file={}".format(args.odex_file))
+  else:
+    if args.debug:
+      raise Exception("Debug requires a real output file. :(")
+    extra.append("--oat-fd={}".format(os.memfd_create("odex_fd", flags=0)))
+    extra.append("--oat-location={}".format("/tmp/odex_fd.odex"))
+    extra.append("--output-vdex-fd={}".format(
+        os.memfd_create("vdex_fd", flags=0)))
+  pre_args = []
+  if args.debug:
+    pre_args = ["lldb-server", "g", ":5039", "--"]
+  pre_args.append(args.dex2oat)
+  print("Running: {}".format(run_print(pre_args + extra)))
+  print("=START=======================================")
+  subprocess.run(pre_args + extra, close_fds=False).check_returncode()
+  print("=END=========================================")
+
+
+if __name__ == "__main__":
+  main()
diff --git a/tools/compile-jar.sh b/tools/compile-jar.sh
index aae2415be2..5a656672ed 100755
--- a/tools/compile-jar.sh
+++ b/tools/compile-jar.sh
@@ -17,6 +17,8 @@
 #
 # This script runs dex2oat on the host to compile a provided JAR or APK.
 #
+# This is a wrapper around the more powerful compile-jar.py script
+#
 
 if [[ "$#" -lt 1 ]]; then
   echo "Usage $0 <jar|apk> <output> <isa> [args]+"
@@ -24,6 +26,7 @@ if [[ "$#" -lt 1 ]]; then
   exit 1
 fi
 
+EXTRA_ARGS=
 FILE=$1
 shift
 OUTPUT=$1
@@ -31,12 +34,4 @@ shift
 ISA=$1
 shift
 
-# Use services.odex as a quick and dirty way to get correct BCP.
-dex2oatd \
-    --runtime-arg -Xms64m --runtime-arg -Xmx512m \
-    --boot-image=${OUT}/apex/com.android.art.debug/javalib/boot.art:${OUT}/system/framework/boot-framework.art \
-    $(${ANDROID_BUILD_TOP}/art/tools/host_bcp.sh ${OUT}/system/framework/oat/${ISA}/services.odex --use-first-dir) \
-    --dex-file=${FILE} --dex-location=/system/framework/${FILE} \
-    --oat-file=${OUTPUT} \
-    --android-root=${OUT}/system --instruction-set=$ISA \
-    $@
+$ANDROID_BUILD_TOP/art/tools/compile-jar.py --arch=$ISA --odex-file=$OUTPUT $FILE $@
diff --git a/tools/pylibdexfile.py b/tools/pylibdexfile.py
new file mode 100644
index 0000000000..30c0b268aa
--- /dev/null
+++ b/tools/pylibdexfile.py
@@ -0,0 +1,232 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script can get info out of dexfiles using libdexfile_external
+#
+
+from abc import ABC
+from ctypes import *
+import os.path
+import functools
+import zipfile
+
+libdexfile_external = CDLL(
+    os.path.expandvars("$ANDROID_HOST_OUT/lib64/libdexfile_external.so"))
+
+DexFileStr = c_void_p
+ExtDexFile = c_void_p
+
+
+class ExtMethodInfo(Structure):
+  """Output format for MethodInfo"""
+  _fields_ = [("offset", c_int32), ("len", c_int32), ("name", DexFileStr)]
+
+
+AllMethodsCallback = CFUNCTYPE(c_int, POINTER(ExtMethodInfo), c_void_p)
+libdexfile_external.ExtDexFileOpenFromFd.argtypes = [
+    c_int, c_size_t, c_char_p,
+    POINTER(DexFileStr),
+    POINTER(ExtDexFile)
+]
+libdexfile_external.ExtDexFileOpenFromFd.restype = c_int
+libdexfile_external.ExtDexFileOpenFromMemory.argtypes = [
+    c_void_p,
+    POINTER(c_size_t), c_char_p,
+    POINTER(DexFileStr),
+    POINTER(ExtDexFile)
+]
+libdexfile_external.ExtDexFileOpenFromMemory.restype = c_int
+libdexfile_external.ExtDexFileFree.argtypes = [ExtDexFile]
+libdexfile_external.ExtDexFileGetAllMethodInfos.argtypes = [
+    ExtDexFile, c_int, AllMethodsCallback, c_void_p
+]
+libdexfile_external.ExtDexFileGetString.argtypes = [
+    DexFileStr, POINTER(c_size_t)
+]
+libdexfile_external.ExtDexFileGetString.restype = c_char_p
+libdexfile_external.ExtDexFileFreeString.argtypes = [DexFileStr]
+
+
+class DexClass(object):
+  """Accessor for DexClass Data"""
+
+  def __init__(self, name):
+    self.name = name.strip()
+    self.arrays = name.count("[")
+    self.base_name = self.name if self.arrays == 0 else self.name[:-(
+        self.arrays * 2)]
+
+  def __repr__(self):
+    return self.name
+
+  @functools.cached_property
+  def descriptor(self):
+    """The name as a descriptor"""
+    if self.base_name == "int":
+      return "[" * self.arrays + "I"
+    elif self.base_name == "short":
+      return "[" * self.arrays + "S"
+    elif self.base_name == "long":
+      return "[" * self.arrays + "J"
+    elif self.base_name == "char":
+      return "[" * self.arrays + "C"
+    elif self.base_name == "boolean":
+      return "[" * self.arrays + "Z"
+    elif self.base_name == "byte":
+      return "[" * self.arrays + "B"
+    elif self.base_name == "float":
+      return "[" * self.arrays + "F"
+    elif self.base_name == "double":
+      return "[" * self.arrays + "D"
+    elif self.base_name == "void":
+      return "[" * self.arrays + "V"
+    else:
+      return "[" * self.arrays + "L{};".format("/".join(
+          self.base_name.split(".")))
+
+
+class Method(object):
+  """Method info wrapper"""
+
+  def __init__(self, mi):
+    self.offset = mi.offset
+    self.len = mi.len
+    self.name = libdexfile_external.ExtDexFileGetString(
+        mi.name, byref(c_size_t(0))).decode("utf-8")
+    libdexfile_external.ExtDexFileFreeString(mi.name)
+
+  def __repr__(self):
+    return "(" + self.name + ")"
+
+  @functools.cached_property
+  def descriptor(self):
+    """name as a descriptor"""
+    ret = DexClass(self.name.split(" ")[0])
+    non_ret = self.name[len(ret.name) + 1:]
+    arg_str = non_ret[non_ret.find("(") + 1:-1]
+    args = [] if arg_str == "" else map(
+        lambda a: DexClass(a.strip()).descriptor, arg_str.split(","))
+    class_and_meth = non_ret[0:non_ret.find("(")]
+    class_only = DexClass(class_and_meth[0:class_and_meth.rfind(".")])
+    meth = class_and_meth[class_and_meth.rfind(".") + 1:]
+    return "{cls}->{meth}({args}){ret}".format(
+        cls=class_only.descriptor,
+        meth=meth,
+        args="".join(args),
+        ret=ret.descriptor)
+
+  @functools.cached_property
+  def name_only(self):
+    """name without the return-type or arguments in java format"""
+    ret = DexClass(self.name.split(" ")[0])
+    non_ret = self.name[len(ret.name) + 1:]
+    class_and_meth = non_ret[0:non_ret.find("(")]
+    return class_and_meth
+
+  @functools.cached_property
+  def klass(self):
+    """declaring DexClass."""
+    ret = DexClass(self.name.split(" ")[0])
+    non_ret = self.name[len(ret.name) + 1:]
+    class_and_meth = non_ret[0:non_ret.find("(")]
+    return DexClass(class_and_meth[0:class_and_meth.rfind(".")])
+
+
+class BaseDexFile(ABC):
+  """DexFile base class"""
+
+  def __init__(self):
+    self.ext_dex_file_ = None
+    return
+
+  @functools.cached_property
+  def methods(self):
+    """Methods in the dex-file"""
+    meths = []
+
+    @AllMethodsCallback
+    def my_cb(info, _):
+      """Callback visitor for method infos"""
+      meths.append(Method(info[0]))
+      return 0
+
+    libdexfile_external.ExtDexFileGetAllMethodInfos(self.ext_dex_file_,
+                                                    c_int(1), my_cb, c_void_p())
+    return meths
+
+
+class FdDexFile(BaseDexFile):
+  """DexFile using an opened file-descriptor"""
+
+  def __init__(self, fd, loc):
+    super().__init__()
+    res_fle_ptr = pointer(c_void_p())
+    err_ptr = pointer(c_void_p())
+    res = libdexfile_external.ExtDexFileOpenFromFd(
+        c_int(fd), 0, create_string_buffer(bytes(loc, "utf-8")), err_ptr,
+        res_fle_ptr)
+    if res == 0:
+      err = libdexfile_external.ExtDexFileGetString(err_ptr.contents,
+                                                    byref(c_size_t()))
+      out = Exception("Failed to open file: {}. Error was: {}".format(loc, err))
+      libdexfile_external.ExtDexFileFreeString(err_ptr.contents)
+      raise out
+    self.ext_dex_file_ = res_fle_ptr.contents
+
+
+class FileDexFile(FdDexFile):
+  """DexFile using a file"""
+
+  def __init__(self, file, loc):
+    if type(file) == str:
+      self.file = open(file, "rb")
+    else:
+      self.file = file
+    super().__init__(self.file.fileno(), loc)
+
+
+class MemDexFile(BaseDexFile):
+  """DexFile using memory"""
+
+  def __init__(self, dat, loc):
+    assert type(dat) == bytes
+    super().__init__()
+    # Don't want GC to screw us over.
+    self.mem_ref = (c_byte * len(dat)).from_buffer_copy(dat)
+    res_fle_ptr = pointer(c_void_p())
+    err_ptr = pointer(c_void_p())
+    res = libdexfile_external.ExtDexFileOpenFromMemory(
+        self.mem_ref, byref(c_size_t(len(dat))),
+        create_string_buffer(bytes(loc, "utf-8")), err_ptr, res_fle_ptr)
+    if res == 0:
+      err = libdexfile_external.ExtDexFileGetString(err_ptr.contents,
+                                                    byref(c_size_t()))
+      out = Exception("Failed to open file: {}. Error was: {}".format(loc, err))
+      libdexfile_external.ExtDexFileFreeString(err_ptr.contents)
+      raise out
+    self.ext_dex_file_ = res_fle_ptr.contents
+
+
+def OpenJar(fle):
+  """Opens all classes[0-9]*.dex files in a zip archive"""
+  res = []
+  with zipfile.ZipFile(fle) as zf:
+    for f in zf.namelist():
+      if f.endswith(".dex") and f.startswith("classes"):
+        res.append(
+            MemDexFile(zf.read(f), "classes" if type(fle) != str else fle))
+  return res