test: Cache repeated soong invocations of get_build_var

This speeds up testing considerably since every call into soong was
taking 10-40s and this had to be done for every run-test to query
the javac libcore bootclasspath.

Now it should only call into soong dumpvars once total.

Also speeds up 674-hiddenapi testrunner time by 2x.

Bug: 74196452
Test: time art/test/testrunner/testrunner.py --host --jit --64 -t 674-hiddenapi
Test: art/test/testrunner/testrunner.py --host
Test: # manual check soong isnt taking all the CPU while running above.
Test: art/test/run-test --host 004-JniTest # or any test really.
Test: art/test/testrunner/run_build_test_target.py art-test
Change-Id: I32f9247db76cfd61993b8f6ea1f0fffa1322a2c5

6 files changed, 401 insertions, 57 deletions

diff --git a/test/testrunner/env.py b/test/testrunner/env.py
index 55569629ea..70efce51ee 100644
--- a/test/testrunner/env.py
+++ b/test/testrunner/env.py
@@ -17,6 +17,16 @@ import re
 import tempfile
 import subprocess
 
+# begin import $ANDROID_BUILD_TOP/art/tools/build/var_cache.py
+_THIS_DIR = os.path.dirname(os.path.realpath(__file__))
+_TOP = os.path.join(_THIS_DIR, "../../..")
+_VAR_CACHE_DIR = os.path.join(_TOP, "art/tools/build/")
+
+import sys
+sys.path.append(_VAR_CACHE_DIR)
+import var_cache
+# end import var_cache.py
+
 _env = dict(os.environ)
 
 def _getEnvBoolean(var, default):
@@ -28,55 +38,8 @@ def _getEnvBoolean(var, default):
       return False
   return default
 
-_DUMP_MANY_VARS_LIST = ['HOST_2ND_ARCH_PREFIX',
-                        'TARGET_2ND_ARCH',
-                        'TARGET_ARCH',
-                        'HOST_PREFER_32_BIT',
-                        'HOST_OUT_EXECUTABLES',
-                        'ANDROID_JAVA_TOOLCHAIN',
-                        'ANDROID_COMPILE_WITH_JACK',
-                        'USE_D8_BY_DEFAULT']
-_DUMP_MANY_VARS = None  # To be set to a dictionary with above list being the keys,
-                        # and the build variable being the value.
-def _dump_many_vars(var_name):
-  """
-  Reach into the Android build system to dump many build vars simultaneously.
-  Since the make system is so slow, we want to avoid calling into build frequently.
-  """
-  global _DUMP_MANY_VARS
-  global _DUMP_MANY_VARS_LIST
-
-  # Look up var from cache.
-  if _DUMP_MANY_VARS:
-    return _DUMP_MANY_VARS[var_name]
-
-  all_vars=" ".join(_DUMP_MANY_VARS_LIST)
-
-  # The command is taken from build/envsetup.sh to fetch build variables.
-  command = ("build/soong/soong_ui.bash --dumpvars-mode --vars=\"%s\"") % (all_vars)
-
-  config = subprocess.Popen(command,
-                            stdout=subprocess.PIPE,
-                            universal_newlines=True,
-                            shell=True,
-                            cwd=ANDROID_BUILD_TOP).communicate()[0] # read until EOF, select stdin
-  # Prints out something like:
-  # TARGET_ARCH='arm64'
-  # HOST_ARCH='x86_64'
-  _DUMP_MANY_VARS = {}
-  for line in config.split("\n"):
-    # Split out "$key='$value'" via regex.
-    match = re.search("([^=]+)='([^']*)", line)
-    if not match:
-      continue
-    key = match.group(1)
-    value = match.group(2)
-    _DUMP_MANY_VARS[key] = value
-
-  return _DUMP_MANY_VARS[var_name]
-
 def _get_build_var(var_name):
-  return _dump_many_vars(var_name)
+  return var_cache.get_build_var(var_name)
 
 def _get_build_var_boolean(var, default):
   val = _get_build_var(var)
diff --git a/test/testrunner/run_build_test_target.py b/test/testrunner/run_build_test_target.py
index fcc5505a95..e0ccc3e14c 100755
--- a/test/testrunner/run_build_test_target.py
+++ b/test/testrunner/run_build_test_target.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2017, The Android Open Source Project
 #
@@ -45,9 +45,9 @@ options = parser.parse_args()
 ##########
 
 if options.list:
-  print "List of all known build_target: "
-  for k in sorted(target_config.iterkeys()):
-    print " * " + k
+  print("List of all known build_target: ")
+  for k in sorted(target_config.keys()):
+    print(" * " + k)
   # TODO: would be nice if this was the same order as the target config file.
   sys.exit(1)
 
@@ -59,10 +59,10 @@ target = target_config[options.build_target]
 n_threads = options.n_threads
 custom_env = target.get('env', {})
 custom_env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
-print custom_env
+print(custom_env)
 os.environ.update(custom_env)
 
-if target.has_key('make'):
+if 'make' in target:
   build_command = 'make'
   build_command += ' DX='
   build_command += ' -j' + str(n_threads)
@@ -75,7 +75,7 @@ if target.has_key('make'):
   if subprocess.call(build_command.split()):
     sys.exit(1)
 
-if target.has_key('golem'):
+if 'golem' in target:
   machine_type = target.get('golem')
   # use art-opt-cc by default since it mimics the default preopt config.
   default_golem_config = 'art-opt-cc'
@@ -93,7 +93,7 @@ if target.has_key('golem'):
   if subprocess.call(cmd):
     sys.exit(1)
 
-if target.has_key('run-test'):
+if 'run-test' in target:
   run_test_command = [os.path.join(env.ANDROID_BUILD_TOP,
                                    'art/test/testrunner/testrunner.py')]
   test_flags = target.get('run-test', [])
diff --git a/tools/bootjars.sh b/tools/bootjars.sh
index f710de99bb..dca209d580 100755
--- a/tools/bootjars.sh
+++ b/tools/bootjars.sh
@@ -21,7 +21,7 @@
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 TOP="$DIR/../.."
 
-source "${TOP}/build/envsetup.sh" >&/dev/null # import get_build_var
+source "${TOP}/art/tools/build/var_cache.sh" >&/dev/null # import get_build_var
 
 selected_env_var=
 core_jars_only=n
diff --git a/tools/build/var_cache.py b/tools/build/var_cache.py
new file mode 100644
index 0000000000..9e616faafc
--- /dev/null
+++ b/tools/build/var_cache.py
@@ -0,0 +1,148 @@
+# Copyright 2018, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# !!! Keep up-to-date with var_cache.sh
+#
+
+#
+# Provide a soong-build variable query mechanism that is cached
+# in the current process and any other subchild process that knows
+# how to parse the exported variable:
+#
+# export ART_TOOLS_BUILD_VAR_CACHE="..."
+#
+# Of the format:
+#
+#   <key1>='<value1>'\n
+#   <key2>='<value2>'\n
+#   ...
+#   <keyN>='<valueN>'
+#
+# Note: This is intentionally the same output format as
+#     build/soong/soong_ui.bash --dumpvars-mode --vars "key1 key2 ... keyN"
+#
+# For example, this would be a valid var-cache:
+#
+# export ART_TOOLS_BUILD_VAR_CACHE="TARGET_CORE_JARS='core-oj core-libart'
+#   HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex'"
+#
+# Calling into soong repeatedly is very slow; whenever it needs to be done
+# more than once, the var_cache.py or var_cache.sh script should be used instead.
+#
+
+import os
+import subprocess
+import sys
+
+def get_build_var(name):
+  """
+  Query soong build for a variable value and return it as a string.
+
+  Var lookup is cached, subsequent var lookups in any child process
+  (that includes a var-cache is free). The var must be in 'var_list'
+  to participate in the cache.
+
+  Example:
+     host_out = var_cache.get_build_var('HOST_OUT')
+
+  Note that build vars can often have spaces in them,
+  so the caller must take care to ensure space-correctness.
+
+  Raises KeyError if the variable name is not in 'var_list'.
+  """
+  _populate()
+  _build_dict()
+
+  value = _var_cache_dict.get(name)
+  if value is None:
+    _debug(_var_cache_dict)
+    raise KeyError("The variable '%s' is not in 'var_list', can't lookup" %(name))
+
+  return value
+
+_var_cache_dict = None
+_THIS_DIR = os.path.dirname(os.path.realpath(__file__))
+_TOP = os.path.join(_THIS_DIR, "../../..")
+_VAR_LIST_PATH = os.path.join(_THIS_DIR, "var_list")
+_SOONG_UI_SCRIPT = os.path.join(_TOP, "build/soong/soong_ui.bash")
+_DEBUG = False
+
+def _populate():
+  if os.environ.get('ART_TOOLS_BUILD_VAR_CACHE'):
+    return
+
+  _debug("Varcache missing (PY)... repopulate")
+
+  interesting_vars=[]
+  with open(_VAR_LIST_PATH) as f:
+    for line in f.readlines():
+      line = line.strip()
+      if not line or line.startswith('#'):
+        continue
+
+      _debug(line)
+
+      interesting_vars.append(line)
+
+  _debug("Interesting vars: ", interesting_vars)
+
+  # Invoke soong exactly once for optimal performance.
+  var_values = subprocess.check_output([
+      _SOONG_UI_SCRIPT, '--dumpvars-mode', '-vars', " ".join(interesting_vars)],
+      cwd=_TOP)
+
+  # Export the ART_TOOLS_BUILD_VAR_CACHE in the same format as soong_ui.bash --dumpvars-mode.
+  os.environb[b'ART_TOOLS_BUILD_VAR_CACHE'] = var_values
+
+  _debug("Soong output: ", var_values)
+
+def _build_dict():
+  global _var_cache_dict
+
+  if _var_cache_dict:
+    return
+
+  _debug("_var_cache_build_dict()")
+
+  _var_cache_dict = {}
+
+  # Parse $ART_TOOLS_BUILD_VAR_CACHE, e.g.
+  #   TARGET_CORE_JARS='core-oj core-libart conscrypt okhttp bouncycastle apache-xml'
+  #   HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex ...'
+
+  for line in os.environ['ART_TOOLS_BUILD_VAR_CACHE'].splitlines():
+    _debug(line)
+    var_name, var_value = line.split("=")
+    var_value = var_value.strip("'")
+
+    _debug("Var name =", var_name)
+    _debug("Var value =", var_value)
+
+    _var_cache_dict[var_name] = var_value
+
+  _debug("Num entries in dict: ", len(_var_cache_dict))
+
+def _debug(*args):
+  if _DEBUG:
+    print(*args, file=sys.stderr)
+
+# Below definitions are for interactive use only, e.g.
+# python -c 'import var_cache; var_cache._dump_cache()'
+
+def _dump_cache():
+  _populate()
+  print(os.environ['ART_TOOLS_BUILD_VAR_CACHE'])
+
+#get_build_var("xyz")
diff --git a/tools/build/var_cache.sh b/tools/build/var_cache.sh
new file mode 100755
index 0000000000..26e9770f95
--- /dev/null
+++ b/tools/build/var_cache.sh
@@ -0,0 +1,195 @@
+#!/bin/bash
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# !!! Keep up-to-date with var_cache.py
+#
+
+#
+# Provide a soong-build variable query mechanism that is cached
+# in the current process and any other subchild process that knows
+# how to parse the exported variable:
+#
+# export ART_TOOLS_BUILD_VAR_CACHE="..."
+#
+# Of the format:
+#
+#   <key1>='<value1>'\n
+#   <key2>='<value2>'\n
+#   ...
+#   <keyN>='<valueN>'
+#
+# Note: This is intentionally the same output format as
+#     build/soong/soong_ui.bash --dumpvars-mode --vars "key1 key2 ... keyN"
+#
+# For example, this would be a valid var-cache:
+#
+# export ART_TOOLS_BUILD_VAR_CACHE="TARGET_CORE_JARS='core-oj core-libart'
+#   HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex'"
+#
+# Calling into soong repeatedly is very slow; whenever it needs to be done
+# more than once, the var_cache.py or var_cache.sh script should be used instead.
+#
+
+# -------------------------------------------------------
+
+# Echoes the result of get_build_var <var_name>.
+# Var lookup is cached, subsequent var lookups in any child process
+# (that includes a var-cache is free). The var must be in 'var_list'
+# to participate in the cache.
+#
+# Example:
+#    local host_out="$(get_build_var HOST_OUT)"
+#
+# Note that build vars can often have spaces in them,
+# so the caller must take care to ensure space-correctness.
+get_build_var() {
+  local var_name="$1"
+
+  _var_cache_populate
+  _var_cache_build_dict
+
+  if [[ ${_VAR_CACHE_DICT[$var_name]+exists} ]]; then
+    echo "${_VAR_CACHE_DICT[$var_name]}"
+    return 0
+  else
+    echo "[ERROR] get_build_var: The variable '$var_name' is not in 'var_list', can't lookup." >&2
+    return 1
+  fi
+}
+
+# The above functions are "public" and are intentionally not exported.
+# User scripts must have "source var_cache.sh" to take advantage of caching.
+
+# -------------------------------------------------------
+# Below functions are "private";
+# do not call them outside of this file.
+
+_var_cache_populate() {
+  if [[ -n $ART_TOOLS_BUILD_VAR_CACHE ]]; then
+    _var_cache_debug "ART_TOOLS_BUILD_VAR_CACHE preset to (quotes added)..."
+    _var_cache_debug \""$ART_TOOLS_BUILD_VAR_CACHE"\"
+    return 0
+  fi
+
+  _var_cache_debug "Varcache missing... repopulate"
+
+  local this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+  local top="$this_dir/../../.."
+
+  local interesting_vars=()
+  while read -r line; do
+    if [[ -z $line ]] || [[ $line == '#'* ]]; then
+      continue;
+    fi
+    interesting_vars+=($line)
+  done < "$this_dir"/var_list
+
+  _var_cache_debug "Interesting vars: " ${interesting_vars[@]}
+
+  local flat_vars="${interesting_vars[*]}"
+
+  local var_values
+  _var_cache_show_command "$top"/build/soong/soong_ui.bash --dumpvars-mode -vars=\"${interesting_vars[*]}\"
+
+  # Invoke soong exactly once for optimal performance.
+  # soong_ui.bash must be invoked from $ANDROID_BUILD_TOP or it gets confused and breaks.
+  var_values="$(cd "$top" && "$top"/build/soong/soong_ui.bash --dumpvars-mode -vars="$flat_vars")"
+
+  # Export the ART_TOOLS_BUILD_VAR_CACHE in the same format as soong_ui.bash --dumpvars-mode.
+  export ART_TOOLS_BUILD_VAR_CACHE="$var_values"
+
+  _var_cache_debug ART_TOOLS_BUILD_VAR_CACHE=\"$var_values\"
+}
+
+_var_cache_build_dict() {
+  if [[ ${#_VAR_CACHE_DICT[@]} -ne 0 ]]; then
+    # Associative arrays cannot be exported, have
+    # a separate step to reconstruct the associative
+    # array from a flat variable.
+    return 0
+  fi
+
+  # Parse $ART_TOOLS_BUILD_VAR_CACHE, e.g.
+  #   TARGET_CORE_JARS='core-oj core-libart conscrypt okhttp bouncycastle apache-xml'
+  #   HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex ...'
+
+  local var_name
+  local var_value
+  local strip_quotes
+
+  _var_cache_debug "_var_cache_build_dict()"
+
+  declare -g -A _VAR_CACHE_DICT  # global associative array.
+  while IFS='=' read -r var_name var_value; do
+    if [[ -z $var_name ]]; then
+      # skip empty lines, e.g. blank newline at the end
+      continue
+    fi
+    _var_cache_debug "Var_name was $var_name"
+    _var_cache_debug "Var_value was $var_value"
+    strip_quotes=${var_value//\'/}
+    _VAR_CACHE_DICT["$var_name"]="$strip_quotes"
+  done < <(echo "$ART_TOOLS_BUILD_VAR_CACHE")
+
+  _var_cache_debug ${#_VAR_CACHE_DICT[@]} -eq 0
+}
+
+_var_cache_debug() {
+  if ((_var_cache_debug_enabled)); then
+    echo "[DBG]: " "$@" >&2
+  fi
+}
+
+_var_cache_show_command() {
+  if (( _var_cache_show_commands || _var_cache_debug_enabled)); then
+    echo "$@" >&2
+  fi
+}
+
+while true; do
+  case $1 in
+    --help)
+      echo "Usage: $0 [--debug] [--show-commands] [--dump-cache] [--var <name>] [--var <name2>...]"
+      echo ""
+      echo "Exposes a function 'get_build_var' which returns the result of"
+      echo "a soong build variable."
+      echo ""
+      echo "Primarily intended to be used as 'source var_cache.sh',"
+      echo "but also allows interactive command line usage for simplifying development."
+      exit 0
+      ;;
+    --var)
+      echo -ne "$2="
+      get_build_var "$2"
+      shift
+      ;;
+    --debug)
+      _var_cache_debug_enabled=1
+      ;;
+    --show-commands)
+      _var_cache_show_commands=1
+      ;;
+    --dump-cache)
+      _var_cache_populate
+      echo "ART_TOOLS_BUILD_VAR_CACHE=\"$ART_TOOLS_BUILD_VAR_CACHE\""
+      ;;
+    *)
+      break
+      ;;
+  esac
+  shift
+done
diff --git a/tools/build/var_list b/tools/build/var_list
new file mode 100644
index 0000000000..3727741dac
--- /dev/null
+++ b/tools/build/var_list
@@ -0,0 +1,38 @@
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This file contains a list of all the build vars that need to be eagerly cached
+# by the var_cache.sh and var_cache.py scripts.
+# Lines starting with '#' or blank lines are ignored.
+#
+
+# javac-helper.sh
+TARGET_CORE_JARS
+PRODUCT_BOOT_JARS
+TARGET_OUT_COMMON_INTERMEDIATES
+HOST_CORE_JARS
+HOST_OUT_COMMON_INTERMEDIATES
+
+# testrunner/env.py
+HOST_2ND_ARCH_PREFIX
+TARGET_2ND_ARCH
+TARGET_ARCH
+HOST_PREFER_32_BIT
+HOST_OUT_EXECUTABLES
+ANDROID_JAVA_TOOLCHAIN
+ANDROID_COMPILE_WITH_JACK
+USE_D8_BY_DEFAULT
+