Merge "test: Add continuous testing for run-test --jvm"

5 files changed, 369 insertions, 33 deletions

diff --git a/test/knownfailures.json b/test/knownfailures.json
index b2e2867dad..cc3d200ed1 100644
--- a/test/knownfailures.json
+++ b/test/knownfailures.json
@@ -673,5 +673,304 @@
         "env_vars": {"SANITIZE_HOST": "address"},
         "bug": "b/73060923",
         "description": ["ASAN issue"]
+    },
+    {
+        "tests": ["1941-dispose-stress", "522-checker-regression-monitor-exit"],
+        "variant": "jvm",
+        "bug": "b/73888836",
+        "description": ["Hangs forever, times out."]
+    },
+    {
+        "tests": [
+          "004-JniTest",
+          "004-NativeAllocations",
+          "004-ReferenceMap",
+          "004-SignalTest",
+          "004-StackWalk",
+          "004-ThreadStress",
+          "004-UnsafeTest",
+          "005-annotations",
+          "008-exceptions",
+          "020-string",
+          "021-string2",
+          "030-bad-finalizer",
+          "031-class-attributes",
+          "034-call-null",
+          "038-inner-null",
+          "044-proxy",
+          "046-reflect",
+          "064-field-access",
+          "068-classloader",
+          "070-nio-buffer",
+          "071-dexfile",
+          "071-dexfile-get-static-size",
+          "071-dexfile-map-clean",
+          "082-inline-execute",
+          "086-null-super",
+          "087-gc-after-link",
+          "088-monitor-verification",
+          "091-override-package-private-method",
+          "097-duplicate-method",
+          "098-ddmc",
+          "099-vmdebug",
+          "100-reflect2",
+          "104-growth-limit",
+          "111-unresolvable-exception",
+          "115-native-bridge",
+          "116-nodex2oat",
+          "117-nopatchoat",
+          "118-noimage-dex2oat",
+          "119-noimage-patchoat",
+          "127-checker-secondarydex",
+          "129-ThreadGetId",
+          "130-hprof",
+          "1337-gc-coverage",
+          "1338-gc-no-los",
+          "134-nodex2oat-nofallback",
+          "134-reg-promotion",
+          "135-MirandaDispatch",
+          "136-daemon-jni-shutdown",
+          "137-cfi",
+          "138-duplicate-classes-check",
+          "138-duplicate-classes-check2",
+          "140-field-packing",
+          "141-class-unload",
+          "142-classloader2",
+          "143-string-value",
+          "144-static-field-sigquit",
+          "145-alloc-tracking-stress",
+          "146-bad-interface",
+          "147-stripped-dex-fallback",
+          "148-multithread-gc-annotations",
+          "150-loadlibrary",
+          "154-gc-loop",
+          "156-register-dex-file-multi-loader",
+          "157-void-class",
+          "158-app-image-class-table",
+          "159-app-image-fields",
+          "161-final-abstract-class",
+          "162-method-resolution",
+          "163-app-image-methods",
+          "164-resolution-trampoline-dex-cache",
+          "167-visit-locks",
+          "168-vmstack-annotated",
+          "201-built-in-except-detail-messages",
+          "203-multi-checkpoint",
+          "304-method-tracing",
+          "305-other-fault-handler",
+          "412-new-array",
+          "416-optimizing-arith-not",
+          "425-invoke-super",
+          "431-type-propagation",
+          "432-optimizing-cmp",
+          "434-invoke-direct",
+          "435-new-instance",
+          "441-checker-inliner",
+          "442-checker-constant-folding",
+          "448-multiple-returns",
+          "449-checker-bce",
+          "452-multiple-returns2",
+          "453-not-byte",
+          "454-get-vreg",
+          "455-checker-gvn",
+          "457-regs",
+          "458-checker-instruct-simplification",
+          "459-dead-phi",
+          "460-multiple-returns3",
+          "461-get-reference-vreg",
+          "466-get-live-vreg",
+          "467-regalloc-pair",
+          "468-checker-bool-simplif-regression",
+          "471-uninitialized-locals",
+          "472-unreachable-if-regression",
+          "475-regression-inliner-ids",
+          "480-checker-dead-blocks",
+          "496-checker-inlining-class-loader",
+          "498-type-propagation",
+          "501-null-constant-dce",
+          "501-regression-packed-switch",
+          "503-dead-instructions",
+          "504-regression-baseline-entry",
+          "506-verify-aput",
+          "509-pre-header",
+          "510-checker-try-catch",
+          "511-clinit-interface",
+          "515-dce-dominator",
+          "516-dead-move-result",
+          "517-checker-builder-fallthrough",
+          "518-null-array-get",
+          "520-equivalent-phi",
+          "529-checker-unresolved",
+          "530-checker-lse",
+          "530-checker-lse-ctor-fences",
+          "530-checker-lse3",
+          "530-checker-regression-reftyp-final",
+          "536-checker-intrinsic-optimization",
+          "536-checker-needs-access-check",
+          "541-regression-inlined-deopt",
+          "543-env-long-ref",
+          "545-tracing-and-jit",
+          "550-checker-regression-wide-store",
+          "551-invoke-super",
+          "552-checker-primitive-typeprop",
+          "552-invoke-non-existent-super",
+          "553-invoke-super",
+          "556-invoke-super",
+          "559-checker-irreducible-loop",
+          "563-checker-fakestring",
+          "564-checker-irreducible-loop",
+          "565-checker-doublenegbitwise",
+          "565-checker-irreducible-loop",
+          "566-polymorphic-inlining",
+          "567-checker-compare",
+          "569-checker-pattern-replacement",
+          "570-checker-osr",
+          "571-irreducible-loop",
+          "574-irreducible-and-constant-area",
+          "575-checker-string-init-alias",
+          "580-checker-string-fact-intrinsics",
+          "585-inline-unresolved",
+          "586-checker-null-array-get",
+          "587-inline-class-error",
+          "588-checker-irreducib-lifetime-hole",
+          "591-new-instance-string",
+          "592-checker-regression-bool-input",
+          "593-checker-boolean-2-integral-conv",
+          "593-checker-shift-and-simplifier",
+          "594-invoke-super",
+          "595-error-class",
+          "595-profile-saving",
+          "596-app-images",
+          "596-checker-dead-phi",
+          "596-monitor-inflation",
+          "597-deopt-busy-loop",
+          "597-deopt-invoke-stub",
+          "597-deopt-new-string",
+          "599-checker-irreducible-loop",
+          "600-verifier-fails",
+          "601-method-access",
+          "602-deoptimizeable",
+          "605-new-string-from-bytes",
+          "608-checker-unresolved-lse",
+          "612-jit-dex-cache",
+          "613-inlining-dex-cache",
+          "616-cha",
+          "616-cha-abstract",
+          "616-cha-interface",
+          "616-cha-interface-default",
+          "616-cha-miranda",
+          "622-simplifyifs-exception-edges",
+          "624-checker-stringops",
+          "626-const-class-linking",
+          "628-vdex",
+          "629-vdex-speed",
+          "630-safecast-array",
+          "631-checker-fp-abs",
+          "633-checker-rtp-getclass",
+          "634-vdex-duplicate",
+          "636-wrong-static-access",
+          "638-checker-inline-cache-intrinsic",
+          "638-checker-inline-caches",
+          "638-no-line-number",
+          "641-irreducible-inline",
+          "643-checker-bogus-ic",
+          "645-checker-abs-simd",
+          "647-jni-get-field-id",
+          "647-sinking-catch",
+          "648-inline-caches-unresolved",
+          "649-vdex-duplicate-method",
+          "652-deopt-intrinsic",
+          "656-annotation-lookup-generic-jni",
+          "656-checker-simd-opt",
+          "659-unpadded-array",
+          "660-clinit",
+          "660-store-8-16",
+          "661-classloader-allocator",
+          "661-oat-writer-layout",
+          "663-checker-select-generator",
+          "663-odd-dex-size",
+          "663-odd-dex-size2",
+          "663-odd-dex-size3",
+          "663-odd-dex-size4",
+          "667-jit-jni-stub",
+          "667-out-of-bounds",
+          "668-aiobe",
+          "674-hiddenapi",
+          "674-hotness-compiled",
+          "674-vdex-uncompress",
+          "675-checker-unverified-method",
+          "676-proxy-jit-at-first-use",
+          "676-resolve-field-type",
+          "706-checker-scheduler",
+          "707-checker-invalid-profile",
+          "714-invoke-custom-lambda-metafactory",
+          "800-smali",
+          "801-VoidCheckCast",
+          "802-deoptimization",
+          "804-class-extends-itself",
+          "900-hello-plugin",
+          "901-hello-ti-agent",
+          "903-hello-tagging",
+          "904-object-allocation",
+          "906-iterate-heap",
+          "909-attach-agent",
+          "910-methods",
+          "911-get-stack-trace",
+          "912-classes",
+          "913-heaps",
+          "918-fields",
+          "920-objects",
+          "922-properties",
+          "924-threads",
+          "925-threadgroups",
+          "927-timers",
+          "929-search",
+          "931-agent-thread",
+          "933-misc-events",
+          "936-search-onload",
+          "939-hello-transformation-bcp",
+          "944-transform-classloaders",
+          "946-obsolete-throw",
+          "948-change-annotations",
+          "950-redefine-intrinsic",
+          "952-invoke-custom",
+          "954-invoke-polymorphic-verifier",
+          "955-methodhandles-smali",
+          "956-methodhandles",
+          "957-methodhandle-transforms",
+          "958-methodhandle-stackframe",
+          "972-default-imt-collision",
+          "972-iface-super-multidex",
+          "973-default-multidex",
+          "974-verify-interface-super",
+          "975-iface-private",
+          "976-conflict-no-methods",
+          "978-virtual-interface",
+          "980-redefine-object",
+          "981-dedup-original-dex",
+          "983-source-transform-verify",
+          "986-native-method-bind",
+          "988-method-trace",
+          "989-method-trace-throw",
+          "993-breakpoints",
+          "1900-track-alloc",
+          "1906-suspend-list-me-first",
+          "1914-get-local-instance",
+          "1926-missed-frame-pop",
+          "1930-monitor-info",
+          "1932-monitor-events-misc",
+          "1935-get-set-current-frame-jit",
+          "1938-transform-abstract-single-impl",
+          "1939-proxy-frames",
+          "1940-ddms-ext",
+          "1945-proxy-method-arguments",
+          "1946-list-descriptors",
+          "1947-breakpoint-redefine-deopt"
+        ],
+        "variant": "jvm",
+        "bug": "b/73888836",
+        "description": ["Failing on JVM. Needs further investigating."]
     }
+
+
 ]
diff --git a/test/testrunner/device_config.py b/test/testrunner/device_config.py
index c7ed6f7fbc..1fad7d297d 100644
--- a/test/testrunner/device_config.py
+++ b/test/testrunner/device_config.py
@@ -1,7 +1,7 @@
 device_config = {
 # Configuration syntax:
 #
-#  device: The value of ro.product.name or 'host'
+#  device: The value of ro.product.name or 'host' or 'jvm'
 #  properties: (Use one or more of these).
 #     * run-test-args: additional run-test-args
 #
diff --git a/test/testrunner/run_build_test_target.py b/test/testrunner/run_build_test_target.py
index 531508e6b3..fcc5505a95 100755
--- a/test/testrunner/run_build_test_target.py
+++ b/test/testrunner/run_build_test_target.py
@@ -96,15 +96,20 @@ if target.has_key('golem'):
 if target.has_key('run-test'):
   run_test_command = [os.path.join(env.ANDROID_BUILD_TOP,
                                    'art/test/testrunner/testrunner.py')]
-  run_test_command += target.get('run-test', [])
+  test_flags = target.get('run-test', [])
+  run_test_command += test_flags
   # Let testrunner compute concurrency based on #cpus.
   # b/65822340
   # run_test_command += ['-j', str(n_threads)]
+
+  # In the config assume everything will run with --host and on ART.
+  # However for only [--jvm] this is undesirable, so don't pass in ART-specific flags.
+  if ['--jvm'] != test_flags:
+    run_test_command += ['--host']
+    run_test_command += ['--dex2oat-jobs']
+    run_test_command += ['4']
   run_test_command += ['-b']
-  run_test_command += ['--host']
   run_test_command += ['--verbose']
-  run_test_command += ['--dex2oat-jobs']
-  run_test_command += ['4']
 
   sys.stdout.write(str(run_test_command) + '\n')
   sys.stdout.flush()
diff --git a/test/testrunner/target_config.py b/test/testrunner/target_config.py
index 3064c76ffd..9d0377510a 100644
--- a/test/testrunner/target_config.py
+++ b/test/testrunner/target_config.py
@@ -31,12 +31,7 @@ target_config = {
     },
 
     'art-test-javac' : {
-        'make' : 'test-art-host-gtest',
-        'run-test' : [],
-        'env' : {
-            'ANDROID_COMPILE_WITH_JACK' : 'false',
-            'ART_USE_READ_BARRIER' : 'true'
-        }
+        'run-test' : ['--jvm']
     },
 
     # ART run-test configurations
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
index 4329ad4863..3974ccb4f9 100755
--- a/test/testrunner/testrunner.py
+++ b/test/testrunner/testrunner.py
@@ -136,7 +136,7 @@ def gather_test_info():
   # TODO: Avoid duplication of the variant names in different lists.
   VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'}
   VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'}
-  VARIANT_TYPE_DICT['target'] = {'target', 'host'}
+  VARIANT_TYPE_DICT['target'] = {'target', 'host', 'jvm'}
   VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'}
   VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'multipicimage'}
   VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'}
@@ -193,7 +193,7 @@ def setup_test_env():
     _user_input_variants['jvmti'].add('no-jvmti')
 
   # By default we run all 'compiler' variants.
-  if not _user_input_variants['compiler']:
+  if not _user_input_variants['compiler'] and _user_input_variants['target'] != 'jvm':
     _user_input_variants['compiler'].add('optimizing')
     _user_input_variants['compiler'].add('jit')
     _user_input_variants['compiler'].add('interpreter')
@@ -215,7 +215,6 @@ def setup_test_env():
   if not _user_input_variants['image']: # Default
     _user_input_variants['image'].add('picimage')
 
-
   if not _user_input_variants['pictest']: # Default
     _user_input_variants['pictest'].add('npictest')
 
@@ -269,11 +268,10 @@ def find_extra_device_arguments(target):
   """
   Gets any extra arguments from the device_config.
   """
-  if target == 'host':
-    return device_config.get(target, { 'run-test-args' : [] })['run-test-args']
-  else:
-    device = get_device_name()
-    return device_config.get(device, { 'run-test-args' : [] })['run-test-args']
+  device_name = target
+  if target == 'target':
+    device_name = get_device_name()
+  return device_config.get(device_name, { 'run-test-args' : [] })['run-test-args']
 
 def get_device_name():
   """
@@ -308,15 +306,27 @@ def run_tests(tests):
     tests: The set of tests to be run.
   """
   options_all = ''
+
+  # jvm does not run with all these combinations,
+  # or at least it doesn't make sense for most of them.
+  # TODO: support some jvm variants like jvmti ?
+  target_input_variants = _user_input_variants['target']
+  uncombinated_target_input_variants = []
+  if 'jvm' in target_input_variants:
+    _user_input_variants['target'].remove('jvm')
+    uncombinated_target_input_variants.append('jvm')
+
   global total_test_count
   total_test_count = len(tests)
-  for variant_type in VARIANT_TYPE_DICT:
-    if not (variant_type == 'target' or 'address_sizes' in variant_type):
-      total_test_count *= len(_user_input_variants[variant_type])
+  if target_input_variants:
+    for variant_type in VARIANT_TYPE_DICT:
+      if not (variant_type == 'target' or 'address_sizes' in variant_type):
+        total_test_count *= len(_user_input_variants[variant_type])
   target_address_combinations = 0
-  for target in _user_input_variants['target']:
+  for target in target_input_variants:
     for address_size in _user_input_variants['address_sizes_target'][target]:
       target_address_combinations += 1
+  target_address_combinations += len(uncombinated_target_input_variants)
   total_test_count *= target_address_combinations
 
   if env.ART_TEST_WITH_STRACE:
@@ -339,17 +349,32 @@ def run_tests(tests):
   if dex2oat_jobs != -1:
     options_all += ' --dex2oat-jobs ' + str(dex2oat_jobs)
 
-  config = itertools.product(tests, _user_input_variants['target'], _user_input_variants['run'],
-                             _user_input_variants['prebuild'], _user_input_variants['compiler'],
-                             _user_input_variants['relocate'], _user_input_variants['trace'],
-                             _user_input_variants['gc'], _user_input_variants['jni'],
-                             _user_input_variants['image'], _user_input_variants['pictest'],
-                             _user_input_variants['debuggable'], _user_input_variants['jvmti'],
-                             _user_input_variants['cdex_level'])
+  def iter_config(tests, input_variants, user_input_variants):
+    config = itertools.product(tests, input_variants, user_input_variants['run'],
+                                 user_input_variants['prebuild'], user_input_variants['compiler'],
+                                 user_input_variants['relocate'], user_input_variants['trace'],
+                                 user_input_variants['gc'], user_input_variants['jni'],
+                                 user_input_variants['image'], user_input_variants['pictest'],
+                                 user_input_variants['debuggable'], user_input_variants['jvmti'],
+                                 user_input_variants['cdex_level'])
+    return config
+
+  # [--host, --target] combines with all the other user input variants.
+  config = iter_config(tests, target_input_variants, _user_input_variants)
+  # [--jvm] currently combines with nothing else. most of the extra flags we'd insert
+  # would be unrecognizable by the 'java' binary, so avoid inserting any extra flags for now.
+  uncombinated_config = iter_config(tests, uncombinated_target_input_variants, { 'run': [''],
+      'prebuild': [''], 'compiler': [''],
+      'relocate': [''], 'trace': [''],
+      'gc': [''], 'jni': [''],
+      'image': [''], 'pictest': [''],
+      'debuggable': [''], 'jvmti': [''],
+      'cdex_level': ['']})
+
+  def start_combination(config_tuple, address_size):
+      test, target, run, prebuild, compiler, relocate, trace, gc, \
+      jni, image, pictest, debuggable, jvmti, cdex_level = config_tuple
 
-  for test, target, run, prebuild, compiler, relocate, trace, gc, \
-      jni, image, pictest, debuggable, jvmti, cdex_level in config:
-    for address_size in _user_input_variants['address_sizes_target'][target]:
       if stop_testrunner:
         # When ART_TEST_KEEP_GOING is set to false, then as soon as a test
         # fails, stop_testrunner is set to True. When this happens, the method
@@ -384,6 +409,8 @@ def run_tests(tests):
 
       if target == 'host':
         options_test += ' --host'
+      elif target == 'jvm':
+        options_test += ' --jvm'
 
       if run == 'ndebug':
         options_test += ' -O'
@@ -488,6 +515,14 @@ def run_tests(tests):
       worker.daemon = True
       worker.start()
 
+  for config_tuple in config:
+    target = config_tuple[1]
+    for address_size in _user_input_variants['address_sizes_target'][target]:
+      start_combination(config_tuple, address_size)
+
+  for config_tuple in uncombinated_config:
+      start_combination(config_tuple, "")  # no address size
+
   while threading.active_count() > 2:
     time.sleep(0.1)
 
@@ -969,6 +1004,8 @@ def main():
       build_targets += 'test-art-host-run-test-dependencies'
     if 'target' in _user_input_variants['target']:
       build_targets += 'test-art-target-run-test-dependencies'
+    if 'jvm' in _user_input_variants['target']:
+      build_targets += 'test-art-host-run-test-dependencies'
     build_command = 'make'
     build_command += ' DX='
     build_command += ' -j'