From 14de5c46174ce3674a93e75523516c5ce6c20905 Mon Sep 17 00:00:00 2001
From: Shubham Ajmera <shubhamajmera@google.com>
Date: Mon, 13 Mar 2017 10:51:14 -0700
Subject: Add verification of knownfailures in the testrunner

Add verification checks in testrunner for
knownfailures entries.
Removed 554-jit-profile-file, 555-checker-regression-x86const entries,
as it doesn't exist anymore.

Also, unified the format of entries to use tests for all the
cases.

Bug: 36164293
Test: ./art/test/testrunner/testrunner.py --host -64
(cherry-picked from commit 2a0e16d09c6888b646402582d909467b11e445c0)

Change-Id: I8dbe18dec952150ede96a248e9639824fc004430
---
 test/testrunner/testrunner.py | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

(limited to 'test/testrunner/testrunner.py')

diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
index be84f898b2..a5504098d8 100755
--- a/test/testrunner/testrunner.py
+++ b/test/testrunner/testrunner.py
@@ -547,6 +547,22 @@ def print_test_info(test_name, result, failed_test_info=""):
   finally:
     print_mutex.release()
 
+def verify_knownfailure_entry(entry):
+  supported_field = {
+      'tests' : (list, unicode),
+      'description' : (list, unicode),
+      'bug' : (unicode,),
+      'variant' : (unicode,),
+      'env_vars' : (dict,),
+  }
+  for field in entry:
+    field_type = type(entry[field])
+    if field_type not in supported_field[field]:
+      raise ValueError('%s is not supported type for %s\n%s' % (
+          str(field_type),
+          field,
+          str(entry)))
+
 def get_disabled_test_info():
   """Generate set of known failures.
 
@@ -563,15 +579,18 @@ def get_disabled_test_info():
 
   disabled_test_info = {}
   for failure in known_failures_info:
-    tests = failure.get('test')
-    if tests:
+    verify_knownfailure_entry(failure)
+    tests = failure.get('tests', [])
+    if isinstance(tests, unicode):
       tests = [tests]
-    else:
-      tests = failure.get('tests', [])
     variants = parse_variants(failure.get('variant'))
     env_vars = failure.get('env_vars')
+
     if check_env_vars(env_vars):
       for test in tests:
+        if test not in RUN_TEST_SET:
+          raise ValueError('%s is not a valid run-test' % (
+              test))
         if test in disabled_test_info:
           disabled_test_info[test] = disabled_test_info[test].union(variants)
         else:
@@ -635,6 +654,9 @@ def parse_variants(variants):
     variant = set()
     for and_variant in and_variants:
       and_variant = and_variant.strip()
+      if and_variant not in TOTAL_VARIANTS_SET:
+        raise ValueError('%s is not a valid variant' % (
+            and_variant))
       variant.add(and_variant)
     variant_list.add(frozenset(variant))
   return variant_list
-- 
cgit v1.2.3-59-g8ed1b