Adds to DumpRenderTree the ability to ignore the results of tests

Change-Id: I7c16d9713fc35c773b810f9d5ce6700f8d9a28e4

5 files changed, 96 insertions, 44 deletions

diff --git a/tests/DumpRenderTree/assets/run_layout_tests.py b/tests/DumpRenderTree/assets/run_layout_tests.py
index d3726c15f069..b6e7bf3395dd 100755
--- a/tests/DumpRenderTree/assets/run_layout_tests.py
+++ b/tests/DumpRenderTree/assets/run_layout_tests.py
@@ -246,6 +246,7 @@ def main(options, args):
 
   result_files = ["/sdcard/layout_tests_passed.txt",
                   "/sdcard/layout_tests_failed.txt",
+                  "/sdcard/layout_tests_ignored.txt",
                   "/sdcard/layout_tests_nontext.txt"]
   for file in result_files:
     shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
@@ -263,10 +264,13 @@ def main(options, args):
   logging.info(str(passed_tests) + " passed")
   failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
   logging.info(str(failed_tests) + " failed")
+  ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
+  logging.info(str(ignored_tests) + " ignored results")
   crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
   logging.info(str(crashed_tests) + " crashed")
   nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
   logging.info(str(nontext_tests) + " no dumpAsText")
+  logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")
 
   logging.info("Results are stored under: " + results_dir + "\n")
 
diff --git a/tests/DumpRenderTree/src/com/android/dumprendertree/FileFilter.java b/tests/DumpRenderTree/src/com/android/dumprendertree/FileFilter.java
index 77adac6bb23d..eecd00aa6eea 100644
--- a/tests/DumpRenderTree/src/com/android/dumprendertree/FileFilter.java
+++ b/tests/DumpRenderTree/src/com/android/dumprendertree/FileFilter.java
@@ -23,27 +23,33 @@ public class FileFilter {
 
     private static final String LOGTAG = "FileFilter";
 
+    // Returns whether we should ignore this test and skip running it.
+    // Currently we use this only for tests that crash the browser.
+    // TODO: Once these crashes are fixed, we should probably eliminate this
+    // method, as no test should crash.
     public static boolean ignoreTest(String file) {
-        // treat files like directories for the time being.
-        for (int i = 0; i < ignoreTestList.length; i ++) {
+        for (int i = 0; i < ignoreTestList.length; i++) {
             if (file.endsWith(ignoreTestList[i])) {
                 Log.v(LOGTAG, "File path in list of ignored tests: " + file);
                 return true;
             }
         }
-        for (int i = 0; i < ignoreTestDirs.length; i++) {
-            if (file.endsWith(ignoreTestDirs[i])) {
-                Log.v(LOGTAG, "File path in list of ignored directories: " + file);
+        return false;
+    }
+
+    // Returns whether a directory does not contain layout tests and so can be
+    // ignored.
+    public static boolean isNonTestDir(String file) {
+        for (int i = 0; i < nonTestDirs.length; i++) {
+            if (file.endsWith(nonTestDirs[i])) {
                 return true;
             }
         }
-        // We should run tests for which the expected result is wrong, as there is
-        // value in checking that they don't cause crashes.
-        // TODO: Run these tests but ignore the result.
-        return ignoreResults(file);
+        return false;
     }
 
-    public static boolean ignoreResults(String file) {
+    // Returns whether we should ignore the result of this test.
+    public static boolean ignoreResult(String file) {
         for (int i = 0; i < ignoreResultList.size(); i++) {
             if (file.endsWith(ignoreResultList.get(i))) {
                 Log.v(LOGTAG, "File path in list of ignored results: " + file);
@@ -51,7 +57,7 @@ public class FileFilter {
             }
         }
         return false;
-      }
+    }
 
     final static Vector<String> ignoreResultList = new Vector<String>();
 
@@ -59,21 +65,16 @@ public class FileFilter {
         fillIgnoreResultList();
     }
 
-    static final String[] ignoreTestDirs = {
+    static final String[] nonTestDirs = {
         ".", // ignore hidden directories and files
         "resources", // ignore resource directories
         ".svn", // don't run anything under .svn folder
-        "profiler",  // profiler is not supported
-        "svg",  // svg is not supported
-        "platform",  // platform specific
+        "platform"  // No-Android specific tests
     };
 
-    static final String [] ignoreTestList = {
-        "editing/selection/move-left-right.html",
+    static final String[] ignoreTestList = {
         "fast/js/regexp-charclass-crash.html", // RegExp is too large, causing OOM
-        "storage/domstorage/localstorage/private-browsing-affects-storage.html", // No notion of private browsing.
-        "storage/domstorage/sessionstorage/private-browsing-affects-storage.html", // No notion of private browsing.
-        "storage/private-browsing-readonly.html", // No notion of private browsing.
+        "editing/selection/move-left-right.html" // Causes DumpRenderTree to hang
     };
 
     static void fillIgnoreResultList() {
@@ -113,6 +114,7 @@ public class FileFilter {
         ignoreResultList.add("fast/workers/shared-worker-shared.html"); // shared workers not supported
         ignoreResultList.add("fast/workers/shared-worker-simple.html"); // shared workers not supported
 
+        // TODO: These need to be triaged
         ignoreResultList.add("fast/css/case-transform.html"); // will not fix #619707
         ignoreResultList.add("fast/dom/Element/offsetLeft-offsetTop-body-quirk.html"); // different screen size result in extra spaces in Apple compared to us
         ignoreResultList.add("fast/dom/Window/Plug-ins.html"); // need test plugin
@@ -183,6 +185,12 @@ public class FileFilter {
         ignoreResultList.add("fast/parser/script-tag-with-trailing-slash.html"); // not capturing the console messages
         ignoreResultList.add("fast/replaced/image-map.html"); // requires eventSender.mouseDown(),mouseUp()
         ignoreResultList.add("fast/text/plain-text-line-breaks.html"); // extra spacing because iFrames rendered next to each other on Apple
+        ignoreResultList.add("profiler"); // profiler is not supported
+        ignoreResultList.add("storage/domstorage/localstorage/private-browsing-affects-storage.html"); // No notion of private browsing.
+        ignoreResultList.add("storage/domstorage/sessionstorage/private-browsing-affects-storage.html"); // No notion of private browsing.
+        ignoreResultList.add("storage/private-browsing-readonly.html"); // No notion of private browsing.
+        ignoreResultList.add("svg"); // svg is not supported
+
     }
 
 }
diff --git a/tests/DumpRenderTree/src/com/android/dumprendertree/FsUtils.java b/tests/DumpRenderTree/src/com/android/dumprendertree/FsUtils.java
index ef0c6c6f90ff..322b0d2b349d 100644
--- a/tests/DumpRenderTree/src/com/android/dumprendertree/FsUtils.java
+++ b/tests/DumpRenderTree/src/com/android/dumprendertree/FsUtils.java
@@ -43,38 +43,45 @@ public class FsUtils {
     }
 
     public static void findLayoutTestsRecursively(BufferedOutputStream bos,
-            String dir) throws IOException {
+            String dir, boolean ignoreResultsInDir) throws IOException {
         Log.v(LOGTAG, "Searching tests under " + dir);
 
         File d = new File(dir);
         if (!d.isDirectory()) {
             throw new AssertionError("A directory expected, but got " + dir);
         }
+        ignoreResultsInDir |= FileFilter.ignoreResult(dir);
 
         String[] files = d.list();
         for (int i = 0; i < files.length; i++) {
             String s = dir + "/" + files[i];
-            if (s.endsWith("TEMPLATE.html")) {
+
+            File f = new File(s);
+            if (f.isDirectory()) {
+                // If this is not a test directory, we don't recurse into it.
+                if (!FileFilter.isNonTestDir(s)) {
+                    Log.v(LOGTAG, "Recursing on " + s);
+                    findLayoutTestsRecursively(bos, s, ignoreResultsInDir);
+                }
                 continue;
             }
+
+            // If this test should be ignored, we skip it completely.
             if (FileFilter.ignoreTest(s)) {
-                Log.v(LOGTAG, "  Ignoring: " + s);
+                Log.v(LOGTAG, "Ignoring: " + s);
                 continue;
             }
-            if (s.toLowerCase().endsWith(".html")
-                    || s.toLowerCase().endsWith(".xml")) {
+
+            if ((s.toLowerCase().endsWith(".html") || s.toLowerCase().endsWith(".xml"))
+                    && !s.endsWith("TEMPLATE.html")) {
+                Log.v(LOGTAG, "Recording " + s);
                 bos.write(s.getBytes());
+                // If the result of this test should be ignored, we still run the test.
+                if (ignoreResultsInDir || FileFilter.ignoreResult(s)) {
+                    bos.write((" IGNORE_RESULT").getBytes());
+                }
                 bos.write('\n');
-                continue;
             }
-
-            File f = new File(s);
-            if (f.isDirectory()) {
-                findLayoutTestsRecursively(bos, s);
-                continue;
-            }
-
-            Log.v(LOGTAG, "Skipping " + s);
         }
     }
 
diff --git a/tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java b/tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java
index ac6933fcfb6e..042158ada564 100644
--- a/tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java
+++ b/tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java
@@ -38,14 +38,16 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Vector;
 
-// TestRecorder creates three files ...
+// TestRecorder creates four files ...
 // - passing tests
 // - failing tests
+// - tests for which results are ignored
 // - tests with no text results available
 // TestRecorder does not have the ability to clear the results.
 class MyTestRecorder {
     private BufferedOutputStream mBufferedOutputPassedStream;
     private BufferedOutputStream mBufferedOutputFailedStream;
+    private BufferedOutputStream mBufferedOutputIgnoreResultStream;
     private BufferedOutputStream mBufferedOutputNoResultStream;
 
     public void passed(String layout_file) {
@@ -68,6 +70,16 @@ class MyTestRecorder {
         }
     }
 
+    public void ignoreResult(String layout_file) {
+        try {
+            mBufferedOutputIgnoreResultStream.write(layout_file.getBytes());
+            mBufferedOutputIgnoreResultStream.write('\n');
+            mBufferedOutputIgnoreResultStream.flush();
+        } catch(Exception e) {
+            e.printStackTrace();
+        }
+    }
+
     public void noResult(String layout_file) {
         try {
             mBufferedOutputNoResultStream.write(layout_file.getBytes());
@@ -82,12 +94,15 @@ class MyTestRecorder {
         try {
             File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
             File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
+            File resultsIgnoreResultFile = new File("/sdcard/layout_tests_ignored.txt");
             File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");
 
             mBufferedOutputPassedStream =
                 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
             mBufferedOutputFailedStream =
                 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
+            mBufferedOutputIgnoreResultStream =
+                new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume));
             mBufferedOutputNoResultStream =
                 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
         } catch (Exception e) {
@@ -99,6 +114,7 @@ class MyTestRecorder {
         try {
             mBufferedOutputPassedStream.close();
             mBufferedOutputFailedStream.close();
+            mBufferedOutputIgnoreResultStream.close();
             mBufferedOutputNoResultStream.close();
         } catch (Exception e) {
             e.printStackTrace();
@@ -132,6 +148,8 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
 
     private MyTestRecorder mResultRecorder;
     private Vector<String> mTestList;
+    // Whether we should ignore the result for the corresponding test. Ordered same as mTestList.
+    private Vector<Boolean> mTestListIgnoreResult;
     private boolean mRebaselineResults;
     // The JavaScript engine currently in use. This determines which set of Android-specific
     // expected test results we use.
@@ -158,8 +176,11 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
             BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE));
             String line = inReader.readLine();
             while (line != null) {
-                if (line.startsWith(mTestPathPrefix))
-                    mTestList.add(line);
+                if (line.startsWith(mTestPathPrefix)) {
+                    String[] components = line.split(" ");
+                    mTestList.add(components[0]);
+                    mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT"));
+                }
                 line = inReader.readLine();
             }
             inReader.close();
@@ -176,6 +197,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
             for (int i = 0; i < mTestList.size(); i++) {
                 if (mTestList.elementAt(i).equals(line)) {
                     mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size()));
+                    mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size()));
                     break;
                 }
             }
@@ -236,14 +258,24 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
         mResultRecorder.passed(file);
     }
 
+    private void ignoreResultCase(String file) {
+        Log.v("Layout test:", file + " ignore result");
+        mResultRecorder.ignoreResult(file);
+    }
+
     private void noResultCase(String file) {
         Log.v("Layout test:", file + " no expected result");
         mResultRecorder.noResult(file);
     }
 
-    private void processResult(String testFile, String actualResultFile, String expectedResultFile) {
+    private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) {
         Log.v(LOGTAG, "  Processing result: " + testFile);
 
+        if (ignoreResult) {
+            ignoreResultCase(testFile);
+            return;
+        }
+
         File actual = new File(actualResultFile);
         File expected = new File(expectedResultFile);
         if (actual.exists() && expected.exists()) {
@@ -266,7 +298,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
         }
     }
 
-    private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout) {
+    private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult) {
         activity.setCallback(new TestShellCallback() {
             public void finished() {
                 synchronized (LayoutTestsAutoTest.this) {
@@ -320,7 +352,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
                 expectedResultFile = getAndroidExpectedResultFile(expectedResultFile);
             }
 
-            processResult(test, resultFile, expectedResultFile);
+            processResult(test, resultFile, expectedResultFile, ignoreResult);
         }
     }
 
@@ -336,6 +368,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
         }
 
         this.mTestList = new Vector<String>();
+        this.mTestListIgnoreResult = new Vector<Boolean>();
 
         // Read settings
         mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath();
@@ -372,9 +405,10 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
         }
         for (int i = 0; i < mTestList.size(); i++) {
             String s = mTestList.elementAt(i);
+            boolean ignoreResult = mTestListIgnoreResult.elementAt(i);
             FsUtils.updateTestStatus(TEST_STATUS_FILE, s);
             // Run tests
-            runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
+            runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult);
         }
 
         FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE");
@@ -399,7 +433,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
         try {
             File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
             BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
-            FsUtils.findLayoutTestsRecursively(bos, getTestPath());
+            FsUtils.findLayoutTestsRecursively(bos, getTestPath(), false); // Don't ignore results
             bos.flush();
             bos.close();
        } catch (Exception e) {
diff --git a/tests/DumpRenderTree/src/com/android/dumprendertree/Menu.java b/tests/DumpRenderTree/src/com/android/dumprendertree/Menu.java
index e15ab6587fac..82671ebb98a4 100644
--- a/tests/DumpRenderTree/src/com/android/dumprendertree/Menu.java
+++ b/tests/DumpRenderTree/src/com/android/dumprendertree/Menu.java
@@ -68,7 +68,7 @@ public class Menu extends FileList {
         try {
             File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
             BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
-            FsUtils.findLayoutTestsRecursively(bos, path);
+            FsUtils.findLayoutTestsRecursively(bos, path, false); // Don't ignore results
             bos.flush();
             bos.close();
        } catch (Exception e) {
@@ -77,4 +77,3 @@ public class Menu extends FileList {
     }
 
 }
-