From 8a6def02473ee4fbffcd1b34173daf751d316202 Mon Sep 17 00:00:00 2001 From: Steve Block Date: Fri, 17 Sep 2010 11:39:22 +0100 Subject: Update DumpRenderTree2 to handle failing tests where no meaningful diff is available Currently, DumpRenderTree2 asigns the following result codes {PASS, FAIL_RESULT_DIFFERS, FAIL_NO_EXPECTED_RESULT, FAIL_TIMED_OUT, FAIL_CRASHED} This is not strictly correct, as a test may fail for any of three reasons - crashing, timing out, or the the result not being as expected. Therefore, it's possible for a test to fail even if the result is as expected. This patch updates AbstractResult to handle each of the three reasons for failure separately. We then test all three to determine if the test has passed. This allows us to correctly report whether or not the result differs from expected for a failing test. Change-Id: I7adcfe72c4dd0bd3de2e1b868d9807be6eb5bddf --- .../android/dumprendertree2/AbstractResult.java | 52 ++++++-- .../dumprendertree2/CrashedDummyResult.java | 20 ++- .../dumprendertree2/LayoutTestsExecutor.java | 8 +- .../com/android/dumprendertree2/Summarizer.java | 135 +++++++++++---------- .../com/android/dumprendertree2/TextResult.java | 34 ++++-- 5 files changed, 161 insertions(+), 88 deletions(-) (limited to 'tests') diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java index 7cbb3970c8da..6048338af496 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java @@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable { public abstract AbstractResult createResult(Bundle bundle); } + /** + * A code representing the result of comparing actual and expected results. + */ public enum ResultCode { - PASS("Passed"), - FAIL_RESULT_DIFFERS("Result differs"), - FAIL_NO_EXPECTED_RESULT("No expected result"), - FAIL_TIMED_OUT("Timed out"), - FAIL_CRASHED("Crashed"); + RESULTS_MATCH("Results match"), + RESULTS_DIFFER("Results differ"), + NO_EXPECTED_RESULT("No expected result"), + NO_ACTUAL_RESULT("No actual result"); private String mTitle; @@ -123,13 +125,47 @@ public abstract class AbstractResult implements Comparable { public abstract String getActualTextResult(); /** - * Returns the code of this result. + * Returns the status code representing the result of comparing actual and expected results. * * @return - * the code of this result + * the status code from comparing actual and expected results */ public abstract ResultCode getResultCode(); + /** + * Returns whether this test crashed. + * + * @return + * whether this test crashed + */ + public abstract boolean didCrash(); + + /** + * Returns whether this test timed out. + * + * @return + * whether this test timed out + */ + public abstract boolean didTimeOut(); + + /** + * Sets that this test timed out. + */ + public abstract void setDidTimeOut(); + + /** + * Returns whether the test passed. + * + * @return + * whether the test passed + */ + public boolean didPass() { + // Tests that crash can't have timed out or have an actual result. + assert !(didCrash() && didTimeOut()); + assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT); + return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH; + } + /** * Return the type of the result data. * @@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable { public abstract String getDiffAsHtml(); public abstract Bundle getBundle(); -} \ No newline at end of file +} diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java index 31da7767ffa7..483116895f8e 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java @@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult { @Override public ResultCode getResultCode() { - return ResultCode.FAIL_CRASHED; + return ResultCode.NO_ACTUAL_RESULT; + } + + @Override + public boolean didCrash() { + return true; + } + + @Override + public boolean didTimeOut() { + return false; + } + + @Override + public void setDidTimeOut() { + /** This method is not applicable for this type of result */ + assert false; } @Override @@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult { public void setExpectedTextResultPath(String relativePath) { /** TODO */ } -} \ No newline at end of file +} diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java index efea1925b47b..b9fc2740d876 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java @@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity { assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name(); Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath); + mResultHandler.removeMessages(MSG_TEST_TIMED_OUT); obtainActualResultsFromWebView(); } @@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity { mCurrentState = CurrentState.OBTAINING_RESULT; + if (mCurrentTestTimedOut) { + mCurrentResult.setDidTimeOut(); + } mCurrentResult.obtainActualResults(mCurrentWebView, mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED)); } @@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity { Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath); mCurrentState = CurrentState.IDLE; - mResultHandler.removeMessages(MSG_TEST_TIMED_OUT); reportResultToService(); mCurrentTestIndex++; updateProgressBar(); @@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity { Bundle bundle = mCurrentResult.getBundle(); bundle.putInt("testIndex", mCurrentTestIndex); - if (mCurrentTestTimedOut) { - bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name()); - } if (!mTestsList.isEmpty()) { bundle.putString("nextTest", mTestsList.get(0)); } diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java index e31e74799c46..b1dea6e8a536 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java @@ -75,9 +75,12 @@ public class Summarizer { " width: 20px;}" + "h3 span.sqr {" + " text-decoration: none;" + - " color: #8ee100;" + " float: left;" + " width: 20px;}" + + "h3 span.sqr_pass {" + + " color: #8ee100;}" + + "h3 span.sqr_fail {" + + " color: #c30000;}" + "span.source {" + " display: block;" + " font-size: 10px;" + @@ -145,16 +148,16 @@ public class Summarizer { " text-transform: uppercase;" + " padding: 3px;" + " -webkit-border-radius: 4px;}" + - "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" + + "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" + " background-color: #ccc;" + " color: black;}" + - "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" + + "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" + " background-color: #a700e4;" + " color: #fff;}" + - "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" + + "span.timed_out {" + " background-color: #f3cb00;" + " color: black;}" + - "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" + + "span.crashed {" + " background-color: #c30000;" + " color: #fff;}" + "span.noLtc {" + @@ -208,11 +211,11 @@ public class Summarizer { public void appendTest(AbstractResult result) { String relativePath = result.getRelativePath(); - if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) { + if (result.didCrash()) { mCrashedTestsCount++; } - if (result.getResultCode() == AbstractResult.ResultCode.PASS) { + if (result.didPass()) { if (mFileFilter.isFail(relativePath)) { mUnexpectedPasses.add(result); } else { @@ -276,13 +279,10 @@ public class Summarizer { createTopSummaryTable(webKitRevision, html); - createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures); - - createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses); - - createResultsListWithDiff(html, "Expected failures", mExpectedFailures); - - createResultsListNoDiff(html, "Expected passes", mExpectedPasses); + createResultsList(html, "Unexpected failures", mUnexpectedFailures); + createResultsList(html, "Unexpected passes", mUnexpectedPasses); + createResultsList(html, "Expected failures", mExpectedFailures); + createResultsList(html, "Expected passes", mExpectedPasses); html.append(""); @@ -357,8 +357,8 @@ public class Summarizer { html.append(""); } - private void createResultsListWithDiff(StringBuilder html, String title, - List resultsList) { + private void createResultsList( + StringBuilder html, String title, List resultsList) { String relativePath; String id = ""; AbstractResult.ResultCode resultCode; @@ -368,7 +368,6 @@ public class Summarizer { for (AbstractResult result : resultsList) { relativePath = result.getRelativePath(); resultCode = result.getResultCode(); - assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode; html.append("

"); @@ -378,61 +377,73 @@ public class Summarizer { * to cause any problems in this case */ id = relativePath.replace(File.separator, ":"); - html.append(""); - html.append("▶ "); - html.append("" + relativePath + ""); - html.append(""); - - html.append(" "); - html.append(resultCode.toString()); - html.append(""); - - /** Detect missing LTC function */ - String additionalTextOutputString = result.getAdditionalTextOutputString(); - if (additionalTextOutputString != null && - additionalTextOutputString.contains("com.android.dumprendertree") && - additionalTextOutputString.contains("has no method")) { - if (additionalTextOutputString.contains("LayoutTestController")) { - html.append(" LTC function missing"); - } - if (additionalTextOutputString.contains("EventSender")) { - html.append(" "); - html.append("ES function missing"); - } + + /** Write the test name */ + if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { + html.append(""); + html.append("▶ "); + html.append("" + relativePath + ""); + html.append(""); + } else { + html.append(""); + html.append("■ "); + html.append("" + result.getRelativePath() + ""); + html.append(""); + } + + if (!result.didPass()) { + appendTags(html, result); } html.append("

"); appendExpectedResultsSources(result, html); - html.append("
"); - html.append(result.getDiffAsHtml()); - html.append("Hide"); - html.append(" | "); - html.append("Show source"); - html.append("
"); + if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { + html.append("
"); + html.append(result.getDiffAsHtml()); + html.append("Hide"); + html.append(" | "); + html.append("Show source"); + html.append("
"); + } html.append("
"); } } - private void createResultsListNoDiff(StringBuilder html, String title, - List resultsList) { - Collections.sort(resultsList); - html.append("

" + title + " [" + resultsList.size() + "]

"); - for (AbstractResult result : resultsList) { - html.append("

"); - html.append(""); - html.append("■ "); - html.append("" + result.getRelativePath() + ""); - html.append(""); - html.append("

"); - appendExpectedResultsSources(result, html); - html.append("
"); + private void appendTags(StringBuilder html, AbstractResult result) { + /** Tag tests which crash, time out or where results don't match */ + if (result.didCrash()) { + html.append(" Crashed"); + } else { + if (result.didTimeOut()) { + html.append(" Timed out"); + } + AbstractResult.ResultCode resultCode = result.getResultCode(); + if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) { + html.append(" "); + html.append(resultCode.toString()); + html.append(""); + } + } + + /** Detect missing LTC function */ + String additionalTextOutputString = result.getAdditionalTextOutputString(); + if (additionalTextOutputString != null && + additionalTextOutputString.contains("com.android.dumprendertree") && + additionalTextOutputString.contains("has no method")) { + if (additionalTextOutputString.contains("LayoutTestController")) { + html.append(" LTC function missing"); + } + if (additionalTextOutputString.contains("EventSender")) { + html.append(" "); + html.append("ES function missing"); + } } } diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java index 9664efe30115..21e54300d444 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java @@ -39,6 +39,7 @@ public class TextResult extends AbstractResult { private String mExpectedResultPath; private String mActualResult; private String mRelativePath; + private boolean mDidTimeOut; private ResultCode mResultCode; private Message mResultObtainedMsg; @@ -74,10 +75,7 @@ public class TextResult extends AbstractResult { mActualResult = bundle.getString("actualTextualResult"); setAdditionalTextOutputString(bundle.getString("additionalTextOutputString")); mRelativePath = bundle.getString("relativePath"); - String resultCode = bundle.getString("resultCode"); - if (resultCode != null) { - mResultCode = ResultCode.valueOf(resultCode); - } + mDidTimeOut = bundle.getBoolean("didTimeOut"); } @Override @@ -87,16 +85,30 @@ public class TextResult extends AbstractResult { } if (mExpectedResult == null) { - mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT; + mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT; } else if (!mExpectedResult.equals(mActualResult)) { - mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS; + mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER; } else { - mResultCode = AbstractResult.ResultCode.PASS; + mResultCode = AbstractResult.ResultCode.RESULTS_MATCH; } - return mResultCode; } + @Override + public boolean didCrash() { + return false; + } + + @Override + public boolean didTimeOut() { + return mDidTimeOut; + } + + @Override + public void setDidTimeOut() { + mDidTimeOut = true; + } + @Override public byte[] getActualImageResult() { return null; @@ -239,9 +251,7 @@ public class TextResult extends AbstractResult { bundle.putString("actualTextualResult", getActualTextResult()); bundle.putString("additionalTextOutputString", getAdditionalTextOutputString()); bundle.putString("relativePath", mRelativePath); - if (mResultCode != null) { - bundle.putString("resultCode", mResultCode.name()); - } + bundle.putBoolean("didTimeOut", mDidTimeOut); bundle.putString("type", getType().name()); return bundle; } @@ -250,4 +260,4 @@ public class TextResult extends AbstractResult { public String getRelativePath() { return mRelativePath; } -} \ No newline at end of file +} -- cgit v1.2.3-59-g8ed1b