Show directories and files with most warnings.

* Now with 4 sections that can be shown/hidden independently.
* After the 'selected_projects' section, add two more sections:
  * 'top_directory_section' contains
    * a table of directories with at least 1% warnings.
    * a 'selected_directory_warnings' subsection to show all warnings
      in the clicked directory from the directory table
  * 'top_file_section' contains
    * a table of files with at least 100 or 1% warnings.
    * a 'selected_file_warnings' subsection to show all warnings
      in the clicked file from the file table
* Adjust button/section spaces and reduce button font size to 100%.
* Rename drawTable to genTables.

Test: warn.py --url=http://cs/android --separator='?l=' build.log > warnings.html
Test: warn.py --gencsv build.log > warnings.csv
Change-Id: I765b09a46adc111cfe781719ba7aa0f917aa6ffc

1 files changed, 213 insertions, 21 deletions

diff --git a/tools/warn/html_writer.py b/tools/warn/html_writer.py
index ef173bcb3f..3fa822a9b2 100644
--- a/tools/warn/html_writer.py
+++ b/tools/warn/html_writer.py
@@ -63,6 +63,11 @@ import sys
 from .severity import Severity
 
 
+# Report files with this number of warnings or more.
+LIMIT_WARNINGS_PER_FILE = 100
+# Report files/directories with this percentage of total warnings or more.
+LIMIT_PERCENT_WARNINGS = 1
+
 HTML_HEAD_SCRIPTS = """\
   <script type="text/javascript">
   function expand(id) {
@@ -89,12 +94,13 @@ HTML_HEAD_SCRIPTS = """\
   </script>
   <style type="text/css">
   th,td{border-collapse:collapse; border:1px solid black;}
-  .button{color:blue;font-size:110%;font-weight:bolder;}
+  .button{color:blue;font-size:100%;font-weight:bolder;}
   .bt{color:black;background-color:transparent;border:none;outline:none;
       font-size:140%;font-weight:bolder;}
   .c0{background-color:#e0e0e0;}
   .c1{background-color:#d0d0d0;}
   .t1{border-collapse:collapse; width:100%; border:1px solid black;}
+  .box{margin:5pt; padding:5pt; border:1px solid;}
   </style>
   <script src="https://www.gstatic.com/charts/loader.js"></script>
 """
@@ -287,14 +293,14 @@ def dump_stats(writer, warn_patterns):
 #     sort by project, severity, warn_id, warning_message
 def emit_buttons(writer):
   """Write the button elements in HTML."""
-  writer('<button class="button" onclick="expandCollapse(1);">'
+  writer('<p><button class="button" onclick="expandCollapse(1);">'
          'Expand all warnings</button>\n'
          '<button class="button" onclick="expandCollapse(0);">'
          'Collapse all warnings</button>\n'
-         '<button class="button" onclick="groupBySeverity();">'
+         '<p><button class="button" onclick="groupBySeverity();">'
          'Group warnings by severity</button>\n'
          '<button class="button" onclick="groupByProject();">'
-         'Group warnings by project</button><br>')
+         'Group warnings by project</button>')
 
 
 def all_patterns(category):
@@ -559,6 +565,11 @@ SCRIPTS_FOR_WARNING_GROUPS = """
 """
 
 
+# Emit a JavaScript const number
+def emit_const_number(name, value, writer):
+  writer('const ' + name + ' = ' + str(value) + ';')
+
+
 # Emit a JavaScript const string
 def emit_const_string(name, value, writer):
   writer('const ' + name + ' = "' + escape_string(value) + '";')
@@ -602,6 +613,8 @@ def emit_js_data(writer, flags, warning_messages, warning_links,
   emit_const_string('FlagPlatform', flags.platform, writer)
   emit_const_string('FlagURL', flags.url, writer)
   emit_const_string('FlagSeparator', flags.separator, writer)
+  emit_const_number('LimitWarningsPerFile', LIMIT_WARNINGS_PER_FILE, writer)
+  emit_const_number('LimitPercentWarnings', LIMIT_PERCENT_WARNINGS, writer)
   emit_const_string_array('SeverityColors', [s.color for s in Severity.levels],
                           writer)
   emit_const_string_array('SeverityHeaders',
@@ -624,8 +637,8 @@ def emit_js_data(writer, flags, warning_messages, warning_links,
 
 DRAW_TABLE_JAVASCRIPT = """
 google.charts.load('current', {'packages':['table']});
-google.charts.setOnLoadCallback(drawTable);
-function drawTable() {
+google.charts.setOnLoadCallback(genTables);
+function genSelectedProjectsTable() {
   var data = new google.visualization.DataTable();
   data.addColumn('string', StatsHeader[0]);
   for (var i=1; i<StatsHeader.length; i++) {
@@ -638,12 +651,167 @@ function drawTable() {
     }
   }
   var table = new google.visualization.Table(
-      document.getElementById('stats_table'));
+      document.getElementById('selected_projects_section'));
   table.draw(data, {allowHtml: true, alternatingRowStyle: true});
 }
+// Global TopDirs and TopFiles are computed later by genTables.
+window.TopDirs = [];
+window.TopFiles = [];
+function computeTopDirsFiles() {
+  var numWarnings = WarningMessages.length;
+  var warningsOfFiles = {};
+  var warningsOfDirs = {};
+  var subDirs = {};
+  function addOneWarning(map, key) {
+    map[key] = 1 + ((key in map) ? map[key] : 0);
+  }
+  for (var i = 0; i < numWarnings; i++) {
+    var file = WarningMessages[i].replace(/:.*/, "");
+    addOneWarning(warningsOfFiles, file);
+    var dirs = file.split("/");
+    var dir = dirs[0];
+    addOneWarning(warningsOfDirs, dir);
+    for (var d = 1; d < dirs.length - 1; d++) {
+      var subDir = dir + "/" + dirs[d];
+      if (!(dir in subDirs)) {
+        subDirs[dir] = {};
+      }
+      subDirs[dir][subDir] = 1;
+      dir = subDir;
+      addOneWarning(warningsOfDirs, dir);
+    }
+  }
+  var minDirWarnings = numWarnings*(LimitPercentWarnings/100);
+  var minFileWarnings = Math.min(LimitWarningsPerFile, minDirWarnings);
+  // Each row in TopDirs and TopFiles has
+  // [index, {v:<num_of_warnings>, f:<percent>}, file_or_dir_name]
+  function countWarnings(minWarnings, warningsOf, isDir) {
+    var rows = [];
+    for (var name in warningsOf) {
+      if (isDir && name in subDirs && Object.keys(subDirs[name]).length < 2) {
+        continue; // skip a directory if it has only one subdir
+      }
+      var count = warningsOf[name];
+      if (count >= minWarnings) {
+        name = isDir ? (name + "/...") : name;
+        var percent = (100*count/numWarnings).toFixed(1);
+        var countFormat = count + ' (' + percent + '%)';
+        rows.push([0, {v:count, f:countFormat}, name]);
+      }
+    }
+    rows.sort((a,b) => b[1].v - a[1].v);
+    for (var i=0; i<rows.length; i++) {
+      rows[i][0] = i;
+    }
+    return rows;
+  }
+  TopDirs = countWarnings(minDirWarnings, warningsOfDirs, true);
+  TopFiles = countWarnings(minFileWarnings, warningsOfFiles, false);
+}
+function genTopDirsFilesTables() {
+  computeTopDirsFiles();
+  function addTable(name, divName, rows, clickFunction) {
+    var data = new google.visualization.DataTable();
+    data.addColumn("number", "index"); // not shown in view
+    data.addColumn("number", "# of warnings");
+    data.addColumn("string", name);
+    data.addRows(rows);
+    var formatter = new google.visualization.PatternFormat(
+      '<p onclick="' + clickFunction + '({0})">{2}</p>');
+    formatter.format(data, [0, 1, 2], 2);
+    var view = new google.visualization.DataView(data);
+    view.setColumns([1,2]); // hide the index column
+    var table = new google.visualization.Table(
+        document.getElementById(divName));
+    table.draw(view, {allowHtml: true, alternatingRowStyle: true});
+  }
+  addTable("Directory", "top_dirs_table", TopDirs, "selectDir");
+  addTable("File", "top_files_table", TopFiles, "selectFile");
+}
+function selectDirFile(idx, rows, dirFile) {
+  if (rows.length <= idx) {
+    return;
+  }
+  var name = rows[idx][2];
+  var spanName = "selected_" + dirFile + "_name";
+  document.getElementById(spanName).innerHTML = name;
+  var divName = "selected_" + dirFile + "_warnings";
+  var numWarnings = rows[idx][1].v;
+  var prefix = name.replace(/\\.\\.\\.$/, "");
+  var data = new google.visualization.DataTable();
+  data.addColumn('string', numWarnings + ' warnings in ' + name);
+  var getWarningMessage = (FlagPlatform == "chrome")
+        ? ((x) => addURLToLine(WarningMessages[Warnings[x][2]],
+                               WarningLinks[Warnings[x][3]]))
+        : ((x) => addURL(WarningMessages[Warnings[x][2]]));
+  for (var i = 0; i < Warnings.length; i++) {
+    if (WarningMessages[Warnings[i][2]].startsWith(prefix)) {
+      data.addRow([getWarningMessage(i)]);
+    }
+  }
+  var table = new google.visualization.Table(
+      document.getElementById(divName));
+  table.draw(data, {allowHtml: true, alternatingRowStyle: true});
+}
+function selectDir(idx) {
+  selectDirFile(idx, TopDirs, "directory")
+}
+function selectFile(idx) {
+  selectDirFile(idx, TopFiles, "file");
+}
+function genTables() {
+  genSelectedProjectsTable();
+  if (WarningMessages.length > 1) {
+    genTopDirsFilesTables();
+  }
+}
 """
 
 
+def dump_boxed_section(writer, func):
+  writer('<div class="box">')
+  func()
+  writer('</div>')
+
+
+def dump_section_header(writer, table_name, section_title):
+  writer('<h3><b><button id="' + table_name + '_mark" class="bt"\n' +
+         ' onclick="expand(\'' + table_name + '\');">&#x2295</button></b>\n' +
+         section_title + '</h3>')
+
+
+def dump_table_section(writer, table_name, section_title):
+  dump_section_header(writer, table_name, section_title)
+  writer('<div id="' + table_name + '" style="display:none;"></div>')
+
+
+def dump_dir_file_section(writer, dir_file, table_name, section_title):
+  section_name = 'top_' + dir_file + '_section'
+  dump_section_header(writer, section_name, section_title)
+  writer('<div id="' + section_name + '" style="display:none;">')
+  writer('<div id="' + table_name + '"></div>')
+  def subsection():
+    subsection_name = 'selected_' + dir_file + '_warnings'
+    subsection_title = ('Warnings in <span id="selected_' + dir_file +
+                        '_name">(click a ' + dir_file +
+                        ' in the above table)</span>')
+    dump_section_header(writer, subsection_name, subsection_title)
+    writer('<div id="' + subsection_name + '" style="display:none;"></div>')
+  dump_boxed_section(writer, subsection)
+  writer('</div>')
+
+
+# HTML output has the following major div elements:
+#  selected_projects_section
+#  top_directory_section
+#    top_dirs_table
+#    selected_directory_warnings
+#  top_file_section
+#    top_files_table
+#    selected_file_warnings
+#  all_warnings_section
+#    warning_groups
+#    fixed_warnings
 def dump_html(flags, output_stream, warning_messages, warning_links,
               warning_records, header_str, warn_patterns, project_names):
   """Dump the flags output to output_stream."""
@@ -651,20 +819,44 @@ def dump_html(flags, output_stream, warning_messages, warning_links,
   dump_html_prologue('Warnings for ' + header_str, writer, warn_patterns,
                      project_names)
   dump_stats(writer, warn_patterns)
-  writer('<br><div id="stats_table"></div><br>')
-  writer('\n<script>')
-  emit_js_data(writer, flags, warning_messages, warning_links, warning_records,
-               warn_patterns, project_names)
-  writer(SCRIPTS_FOR_WARNING_GROUPS)
-  writer('</script>')
-  emit_buttons(writer)
-  # Warning messages are grouped by severities or project names.
-  writer('<br><div id="warning_groups"></div>')
-  if flags.byproject:
-    writer('<script>groupByProject();</script>')
-  else:
-    writer('<script>groupBySeverity();</script>')
-  dump_fixed(writer, warn_patterns)
+  writer('<br><br>Press &#x2295 to show section content,'
+         ' and &#x2296 to hide the content.')
+  def section1():
+    dump_table_section(writer, 'selected_projects_section',
+                       'Number of warnings in preselected project directories')
+  def section2():
+    dump_dir_file_section(
+        writer, 'directory', 'top_dirs_table',
+        'Directories with at least ' +
+        str(LIMIT_PERCENT_WARNINGS) + '% warnings')
+  def section3():
+    dump_dir_file_section(
+        writer, 'file', 'top_files_table',
+        'Files with at least ' +
+        str(LIMIT_PERCENT_WARNINGS) + '% or ' +
+        str(LIMIT_WARNINGS_PER_FILE) + ' warnings')
+  def section4():
+    writer('<script>')
+    emit_js_data(writer, flags, warning_messages, warning_links,
+                 warning_records, warn_patterns, project_names)
+    writer(SCRIPTS_FOR_WARNING_GROUPS)
+    writer('</script>')
+    dump_section_header(writer, 'all_warnings_section',
+                        'All warnings grouped by severities or projects')
+    writer('<div id="all_warnings_section" style="display:none;">')
+    emit_buttons(writer)
+    # Warning messages are grouped by severities or project names.
+    writer('<br><div id="warning_groups"></div>')
+    if flags.byproject:
+      writer('<script>groupByProject();</script>')
+    else:
+      writer('<script>groupBySeverity();</script>')
+    dump_fixed(writer, warn_patterns)
+    writer('</div>')
+  dump_boxed_section(writer, section1)
+  dump_boxed_section(writer, section2)
+  dump_boxed_section(writer, section3)
+  dump_boxed_section(writer, section4)
   dump_html_epilogue(writer)