Split warn.py into multiple files

* Split warning patterns into *_warn_patterns.py
* Split project list into android_project_list.py
* Split out the Severity class, to be changed later
* Split core of warn.py into warn_common.py and
  leave only platform dependent code in warn.py.
  This allows the core logic be used with different
  parallel processing libraries.
* Old warn.py just calls -m warn.warn.

Test: path_to_build/tools/warn.py build.log
Test: warn.py --url=http://cs/android --separator='?l=' build.log > warnings.html
Change-Id: I6734e4472a21018cd5ce06d549f6dbca24f4de54

1 files changed, 888 insertions, 0 deletions

diff --git a/tools/warn/warn_common.py b/tools/warn/warn_common.py
new file mode 100755
index 0000000000..de4b80595f
--- /dev/null
+++ b/tools/warn/warn_common.py
@@ -0,0 +1,888 @@
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Grep warnings messages and output HTML tables or warning counts in CSV.
+
+Default is to output warnings in HTML tables grouped by warning severity.
+Use option --byproject to output tables grouped by source file projects.
+Use option --gencsv to output warning counts in CSV format.
+"""
+
+# List of important data structures and functions in this script.
+#
+# To parse and keep warning message in the input file:
+#   severity:                classification of message severity
+#   severity.range           [0, 1, ... last_severity_level]
+#   severity.colors          for header background
+#   severity.column_headers  for the warning count table
+#   severity.headers         for warning message tables
+#   warn_patterns:
+#   warn_patterns[w]['category']     tool that issued the warning, not used now
+#   warn_patterns[w]['description']  table heading
+#   warn_patterns[w]['members']      matched warnings from input
+#   warn_patterns[w]['option']       compiler flag to control the warning
+#   warn_patterns[w]['patterns']     regular expressions to match warnings
+#   warn_patterns[w]['projects'][p]  number of warnings of pattern w in p
+#   warn_patterns[w]['severity']     severity level
+#   project_list[p][0]               project name
+#   project_list[p][1]               regular expression to match a project path
+#   project_patterns[p]              re.compile(project_list[p][1])
+#   project_names[p]                 project_list[p][0]
+#   warning_messages     array of each warning message, without source url
+#   warning_records      array of [idx to warn_patterns,
+#                                  idx to project_names,
+#                                  idx to warning_messages]
+#   android_root
+#   platform_version
+#   target_product
+#   target_variant
+#   compile_patterns, parse_input_file
+#
+# To emit html page of warning messages:
+#   flags: --byproject, --url, --separator
+# Old stuff for static html components:
+#   html_script_style:  static html scripts and styles
+#   htmlbig:
+#   dump_stats, dump_html_prologue, dump_html_epilogue:
+#   emit_buttons:
+#   dump_fixed
+#   sort_warnings:
+#   emit_stats_by_project:
+#   all_patterns,
+#   findproject, classify_warning
+#   dump_html
+#
+# New dynamic HTML page's static JavaScript data:
+#   Some data are copied from Python to JavaScript, to generate HTML elements.
+#   FlagURL                args.url
+#   FlagSeparator          args.separator
+#   SeverityColors:        severity.colors
+#   SeverityHeaders:       severity.headers
+#   SeverityColumnHeaders: severity.column_headers
+#   ProjectNames:          project_names, or project_list[*][0]
+#   WarnPatternsSeverity:     warn_patterns[*]['severity']
+#   WarnPatternsDescription:  warn_patterns[*]['description']
+#   WarnPatternsOption:       warn_patterns[*]['option']
+#   WarningMessages:          warning_messages
+#   Warnings:                 warning_records
+#   StatsHeader:           warning count table header row
+#   StatsRows:             array of warning count table rows
+#
+# New dynamic HTML page's dynamic JavaScript data:
+#
+# New dynamic HTML related function to emit data:
+#   escape_string, strip_escape_string, emit_warning_arrays
+#   emit_js_data():
+
+from __future__ import print_function
+import argparse
+import cgi
+import csv
+import io
+import multiprocessing
+import os
+import re
+import signal
+import sys
+
+# pylint:disable=relative-beyond-top-level
+from . import cpp_warn_patterns
+from . import java_warn_patterns
+from . import make_warn_patterns
+from . import other_warn_patterns
+from . import tidy_warn_patterns
+from .android_project_list import project_list
+from .severity import Severity
+
+parser = argparse.ArgumentParser(description='Convert a build log into HTML')
+parser.add_argument('--csvpath',
+                    help='Save CSV warning file to the passed absolute path',
+                    default=None)
+parser.add_argument('--gencsv',
+                    help='Generate a CSV file with number of various warnings',
+                    action='store_true',
+                    default=False)
+parser.add_argument('--byproject',
+                    help='Separate warnings in HTML output by project names',
+                    action='store_true',
+                    default=False)
+parser.add_argument('--url',
+                    help='Root URL of an Android source code tree prefixed '
+                    'before files in warnings')
+parser.add_argument('--separator',
+                    help='Separator between the end of a URL and the line '
+                    'number argument. e.g. #')
+parser.add_argument('--processes',
+                    type=int,
+                    default=multiprocessing.cpu_count(),
+                    help='Number of parallel processes to process warnings')
+parser.add_argument(dest='buildlog', metavar='build.log',
+                    help='Path to build.log file')
+args = parser.parse_args()
+
+warn_patterns = make_warn_patterns.patterns
+warn_patterns.extend(cpp_warn_patterns.patterns)
+warn_patterns.extend(java_warn_patterns.patterns)
+warn_patterns.extend(tidy_warn_patterns.patterns)
+warn_patterns.extend(other_warn_patterns.patterns)
+
+project_patterns = []
+project_names = []
+warning_messages = []
+warning_records = []
+
+
+def initialize_arrays():
+  """Complete global arrays before they are used."""
+  global project_names, project_patterns
+  project_names = [p[0] for p in project_list]
+  project_patterns = [re.compile(p[1]) for p in project_list]
+  for w in warn_patterns:
+    w['members'] = []
+    if 'option' not in w:
+      w['option'] = ''
+    # Each warning pattern has a 'projects' dictionary, that
+    # maps a project name to number of warnings in that project.
+    w['projects'] = {}
+
+
+initialize_arrays()
+
+
+android_root = ''
+platform_version = 'unknown'
+target_product = 'unknown'
+target_variant = 'unknown'
+
+
+##### Data and functions to dump html file. ##################################
+
+html_head_scripts = """\
+  <script type="text/javascript">
+  function expand(id) {
+    var e = document.getElementById(id);
+    var f = document.getElementById(id + "_mark");
+    if (e.style.display == 'block') {
+       e.style.display = 'none';
+       f.innerHTML = '&#x2295';
+    }
+    else {
+       e.style.display = 'block';
+       f.innerHTML = '&#x2296';
+    }
+  };
+  function expandCollapse(show) {
+    for (var id = 1; ; id++) {
+      var e = document.getElementById(id + "");
+      var f = document.getElementById(id + "_mark");
+      if (!e || !f) break;
+      e.style.display = (show ? 'block' : 'none');
+      f.innerHTML = (show ? '&#x2296' : '&#x2295');
+    }
+  };
+  </script>
+  <style type="text/css">
+  th,td{border-collapse:collapse; border:1px solid black;}
+  .button{color:blue;font-size:110%;font-weight:bolder;}
+  .bt{color:black;background-color:transparent;border:none;outline:none;
+      font-size:140%;font-weight:bolder;}
+  .c0{background-color:#e0e0e0;}
+  .c1{background-color:#d0d0d0;}
+  .t1{border-collapse:collapse; width:100%; border:1px solid black;}
+  </style>
+  <script src="https://www.gstatic.com/charts/loader.js"></script>
+"""
+
+
+def html_big(param):
+  return '<font size="+2">' + param + '</font>'
+
+
+def dump_html_prologue(title):
+  print('<html>\n<head>')
+  print('<title>' + title + '</title>')
+  print(html_head_scripts)
+  emit_stats_by_project()
+  print('</head>\n<body>')
+  print(html_big(title))
+  print('<p>')
+
+
+def dump_html_epilogue():
+  print('</body>\n</head>\n</html>')
+
+
+def sort_warnings():
+  for i in warn_patterns:
+    i['members'] = sorted(set(i['members']))
+
+
+def emit_stats_by_project():
+  """Dump a google chart table of warnings per project and severity."""
+  # warnings[p][s] is number of warnings in project p of severity s.
+  # pylint:disable=g-complex-comprehension
+  warnings = {p: {s: 0 for s in Severity.range} for p in project_names}
+  for i in warn_patterns:
+    s = i['severity']
+    for p in i['projects']:
+      warnings[p][s] += i['projects'][p]
+
+  # total_by_project[p] is number of warnings in project p.
+  total_by_project = {p: sum(warnings[p][s] for s in Severity.range)
+                      for p in project_names}
+
+  # total_by_severity[s] is number of warnings of severity s.
+  total_by_severity = {s: sum(warnings[p][s] for p in project_names)
+                       for s in Severity.range}
+
+  # emit table header
+  stats_header = ['Project']
+  for s in Severity.range:
+    if total_by_severity[s]:
+      stats_header.append("<span style='background-color:{}'>{}</span>".
+                          format(Severity.colors[s],
+                                 Severity.column_headers[s]))
+  stats_header.append('TOTAL')
+
+  # emit a row of warning counts per project, skip no-warning projects
+  total_all_projects = 0
+  stats_rows = []
+  for p in project_names:
+    if total_by_project[p]:
+      one_row = [p]
+      for s in Severity.range:
+        if total_by_severity[s]:
+          one_row.append(warnings[p][s])
+      one_row.append(total_by_project[p])
+      stats_rows.append(one_row)
+      total_all_projects += total_by_project[p]
+
+  # emit a row of warning counts per severity
+  total_all_severities = 0
+  one_row = ['<b>TOTAL</b>']
+  for s in Severity.range:
+    if total_by_severity[s]:
+      one_row.append(total_by_severity[s])
+      total_all_severities += total_by_severity[s]
+  one_row.append(total_all_projects)
+  stats_rows.append(one_row)
+  print('<script>')
+  emit_const_string_array('StatsHeader', stats_header)
+  emit_const_object_array('StatsRows', stats_rows)
+  print(draw_table_javascript)
+  print('</script>')
+
+
+def dump_stats():
+  """Dump some stats about total number of warnings and such."""
+  known = 0
+  skipped = 0
+  unknown = 0
+  sort_warnings()
+  for i in warn_patterns:
+    if i['severity'] == Severity.UNKNOWN:
+      unknown += len(i['members'])
+    elif i['severity'] == Severity.SKIP:
+      skipped += len(i['members'])
+    else:
+      known += len(i['members'])
+  print('Number of classified warnings: <b>' + str(known) + '</b><br>')
+  print('Number of skipped warnings: <b>' + str(skipped) + '</b><br>')
+  print('Number of unclassified warnings: <b>' + str(unknown) + '</b><br>')
+  total = unknown + known + skipped
+  extra_msg = ''
+  if total < 1000:
+    extra_msg = ' (low count may indicate incremental build)'
+  print('Total number of warnings: <b>' + str(total) + '</b>' + extra_msg)
+
+
+# New base table of warnings, [severity, warn_id, project, warning_message]
+# Need buttons to show warnings in different grouping options.
+# (1) Current, group by severity, id for each warning pattern
+#     sort by severity, warn_id, warning_message
+# (2) Current --byproject, group by severity,
+#     id for each warning pattern + project name
+#     sort by severity, warn_id, project, warning_message
+# (3) New, group by project + severity,
+#     id for each warning pattern
+#     sort by project, severity, warn_id, warning_message
+def emit_buttons():
+  print('<button class="button" onclick="expandCollapse(1);">'
+        'Expand all warnings</button>\n'
+        '<button class="button" onclick="expandCollapse(0);">'
+        'Collapse all warnings</button>\n'
+        '<button class="button" onclick="groupBySeverity();">'
+        'Group warnings by severity</button>\n'
+        '<button class="button" onclick="groupByProject();">'
+        'Group warnings by project</button><br>')
+
+
+def all_patterns(category):
+  patterns = ''
+  for i in category['patterns']:
+    patterns += i
+    patterns += ' / '
+  return patterns
+
+
+def dump_fixed():
+  """Show which warnings no longer occur."""
+  anchor = 'fixed_warnings'
+  mark = anchor + '_mark'
+  print('\n<br><p style="background-color:lightblue"><b>'
+        '<button id="' + mark + '" '
+        'class="bt" onclick="expand(\'' + anchor + '\');">'
+        '&#x2295</button> Fixed warnings. '
+        'No more occurrences. Please consider turning these into '
+        'errors if possible, before they are reintroduced in to the build'
+        ':</b></p>')
+  print('<blockquote>')
+  fixed_patterns = []
+  for i in warn_patterns:
+    if not i['members']:
+      fixed_patterns.append(i['description'] + ' (' +
+                            all_patterns(i) + ')')
+    if i['option']:
+      fixed_patterns.append(' ' + i['option'])
+  fixed_patterns = sorted(fixed_patterns)
+  print('<div id="' + anchor + '" style="display:none;"><table>')
+  cur_row_class = 0
+  for text in fixed_patterns:
+    cur_row_class = 1 - cur_row_class
+    # remove last '\n'
+    t = text[:-1] if text[-1] == '\n' else text
+    print('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>')
+  print('</table></div>')
+  print('</blockquote>')
+
+
+def find_project_index(line):
+  for p in range(len(project_patterns)):
+    if project_patterns[p].match(line):
+      return p
+  return -1
+
+
+def classify_one_warning(line, results):
+  """Classify one warning line."""
+  for i in range(len(warn_patterns)):
+    w = warn_patterns[i]
+    for cpat in w['compiled_patterns']:
+      if cpat.match(line):
+        p = find_project_index(line)
+        results.append([line, i, p])
+        return
+      else:
+        # If we end up here, there was a problem parsing the log
+        # probably caused by 'make -j' mixing the output from
+        # 2 or more concurrent compiles
+        pass
+
+
+def classify_warnings(lines):
+  results = []
+  for line in lines:
+    classify_one_warning(line, results)
+  # After the main work, ignore all other signals to a child process,
+  # to avoid bad warning/error messages from the exit clean-up process.
+  if args.processes > 1:
+    signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM))
+  return results
+
+
+def parallel_classify_warnings(warning_lines, parallel_process):
+  """Classify all warning lines with num_cpu parallel processes."""
+  compile_patterns()
+  num_cpu = args.processes
+  if num_cpu > 1:
+    groups = [[] for x in range(num_cpu)]
+    i = 0
+    for x in warning_lines:
+      groups[i].append(x)
+      i = (i + 1) % num_cpu
+    group_results = parallel_process(num_cpu, classify_warnings, groups)
+  else:
+    group_results = [classify_warnings(warning_lines)]
+
+  for result in group_results:
+    for line, pattern_idx, project_idx in result:
+      pattern = warn_patterns[pattern_idx]
+      pattern['members'].append(line)
+      message_idx = len(warning_messages)
+      warning_messages.append(line)
+      warning_records.append([pattern_idx, project_idx, message_idx])
+      pname = '???' if project_idx < 0 else project_names[project_idx]
+      # Count warnings by project.
+      if pname in pattern['projects']:
+        pattern['projects'][pname] += 1
+      else:
+        pattern['projects'][pname] = 1
+
+
+def compile_patterns():
+  """Precompiling every pattern speeds up parsing by about 30x."""
+  for i in warn_patterns:
+    i['compiled_patterns'] = []
+    for pat in i['patterns']:
+      i['compiled_patterns'].append(re.compile(pat))
+
+
+def find_warn_py_and_android_root(path):
+  """Set and return android_root path if it is found."""
+  global android_root
+  parts = path.split('/')
+  for idx in reversed(range(2, len(parts))):
+    root_path = '/'.join(parts[:idx])
+    # Android root directory should contain this script.
+    if os.path.exists(root_path + '/build/make/tools/warn.py'):
+      android_root = root_path
+      return True
+  return False
+
+
+def find_android_root():
+  """Guess android_root from common prefix of file paths."""
+  # Use the longest common prefix of the absolute file paths
+  # of the first 10000 warning messages as the android_root.
+  global android_root
+  warning_lines = set()
+  warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*')
+  count = 0
+  infile = io.open(args.buildlog, mode='r', encoding='utf-8')
+  for line in infile:
+    if warning_pattern.match(line):
+      warning_lines.add(line)
+      count += 1
+      if count > 9999:
+        break
+      # Try to find warn.py and use its location to find
+      # the source tree root.
+      if count < 100:
+        path = os.path.normpath(re.sub(':.*$', '', line))
+        if find_warn_py_and_android_root(path):
+          return
+  # Do not use common prefix of a small number of paths.
+  if count > 10:
+    root_path = os.path.commonprefix(warning_lines)
+    if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
+      android_root = root_path[:-1]
+
+
+def remove_android_root_prefix(path):
+  """Remove android_root prefix from path if it is found."""
+  if path.startswith(android_root):
+    return path[1 + len(android_root):]
+  else:
+    return path
+
+
+def normalize_path(path):
+  """Normalize file path relative to android_root."""
+  # If path is not an absolute path, just normalize it.
+  path = os.path.normpath(path)
+  # Remove known prefix of root path and normalize the suffix.
+  if path[0] == '/' and android_root:
+    return remove_android_root_prefix(path)
+  return path
+
+
+def normalize_warning_line(line):
+  """Normalize file path relative to android_root in a warning line."""
+  # replace fancy quotes with plain ol' quotes
+  line = re.sub(u'[\u2018\u2019]', '\'', line)
+  # replace non-ASCII chars to spaces
+  line = re.sub(u'[^\x00-\x7f]', ' ', line)
+  line = line.strip()
+  first_column = line.find(':')
+  if first_column > 0:
+    return normalize_path(line[:first_column]) + line[first_column:]
+  else:
+    return line
+
+
+def parse_input_file(infile):
+  """Parse input file, collect parameters and warning lines."""
+  global android_root
+  global platform_version
+  global target_product
+  global target_variant
+  line_counter = 0
+
+  # rustc warning messages have two lines that should be combined:
+  #     warning: description
+  #        --> file_path:line_number:column_number
+  # Some warning messages have no file name:
+  #     warning: macro replacement list ... [bugprone-macro-parentheses]
+  # Some makefile warning messages have no line number:
+  #     some/path/file.mk: warning: description
+  # C/C++ compiler warning messages have line and column numbers:
+  #     some/path/file.c:line_number:column_number: warning: description
+  warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)')
+  warning_without_file = re.compile('^warning: .*')
+  rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')
+
+  # Collect all warnings into the warning_lines set.
+  warning_lines = set()
+  prev_warning = ''
+  for line in infile:
+    if prev_warning:
+      if rustc_file_position.match(line):
+        # must be a rustc warning, combine 2 lines into one warning
+        line = line.strip().replace('--> ', '') + ': ' + prev_warning
+        warning_lines.add(normalize_warning_line(line))
+        prev_warning = ''
+        continue
+      # add prev_warning, and then process the current line
+      prev_warning = 'unknown_source_file: ' + prev_warning
+      warning_lines.add(normalize_warning_line(prev_warning))
+      prev_warning = ''
+    if warning_pattern.match(line):
+      if warning_without_file.match(line):
+        # save this line and combine it with the next line
+        prev_warning = line
+      else:
+        warning_lines.add(normalize_warning_line(line))
+      continue
+    if line_counter < 100:
+      # save a little bit of time by only doing this for the first few lines
+      line_counter += 1
+      m = re.search('(?<=^PLATFORM_VERSION=).*', line)
+      if m is not None:
+        platform_version = m.group(0)
+      m = re.search('(?<=^TARGET_PRODUCT=).*', line)
+      if m is not None:
+        target_product = m.group(0)
+      m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
+      if m is not None:
+        target_variant = m.group(0)
+      m = re.search('.* TOP=([^ ]*) .*', line)
+      if m is not None:
+        android_root = m.group(1)
+  return warning_lines
+
+
+# Return s with escaped backslash and quotation characters.
+def escape_string(s):
+  return s.replace('\\', '\\\\').replace('"', '\\"')
+
+
+# Return s without trailing '\n' and escape the quotation characters.
+def strip_escape_string(s):
+  if not s:
+    return s
+  s = s[:-1] if s[-1] == '\n' else s
+  return escape_string(s)
+
+
+def emit_warning_array(name):
+  print('var warning_{} = ['.format(name))
+  for i in range(len(warn_patterns)):
+    print('{},'.format(warn_patterns[i][name]))
+  print('];')
+
+
+def emit_warning_arrays():
+  emit_warning_array('severity')
+  print('var warning_description = [')
+  for i in range(len(warn_patterns)):
+    if warn_patterns[i]['members']:
+      print('"{}",'.format(escape_string(warn_patterns[i]['description'])))
+    else:
+      print('"",')  # no such warning
+  print('];')
+
+
+scripts_for_warning_groups = """
+  function compareMessages(x1, x2) { // of the same warning type
+    return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1;
+  }
+  function byMessageCount(x1, x2) {
+    return x2[2] - x1[2];  // reversed order
+  }
+  function bySeverityMessageCount(x1, x2) {
+    // orer by severity first
+    if (x1[1] != x2[1])
+      return  x1[1] - x2[1];
+    return byMessageCount(x1, x2);
+  }
+  const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/;
+  function addURL(line) {
+    if (FlagURL == "") return line;
+    if (FlagSeparator == "") {
+      return line.replace(ParseLinePattern,
+        "<a target='_blank' href='" + FlagURL + "/$1'>$1</a>:$2:$3");
+    }
+    return line.replace(ParseLinePattern,
+      "<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator +
+        "$2'>$1:$2</a>:$3");
+  }
+  function createArrayOfDictionaries(n) {
+    var result = [];
+    for (var i=0; i<n; i++) result.push({});
+    return result;
+  }
+  function groupWarningsBySeverity() {
+    // groups is an array of dictionaries,
+    // each dictionary maps from warning type to array of warning messages.
+    var groups = createArrayOfDictionaries(SeverityColors.length);
+    for (var i=0; i<Warnings.length; i++) {
+      var w = Warnings[i][0];
+      var s = WarnPatternsSeverity[w];
+      var k = w.toString();
+      if (!(k in groups[s]))
+        groups[s][k] = [];
+      groups[s][k].push(Warnings[i]);
+    }
+    return groups;
+  }
+  function groupWarningsByProject() {
+    var groups = createArrayOfDictionaries(ProjectNames.length);
+    for (var i=0; i<Warnings.length; i++) {
+      var w = Warnings[i][0];
+      var p = Warnings[i][1];
+      var k = w.toString();
+      if (!(k in groups[p]))
+        groups[p][k] = [];
+      groups[p][k].push(Warnings[i]);
+    }
+    return groups;
+  }
+  var GlobalAnchor = 0;
+  function createWarningSection(header, color, group) {
+    var result = "";
+    var groupKeys = [];
+    var totalMessages = 0;
+    for (var k in group) {
+       totalMessages += group[k].length;
+       groupKeys.push([k, WarnPatternsSeverity[parseInt(k)], group[k].length]);
+    }
+    groupKeys.sort(bySeverityMessageCount);
+    for (var idx=0; idx<groupKeys.length; idx++) {
+      var k = groupKeys[idx][0];
+      var messages = group[k];
+      var w = parseInt(k);
+      var wcolor = SeverityColors[WarnPatternsSeverity[w]];
+      var description = WarnPatternsDescription[w];
+      if (description.length == 0)
+          description = "???";
+      GlobalAnchor += 1;
+      result += "<table class='t1'><tr bgcolor='" + wcolor + "'><td>" +
+                "<button class='bt' id='" + GlobalAnchor + "_mark" +
+                "' onclick='expand(\\"" + GlobalAnchor + "\\");'>" +
+                "&#x2295</button> " +
+                description + " (" + messages.length + ")</td></tr></table>";
+      result += "<div id='" + GlobalAnchor +
+                "' style='display:none;'><table class='t1'>";
+      var c = 0;
+      messages.sort(compareMessages);
+      for (var i=0; i<messages.length; i++) {
+        result += "<tr><td class='c" + c + "'>" +
+                  addURL(WarningMessages[messages[i][2]]) + "</td></tr>";
+        c = 1 - c;
+      }
+      result += "</table></div>";
+    }
+    if (result.length > 0) {
+      return "<br><span style='background-color:" + color + "'><b>" +
+             header + ": " + totalMessages +
+             "</b></span><blockquote><table class='t1'>" +
+             result + "</table></blockquote>";
+
+    }
+    return "";  // empty section
+  }
+  function generateSectionsBySeverity() {
+    var result = "";
+    var groups = groupWarningsBySeverity();
+    for (s=0; s<SeverityColors.length; s++) {
+      result += createWarningSection(SeverityHeaders[s], SeverityColors[s], groups[s]);
+    }
+    return result;
+  }
+  function generateSectionsByProject() {
+    var result = "";
+    var groups = groupWarningsByProject();
+    for (i=0; i<groups.length; i++) {
+      result += createWarningSection(ProjectNames[i], 'lightgrey', groups[i]);
+    }
+    return result;
+  }
+  function groupWarnings(generator) {
+    GlobalAnchor = 0;
+    var e = document.getElementById("warning_groups");
+    e.innerHTML = generator();
+  }
+  function groupBySeverity() {
+    groupWarnings(generateSectionsBySeverity);
+  }
+  function groupByProject() {
+    groupWarnings(generateSectionsByProject);
+  }
+"""
+
+
+# Emit a JavaScript const string
+def emit_const_string(name, value):
+  print('const ' + name + ' = "' + escape_string(value) + '";')
+
+
+# Emit a JavaScript const integer array.
+def emit_const_int_array(name, array):
+  print('const ' + name + ' = [')
+  for n in array:
+    print(str(n) + ',')
+  print('];')
+
+
+# Emit a JavaScript const string array.
+def emit_const_string_array(name, array):
+  print('const ' + name + ' = [')
+  for s in array:
+    print('"' + strip_escape_string(s) + '",')
+  print('];')
+
+
+# Emit a JavaScript const string array for HTML.
+def emit_const_html_string_array(name, array):
+  print('const ' + name + ' = [')
+  for s in array:
+    # Not using html.escape yet, to work for both python 2 and 3,
+    # until all users switch to python 3.
+    # pylint:disable=deprecated-method
+    print('"' + cgi.escape(strip_escape_string(s)) + '",')
+  print('];')
+
+
+# Emit a JavaScript const object array.
+def emit_const_object_array(name, array):
+  print('const ' + name + ' = [')
+  for x in array:
+    print(str(x) + ',')
+  print('];')
+
+
+def emit_js_data():
+  """Dump dynamic HTML page's static JavaScript data."""
+  emit_const_string('FlagURL', args.url if args.url else '')
+  emit_const_string('FlagSeparator', args.separator if args.separator else '')
+  emit_const_string_array('SeverityColors', Severity.colors)
+  emit_const_string_array('SeverityHeaders', Severity.headers)
+  emit_const_string_array('SeverityColumnHeaders', Severity.column_headers)
+  emit_const_string_array('ProjectNames', project_names)
+  emit_const_int_array('WarnPatternsSeverity',
+                       [w['severity'] for w in warn_patterns])
+  emit_const_html_string_array('WarnPatternsDescription',
+                               [w['description'] for w in warn_patterns])
+  emit_const_html_string_array('WarnPatternsOption',
+                               [w['option'] for w in warn_patterns])
+  emit_const_html_string_array('WarningMessages', warning_messages)
+  emit_const_object_array('Warnings', warning_records)
+
+draw_table_javascript = """
+google.charts.load('current', {'packages':['table']});
+google.charts.setOnLoadCallback(drawTable);
+function drawTable() {
+  var data = new google.visualization.DataTable();
+  data.addColumn('string', StatsHeader[0]);
+  for (var i=1; i<StatsHeader.length; i++) {
+    data.addColumn('number', StatsHeader[i]);
+  }
+  data.addRows(StatsRows);
+  for (var i=0; i<StatsRows.length; i++) {
+    for (var j=0; j<StatsHeader.length; j++) {
+      data.setProperty(i, j, 'style', 'border:1px solid black;');
+    }
+  }
+  var table = new google.visualization.Table(document.getElementById('stats_table'));
+  table.draw(data, {allowHtml: true, alternatingRowStyle: true});
+}
+"""
+
+
+def dump_html():
+  """Dump the html output to stdout."""
+  dump_html_prologue('Warnings for ' + platform_version + ' - ' +
+                     target_product + ' - ' + target_variant)
+  dump_stats()
+  print('<br><div id="stats_table"></div><br>')
+  print('\n<script>')
+  emit_js_data()
+  print(scripts_for_warning_groups)
+  print('</script>')
+  emit_buttons()
+  # Warning messages are grouped by severities or project names.
+  print('<br><div id="warning_groups"></div>')
+  if args.byproject:
+    print('<script>groupByProject();</script>')
+  else:
+    print('<script>groupBySeverity();</script>')
+  dump_fixed()
+  dump_html_epilogue()
+
+
+##### Functions to count warnings and dump csv file. #########################
+
+
+def description_for_csv(category):
+  if not category['description']:
+    return '?'
+  return category['description']
+
+
+def count_severity(writer, sev, kind):
+  """Count warnings of given severity."""
+  total = 0
+  for i in warn_patterns:
+    if i['severity'] == sev and i['members']:
+      n = len(i['members'])
+      total += n
+      warning = kind + ': ' + description_for_csv(i)
+      writer.writerow([n, '', warning])
+      # print number of warnings for each project, ordered by project name.
+      projects = sorted(i['projects'].keys())
+      for p in projects:
+        writer.writerow([i['projects'][p], p, warning])
+  writer.writerow([total, '', kind + ' warnings'])
+
+  return total
+
+
+# dump number of warnings in csv format to stdout
+def dump_csv(writer):
+  """Dump number of warnings in csv format to stdout."""
+  sort_warnings()
+  total = 0
+  for s in Severity.range:
+    total += count_severity(writer, s, Severity.column_headers[s])
+  writer.writerow([total, '', 'All warnings'])
+
+
+def common_main(parallel_process):
+  """Real main function to classify warnings and generate .html file."""
+  find_android_root()
+  # We must use 'utf-8' codec to parse some non-ASCII code in warnings.
+  warning_lines = parse_input_file(
+      io.open(args.buildlog, mode='r', encoding='utf-8'))
+  parallel_classify_warnings(warning_lines, parallel_process)
+  # If a user pases a csv path, save the fileoutput to the path
+  # If the user also passed gencsv write the output to stdout
+  # If the user did not pass gencsv flag dump the html report to stdout.
+  if args.csvpath:
+    with open(args.csvpath, 'w') as f:
+      dump_csv(csv.writer(f, lineterminator='\n'))
+  if args.gencsv:
+    dump_csv(csv.writer(sys.stdout, lineterminator='\n'))
+  else:
+    dump_html()