Merge "Move boot jars package check from make"

9 files changed, 505 insertions, 4 deletions

diff --git a/android/apex.go b/android/apex.go
index 3039e7944..e70ec4f3c 100644
--- a/android/apex.go
+++ b/android/apex.go
@@ -67,6 +67,15 @@ func (i ApexInfo) IsForPlatform() bool {
 	return i.ApexVariationName == ""
 }
 
+func (i ApexInfo) InApex(apex string) bool {
+	for _, a := range i.InApexes {
+		if a == apex {
+			return true
+		}
+	}
+	return false
+}
+
 // ApexTestForInfo stores the contents of APEXes for which this module is a test and thus has
 // access to APEX internals.
 type ApexTestForInfo struct {
diff --git a/android/config.go b/android/config.go
index dbae7f78c..a4990576e 100644
--- a/android/config.go
+++ b/android/config.go
@@ -793,6 +793,11 @@ func (c *config) AlwaysUsePrebuiltSdks() bool {
 	return Bool(c.productVariables.Always_use_prebuilt_sdks)
 }
 
+// Returns true if the boot jars check should be skipped.
+func (c *config) SkipBootJarsCheck() bool {
+	return Bool(c.productVariables.Skip_boot_jars_check)
+}
+
 func (c *config) Fuchsia() bool {
 	return Bool(c.productVariables.Fuchsia)
 }
@@ -1341,6 +1346,11 @@ func (l *ConfiguredJarList) Jar(idx int) string {
 	return l.jars[idx]
 }
 
+// Apex component of idx-th pair on the list.
+func (l *ConfiguredJarList) Apex(idx int) string {
+	return l.apexes[idx]
+}
+
 // If the list contains a pair with the given jar.
 func (l *ConfiguredJarList) ContainsJar(jar string) bool {
 	return InList(jar, l.jars)
@@ -1538,3 +1548,11 @@ func (c *config) BootJars() []string {
 		return list
 	}).([]string)
 }
+
+func (c *config) NonUpdatableBootJars() ConfiguredJarList {
+	return c.productVariables.BootJars
+}
+
+func (c *config) UpdatableBootJars() ConfiguredJarList {
+	return c.productVariables.UpdatableBootJars
+}
diff --git a/android/variable.go b/android/variable.go
index 7999f0fb0..a9a9c87c2 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -224,6 +224,7 @@ type productVariables struct {
 	Unbundled_build              *bool `json:",omitempty"`
 	Unbundled_build_apps         *bool `json:",omitempty"`
 	Always_use_prebuilt_sdks     *bool `json:",omitempty"`
+	Skip_boot_jars_check         *bool `json:",omitempty"`
 	Malloc_not_svelte            *bool `json:",omitempty"`
 	Malloc_zero_contents         *bool `json:",omitempty"`
 	Malloc_pattern_fill_contents *bool `json:",omitempty"`
diff --git a/java/Android.bp b/java/Android.bp
index 92e8ca458..9e8dc786b 100644
--- a/java/Android.bp
+++ b/java/Android.bp
@@ -22,6 +22,7 @@ bootstrap_go_package {
         "androidmk.go",
         "app_builder.go",
         "app.go",
+        "boot_jars.go",
         "builder.go",
         "device_host_converter.go",
         "dex.go",
diff --git a/java/androidmk.go b/java/androidmk.go
index c21c83ab5..e1a661fc1 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -217,10 +217,6 @@ func (prebuilt *DexImport) AndroidMkEntries() []android.AndroidMkEntries {
 			func(entries *android.AndroidMkEntries) {
 				if prebuilt.dexJarFile != nil {
 					entries.SetPath("LOCAL_SOONG_DEX_JAR", prebuilt.dexJarFile)
-					// TODO(b/125517186): export the dex jar as a classes jar to match some mis-uses in Make until
-					// boot_jars_package_check.mk can check dex jars.
-					entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.dexJarFile)
-					entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.dexJarFile)
 				}
 				if len(prebuilt.dexpreopter.builtInstalled) > 0 {
 					entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", prebuilt.dexpreopter.builtInstalled)
diff --git a/java/boot_jars.go b/java/boot_jars.go
new file mode 100644
index 000000000..900eb7adf
--- /dev/null
+++ b/java/boot_jars.go
@@ -0,0 +1,123 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+	"android/soong/android"
+)
+
+func init() {
+	android.RegisterSingletonType("boot_jars", bootJarsSingletonFactory)
+}
+
+func bootJarsSingletonFactory() android.Singleton {
+	return &bootJarsSingleton{}
+}
+
+type bootJarsSingleton struct{}
+
+func populateMapFromConfiguredJarList(ctx android.SingletonContext, moduleToApex map[string]string, list android.ConfiguredJarList, name string) bool {
+	for i := 0; i < list.Len(); i++ {
+		module := list.Jar(i)
+		// Ignore jacocoagent it is only added when instrumenting and so has no impact on
+		// app compatibility.
+		if module == "jacocoagent" {
+			continue
+		}
+		apex := list.Apex(i)
+		if existing, ok := moduleToApex[module]; ok {
+			ctx.Errorf("Configuration property %q is invalid as it contains multiple references to module (%s) in APEXes (%s and %s)",
+				module, existing, apex)
+			return false
+		}
+
+		moduleToApex[module] = apex
+	}
+
+	return true
+}
+
+func (b *bootJarsSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+	config := ctx.Config()
+	if config.SkipBootJarsCheck() {
+		return
+	}
+
+	// Populate a map from module name to APEX from the boot jars. If there is a problem
+	// such as duplicate modules then fail and return immediately.
+	moduleToApex := make(map[string]string)
+	if !populateMapFromConfiguredJarList(ctx, moduleToApex, config.NonUpdatableBootJars(), "BootJars") ||
+		!populateMapFromConfiguredJarList(ctx, moduleToApex, config.UpdatableBootJars(), "UpdatableBootJars") {
+		return
+	}
+
+	// Map from module name to the correct apex variant.
+	nameToApexVariant := make(map[string]android.Module)
+
+	// Scan all the modules looking for the module/apex variants corresponding to the
+	// boot jars.
+	ctx.VisitAllModules(func(module android.Module) {
+		name := ctx.ModuleName(module)
+		if apex, ok := moduleToApex[name]; ok {
+			apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo)
+			if (apex == "platform" && apexInfo.IsForPlatform()) || apexInfo.InApex(apex) {
+				// The module name/apex variant should be unique in the system but double check
+				// just in case something has gone wrong.
+				if existing, ok := nameToApexVariant[name]; ok {
+					ctx.Errorf("found multiple variants matching %s:%s: %q and %q", apex, name, existing, module)
+				}
+				nameToApexVariant[name] = module
+			}
+		}
+	})
+
+	timestamp := android.PathForOutput(ctx, "boot-jars-package-check/stamp")
+
+	rule := android.NewRuleBuilder()
+	checkBootJars := rule.Command().BuiltTool(ctx, "check_boot_jars").
+		Input(android.PathForSource(ctx, "build/soong/scripts/check_boot_jars/package_allowed_list.txt"))
+
+	// If this is not an unbundled build and missing dependencies are not allowed
+	// then all the boot jars listed must have been found.
+	strict := !config.UnbundledBuild() && !config.AllowMissingDependencies()
+
+	// Iterate over the module names on the boot classpath in order
+	for _, name := range android.SortedStringKeys(moduleToApex) {
+		if apexVariant, ok := nameToApexVariant[name]; ok {
+			if dep, ok := apexVariant.(Dependency); ok {
+				// Add the implementation jars for the module to be checked. This uses implementation
+				// and resources jar as that is what the previous make based check uses.
+				for _, jar := range dep.ImplementationAndResourcesJars() {
+					checkBootJars.Input(jar)
+				}
+			} else if _, ok := apexVariant.(*DexImport); ok {
+				// TODO(b/171479578): ignore deximport when doing package check until boot_jars.go can check dex jars.
+			} else {
+				ctx.Errorf("module %q is of type %q which is not supported as a boot jar", name, ctx.ModuleType(apexVariant))
+			}
+		} else if strict {
+			ctx.Errorf("boot jars package check failed as it could not find module %q for apex %q", name, moduleToApex[name])
+		}
+	}
+
+	checkBootJars.Text("&& touch").Output(timestamp)
+	rule.Build(pctx, ctx, "boot_jars_package_check", "check boot jar packages")
+
+	// The check-boot-jars phony target depends on the timestamp created if the check succeeds.
+	ctx.Phony("check-boot-jars", timestamp)
+
+	// The droidcore phony target depends on the check-boot-jars phony target
+	ctx.Phony("droidcore", android.PathForPhony(ctx, "check-boot-jars"))
+}
diff --git a/scripts/Android.bp b/scripts/Android.bp
index 92f5c5335..dd03f283a 100644
--- a/scripts/Android.bp
+++ b/scripts/Android.bp
@@ -1,4 +1,20 @@
 python_binary_host {
+    name: "check_boot_jars",
+    main: "check_boot_jars/check_boot_jars.py",
+    srcs: [
+        "check_boot_jars/check_boot_jars.py",
+    ],
+    version: {
+        py2: {
+            enabled: true,
+        },
+        py3: {
+            enabled: false,
+        },
+    },
+}
+
+python_binary_host {
     name: "manifest_fixer",
     main: "manifest_fixer.py",
     srcs: [
diff --git a/scripts/check_boot_jars/check_boot_jars.py b/scripts/check_boot_jars/check_boot_jars.py
new file mode 100755
index 000000000..cf4ef2782
--- /dev/null
+++ b/scripts/check_boot_jars/check_boot_jars.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+"""
+Check boot jars.
+
+Usage: check_boot_jars.py <package_allow_list_file> <jar1> <jar2> ...
+"""
+import logging
+import os.path
+import re
+import subprocess
+import sys
+
+
+# The compiled allow list RE.
+allow_list_re = None
+
+
+def LoadAllowList(filename):
+  """ Load and compile allow list regular expressions from filename.
+  """
+  lines = []
+  with open(filename, 'r') as f:
+    for line in f:
+      line = line.strip()
+      if not line or line.startswith('#'):
+        continue
+      lines.append(line)
+  combined_re = r'^(%s)$' % '|'.join(lines)
+  global allow_list_re
+  try:
+    allow_list_re = re.compile(combined_re)
+  except re.error:
+    logging.exception(
+        'Cannot compile package allow list regular expression: %r',
+        combined_re)
+    allow_list_re = None
+    return False
+  return True
+
+
+def CheckJar(allow_list_path, jar):
+  """Check a jar file.
+  """
+  # Get the list of files inside the jar file.
+  p = subprocess.Popen(args='jar tf %s' % jar,
+      stdout=subprocess.PIPE, shell=True)
+  stdout, _ = p.communicate()
+  if p.returncode != 0:
+    return False
+  items = stdout.split()
+  classes = 0
+  for f in items:
+    if f.endswith('.class'):
+      classes += 1
+      package_name = os.path.dirname(f)
+      package_name = package_name.replace('/', '.')
+      if not package_name or not allow_list_re.match(package_name):
+        print >> sys.stderr, ('Error: %s contains class file %s, whose package name %s is empty or'
+                              ' not in the allow list %s of packages allowed on the bootclasspath.'
+                              % (jar, f, package_name, allow_list_path))
+        return False
+  if classes == 0:
+    print >> sys.stderr, ('Error: %s does not contain any class files.' % jar)
+    return False
+  return True
+
+
+def main(argv):
+  if len(argv) < 2:
+    print __doc__
+    return 1
+  allow_list_path = argv[0]
+
+  if not LoadAllowList(allow_list_path):
+    return 1
+
+  passed = True
+  for jar in argv[1:]:
+    if not CheckJar(allow_list_path, jar):
+      passed = False
+  if not passed:
+    return 1
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))
diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt
new file mode 100644
index 000000000..18ab427b5
--- /dev/null
+++ b/scripts/check_boot_jars/package_allowed_list.txt
@@ -0,0 +1,248 @@
+# Boot jar package name allowed list.
+# Each line is interpreted as a regular expression.
+
+###################################################
+# core-libart.jar & core-oj.jar
+java\.awt\.font
+java\.beans
+java\.io
+java\.lang
+java\.lang\.annotation
+java\.lang\.invoke
+java\.lang\.ref
+java\.lang\.reflect
+java\.math
+java\.net
+java\.nio
+java\.nio\.file
+java\.nio\.file\.spi
+java\.nio\.file\.attribute
+java\.nio\.channels
+java\.nio\.channels\.spi
+java\.nio\.charset
+java\.nio\.charset\.spi
+java\.security
+java\.security\.acl
+java\.security\.cert
+java\.security\.interfaces
+java\.security\.spec
+java\.sql
+java\.text
+java\.text\.spi
+java\.time
+java\.time\.chrono
+java\.time\.format
+java\.time\.temporal
+java\.time\.zone
+java\.util
+java\.util\.concurrent
+java\.util\.concurrent\.atomic
+java\.util\.concurrent\.locks
+java\.util\.function
+java\.util\.jar
+java\.util\.logging
+java\.util\.prefs
+java\.util\.regex
+java\.util\.spi
+java\.util\.stream
+java\.util\.zip
+# TODO: Remove javax.annotation.processing if possible, see http://b/132338110:
+javax\.annotation\.processing
+javax\.crypto
+javax\.crypto\.interfaces
+javax\.crypto\.spec
+javax\.net
+javax\.net\.ssl
+javax\.security\.auth
+javax\.security\.auth\.callback
+javax\.security\.auth\.login
+javax\.security\.auth\.x500
+javax\.security\.cert
+javax\.sql
+javax\.xml
+javax\.xml\.datatype
+javax\.xml\.namespace
+javax\.xml\.parsers
+javax\.xml\.transform
+javax\.xml\.transform\.dom
+javax\.xml\.transform\.sax
+javax\.xml\.transform\.stream
+javax\.xml\.validation
+javax\.xml\.xpath
+jdk\.internal\.util
+jdk\.internal\.vm\.annotation
+jdk\.net
+org\.w3c\.dom
+org\.w3c\.dom\.ls
+org\.w3c\.dom\.traversal
+# OpenJdk internal implementation.
+sun\.invoke\.util
+sun\.invoke\.empty
+sun\.misc
+sun\.util.*
+sun\.text.*
+sun\.security.*
+sun\.reflect.*
+sun\.nio.*
+sun\.net.*
+com\.sun\..*
+
+# TODO: Move these internal org.apache.harmony classes to libcore.*
+org\.apache\.harmony\.crypto\.internal
+org\.apache\.harmony\.dalvik
+org\.apache\.harmony\.dalvik\.ddmc
+org\.apache\.harmony\.luni\.internal\.util
+org\.apache\.harmony\.security
+org\.apache\.harmony\.security\.asn1
+org\.apache\.harmony\.security\.fortress
+org\.apache\.harmony\.security\.pkcs10
+org\.apache\.harmony\.security\.pkcs7
+org\.apache\.harmony\.security\.pkcs8
+org\.apache\.harmony\.security\.provider\.crypto
+org\.apache\.harmony\.security\.utils
+org\.apache\.harmony\.security\.x501
+org\.apache\.harmony\.security\.x509
+org\.apache\.harmony\.security\.x509\.tsp
+org\.apache\.harmony\.xml
+org\.apache\.harmony\.xml\.dom
+org\.apache\.harmony\.xml\.parsers
+
+org\.json
+org\.xmlpull\.v1
+org\.xmlpull\.v1\.sax2
+
+# TODO:  jarjar org.kxml2.io to com.android org\.kxml2\.io
+org\.kxml2\.io
+org\.xml
+org\.xml\.sax
+org\.xml\.sax\.ext
+org\.xml\.sax\.helpers
+
+dalvik\..*
+libcore\..*
+android\..*
+com\.android\..*
+###################################################
+# android.test.base.jar
+junit\.extensions
+junit\.framework
+android\.test
+android\.test\.suitebuilder\.annotation
+
+
+###################################################
+# ext.jar
+# TODO: jarjar javax.sip to com.android
+javax\.sip
+javax\.sip\.address
+javax\.sip\.header
+javax\.sip\.message
+
+# TODO: jarjar org.apache.commons to com.android
+org\.apache\.commons\.codec
+org\.apache\.commons\.codec\.binary
+org\.apache\.commons\.codec\.language
+org\.apache\.commons\.codec\.net
+org\.apache\.commons\.logging
+org\.apache\.commons\.logging\.impl
+org\.apache\.http
+org\.apache\.http\.auth
+org\.apache\.http\.auth\.params
+org\.apache\.http\.client
+org\.apache\.http\.client\.entity
+org\.apache\.http\.client\.methods
+org\.apache\.http\.client\.params
+org\.apache\.http\.client\.protocol
+org\.apache\.http\.client\.utils
+org\.apache\.http\.conn
+org\.apache\.http\.conn\.params
+org\.apache\.http\.conn\.routing
+org\.apache\.http\.conn\.scheme
+org\.apache\.http\.conn\.ssl
+org\.apache\.http\.conn\.util
+org\.apache\.http\.cookie
+org\.apache\.http\.cookie\.params
+org\.apache\.http\.entity
+org\.apache\.http\.impl
+org\.apache\.http\.impl\.auth
+org\.apache\.http\.impl\.client
+org\.apache\.http\.impl\.client
+org\.apache\.http\.impl\.conn
+org\.apache\.http\.impl\.conn\.tsccm
+org\.apache\.http\.impl\.cookie
+org\.apache\.http\.impl\.entity
+org\.apache\.http\.impl\.io
+org\.apache\.http\.impl\.io
+org\.apache\.http\.io
+org\.apache\.http\.message
+org\.apache\.http\.params
+org\.apache\.http\.protocol
+org\.apache\.http\.util
+
+# TODO: jarjar gov.nist to com.android
+gov\.nist\.core
+gov\.nist\.core\.net
+gov\.nist\.javax\.sip
+gov\.nist\.javax\.sip\.address
+gov\.nist\.javax\.sip\.clientauthutils
+gov\.nist\.javax\.sip\.header
+gov\.nist\.javax\.sip\.header\.extensions
+gov\.nist\.javax\.sip\.header\.ims
+gov\.nist\.javax\.sip\.message
+gov\.nist\.javax\.sip\.parser
+gov\.nist\.javax\.sip\.parser\.extensions
+gov\.nist\.javax\.sip\.parser\.ims
+gov\.nist\.javax\.sip\.stack
+
+org\.ccil\.cowan\.tagsoup
+org\.ccil\.cowan\.tagsoup\.jaxp
+
+###################################################
+# framework.jar
+javax\.microedition\.khronos\.opengles
+javax\.microedition\.khronos\.egl
+
+android
+
+###################################################
+# apache-xml.jar
+org\.apache\.xml\.res
+org\.apache\.xml\.utils
+org\.apache\.xml\.utils\.res
+org\.apache\.xml\.dtm
+org\.apache\.xml\.dtm\.ref
+org\.apache\.xml\.dtm\.ref\.dom2dtm
+org\.apache\.xml\.dtm\.ref\.sax2dtm
+org\.apache\.xml\.serializer
+org\.apache\.xml\.serializer\.utils
+org\.apache\.xml\.serializer\.dom3
+org\.apache\.xpath
+org\.apache\.xpath\.operations
+org\.apache\.xpath\.domapi
+org\.apache\.xpath\.functions
+org\.apache\.xpath\.res
+org\.apache\.xpath\.axes
+org\.apache\.xpath\.objects
+org\.apache\.xpath\.patterns
+org\.apache\.xpath\.jaxp
+org\.apache\.xpath\.compiler
+org\.apache\.xalan
+org\.apache\.xalan\.res
+org\.apache\.xalan\.templates
+org\.apache\.xalan\.serialize
+org\.apache\.xalan\.extensions
+org\.apache\.xalan\.processor
+org\.apache\.xalan\.transformer
+org\.apache\.xalan\.xslt
+
+###################################################
+# Packages in the google namespace across all bootclasspath jars.
+com\.google\.android\..*
+com\.google\.vr\.platform.*
+com\.google\.i18n\.phonenumbers\..*
+com\.google\.i18n\.phonenumbers
+
+###################################################
+# Packages used for Android in Chrome OS
+org\.chromium\.arc
+org\.chromium\.arc\..*