diff options
| -rw-r--r-- | android/bazel.go | 13 | ||||
| -rw-r--r-- | android/hooks.go | 17 | ||||
| -rw-r--r-- | android/register.go | 6 | ||||
| -rw-r--r-- | bp2build/android_app_conversion_test.go | 39 | ||||
| -rw-r--r-- | java/droidstubs.go | 4 | ||||
| -rw-r--r-- | java/java.go | 12 | ||||
| -rw-r--r-- | mk2rbc/mk2rbc.go | 21 | ||||
| -rw-r--r-- | mk2rbc/mk2rbc_test.go | 37 | ||||
| -rw-r--r-- | mk2rbc/node.go | 21 | ||||
| -rw-r--r-- | mk2rbc/variable.go | 54 | ||||
| -rw-r--r-- | rust/config/allowed_list.go | 1 | ||||
| -rw-r--r-- | scripts/hiddenapi/Android.bp | 30 | ||||
| -rw-r--r-- | scripts/hiddenapi/signature_trie.py | 256 | ||||
| -rwxr-xr-x | scripts/hiddenapi/signature_trie_test.py | 192 | ||||
| -rwxr-xr-x | scripts/hiddenapi/verify_overlaps.py | 254 | ||||
| -rwxr-xr-x | scripts/hiddenapi/verify_overlaps_test.py | 61 |
16 files changed, 667 insertions, 351 deletions
diff --git a/android/bazel.go b/android/bazel.go index 342b840da..97226c6da 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -485,11 +485,12 @@ var ( "conscrypt", // b/210751803, we don't handle path property for filegroups "conscrypt-for-host", // b/210751803, we don't handle path property for filegroups - "host-libprotobuf-java-lite", // b/217236083, java_library cannot have deps without srcs - "host-libprotobuf-java-micro", // b/217236083, java_library cannot have deps without srcs - "host-libprotobuf-java-nano", // b/217236083, java_library cannot have deps without srcs - "error_prone_core", // b/217236083, java_library cannot have deps without srcs - "bouncycastle-host", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-lite", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-micro", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-nano", // b/217236083, java_library cannot have deps without srcs + "error_prone_core", // b/217236083, java_library cannot have deps without srcs + "bouncycastle-host", // b/217236083, java_library cannot have deps without srcs + "mockito-robolectric-prebuilt", // b/217236083, java_library cannot have deps without srcs "apex_manifest_proto_java", // b/215230097, we don't handle .proto files in java_library srcs attribute @@ -559,6 +560,8 @@ var ( "dex2oat-script", // depends on unconverted modules: dex2oat "error_prone_checkerframework_dataflow_nullaway", // TODO(b/219908977): "Error in fail: deps not allowed without srcs; move to runtime_deps?" + + "libprotobuf-java-nano", // b/220869005, depends on non-public_current SDK } // Per-module denylist of cc_library modules to only generate the static diff --git a/android/hooks.go b/android/hooks.go index bded76467..5e3a4a7e7 100644 --- a/android/hooks.go +++ b/android/hooks.go @@ -15,7 +15,10 @@ package android import ( + "fmt" + "path" "reflect" + "runtime" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -88,7 +91,19 @@ func (l *loadHookContext) PrependProperties(props ...interface{}) { func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { inherited := []interface{}{&l.Module().base().commonProperties} - module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module) + + var typeName string + if typeNameLookup, ok := ModuleTypeByFactory()[reflect.ValueOf(factory)]; ok { + typeName = typeNameLookup + } else { + factoryPtr := reflect.ValueOf(factory).Pointer() + factoryFunc := runtime.FuncForPC(factoryPtr) + filePath, _ := factoryFunc.FileLine(factoryPtr) + typeName = fmt.Sprintf("%s_%s", path.Base(filePath), factoryFunc.Name()) + } + typeName = typeName + "_loadHookModule" + + module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module) if l.Module().base().variableProperties != nil && module.base().variableProperties != nil { src := l.Module().base().variableProperties diff --git a/android/register.go b/android/register.go index 10e14e04d..c50583322 100644 --- a/android/register.go +++ b/android/register.go @@ -59,6 +59,7 @@ func (t moduleType) register(ctx *Context) { var moduleTypes []moduleType var moduleTypesForDocs = map[string]reflect.Value{} +var moduleTypeByFactory = map[reflect.Value]string{} type singleton struct { // True if this should be registered as a pre-singleton, false otherwise. @@ -140,6 +141,7 @@ func RegisterModuleType(name string, factory ModuleFactory) { // RegisterModuleType was a lambda. func RegisterModuleTypeForDocs(name string, factory reflect.Value) { moduleTypesForDocs[name] = factory + moduleTypeByFactory[factory] = name } func RegisterSingletonType(name string, factory SingletonFactory) { @@ -228,6 +230,10 @@ func ModuleTypeFactoriesForDocs() map[string]reflect.Value { return moduleTypesForDocs } +func ModuleTypeByFactory() map[reflect.Value]string { + return moduleTypeByFactory +} + // Interface for registering build components. // // Provided to allow registration of build components to be shared between the runtime diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go index 42c1a5458..b6095b2ef 100644 --- a/bp2build/android_app_conversion_test.go +++ b/bp2build/android_app_conversion_test.go @@ -94,3 +94,42 @@ android_app { }), }}) } + +func TestAndroidAppArchVariantSrcs(t *testing.T) { + runAndroidAppTestCase(t, bp2buildTestCase{ + description: "Android app - arch variant srcs", + moduleTypeUnderTest: "android_app", + moduleTypeUnderTestFactory: java.AndroidAppFactory, + filesystem: map[string]string{ + "arm.java": "", + "x86.java": "", + "res/res.png": "", + "AndroidManifest.xml": "", + }, + blueprint: ` +android_app { + name: "TestApp", + sdk_version: "current", + arch: { + arm: { + srcs: ["arm.java"], + }, + x86: { + srcs: ["x86.java"], + } + } +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("android_binary", "TestApp", attrNameToString{ + "srcs": `select({ + "//build/bazel/platforms/arch:arm": ["arm.java"], + "//build/bazel/platforms/arch:x86": ["x86.java"], + "//conditions:default": [], + })`, + "manifest": `"AndroidManifest.xml"`, + "resource_files": `["res/res.png"]`, + "deps": `["//prebuilts/sdk:public_current_android_sdk_java_import"]`, + }), + }}) +} diff --git a/java/droidstubs.go b/java/droidstubs.go index 272cf1e6e..2921c3e82 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -477,7 +477,9 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi Flag("--format=v2"). FlagWithArg("--repeat-errors-max ", "10"). FlagWithArg("--hide ", "UnresolvedImport"). - FlagWithArg("--hide ", "InvalidNullability") + FlagWithArg("--hide ", "InvalidNullability"). + // b/223382732 + FlagWithArg("--hide ", "ChangedDefault") return cmd } diff --git a/java/java.go b/java/java.go index 0a35908dd..895ce7af1 100644 --- a/java/java.go +++ b/java/java.go @@ -2011,8 +2011,16 @@ type javaLibraryAttributes struct { } func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) *javaLibraryAttributes { - //TODO(b/209577426): Support multiple arch variants - srcs := bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs)) + var srcs bazel.LabelListAttribute + archVariantProps := m.GetArchVariantProperties(ctx, &CommonProperties{}) + for axis, configToProps := range archVariantProps { + for config, _props := range configToProps { + if archProps, ok := _props.(*CommonProperties); ok { + archSrcs := android.BazelLabelForModuleSrcExcludes(ctx, archProps.Srcs, archProps.Exclude_srcs) + srcs.SetSelectValue(axis, config, archSrcs) + } + } + } javaSrcPartition := "java" protoSrcPartition := "proto" diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index d108a0d04..d8b88b2cd 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -50,15 +50,12 @@ const ( soongNsPrefix = "SOONG_CONFIG_" // And here are the functions and variables: - cfnGetCfg = baseName + ".cfg" - cfnMain = baseName + ".product_configuration" - cfnBoardMain = baseName + ".board_configuration" - cfnPrintVars = baseName + ".printvars" - cfnWarning = baseName + ".warning" - cfnLocalAppend = baseName + ".local_append" - cfnLocalSetDefault = baseName + ".local_set_default" - cfnInherit = baseName + ".inherit" - cfnSetListDefault = baseName + ".setdefault" + cfnGetCfg = baseName + ".cfg" + cfnMain = baseName + ".product_configuration" + cfnBoardMain = baseName + ".board_configuration" + cfnPrintVars = baseName + ".printvars" + cfnInherit = baseName + ".inherit" + cfnSetListDefault = baseName + ".setdefault" ) const ( @@ -571,11 +568,7 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode case "=", ":=": asgn.flavor = asgnSet case "+=": - if asgn.previous == nil && !asgn.lhs.isPreset() { - asgn.flavor = asgnMaybeAppend - } else { - asgn.flavor = asgnAppend - } + asgn.flavor = asgnAppend case "?=": asgn.flavor = asgnMaybeSet default: diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index 556dcaa0c..35c54d268 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -901,6 +901,43 @@ def init(g, handle): `, }, { + desc: "assigment setdefaults", + mkname: "product.mk", + in: ` +# All of these should have a setdefault because they're self-referential and not defined before +PRODUCT_LIST1 = a $(PRODUCT_LIST1) +PRODUCT_LIST2 ?= a $(PRODUCT_LIST2) +PRODUCT_LIST3 += a + +# Now doing them again should not have a setdefault because they've already been set +PRODUCT_LIST1 = a $(PRODUCT_LIST1) +PRODUCT_LIST2 ?= a $(PRODUCT_LIST2) +PRODUCT_LIST3 += a +`, + expected: `# All of these should have a setdefault because they're self-referential and not defined before +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.setdefault(handle, "PRODUCT_LIST1") + cfg["PRODUCT_LIST1"] = (["a"] + + cfg.get("PRODUCT_LIST1", [])) + if cfg.get("PRODUCT_LIST2") == None: + rblf.setdefault(handle, "PRODUCT_LIST2") + cfg["PRODUCT_LIST2"] = (["a"] + + cfg.get("PRODUCT_LIST2", [])) + rblf.setdefault(handle, "PRODUCT_LIST3") + cfg["PRODUCT_LIST3"] += ["a"] + # Now doing them again should not have a setdefault because they've already been set + cfg["PRODUCT_LIST1"] = (["a"] + + cfg["PRODUCT_LIST1"]) + if cfg.get("PRODUCT_LIST2") == None: + cfg["PRODUCT_LIST2"] = (["a"] + + cfg["PRODUCT_LIST2"]) + cfg["PRODUCT_LIST3"] += ["a"] +`, + }, + { desc: "soong namespace assignments", mkname: "product.mk", in: ` diff --git a/mk2rbc/node.go b/mk2rbc/node.go index 5d98d7bc1..9d5af91c4 100644 --- a/mk2rbc/node.go +++ b/mk2rbc/node.go @@ -184,10 +184,9 @@ type assignmentFlavor int const ( // Assignment flavors - asgnSet assignmentFlavor = iota // := or = - asgnMaybeSet assignmentFlavor = iota // ?= and variable may be unset - asgnAppend assignmentFlavor = iota // += and variable has been set before - asgnMaybeAppend assignmentFlavor = iota // += and variable may be unset + asgnSet assignmentFlavor = iota // := or = + asgnMaybeSet assignmentFlavor = iota // ?= + asgnAppend assignmentFlavor = iota // += ) type assignmentNode struct { @@ -215,6 +214,20 @@ func (asgn *assignmentNode) emit(gctx *generationContext) { } } +func (asgn *assignmentNode) isSelfReferential() bool { + if asgn.flavor == asgnAppend { + return true + } + isSelfReferential := false + asgn.value.transform(func(expr starlarkExpr) starlarkExpr { + if ref, ok := expr.(*variableRefExpr); ok && ref.ref.name() == asgn.lhs.name() { + isSelfReferential = true + } + return nil + }) + return isSelfReferential +} + type exprNode struct { expr starlarkExpr } diff --git a/mk2rbc/variable.go b/mk2rbc/variable.go index 6805744a7..3241a3854 100644 --- a/mk2rbc/variable.go +++ b/mk2rbc/variable.go @@ -97,29 +97,27 @@ func (pcv productConfigVariable) emitSet(gctx *generationContext, asgn *assignme gctx.newLine() } + // If we are not sure variable has been assigned before, emit setdefault + needsSetDefault := asgn.previous == nil && !pcv.isPreset() && asgn.isSelfReferential() + switch asgn.flavor { case asgnSet: - isSelfReferential := false - asgn.value.transform(func(expr starlarkExpr) starlarkExpr { - if ref, ok := expr.(*variableRefExpr); ok && ref.ref.name() == pcv.name() { - isSelfReferential = true - } - return nil - }) - if isSelfReferential { + if needsSetDefault { emitSetDefault() } emitAssignment() case asgnAppend: - emitAppend() - case asgnMaybeAppend: - // If we are not sure variable has been assigned before, emit setdefault - emitSetDefault() + if needsSetDefault { + emitSetDefault() + } emitAppend() case asgnMaybeSet: gctx.writef("if cfg.get(%q) == None:", pcv.nam) gctx.indentLevel++ gctx.newLine() + if needsSetDefault { + emitSetDefault() + } emitAssignment() gctx.indentLevel-- } @@ -134,7 +132,7 @@ func (pcv productConfigVariable) emitGet(gctx *generationContext, isDefined bool } func (pcv productConfigVariable) emitDefined(gctx *generationContext) { - gctx.writef("g.get(%q) != None", pcv.name()) + gctx.writef("cfg.get(%q) != None", pcv.name()) } type otherGlobalVariable struct { @@ -159,20 +157,30 @@ func (scv otherGlobalVariable) emitSet(gctx *generationContext, asgn *assignment value.emit(gctx) } + // If we are not sure variable has been assigned before, emit setdefault + needsSetDefault := asgn.previous == nil && !scv.isPreset() && asgn.isSelfReferential() + switch asgn.flavor { case asgnSet: + if needsSetDefault { + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + } emitAssignment() case asgnAppend: - emitAppend() - case asgnMaybeAppend: - // If we are not sure variable has been assigned before, emit setdefault - gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) - gctx.newLine() + if needsSetDefault { + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + } emitAppend() case asgnMaybeSet: gctx.writef("if g.get(%q) == None:", scv.nam) gctx.indentLevel++ gctx.newLine() + if needsSetDefault { + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + } emitAssignment() gctx.indentLevel-- } @@ -204,7 +212,7 @@ func (lv localVariable) String() string { func (lv localVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { switch asgn.flavor { - case asgnSet: + case asgnSet, asgnMaybeSet: gctx.writef("%s = ", lv) asgn.value.emitListVarCopy(gctx) case asgnAppend: @@ -216,14 +224,6 @@ func (lv localVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { value = &toStringExpr{expr: value} } value.emit(gctx) - case asgnMaybeAppend: - gctx.writef("%s(%q, ", cfnLocalAppend, lv) - asgn.value.emit(gctx) - gctx.write(")") - case asgnMaybeSet: - gctx.writef("%s(%q, ", cfnLocalSetDefault, lv) - asgn.value.emit(gctx) - gctx.write(")") } } diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go index 14fcb028f..bc36b205f 100644 --- a/rust/config/allowed_list.go +++ b/rust/config/allowed_list.go @@ -24,6 +24,7 @@ var ( "packages/modules/DnsResolver", "packages/modules/Uwb", "packages/modules/Virtualization", + "platform_testing/tests/codecoverage/native/rust", "prebuilts/rust", "system/core/libstats/pull_rust", "system/extras/profcollectd", diff --git a/scripts/hiddenapi/Android.bp b/scripts/hiddenapi/Android.bp index 7ffda62d1..8a47c5dd8 100644 --- a/scripts/hiddenapi/Android.bp +++ b/scripts/hiddenapi/Android.bp @@ -69,10 +69,37 @@ python_test_host { }, } +python_library_host { + name: "signature_trie", + srcs: ["signature_trie.py"], +} + +python_test_host { + name: "signature_trie_test", + main: "signature_trie_test.py", + srcs: ["signature_trie_test.py"], + libs: ["signature_trie"], + version: { + py2: { + enabled: false, + }, + py3: { + enabled: true, + embedded_launcher: true, + }, + }, + test_options: { + unit_test: true, + }, +} + python_binary_host { name: "verify_overlaps", main: "verify_overlaps.py", srcs: ["verify_overlaps.py"], + libs: [ + "signature_trie", + ], version: { py2: { enabled: false, @@ -91,6 +118,9 @@ python_test_host { "verify_overlaps.py", "verify_overlaps_test.py", ], + libs: [ + "signature_trie", + ], version: { py2: { enabled: false, diff --git a/scripts/hiddenapi/signature_trie.py b/scripts/hiddenapi/signature_trie.py new file mode 100644 index 000000000..2ea8c49b1 --- /dev/null +++ b/scripts/hiddenapi/signature_trie.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Verify that one set of hidden API flags is a subset of another.""" +import dataclasses +import typing + +from itertools import chain + + +@dataclasses.dataclass() +class Node: + + def values(self, selector): + """Get the values from a set of selected nodes. + + :param selector: a function that can be applied to a key in the nodes + attribute to determine whether to return its values. + + :return: A list of iterables of all the values associated with + this node and its children. + """ + raise NotImplementedError("Please Implement this method") + + def append_values(self, values, selector): + """Append the values associated with this node and its children. + + For each item (key, child) in nodes the child node's values are returned + if and only if the selector returns True when called on its key. A child + node's values are all the values associated with it and all its + descendant nodes. + + :param selector: a function that can be applied to a key in the nodes + attribute to determine whether to return its values. + :param values: a list of a iterables of values. + """ + raise NotImplementedError("Please Implement this method") + + +# pylint: disable=line-too-long +@dataclasses.dataclass() +class InteriorNode(Node): + """An interior node in a trie. + + Each interior node has a dict that maps from an element of a signature to + either another interior node or a leaf. Each interior node represents either + a package, class or nested class. Class members are represented by a Leaf. + + Associating the set of flags [public-api] with the signature + "Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following + nodes to be created: + Node() + ^- package:java -> Node() + ^- package:lang -> Node() + ^- class:Object -> Node() + ^- member:String()Ljava/lang/String; -> Leaf([public-api]) + + Associating the set of flags [blocked,core-platform-api] with the signature + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;" + will cause the following nodes to be created: + Node() + ^- package:java -> Node() + ^- package:lang -> Node() + ^- class:Character -> Node() + ^- class:UnicodeScript -> Node() + ^- member:of(I)Ljava/lang/Character$UnicodeScript; + -> Leaf([blocked,core-platform-api]) + """ + + # pylint: enable=line-too-long + + # A dict from an element of the signature to the Node/Leaf containing the + # next element/value. + nodes: typing.Dict[str, Node] = dataclasses.field(default_factory=dict) + + # pylint: disable=line-too-long + @staticmethod + def signature_to_elements(signature): + """Split a signature or a prefix into a number of elements: + + 1. The packages (excluding the leading L preceding the first package). + 2. The class names, from outermost to innermost. + 3. The member signature. + e.g. + Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; + will be broken down into these elements: + 1. package:java + 2. package:lang + 3. class:Character + 4. class:UnicodeScript + 5. member:of(I)Ljava/lang/Character$UnicodeScript; + """ + # Remove the leading L. + # - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; + text = signature.removeprefix("L") + # Split the signature between qualified class name and the class member + # signature. + # 0 - java/lang/Character$UnicodeScript + # 1 - of(I)Ljava/lang/Character$UnicodeScript; + parts = text.split(";->") + member = parts[1:] + # Split the qualified class name into packages, and class name. + # 0 - java + # 1 - lang + # 2 - Character$UnicodeScript + elements = parts[0].split("/") + packages = elements[0:-1] + class_name = elements[-1] + if class_name in ("*", "**"): # pylint: disable=no-else-return + # Cannot specify a wildcard and target a specific member + if len(member) != 0: + raise Exception(f"Invalid signature {signature}: contains " + f"wildcard {class_name} and " + f"member signature {member[0]}") + wildcard = [class_name] + # Assemble the parts into a single list, adding prefixes to identify + # the different parts. + # 0 - package:java + # 1 - package:lang + # 2 - * + return list(chain(["package:" + x for x in packages], wildcard)) + else: + # Split the class name into outer / inner classes + # 0 - Character + # 1 - UnicodeScript + classes = class_name.split("$") + # Assemble the parts into a single list, adding prefixes to identify + # the different parts. + # 0 - package:java + # 1 - package:lang + # 2 - class:Character + # 3 - class:UnicodeScript + # 4 - member:of(I)Ljava/lang/Character$UnicodeScript; + return list( + chain(["package:" + x for x in packages], + ["class:" + x for x in classes], + ["member:" + x for x in member])) + + # pylint: enable=line-too-long + + def add(self, signature, value): + """Associate the value with the specific signature. + + :param signature: the member signature + :param value: the value to associated with the signature + :return: n/a + """ + # Split the signature into elements. + elements = self.signature_to_elements(signature) + # Find the Node associated with the deepest class. + node = self + for element in elements[:-1]: + if element in node.nodes: + node = node.nodes[element] + else: + next_node = InteriorNode() + node.nodes[element] = next_node + node = next_node + # Add a Leaf containing the value and associate it with the member + # signature within the class. + last_element = elements[-1] + if not last_element.startswith("member:"): + raise Exception( + f"Invalid signature: {signature}, does not identify a " + "specific member") + if last_element in node.nodes: + raise Exception(f"Duplicate signature: {signature}") + node.nodes[last_element] = Leaf(value) + + def get_matching_rows(self, pattern): + """Get the values (plural) associated with the pattern. + + e.g. If the pattern is a full signature then this will return a list + containing the value associated with that signature. + + If the pattern is a class then this will return a list containing the + values associated with all members of that class. + + If the pattern is a package then this will return a list containing the + values associated with all the members of all the classes in that + package and sub-packages. + + If the pattern ends with "*" then the preceding part is treated as a + package and this will return a list containing the values associated + with all the members of all the classes in that package. + + If the pattern ends with "**" then the preceding part is treated + as a package and this will return a list containing the values + associated with all the members of all the classes in that package and + all sub-packages. + + :param pattern: the pattern which could be a complete signature or a + class, or package wildcard. + :return: an iterable containing all the values associated with the + pattern. + """ + elements = self.signature_to_elements(pattern) + node = self + + # Include all values from this node and all its children. + selector = lambda x: True + + last_element = elements[-1] + if last_element in ("*", "**"): + elements = elements[:-1] + if last_element == "*": + # Do not include values from sub-packages. + selector = lambda x: not x.startswith("package:") + + for element in elements: + if element in node.nodes: + node = node.nodes[element] + else: + return [] + return chain.from_iterable(node.values(selector)) + + def values(self, selector): + values = [] + self.append_values(values, selector) + return values + + def append_values(self, values, selector): + for key, node in self.nodes.items(): + if selector(key): + node.append_values(values, lambda x: True) + + + +@dataclasses.dataclass() +class Leaf(Node): + """A leaf of the trie""" + + # The value associated with this leaf. + value: typing.Any + + def values(self, selector): + return [[self.value]] + + def append_values(self, values, selector): + values.append([self.value]) + + +def signature_trie(): + return InteriorNode() diff --git a/scripts/hiddenapi/signature_trie_test.py b/scripts/hiddenapi/signature_trie_test.py new file mode 100755 index 000000000..2dc79d0af --- /dev/null +++ b/scripts/hiddenapi/signature_trie_test.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for verify_overlaps_test.py.""" +import io +import unittest + +from signature_trie import InteriorNode +from signature_trie import signature_trie + + +class TestSignatureToElements(unittest.TestCase): + + @staticmethod + def signature_to_elements(signature): + return InteriorNode.signature_to_elements(signature) + + def test_nested_inner_classes(self): + elements = [ + "package:java", + "package:lang", + "class:ProcessBuilder", + "class:Redirect", + "class:1", + "member:<init>()V", + ] + signature = "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_basic_member(self): + elements = [ + "package:java", + "package:lang", + "class:Object", + "member:hashCode()I", + ] + signature = "Ljava/lang/Object;->hashCode()I" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_double_dollar_class(self): + elements = [ + "package:java", + "package:lang", + "class:CharSequence", + "class:", + "class:ExternalSyntheticLambda0", + "member:<init>(Ljava/lang/CharSequence;)V", + ] + signature = "Ljava/lang/CharSequence$$ExternalSyntheticLambda0;" \ + "-><init>(Ljava/lang/CharSequence;)V" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_no_member(self): + elements = [ + "package:java", + "package:lang", + "class:CharSequence", + "class:", + "class:ExternalSyntheticLambda0", + ] + signature = "Ljava/lang/CharSequence$$ExternalSyntheticLambda0" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_wildcard(self): + elements = [ + "package:java", + "package:lang", + "*", + ] + signature = "java/lang/*" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_recursive_wildcard(self): + elements = [ + "package:java", + "package:lang", + "**", + ] + signature = "java/lang/**" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_no_packages_wildcard(self): + elements = [ + "*", + ] + signature = "*" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_no_packages_recursive_wildcard(self): + elements = [ + "**", + ] + signature = "**" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_non_standard_class_name(self): + elements = [ + "package:javax", + "package:crypto", + "class:extObjectInputStream", + ] + signature = "Ljavax/crypto/extObjectInputStream" + self.assertEqual(elements, self.signature_to_elements(signature)) + + def test_invalid_pattern_wildcard_and_member(self): + pattern = "Ljava/lang/*;->hashCode()I" + with self.assertRaises(Exception) as context: + self.signature_to_elements(pattern) + self.assertIn("contains wildcard * and member signature hashCode()I", + str(context.exception)) + + +class TestGetMatchingRows(unittest.TestCase): + extractInput = """ +Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; +Ljava/lang/Character;->serialVersionUID:J +Ljava/lang/Object;->hashCode()I +Ljava/lang/Object;->toString()Ljava/lang/String; +Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V +Ljava/util/zip/ZipFile;-><clinit>()V +""" + + def read_trie(self): + trie = signature_trie() + with io.StringIO(self.extractInput.strip()) as f: + for line in iter(f.readline, ""): + line = line.rstrip() + trie.add(line, line) + return trie + + def check_patterns(self, pattern, expected): + trie = self.read_trie() + self.check_node_patterns(trie, pattern, expected) + + def check_node_patterns(self, node, pattern, expected): + actual = list(node.get_matching_rows(pattern)) + actual.sort() + self.assertEqual(expected, actual) + + def test_member_pattern(self): + self.check_patterns("java/util/zip/ZipFile;-><clinit>()V", + ["Ljava/util/zip/ZipFile;-><clinit>()V"]) + + def test_class_pattern(self): + self.check_patterns("java/lang/Object", [ + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + ]) + + # pylint: disable=line-too-long + def test_nested_class_pattern(self): + self.check_patterns("java/lang/Character", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + ]) + + def test_wildcard(self): + self.check_patterns("java/lang/*", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V", + ]) + + def test_recursive_wildcard(self): + self.check_patterns("java/**", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V", + "Ljava/util/zip/ZipFile;-><clinit>()V", + ]) + + # pylint: enable=line-too-long + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/hiddenapi/verify_overlaps.py b/scripts/hiddenapi/verify_overlaps.py index 4cd7e63ac..e5214dfc8 100755 --- a/scripts/hiddenapi/verify_overlaps.py +++ b/scripts/hiddenapi/verify_overlaps.py @@ -13,239 +13,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Verify that one set of hidden API flags is a subset of another. -""" +"""Verify that one set of hidden API flags is a subset of another.""" import argparse import csv import sys from itertools import chain -#pylint: disable=line-too-long -class InteriorNode: - """An interior node in a trie. - - Each interior node has a dict that maps from an element of a signature to - either another interior node or a leaf. Each interior node represents either - a package, class or nested class. Class members are represented by a Leaf. - - Associating the set of flags [public-api] with the signature - "Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following - nodes to be created: - Node() - ^- package:java -> Node() - ^- package:lang -> Node() - ^- class:Object -> Node() - ^- member:String()Ljava/lang/String; -> Leaf([public-api]) - - Associating the set of flags [blocked,core-platform-api] with the signature - "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;" - will cause the following nodes to be created: - Node() - ^- package:java -> Node() - ^- package:lang -> Node() - ^- class:Character -> Node() - ^- class:UnicodeScript -> Node() - ^- member:of(I)Ljava/lang/Character$UnicodeScript; - -> Leaf([blocked,core-platform-api]) - - Attributes: - nodes: a dict from an element of the signature to the Node/Leaf - containing the next element/value. - """ - #pylint: enable=line-too-long - - def __init__(self): - self.nodes = {} - - #pylint: disable=line-too-long - def signatureToElements(self, signature): - """Split a signature or a prefix into a number of elements: - 1. The packages (excluding the leading L preceding the first package). - 2. The class names, from outermost to innermost. - 3. The member signature. - e.g. - Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; - will be broken down into these elements: - 1. package:java - 2. package:lang - 3. class:Character - 4. class:UnicodeScript - 5. member:of(I)Ljava/lang/Character$UnicodeScript; - """ - # Remove the leading L. - # - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; - text = signature.removeprefix("L") - # Split the signature between qualified class name and the class member - # signature. - # 0 - java/lang/Character$UnicodeScript - # 1 - of(I)Ljava/lang/Character$UnicodeScript; - parts = text.split(";->") - member = parts[1:] - # Split the qualified class name into packages, and class name. - # 0 - java - # 1 - lang - # 2 - Character$UnicodeScript - elements = parts[0].split("/") - packages = elements[0:-1] - className = elements[-1] - if className in ("*" , "**"): #pylint: disable=no-else-return - # Cannot specify a wildcard and target a specific member - if len(member) != 0: - raise Exception( - "Invalid signature %s: contains wildcard %s and member " \ - "signature %s" - % (signature, className, member[0])) - wildcard = [className] - # Assemble the parts into a single list, adding prefixes to identify - # the different parts. - # 0 - package:java - # 1 - package:lang - # 2 - * - return list( - chain(["package:" + x for x in packages], wildcard)) - else: - # Split the class name into outer / inner classes - # 0 - Character - # 1 - UnicodeScript - classes = className.split("$") - # Assemble the parts into a single list, adding prefixes to identify - # the different parts. - # 0 - package:java - # 1 - package:lang - # 2 - class:Character - # 3 - class:UnicodeScript - # 4 - member:of(I)Ljava/lang/Character$UnicodeScript; - return list( - chain( - ["package:" + x for x in packages], - ["class:" + x for x in classes], - ["member:" + x for x in member])) - #pylint: enable=line-too-long - - def add(self, signature, value): - """Associate the value with the specific signature. - - :param signature: the member signature - :param value: the value to associated with the signature - :return: n/a - """ - # Split the signature into elements. - elements = self.signatureToElements(signature) - # Find the Node associated with the deepest class. - node = self - for element in elements[:-1]: - if element in node.nodes: - node = node.nodes[element] - else: - next_node = InteriorNode() - node.nodes[element] = next_node - node = next_node - # Add a Leaf containing the value and associate it with the member - # signature within the class. - lastElement = elements[-1] - if not lastElement.startswith("member:"): - raise Exception( - "Invalid signature: %s, does not identify a specific member" % - signature) - if lastElement in node.nodes: - raise Exception("Duplicate signature: %s" % signature) - node.nodes[lastElement] = Leaf(value) - - def getMatchingRows(self, pattern): - """Get the values (plural) associated with the pattern. - - e.g. If the pattern is a full signature then this will return a list - containing the value associated with that signature. - - If the pattern is a class then this will return a list containing the - values associated with all members of that class. - - If the pattern is a package then this will return a list containing the - values associated with all the members of all the classes in that - package and sub-packages. - - If the pattern ends with "*" then the preceding part is treated as a - package and this will return a list containing the values associated - with all the members of all the classes in that package. - - If the pattern ends with "**" then the preceding part is treated - as a package and this will return a list containing the values - associated with all the members of all the classes in that package and - all sub-packages. - - :param pattern: the pattern which could be a complete signature or a - class, or package wildcard. - :return: an iterable containing all the values associated with the - pattern. - """ - elements = self.signatureToElements(pattern) - node = self - # Include all values from this node and all its children. - selector = lambda x: True - lastElement = elements[-1] - if lastElement in ("*", "**"): - elements = elements[:-1] - if lastElement == "*": - # Do not include values from sub-packages. - selector = lambda x: not x.startswith("package:") - for element in elements: - if element in node.nodes: - node = node.nodes[element] - else: - return [] - return chain.from_iterable(node.values(selector)) - - def values(self, selector): - """:param selector: a function that can be applied to a key in the nodes - attribute to determine whether to return its values. - - :return: A list of iterables of all the values associated with - this node and its children. - """ - values = [] - self.appendValues(values, selector) - return values - - def appendValues(self, values, selector): - """Append the values associated with this node and its children to the - list. - - For each item (key, child) in nodes the child node's values are returned - if and only if the selector returns True when called on its key. A child - node's values are all the values associated with it and all its - descendant nodes. - - :param selector: a function that can be applied to a key in the nodes - attribute to determine whether to return its values. - :param values: a list of a iterables of values. - """ - for key, node in self.nodes.items(): - if selector(key): - node.appendValues(values, lambda x: True) - - -class Leaf: - """A leaf of the trie - - Attributes: - value: the value associated with this leaf. - """ - - def __init__(self, value): - self.value = value - - def values(self, selector): #pylint: disable=unused-argument - """:return: A list of a list of the value associated with this node. - """ - return [[self.value]] - - def appendValues(self, values, selector): #pylint: disable=unused-argument - """Appends a list of the value associated with this node to the list. - - :param values: a list of a iterables of values. - """ - values.append([self.value]) +from signature_trie import signature_trie def dict_reader(csvfile): @@ -259,7 +34,7 @@ def read_flag_trie_from_file(file): def read_flag_trie_from_stream(stream): - trie = InteriorNode() + trie = signature_trie() reader = dict_reader(stream) for row in reader: signature = row["signature"] @@ -269,8 +44,7 @@ def read_flag_trie_from_stream(stream): def extract_subset_from_monolithic_flags_as_dict_from_file( monolithicTrie, patternsFile): - """Extract a subset of flags from the dict containing all the monolithic - flags. + """Extract a subset of flags from the dict of monolithic flags. :param monolithicFlagsDict: the dict containing all the monolithic flags. :param patternsFile: a file containing a list of signature patterns that @@ -284,8 +58,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_file( def extract_subset_from_monolithic_flags_as_dict_from_stream( monolithicTrie, stream): - """Extract a subset of flags from the trie containing all the monolithic - flags. + """Extract a subset of flags from the trie of monolithic flags. :param monolithicTrie: the trie containing all the monolithic flags. :param stream: a stream containing a list of signature patterns that define @@ -295,7 +68,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream( dict_signature_to_row = {} for pattern in stream: pattern = pattern.rstrip() - rows = monolithicTrie.getMatchingRows(pattern) + rows = monolithicTrie.get_matching_rows(pattern) for row in rows: signature = row["signature"] dict_signature_to_row[signature] = row @@ -303,8 +76,10 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream( def read_signature_csv_from_stream_as_dict(stream): - """Read the csv contents from the stream into a dict. The first column is - assumed to be the signature and used as the key. + """Read the csv contents from the stream into a dict. + + The first column is assumed to be the signature and used as the + key. The whole row is stored as the value. :param stream: the csv contents to read @@ -319,8 +94,10 @@ def read_signature_csv_from_stream_as_dict(stream): def read_signature_csv_from_file_as_dict(csvFile): - """Read the csvFile into a dict. The first column is assumed to be the - signature and used as the key. + """Read the csvFile into a dict. + + The first column is assumed to be the signature and used as the + key. The whole row is stored as the value. :param csvFile: the csv file to read @@ -363,8 +140,7 @@ def compare_signature_flags(monolithicFlagsDict, modularFlagsDict): def main(argv): args_parser = argparse.ArgumentParser( description="Verify that sets of hidden API flags are each a subset of " - "the monolithic flag file." - ) + "the monolithic flag file.") args_parser.add_argument("monolithicFlags", help="The monolithic flag file") args_parser.add_argument( "modularFlags", diff --git a/scripts/hiddenapi/verify_overlaps_test.py b/scripts/hiddenapi/verify_overlaps_test.py index 22a1cdf6d..8cf295962 100755 --- a/scripts/hiddenapi/verify_overlaps_test.py +++ b/scripts/hiddenapi/verify_overlaps_test.py @@ -17,54 +17,9 @@ import io import unittest -from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import +from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import -class TestSignatureToElements(unittest.TestCase): - - def signatureToElements(self, signature): - return InteriorNode().signatureToElements(signature) - - def test_signatureToElements_1(self): - expected = [ - 'package:java', - 'package:lang', - 'class:ProcessBuilder', - 'class:Redirect', - 'class:1', - 'member:<init>()V', - ] - self.assertEqual( - expected, - self.signatureToElements( - 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V')) - - def test_signatureToElements_2(self): - expected = [ - 'package:java', - 'package:lang', - 'class:Object', - 'member:hashCode()I', - ] - self.assertEqual( - expected, - self.signatureToElements('Ljava/lang/Object;->hashCode()I')) - - def test_signatureToElements_3(self): - expected = [ - 'package:java', - 'package:lang', - 'class:CharSequence', - 'class:', - 'class:ExternalSyntheticLambda0', - 'member:<init>(Ljava/lang/CharSequence;)V', - ] - self.assertEqual( - expected, - self.signatureToElements( - 'Ljava/lang/CharSequence$$ExternalSyntheticLambda0;' - '-><init>(Ljava/lang/CharSequence;)V')) - #pylint: disable=line-too-long class TestDetectOverlaps(unittest.TestCase): @@ -239,18 +194,6 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked } self.assertEqual(expected, subset) - def test_extract_subset_invalid_pattern_wildcard_and_member(self): - monolithic = self.read_flag_trie_from_string( - TestDetectOverlaps.extractInput) - - patterns = 'Ljava/lang/*;->hashCode()I' - - with self.assertRaises(Exception) as context: - self.extract_subset_from_monolithic_flags_as_dict_from_string( - monolithic, patterns) - self.assertTrue('contains wildcard * and member signature hashCode()I' - in str(context.exception)) - def test_read_trie_duplicate(self): with self.assertRaises(Exception) as context: self.read_flag_trie_from_string(""" @@ -369,6 +312,8 @@ Ljava/lang/Object;->hashCode()I,blocked mismatches = compare_signature_flags(monolithic, modular) expected = [] self.assertEqual(expected, mismatches) + + #pylint: enable=line-too-long if __name__ == '__main__': |