diff options
65 files changed, 1905 insertions, 628 deletions
@@ -6,6 +6,7 @@ ahumesky@google.com alexmarquez@google.com asmundak@google.com ccross@android.com +colefaust@google.com cparsons@google.com delmerico@google.com dwillemsen@google.com diff --git a/android/api_levels.go b/android/api_levels.go index 926d29794..de566250c 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -321,6 +321,7 @@ func getFinalCodenamesMap(config Config) map[string]int { "Q": 29, "R": 30, "S": 31, + "S-V2": 32, } // TODO: Differentiate "current" and "future". @@ -364,6 +365,7 @@ func GetApiLevelsMap(config Config) map[string]int { "Q": 29, "R": 30, "S": 31, + "S-V2": 32, } for i, codename := range config.PlatformVersionActiveCodenames() { apiLevelsMap[codename] = previewAPILevelBase + i diff --git a/android/arch.go b/android/arch.go index 8aa8d4043..6b81022bf 100644 --- a/android/arch.go +++ b/android/arch.go @@ -909,6 +909,7 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { "Glibc", "Musl", "Linux", + "Host_linux", "Not_windows", "Arm_on_x86", "Arm_on_x86_64", @@ -930,6 +931,12 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { targets = append(targets, target) } } + if os.Linux() && os.Class == Host { + target := "Host_linux_" + archType.Name + if !InList(target, targets) { + targets = append(targets, target) + } + } if os.Bionic() { target := "Bionic_" + archType.Name if !InList(target, targets) { @@ -1162,6 +1169,14 @@ func (m *ModuleBase) setOSProperties(ctx BottomUpMutatorContext) { } } + if os.Linux() && os.Class == Host { + field := "Host_linux" + prefix := "target.host_linux" + if linuxProperties, ok := getChildPropertyStruct(ctx, targetProp, field, prefix); ok { + mergePropertyStruct(ctx, genProps, linuxProperties) + } + } + if os.Bionic() { field := "Bionic" prefix := "target.bionic" @@ -2127,6 +2142,7 @@ func (m *ModuleBase) GetArchVariantProperties(ctx ArchVariantContext, propertySe linuxStructs := getTargetStructs(ctx, archProperties, "Linux") bionicStructs := getTargetStructs(ctx, archProperties, "Bionic") hostStructs := getTargetStructs(ctx, archProperties, "Host") + hostLinuxStructs := getTargetStructs(ctx, archProperties, "Host_linux") hostNotWindowsStructs := getTargetStructs(ctx, archProperties, "Not_windows") // For android, linux, ... @@ -2147,6 +2163,9 @@ func (m *ModuleBase) GetArchVariantProperties(ctx ArchVariantContext, propertySe if os.Bionic() { osStructs = append(osStructs, bionicStructs...) } + if os.Linux() && os.Class == Host { + osStructs = append(osStructs, hostLinuxStructs...) + } if os == LinuxMusl { osStructs = append(osStructs, getTargetStructs(ctx, archProperties, "Musl")...) @@ -2179,6 +2198,16 @@ func (m *ModuleBase) GetArchVariantProperties(ctx ArchVariantContext, propertySe targetStructs := getTargetStructs(ctx, archProperties, targetField) osArchStructs = append(osArchStructs, targetStructs...) } + if os == LinuxMusl { + targetField := "Musl_" + arch.Name + targetStructs := getTargetStructs(ctx, archProperties, targetField) + osArchStructs = append(osArchStructs, targetStructs...) + } + if os == Linux { + targetField := "Glibc_" + arch.Name + targetStructs := getTargetStructs(ctx, archProperties, targetField) + osArchStructs = append(osArchStructs, targetStructs...) + } targetField := GetCompoundTargetField(os, arch) targetName := fmt.Sprintf("%s_%s", os.Name, arch.Name) diff --git a/android/arch_test.go b/android/arch_test.go index 7caf8378f..68dc7f52a 100644 --- a/android/arch_test.go +++ b/android/arch_test.go @@ -510,6 +510,7 @@ func TestArchProperties(t *testing.T) { musl: { a: ["musl"] }, linux_bionic: { a: ["linux_bionic"] }, linux: { a: ["linux"] }, + host_linux: { a: ["host_linux"] }, linux_glibc: { a: ["linux_glibc"] }, linux_musl: { a: ["linux_musl"] }, windows: { a: ["windows"], enabled: true }, @@ -566,12 +567,12 @@ func TestArchProperties(t *testing.T) { { module: "foo", variant: "linux_glibc_x86_64", - property: []string{"root", "host", "linux", "glibc", "linux_glibc", "not_windows", "x86_64", "lib64", "linux_x86_64", "linux_glibc_x86_64"}, + property: []string{"root", "host", "linux", "host_linux", "glibc", "linux_glibc", "not_windows", "x86_64", "lib64", "linux_x86_64", "linux_glibc_x86_64"}, }, { module: "foo", variant: "linux_glibc_x86", - property: []string{"root", "host", "linux", "glibc", "linux_glibc", "not_windows", "x86", "lib32", "linux_x86", "linux_glibc_x86"}, + property: []string{"root", "host", "linux", "host_linux", "glibc", "linux_glibc", "not_windows", "x86", "lib32", "linux_x86", "linux_glibc_x86"}, }, }, }, @@ -605,12 +606,12 @@ func TestArchProperties(t *testing.T) { { module: "foo", variant: "linux_musl_x86_64", - property: []string{"root", "host", "linux", "musl", "linux_glibc", "linux_musl", "not_windows", "x86_64", "lib64", "linux_x86_64", "linux_musl_x86_64", "linux_glibc_x86_64"}, + property: []string{"root", "host", "linux", "host_linux", "musl", "linux_glibc", "linux_musl", "not_windows", "x86_64", "lib64", "linux_x86_64", "linux_musl_x86_64", "linux_glibc_x86_64"}, }, { module: "foo", variant: "linux_musl_x86", - property: []string{"root", "host", "linux", "musl", "linux_glibc", "linux_musl", "not_windows", "x86", "lib32", "linux_x86", "linux_musl_x86", "linux_glibc_x86"}, + property: []string{"root", "host", "linux", "host_linux", "musl", "linux_glibc", "linux_musl", "not_windows", "x86", "lib32", "linux_x86", "linux_musl_x86", "linux_glibc_x86"}, }, }, }, diff --git a/android/bazel.go b/android/bazel.go index 410cec3a7..061736761 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -208,6 +208,7 @@ var ( "build/bazel/tests":/* recursive = */ true, "build/bazel/platforms":/* recursive = */ true, "build/bazel/product_variables":/* recursive = */ true, + "build/bazel/vendor/google":/* recursive = */ true, "build/bazel_common_rules":/* recursive = */ true, // build/make/tools/signapk BUILD file is generated, so build/make/tools is not recursive. "build/make/tools":/* recursive = */ false, @@ -225,6 +226,7 @@ var ( "packages/apps/QuickSearchBox":/* recursive = */ true, "packages/apps/WallpaperPicker":/* recursive = */ false, + "prebuilts/bundletool":/* recursive = */ true, "prebuilts/gcc":/* recursive = */ true, "prebuilts/build-tools":/* recursive = */ false, "prebuilts/jdk/jdk11":/* recursive = */ false, @@ -291,12 +293,14 @@ var ( "development/samples/WiFiDirectDemo": Bp2BuildDefaultTrue, "development/sdk": Bp2BuildDefaultTrueRecursively, "external/arm-optimized-routines": Bp2BuildDefaultTrueRecursively, + "external/auto/android-annotation-stubs": Bp2BuildDefaultTrueRecursively, "external/auto/common": Bp2BuildDefaultTrueRecursively, "external/auto/service": Bp2BuildDefaultTrueRecursively, "external/boringssl": Bp2BuildDefaultTrueRecursively, "external/bouncycastle": Bp2BuildDefaultTrue, "external/brotli": Bp2BuildDefaultTrue, "external/conscrypt": Bp2BuildDefaultTrue, + "external/e2fsprogs": Bp2BuildDefaultTrueRecursively, "external/error_prone": Bp2BuildDefaultTrueRecursively, "external/fmtlib": Bp2BuildDefaultTrueRecursively, "external/google-benchmark": Bp2BuildDefaultTrueRecursively, @@ -383,8 +387,36 @@ var ( } // Per-module allowlist to always opt modules in of both bp2build and mixed builds. + // These modules are usually in directories with many other modules that are not ready for + // conversion. + // + // A module can either be in this list or its directory allowlisted entirely + // in bp2buildDefaultConfig, but not both at the same time. bp2buildModuleAlwaysConvertList = []string{ "junit-params-assertj-core", + + //external/avb + "avbtool", + "libavb", + "avb_headers", + + //external/fec + "libfec_rs", + + //system/core/libsparse + "libsparse", + + //system/extras/ext4_utils + "libext4_utils", + + //system/extras/libfec + "libfec", + + //system/extras/squashfs_utils + "libsquashfs_utils", + + //system/extras/verity/fec + "fec", } // Per-module denylist to always opt modules out of both bp2build and mixed builds. @@ -453,11 +485,12 @@ var ( "conscrypt", // b/210751803, we don't handle path property for filegroups "conscrypt-for-host", // b/210751803, we don't handle path property for filegroups - "host-libprotobuf-java-lite", // b/217236083, java_library cannot have deps without srcs - "host-libprotobuf-java-micro", // b/217236083, java_library cannot have deps without srcs - "host-libprotobuf-java-nano", // b/217236083, java_library cannot have deps without srcs - "error_prone_core", // b/217236083, java_library cannot have deps without srcs - "bouncycastle-host", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-lite", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-micro", // b/217236083, java_library cannot have deps without srcs + "host-libprotobuf-java-nano", // b/217236083, java_library cannot have deps without srcs + "error_prone_core", // b/217236083, java_library cannot have deps without srcs + "bouncycastle-host", // b/217236083, java_library cannot have deps without srcs + "mockito-robolectric-prebuilt", // b/217236083, java_library cannot have deps without srcs "apex_manifest_proto_java", // b/215230097, we don't handle .proto files in java_library srcs attribute @@ -527,6 +560,8 @@ var ( "dex2oat-script", // depends on unconverted modules: dex2oat "error_prone_checkerframework_dataflow_nullaway", // TODO(b/219908977): "Error in fail: deps not allowed without srcs; move to runtime_deps?" + + "libprotobuf-java-nano", // b/220869005, depends on non-public_current SDK } // Per-module denylist of cc_library modules to only generate the static @@ -679,14 +714,21 @@ func (b *BazelModuleBase) shouldConvertWithBp2build(ctx BazelConversionContext, } packagePath := ctx.OtherModuleDir(module) - config := ctx.Config().bp2buildPackageConfig + if alwaysConvert && ShouldKeepExistingBuildFileForDir(packagePath) { + ctx.(BaseModuleContext).ModuleErrorf("A module cannot be in a directory listed in bp2buildKeepExistingBuildFile"+ + " and also be in bp2buildModuleAlwaysConvert. Directory: '%s'", packagePath) + return false + } + + config := ctx.Config().bp2buildPackageConfig // This is a tristate value: true, false, or unset. propValue := b.bazelProperties.Bazel_module.Bp2build_available if bp2buildDefaultTrueRecursively(packagePath, config) { if alwaysConvert { - ctx.(BaseModuleContext).ModuleErrorf("a module cannot be in a directory marked Bp2BuildDefaultTrue" + - " or Bp2BuildDefaultTrueRecursively and also be in bp2buildModuleAlwaysConvert") + ctx.(BaseModuleContext).ModuleErrorf("A module cannot be in a directory marked Bp2BuildDefaultTrue"+ + " or Bp2BuildDefaultTrueRecursively and also be in bp2buildModuleAlwaysConvert. Directory: '%s'", + packagePath) } // Allow modules to explicitly opt-out. diff --git a/android/config.go b/android/config.go index 3d8bc31a5..e8ca84c6f 100644 --- a/android/config.go +++ b/android/config.go @@ -1256,6 +1256,10 @@ func (c *deviceConfig) ClangCoverageEnabled() bool { return Bool(c.config.productVariables.ClangCoverage) } +func (c *deviceConfig) ClangCoverageContinuousMode() bool { + return Bool(c.config.productVariables.ClangCoverageContinuousMode) +} + func (c *deviceConfig) GcovCoverageEnabled() bool { return Bool(c.config.productVariables.GcovCoverage) } diff --git a/android/hooks.go b/android/hooks.go index bded76467..5e3a4a7e7 100644 --- a/android/hooks.go +++ b/android/hooks.go @@ -15,7 +15,10 @@ package android import ( + "fmt" + "path" "reflect" + "runtime" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -88,7 +91,19 @@ func (l *loadHookContext) PrependProperties(props ...interface{}) { func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module { inherited := []interface{}{&l.Module().base().commonProperties} - module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module) + + var typeName string + if typeNameLookup, ok := ModuleTypeByFactory()[reflect.ValueOf(factory)]; ok { + typeName = typeNameLookup + } else { + factoryPtr := reflect.ValueOf(factory).Pointer() + factoryFunc := runtime.FuncForPC(factoryPtr) + filePath, _ := factoryFunc.FileLine(factoryPtr) + typeName = fmt.Sprintf("%s_%s", path.Base(filePath), factoryFunc.Name()) + } + typeName = typeName + "_loadHookModule" + + module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module) if l.Module().base().variableProperties != nil && module.base().variableProperties != nil { src := l.Module().base().variableProperties diff --git a/android/module.go b/android/module.go index 03d3f80bc..43509c0b5 100644 --- a/android/module.go +++ b/android/module.go @@ -456,6 +456,10 @@ type ModuleContext interface { // GetMissingDependencies returns the list of dependencies that were passed to AddDependencies or related methods, // but do not exist. GetMissingDependencies() []string + + // LicenseMetadataFile returns the path where the license metadata for this module will be + // generated. + LicenseMetadataFile() Path } type Module interface { @@ -3279,6 +3283,10 @@ func (m *moduleContext) blueprintModuleContext() blueprint.ModuleContext { return m.bp } +func (m *moduleContext) LicenseMetadataFile() Path { + return m.module.base().licenseMetadataFile +} + // SrcIsModule decodes module references in the format ":unqualified-name" or "//namespace:name" // into the module name, or empty string if the input was not a module reference. func SrcIsModule(s string) (module string) { diff --git a/android/paths.go b/android/paths.go index 05caebd18..e7829b961 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1474,14 +1474,11 @@ func pathForModuleOut(ctx ModuleOutPathContext) OutputPath { func PathForVndkRefAbiDump(ctx ModuleInstallPathContext, version, fileName string, isNdk, isLlndkOrVndk, isGzip bool) OptionalPath { - arches := ctx.DeviceConfig().Arches() - if len(arches) == 0 { - panic("device build with no primary arch") - } - currentArch := ctx.Arch() - archNameAndVariant := currentArch.ArchType.String() - if currentArch.ArchVariant != "" { - archNameAndVariant += "_" + currentArch.ArchVariant + currentArchType := ctx.Arch().ArchType + primaryArchType := ctx.Config().DevicePrimaryArchType() + archName := currentArchType.String() + if currentArchType != primaryArchType { + archName += "_" + primaryArchType.String() } var dirName string @@ -1503,7 +1500,7 @@ func PathForVndkRefAbiDump(ctx ModuleInstallPathContext, version, fileName strin } return ExistentPathForSource(ctx, "prebuilts", "abi-dumps", dirName, - version, binderBitness, archNameAndVariant, "source-based", + version, binderBitness, archName, "source-based", fileName+ext) } diff --git a/android/register.go b/android/register.go index 10e14e04d..c50583322 100644 --- a/android/register.go +++ b/android/register.go @@ -59,6 +59,7 @@ func (t moduleType) register(ctx *Context) { var moduleTypes []moduleType var moduleTypesForDocs = map[string]reflect.Value{} +var moduleTypeByFactory = map[reflect.Value]string{} type singleton struct { // True if this should be registered as a pre-singleton, false otherwise. @@ -140,6 +141,7 @@ func RegisterModuleType(name string, factory ModuleFactory) { // RegisterModuleType was a lambda. func RegisterModuleTypeForDocs(name string, factory reflect.Value) { moduleTypesForDocs[name] = factory + moduleTypeByFactory[factory] = name } func RegisterSingletonType(name string, factory SingletonFactory) { @@ -228,6 +230,10 @@ func ModuleTypeFactoriesForDocs() map[string]reflect.Value { return moduleTypesForDocs } +func ModuleTypeByFactory() map[reflect.Value]string { + return moduleTypeByFactory +} + // Interface for registering build components. // // Provided to allow registration of build components to be shared between the runtime diff --git a/android/variable.go b/android/variable.go index 68f19b993..37037ebdb 100644 --- a/android/variable.go +++ b/android/variable.go @@ -306,10 +306,11 @@ type productVariables struct { JavaCoveragePaths []string `json:",omitempty"` JavaCoverageExcludePaths []string `json:",omitempty"` - GcovCoverage *bool `json:",omitempty"` - ClangCoverage *bool `json:",omitempty"` - NativeCoveragePaths []string `json:",omitempty"` - NativeCoverageExcludePaths []string `json:",omitempty"` + GcovCoverage *bool `json:",omitempty"` + ClangCoverage *bool `json:",omitempty"` + NativeCoveragePaths []string `json:",omitempty"` + NativeCoverageExcludePaths []string `json:",omitempty"` + ClangCoverageContinuousMode *bool `json:",omitempty"` // Set by NewConfig Native_coverage *bool `json:",omitempty"` diff --git a/androidmk/androidmk/android.go b/androidmk/androidmk/android.go index 295b0e50e..954f8d0ea 100644 --- a/androidmk/androidmk/android.go +++ b/androidmk/androidmk/android.go @@ -17,6 +17,7 @@ package androidmk import ( "fmt" "sort" + "strconv" "strings" mkparser "android/soong/androidmk/parser" @@ -623,6 +624,16 @@ func makeBlueprintStringAssignment(file *bpFile, prefix string, suffix string, v return err } +// Assigns a given boolean value to a given variable in the result bp file. See +// setVariable documentation for more information about prefix and name. +func makeBlueprintBoolAssignment(ctx variableAssignmentContext, prefix, name string, value bool) error { + expressionValue, err := stringToBoolValue(strconv.FormatBool(value)) + if err == nil { + err = setVariable(ctx.file, false, prefix, name, expressionValue, true) + } + return err +} + // If variable is a literal variable name, return the name, otherwise return "" func varLiteralName(variable mkparser.Variable) string { if len(variable.Name.Variables) == 0 { @@ -647,7 +658,11 @@ func prebuiltModulePath(ctx variableAssignmentContext) error { varname := "" fixed := "" val := ctx.mkvalue + if len(val.Variables) == 1 && varLiteralName(val.Variables[0]) != "" && len(val.Strings) == 2 && val.Strings[0] == "" { + if varLiteralName(val.Variables[0]) == "PRODUCT_OUT" && val.Strings[1] == "/system/priv-app" { + return makeBlueprintBoolAssignment(ctx, "", "privileged", true) + } fixed = val.Strings[1] varname = val.Variables[0].Name.Strings[0] // TARGET_OUT_OPTIONAL_EXECUTABLES puts the artifact in xbin, which is diff --git a/androidmk/androidmk/androidmk.go b/androidmk/androidmk/androidmk.go index b8316a361..aaafdc758 100644 --- a/androidmk/androidmk/androidmk.go +++ b/androidmk/androidmk/androidmk.go @@ -411,6 +411,24 @@ func makeVariableToBlueprint(file *bpFile, val *mkparser.MakeString, return exp, nil } +// If local is set to true, then the variable will be added as a part of the +// variable at file.bpPos. For example, if file.bpPos references a module, +// then calling this method will set a property on that module if local is set +// to true. Otherwise, the Variable will be created at the root of the file. +// +// prefix should be populated with the top level value to be assigned, and +// name with a sub-value. If prefix is empty, then name is the top level value. +// For example, if prefix is "foo" and name is "bar" with a value of "baz", then +// the following variable will be generated: +// +// foo { +// bar: "baz" +// } +// +// If prefix is the empty string and name is "foo" with a value of "bar", the +// following variable will be generated (if it is a property): +// +// foo: "bar" func setVariable(file *bpFile, plusequals bool, prefix, name string, value bpparser.Expression, local bool) error { if prefix != "" { name = prefix + "." + name diff --git a/androidmk/androidmk/androidmk_test.go b/androidmk/androidmk/androidmk_test.go index e8b6f78cf..21763613a 100644 --- a/androidmk/androidmk/androidmk_test.go +++ b/androidmk/androidmk/androidmk_test.go @@ -1675,6 +1675,21 @@ android_app { } `, }, + { + desc: "privileged app", + in: ` +include $(CLEAR_VARS) +LOCAL_MODULE := foo +LOCAL_MODULE_PATH := $(PRODUCT_OUT)/system/priv-app +include $(BUILD_PACKAGE) + `, + expected: ` +android_app { + name: "foo", + privileged: true +} +`, + }, } func TestEndToEnd(t *testing.T) { diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go index aac4c4efe..803032649 100644 --- a/androidmk/parser/make_strings.go +++ b/androidmk/parser/make_strings.go @@ -24,14 +24,24 @@ import ( // A MakeString is a string that may contain variable substitutions in it. // It can be considered as an alternating list of raw Strings and variable // substitutions, where the first and last entries in the list must be raw -// Strings (possibly empty). A MakeString that starts with a variable -// will have an empty first raw string, and a MakeString that ends with a -// variable will have an empty last raw string. Two sequential Variables -// will have an empty raw string between them. +// Strings (possibly empty). The entirety of the text before the first variable, +// between two variables, and after the last variable will be considered a +// single String value. A MakeString that starts with a variable will have an +// empty first raw string, and a MakeString that ends with a variable will have +// an empty last raw string. Two sequential Variables will have an empty raw +// string between them. // // The MakeString is stored as two lists, a list of raw Strings and a list // of Variables. The raw string list is always one longer than the variable // list. +// +// For example, "$(FOO)/bar/baz" will be represented as the +// following lists: +// +// { +// Strings: ["", "/bar/baz"], +// Variables: ["FOO"] +// } type MakeString struct { StringPos Pos Strings []string diff --git a/apex/androidmk.go b/apex/androidmk.go index 8785ca0f3..059b4d76c 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -373,8 +373,10 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { } fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+stemSuffix) fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !a.installable()) - fmt.Fprintln(w, "LOCAL_SOONG_INSTALLED_MODULE :=", a.installedFile.String()) - fmt.Fprintln(w, "LOCAL_SOONG_INSTALL_PAIRS :=", a.outputFile.String()+":"+a.installedFile.String()) + if a.installable() { + fmt.Fprintln(w, "LOCAL_SOONG_INSTALLED_MODULE :=", a.installedFile.String()) + fmt.Fprintln(w, "LOCAL_SOONG_INSTALL_PAIRS :=", a.outputFile.String()+":"+a.installedFile.String()) + } // Because apex writes .mk with Custom(), we need to write manually some common properties // which are available via data.Entries diff --git a/apex/apex.go b/apex/apex.go index 9031a4ea9..ac67feea2 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -1415,7 +1415,7 @@ func (a *apexBundle) AddSanitizerDependencies(ctx android.BottomUpMutatorContext for _, target := range ctx.MultiTargets() { if target.Arch.ArchType.Multilib == "lib64" { addDependenciesForNativeModules(ctx, ApexNativeDependencies{ - Native_shared_libs: []string{"libclang_rt.hwasan-aarch64-android"}, + Native_shared_libs: []string{"libclang_rt.hwasan"}, Tests: nil, Jni_libs: nil, Binaries: nil, diff --git a/apex/apex_test.go b/apex/apex_test.go index 1c36c759f..4f2a58348 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -1415,13 +1415,14 @@ func TestRuntimeApexShouldInstallHwasanIfLibcDependsOnIt(t *testing.T) { } cc_prebuilt_library_shared { - name: "libclang_rt.hwasan-aarch64-android", + name: "libclang_rt.hwasan", no_libcrt: true, nocrt: true, stl: "none", system_shared_libs: [], srcs: [""], stubs: { versions: ["1"] }, + stem: "libclang_rt.hwasan-aarch64-android", sanitize: { never: true, @@ -1434,7 +1435,7 @@ func TestRuntimeApexShouldInstallHwasanIfLibcDependsOnIt(t *testing.T) { "lib64/bionic/libclang_rt.hwasan-aarch64-android.so", }) - hwasan := ctx.ModuleForTests("libclang_rt.hwasan-aarch64-android", "android_arm64_armv8-a_shared") + hwasan := ctx.ModuleForTests("libclang_rt.hwasan", "android_arm64_armv8-a_shared") installed := hwasan.Description("install libclang_rt.hwasan") ensureContains(t, installed.Output.String(), "/system/lib64/bootstrap/libclang_rt.hwasan-aarch64-android.so") @@ -1462,13 +1463,14 @@ func TestRuntimeApexShouldInstallHwasanIfHwaddressSanitized(t *testing.T) { } cc_prebuilt_library_shared { - name: "libclang_rt.hwasan-aarch64-android", + name: "libclang_rt.hwasan", no_libcrt: true, nocrt: true, stl: "none", system_shared_libs: [], srcs: [""], stubs: { versions: ["1"] }, + stem: "libclang_rt.hwasan-aarch64-android", sanitize: { never: true, @@ -1482,7 +1484,7 @@ func TestRuntimeApexShouldInstallHwasanIfHwaddressSanitized(t *testing.T) { "lib64/bionic/libclang_rt.hwasan-aarch64-android.so", }) - hwasan := ctx.ModuleForTests("libclang_rt.hwasan-aarch64-android", "android_arm64_armv8-a_shared") + hwasan := ctx.ModuleForTests("libclang_rt.hwasan", "android_arm64_armv8-a_shared") installed := hwasan.Description("install libclang_rt.hwasan") ensureContains(t, installed.Output.String(), "/system/lib64/bootstrap/libclang_rt.hwasan-aarch64-android.so") diff --git a/apex/builder.go b/apex/builder.go index 183c21562..8c5f99bf5 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -856,6 +856,10 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { installSuffix = imageCapexSuffix } + if !a.installable() { + a.SkipInstall() + } + // Install to $OUT/soong/{target,host}/.../apex. a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile, a.compatSymlinks.Paths()...) diff --git a/bp2build/Android.bp b/bp2build/Android.bp index b904c3533..8a171d402 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -44,9 +44,16 @@ bootstrap_go_package { "cc_library_shared_conversion_test.go", "cc_library_static_conversion_test.go", "cc_object_conversion_test.go", + "cc_prebuilt_library_shared_test.go", "conversion_test.go", "filegroup_conversion_test.go", "genrule_conversion_test.go", + "java_binary_host_conversion_test.go", + "java_import_conversion_test.go", + "java_library_conversion_test.go", + "java_library_host_conversion_test.go", + "java_plugin_conversion_test.go", + "java_proto_conversion_test.go", "performance_test.go", "prebuilt_etc_conversion_test.go", "python_binary_conversion_test.go", diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go index 28de06c00..b6095b2ef 100644 --- a/bp2build/android_app_conversion_test.go +++ b/bp2build/android_app_conversion_test.go @@ -51,6 +51,7 @@ android_app { "srcs": `["app.java"]`, "manifest": `"AndroidManifest.xml"`, "resource_files": `["res/res.png"]`, + "deps": `["//prebuilts/sdk:public_current_android_sdk_java_import"]`, }), }}) } @@ -86,7 +87,49 @@ android_app { "resb/res.png", ]`, "custom_package": `"com.google"`, - "deps": `[":static_lib_dep"]`, + "deps": `[ + "//prebuilts/sdk:public_current_android_sdk_java_import", + ":static_lib_dep", + ]`, + }), + }}) +} + +func TestAndroidAppArchVariantSrcs(t *testing.T) { + runAndroidAppTestCase(t, bp2buildTestCase{ + description: "Android app - arch variant srcs", + moduleTypeUnderTest: "android_app", + moduleTypeUnderTestFactory: java.AndroidAppFactory, + filesystem: map[string]string{ + "arm.java": "", + "x86.java": "", + "res/res.png": "", + "AndroidManifest.xml": "", + }, + blueprint: ` +android_app { + name: "TestApp", + sdk_version: "current", + arch: { + arm: { + srcs: ["arm.java"], + }, + x86: { + srcs: ["x86.java"], + } + } +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("android_binary", "TestApp", attrNameToString{ + "srcs": `select({ + "//build/bazel/platforms/arch:arm": ["arm.java"], + "//build/bazel/platforms/arch:x86": ["x86.java"], + "//conditions:default": [], + })`, + "manifest": `"AndroidManifest.xml"`, + "resource_files": `["res/res.png"]`, + "deps": `["//prebuilts/sdk:public_current_android_sdk_java_import"]`, }), }}) } diff --git a/bp2build/java_proto_conversion_test.go b/bp2build/java_proto_conversion_test.go index 93b0677a6..61a398cd9 100644 --- a/bp2build/java_proto_conversion_test.go +++ b/bp2build/java_proto_conversion_test.go @@ -71,8 +71,7 @@ func TestJavaProto(t *testing.T) { }` protoLibrary := makeBazelTarget("proto_library", "java-protos_proto", attrNameToString{ - "srcs": `["a.proto"]`, - "strip_import_prefix": `""`, + "srcs": `["a.proto"]`, }) for _, tc := range testCases { @@ -107,8 +106,7 @@ func TestJavaProtoDefault(t *testing.T) { `, expectedBazelTargets: []string{ makeBazelTarget("proto_library", "java-protos_proto", attrNameToString{ - "srcs": `["a.proto"]`, - "strip_import_prefix": `""`, + "srcs": `["a.proto"]`, }), makeBazelTarget( "java_lite_proto_library", diff --git a/cc/binary.go b/cc/binary.go index 0fe44903c..9262f217e 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -220,18 +220,18 @@ func newBinary(hod android.HostOrDeviceSupported, bazelable bool) (*Module, *bin func (binary *binaryDecorator) linkerInit(ctx BaseModuleContext) { binary.baseLinker.linkerInit(ctx) - if !ctx.toolchain().Bionic() && !ctx.toolchain().Musl() { - if ctx.Os() == android.Linux { - // Unless explicitly specified otherwise, host static binaries are built with -static - // if HostStaticBinaries is true for the product configuration. - if binary.Properties.Static_executable == nil && ctx.Config().HostStaticBinaries() { - binary.Properties.Static_executable = BoolPtr(true) - } - } else { - // Static executables are not supported on Darwin or Windows - binary.Properties.Static_executable = nil + if ctx.Os().Linux() && ctx.Host() { + // Unless explicitly specified otherwise, host static binaries are built with -static + // if HostStaticBinaries is true for the product configuration. + if binary.Properties.Static_executable == nil && ctx.Config().HostStaticBinaries() { + binary.Properties.Static_executable = BoolPtr(true) } } + + if ctx.Darwin() || ctx.Windows() { + // Static executables are not supported on Darwin or Windows + binary.Properties.Static_executable = nil + } } func (binary *binaryDecorator) static() bool { @@ -1383,7 +1383,7 @@ func isBionic(name string) bool { } func InstallToBootstrap(name string, config android.Config) bool { - if name == "libclang_rt.hwasan-aarch64-android" { + if name == "libclang_rt.hwasan" { return true } return isBionic(name) diff --git a/cc/cc_test.go b/cc/cc_test.go index 51a6a271f..278efa174 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -2944,13 +2944,13 @@ func TestStaticLibDepExport(t *testing.T) { // Check the shared version of lib2. variant := "android_arm64_armv8-a_shared" module := ctx.ModuleForTests("lib2", variant).Module().(*Module) - checkStaticLibs(t, []string{"lib1", "libc++demangle", "libclang_rt.builtins-aarch64-android"}, module) + checkStaticLibs(t, []string{"lib1", "libc++demangle", "libclang_rt.builtins"}, module) // Check the static version of lib2. variant = "android_arm64_armv8-a_static" module = ctx.ModuleForTests("lib2", variant).Module().(*Module) // libc++_static is linked additionally. - checkStaticLibs(t, []string{"lib1", "libc++_static", "libc++demangle", "libclang_rt.builtins-aarch64-android"}, module) + checkStaticLibs(t, []string{"lib1", "libc++_static", "libc++demangle", "libclang_rt.builtins"}, module) } var compilerFlagsTestCases = []struct { diff --git a/cc/config/toolchain.go b/cc/config/toolchain.go index 6cede11a0..7175fdc1a 100644 --- a/cc/config/toolchain.go +++ b/cc/config/toolchain.go @@ -227,14 +227,7 @@ func addPrefix(list []string, prefix string) []string { } func LibclangRuntimeLibrary(t Toolchain, library string) string { - arch := t.LibclangRuntimeLibraryArch() - if arch == "" { - return "" - } - if !t.Bionic() { - return "libclang_rt." + library + "-" + arch - } - return "libclang_rt." + library + "-" + arch + "-android" + return "libclang_rt." + library } func BuiltinsRuntimeLibrary(t Toolchain) string { diff --git a/cc/coverage.go b/cc/coverage.go index f2b5425e6..d0902eab8 100644 --- a/cc/coverage.go +++ b/cc/coverage.go @@ -77,6 +77,10 @@ func (cov *coverage) deps(ctx DepsContext, deps Deps) Deps { return deps } +func EnableContinuousCoverage(ctx android.BaseModuleContext) bool { + return ctx.DeviceConfig().ClangCoverageContinuousMode() +} + func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags, PathDeps) { clangCoverage := ctx.DeviceConfig().ClangCoverageEnabled() gcovCoverage := ctx.DeviceConfig().GcovCoverageEnabled() @@ -101,6 +105,9 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags // Override -Wframe-larger-than. We can expect frame size increase after // coverage instrumentation. flags.Local.CFlags = append(flags.Local.CFlags, "-Wno-frame-larger-than=") + if EnableContinuousCoverage(ctx) { + flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-mllvm", "-runtime-counter-relocation") + } } } @@ -152,6 +159,9 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--wrap,getenv") } else if clangCoverage { flags.Local.LdFlags = append(flags.Local.LdFlags, profileInstrFlag) + if EnableContinuousCoverage(ctx) { + flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-mllvm=-runtime-counter-relocation") + } coverage := ctx.GetDirectDepWithTag(getClangProfileLibraryName(ctx), CoverageDepTag).(*Module) deps.WholeStaticLibs = append(deps.WholeStaticLibs, coverage.OutputFile().Path()) diff --git a/cc/sanitize_test.go b/cc/sanitize_test.go index 0070e4026..c1ca03408 100644 --- a/cc/sanitize_test.go +++ b/cc/sanitize_test.go @@ -24,11 +24,7 @@ import ( var prepareForAsanTest = android.FixtureAddFile("asan/Android.bp", []byte(` cc_library_shared { - name: "libclang_rt.asan-aarch64-android", - } - - cc_library_shared { - name: "libclang_rt.asan-arm-android", + name: "libclang_rt.asan", } `)) diff --git a/cc/testing.go b/cc/testing.go index a03d147a0..32f7c6080 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -86,51 +86,17 @@ func commonDefaultModules() string { } cc_prebuilt_library_static { - name: "libclang_rt.builtins-arm-android", + name: "libclang_rt.builtins", defaults: ["toolchain_libs_defaults"], - native_bridge_supported: true, + host_supported: true, + vendor_available: true, vendor_ramdisk_available: true, - } - - cc_prebuilt_library_static { - name: "libclang_rt.builtins-aarch64-android", - defaults: ["toolchain_libs_defaults"], native_bridge_supported: true, - vendor_ramdisk_available: true, - } - - cc_prebuilt_library_static { - name: "libclang_rt.builtins-x86_64", - defaults: ["toolchain_libs_defaults"], - host_supported: true, - } - - cc_prebuilt_library_static { - name: "libclang_rt.builtins-i386", - defaults: ["toolchain_libs_defaults"], - host_supported: true, } cc_prebuilt_library_shared { - name: "libclang_rt.hwasan-aarch64-android", - defaults: ["toolchain_libs_defaults"], - } - - cc_prebuilt_library_static { - name: "libclang_rt.builtins-i686-android", + name: "libclang_rt.hwasan", defaults: ["toolchain_libs_defaults"], - vendor_ramdisk_available: true, - native_bridge_supported: true, - } - - cc_prebuilt_library_static { - name: "libclang_rt.builtins-x86_64-android", - defaults: [ - "linux_bionic_supported", - "toolchain_libs_defaults", - ], - native_bridge_supported: true, - vendor_ramdisk_available: true, } cc_prebuilt_library_static { @@ -144,30 +110,7 @@ func commonDefaultModules() string { } cc_prebuilt_library_static { - name: "libclang_rt.fuzzer-arm-android", - defaults: ["toolchain_libs_defaults"], - } - - cc_prebuilt_library_static { - name: "libclang_rt.fuzzer-aarch64-android", - defaults: ["toolchain_libs_defaults"], - } - - cc_prebuilt_library_static { - name: "libclang_rt.fuzzer-i686-android", - defaults: ["toolchain_libs_defaults"], - } - - cc_prebuilt_library_static { - name: "libclang_rt.fuzzer-x86_64-android", - defaults: [ - "linux_bionic_supported", - "toolchain_libs_defaults", - ], - } - - cc_prebuilt_library_static { - name: "libclang_rt.fuzzer-x86_64", + name: "libclang_rt.fuzzer", defaults: [ "linux_bionic_supported", "toolchain_libs_defaults", @@ -176,17 +119,12 @@ func commonDefaultModules() string { // Needed for sanitizer cc_prebuilt_library_shared { - name: "libclang_rt.ubsan_standalone-aarch64-android", - defaults: ["toolchain_libs_defaults"], - } - - cc_prebuilt_library_static { - name: "libclang_rt.ubsan_minimal-aarch64-android", + name: "libclang_rt.ubsan_standalone", defaults: ["toolchain_libs_defaults"], } cc_prebuilt_library_static { - name: "libclang_rt.ubsan_minimal-arm-android", + name: "libclang_rt.ubsan_minimal", defaults: ["toolchain_libs_defaults"], } diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go index 8a17e2e54..e7c05aca8 100644 --- a/cc/vendor_snapshot.go +++ b/cc/vendor_snapshot.go @@ -146,6 +146,7 @@ type snapshotJsonFlags struct { // binary flags Symlinks []string `json:",omitempty"` StaticExecutable bool `json:",omitempty"` + InstallInRoot bool `json:",omitempty"` // dependencies SharedLibs []string `json:",omitempty"` @@ -320,6 +321,7 @@ var ccSnapshotAction snapshot.GenerateSnapshotAction = func(s snapshot.SnapshotS // binary flags prop.Symlinks = m.Symlinks() prop.StaticExecutable = m.StaticExecutable() + prop.InstallInRoot = m.InstallInRoot() prop.SharedLibs = m.SnapshotSharedLibs() // static libs dependencies are required to collect the NOTICE files. prop.StaticLibs = m.SnapshotStaticLibs() diff --git a/cc/vndk.go b/cc/vndk.go index c9c9f2c08..bf6148b1c 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -450,7 +450,7 @@ var _ android.OutputFileProducer = &vndkLibrariesTxt{} // Therefore, by removing the library here, we cause it to only be installed if libc // depends on it. func llndkLibrariesTxtFactory() android.SingletonModule { - return newVndkLibrariesWithMakeVarFilter(llndkLibraries, "LLNDK_LIBRARIES", "libclang_rt.hwasan-") + return newVndkLibrariesWithMakeVarFilter(llndkLibraries, "LLNDK_LIBRARIES", "libclang_rt.hwasan") } // vndksp_libraries_txt is a singleton module whose content is a list of VNDKSP libraries diff --git a/dexpreopt/DEXPREOPT_IMPLEMENTATION.md b/dexpreopt/DEXPREOPT_IMPLEMENTATION.md new file mode 100644 index 000000000..c3a17307d --- /dev/null +++ b/dexpreopt/DEXPREOPT_IMPLEMENTATION.md @@ -0,0 +1,258 @@ +## Dexpreopt implementation + +### Introduction + +All dexpreopted Java code falls into three categories: + +- bootclasspath +- system server +- apps and libraries + +Dexpreopt implementation for bootclasspath libraries (boot images) is located in +[soong/java] (see e.g. [soong/java/dexpreopt_bootjars.go]), and install rules +are in [make/core/dex_preopt.mk]. + +Dexpreopt implementation for system server, libraries and apps is located in +[soong/dexpreopt]. For the rest of this section we focus primarily on it (and +not boot images). + +Dexpeopt implementation is split across the Soong part and the Make part. The +core logic is in Soong, and Make only generates configs and scripts to pass +information to Soong. + +### Global and module dexpreopt.config + +The build system generates a global JSON dexpreopt config that is populated from +product variables. This is static configuration that is passed to both Soong and +Make. The `$OUT/soong/dexpreopt.config` file is generated in +[make/core/dex_preopt_config.mk]. Soong reads it in [soong/dexpreopt/config.go] +and makes a device-specific copy (this is needed to ensure incremental build +correctness). The global config contains lists of bootclasspath jars, system +server jars, dex2oat options, global switches that enable and disable parts of +dexpreopt and so on. + +The build system also generates a module config for each dexpreopted package. It +contains package-specific configuration that is derived from the global +configuration and Android.bp or Android.mk module for the package. + +Module configs for Make packages are generated in +[make/core/dex_preopt_odex_install.mk]; they are materialized as per-package +JSON dexpreopt.config files. + +Module configs in Soong are not materialized as dexpreopt.config files and exist +as Go structures in memory, unless it is necessary to materialize them as a file +for dependent Make packages or for post-dexpreopting. Module configs are defined +in [soong/dexpreopt/config.go]. + +### Dexpreopt in Soong + +The Soong implementation of dexpreopt consists roughly of the following steps: + +- Read global dexpreopt config passed from Make ([soong/dexpreopt/config.go]). + +- Construct a static boot image config ([soong/java/dexpreopt_config.go]). + +- During dependency mutator pass, for each suitable module: + - add uses-library dependencies (e.g. for apps: [soong/java/app.go:deps]) + +- During rule generation pass, for each suitable module: + - compute transitive uses-library dependency closure + ([soong/java/java.go:addCLCFromDep]) + + - construct CLC from the dependency closure + ([soong/dexpreopt/class_loader_context.go]) + + - construct module config with CLC, boot image locations, etc. + ([soong/java/dexpreopt.go]) + + - generate build rules to verify build-time CLC against the manifest (e.g. + for apps: [soong/java/app.go:verifyUsesLibraries]) + + - generate dexpreopt build rule ([soong/dexpreopt/dexpreopt.go]) + +- At the end of rule generation pass: + - generate build rules for boot images ([soong/java/dexpreopt_bootjars.go], + [soong/java/bootclasspath_fragment.go] and + [soong/java/platform_bootclasspath.go]) + +### Dexpreopt in Make - dexpreopt_gen + +In order to reuse the same dexpreopt implementation for both Soong and Make +packages, part of Soong is compiled into a standalone binary dexpreopt_gen. It +runs during the Ninja stage of the build and generates shell scripts with +dexpreopt build rules for Make packages, and then executes them. + +This setup causes many inconveniences. To name a few: + +- Errors in the build rules are only revealed at the late stage of the build. + +- These rules are not tested by the presubmit builds that run `m nothing` on + many build targets/products. + +- It is impossible to find dexpreopt build rules in the generated Ninja files. + +However all these issues are a lesser evil compared to having a duplicate +dexpreopt implementation in Make. Also note that it would be problematic to +reimplement the logic in Make anyway, because Android.mk modules are not +processed in the order of uses-library dependencies and propagating dependency +information from one module to another would require a similar workaround with +a script. + +Dexpreopt for Make packages involves a few steps: + +- At Soong phase (during `m nothing`), see dexpreopt_gen: + - generate build rules for dexpreopt_gen binary + +- At Make/Kati phase (during `m nothing`), see + [make/core/dex_preopt_odex_install.mk]: + - generate build rules for module dexpreopt.config + + - generate build rules for merging dependency dexpreopt.config files (see + [make/core/dex_preopt_config_merger.py]) + + - generate build rules for dexpreopt_gen invocation + + - generate build rules for executing dexpreopt.sh scripts + +- At Ninja phase (during `m`): + - generate dexpreopt.config files + + - execute dexpreopt_gen rules (generate dexpreopt.sh scripts) + + - execute dexpreopt.sh scripts (this runs the actual dexpreopt) + +The Make/Kati phase adds all the necessary dependencies that trigger +dexpreopt_gen and dexpreopt.sh rules. The real dexpreopt command (dex2oat +invocation that will be executed to AOT-compile a package) is in the +dexpreopt.sh script, which is generated close to the end of the build. + +### Indirect build rules + +The process described above for Make packages involves "indirect build rules", +i.e. build rules that are generated not at the time when the build system is +created (which is a small step at the very beginning of the build triggered with +`m nothing`), but at the time when the actual build is done (`m` phase). + +Some build systems, such as Make, allow modifications of the build graph during +the build. Other build systems, such as Soong, have a clear separation into the +first "generation phase" (this is when build rules are created) and the second +"build phase" (this is when the build rules are executed), and they do not allow +modifications of the dependency graph during the second phase. The Soong +approach is better from performance standpoint, because with the Make approach +there are no guarantees regarding the time of the build --- recursive build +graph modfications continue until fixpoint. However the Soong approach is also +more restictive, as it can only generate build rules from the information that +is passed to the build system via global configuration, Android.bp files or +encoded in the Go code. Any other information (such as the contents of the Java +manifest files) are not accessible and cannot be used to generate build rules. + +Hence the need for the "indirect build rules": during the generation phase only +stubs of the build rules are generated, and the real rules are generated by the +stub rules during the build phase (and executed immediately). Note that the +build system still has to add all the necessary dependencies during the +generation phase, because it will not be possible to change build order during +the build phase. + +Indirect buils rules are used in a couple of places in dexpreopt: + +- [soong/scripts/manifest_check.py]: first to extract targetSdkVersion from the + manifest, and later to extract `<uses-library/>` tags from the manifest and + compare them to the uses-library list known to the build system + +- [soong/scripts/construct_context.py]: to trim compatibility libraries in CLC + +- [make/core/dex_preopt_config_merger.py]: to merge information from + dexpreopt.config files for uses-library dependencies into the dependent's + dexpreopt.config file (mostly the CLC) + +- autogenerated dexpreopt.sh scripts: to call dexpreopt_gen + +### Consistency check - manifest_check.py + +Because the information from the manifests has to be duplicated in the +Android.bp/Android.mk files, there is a danger that it may get out of sync. To +guard against that, the build system generates a rule that verifies +uses-libraries: checks the metadata in the build files against the contents of a +manifest. The manifest can be available as a source file, or as part of a +prebuilt APK. + +The check is implemented in [soong/scripts/manifest_check.py]. + +It is possible to turn off the check globally for a product by setting +`PRODUCT_BROKEN_VERIFY_USES_LIBRARIES := true` in a product makefile, or for a +particular build by setting `RELAX_USES_LIBRARY_CHECK=true`. + +### Compatibility libraries - construct_context.py + +Compatibility libraries are libraries that didn’t exist prior to a certain SDK +version (say, `N`), but classes in them were in the bootclasspath jars, etc., +and in version `N` they have been separated into a standalone uses-library. +Compatibility libraries should only be in the CLC of an app if its +`targetSdkVersion` in the manifest is less than `N`. + +Currently compatibility libraries only affect apps (but not other libraries). + +The build system cannot see `targetSdkVersion` of an app at the time it +generates dexpreopt build rules, so it doesn't know whether to add compatibility +libaries to CLC or not. As a workaround, the build system includes all +compatibility libraries regardless of the app version, and appends some extra +logic to the dexpreopt rule that will extract `targetSdkVersion` from the +manifest and filter CLC based on that version during Ninja stage of the build, +immediately before executing the dexpreopt command (see the +soong/scripts/construct_context.py script). + +As of the time of writing (January 2022), there are the following compatibility +libraries: + +- org.apache.http.legacy (SDK 28) +- android.hidl.base-V1.0-java (SDK 29) +- android.hidl.manager-V1.0-java (SDK 29) +- android.test.base (SDK 30) +- android.test.mock (SDK 30) + +### Manifest fixer + +Sometimes uses-library tags are missing from the source manifest of a +library/app. This may happen for example if one of the transitive dependencies +of the library/app starts using another uses-library, and the library/app's +manifest isn't updated to include it. + +Soong can compute some of the missing uses-library tags for a given library/app +automatically as SDK libraries in the transitive dependency closure of the +library/app. The closure is needed because a library/app may depend on a static +library that may in turn depend on an SDK library (possibly transitively via +another library). + +Not all uses-library tags can be computed in this way, because some of the +uses-library dependencies are not SDK libraries, or they are not reachable via +transitive dependency closure. But when possible, allowing Soong to calculate +the manifest entries is less prone to errors and simplifies maintenance. For +example, consider a situation when many apps use some static library that adds a +new uses-library dependency -- all the apps will have to be updated. That is +difficult to maintain. + +There is also a manifest merger, because sometimes the final manifest of an app +is merged from a few dependency manifests, so the final manifest installed on +devices contains a superset of uses-library tags of the source manifest of the +app. + + +[make/core/dex_preopt.mk]: https://cs.android.com/android/platform/superproject/+/master:build/make/core/dex_preopt.mk +[make/core/dex_preopt_config.mk]: https://cs.android.com/android/platform/superproject/+/master:build/make/core/dex_preopt_config.mk +[make/core/dex_preopt_config_merger.py]: https://cs.android.com/android/platform/superproject/+/master:build/make/core/dex_preopt_config_merger.py +[make/core/dex_preopt_odex_install.mk]: https://cs.android.com/android/platform/superproject/+/master:build/make/core/dex_preopt_odex_install.mk +[soong/dexpreopt]: https://cs.android.com/android/platform/superproject/+/master:build/soong/dexpreopt +[soong/dexpreopt/class_loader_context.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/dexpreopt/class_loader_context.go +[soong/dexpreopt/config.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/dexpreopt/config.go +[soong/dexpreopt/dexpreopt.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/dexpreopt/dexpreopt.go +[soong/java]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java +[soong/java/app.go:deps]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/app.go?q=%22func%20\(u%20*usesLibrary\)%20deps%22 +[soong/java/app.go:verifyUsesLibraries]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/app.go?q=%22func%20\(u%20*usesLibrary\)%20verifyUsesLibraries%22 +[soong/java/bootclasspath_fragment.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/bootclasspath_fragment.go +[soong/java/dexpreopt.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/dexpreopt.go +[soong/java/dexpreopt_bootjars.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/dexpreopt_bootjars.go +[soong/java/dexpreopt_config.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/dexpreopt_config.go +[soong/java/java.go:addCLCFromDep]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/java.go?q=%22func%20addCLCfromDep%22 +[soong/java/platform_bootclasspath.go]: https://cs.android.com/android/platform/superproject/+/master:build/soong/java/platform_bootclasspath.go +[soong/scripts/construct_context.py]: https://cs.android.com/android/platform/superproject/+/master:build/soong/scripts/construct_context.py +[soong/scripts/manifest_check.py]: https://cs.android.com/android/platform/superproject/+/master:build/soong/scripts/manifest_check.py diff --git a/docs/rbe.json b/docs/rbe.json new file mode 100644 index 000000000..f6ff10772 --- /dev/null +++ b/docs/rbe.json @@ -0,0 +1,24 @@ +{ + "env": { + "USE_RBE": "1", + + "RBE_R8_EXEC_STRATEGY": "remote_local_fallback", + "RBE_CXX_EXEC_STRATEGY": "remote_local_fallback", + "RBE_D8_EXEC_STRATEGY": "remote_local_fallback", + "RBE_JAVAC_EXEC_STRATEGY": "remote_local_fallback", + "RBE_JAVAC": "1", + "RBE_R8": "1", + "RBE_D8": "1", + + "RBE_instance": "[replace with your RBE instance]", + "RBE_service": "[replace with your RBE service endpoint]", + + "RBE_DIR": "prebuilts/remoteexecution-client/live", + + "RBE_use_application_default_credentials": "true", + + "RBE_log_dir": "/tmp", + "RBE_output_dir": "/tmp", + "RBE_proxy_log_dir": "/tmp" + } +} diff --git a/docs/rbe.md b/docs/rbe.md new file mode 100644 index 000000000..cfe86d765 --- /dev/null +++ b/docs/rbe.md @@ -0,0 +1,70 @@ +# Build Android Platform on Remote Build Execution + +Soong is integrated with Google's Remote Build Execution(RBE) service, which +implements the +[Remote Executaion API](https://github.com/bazelbuild/remote-apis). + +With RBE enabled, it can speed up the Android Platform builds by distributing +build actions through a worker pool sharing a central cache of build results. + +## Configuration + +To enable RBE, you need to set several environment variables before triggering +the build. You can set them through a +[environment variables config file](https://android.googlesource.com/platform/build/soong/+/master/README.md#environment-variables-config-file). +As an example, [build/soong/docs/rbe.json](rbe.json) is a config that enables +RBE in the build. Once the config file is created, you need to let Soong load +the config file by specifying `ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR` environment +variable and `ANDROID_BUILD_ENVIRONMENT_CONFIG` environment variable. The +following command starts Soong with [build/soong/docs/rbe.json](rbe.json) +loaded: + +```shell +ANDROID_BUILD_ENVIRONMENT_CONFIG=rbe \ +ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR=build/soong/doc \ + build/soong/soong_ui.bash +``` + +### Configuration Explanation + +Below a brief explanation of each field in +[build/soong/docs/rbe.json](rbe.json): + +##### USE\_RBE: +If set to 1, enable RBE for the build. + +##### RBE\_CXX\_EXEC\_STRATEGY / RBE\_JAVAC\_EXEC\_STRATEGY / RBE\_R8\_EXEC\_STRATEGY / RBE\_D8\_EXEC\_STRATEGY: + +Sets strategies for C++/javac/r8/d8 action types. Available options are +(**Note**: all options will update the remote cache if the right permissions to +update cache are given to the user.): + +* **local**: Only execute locally. +* **remote**: Only execute remotely. +* **remote_local_fallback**: Try executing remotely and fall back to local + execution if failed. +* **racing**: Race remote execution and local execution and use the earlier + result. + +##### RBE\_JAVAC / RBE\_R8 / RBE\_D8 + +If set to 1, enable javac/r8/d8 support. C++ compilation is enabled by default. + +##### RBE\_service / RBE\_instance + +The remote execution service endpoint and instance ID to target when calling +remote execution via gRPC to execute actions. + +##### RBE\_DIR + +Where to find remote client binaries (rewrapper, reproxy) + +##### RBE\_use\_application\_default\_credentials + +reclient uses +[application default credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login) +for autentication, as generated by `gcloud auth application-default login` + +##### RBE\_log\_dir/RBE\_proxy\_log\_dir/RBE\_output\_dir + +Logs generated by rewrapper and reproxy will go here. diff --git a/java/base.go b/java/base.go index 8747039da..9978a66fc 100644 --- a/java/base.go +++ b/java/base.go @@ -481,6 +481,8 @@ type Module struct { sdkVersion android.SdkSpec minSdkVersion android.SdkSpec maxSdkVersion android.SdkSpec + + sourceExtensions []string } func (j *Module) CheckStableSdkVersion(ctx android.BaseModuleContext) error { @@ -982,6 +984,14 @@ func (j *Module) collectJavacFlags( return flags } +func (j *Module) AddJSONData(d *map[string]interface{}) { + (&j.ModuleBase).AddJSONData(d) + (*d)["Java"] = map[string]interface{}{ + "SourceExtensions": j.sourceExtensions, + } + +} + func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.deviceProperties.Aidl.Export_include_dirs) @@ -993,6 +1003,12 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { } srcFiles := android.PathsForModuleSrcExcludes(ctx, j.properties.Srcs, j.properties.Exclude_srcs) + j.sourceExtensions = []string{} + for _, ext := range []string{".kt", ".proto", ".aidl", ".java", ".logtags"} { + if hasSrcExt(srcFiles.Strings(), ext) { + j.sourceExtensions = append(j.sourceExtensions, ext) + } + } if hasSrcExt(srcFiles.Strings(), ".proto") { flags = protoFlags(ctx, &j.properties, &j.protoProperties, flags) } diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index 5fe409e25..c3a5d5f70 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -16,6 +16,7 @@ package java import ( "fmt" + "io" "path/filepath" "reflect" "strings" @@ -139,7 +140,7 @@ type bootclasspathFragmentProperties struct { BootclasspathFragmentsDepsProperties } -type SourceOnlyBootclasspathProperties struct { +type HiddenApiPackageProperties struct { Hidden_api struct { // Contains prefixes of a package hierarchy that is provided solely by this // bootclasspath_fragment. @@ -148,6 +149,14 @@ type SourceOnlyBootclasspathProperties struct { // hidden API flags. See split_packages property for more details. Package_prefixes []string + // A list of individual packages that are provided solely by this + // bootclasspath_fragment but which cannot be listed in package_prefixes + // because there are sub-packages which are provided by other modules. + // + // This should only be used for legacy packages. New packages should be + // covered by a package prefix. + Single_packages []string + // The list of split packages provided by this bootclasspath_fragment. // // A split package is one that contains classes which are provided by multiple @@ -207,6 +216,11 @@ type SourceOnlyBootclasspathProperties struct { } } +type SourceOnlyBootclasspathProperties struct { + HiddenApiPackageProperties + Coverage HiddenApiPackageProperties +} + type BootclasspathFragmentModule struct { android.ModuleBase android.ApexModuleBase @@ -270,6 +284,12 @@ func bootclasspathFragmentFactory() android.Module { ctx.PropertyErrorf("coverage", "error trying to append coverage specific properties: %s", err) return } + + err = proptools.AppendProperties(&m.sourceOnlyProperties.HiddenApiPackageProperties, &m.sourceOnlyProperties.Coverage, nil) + if err != nil { + ctx.PropertyErrorf("coverage", "error trying to append hidden api coverage specific properties: %s", err) + return + } } // Initialize the contents property from the image_name. @@ -588,6 +608,19 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo // Provide the apex content info. b.provideApexContentInfo(ctx, imageConfig, hiddenAPIOutput, bootImageFilesByArch) } + } else { + // Versioned fragments are not needed by make. + b.HideFromMake() + } + + // In order for information about bootclasspath_fragment modules to be added to module-info.json + // it is necessary to output an entry to Make. As bootclasspath_fragment modules are part of an + // APEX there can be multiple variants, including the default/platform variant and only one can + // be output to Make but it does not really matter which variant is output. The default/platform + // variant is the first (ctx.PrimaryModule()) and is usually hidden from make so this just picks + // the last variant (ctx.FinalModule()). + if ctx.Module() != ctx.FinalModule() { + b.HideFromMake() } } @@ -717,7 +750,8 @@ func (b *BootclasspathFragmentModule) generateHiddenAPIBuildActions(ctx android. // TODO(b/192868581): Remove once the source and prebuilts provide a signature patterns file of // their own. if output.SignaturePatternsPath == nil { - output.SignaturePatternsPath = buildRuleSignaturePatternsFile(ctx, output.AllFlagsPath, []string{"*"}, nil) + output.SignaturePatternsPath = buildRuleSignaturePatternsFile( + ctx, output.AllFlagsPath, []string{"*"}, nil, nil) } // Initialize a HiddenAPIInfo structure. @@ -792,11 +826,13 @@ func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleC // signature patterns. splitPackages := b.sourceOnlyProperties.Hidden_api.Split_packages packagePrefixes := b.sourceOnlyProperties.Hidden_api.Package_prefixes - if splitPackages != nil || packagePrefixes != nil { + singlePackages := b.sourceOnlyProperties.Hidden_api.Single_packages + if splitPackages != nil || packagePrefixes != nil || singlePackages != nil { if splitPackages == nil { splitPackages = []string{"*"} } - output.SignaturePatternsPath = buildRuleSignaturePatternsFile(ctx, output.AllFlagsPath, splitPackages, packagePrefixes) + output.SignaturePatternsPath = buildRuleSignaturePatternsFile( + ctx, output.AllFlagsPath, splitPackages, packagePrefixes, singlePackages) } return output @@ -849,7 +885,22 @@ func (b *BootclasspathFragmentModule) generateBootImageBuildActions(ctx android. } func (b *BootclasspathFragmentModule) AndroidMkEntries() []android.AndroidMkEntries { - var entriesList []android.AndroidMkEntries + // Use the generated classpath proto as the output. + outputFile := b.outputFilepath + // Create a fake entry that will cause this to be added to the module-info.json file. + entriesList := []android.AndroidMkEntries{{ + Class: "FAKE", + OutputFile: android.OptionalPathForPath(outputFile), + Include: "$(BUILD_PHONY_PACKAGE)", + ExtraFooters: []android.AndroidMkExtraFootersFunc{ + func(w io.Writer, name, prefix, moduleDir string) { + // Allow the bootclasspath_fragment to be built by simply passing its name on the command + // line. + fmt.Fprintln(w, ".PHONY:", b.Name()) + fmt.Fprintln(w, b.Name()+":", outputFile.String()) + }, + }, + }} for _, install := range b.bootImageDeviceInstalls { entriesList = append(entriesList, install.ToMakeEntries()) } diff --git a/java/config/config.go b/java/config/config.go index 39584cb83..05dfde635 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -105,7 +105,12 @@ func init() { if override := ctx.Config().Getenv("OVERRIDE_JLINK_VERSION_NUMBER"); override != "" { return override } - return "11" + switch ctx.Config().Getenv("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN") { + case "true": + return "17" + default: + return "11" + } }) pctx.SourcePathVariable("JavaToolchain", "${JavaHome}/bin") diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index cad9c332a..3d91aec91 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -276,11 +276,17 @@ type bootImageConfig struct { // Rules which should be used in make to install the outputs. profileInstalls android.RuleBuilderInstalls + // Path to the license metadata file for the module that built the profile. + profileLicenseMetadataFile android.OptionalPath + // Path to the image profile file on host (or empty, if profile is not generated). profilePathOnHost android.Path // Target-dependent fields. variants []*bootImageVariant + + // Path of the preloaded classes file. + preloadedClassesFile string } // Target-dependent description of a boot image. @@ -320,6 +326,9 @@ type bootImageVariant struct { // Rules which should be used in make to install the outputs on device. deviceInstalls android.RuleBuilderInstalls + + // Path to the license metadata file for the module that built the image. + licenseMetadataFile android.OptionalPath } // Get target-specific boot image variant for the given boot image config and target. @@ -680,6 +689,13 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress()) } + // We always expect a preloaded classes file to be available. However, if we cannot find it, it's + // OK to not pass the flag to dex2oat. + preloadedClassesPath := android.ExistentPathForSource(ctx, image.preloadedClassesFile) + if preloadedClassesPath.Valid() { + cmd.FlagWithInput("--preloaded-classes=", preloadedClassesPath.Path()) + } + cmd. FlagForEachInput("--dex-file=", image.dexPaths.Paths()). FlagForEachArg("--dex-location=", image.dexLocations). @@ -759,6 +775,7 @@ func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, p image.vdexInstalls = vdexInstalls image.unstrippedInstalls = unstrippedInstalls image.deviceInstalls = deviceInstalls + image.licenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) } const failureMessage = `ERROR: Dex2oat failed to compile a boot image. @@ -807,6 +824,7 @@ func bootImageProfileRule(ctx android.ModuleContext, image *bootImageConfig) and if image == defaultBootImageConfig(ctx) { rule.Install(profile, "/system/etc/boot-image.prof") image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + image.profileLicenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) } rule.Build("bootJarsProfile", "profile boot jars") @@ -844,6 +862,7 @@ func bootFrameworkProfileRule(ctx android.ModuleContext, image *bootImageConfig) rule.Install(profile, "/system/etc/boot-image.bprof") rule.Build("bootFrameworkProfile", "profile boot framework jars") image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + image.profileLicenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) return profile } @@ -909,6 +928,9 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { image := d.defaultBootImage if image != nil { ctx.Strict("DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED", image.profileInstalls.String()) + if image.profileLicenseMetadataFile.Valid() { + ctx.Strict("DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA", image.profileLicenseMetadataFile.String()) + } global := dexpreopt.GetGlobalConfig(ctx) dexPaths, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) @@ -934,6 +956,9 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_IMAGE_DEPS_"+sfx, strings.Join(variant.imagesDeps.Strings(), " ")) ctx.Strict("DEXPREOPT_IMAGE_BUILT_INSTALLED_"+sfx, variant.installs.String()) ctx.Strict("DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_"+sfx, variant.unstrippedInstalls.String()) + if variant.licenseMetadataFile.Valid() { + ctx.Strict("DEXPREOPT_IMAGE_LICENSE_METADATA_"+sfx, variant.licenseMetadataFile.String()) + } } imageLocationsOnHost, imageLocationsOnDevice := current.getAnyAndroidVariant().imageLocations() ctx.Strict("DEXPREOPT_IMAGE_LOCATIONS_ON_HOST"+current.name, strings.Join(imageLocationsOnHost, ":")) diff --git a/java/dexpreopt_config.go b/java/dexpreopt_config.go index 21e1d12c0..4d0bd09c6 100644 --- a/java/dexpreopt_config.go +++ b/java/dexpreopt_config.go @@ -62,18 +62,20 @@ func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig installDirOnDevice: "system/framework", profileInstallPathInApex: "etc/boot-image.prof", modules: artModules, + preloadedClassesFile: "art/build/boot/preloaded-classes", } // Framework config for the boot image extension. // It includes framework libraries and depends on the ART config. frameworkSubdir := "system/framework" frameworkCfg := bootImageConfig{ - extends: &artCfg, - name: frameworkBootImageName, - stem: "boot", - installDirOnHost: frameworkSubdir, - installDirOnDevice: frameworkSubdir, - modules: frameworkModules, + extends: &artCfg, + name: frameworkBootImageName, + stem: "boot", + installDirOnHost: frameworkSubdir, + installDirOnDevice: frameworkSubdir, + modules: frameworkModules, + preloadedClassesFile: "frameworks/base/config/preloaded-classes", } return map[string]*bootImageConfig{ diff --git a/java/droidstubs.go b/java/droidstubs.go index 5dc7bc9ab..2921c3e82 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -334,7 +334,11 @@ func (d *Droidstubs) annotationsFlags(ctx android.ModuleContext, cmd *android.Ru // TODO(tnorbye): find owners to fix these warnings when annotation was enabled. cmd.FlagWithArg("--hide ", "HiddenTypedefConstant"). FlagWithArg("--hide ", "SuperfluousPrefix"). - FlagWithArg("--hide ", "AnnotationExtraction") + FlagWithArg("--hide ", "AnnotationExtraction"). + // b/222738070 + FlagWithArg("--hide ", "BannedThrow"). + // b/223382732 + FlagWithArg("--hide ", "ChangedDefault") } } @@ -473,7 +477,9 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi Flag("--format=v2"). FlagWithArg("--repeat-errors-max ", "10"). FlagWithArg("--hide ", "UnresolvedImport"). - FlagWithArg("--hide ", "InvalidNullability") + FlagWithArg("--hide ", "InvalidNullability"). + // b/223382732 + FlagWithArg("--hide ", "ChangedDefault") return cmd } diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index 0cc960d5c..95ded34bb 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -943,7 +943,9 @@ func (s SignatureCsvSubsets) RelativeToTop() []string { // buildRuleSignaturePatternsFile creates a rule to generate a file containing the set of signature // patterns that will select a subset of the monolithic flags. -func buildRuleSignaturePatternsFile(ctx android.ModuleContext, flagsPath android.Path, splitPackages []string, packagePrefixes []string) android.Path { +func buildRuleSignaturePatternsFile( + ctx android.ModuleContext, flagsPath android.Path, + splitPackages []string, packagePrefixes []string, singlePackages []string) android.Path { patternsFile := android.PathForModuleOut(ctx, "modular-hiddenapi", "signature-patterns.csv") // Create a rule to validate the output from the following rule. rule := android.NewRuleBuilder(pctx, ctx) @@ -959,6 +961,7 @@ func buildRuleSignaturePatternsFile(ctx android.ModuleContext, flagsPath android FlagWithInput("--flags ", flagsPath). FlagForEachArg("--split-package ", quotedSplitPackages). FlagForEachArg("--package-prefix ", packagePrefixes). + FlagForEachArg("--single-package ", singlePackages). FlagWithOutput("--output ", patternsFile) rule.Build("hiddenAPISignaturePatterns", "hidden API signature patterns") diff --git a/java/java.go b/java/java.go index d0f0abc7a..895ce7af1 100644 --- a/java/java.go +++ b/java/java.go @@ -2011,8 +2011,16 @@ type javaLibraryAttributes struct { } func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) *javaLibraryAttributes { - //TODO(b/209577426): Support multiple arch variants - srcs := bazel.MakeLabelListAttribute(android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs)) + var srcs bazel.LabelListAttribute + archVariantProps := m.GetArchVariantProperties(ctx, &CommonProperties{}) + for axis, configToProps := range archVariantProps { + for config, _props := range configToProps { + if archProps, ok := _props.(*CommonProperties); ok { + archSrcs := android.BazelLabelForModuleSrcExcludes(ctx, archProps.Srcs, archProps.Exclude_srcs) + srcs.SetSelectValue(axis, config, archSrcs) + } + } + } javaSrcPartition := "java" protoSrcPartition := "proto" @@ -2030,6 +2038,11 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) } var deps bazel.LabelList + sdkVersion := m.SdkVersion(ctx) + if sdkVersion.Kind == android.SdkPublic && sdkVersion.ApiLevel == android.FutureApiLevel { + // TODO(b/220869005) remove forced dependency on current public android.jar + deps.Add(&bazel.Label{Label: "//prebuilts/sdk:public_current_android_sdk_java_import"}) + } if m.properties.Libs != nil { deps.Append(android.BazelLabelForModuleDeps(ctx, m.properties.Libs)) } diff --git a/java/legacy_core_platform_api_usage.go b/java/legacy_core_platform_api_usage.go index e3396c12f..8e224914d 100644 --- a/java/legacy_core_platform_api_usage.go +++ b/java/legacy_core_platform_api_usage.go @@ -81,7 +81,6 @@ var legacyCorePlatformApiModules = []string{ "ds-car-docs", // for AAOS API documentation only "DynamicSystemInstallationService", "EmergencyInfo-lib", - "ethernet-service", "EthernetServiceTests", "ExternalStorageProvider", "face-V1-0-javalib", diff --git a/linkerconfig/linkerconfig.go b/linkerconfig/linkerconfig.go index dbc112e5a..003b27507 100644 --- a/linkerconfig/linkerconfig.go +++ b/linkerconfig/linkerconfig.go @@ -158,7 +158,6 @@ func (l *linkerConfig) AndroidMkEntries() []android.AndroidMkEntries { entries.SetString("LOCAL_MODULE_PATH", l.installDirPath.String()) entries.SetString("LOCAL_INSTALLED_MODULE_STEM", l.outputFilePath.Base()) entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !installable) - entries.SetString("LINKER_CONFIG_PATH_"+l.Name(), l.OutputFile().String()) }, }, }} diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 74053fefb..212b7f658 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -50,15 +50,12 @@ const ( soongNsPrefix = "SOONG_CONFIG_" // And here are the functions and variables: - cfnGetCfg = baseName + ".cfg" - cfnMain = baseName + ".product_configuration" - cfnBoardMain = baseName + ".board_configuration" - cfnPrintVars = baseName + ".printvars" - cfnWarning = baseName + ".warning" - cfnLocalAppend = baseName + ".local_append" - cfnLocalSetDefault = baseName + ".local_set_default" - cfnInherit = baseName + ".inherit" - cfnSetListDefault = baseName + ".setdefault" + cfnGetCfg = baseName + ".cfg" + cfnMain = baseName + ".product_configuration" + cfnBoardMain = baseName + ".board_configuration" + cfnPrintVars = baseName + ".printvars" + cfnInherit = baseName + ".inherit" + cfnSetListDefault = baseName + ".setdefault" ) const ( @@ -410,6 +407,8 @@ type parseContext struct { dependentModules map[string]*moduleInfo soongNamespaces map[string]map[string]bool includeTops []string + typeHints map[string]starlarkType + atTopOfMakefile bool } func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { @@ -453,6 +452,8 @@ func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { dependentModules: make(map[string]*moduleInfo), soongNamespaces: make(map[string]map[string]bool), includeTops: []string{}, + typeHints: make(map[string]starlarkType), + atTopOfMakefile: true, } ctx.pushVarAssignments() for _, item := range predefined { @@ -465,17 +466,17 @@ func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { return ctx } -func (ctx *parseContext) lastAssignment(name string) *assignmentNode { +func (ctx *parseContext) lastAssignment(v variable) *assignmentNode { for va := ctx.varAssignments; va != nil; va = va.outer { - if v, ok := va.vars[name]; ok { + if v, ok := va.vars[v.name()]; ok { return v } } return nil } -func (ctx *parseContext) setLastAssignment(name string, asgn *assignmentNode) { - ctx.varAssignments.vars[name] = asgn +func (ctx *parseContext) setLastAssignment(v variable, asgn *assignmentNode) { + ctx.varAssignments.vars[v.name()] = asgn } func (ctx *parseContext) pushVarAssignments() { @@ -532,7 +533,7 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs == nil { return []starlarkNode{ctx.newBadNode(a, "unknown variable %s", name)} } - _, isTraced := ctx.tracedVariables[name] + _, isTraced := ctx.tracedVariables[lhs.name()] asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced, location: ctx.errorLocation(a)} if lhs.valueType() == starlarkTypeUnknown { // Try to divine variable type from the RHS @@ -565,17 +566,19 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode } } - asgn.previous = ctx.lastAssignment(name) - ctx.setLastAssignment(name, asgn) + if asgn.lhs.valueType() == starlarkTypeString && + asgn.value.typ() != starlarkTypeUnknown && + asgn.value.typ() != starlarkTypeString { + asgn.value = &toStringExpr{expr: asgn.value} + } + + asgn.previous = ctx.lastAssignment(lhs) + ctx.setLastAssignment(lhs, asgn) switch a.Type { case "=", ":=": asgn.flavor = asgnSet case "+=": - if asgn.previous == nil && !asgn.lhs.isPreset() { - asgn.flavor = asgnMaybeAppend - } else { - asgn.flavor = asgnAppend - } + asgn.flavor = asgnAppend case "?=": asgn.flavor = asgnMaybeSet default: @@ -1268,12 +1271,12 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt args: []starlarkExpr{ &stringLiteralExpr{literal: substParts[0]}, &stringLiteralExpr{literal: substParts[1]}, - NewVariableRefExpr(v, ctx.lastAssignment(v.name()) != nil), + NewVariableRefExpr(v, ctx.lastAssignment(v) != nil), }, } } if v := ctx.addVariable(refDump); v != nil { - return NewVariableRefExpr(v, ctx.lastAssignment(v.name()) != nil) + return NewVariableRefExpr(v, ctx.lastAssignment(v) != nil) } return ctx.newBadExpr(node, "unknown variable %s", refDump) } @@ -1687,7 +1690,8 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNod // Clear the includeTops after each non-comment statement // so that include annotations placed on certain statements don't apply // globally for the rest of the makefile was well. - if _, wasComment := node.(*mkparser.Comment); !wasComment && len(ctx.includeTops) > 0 { + if _, wasComment := node.(*mkparser.Comment); !wasComment { + ctx.atTopOfMakefile = false ctx.includeTops = []string{} } @@ -1697,6 +1701,12 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNod return result } +// The types allowed in a type_hint +var typeHintMap = map[string]starlarkType{ + "string": starlarkTypeString, + "list": starlarkTypeList, +} + // Processes annotation. An annotation is a comment that starts with #RBC# and provides // a conversion hint -- say, where to look for the dynamically calculated inherit/include // paths. Returns true if the comment was a successfully-handled annotation. @@ -1721,6 +1731,35 @@ func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) (starlar } ctx.includeTops = append(ctx.includeTops, p) return nil, true + } else if p, ok := maybeTrim(annotation, "type_hint"); ok { + // Type hints must come at the beginning the file, to avoid confusion + // if a type hint was specified later and thus only takes effect for half + // of the file. + if !ctx.atTopOfMakefile { + return ctx.newBadNode(cnode, "type_hint annotations must come before the first Makefile statement"), true + } + + parts := strings.Fields(p) + if len(parts) <= 1 { + return ctx.newBadNode(cnode, "Invalid type_hint annotation: %s. Must be a variable type followed by a list of variables of that type", p), true + } + + var varType starlarkType + if varType, ok = typeHintMap[parts[0]]; !ok { + varType = starlarkTypeUnknown + } + if varType == starlarkTypeUnknown { + return ctx.newBadNode(cnode, "Invalid type_hint annotation. Only list/string types are accepted, found %s", parts[0]), true + } + + for _, name := range parts[1:] { + // Don't allow duplicate type hints + if _, ok := ctx.typeHints[name]; ok { + return ctx.newBadNode(cnode, "Duplicate type hint for variable %s", name), true + } + ctx.typeHints[name] = varType + } + return nil, true } return ctx.newBadNode(cnode, "unsupported annotation %s", cnode.Comment), true } diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index 55d48ff54..08926e5fa 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -65,6 +65,10 @@ def init(g, handle): PRODUCT_NAME := Pixel 3 PRODUCT_MODEL := local_var = foo +local-var-with-dashes := bar +$(warning local-var-with-dashes: $(local-var-with-dashes)) +GLOBAL-VAR-WITH-DASHES := baz +$(warning GLOBAL-VAR-WITH-DASHES: $(GLOBAL-VAR-WITH-DASHES)) `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -73,6 +77,10 @@ def init(g, handle): cfg["PRODUCT_NAME"] = "Pixel 3" cfg["PRODUCT_MODEL"] = "" _local_var = "foo" + _local_var_with_dashes = "bar" + rblf.mkwarning("pixel3.mk", "local-var-with-dashes: %s" % _local_var_with_dashes) + g["GLOBAL-VAR-WITH-DASHES"] = "baz" + rblf.mkwarning("pixel3.mk", "GLOBAL-VAR-WITH-DASHES: %s" % g["GLOBAL-VAR-WITH-DASHES"]) `, }, { @@ -893,6 +901,43 @@ def init(g, handle): `, }, { + desc: "assigment setdefaults", + mkname: "product.mk", + in: ` +# All of these should have a setdefault because they're self-referential and not defined before +PRODUCT_LIST1 = a $(PRODUCT_LIST1) +PRODUCT_LIST2 ?= a $(PRODUCT_LIST2) +PRODUCT_LIST3 += a + +# Now doing them again should not have a setdefault because they've already been set +PRODUCT_LIST1 = a $(PRODUCT_LIST1) +PRODUCT_LIST2 ?= a $(PRODUCT_LIST2) +PRODUCT_LIST3 += a +`, + expected: `# All of these should have a setdefault because they're self-referential and not defined before +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.setdefault(handle, "PRODUCT_LIST1") + cfg["PRODUCT_LIST1"] = (["a"] + + cfg.get("PRODUCT_LIST1", [])) + if cfg.get("PRODUCT_LIST2") == None: + rblf.setdefault(handle, "PRODUCT_LIST2") + cfg["PRODUCT_LIST2"] = (["a"] + + cfg.get("PRODUCT_LIST2", [])) + rblf.setdefault(handle, "PRODUCT_LIST3") + cfg["PRODUCT_LIST3"] += ["a"] + # Now doing them again should not have a setdefault because they've already been set + cfg["PRODUCT_LIST1"] = (["a"] + + cfg["PRODUCT_LIST1"]) + if cfg.get("PRODUCT_LIST2") == None: + cfg["PRODUCT_LIST2"] = (["a"] + + cfg["PRODUCT_LIST2"]) + cfg["PRODUCT_LIST3"] += ["a"] +`, + }, + { desc: "soong namespace assignments", mkname: "product.mk", in: ` @@ -985,6 +1030,7 @@ endif def init(g, handle): cfg = rblf.cfg(handle) if "hwaddress" not in cfg.get("PRODUCT_PACKAGES", []): + rblf.setdefault(handle, "PRODUCT_PACKAGES") cfg["PRODUCT_PACKAGES"] = (rblf.mkstrip("%s hwaddress" % " ".join(cfg.get("PRODUCT_PACKAGES", [])))).split() `, }, @@ -1219,7 +1265,7 @@ TEST_VAR_LIST := foo TEST_VAR_LIST += bar TEST_VAR_2 := $(if $(TEST_VAR),bar) TEST_VAR_3 := $(if $(TEST_VAR),bar,baz) -TEST_VAR_3 := $(if $(TEST_VAR),$(TEST_VAR_LIST)) +TEST_VAR_4 := $(if $(TEST_VAR),$(TEST_VAR_LIST)) `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -1230,7 +1276,7 @@ def init(g, handle): g["TEST_VAR_LIST"] += ["bar"] g["TEST_VAR_2"] = ("bar" if g["TEST_VAR"] else "") g["TEST_VAR_3"] = ("bar" if g["TEST_VAR"] else "baz") - g["TEST_VAR_3"] = (g["TEST_VAR_LIST"] if g["TEST_VAR"] else []) + g["TEST_VAR_4"] = (g["TEST_VAR_LIST"] if g["TEST_VAR"] else []) `, }, { @@ -1360,6 +1406,57 @@ def init(g, handle): pass `, }, + { + desc: "Type hints", + mkname: "product.mk", + in: ` +# Test type hints +#RBC# type_hint list MY_VAR MY_VAR_2 +# Unsupported type +#RBC# type_hint bool MY_VAR_3 +# Invalid syntax +#RBC# type_hint list +# Duplicated variable +#RBC# type_hint list MY_VAR_2 +#RBC# type_hint list my-local-var-with-dashes +#RBC# type_hint string MY_STRING_VAR + +MY_VAR := foo +MY_VAR_UNHINTED := foo + +# Vars set after other statements still get the hint +MY_VAR_2 := foo + +# You can't specify a type hint after the first statement +#RBC# type_hint list MY_VAR_4 +MY_VAR_4 := foo + +my-local-var-with-dashes := foo + +MY_STRING_VAR := $(wildcard foo/bar.mk) +`, + expected: `# Test type hints +# Unsupported type +load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + rblf.mk2rbc_error("product.mk:5", "Invalid type_hint annotation. Only list/string types are accepted, found bool") + # Invalid syntax + rblf.mk2rbc_error("product.mk:7", "Invalid type_hint annotation: list. Must be a variable type followed by a list of variables of that type") + # Duplicated variable + rblf.mk2rbc_error("product.mk:9", "Duplicate type hint for variable MY_VAR_2") + g["MY_VAR"] = ["foo"] + g["MY_VAR_UNHINTED"] = "foo" + # Vars set after other statements still get the hint + g["MY_VAR_2"] = ["foo"] + # You can't specify a type hint after the first statement + rblf.mk2rbc_error("product.mk:20", "type_hint annotations must come before the first Makefile statement") + g["MY_VAR_4"] = "foo" + _my_local_var_with_dashes = ["foo"] + g["MY_STRING_VAR"] = " ".join(rblf.expand_wildcard("foo/bar.mk")) +`, + }, } var known_variables = []struct { diff --git a/mk2rbc/node.go b/mk2rbc/node.go index 5d98d7bc1..9d5af91c4 100644 --- a/mk2rbc/node.go +++ b/mk2rbc/node.go @@ -184,10 +184,9 @@ type assignmentFlavor int const ( // Assignment flavors - asgnSet assignmentFlavor = iota // := or = - asgnMaybeSet assignmentFlavor = iota // ?= and variable may be unset - asgnAppend assignmentFlavor = iota // += and variable has been set before - asgnMaybeAppend assignmentFlavor = iota // += and variable may be unset + asgnSet assignmentFlavor = iota // := or = + asgnMaybeSet assignmentFlavor = iota // ?= + asgnAppend assignmentFlavor = iota // += ) type assignmentNode struct { @@ -215,6 +214,20 @@ func (asgn *assignmentNode) emit(gctx *generationContext) { } } +func (asgn *assignmentNode) isSelfReferential() bool { + if asgn.flavor == asgnAppend { + return true + } + isSelfReferential := false + asgn.value.transform(func(expr starlarkExpr) starlarkExpr { + if ref, ok := expr.(*variableRefExpr); ok && ref.ref.name() == asgn.lhs.name() { + isSelfReferential = true + } + return nil + }) + return isSelfReferential +} + type exprNode struct { expr starlarkExpr } diff --git a/mk2rbc/variable.go b/mk2rbc/variable.go index f7adca568..506266a29 100644 --- a/mk2rbc/variable.go +++ b/mk2rbc/variable.go @@ -88,25 +88,36 @@ func (pcv productConfigVariable) emitSet(gctx *generationContext, asgn *assignme } value.emit(gctx) } - - switch asgn.flavor { - case asgnSet: - emitAssignment() - case asgnAppend: - emitAppend() - case asgnMaybeAppend: - // If we are not sure variable has been assigned before, emit setdefault + emitSetDefault := func() { if pcv.typ == starlarkTypeList { gctx.writef("%s(handle, %q)", cfnSetListDefault, pcv.name()) } else { gctx.writef("cfg.setdefault(%q, %s)", pcv.name(), pcv.defaultValueString()) } gctx.newLine() + } + + // If we are not sure variable has been assigned before, emit setdefault + needsSetDefault := asgn.previous == nil && !pcv.isPreset() && asgn.isSelfReferential() + + switch asgn.flavor { + case asgnSet: + if needsSetDefault { + emitSetDefault() + } + emitAssignment() + case asgnAppend: + if needsSetDefault { + emitSetDefault() + } emitAppend() case asgnMaybeSet: gctx.writef("if cfg.get(%q) == None:", pcv.nam) gctx.indentLevel++ gctx.newLine() + if needsSetDefault { + emitSetDefault() + } emitAssignment() gctx.indentLevel-- } @@ -121,7 +132,7 @@ func (pcv productConfigVariable) emitGet(gctx *generationContext, isDefined bool } func (pcv productConfigVariable) emitDefined(gctx *generationContext) { - gctx.writef("g.get(%q) != None", pcv.name()) + gctx.writef("cfg.get(%q) != None", pcv.name()) } type otherGlobalVariable struct { @@ -146,20 +157,30 @@ func (scv otherGlobalVariable) emitSet(gctx *generationContext, asgn *assignment value.emit(gctx) } + // If we are not sure variable has been assigned before, emit setdefault + needsSetDefault := asgn.previous == nil && !scv.isPreset() && asgn.isSelfReferential() + switch asgn.flavor { case asgnSet: + if needsSetDefault { + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + } emitAssignment() case asgnAppend: - emitAppend() - case asgnMaybeAppend: - // If we are not sure variable has been assigned before, emit setdefault - gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) - gctx.newLine() + if needsSetDefault { + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + } emitAppend() case asgnMaybeSet: gctx.writef("if g.get(%q) == None:", scv.nam) gctx.indentLevel++ gctx.newLine() + if needsSetDefault { + gctx.writef("g.setdefault(%q, %s)", scv.name(), scv.defaultValueString()) + gctx.newLine() + } emitAssignment() gctx.indentLevel-- } @@ -191,7 +212,7 @@ func (lv localVariable) String() string { func (lv localVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { switch asgn.flavor { - case asgnSet: + case asgnSet, asgnMaybeSet: gctx.writef("%s = ", lv) asgn.value.emitListVarCopy(gctx) case asgnAppend: @@ -203,14 +224,6 @@ func (lv localVariable) emitSet(gctx *generationContext, asgn *assignmentNode) { value = &toStringExpr{expr: value} } value.emit(gctx) - case asgnMaybeAppend: - gctx.writef("%s(%q, ", cfnLocalAppend, lv) - asgn.value.emit(gctx) - gctx.write(")") - case asgnMaybeSet: - gctx.writef("%s(%q, ", cfnLocalSetDefault, lv) - asgn.value.emit(gctx) - gctx.write(")") } } @@ -278,23 +291,35 @@ var presetVariables = map[string]bool{ // addVariable returns a variable with a given name. A variable is // added if it does not exist yet. func (ctx *parseContext) addVariable(name string) variable { + // Get the hintType before potentially changing the variable name + var hintType starlarkType + var ok bool + if hintType, ok = ctx.typeHints[name]; !ok { + hintType = starlarkTypeUnknown + } + // Heuristics: if variable's name is all lowercase, consider it local + // string variable. + isLocalVariable := name == strings.ToLower(name) + // Local variables can't have special characters in them, because they + // will be used as starlark identifiers + if isLocalVariable { + name = strings.ReplaceAll(strings.TrimSpace(name), "-", "_") + } v, found := ctx.variables[name] if !found { - _, preset := presetVariables[name] if vi, found := KnownVariables[name]; found { + _, preset := presetVariables[name] switch vi.class { case VarClassConfig: v = &productConfigVariable{baseVariable{nam: name, typ: vi.valueType, preset: preset}} case VarClassSoong: v = &otherGlobalVariable{baseVariable{nam: name, typ: vi.valueType, preset: preset}} } - } else if name == strings.ToLower(name) { - // Heuristics: if variable's name is all lowercase, consider it local - // string variable. - v = &localVariable{baseVariable{nam: name, typ: starlarkTypeUnknown}} + } else if isLocalVariable { + v = &localVariable{baseVariable{nam: name, typ: hintType}} } else { - vt := starlarkTypeUnknown - if strings.HasPrefix(name, "LOCAL_") { + vt := hintType + if strings.HasPrefix(name, "LOCAL_") && vt == starlarkTypeUnknown { // Heuristics: local variables that contribute to corresponding config variables if cfgVarName, found := localProductConfigVariables[name]; found { vi, found2 := KnownVariables[cfgVarName] diff --git a/python/python.go b/python/python.go index 734ac57f1..b100cc318 100644 --- a/python/python.go +++ b/python/python.go @@ -423,6 +423,9 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { if ctx.Target().Os.Bionic() { launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") } + if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") + } switch p.properties.Actual_version { case pyVersion2: @@ -432,6 +435,7 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { if p.bootstrapper.autorun() { launcherModule = "py2-launcher-autorun" } + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") case pyVersion3: @@ -441,6 +445,9 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { if p.bootstrapper.autorun() { launcherModule = "py3-launcher-autorun" } + if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl { + launcherModule += "-static" + } if ctx.Device() { launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") diff --git a/rust/compiler.go b/rust/compiler.go index c5d40f4dc..19499fa36 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -121,6 +121,12 @@ type BaseCompilerProperties struct { // include all of the static libraries symbols in any dylibs or binaries which use this rlib as well. Whole_static_libs []string `android:"arch_variant"` + // list of Rust system library dependencies. + // + // This is usually only needed when `no_stdlibs` is true, in which case it can be used to depend on system crates + // like `core` and `alloc`. + Stdlibs []string `android:"arch_variant"` + // crate name, required for modules which produce Rust libraries: rust_library, rust_ffi and SourceProvider // modules which create library variants (rust_bindgen). This must be the expected extern crate name used in // source, and is required to conform to an enforced format matching library output files (if the output file is @@ -360,6 +366,7 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { deps.StaticLibs = append(deps.StaticLibs, compiler.Properties.Static_libs...) deps.WholeStaticLibs = append(deps.WholeStaticLibs, compiler.Properties.Whole_static_libs...) deps.SharedLibs = append(deps.SharedLibs, compiler.Properties.Shared_libs...) + deps.Stdlibs = append(deps.Stdlibs, compiler.Properties.Stdlibs...) if !Bool(compiler.Properties.No_stdlibs) { for _, stdlib := range config.Stdlibs { diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go index 14fcb028f..bc36b205f 100644 --- a/rust/config/allowed_list.go +++ b/rust/config/allowed_list.go @@ -24,6 +24,7 @@ var ( "packages/modules/DnsResolver", "packages/modules/Uwb", "packages/modules/Virtualization", + "platform_testing/tests/codecoverage/native/rust", "prebuilts/rust", "system/core/libstats/pull_rust", "system/extras/profcollectd", diff --git a/rust/config/toolchain.go b/rust/config/toolchain.go index a769f121c..9c9d5724a 100644 --- a/rust/config/toolchain.go +++ b/rust/config/toolchain.go @@ -121,14 +121,7 @@ func LibFuzzerRuntimeLibrary(t Toolchain) string { } func LibclangRuntimeLibrary(t Toolchain, library string) string { - arch := t.LibclangRuntimeLibraryArch() - if arch == "" { - return "" - } - if !t.Bionic() { - return "libclang_rt." + library + "-" + arch - } - return "libclang_rt." + library + "-" + arch + "-android" + return "libclang_rt." + library } func LibRustRuntimeLibrary(t Toolchain, library string) string { diff --git a/rust/coverage.go b/rust/coverage.go index 050b811c7..651ce6e16 100644 --- a/rust/coverage.go +++ b/rust/coverage.go @@ -22,6 +22,7 @@ import ( var CovLibraryName = "libprofile-clang-extras" +// Add '%c' to default specifier after we resolve http://b/210012154 const profileInstrFlag = "-fprofile-instr-generate=/data/misc/trace/clang-%p-%m.profraw" type coverage struct { @@ -59,6 +60,10 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags flags.LinkFlags = append(flags.LinkFlags, profileInstrFlag, "-g", coverage.OutputFile().Path().String(), "-Wl,--wrap,open") deps.StaticLibs = append(deps.StaticLibs, coverage.OutputFile().Path()) + if cc.EnableContinuousCoverage(ctx) { + flags.RustFlags = append(flags.RustFlags, "-C llvm-args=--runtime-counter-relocation") + flags.LinkFlags = append(flags.LinkFlags, "-Wl,-mllvm,-runtime-counter-relocation") + } } return flags, deps diff --git a/rust/image.go b/rust/image.go index 5d57f1522..dfc7f7431 100644 --- a/rust/image.go +++ b/rust/image.go @@ -149,6 +149,10 @@ func (mod *Module) InRecovery() bool { return mod.ModuleBase.InRecovery() || mod.ModuleBase.InstallInRecovery() } +func (mod *Module) InRamdisk() bool { + return mod.ModuleBase.InRamdisk() || mod.ModuleBase.InstallInRamdisk() +} + func (mod *Module) InVendorRamdisk() bool { return mod.ModuleBase.InVendorRamdisk() || mod.ModuleBase.InstallInVendorRamdisk() } diff --git a/rust/rust.go b/rust/rust.go index f40f1a8bb..1c718a414 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -968,6 +968,7 @@ func (mod *Module) deps(ctx DepsContext) Deps { deps.ProcMacros = android.LastUniqueStrings(deps.ProcMacros) deps.SharedLibs = android.LastUniqueStrings(deps.SharedLibs) deps.StaticLibs = android.LastUniqueStrings(deps.StaticLibs) + deps.Stdlibs = android.LastUniqueStrings(deps.Stdlibs) deps.WholeStaticLibs = android.LastUniqueStrings(deps.WholeStaticLibs) return deps diff --git a/rust/testing.go b/rust/testing.go index 1b34dfe8e..cb98bed65 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -88,13 +88,13 @@ func GatherRequiredDepsForTest() string { export_include_dirs: ["libprotobuf-cpp-full-includes"], } cc_library { - name: "libclang_rt.asan-aarch64-android", + name: "libclang_rt.asan", no_libcrt: true, nocrt: true, system_shared_libs: [], } cc_library { - name: "libclang_rt.hwasan_static-aarch64-android", + name: "libclang_rt.hwasan_static", no_libcrt: true, nocrt: true, system_shared_libs: [], diff --git a/rust/vendor_snapshot_test.go b/rust/vendor_snapshot_test.go index 03bd867a3..7be00425e 100644 --- a/rust/vendor_snapshot_test.go +++ b/rust/vendor_snapshot_test.go @@ -561,7 +561,7 @@ func TestVendorSnapshotUse(t *testing.T) { static_libs: [ "libvendor", "libvndk", - "libclang_rt.builtins-aarch64-android", + "libclang_rt.builtins", "note_memtag_heap_sync", ], shared_libs: [ @@ -589,7 +589,7 @@ func TestVendorSnapshotUse(t *testing.T) { static_libs: [ "libvendor", "libvndk", - "libclang_rt.builtins-arm-android", + "libclang_rt.builtins", ], shared_libs: [ "libvendor_available", @@ -731,19 +731,7 @@ func TestVendorSnapshotUse(t *testing.T) { } vendor_snapshot_static { - name: "libclang_rt.builtins-aarch64-android", - version: "30", - target_arch: "arm64", - vendor: true, - arch: { - arm64: { - src: "libclang_rt.builtins-aarch64-android.a", - }, - }, - } - - vendor_snapshot_static { - name: "libclang_rt.builtins-arm-android", + name: "libclang_rt.builtins", version: "30", target_arch: "arm64", vendor: true, @@ -751,6 +739,9 @@ func TestVendorSnapshotUse(t *testing.T) { arm: { src: "libclang_rt.builtins-arm-android.a", }, + arm64: { + src: "libclang_rt.builtins-aarch64-android.a", + }, }, } @@ -967,7 +958,7 @@ func TestVendorSnapshotUse(t *testing.T) { } libclientAndroidMkStaticLibs := ctx.ModuleForTests("libclient", sharedVariant).Module().(*Module).Properties.AndroidMkStaticLibs - if g, w := libclientAndroidMkStaticLibs, []string{"libvendor", "libvendor_without_snapshot", "libclang_rt.builtins-aarch64-android.vendor"}; !reflect.DeepEqual(g, w) { + if g, w := libclientAndroidMkStaticLibs, []string{"libvendor", "libvendor_without_snapshot", "libclang_rt.builtins.vendor"}; !reflect.DeepEqual(g, w) { t.Errorf("wanted libclient AndroidMkStaticLibs %q, got %q", w, g) } @@ -1024,7 +1015,7 @@ func TestVendorSnapshotUse(t *testing.T) { } memtagStaticLibs := ctx.ModuleForTests("memtag_binary", "android_vendor.30_arm64_armv8-a").Module().(*Module).Properties.AndroidMkStaticLibs - if g, w := memtagStaticLibs, []string{"libclang_rt.builtins-aarch64-android.vendor", "note_memtag_heap_sync.vendor"}; !reflect.DeepEqual(g, w) { + if g, w := memtagStaticLibs, []string{"libclang_rt.builtins.vendor", "note_memtag_heap_sync.vendor"}; !reflect.DeepEqual(g, w) { t.Errorf("wanted memtag_binary AndroidMkStaticLibs %q, got %q", w, g) } } diff --git a/scripts/hiddenapi/Android.bp b/scripts/hiddenapi/Android.bp index 7ffda62d1..8a47c5dd8 100644 --- a/scripts/hiddenapi/Android.bp +++ b/scripts/hiddenapi/Android.bp @@ -69,10 +69,37 @@ python_test_host { }, } +python_library_host { + name: "signature_trie", + srcs: ["signature_trie.py"], +} + +python_test_host { + name: "signature_trie_test", + main: "signature_trie_test.py", + srcs: ["signature_trie_test.py"], + libs: ["signature_trie"], + version: { + py2: { + enabled: false, + }, + py3: { + enabled: true, + embedded_launcher: true, + }, + }, + test_options: { + unit_test: true, + }, +} + python_binary_host { name: "verify_overlaps", main: "verify_overlaps.py", srcs: ["verify_overlaps.py"], + libs: [ + "signature_trie", + ], version: { py2: { enabled: false, @@ -91,6 +118,9 @@ python_test_host { "verify_overlaps.py", "verify_overlaps_test.py", ], + libs: [ + "signature_trie", + ], version: { py2: { enabled: false, diff --git a/scripts/hiddenapi/signature_patterns.py b/scripts/hiddenapi/signature_patterns.py index e75ee9566..5a82be7b2 100755 --- a/scripts/hiddenapi/signature_patterns.py +++ b/scripts/hiddenapi/signature_patterns.py @@ -25,92 +25,138 @@ import csv import sys -def dict_reader(csvfile): +def dict_reader(csv_file): return csv.DictReader( - csvfile, delimiter=',', quotechar='|', fieldnames=['signature']) + csv_file, delimiter=',', quotechar='|', fieldnames=['signature']) -def dotPackageToSlashPackage(pkg): +def dot_package_to_slash_package(pkg): return pkg.replace('.', '/') -def slashPackageToDotPackage(pkg): +def dot_packages_to_slash_packages(pkgs): + return [dot_package_to_slash_package(p) for p in pkgs] + + +def slash_package_to_dot_package(pkg): return pkg.replace('/', '.') -def isSplitPackage(splitPackages, pkg): - return splitPackages and (pkg in splitPackages or '*' in splitPackages) +def slash_packages_to_dot_packages(pkgs): + return [slash_package_to_dot_package(p) for p in pkgs] + +def is_split_package(split_packages, pkg): + return split_packages and (pkg in split_packages or '*' in split_packages) -def matchedByPackagePrefixPattern(packagePrefixes, prefix): - for packagePrefix in packagePrefixes: + +def matched_by_package_prefix_pattern(package_prefixes, prefix): + for packagePrefix in package_prefixes: if prefix == packagePrefix: return packagePrefix - elif prefix.startswith(packagePrefix) and prefix[len( - packagePrefix)] == '/': + if (prefix.startswith(packagePrefix) and + prefix[len(packagePrefix)] == '/'): return packagePrefix return False -def validate_package_prefixes(splitPackages, packagePrefixes): +def validate_package_is_not_matched_by_package_prefix(package_type, pkg, + package_prefixes): + package_prefix = matched_by_package_prefix_pattern(package_prefixes, pkg) + if package_prefix: + # A package prefix matches the package. + package_for_output = slash_package_to_dot_package(pkg) + package_prefix_for_output = slash_package_to_dot_package(package_prefix) + return [ + f'{package_type} {package_for_output} is matched by ' + f'package prefix {package_prefix_for_output}' + ] + return [] + + +def validate_package_prefixes(split_packages, single_packages, + package_prefixes): # If there are no package prefixes then there is no possible conflict # between them and the split packages. - if len(packagePrefixes) == 0: - return + if len(package_prefixes) == 0: + return [] # Check to make sure that the split packages and package prefixes do not # overlap. errors = [] - for splitPackage in splitPackages: - if splitPackage == '*': + for split_package in split_packages: + if split_package == '*': # A package prefix matches a split package. - packagePrefixesForOutput = ', '.join( - map(slashPackageToDotPackage, packagePrefixes)) + package_prefixes_for_output = ', '.join( + slash_packages_to_dot_packages(package_prefixes)) errors.append( - 'split package "*" conflicts with all package prefixes %s\n' - ' add split_packages:[] to fix' % packagePrefixesForOutput) + "split package '*' conflicts with all package prefixes " + f'{package_prefixes_for_output}\n' + ' add split_packages:[] to fix') else: - packagePrefix = matchedByPackagePrefixPattern( - packagePrefixes, splitPackage) - if packagePrefix: - # A package prefix matches a split package. - splitPackageForOutput = slashPackageToDotPackage(splitPackage) - packagePrefixForOutput = slashPackageToDotPackage(packagePrefix) - errors.append( - 'split package %s is matched by package prefix %s' % - (splitPackageForOutput, packagePrefixForOutput)) + errs = validate_package_is_not_matched_by_package_prefix( + 'split package', split_package, package_prefixes) + errors.extend(errs) + + # Check to make sure that the single packages and package prefixes do not + # overlap. + for single_package in single_packages: + errs = validate_package_is_not_matched_by_package_prefix( + 'single package', single_package, package_prefixes) + errors.extend(errs) return errors -def validate_split_packages(splitPackages): +def validate_split_packages(split_packages): errors = [] - if '*' in splitPackages and len(splitPackages) > 1: + if '*' in split_packages and len(split_packages) > 1: errors.append('split packages are invalid as they contain both the' ' wildcard (*) and specific packages, use the wildcard or' ' specific packages, not a mixture') return errors -def produce_patterns_from_file(file, splitPackages=None, packagePrefixes=None): - with open(file, 'r') as f: - return produce_patterns_from_stream(f, splitPackages, packagePrefixes) +def validate_single_packages(split_packages, single_packages): + overlaps = [] + for single_package in single_packages: + if single_package in split_packages: + overlaps.append(single_package) + if overlaps: + indented = ''.join([f'\n {o}' for o in overlaps]) + return [ + f'single_packages and split_packages overlap, please ensure the ' + f'following packages are only present in one:{indented}' + ] + return [] + + +def produce_patterns_from_file(file, + split_packages=None, + single_packages=None, + package_prefixes=None): + with open(file, 'r', encoding='utf8') as f: + return produce_patterns_from_stream(f, split_packages, single_packages, + package_prefixes) def produce_patterns_from_stream(stream, - splitPackages=None, - packagePrefixes=None): - splitPackages = set(splitPackages or []) - packagePrefixes = list(packagePrefixes or []) + split_packages=None, + single_packages=None, + package_prefixes=None): + split_packages = set(split_packages or []) + single_packages = set(single_packages or []) + package_prefixes = list(package_prefixes or []) # Read in all the signatures into a list and remove any unnecessary class # and member names. patterns = set() + unmatched_packages = set() for row in dict_reader(stream): signature = row['signature'] text = signature.removeprefix('L') # Remove the class specific member signature pieces = text.split(';->') - qualifiedClassName = pieces[0] - pieces = qualifiedClassName.rsplit('/', maxsplit=1) + qualified_class_name = pieces[0] + pieces = qualified_class_name.rsplit('/', maxsplit=1) pkg = pieces[0] # If the package is split across multiple modules then it cannot be used # to select the subset of the monolithic flags that this module @@ -121,27 +167,54 @@ def produce_patterns_from_stream(stream, # If the package is not split then every class in the package must be # provided by this module so there is no need to list the classes # explicitly so just use the package name instead. - if isSplitPackage(splitPackages, pkg): + if is_split_package(split_packages, pkg): # Remove inner class names. - pieces = qualifiedClassName.split('$', maxsplit=1) + pieces = qualified_class_name.split('$', maxsplit=1) pattern = pieces[0] - else: + patterns.add(pattern) + elif pkg in single_packages: # Add a * to ensure that the pattern matches the classes in that # package. pattern = pkg + '/*' - patterns.add(pattern) + patterns.add(pattern) + else: + unmatched_packages.add(pkg) + + # Remove any unmatched packages that would be matched by a package prefix + # pattern. + unmatched_packages = [ + p for p in unmatched_packages + if not matched_by_package_prefix_pattern(package_prefixes, p) + ] + errors = [] + if unmatched_packages: + unmatched_packages.sort() + indented = ''.join([ + f'\n {slash_package_to_dot_package(p)}' + for p in unmatched_packages + ]) + errors.append('The following packages were unexpected, please add them ' + 'to one of the hidden_api properties, split_packages, ' + f'single_packages or package_prefixes:{indented}') # Remove any patterns that would be matched by a package prefix pattern. - patterns = list( - filter(lambda p: not matchedByPackagePrefixPattern(packagePrefixes, p), - patterns)) + patterns = [ + p for p in patterns + if not matched_by_package_prefix_pattern(package_prefixes, p) + ] # Add the package prefix patterns to the list. Add a ** to ensure that each # package prefix pattern will match the classes in that package and all # sub-packages. - patterns = patterns + list(map(lambda x: x + '/**', packagePrefixes)) + patterns = patterns + [f'{p}/**' for p in package_prefixes] # Sort the patterns. patterns.sort() - return patterns + return patterns, errors + + +def print_and_exit(errors): + for error in errors: + print(error) + sys.exit(1) def main(args): @@ -155,35 +228,50 @@ def main(args): args_parser.add_argument( '--split-package', action='append', - help='A package that is split across multiple bootclasspath_fragment modules' - ) + help='A package that is split across multiple bootclasspath_fragment ' + 'modules') args_parser.add_argument( '--package-prefix', action='append', help='A package prefix unique to this set of flags') + args_parser.add_argument( + '--single-package', + action='append', + help='A single package unique to this set of flags') args_parser.add_argument('--output', help='Generated signature prefixes') args = args_parser.parse_args(args) - splitPackages = set(map(dotPackageToSlashPackage, args.split_package or [])) - errors = validate_split_packages(splitPackages) + split_packages = set( + dot_packages_to_slash_packages(args.split_package or [])) + errors = validate_split_packages(split_packages) + if errors: + print_and_exit(errors) + + single_packages = list( + dot_packages_to_slash_packages(args.single_package or [])) - packagePrefixes = list( - map(dotPackageToSlashPackage, args.package_prefix or [])) + errors = validate_single_packages(split_packages, single_packages) + if errors: + print_and_exit(errors) - if not errors: - errors = validate_package_prefixes(splitPackages, packagePrefixes) + package_prefixes = dot_packages_to_slash_packages(args.package_prefix or []) + errors = validate_package_prefixes(split_packages, single_packages, + package_prefixes) if errors: - for error in errors: - print(error) - sys.exit(1) + print_and_exit(errors) + patterns = [] # Read in all the patterns into a list. - patterns = produce_patterns_from_file(args.flags, splitPackages, - packagePrefixes) + patterns, errors = produce_patterns_from_file(args.flags, split_packages, + single_packages, + package_prefixes) + + if errors: + print_and_exit(errors) # Write out all the patterns. - with open(args.output, 'w') as outputFile: + with open(args.output, 'w', encoding='utf8') as outputFile: for pattern in patterns: outputFile.write(pattern) outputFile.write('\n') diff --git a/scripts/hiddenapi/signature_patterns_test.py b/scripts/hiddenapi/signature_patterns_test.py index b59dfd71b..90b3d0b09 100755 --- a/scripts/hiddenapi/signature_patterns_test.py +++ b/scripts/hiddenapi/signature_patterns_test.py @@ -17,37 +17,52 @@ import io import unittest -from signature_patterns import * #pylint: disable=unused-wildcard-import,wildcard-import +import signature_patterns class TestGeneratedPatterns(unittest.TestCase): - csvFlags = """ + csv_flags = """ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;,public-api Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api Ljava/lang/Object;->toString()Ljava/lang/String;,blocked """ - def produce_patterns_from_string(self, - csv, - splitPackages=None, - packagePrefixes=None): - with io.StringIO(csv) as f: - return produce_patterns_from_stream(f, splitPackages, - packagePrefixes) + @staticmethod + def produce_patterns_from_string(csv_text, + split_packages=None, + single_packages=None, + package_prefixes=None): + with io.StringIO(csv_text) as f: + return signature_patterns.produce_patterns_from_stream( + f, split_packages, single_packages, package_prefixes) + + def test_generate_unmatched(self): + _, errors = self.produce_patterns_from_string( + TestGeneratedPatterns.csv_flags) + self.assertEqual([ + 'The following packages were unexpected, please add them to one of ' + 'the hidden_api properties, split_packages, single_packages or ' + 'package_prefixes:\n' + ' java.lang' + ], errors) def test_generate_default(self): - patterns = self.produce_patterns_from_string( - TestGeneratedPatterns.csvFlags) + patterns, errors = self.produce_patterns_from_string( + TestGeneratedPatterns.csv_flags, single_packages=['java/lang']) + self.assertEqual([], errors) + expected = [ 'java/lang/*', ] self.assertEqual(expected, patterns) def test_generate_split_package(self): - patterns = self.produce_patterns_from_string( - TestGeneratedPatterns.csvFlags, splitPackages={'java/lang'}) + patterns, errors = self.produce_patterns_from_string( + TestGeneratedPatterns.csv_flags, split_packages={'java/lang'}) + self.assertEqual([], errors) + expected = [ 'java/lang/Character', 'java/lang/Object', @@ -56,8 +71,10 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked self.assertEqual(expected, patterns) def test_generate_split_package_wildcard(self): - patterns = self.produce_patterns_from_string( - TestGeneratedPatterns.csvFlags, splitPackages={'*'}) + patterns, errors = self.produce_patterns_from_string( + TestGeneratedPatterns.csv_flags, split_packages={'*'}) + self.assertEqual([], errors) + expected = [ 'java/lang/Character', 'java/lang/Object', @@ -66,23 +83,27 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked self.assertEqual(expected, patterns) def test_generate_package_prefix(self): - patterns = self.produce_patterns_from_string( - TestGeneratedPatterns.csvFlags, packagePrefixes={'java/lang'}) + patterns, errors = self.produce_patterns_from_string( + TestGeneratedPatterns.csv_flags, package_prefixes={'java/lang'}) + self.assertEqual([], errors) + expected = [ 'java/lang/**', ] self.assertEqual(expected, patterns) def test_generate_package_prefix_top_package(self): - patterns = self.produce_patterns_from_string( - TestGeneratedPatterns.csvFlags, packagePrefixes={'java'}) + patterns, errors = self.produce_patterns_from_string( + TestGeneratedPatterns.csv_flags, package_prefixes={'java'}) + self.assertEqual([], errors) + expected = [ 'java/**', ] self.assertEqual(expected, patterns) def test_split_package_wildcard_conflicts_with_other_split_packages(self): - errors = validate_split_packages({'*', 'java'}) + errors = signature_patterns.validate_split_packages({'*', 'java'}) expected = [ 'split packages are invalid as they contain both the wildcard (*)' ' and specific packages, use the wildcard or specific packages,' @@ -91,21 +112,39 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked self.assertEqual(expected, errors) def test_split_package_wildcard_conflicts_with_package_prefixes(self): - errors = validate_package_prefixes({'*'}, packagePrefixes={'java'}) + errors = signature_patterns.validate_package_prefixes( + {'*'}, [], package_prefixes={'java'}) expected = [ - 'split package "*" conflicts with all package prefixes java\n' + "split package '*' conflicts with all package prefixes java\n" ' add split_packages:[] to fix', ] self.assertEqual(expected, errors) - def test_split_package_conflict(self): - errors = validate_package_prefixes({'java/split'}, - packagePrefixes={'java'}) + def test_split_package_conflicts_with_package_prefixes(self): + errors = signature_patterns.validate_package_prefixes( + {'java/split'}, [], package_prefixes={'java'}) expected = [ 'split package java.split is matched by package prefix java', ] self.assertEqual(expected, errors) + def test_single_package_conflicts_with_package_prefixes(self): + errors = signature_patterns.validate_package_prefixes( + {}, ['java/single'], package_prefixes={'java'}) + expected = [ + 'single package java.single is matched by package prefix java', + ] + self.assertEqual(expected, errors) + + def test_single_package_conflicts_with_split_packages(self): + errors = signature_patterns.validate_single_packages({'java/pkg'}, + ['java/pkg']) + expected = [ + 'single_packages and split_packages overlap, please ensure the ' + 'following packages are only present in one:\n java/pkg' + ] + self.assertEqual(expected, errors) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/scripts/hiddenapi/signature_trie.py b/scripts/hiddenapi/signature_trie.py new file mode 100644 index 000000000..e813a9781 --- /dev/null +++ b/scripts/hiddenapi/signature_trie.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Verify that one set of hidden API flags is a subset of another.""" +import dataclasses +import typing + +from itertools import chain + + +@dataclasses.dataclass() +class Node: + """A node in the signature trie.""" + + # The type of the node. + # + # Leaf nodes are of type "member". + # Interior nodes can be either "package", or "class". + type: str + + # The selector of the node. + # + # That is a string that can be used to select the node, e.g. in a pattern + # that is passed to InteriorNode.get_matching_rows(). + selector: str + + def values(self, selector): + """Get the values from a set of selected nodes. + + :param selector: a function that can be applied to a key in the nodes + attribute to determine whether to return its values. + + :return: A list of iterables of all the values associated with + this node and its children. + """ + raise NotImplementedError("Please Implement this method") + + def append_values(self, values, selector): + """Append the values associated with this node and its children. + + For each item (key, child) in nodes the child node's values are returned + if and only if the selector returns True when called on its key. A child + node's values are all the values associated with it and all its + descendant nodes. + + :param selector: a function that can be applied to a key in the nodes + attribute to determine whether to return its values. + :param values: a list of a iterables of values. + """ + raise NotImplementedError("Please Implement this method") + + def child_nodes(self): + """Get an iterable of the child nodes of this node.""" + raise NotImplementedError("Please Implement this method") + + +# pylint: disable=line-too-long +@dataclasses.dataclass() +class InteriorNode(Node): + """An interior node in a trie. + + Each interior node has a dict that maps from an element of a signature to + either another interior node or a leaf. Each interior node represents either + a package, class or nested class. Class members are represented by a Leaf. + + Associating the set of flags [public-api] with the signature + "Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following + nodes to be created: + Node() + ^- package:java -> Node() + ^- package:lang -> Node() + ^- class:Object -> Node() + ^- member:String()Ljava/lang/String; -> Leaf([public-api]) + + Associating the set of flags [blocked,core-platform-api] with the signature + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;" + will cause the following nodes to be created: + Node() + ^- package:java -> Node() + ^- package:lang -> Node() + ^- class:Character -> Node() + ^- class:UnicodeScript -> Node() + ^- member:of(I)Ljava/lang/Character$UnicodeScript; + -> Leaf([blocked,core-platform-api]) + """ + + # pylint: enable=line-too-long + + # A dict from an element of the signature to the Node/Leaf containing the + # next element/value. + nodes: typing.Dict[str, Node] = dataclasses.field(default_factory=dict) + + # pylint: disable=line-too-long + @staticmethod + def signature_to_elements(signature): + """Split a signature or a prefix into a number of elements: + + 1. The packages (excluding the leading L preceding the first package). + 2. The class names, from outermost to innermost. + 3. The member signature. + e.g. + Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; + will be broken down into these elements: + 1. package:java + 2. package:lang + 3. class:Character + 4. class:UnicodeScript + 5. member:of(I)Ljava/lang/Character$UnicodeScript; + """ + # Remove the leading L. + # - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; + text = signature.removeprefix("L") + # Split the signature between qualified class name and the class member + # signature. + # 0 - java/lang/Character$UnicodeScript + # 1 - of(I)Ljava/lang/Character$UnicodeScript; + parts = text.split(";->") + # If there is no member then this will be an empty list. + member = parts[1:] + # Split the qualified class name into packages, and class name. + # 0 - java + # 1 - lang + # 2 - Character$UnicodeScript + elements = parts[0].split("/") + last_element = elements[-1] + wildcard = [] + classes = [] + if "*" in last_element: + if last_element not in ("*", "**"): + raise Exception(f"Invalid signature '{signature}': invalid " + f"wildcard '{last_element}'") + packages = elements[0:-1] + # Cannot specify a wildcard and target a specific member + if member: + raise Exception(f"Invalid signature '{signature}': contains " + f"wildcard '{last_element}' and " + f"member signature '{member[0]}'") + wildcard = [last_element] + elif last_element.islower(): + raise Exception(f"Invalid signature '{signature}': last element " + f"'{last_element}' is lower case but should be an " + f"upper case class name or wildcard") + else: + packages = elements[0:-1] + # Split the class name into outer / inner classes + # 0 - Character + # 1 - UnicodeScript + classes = last_element.removesuffix(";").split("$") + + # Assemble the parts into a single list, adding prefixes to identify + # the different parts. If a wildcard is provided then it looks something + # like this: + # 0 - package:java + # 1 - package:lang + # 2 - * + # + # Otherwise, it looks something like this: + # 0 - package:java + # 1 - package:lang + # 2 - class:Character + # 3 - class:UnicodeScript + # 4 - member:of(I)Ljava/lang/Character$UnicodeScript; + return list( + chain([("package", x) for x in packages], + [("class", x) for x in classes], + [("member", x) for x in member], + [("wildcard", x) for x in wildcard])) + + # pylint: enable=line-too-long + + @staticmethod + def split_element(element): + element_type, element_value = element + return element_type, element_value + + @staticmethod + def element_type(element): + element_type, _ = InteriorNode.split_element(element) + return element_type + + @staticmethod + def elements_to_selector(elements): + """Compute a selector for a set of elements. + + A selector uniquely identifies a specific Node in the trie. It is + essentially a prefix of a signature (without the leading L). + + e.g. a trie containing "Ljava/lang/Object;->String()Ljava/lang/String;" + would contain nodes with the following selectors: + * "java" + * "java/lang" + * "java/lang/Object" + * "java/lang/Object;->String()Ljava/lang/String;" + """ + signature = "" + preceding_type = "" + for element in elements: + element_type, element_value = InteriorNode.split_element(element) + separator = "" + if element_type == "package": + separator = "/" + elif element_type == "class": + if preceding_type == "class": + separator = "$" + else: + separator = "/" + elif element_type == "wildcard": + separator = "/" + elif element_type == "member": + separator += ";->" + + if signature: + signature += separator + + signature += element_value + + preceding_type = element_type + + return signature + + def add(self, signature, value, only_if_matches=False): + """Associate the value with the specific signature. + + :param signature: the member signature + :param value: the value to associated with the signature + :param only_if_matches: True if the value is added only if the signature + matches at least one of the existing top level packages. + :return: n/a + """ + # Split the signature into elements. + elements = self.signature_to_elements(signature) + # Find the Node associated with the deepest class. + node = self + for index, element in enumerate(elements[:-1]): + if element in node.nodes: + node = node.nodes[element] + elif only_if_matches and index == 0: + return + else: + selector = self.elements_to_selector(elements[0:index + 1]) + next_node = InteriorNode( + type=InteriorNode.element_type(element), selector=selector) + node.nodes[element] = next_node + node = next_node + # Add a Leaf containing the value and associate it with the member + # signature within the class. + last_element = elements[-1] + last_element_type = self.element_type(last_element) + if last_element_type != "member": + raise Exception( + f"Invalid signature: {signature}, does not identify a " + "specific member") + if last_element in node.nodes: + raise Exception(f"Duplicate signature: {signature}") + leaf = Leaf( + type=last_element_type, + selector=signature, + value=value, + ) + node.nodes[last_element] = leaf + + def get_matching_rows(self, pattern): + """Get the values (plural) associated with the pattern. + + e.g. If the pattern is a full signature then this will return a list + containing the value associated with that signature. + + If the pattern is a class then this will return a list containing the + values associated with all members of that class. + + If the pattern ends with "*" then the preceding part is treated as a + package and this will return a list containing the values associated + with all the members of all the classes in that package. + + If the pattern ends with "**" then the preceding part is treated + as a package and this will return a list containing the values + associated with all the members of all the classes in that package and + all sub-packages. + + :param pattern: the pattern which could be a complete signature or a + class, or package wildcard. + :return: an iterable containing all the values associated with the + pattern. + """ + elements = self.signature_to_elements(pattern) + node = self + + # Include all values from this node and all its children. + selector = lambda x: True + + last_element = elements[-1] + last_element_type, last_element_value = self.split_element(last_element) + if last_element_type == "wildcard": + elements = elements[:-1] + if last_element_value == "*": + # Do not include values from sub-packages. + selector = lambda x: InteriorNode.element_type(x) != "package" + + for element in elements: + if element in node.nodes: + node = node.nodes[element] + else: + return [] + return chain.from_iterable(node.values(selector)) + + def values(self, selector): + values = [] + self.append_values(values, selector) + return values + + def append_values(self, values, selector): + for key, node in self.nodes.items(): + if selector(key): + node.append_values(values, lambda x: True) + + def child_nodes(self): + return self.nodes.values() + + +@dataclasses.dataclass() +class Leaf(Node): + """A leaf of the trie""" + + # The value associated with this leaf. + value: typing.Any + + def values(self, selector): + return [[self.value]] + + def append_values(self, values, selector): + values.append([self.value]) + + def child_nodes(self): + return [] + + +def signature_trie(): + return InteriorNode(type="root", selector="") diff --git a/scripts/hiddenapi/signature_trie_test.py b/scripts/hiddenapi/signature_trie_test.py new file mode 100755 index 000000000..129569107 --- /dev/null +++ b/scripts/hiddenapi/signature_trie_test.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for verify_overlaps_test.py.""" +import io +import unittest + +from signature_trie import InteriorNode +from signature_trie import signature_trie + + +class TestSignatureToElements(unittest.TestCase): + + @staticmethod + def signature_to_elements(signature): + return InteriorNode.signature_to_elements(signature) + + @staticmethod + def elements_to_signature(elements): + return InteriorNode.elements_to_selector(elements) + + def test_nested_inner_classes(self): + elements = [ + ("package", "java"), + ("package", "lang"), + ("class", "ProcessBuilder"), + ("class", "Redirect"), + ("class", "1"), + ("member", "<init>()V"), + ] + signature = "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, "L" + self.elements_to_signature(elements)) + + def test_basic_member(self): + elements = [ + ("package", "java"), + ("package", "lang"), + ("class", "Object"), + ("member", "hashCode()I"), + ] + signature = "Ljava/lang/Object;->hashCode()I" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, "L" + self.elements_to_signature(elements)) + + def test_double_dollar_class(self): + elements = [ + ("package", "java"), + ("package", "lang"), + ("class", "CharSequence"), + ("class", ""), + ("class", "ExternalSyntheticLambda0"), + ("member", "<init>(Ljava/lang/CharSequence;)V"), + ] + signature = "Ljava/lang/CharSequence$$ExternalSyntheticLambda0;" \ + "-><init>(Ljava/lang/CharSequence;)V" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, "L" + self.elements_to_signature(elements)) + + def test_no_member(self): + elements = [ + ("package", "java"), + ("package", "lang"), + ("class", "CharSequence"), + ("class", ""), + ("class", "ExternalSyntheticLambda0"), + ] + signature = "Ljava/lang/CharSequence$$ExternalSyntheticLambda0" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, "L" + self.elements_to_signature(elements)) + + def test_wildcard(self): + elements = [ + ("package", "java"), + ("package", "lang"), + ("wildcard", "*"), + ] + signature = "java/lang/*" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, self.elements_to_signature(elements)) + + def test_recursive_wildcard(self): + elements = [ + ("package", "java"), + ("package", "lang"), + ("wildcard", "**"), + ] + signature = "java/lang/**" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, self.elements_to_signature(elements)) + + def test_no_packages_wildcard(self): + elements = [ + ("wildcard", "*"), + ] + signature = "*" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, self.elements_to_signature(elements)) + + def test_no_packages_recursive_wildcard(self): + elements = [ + ("wildcard", "**"), + ] + signature = "**" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, self.elements_to_signature(elements)) + + def test_invalid_no_class_or_wildcard(self): + signature = "java/lang" + with self.assertRaises(Exception) as context: + self.signature_to_elements(signature) + self.assertIn( + "last element 'lang' is lower case but should be an " + "upper case class name or wildcard", str(context.exception)) + + def test_non_standard_class_name(self): + elements = [ + ("package", "javax"), + ("package", "crypto"), + ("class", "extObjectInputStream"), + ] + signature = "Ljavax/crypto/extObjectInputStream" + self.assertEqual(elements, self.signature_to_elements(signature)) + self.assertEqual(signature, "L" + self.elements_to_signature(elements)) + + def test_invalid_pattern_wildcard(self): + pattern = "Ljava/lang/Class*" + with self.assertRaises(Exception) as context: + self.signature_to_elements(pattern) + self.assertIn("invalid wildcard 'Class*'", str(context.exception)) + + def test_invalid_pattern_wildcard_and_member(self): + pattern = "Ljava/lang/*;->hashCode()I" + with self.assertRaises(Exception) as context: + self.signature_to_elements(pattern) + self.assertIn( + "contains wildcard '*' and member signature 'hashCode()I'", + str(context.exception)) + + +class TestGetMatchingRows(unittest.TestCase): + extractInput = """ +Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; +Ljava/lang/Character;->serialVersionUID:J +Ljava/lang/Object;->hashCode()I +Ljava/lang/Object;->toString()Ljava/lang/String; +Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V +Ljava/util/zip/ZipFile;-><clinit>()V +""" + + def read_trie(self): + trie = signature_trie() + with io.StringIO(self.extractInput.strip()) as f: + for line in iter(f.readline, ""): + line = line.rstrip() + trie.add(line, line) + return trie + + def check_patterns(self, pattern, expected): + trie = self.read_trie() + self.check_node_patterns(trie, pattern, expected) + + def check_node_patterns(self, node, pattern, expected): + actual = list(node.get_matching_rows(pattern)) + actual.sort() + self.assertEqual(expected, actual) + + def test_member_pattern(self): + self.check_patterns("java/util/zip/ZipFile;-><clinit>()V", + ["Ljava/util/zip/ZipFile;-><clinit>()V"]) + + def test_class_pattern(self): + self.check_patterns("java/lang/Object", [ + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + ]) + + # pylint: disable=line-too-long + def test_nested_class_pattern(self): + self.check_patterns("java/lang/Character", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + ]) + + def test_wildcard(self): + self.check_patterns("java/lang/*", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V", + ]) + + def test_recursive_wildcard(self): + self.check_patterns("java/**", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V", + "Ljava/util/zip/ZipFile;-><clinit>()V", + ]) + + def test_node_wildcard(self): + trie = self.read_trie() + node = list(trie.child_nodes())[0] + self.check_node_patterns(node, "**", [ + "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;", + "Ljava/lang/Character;->serialVersionUID:J", + "Ljava/lang/Object;->hashCode()I", + "Ljava/lang/Object;->toString()Ljava/lang/String;", + "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V", + "Ljava/util/zip/ZipFile;-><clinit>()V", + ]) + + # pylint: enable=line-too-long + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/hiddenapi/verify_overlaps.py b/scripts/hiddenapi/verify_overlaps.py index 4cd7e63ac..e5214dfc8 100755 --- a/scripts/hiddenapi/verify_overlaps.py +++ b/scripts/hiddenapi/verify_overlaps.py @@ -13,239 +13,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Verify that one set of hidden API flags is a subset of another. -""" +"""Verify that one set of hidden API flags is a subset of another.""" import argparse import csv import sys from itertools import chain -#pylint: disable=line-too-long -class InteriorNode: - """An interior node in a trie. - - Each interior node has a dict that maps from an element of a signature to - either another interior node or a leaf. Each interior node represents either - a package, class or nested class. Class members are represented by a Leaf. - - Associating the set of flags [public-api] with the signature - "Ljava/lang/Object;->String()Ljava/lang/String;" will cause the following - nodes to be created: - Node() - ^- package:java -> Node() - ^- package:lang -> Node() - ^- class:Object -> Node() - ^- member:String()Ljava/lang/String; -> Leaf([public-api]) - - Associating the set of flags [blocked,core-platform-api] with the signature - "Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;" - will cause the following nodes to be created: - Node() - ^- package:java -> Node() - ^- package:lang -> Node() - ^- class:Character -> Node() - ^- class:UnicodeScript -> Node() - ^- member:of(I)Ljava/lang/Character$UnicodeScript; - -> Leaf([blocked,core-platform-api]) - - Attributes: - nodes: a dict from an element of the signature to the Node/Leaf - containing the next element/value. - """ - #pylint: enable=line-too-long - - def __init__(self): - self.nodes = {} - - #pylint: disable=line-too-long - def signatureToElements(self, signature): - """Split a signature or a prefix into a number of elements: - 1. The packages (excluding the leading L preceding the first package). - 2. The class names, from outermost to innermost. - 3. The member signature. - e.g. - Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; - will be broken down into these elements: - 1. package:java - 2. package:lang - 3. class:Character - 4. class:UnicodeScript - 5. member:of(I)Ljava/lang/Character$UnicodeScript; - """ - # Remove the leading L. - # - java/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript; - text = signature.removeprefix("L") - # Split the signature between qualified class name and the class member - # signature. - # 0 - java/lang/Character$UnicodeScript - # 1 - of(I)Ljava/lang/Character$UnicodeScript; - parts = text.split(";->") - member = parts[1:] - # Split the qualified class name into packages, and class name. - # 0 - java - # 1 - lang - # 2 - Character$UnicodeScript - elements = parts[0].split("/") - packages = elements[0:-1] - className = elements[-1] - if className in ("*" , "**"): #pylint: disable=no-else-return - # Cannot specify a wildcard and target a specific member - if len(member) != 0: - raise Exception( - "Invalid signature %s: contains wildcard %s and member " \ - "signature %s" - % (signature, className, member[0])) - wildcard = [className] - # Assemble the parts into a single list, adding prefixes to identify - # the different parts. - # 0 - package:java - # 1 - package:lang - # 2 - * - return list( - chain(["package:" + x for x in packages], wildcard)) - else: - # Split the class name into outer / inner classes - # 0 - Character - # 1 - UnicodeScript - classes = className.split("$") - # Assemble the parts into a single list, adding prefixes to identify - # the different parts. - # 0 - package:java - # 1 - package:lang - # 2 - class:Character - # 3 - class:UnicodeScript - # 4 - member:of(I)Ljava/lang/Character$UnicodeScript; - return list( - chain( - ["package:" + x for x in packages], - ["class:" + x for x in classes], - ["member:" + x for x in member])) - #pylint: enable=line-too-long - - def add(self, signature, value): - """Associate the value with the specific signature. - - :param signature: the member signature - :param value: the value to associated with the signature - :return: n/a - """ - # Split the signature into elements. - elements = self.signatureToElements(signature) - # Find the Node associated with the deepest class. - node = self - for element in elements[:-1]: - if element in node.nodes: - node = node.nodes[element] - else: - next_node = InteriorNode() - node.nodes[element] = next_node - node = next_node - # Add a Leaf containing the value and associate it with the member - # signature within the class. - lastElement = elements[-1] - if not lastElement.startswith("member:"): - raise Exception( - "Invalid signature: %s, does not identify a specific member" % - signature) - if lastElement in node.nodes: - raise Exception("Duplicate signature: %s" % signature) - node.nodes[lastElement] = Leaf(value) - - def getMatchingRows(self, pattern): - """Get the values (plural) associated with the pattern. - - e.g. If the pattern is a full signature then this will return a list - containing the value associated with that signature. - - If the pattern is a class then this will return a list containing the - values associated with all members of that class. - - If the pattern is a package then this will return a list containing the - values associated with all the members of all the classes in that - package and sub-packages. - - If the pattern ends with "*" then the preceding part is treated as a - package and this will return a list containing the values associated - with all the members of all the classes in that package. - - If the pattern ends with "**" then the preceding part is treated - as a package and this will return a list containing the values - associated with all the members of all the classes in that package and - all sub-packages. - - :param pattern: the pattern which could be a complete signature or a - class, or package wildcard. - :return: an iterable containing all the values associated with the - pattern. - """ - elements = self.signatureToElements(pattern) - node = self - # Include all values from this node and all its children. - selector = lambda x: True - lastElement = elements[-1] - if lastElement in ("*", "**"): - elements = elements[:-1] - if lastElement == "*": - # Do not include values from sub-packages. - selector = lambda x: not x.startswith("package:") - for element in elements: - if element in node.nodes: - node = node.nodes[element] - else: - return [] - return chain.from_iterable(node.values(selector)) - - def values(self, selector): - """:param selector: a function that can be applied to a key in the nodes - attribute to determine whether to return its values. - - :return: A list of iterables of all the values associated with - this node and its children. - """ - values = [] - self.appendValues(values, selector) - return values - - def appendValues(self, values, selector): - """Append the values associated with this node and its children to the - list. - - For each item (key, child) in nodes the child node's values are returned - if and only if the selector returns True when called on its key. A child - node's values are all the values associated with it and all its - descendant nodes. - - :param selector: a function that can be applied to a key in the nodes - attribute to determine whether to return its values. - :param values: a list of a iterables of values. - """ - for key, node in self.nodes.items(): - if selector(key): - node.appendValues(values, lambda x: True) - - -class Leaf: - """A leaf of the trie - - Attributes: - value: the value associated with this leaf. - """ - - def __init__(self, value): - self.value = value - - def values(self, selector): #pylint: disable=unused-argument - """:return: A list of a list of the value associated with this node. - """ - return [[self.value]] - - def appendValues(self, values, selector): #pylint: disable=unused-argument - """Appends a list of the value associated with this node to the list. - - :param values: a list of a iterables of values. - """ - values.append([self.value]) +from signature_trie import signature_trie def dict_reader(csvfile): @@ -259,7 +34,7 @@ def read_flag_trie_from_file(file): def read_flag_trie_from_stream(stream): - trie = InteriorNode() + trie = signature_trie() reader = dict_reader(stream) for row in reader: signature = row["signature"] @@ -269,8 +44,7 @@ def read_flag_trie_from_stream(stream): def extract_subset_from_monolithic_flags_as_dict_from_file( monolithicTrie, patternsFile): - """Extract a subset of flags from the dict containing all the monolithic - flags. + """Extract a subset of flags from the dict of monolithic flags. :param monolithicFlagsDict: the dict containing all the monolithic flags. :param patternsFile: a file containing a list of signature patterns that @@ -284,8 +58,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_file( def extract_subset_from_monolithic_flags_as_dict_from_stream( monolithicTrie, stream): - """Extract a subset of flags from the trie containing all the monolithic - flags. + """Extract a subset of flags from the trie of monolithic flags. :param monolithicTrie: the trie containing all the monolithic flags. :param stream: a stream containing a list of signature patterns that define @@ -295,7 +68,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream( dict_signature_to_row = {} for pattern in stream: pattern = pattern.rstrip() - rows = monolithicTrie.getMatchingRows(pattern) + rows = monolithicTrie.get_matching_rows(pattern) for row in rows: signature = row["signature"] dict_signature_to_row[signature] = row @@ -303,8 +76,10 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream( def read_signature_csv_from_stream_as_dict(stream): - """Read the csv contents from the stream into a dict. The first column is - assumed to be the signature and used as the key. + """Read the csv contents from the stream into a dict. + + The first column is assumed to be the signature and used as the + key. The whole row is stored as the value. :param stream: the csv contents to read @@ -319,8 +94,10 @@ def read_signature_csv_from_stream_as_dict(stream): def read_signature_csv_from_file_as_dict(csvFile): - """Read the csvFile into a dict. The first column is assumed to be the - signature and used as the key. + """Read the csvFile into a dict. + + The first column is assumed to be the signature and used as the + key. The whole row is stored as the value. :param csvFile: the csv file to read @@ -363,8 +140,7 @@ def compare_signature_flags(monolithicFlagsDict, modularFlagsDict): def main(argv): args_parser = argparse.ArgumentParser( description="Verify that sets of hidden API flags are each a subset of " - "the monolithic flag file." - ) + "the monolithic flag file.") args_parser.add_argument("monolithicFlags", help="The monolithic flag file") args_parser.add_argument( "modularFlags", diff --git a/scripts/hiddenapi/verify_overlaps_test.py b/scripts/hiddenapi/verify_overlaps_test.py index 22a1cdf6d..8cf295962 100755 --- a/scripts/hiddenapi/verify_overlaps_test.py +++ b/scripts/hiddenapi/verify_overlaps_test.py @@ -17,54 +17,9 @@ import io import unittest -from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import +from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import -class TestSignatureToElements(unittest.TestCase): - - def signatureToElements(self, signature): - return InteriorNode().signatureToElements(signature) - - def test_signatureToElements_1(self): - expected = [ - 'package:java', - 'package:lang', - 'class:ProcessBuilder', - 'class:Redirect', - 'class:1', - 'member:<init>()V', - ] - self.assertEqual( - expected, - self.signatureToElements( - 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V')) - - def test_signatureToElements_2(self): - expected = [ - 'package:java', - 'package:lang', - 'class:Object', - 'member:hashCode()I', - ] - self.assertEqual( - expected, - self.signatureToElements('Ljava/lang/Object;->hashCode()I')) - - def test_signatureToElements_3(self): - expected = [ - 'package:java', - 'package:lang', - 'class:CharSequence', - 'class:', - 'class:ExternalSyntheticLambda0', - 'member:<init>(Ljava/lang/CharSequence;)V', - ] - self.assertEqual( - expected, - self.signatureToElements( - 'Ljava/lang/CharSequence$$ExternalSyntheticLambda0;' - '-><init>(Ljava/lang/CharSequence;)V')) - #pylint: disable=line-too-long class TestDetectOverlaps(unittest.TestCase): @@ -239,18 +194,6 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked } self.assertEqual(expected, subset) - def test_extract_subset_invalid_pattern_wildcard_and_member(self): - monolithic = self.read_flag_trie_from_string( - TestDetectOverlaps.extractInput) - - patterns = 'Ljava/lang/*;->hashCode()I' - - with self.assertRaises(Exception) as context: - self.extract_subset_from_monolithic_flags_as_dict_from_string( - monolithic, patterns) - self.assertTrue('contains wildcard * and member signature hashCode()I' - in str(context.exception)) - def test_read_trie_duplicate(self): with self.assertRaises(Exception) as context: self.read_flag_trie_from_string(""" @@ -369,6 +312,8 @@ Ljava/lang/Object;->hashCode()I,blocked mismatches = compare_signature_flags(monolithic, modular) expected = [] self.assertEqual(expected, mismatches) + + #pylint: enable=line-too-long if __name__ == '__main__': diff --git a/ui/build/config.go b/ui/build/config.go index 077a4d199..01fe8fa4a 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -368,10 +368,14 @@ func NewConfig(ctx Context, args ...string) Config { java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag()) java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag()) + java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag()) javaHome := func() string { if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { return override } + if ret.environ.IsEnvTrue("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN") { + return java17Home + } if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.") } |