diff options
55 files changed, 3369 insertions, 2146 deletions
diff --git a/android/Android.bp b/android/Android.bp index 7bd145080..66d361efa 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "apex.go", "api_levels.go", "arch.go", + "arch_list.go", "bazel_handler.go", "config.go", "csuite_config.go", @@ -32,11 +33,13 @@ bootstrap_go_package { "mutator.go", "namespace.go", "neverallow.go", + "ninja_deps.go", "notices.go", "onceper.go", "override_module.go", "package.go", "package_ctx.go", + "packaging.go", "path_properties.go", "paths.go", "phony.go", @@ -73,8 +76,10 @@ bootstrap_go_package { "mutator_test.go", "namespace_test.go", "neverallow_test.go", + "ninja_deps_test.go", "onceper_test.go", "package_test.go", + "packaging_test.go", "path_properties_test.go", "paths_test.go", "prebuilt_test.go", diff --git a/android/arch.go b/android/arch.go index 98ff07a8e..eb651e618 100644 --- a/android/arch.go +++ b/android/arch.go @@ -27,28 +27,6 @@ import ( "github.com/google/blueprint/proptools" ) -const COMMON_VARIANT = "common" - -var ( - archTypeList []ArchType - - Arm = newArch("arm", "lib32") - Arm64 = newArch("arm64", "lib64") - X86 = newArch("x86", "lib32") - X86_64 = newArch("x86_64", "lib64") - - Common = ArchType{ - Name: COMMON_VARIANT, - } -) - -var archTypeMap = map[string]ArchType{ - "arm": Arm, - "arm64": Arm64, - "x86": X86, - "x86_64": X86_64, -} - /* Example blueprints file containing all variant property groups, with comment listing what type of variants get properties in that group: @@ -111,405 +89,26 @@ module { } */ -var archVariants = map[ArchType][]string{ - Arm: { - "armv7-a", - "armv7-a-neon", - "armv8-a", - "armv8-2a", - "cortex-a7", - "cortex-a8", - "cortex-a9", - "cortex-a15", - "cortex-a53", - "cortex-a53-a57", - "cortex-a55", - "cortex-a72", - "cortex-a73", - "cortex-a75", - "cortex-a76", - "krait", - "kryo", - "kryo385", - "exynos-m1", - "exynos-m2", - }, - Arm64: { - "armv8_a", - "armv8_2a", - "armv8-2a-dotprod", - "cortex-a53", - "cortex-a55", - "cortex-a72", - "cortex-a73", - "cortex-a75", - "cortex-a76", - "kryo", - "kryo385", - "exynos-m1", - "exynos-m2", - }, - X86: { - "amberlake", - "atom", - "broadwell", - "haswell", - "icelake", - "ivybridge", - "kabylake", - "sandybridge", - "silvermont", - "skylake", - "stoneyridge", - "tigerlake", - "whiskeylake", - "x86_64", - }, - X86_64: { - "amberlake", - "broadwell", - "haswell", - "icelake", - "ivybridge", - "kabylake", - "sandybridge", - "silvermont", - "skylake", - "stoneyridge", - "tigerlake", - "whiskeylake", - }, -} - -var archFeatures = map[ArchType][]string{ - Arm: { - "neon", - }, - Arm64: { - "dotprod", - }, - X86: { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "avx2", - "avx512", - "popcnt", - "movbe", - }, - X86_64: { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "avx2", - "avx512", - "popcnt", - }, -} - -var archFeatureMap = map[ArchType]map[string][]string{ - Arm: { - "armv7-a-neon": { - "neon", - }, - "armv8-a": { - "neon", - }, - "armv8-2a": { - "neon", - }, - }, - Arm64: { - "armv8-2a-dotprod": { - "dotprod", - }, - }, - X86: { - "amberlake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "aes_ni", - "popcnt", - }, - "atom": { - "ssse3", - "movbe", - }, - "broadwell": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "aes_ni", - "popcnt", - }, - "haswell": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "popcnt", - "movbe", - }, - "icelake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "ivybridge": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "popcnt", - }, - "kabylake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "aes_ni", - "popcnt", - }, - "sandybridge": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "popcnt", - }, - "silvermont": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "popcnt", - "movbe", - }, - "skylake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "stoneyridge": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "avx2", - "popcnt", - "movbe", - }, - "tigerlake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "whiskeylake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "x86_64": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "popcnt", - }, - }, - X86_64: { - "amberlake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "aes_ni", - "popcnt", - }, - "broadwell": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "aes_ni", - "popcnt", - }, - "haswell": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "popcnt", - }, - "icelake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "ivybridge": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "popcnt", - }, - "kabylake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "aes_ni", - "popcnt", - }, - "sandybridge": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "popcnt", - }, - "silvermont": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "popcnt", - }, - "skylake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "stoneyridge": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "aes_ni", - "avx", - "avx2", - "popcnt", - }, - "tigerlake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - "whiskeylake": { - "ssse3", - "sse4", - "sse4_1", - "sse4_2", - "avx", - "avx2", - "avx512", - "aes_ni", - "popcnt", - }, - }, -} +// An Arch indicates a single CPU architecture. +type Arch struct { + // The type of the architecture (arm, arm64, x86, or x86_64). + ArchType ArchType -var defaultArchFeatureMap = map[OsType]map[ArchType][]string{} + // The variant of the architecture, for example "armv7-a" or "armv7-a-neon" for arm. + ArchVariant string -func RegisterDefaultArchVariantFeatures(os OsType, arch ArchType, features ...string) { - checkCalledFromInit() + // The variant of the CPU, for example "cortex-a53" for arm64. + CpuVariant string - for _, feature := range features { - if !InList(feature, archFeatures[arch]) { - panic(fmt.Errorf("Invalid feature %q for arch %q variant \"\"", feature, arch)) - } - } + // The list of Android app ABIs supported by the CPU architecture, for example "arm64-v8a". + Abi []string - if defaultArchFeatureMap[os] == nil { - defaultArchFeatureMap[os] = make(map[ArchType][]string) - } - defaultArchFeatureMap[os][arch] = features -} - -// An Arch indicates a single CPU architecture. -type Arch struct { - ArchType ArchType - ArchVariant string - CpuVariant string - Abi []string + // The list of arch-specific features supported by the CPU architecture, for example "neon". ArchFeatures []string } +// String returns the Arch as a string. The value is used as the name of the variant created +// by archMutator. func (a Arch) String() string { s := a.ArchType.String() if a.ArchVariant != "" { @@ -521,12 +120,42 @@ func (a Arch) String() string { return s } +// ArchType is used to define the 4 supported architecture types (arm, arm64, x86, x86_64), as +// well as the "common" architecture used for modules that support multiple architectures, for +// example Java modules. type ArchType struct { - Name string - Field string + // Name is the name of the architecture type, "arm", "arm64", "x86", or "x86_64". + Name string + + // Field is the name of the field used in properties that refer to the architecture, e.g. "Arm64". + Field string + + // Multilib is either "lib32" or "lib64" for 32-bit or 64-bit architectures. Multilib string } +// String returns the name of the ArchType. +func (a ArchType) String() string { + return a.Name +} + +const COMMON_VARIANT = "common" + +var ( + archTypeList []ArchType + + Arm = newArch("arm", "lib32") + Arm64 = newArch("arm64", "lib64") + X86 = newArch("x86", "lib32") + X86_64 = newArch("x86_64", "lib64") + + Common = ArchType{ + Name: COMMON_VARIANT, + } +) + +var archTypeMap = map[string]ArchType{} + func newArch(name, multilib string) ArchType { archType := ArchType{ Name: name, @@ -534,25 +163,25 @@ func newArch(name, multilib string) ArchType { Multilib: multilib, } archTypeList = append(archTypeList, archType) + archTypeMap[name] = archType return archType } +// ArchTypeList returns the 4 supported ArchTypes for arm, arm64, x86 and x86_64. func ArchTypeList() []ArchType { return append([]ArchType(nil), archTypeList...) } -func (a ArchType) String() string { - return a.Name -} - -var _ encoding.TextMarshaler = ArchType{} - +// MarshalText allows an ArchType to be serialized through any encoder that supports +// encoding.TextMarshaler. func (a ArchType) MarshalText() ([]byte, error) { return []byte(strconv.Quote(a.String())), nil } -var _ encoding.TextUnmarshaler = &ArchType{} +var _ encoding.TextMarshaler = ArchType{} +// UnmarshalText allows an ArchType to be deserialized through any decoder that supports +// encoding.TextUnmarshaler. func (a *ArchType) UnmarshalText(text []byte) error { if u, ok := archTypeMap[string(text)]; ok { *a = u @@ -562,67 +191,22 @@ func (a *ArchType) UnmarshalText(text []byte) error { return fmt.Errorf("unknown ArchType %q", text) } -var BuildOs = func() OsType { - switch runtime.GOOS { - case "linux": - return Linux - case "darwin": - return Darwin - default: - panic(fmt.Sprintf("unsupported OS: %s", runtime.GOOS)) - } -}() - -var BuildArch = func() ArchType { - switch runtime.GOARCH { - case "amd64": - return X86_64 - default: - panic(fmt.Sprintf("unsupported Arch: %s", runtime.GOARCH)) - } -}() - -var ( - OsTypeList []OsType - commonTargetMap = make(map[string]Target) - - NoOsType OsType - Linux = NewOsType("linux_glibc", Host, false) - Darwin = NewOsType("darwin", Host, false) - LinuxBionic = NewOsType("linux_bionic", Host, false) - Windows = NewOsType("windows", Host, true) - Android = NewOsType("android", Device, false) - Fuchsia = NewOsType("fuchsia", Device, false) - - // A pseudo OSType for a common os variant, which is OSType agnostic and which - // has dependencies on all the OS variants. - CommonOS = NewOsType("common_os", Generic, false) - - osArchTypeMap = map[OsType][]ArchType{ - Linux: []ArchType{X86, X86_64}, - LinuxBionic: []ArchType{Arm64, X86_64}, - Darwin: []ArchType{X86_64}, - Windows: []ArchType{X86, X86_64}, - Android: []ArchType{Arm, Arm64, X86, X86_64}, - Fuchsia: []ArchType{Arm64, X86_64}, - } -) - -type OsType struct { - Name, Field string - Class OsClass - - DefaultDisabled bool -} +var _ encoding.TextUnmarshaler = &ArchType{} +// OsClass is an enum that describes whether a variant of a module runs on the host, on the device, +// or is generic. type OsClass int const ( + // Generic is used for variants of modules that are not OS-specific. Generic OsClass = iota + // Device is used for variants of modules that run on the device. Device + // Host is used for variants of modules that run on the host. Host ) +// String returns the OsClass as a string. func (class OsClass) String() string { switch class { case Generic: @@ -636,22 +220,48 @@ func (class OsClass) String() string { } } +// OsType describes an OS variant of a module. +type OsType struct { + // Name is the name of the OS. It is also used as the name of the property in Android.bp + // files. + Name string + + // Field is the name of the OS converted to an exported field name, i.e. with the first + // character capitalized. + Field string + + // Class is the OsClass of the OS. + Class OsClass + + // DefaultDisabled is set when the module variants for the OS should not be created unless + // the module explicitly requests them. This is used to limit Windows cross compilation to + // only modules that need it. + DefaultDisabled bool +} + +// String returns the name of the OsType. func (os OsType) String() string { return os.Name } +// Bionic returns true if the OS uses the Bionic libc runtime, i.e. if the OS is Android or +// is Linux with Bionic. func (os OsType) Bionic() bool { return os == Android || os == LinuxBionic } +// Linux returns true if the OS uses the Linux kernel, i.e. if the OS is Android or is Linux +// with or without the Bionic libc runtime. func (os OsType) Linux() bool { return os == Android || os == Linux || os == LinuxBionic } -func NewOsType(name string, class OsClass, defDisabled bool) OsType { +// newOsType constructs an OsType and adds it to the global lists. +func newOsType(name string, class OsClass, defDisabled bool, archTypes ...ArchType) OsType { + checkCalledFromInit() os := OsType{ Name: name, - Field: strings.Title(name), + Field: proptools.FieldNameForProperty(name), Class: class, DefaultDisabled: defDisabled, @@ -663,10 +273,12 @@ func NewOsType(name string, class OsClass, defDisabled bool) OsType { } else { commonTargetMap[name] = Target{Os: os, Arch: Arch{ArchType: Common}} } + osArchTypeMap[os] = archTypes return os } +// osByName returns the OsType that has the given name, or NoOsType if none match. func osByName(name string) OsType { for _, os := range OsTypeList { if os.Name == name { @@ -677,18 +289,74 @@ func osByName(name string) OsType { return NoOsType } -type NativeBridgeSupport bool +// BuildOs returns the OsType for the OS that the build is running on. +var BuildOs = func() OsType { + switch runtime.GOOS { + case "linux": + return Linux + case "darwin": + return Darwin + default: + panic(fmt.Sprintf("unsupported OS: %s", runtime.GOOS)) + } +}() -const ( - NativeBridgeDisabled NativeBridgeSupport = false - NativeBridgeEnabled NativeBridgeSupport = true +// BuildArch returns the ArchType for the CPU that the build is running on. +var BuildArch = func() ArchType { + switch runtime.GOARCH { + case "amd64": + return X86_64 + default: + panic(fmt.Sprintf("unsupported Arch: %s", runtime.GOARCH)) + } +}() + +var ( + // OsTypeList contains a list of all the supported OsTypes, including ones not supported + // by the current build host or the target device. + OsTypeList []OsType + // commonTargetMap maps names of OsTypes to the corresponding common Target, i.e. the + // Target with the same OsType and the common ArchType. + commonTargetMap = make(map[string]Target) + // osArchTypeMap maps OsTypes to the list of supported ArchTypes for that OS. + osArchTypeMap = map[OsType][]ArchType{} + + // NoOsType is a placeholder for when no OS is needed. + NoOsType OsType + // Linux is the OS for the Linux kernel plus the glibc runtime. + Linux = newOsType("linux_glibc", Host, false, X86, X86_64) + // Darwin is the OS for MacOS/Darwin host machines. + Darwin = newOsType("darwin", Host, false, X86_64) + // LinuxBionic is the OS for the Linux kernel plus the Bionic libc runtime, but without the + // rest of Android. + LinuxBionic = newOsType("linux_bionic", Host, false, Arm64, X86_64) + // Windows the OS for Windows host machines. + Windows = newOsType("windows", Host, true, X86, X86_64) + // Android is the OS for target devices that run all of Android, including the Linux kernel + // and the Bionic libc runtime. + Android = newOsType("android", Device, false, Arm, Arm64, X86, X86_64) + // Fuchsia is the OS for target devices that run Fuchsia. + Fuchsia = newOsType("fuchsia", Device, false, Arm64, X86_64) + + // CommonOS is a pseudo OSType for a common OS variant, which is OsType agnostic and which + // has dependencies on all the OS variants. + CommonOS = newOsType("common_os", Generic, false) ) +// Target specifies the OS and architecture that a module is being compiled for. type Target struct { - Os OsType - Arch Arch - NativeBridge NativeBridgeSupport + // Os the OS that the module is being compiled for (e.g. "linux_glibc", "android"). + Os OsType + // Arch is the architecture that the module is being compiled for. + Arch Arch + // NativeBridge is NativeBridgeEnabled if the architecture is supported using NativeBridge + // (i.e. arm on x86) for this device. + NativeBridge NativeBridgeSupport + // NativeBridgeHostArchName is the name of the real architecture that is used to implement + // the NativeBridge architecture. For example, for arm on x86 this would be "x86". NativeBridgeHostArchName string + // NativeBridgeRelativePath is the name of the subdirectory that will contain NativeBridge + // libraries and binaries. NativeBridgeRelativePath string // HostCross is true when the target cannot run natively on the current build host. @@ -697,14 +365,25 @@ type Target struct { HostCross bool } +// NativeBridgeSupport is an enum that specifies if a Target supports NativeBridge. +type NativeBridgeSupport bool + +const ( + NativeBridgeDisabled NativeBridgeSupport = false + NativeBridgeEnabled NativeBridgeSupport = true +) + +// String returns the OS and arch variations used for the Target. func (target Target) String() string { return target.OsVariation() + "_" + target.ArchVariation() } +// OsVariation returns the name of the variation used by the osMutator for the Target. func (target Target) OsVariation() string { return target.Os.String() } +// ArchVariation returns the name of the variation used by the archMutator for the Target. func (target Target) ArchVariation() string { var variation string if target.NativeBridge { @@ -715,6 +394,8 @@ func (target Target) ArchVariation() string { return variation } +// Variations returns a list of blueprint.Variations for the osMutator and archMutator for the +// Target. func (target Target) Variations() []blueprint.Variation { return []blueprint.Variation{ {Mutator: "os", Variation: target.OsVariation()}, @@ -722,10 +403,16 @@ func (target Target) Variations() []blueprint.Variation { } } +// osMutator splits an arch-specific module into a variant for each OS that is enabled for the +// module. It uses the HostOrDevice value passed to InitAndroidArchModule and the +// device_supported and host_supported properties to determine which OsTypes are enabled for this +// module, then searches through the Targets to determine which have enabled Targets for this +// module. func osMutator(bpctx blueprint.BottomUpMutatorContext) { var module Module var ok bool if module, ok = bpctx.Module().(Module); !ok { + // The module is not a Soong module, it is a Blueprint module. if bootstrap.IsBootstrapModule(bpctx.Module()) { // Bootstrap Go modules are always the build OS or linux bionic. config := bpctx.Config().(Config) @@ -749,35 +436,38 @@ func osMutator(bpctx blueprint.BottomUpMutatorContext) { base := module.base() + // Nothing to do for modules that are not architecture specific (e.g. a genrule). if !base.ArchSpecific() { return } + // Collect a list of OSTypes supported by this module based on the HostOrDevice value + // passed to InitAndroidArchModule and the device_supported and host_supported properties. var moduleOSList []OsType - for _, os := range OsTypeList { for _, t := range mctx.Config().Targets[os] { - if base.supportsTarget(t, mctx.Config()) { + if base.supportsTarget(t) { moduleOSList = append(moduleOSList, os) break } } } + // If there are no supported OSes then disable the module. if len(moduleOSList) == 0 { base.Disable() return } + // Convert the list of supported OsTypes to the variation names. osNames := make([]string, len(moduleOSList)) - for i, os := range moduleOSList { osNames[i] = os.String() } createCommonOSVariant := base.commonProperties.CreateCommonOSVariant if createCommonOSVariant { - // A CommonOS variant was requested so add it to the list of OS's variants to + // A CommonOS variant was requested so add it to the list of OS variants to // create. It needs to be added to the end because it needs to depend on the // the other variants in the list returned by CreateVariations(...) and inter // variant dependencies can only be created from a later variant in that list to @@ -787,6 +477,8 @@ func osMutator(bpctx blueprint.BottomUpMutatorContext) { moduleOSList = append(moduleOSList, CommonOS) } + // Create the variations, annotate each one with which OS it was created for, and + // squash the appropriate OS-specific properties into the top level properties. modules := mctx.CreateVariations(osNames...) for i, m := range modules { m.base().commonProperties.CompileOS = moduleOSList[i] @@ -841,7 +533,7 @@ func GetOsSpecificVariantsOfCommonOSVariant(mctx BaseModuleContext) []Module { } // archMutator splits a module into a variant for each Target requested by the module. Target selection -// for a module is in three levels, OsClass, mulitlib, and then Target. +// for a module is in three levels, OsClass, multilib, and then Target. // OsClass selection is determined by: // - The HostOrDeviceSupported value passed in to InitAndroidArchModule by the module type factory, which selects // whether the module type can compile for host, device or both. @@ -858,7 +550,11 @@ func GetOsSpecificVariantsOfCommonOSVariant(mctx BaseModuleContext) []Module { // but may be arm for a 32-bit only build. // "32": compile for only a single 32-bit Target supported by the OsClass. // "64": compile for only a single 64-bit Target supported by the OsClass. -// "common": compile a for a single Target that will work on all Targets suported by the OsClass (for example Java). +// "common": compile a for a single Target that will work on all Targets supported by the OsClass (for example Java). +// "common_first": compile a for a Target that will work on all Targets supported by the OsClass +// (same as "common"), plus a second Target for the preferred Target supported by the OsClass +// (same as "first"). This is used for java_binary that produces a common .jar and a wrapper +// executable script. // // Once the list of Targets is determined, the module is split into a variant for each Target. // @@ -899,8 +595,8 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) { osTargets := mctx.Config().Targets[os] image := base.commonProperties.ImageVariation - // Filter NativeBridge targets unless they are explicitly supported - // Skip creating native bridge variants for vendor modules + // Filter NativeBridge targets unless they are explicitly supported. + // Skip creating native bridge variants for non-core modules. if os == Android && !(Bool(base.commonProperties.Native_bridge_supported) && image == CoreVariation) { @@ -919,17 +615,24 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) { osTargets = []Target{osTargets[0]} } + // Some modules want compile_multilib: "first" to mean 32-bit, not 64-bit. + // This is used for Windows support and for HOST_PREFER_32_BIT=true support for Art modules. prefer32 := false if base.prefer32 != nil { prefer32 = base.prefer32(mctx, base, os) } + // Determine the multilib selection for this module. multilib, extraMultilib := decodeMultilib(base, os.Class) + + // Convert the multilib selection into a list of Targets. targets, err := decodeMultilibTargets(multilib, osTargets, prefer32) if err != nil { mctx.ModuleErrorf("%s", err.Error()) } + // If the module is using extraMultilib, decode the extraMultilib selection into + // a separate list of Targets. var multiTargets []Target if extraMultilib != "" { multiTargets, err = decodeMultilibTargets(extraMultilib, osTargets, prefer32) @@ -938,23 +641,27 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) { } } + // Recovery is always the primary architecture, filter out any other architectures. if image == RecoveryVariation { primaryArch := mctx.Config().DevicePrimaryArchType() targets = filterToArch(targets, primaryArch) multiTargets = filterToArch(multiTargets, primaryArch) } + // If there are no supported targets disable the module. if len(targets) == 0 { base.Disable() return } + // Convert the targets into a list of arch variation names. targetNames := make([]string, len(targets)) - for i, target := range targets { targetNames[i] = target.ArchVariation() } + // Create the variations, annotate each one with which Target it was created for, and + // squash the appropriate arch-specific properties into the top level properties. modules := mctx.CreateVariations(targetNames...) for i, m := range modules { addTargetProperties(m, targets[i], multiTargets, i == 0) @@ -968,22 +675,34 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) { } } +// addTargetProperties annotates a variant with the Target is is being compiled for, the list +// of additional Targets it is supporting (if any), and whether it is the primary Target for +// the module. func addTargetProperties(m Module, target Target, multiTargets []Target, primaryTarget bool) { m.base().commonProperties.CompileTarget = target m.base().commonProperties.CompileMultiTargets = multiTargets m.base().commonProperties.CompilePrimary = primaryTarget } +// decodeMultilib returns the appropriate compile_multilib property for the module, or the default +// multilib from the factory's call to InitAndroidArchModule if none was set. For modules that +// called InitAndroidMultiTargetsArchModule it always returns "common" for multilib, and returns +// the actual multilib in extraMultilib. func decodeMultilib(base *ModuleBase, class OsClass) (multilib, extraMultilib string) { + // First check the "android.compile_multilib" or "host.compile_multilib" properties. switch class { case Device: multilib = String(base.commonProperties.Target.Android.Compile_multilib) case Host: multilib = String(base.commonProperties.Target.Host.Compile_multilib) } + + // If those aren't set, try the "compile_multilib" property. if multilib == "" { multilib = String(base.commonProperties.Compile_multilib) } + + // If that wasn't set, use the default multilib set by the factory. if multilib == "" { multilib = base.commonProperties.Default_multilib } @@ -1000,6 +719,8 @@ func decodeMultilib(base *ModuleBase, class OsClass) (multilib, extraMultilib st } } +// filterToArch takes a list of Targets and an ArchType, and returns a modified list that contains +// only Targets that have the specified ArchType. func filterToArch(targets []Target, arch ArchType) []Target { for i := 0; i < len(targets); i++ { if targets[i].Arch.ArchType != arch { @@ -1010,17 +731,27 @@ func filterToArch(targets []Target, arch ArchType) []Target { return targets } -type archPropTypeDesc struct { - arch, multilib, target reflect.Type -} - +// archPropRoot is a struct type used as the top level of the arch-specific properties. It +// contains the "arch", "multilib", and "target" property structs. It is used to split up the +// property structs to limit how much is allocated when a single arch-specific property group is +// used. The types are interface{} because they will hold instances of runtime-created types. type archPropRoot struct { Arch, Multilib, Target interface{} } +// archPropTypeDesc holds the runtime-created types for the property structs to instantiate to +// create an archPropRoot property struct. +type archPropTypeDesc struct { + arch, multilib, target reflect.Type +} + // createArchPropTypeDesc takes a reflect.Type that is either a struct or a pointer to a struct, and // returns lists of reflect.Types that contains the arch-variant properties inside structs for each // arch, multilib and target property. +// +// This is a relatively expensive operation, so the results are cached in the global +// archPropTypeMap. It is constructed entirely based on compile-time data, so there is no need +// to isolate the results between multiple tests running in parallel. func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { // Each property struct shard will be nested many times under the runtime generated arch struct, // which can hit the limit of 64kB for the name of runtime generated structs. They are nested @@ -1029,7 +760,12 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { // could cause problems if a single deeply nested property no longer fits in the name. const maxArchTypeNameSize = 500 + // Convert the type to a new set of types that contains only the arch-specific properties + // (those that are tagged with `android:"arch_specific"`), and sharded into multiple types + // to keep the runtime-generated names under the limit. propShards, _ := proptools.FilterPropertyStructSharded(props, maxArchTypeNameSize, filterArchStruct) + + // If the type has no arch-specific properties there is nothing to do. if len(propShards) == 0 { return nil } @@ -1037,6 +773,8 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { var ret []archPropTypeDesc for _, props := range propShards { + // variantFields takes a list of variant property field names and returns a list the + // StructFields with the names and the type of the current shard. variantFields := func(names []string) []reflect.StructField { ret := make([]reflect.StructField, len(names)) @@ -1048,9 +786,11 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { return ret } + // Create a type that contains the properties in this shard repeated for each + // architecture, architecture variant, and architecture feature. archFields := make([]reflect.StructField, len(archTypeList)) for i, arch := range archTypeList { - variants := []string{} + var variants []string for _, archVariant := range archVariants[arch] { archVariant := variantReplacer.Replace(archVariant) @@ -1061,8 +801,13 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { variants = append(variants, proptools.FieldNameForProperty(feature)) } + // Create the StructFields for each architecture variant architecture feature + // (e.g. "arch.arm.cortex-a53" or "arch.arm.neon"). fields := variantFields(variants) + // Create the StructField for the architecture itself (e.g. "arch.arm"). The special + // "BlueprintEmbed" name is used by Blueprint to put the properties in the + // parent struct. fields = append([]reflect.StructField{{ Name: "BlueprintEmbed", Type: props, @@ -1074,10 +819,15 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { Type: reflect.StructOf(fields), } } + + // Create the type of the "arch" property struct for this shard. archType := reflect.StructOf(archFields) + // Create the type for the "multilib" property struct for this shard, containing the + // "multilib.lib32" and "multilib.lib64" property structs. multilibType := reflect.StructOf(variantFields([]string{"Lib32", "Lib64"})) + // Start with a list of the special targets targets := []string{ "Host", "Android64", @@ -1090,11 +840,14 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { "Native_bridge", } for _, os := range OsTypeList { + // Add all the OSes. targets = append(targets, os.Field) + // Add the OS/Arch combinations, e.g. "android_arm64". for _, archType := range osArchTypeMap[os] { targets = append(targets, os.Field+"_"+archType.Name) + // Also add the special "linux_<arch>" and "bionic_<arch>" property structs. if os.Linux() { target := "Linux_" + archType.Name if !InList(target, targets) { @@ -1110,8 +863,10 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { } } + // Create the type for the "target" property struct for this shard. targetType := reflect.StructOf(variantFields(targets)) + // Return a descriptor of the 3 runtime-created types. ret = append(ret, archPropTypeDesc{ arch: reflect.PtrTo(archType), multilib: reflect.PtrTo(multilibType), @@ -1121,6 +876,11 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc { return ret } +// variantReplacer converts architecture variant or architecture feature names into names that +// are valid for an Android.bp file. +var variantReplacer = strings.NewReplacer("-", "_", ".", "_") + +// filterArchStruct returns true if the given field is an architecture specific property. func filterArchStruct(field reflect.StructField, prefix string) (bool, reflect.StructField) { if proptools.HasTag(field, "android", "arch_variant") { // The arch_variant field isn't necessary past this point @@ -1147,12 +907,17 @@ func filterArchStruct(field reflect.StructField, prefix string) (bool, reflect.S return false, field } +// archPropTypeMap contains a cache of the results of createArchPropTypeDesc for each type. It is +// shared across all Contexts, but is constructed based only on compile-time information so there +// is no risk of contaminating one Context with data from another. var archPropTypeMap OncePer -func InitArchModule(m Module) { +// initArchModule adds the architecture-specific property structs to a Module. +func initArchModule(m Module) { base := m.base() + // Store the original list of top level property structs base.generalProperties = m.GetProperties() for _, properties := range base.generalProperties { @@ -1169,10 +934,13 @@ func InitArchModule(m Module) { propertiesValue.Interface())) } + // Get or create the arch-specific property struct types for this property struct type. archPropTypes := archPropTypeMap.Once(NewCustomOnceKey(t), func() interface{} { return createArchPropTypeDesc(t) }).([]archPropTypeDesc) + // Instantiate one of each arch-specific property struct type and add it to the + // properties for the Module. var archProperties []interface{} for _, t := range archPropTypes { archProperties = append(archProperties, &archPropRoot{ @@ -1185,14 +953,18 @@ func InitArchModule(m Module) { m.AddProperties(archProperties...) } + // Update the list of properties that can be set by a defaults module or a call to + // AppendMatchingProperties or PrependMatchingProperties. base.customizableProperties = m.GetProperties() } -var variantReplacer = strings.NewReplacer("-", "_", ".", "_") - +// appendProperties squashes properties from the given field of the given src property struct +// into the dst property struct. Returns the reflect.Value of the field in the src property +// struct to be used for further appendProperties calls on fields of that property struct. func (m *ModuleBase) appendProperties(ctx BottomUpMutatorContext, dst interface{}, src reflect.Value, field, srcPrefix string) reflect.Value { + // Step into non-nil pointers to structs in the src value. if src.Kind() == reflect.Ptr { if src.IsNil() { return src @@ -1200,18 +972,25 @@ func (m *ModuleBase) appendProperties(ctx BottomUpMutatorContext, src = src.Elem() } + // Find the requested field in the src struct. src = src.FieldByName(field) if !src.IsValid() { ctx.ModuleErrorf("field %q does not exist", srcPrefix) return src } + // Save the value of the field in the src struct to return. ret := src + // If the value of the field is a struct (as opposed to a pointer to a struct) then step + // into the BlueprintEmbed field. if src.Kind() == reflect.Struct { src = src.FieldByName("BlueprintEmbed") } + // order checks the `android:"variant_prepend"` tag to handle properties where the + // arch-specific value needs to come before the generic value, for example for lists of + // include directories. order := func(property string, dstField, srcField reflect.StructField, dstValue, srcValue interface{}) (proptools.Order, error) { @@ -1222,6 +1001,7 @@ func (m *ModuleBase) appendProperties(ctx BottomUpMutatorContext, } } + // Squash the located property struct into the destination property struct. err := proptools.ExtendMatchingProperties([]interface{}{dst}, src.Interface(), nil, order) if err != nil { if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { @@ -1234,7 +1014,8 @@ func (m *ModuleBase) appendProperties(ctx BottomUpMutatorContext, return ret } -// Rewrite the module's properties structs to contain os-specific values. +// Squash the appropriate OS-specific property structs into the matching top level property structs +// based on the CompileOS value that was annotated on the variant. func (m *ModuleBase) setOSProperties(ctx BottomUpMutatorContext) { os := m.commonProperties.CompileOS @@ -1328,7 +1109,8 @@ func (m *ModuleBase) setOSProperties(ctx BottomUpMutatorContext) { } } -// Rewrite the module's properties structs to contain arch-specific values. +// Squash the appropriate arch-specific property structs into the matching top level property +// structs based on the CompileTarget value that was annotated on the variant. func (m *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { arch := m.Arch() os := m.Os() @@ -1480,22 +1262,7 @@ func (m *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { } } -func forEachInterface(v reflect.Value, f func(reflect.Value)) { - switch v.Kind() { - case reflect.Interface: - f(v) - case reflect.Struct: - for i := 0; i < v.NumField(); i++ { - forEachInterface(v.Field(i), f) - } - case reflect.Ptr: - forEachInterface(v.Elem(), f) - default: - panic(fmt.Errorf("Unsupported kind %s", v.Kind())) - } -} - -// Convert the arch product variables into a list of targets for each os class structs +// Convert the arch product variables into a list of targets for each OsType. func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { variables := config.productVariables @@ -1567,19 +1334,26 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { return nil, fmt.Errorf("No host primary architecture set") } + // The primary host target, which must always exist. addTarget(BuildOs, *variables.HostArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) + // An optional secondary host target. if variables.HostSecondaryArch != nil && *variables.HostSecondaryArch != "" { addTarget(BuildOs, *variables.HostSecondaryArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) } + // An optional host target that uses the Bionic glibc runtime. if Bool(config.Host_bionic) { addTarget(LinuxBionic, "x86_64", nil, nil, nil, NativeBridgeDisabled, nil, nil) } + + // An optional cross-compiled host target that uses the Bionic glibc runtime on an arm64 + // architecture. if Bool(config.Host_bionic_arm64) { addTarget(LinuxBionic, "arm64", nil, nil, nil, NativeBridgeDisabled, nil, nil) } + // Optional cross-compiled host targets, generally Windows. if String(variables.CrossHost) != "" { crossHostOs := osByName(*variables.CrossHost) if crossHostOs == NoOsType { @@ -1590,28 +1364,34 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { return nil, fmt.Errorf("No cross-host primary architecture set") } + // The primary cross-compiled host target. addTarget(crossHostOs, *variables.CrossHostArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) + // An optional secondary cross-compiled host target. if variables.CrossHostSecondaryArch != nil && *variables.CrossHostSecondaryArch != "" { addTarget(crossHostOs, *variables.CrossHostSecondaryArch, nil, nil, nil, NativeBridgeDisabled, nil, nil) } } + // Optional device targets if variables.DeviceArch != nil && *variables.DeviceArch != "" { var target = Android if Bool(variables.Fuchsia) { target = Fuchsia } + // The primary device target. addTarget(target, *variables.DeviceArch, variables.DeviceArchVariant, variables.DeviceCpuVariant, variables.DeviceAbi, NativeBridgeDisabled, nil, nil) + // An optional secondary device target. if variables.DeviceSecondaryArch != nil && *variables.DeviceSecondaryArch != "" { addTarget(Android, *variables.DeviceSecondaryArch, variables.DeviceSecondaryArchVariant, variables.DeviceSecondaryCpuVariant, variables.DeviceSecondaryAbi, NativeBridgeDisabled, nil, nil) } + // An optional NativeBridge device target. if variables.NativeBridgeArch != nil && *variables.NativeBridgeArch != "" { addTarget(Android, *variables.NativeBridgeArch, variables.NativeBridgeArchVariant, variables.NativeBridgeCpuVariant, @@ -1619,6 +1399,7 @@ func decodeTargetProductVariables(config *config) (map[OsType][]Target, error) { variables.NativeBridgeRelativePath) } + // An optional secondary NativeBridge device target. if variables.DeviceSecondaryArch != nil && *variables.DeviceSecondaryArch != "" && variables.NativeBridgeSecondaryArch != nil && *variables.NativeBridgeSecondaryArch != "" { addTarget(Android, *variables.NativeBridgeSecondaryArch, @@ -1653,6 +1434,7 @@ func hasArmAndroidArch(targets []Target) bool { return false } +// archConfig describes a built-in configuration. type archConfig struct { arch string archVariant string @@ -1660,6 +1442,7 @@ type archConfig struct { abi []string } +// getMegaDeviceConfig returns a list of archConfigs for every architecture simultaneously. func getMegaDeviceConfig() []archConfig { return []archConfig{ {"arm", "armv7-a", "generic", []string{"armeabi-v7a"}}, @@ -1706,6 +1489,7 @@ func getMegaDeviceConfig() []archConfig { } } +// getNdkAbisConfig returns a list of archConfigs for the ABIs supported by the NDK. func getNdkAbisConfig() []archConfig { return []archConfig{ {"arm", "armv7-a", "", []string{"armeabi-v7a"}}, @@ -1715,6 +1499,7 @@ func getNdkAbisConfig() []archConfig { } } +// getAmlAbisConfig returns a list of archConfigs for the ABIs supported by mainline modules. func getAmlAbisConfig() []archConfig { return []archConfig{ {"arm", "armv7-a-neon", "", []string{"armeabi-v7a"}}, @@ -1724,6 +1509,7 @@ func getAmlAbisConfig() []archConfig { } } +// decodeArchSettings converts a list of archConfigs into a list of Targets for the given OsType. func decodeArchSettings(os OsType, archConfigs []archConfig) ([]Target, error) { var ret []Target @@ -1743,15 +1529,9 @@ func decodeArchSettings(os OsType, archConfigs []archConfig) ([]Target, error) { return ret, nil } -// Convert a set of strings from product variables into a single Arch struct +// decodeArch converts a set of strings from product variables into an Arch struct. func decodeArch(os OsType, arch string, archVariant, cpuVariant *string, abi []string) (Arch, error) { - stringPtr := func(p *string) string { - if p != nil { - return *p - } - return "" - } - + // Verify the arch is valid archType, ok := archTypeMap[arch] if !ok { return Arch{}, fmt.Errorf("unknown arch %q", arch) @@ -1759,19 +1539,22 @@ func decodeArch(os OsType, arch string, archVariant, cpuVariant *string, abi []s a := Arch{ ArchType: archType, - ArchVariant: stringPtr(archVariant), - CpuVariant: stringPtr(cpuVariant), + ArchVariant: String(archVariant), + CpuVariant: String(cpuVariant), Abi: abi, } + // Convert generic arch variants into the empty string. if a.ArchVariant == a.ArchType.Name || a.ArchVariant == "generic" { a.ArchVariant = "" } + // Convert generic CPU variants into the empty string. if a.CpuVariant == a.ArchType.Name || a.CpuVariant == "generic" { a.CpuVariant = "" } + // Filter empty ABIs out of the list. for i := 0; i < len(a.Abi); i++ { if a.Abi[i] == "" { a.Abi = append(a.Abi[:i], a.Abi[i+1:]...) @@ -1780,10 +1563,12 @@ func decodeArch(os OsType, arch string, archVariant, cpuVariant *string, abi []s } if a.ArchVariant == "" { + // Set ArchFeatures from the default arch features. if featureMap, ok := defaultArchFeatureMap[os]; ok { a.ArchFeatures = featureMap[archType] } } else { + // Set ArchFeatures from the arch type. if featureMap, ok := archFeatureMap[archType]; ok { a.ArchFeatures = featureMap[a.ArchVariant] } @@ -1792,6 +1577,8 @@ func decodeArch(os OsType, arch string, archVariant, cpuVariant *string, abi []s return a, nil } +// filterMultilibTargets takes a list of Targets and a multilib value and returns a new list of +// Targets containing only those that have the given multilib value. func filterMultilibTargets(targets []Target, multilib string) []Target { var ret []Target for _, t := range targets { @@ -1802,8 +1589,8 @@ func filterMultilibTargets(targets []Target, multilib string) []Target { return ret } -// Return the set of Os specific common architecture targets for each Os in a list of -// targets. +// getCommonTargets returns the set of Os specific common architecture targets for each Os in a list +// of targets. func getCommonTargets(targets []Target) []Target { var ret []Target set := make(map[string]bool) @@ -1818,6 +1605,9 @@ func getCommonTargets(targets []Target) []Target { return ret } +// firstTarget takes a list of Targets and a list of multilib values and returns a list of Targets +// that contains zero or one Target for each OsType, selecting the one that matches the earliest +// filter. func firstTarget(targets []Target, filters ...string) []Target { // find the first target from each OS var ret []Target @@ -1837,9 +1627,10 @@ func firstTarget(targets []Target, filters ...string) []Target { return ret } -// Use the module multilib setting to select one or more targets from a target list +// decodeMultilibTargets uses the module's multilib setting to select one or more targets from a +// list of Targets. func decodeMultilibTargets(multilib string, targets []Target, prefer32 bool) ([]Target, error) { - buildTargets := []Target{} + var buildTargets []Target switch multilib { case "common": diff --git a/android/arch_list.go b/android/arch_list.go new file mode 100644 index 000000000..0c33b9d34 --- /dev/null +++ b/android/arch_list.go @@ -0,0 +1,410 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import "fmt" + +var archVariants = map[ArchType][]string{ + Arm: { + "armv7-a", + "armv7-a-neon", + "armv8-a", + "armv8-2a", + "cortex-a7", + "cortex-a8", + "cortex-a9", + "cortex-a15", + "cortex-a53", + "cortex-a53-a57", + "cortex-a55", + "cortex-a72", + "cortex-a73", + "cortex-a75", + "cortex-a76", + "krait", + "kryo", + "kryo385", + "exynos-m1", + "exynos-m2", + }, + Arm64: { + "armv8_a", + "armv8_2a", + "armv8-2a-dotprod", + "cortex-a53", + "cortex-a55", + "cortex-a72", + "cortex-a73", + "cortex-a75", + "cortex-a76", + "kryo", + "kryo385", + "exynos-m1", + "exynos-m2", + }, + X86: { + "amberlake", + "atom", + "broadwell", + "haswell", + "icelake", + "ivybridge", + "kabylake", + "sandybridge", + "silvermont", + "skylake", + "stoneyridge", + "tigerlake", + "whiskeylake", + "x86_64", + }, + X86_64: { + "amberlake", + "broadwell", + "haswell", + "icelake", + "ivybridge", + "kabylake", + "sandybridge", + "silvermont", + "skylake", + "stoneyridge", + "tigerlake", + "whiskeylake", + }, +} + +var archFeatures = map[ArchType][]string{ + Arm: { + "neon", + }, + Arm64: { + "dotprod", + }, + X86: { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "avx2", + "avx512", + "popcnt", + "movbe", + }, + X86_64: { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "avx2", + "avx512", + "popcnt", + }, +} + +var archFeatureMap = map[ArchType]map[string][]string{ + Arm: { + "armv7-a-neon": { + "neon", + }, + "armv8-a": { + "neon", + }, + "armv8-2a": { + "neon", + }, + }, + Arm64: { + "armv8-2a-dotprod": { + "dotprod", + }, + }, + X86: { + "amberlake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "aes_ni", + "popcnt", + }, + "atom": { + "ssse3", + "movbe", + }, + "broadwell": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "aes_ni", + "popcnt", + }, + "haswell": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "popcnt", + "movbe", + }, + "icelake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "ivybridge": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "popcnt", + }, + "kabylake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "aes_ni", + "popcnt", + }, + "sandybridge": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "popcnt", + }, + "silvermont": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "popcnt", + "movbe", + }, + "skylake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "stoneyridge": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "avx2", + "popcnt", + "movbe", + }, + "tigerlake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "whiskeylake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "x86_64": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "popcnt", + }, + }, + X86_64: { + "amberlake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "aes_ni", + "popcnt", + }, + "broadwell": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "aes_ni", + "popcnt", + }, + "haswell": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "popcnt", + }, + "icelake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "ivybridge": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "popcnt", + }, + "kabylake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "aes_ni", + "popcnt", + }, + "sandybridge": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "popcnt", + }, + "silvermont": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "popcnt", + }, + "skylake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "stoneyridge": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "aes_ni", + "avx", + "avx2", + "popcnt", + }, + "tigerlake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + "whiskeylake": { + "ssse3", + "sse4", + "sse4_1", + "sse4_2", + "avx", + "avx2", + "avx512", + "aes_ni", + "popcnt", + }, + }, +} + +var defaultArchFeatureMap = map[OsType]map[ArchType][]string{} + +// RegisterDefaultArchVariantFeatures is called by files that define Toolchains to specify the +// arch features that are available for the default arch variant. It must be called from an +// init() function. +func RegisterDefaultArchVariantFeatures(os OsType, arch ArchType, features ...string) { + checkCalledFromInit() + + for _, feature := range features { + if !InList(feature, archFeatures[arch]) { + panic(fmt.Errorf("Invalid feature %q for arch %q variant \"\"", feature, arch)) + } + } + + if defaultArchFeatureMap[os] == nil { + defaultArchFeatureMap[os] = make(map[ArchType][]string) + } + defaultArchFeatureMap[os][arch] = features +} diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 5c7d9fc67..7d8d12f19 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -194,6 +194,7 @@ func (context *bazelContext) issueBazelCommand(command string, labels []string, cmdFlags := []string{"--output_base=" + context.outputBase, command} cmdFlags = append(cmdFlags, labels...) + cmdFlags = append(cmdFlags, "--package_path=%workspace%/"+context.intermediatesDir()) cmdFlags = append(cmdFlags, extraFlags...) bazelCmd := exec.Command(context.bazelPath, cmdFlags...) @@ -210,6 +211,21 @@ func (context *bazelContext) issueBazelCommand(command string, labels []string, } } +// Returns the string contents of a workspace file that should be output +// adjacent to the main bzl file and build file. +// This workspace file allows, via local_repository rule, sourcetree-level +// BUILD targets to be referenced via @sourceroot. +func (context *bazelContext) workspaceFileContents() []byte { + formatString := ` +# This file is generated by soong_build. Do not edit. +local_repository( + name = "sourceroot", + path = "%s", +) +` + return []byte(fmt.Sprintf(formatString, context.workspaceDir)) +} + func (context *bazelContext) mainBzlFileContents() []byte { contents := ` # This file is generated by soong_build. Do not edit. @@ -224,6 +240,18 @@ mixed_build_root = rule( return []byte(contents) } +// Returns a "canonicalized" corresponding to the given sourcetree-level label. +// This abstraction is required because a sourcetree label such as //foo/bar:baz +// must be referenced via the local repository prefix, such as +// @sourceroot//foo/bar:baz. +func canonicalizeLabel(label string) string { + if strings.HasPrefix(label, "//") { + return "@sourceroot" + label + } else { + return "@sourceroot//" + label + } +} + func (context *bazelContext) mainBuildFileContents() []byte { formatString := ` # This file is generated by soong_build. Do not edit. @@ -235,7 +263,7 @@ mixed_build_root(name = "buildroot", ` var buildRootDeps []string = nil for val, _ := range context.requests { - buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", val.label)) + buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", canonicalizeLabel(val.label))) } buildRootDepsString := strings.Join(buildRootDeps, ",\n ") @@ -261,13 +289,19 @@ def format(target): // TODO(cparsons): Sort by request type instead of assuming all requests // are of GetAllFiles type. for val, _ := range context.requests { - buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", val.label)) + buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", canonicalizeLabel(val.label))) } buildRootDepsString := strings.Join(buildRootDeps, ",\n ") return []byte(fmt.Sprintf(formatString, buildRootDepsString)) } +// Returns a workspace-relative path containing build-related metadata required +// for interfacing with Bazel. Example: out/soong/bazel. +func (context *bazelContext) intermediatesDir() string { + return filepath.Join(context.buildDir, "bazel") +} + // Issues commands to Bazel to receive results for all cquery requests // queued in the BazelContext. func (context *bazelContext) InvokeBazel() error { @@ -275,26 +309,38 @@ func (context *bazelContext) InvokeBazel() error { var cqueryOutput string var err error + + err = os.Mkdir(absolutePath(context.intermediatesDir()), 0777) + if err != nil { + return err + } err = ioutil.WriteFile( - absolutePath(filepath.Join(context.buildDir, "main.bzl")), + absolutePath(filepath.Join(context.intermediatesDir(), "main.bzl")), context.mainBzlFileContents(), 0666) if err != nil { return err } err = ioutil.WriteFile( - absolutePath(filepath.Join(context.buildDir, "BUILD.bazel")), + absolutePath(filepath.Join(context.intermediatesDir(), "BUILD.bazel")), context.mainBuildFileContents(), 0666) if err != nil { return err } - cquery_file_relpath := filepath.Join(context.buildDir, "buildroot.cquery") + cquery_file_relpath := filepath.Join(context.intermediatesDir(), "buildroot.cquery") err = ioutil.WriteFile( absolutePath(cquery_file_relpath), context.cqueryStarlarkFileContents(), 0666) if err != nil { return err } - buildroot_label := fmt.Sprintf("//%s:buildroot", context.buildDir) + workspace_file_relpath := filepath.Join(context.intermediatesDir(), "WORKSPACE.bazel") + err = ioutil.WriteFile( + absolutePath(workspace_file_relpath), + context.workspaceFileContents(), 0666) + if err != nil { + return err + } + buildroot_label := "//:buildroot" cqueryOutput, err = context.issueBazelCommand("cquery", []string{fmt.Sprintf("deps(%s)", buildroot_label)}, "--output=starlark", @@ -313,7 +359,7 @@ func (context *bazelContext) InvokeBazel() error { } for val, _ := range context.requests { - if cqueryResult, ok := cqueryResults[val.label]; ok { + if cqueryResult, ok := cqueryResults[canonicalizeLabel(val.label)]; ok { context.results[val] = string(cqueryResult) } else { return fmt.Errorf("missing result for bazel target %s", val.label) diff --git a/android/config.go b/android/config.go index e87a4ac2a..d833c5c4d 100644 --- a/android/config.go +++ b/android/config.go @@ -126,6 +126,8 @@ type config struct { // in tests when a path doesn't exist. testAllowNonExistentPaths bool + ninjaFileDepsSet sync.Map + OncePer } diff --git a/android/defaults.go b/android/defaults.go index eb013d7ab..44753ce22 100644 --- a/android/defaults.go +++ b/android/defaults.go @@ -183,7 +183,7 @@ func InitDefaultsModule(module DefaultsModule) { initAndroidModuleBase(module) initProductVariableModule(module) - InitArchModule(module) + initArchModule(module) InitDefaultableModule(module) // Add properties that will not have defaults applied to them. diff --git a/android/makevars.go b/android/makevars.go index 3ca7792d4..f78439596 100644 --- a/android/makevars.go +++ b/android/makevars.go @@ -139,15 +139,24 @@ type SingletonMakeVarsProvider interface { MakeVars(ctx MakeVarsContext) } -// registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to the list of -// MakeVarsProviders to run. -func registerSingletonMakeVarsProvider(singleton SingletonMakeVarsProvider) { - singletonMakeVarsProviders = append(singletonMakeVarsProviders, - makeVarsProvider{pctx, SingletonmakeVarsProviderAdapter(singleton)}) +var singletonMakeVarsProvidersKey = NewOnceKey("singletonMakeVarsProvidersKey") + +// registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to +// the list of MakeVarsProviders to run. +func registerSingletonMakeVarsProvider(config Config, singleton SingletonMakeVarsProvider) { + // Singletons are registered on the Context and may be different between different Contexts, + // for example when running multiple tests. Store the SingletonMakeVarsProviders in the + // Config so they are attached to the Context. + singletonMakeVarsProviders := config.Once(singletonMakeVarsProvidersKey, func() interface{} { + return &[]makeVarsProvider{} + }).(*[]makeVarsProvider) + + *singletonMakeVarsProviders = append(*singletonMakeVarsProviders, + makeVarsProvider{pctx, singletonMakeVarsProviderAdapter(singleton)}) } -// SingletonmakeVarsProviderAdapter converts a SingletonMakeVarsProvider to a MakeVarsProvider. -func SingletonmakeVarsProviderAdapter(singleton SingletonMakeVarsProvider) MakeVarsProvider { +// singletonMakeVarsProviderAdapter converts a SingletonMakeVarsProvider to a MakeVarsProvider. +func singletonMakeVarsProviderAdapter(singleton SingletonMakeVarsProvider) MakeVarsProvider { return func(ctx MakeVarsContext) { singleton.MakeVars(ctx) } } @@ -175,9 +184,6 @@ type makeVarsProvider struct { // Collection of makevars providers that are registered in init() methods. var makeVarsInitProviders []makeVarsProvider -// Collection of singleton makevars providers that are not registered as part of init() methods. -var singletonMakeVarsProviders []makeVarsProvider - type makeVarsContext struct { SingletonContext config Config @@ -224,20 +230,11 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { var vars []makeVarsVariable var dists []dist var phonies []phony - for _, provider := range append(makeVarsInitProviders) { - mctx := &makeVarsContext{ - SingletonContext: ctx, - pctx: provider.pctx, - } - provider.call(mctx) + providers := append([]makeVarsProvider(nil), makeVarsInitProviders...) + providers = append(providers, *ctx.Config().Get(singletonMakeVarsProvidersKey).(*[]makeVarsProvider)...) - vars = append(vars, mctx.vars...) - phonies = append(phonies, mctx.phonies...) - dists = append(dists, mctx.dists...) - } - - for _, provider := range append(singletonMakeVarsProviders) { + for _, provider := range providers { mctx := &makeVarsContext{ SingletonContext: ctx, pctx: provider.pctx, @@ -250,12 +247,6 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { dists = append(dists, mctx.dists...) } - // Clear singleton makevars providers after use. Since these are in-memory - // singletons, this ensures state is reset if the build tree is processed - // multiple times. - // TODO(cparsons): Clean up makeVarsProviders to be part of the context. - singletonMakeVarsProviders = nil - ctx.VisitAllModules(func(m Module) { if provider, ok := m.(ModuleMakeVarsProvider); ok && m.Enabled() { mctx := &makeVarsContext{ diff --git a/android/module.go b/android/module.go index d677406a6..6b659d2cd 100644 --- a/android/module.go +++ b/android/module.go @@ -440,6 +440,7 @@ type Module interface { TargetRequiredModuleNames() []string FilesToInstall() InstallPaths + PackagingSpecs() []PackagingSpec } // Qualified id for a module @@ -716,7 +717,9 @@ type commonProperties struct { DebugMutators []string `blueprint:"mutated"` DebugVariations []string `blueprint:"mutated"` - // set by ImageMutator + // ImageVariation is set by ImageMutator to specify which image this variation is for, + // for example "" for core or "recovery" for recovery. It will often be set to one of the + // constants in image.go, but can also be set to a custom value by individual module types. ImageVariation string `blueprint:"mutated"` } @@ -765,27 +768,32 @@ const ( type HostOrDeviceSupported int const ( - _ HostOrDeviceSupported = iota + hostSupported = 1 << iota + hostCrossSupported + deviceSupported + hostDefault + deviceDefault // Host and HostCross are built by default. Device is not supported. - HostSupported + HostSupported = hostSupported | hostCrossSupported | hostDefault // Host is built by default. HostCross and Device are not supported. - HostSupportedNoCross + HostSupportedNoCross = hostSupported | hostDefault // Device is built by default. Host and HostCross are not supported. - DeviceSupported + DeviceSupported = deviceSupported | deviceDefault // Device is built by default. Host and HostCross are supported. - HostAndDeviceSupported + HostAndDeviceSupported = hostSupported | hostCrossSupported | deviceSupported | deviceDefault // Host, HostCross, and Device are built by default. - HostAndDeviceDefault + HostAndDeviceDefault = hostSupported | hostCrossSupported | hostDefault | + deviceSupported | deviceDefault // Nothing is supported. This is not exposed to the user, but used to mark a // host only module as unsupported when the module type is not supported on // the host OS. E.g. benchmarks are supported on Linux but not Darwin. - NeitherHostNorDeviceSupported + NeitherHostNorDeviceSupported = 0 ) type moduleKind int @@ -819,6 +827,8 @@ func initAndroidModuleBase(m Module) { m.base().module = m } +// InitAndroidModule initializes the Module as an Android module that is not architecture-specific. +// It adds the common properties, for example "name" and "enabled". func InitAndroidModule(m Module) { initAndroidModuleBase(m) base := m.base() @@ -838,6 +848,12 @@ func InitAndroidModule(m Module) { setPrimaryVisibilityProperty(m, "visibility", &base.commonProperties.Visibility) } +// InitAndroidArchModule initializes the Module as an Android module that is architecture-specific. +// It adds the common properties, for example "name" and "enabled", as well as runtime generated +// property structs for architecture-specific versions of generic properties tagged with +// `android:"arch_variant"`. +// +// InitAndroidModule should not be called if InitAndroidArchModule was called. func InitAndroidArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib Multilib) { InitAndroidModule(m) @@ -847,21 +863,37 @@ func InitAndroidArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib base.commonProperties.ArchSpecific = true base.commonProperties.UseTargetVariants = true - switch hod { - case HostAndDeviceSupported, HostAndDeviceDefault: + if hod&hostSupported != 0 && hod&deviceSupported != 0 { m.AddProperties(&base.hostAndDeviceProperties) } - InitArchModule(m) + initArchModule(m) } +// InitAndroidMultiTargetsArchModule initializes the Module as an Android module that is +// architecture-specific, but will only have a single variant per OS that handles all the +// architectures simultaneously. The list of Targets that it must handle will be available from +// ModuleContext.MultiTargets. It adds the common properties, for example "name" and "enabled", as +// well as runtime generated property structs for architecture-specific versions of generic +// properties tagged with `android:"arch_variant"`. +// +// InitAndroidModule or InitAndroidArchModule should not be called if +// InitAndroidMultiTargetsArchModule was called. func InitAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib Multilib) { InitAndroidArchModule(m, hod, defaultMultilib) m.base().commonProperties.UseTargetVariants = false } -// As InitAndroidMultiTargetsArchModule except it creates an additional CommonOS variant that -// has dependencies on all the OsType specific variants. +// InitCommonOSAndroidMultiTargetsArchModule initializes the Module as an Android module that is +// architecture-specific, but will only have a single variant per OS that handles all the +// architectures simultaneously, and will also have an additional CommonOS variant that has +// dependencies on all the OS-specific variants. The list of Targets that it must handle will be +// available from ModuleContext.MultiTargets. It adds the common properties, for example "name" and +// "enabled", as well as runtime generated property structs for architecture-specific versions of +// generic properties tagged with `android:"arch_variant"`. +// +// InitAndroidModule, InitAndroidArchModule or InitAndroidMultiTargetsArchModule should not be +// called if InitCommonOSAndroidMultiTargetsArchModule was called. func InitCommonOSAndroidMultiTargetsArchModule(m Module, hod HostOrDeviceSupported, defaultMultilib Multilib) { InitAndroidArchModule(m, hod, defaultMultilib) m.base().commonProperties.UseTargetVariants = false @@ -934,6 +966,7 @@ type ModuleBase struct { noAddressSanitizer bool installFiles InstallPaths checkbuildFiles Paths + packagingSpecs []PackagingSpec noticeFiles Paths phonies map[string]Paths @@ -1098,43 +1131,54 @@ func (m *ModuleBase) IsCommonOSVariant() bool { return m.commonProperties.CommonOSVariant } -func (m *ModuleBase) supportsTarget(target Target, config Config) bool { - switch m.commonProperties.HostOrDeviceSupported { - case HostSupported: - return target.Os.Class == Host - case HostSupportedNoCross: - return target.Os.Class == Host && !target.HostCross - case DeviceSupported: - return target.Os.Class == Device - case HostAndDeviceSupported, HostAndDeviceDefault: - supported := false - if Bool(m.hostAndDeviceProperties.Host_supported) || - (m.commonProperties.HostOrDeviceSupported == HostAndDeviceDefault && - m.hostAndDeviceProperties.Host_supported == nil) { - supported = supported || target.Os.Class == Host - } - if m.hostAndDeviceProperties.Device_supported == nil || - *m.hostAndDeviceProperties.Device_supported { - supported = supported || target.Os.Class == Device +// supportsTarget returns true if the given Target is supported by the current module. +func (m *ModuleBase) supportsTarget(target Target) bool { + switch target.Os.Class { + case Host: + if target.HostCross { + return m.HostCrossSupported() + } else { + return m.HostSupported() } - return supported + case Device: + return m.DeviceSupported() default: return false } } +// DeviceSupported returns true if the current module is supported and enabled for device targets, +// i.e. the factory method set the HostOrDeviceSupported value to include device support and +// the device support is enabled by default or enabled by the device_supported property. func (m *ModuleBase) DeviceSupported() bool { - return m.commonProperties.HostOrDeviceSupported == DeviceSupported || - m.commonProperties.HostOrDeviceSupported == HostAndDeviceSupported && - (m.hostAndDeviceProperties.Device_supported == nil || - *m.hostAndDeviceProperties.Device_supported) + hod := m.commonProperties.HostOrDeviceSupported + // deviceEnabled is true if the device_supported property is true or the HostOrDeviceSupported + // value has the deviceDefault bit set. + deviceEnabled := proptools.BoolDefault(m.hostAndDeviceProperties.Device_supported, hod&deviceDefault != 0) + return hod&deviceSupported != 0 && deviceEnabled } +// HostSupported returns true if the current module is supported and enabled for host targets, +// i.e. the factory method set the HostOrDeviceSupported value to include host support and +// the host support is enabled by default or enabled by the host_supported property. func (m *ModuleBase) HostSupported() bool { - return m.commonProperties.HostOrDeviceSupported == HostSupported || - m.commonProperties.HostOrDeviceSupported == HostAndDeviceSupported && - (m.hostAndDeviceProperties.Host_supported != nil && - *m.hostAndDeviceProperties.Host_supported) + hod := m.commonProperties.HostOrDeviceSupported + // hostEnabled is true if the host_supported property is true or the HostOrDeviceSupported + // value has the hostDefault bit set. + hostEnabled := proptools.BoolDefault(m.hostAndDeviceProperties.Host_supported, hod&hostDefault != 0) + return hod&hostSupported != 0 && hostEnabled +} + +// HostCrossSupported returns true if the current module is supported and enabled for host cross +// targets, i.e. the factory method set the HostOrDeviceSupported value to include host cross +// support and the host cross support is enabled by default or enabled by the +// host_supported property. +func (m *ModuleBase) HostCrossSupported() bool { + hod := m.commonProperties.HostOrDeviceSupported + // hostEnabled is true if the host_supported property is true or the HostOrDeviceSupported + // value has the hostDefault bit set. + hostEnabled := proptools.BoolDefault(m.hostAndDeviceProperties.Host_supported, hod&hostDefault != 0) + return hod&hostCrossSupported != 0 && hostEnabled } func (m *ModuleBase) Platform() bool { @@ -1259,6 +1303,10 @@ func (m *ModuleBase) FilesToInstall() InstallPaths { return m.installFiles } +func (m *ModuleBase) PackagingSpecs() []PackagingSpec { + return m.packagingSpecs +} + func (m *ModuleBase) NoAddressSanitizer() bool { return m.noAddressSanitizer } @@ -1581,6 +1629,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.installFiles = append(m.installFiles, ctx.installFiles...) m.checkbuildFiles = append(m.checkbuildFiles, ctx.checkbuildFiles...) + m.packagingSpecs = append(m.packagingSpecs, ctx.packagingSpecs...) m.initRcPaths = PathsForModuleSrc(ctx, m.commonProperties.Init_rc) m.vintfFragmentsPaths = PathsForModuleSrc(ctx, m.commonProperties.Vintf_fragments) for k, v := range ctx.phonies { @@ -1748,6 +1797,7 @@ func (b *baseModuleContext) blueprintBaseModuleContext() blueprint.BaseModuleCon type moduleContext struct { bp blueprint.ModuleContext baseModuleContext + packagingSpecs []PackagingSpec installDeps InstallPaths installFiles InstallPaths checkbuildFiles Paths @@ -2284,16 +2334,15 @@ func (m *moduleContext) skipInstall(fullInstallPath InstallPath) bool { func (m *moduleContext) InstallFile(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath { - return m.installFile(installPath, name, srcPath, Cp, deps) + return m.installFile(installPath, name, srcPath, deps, false) } func (m *moduleContext) InstallExecutable(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath { - return m.installFile(installPath, name, srcPath, CpExecutable, deps) + return m.installFile(installPath, name, srcPath, deps, true) } -func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, - rule blueprint.Rule, deps []Path) InstallPath { +func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, deps []Path, executable bool) InstallPath { fullInstallPath := installPath.Join(m, name) m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, false) @@ -2312,6 +2361,11 @@ func (m *moduleContext) installFile(installPath InstallPath, name string, srcPat orderOnlyDeps = deps } + rule := Cp + if executable { + rule = CpExecutable + } + m.Build(pctx, BuildParams{ Rule: rule, Description: "install " + fullInstallPath.Base(), @@ -2324,6 +2378,14 @@ func (m *moduleContext) installFile(installPath InstallPath, name string, srcPat m.installFiles = append(m.installFiles, fullInstallPath) } + + m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ + relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), + srcPath: srcPath, + symlinkTarget: "", + executable: executable, + }) + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) return fullInstallPath } @@ -2332,12 +2394,12 @@ func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, src fullInstallPath := installPath.Join(m, name) m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, true) + relPath, err := filepath.Rel(path.Dir(fullInstallPath.String()), srcPath.String()) + if err != nil { + panic(fmt.Sprintf("Unable to generate symlink between %q and %q: %s", fullInstallPath.Base(), srcPath.Base(), err)) + } if !m.skipInstall(fullInstallPath) { - relPath, err := filepath.Rel(path.Dir(fullInstallPath.String()), srcPath.String()) - if err != nil { - panic(fmt.Sprintf("Unable to generate symlink between %q and %q: %s", fullInstallPath.Base(), srcPath.Base(), err)) - } m.Build(pctx, BuildParams{ Rule: Symlink, Description: "install symlink " + fullInstallPath.Base(), @@ -2352,6 +2414,14 @@ func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, src m.installFiles = append(m.installFiles, fullInstallPath) m.checkbuildFiles = append(m.checkbuildFiles, srcPath) } + + m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ + relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), + srcPath: nil, + symlinkTarget: relPath, + executable: false, + }) + return fullInstallPath } @@ -2374,6 +2444,14 @@ func (m *moduleContext) InstallAbsoluteSymlink(installPath InstallPath, name str m.installFiles = append(m.installFiles, fullInstallPath) } + + m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ + relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), + srcPath: nil, + symlinkTarget: absPath, + executable: false, + }) + return fullInstallPath } diff --git a/android/ninja_deps.go b/android/ninja_deps.go new file mode 100644 index 000000000..2f442d5f0 --- /dev/null +++ b/android/ninja_deps.go @@ -0,0 +1,43 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import "sort" + +func (c *config) addNinjaFileDeps(deps ...string) { + for _, dep := range deps { + c.ninjaFileDepsSet.Store(dep, true) + } +} + +func (c *config) ninjaFileDeps() []string { + var deps []string + c.ninjaFileDepsSet.Range(func(key, value interface{}) bool { + deps = append(deps, key.(string)) + return true + }) + sort.Strings(deps) + return deps +} + +func ninjaDepsSingletonFactory() Singleton { + return &ninjaDepsSingleton{} +} + +type ninjaDepsSingleton struct{} + +func (ninjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) { + ctx.AddNinjaFileDeps(ctx.Config().ninjaFileDeps()...) +} diff --git a/android/ninja_deps_test.go b/android/ninja_deps_test.go new file mode 100644 index 000000000..d3775edd9 --- /dev/null +++ b/android/ninja_deps_test.go @@ -0,0 +1,75 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "testing" +) + +func init() { + // This variable uses ExistentPathForSource on a PackageVarContext, which is a PathContext + // that is not a PathGlobContext. That requires the deps to be stored in the Config. + pctx.VariableFunc("test_ninja_deps_variable", func(ctx PackageVarContext) string { + // Using ExistentPathForSource to look for a file that does not exist in a directory that + // does exist (test_ninja_deps) from a PackageVarContext adds a dependency from build.ninja + // to the directory. + if ExistentPathForSource(ctx, "test_ninja_deps/does_not_exist").Valid() { + return "true" + } else { + return "false" + } + }) +} + +func testNinjaDepsSingletonFactory() Singleton { + return testNinjaDepsSingleton{} +} + +type testNinjaDepsSingleton struct{} + +func (testNinjaDepsSingleton) GenerateBuildActions(ctx SingletonContext) { + // Reference the test_ninja_deps_variable in a build statement so Blueprint is forced to + // evaluate it. + ctx.Build(pctx, BuildParams{ + Rule: Cp, + Input: PathForTesting("foo"), + Output: PathForOutput(ctx, "test_ninja_deps_out"), + Args: map[string]string{ + "cpFlags": "${test_ninja_deps_variable}", + }, + }) +} + +func TestNinjaDeps(t *testing.T) { + fs := map[string][]byte{ + "test_ninja_deps/exists": nil, + } + config := TestConfig(buildDir, nil, "", fs) + + ctx := NewTestContext(config) + ctx.RegisterSingletonType("test_ninja_deps_singleton", testNinjaDepsSingletonFactory) + ctx.RegisterSingletonType("ninja_deps_singleton", ninjaDepsSingletonFactory) + ctx.Register() + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + ninjaDeps, errs := ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + // Verify that the ninja file has a dependency on the test_ninja_deps directory. + if g, w := ninjaDeps, "test_ninja_deps"; !InList(w, g) { + t.Errorf("expected %q in %q", w, g) + } +} diff --git a/android/package_ctx.go b/android/package_ctx.go index 0de356e1c..6d0fcb33f 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -56,7 +56,7 @@ func (e *configErrorWrapper) Errorf(format string, args ...interface{}) { e.errors = append(e.errors, fmt.Errorf(format, args...)) } func (e *configErrorWrapper) AddNinjaFileDeps(deps ...string) { - e.pctx.AddNinjaFileDeps(deps...) + e.config.addNinjaFileDeps(deps...) } type PackageVarContext interface { diff --git a/android/packaging.go b/android/packaging.go new file mode 100644 index 000000000..512e4ba53 --- /dev/null +++ b/android/packaging.go @@ -0,0 +1,205 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "path/filepath" + + "github.com/google/blueprint" +) + +// PackagingSpec abstracts a request to place a built artifact at a certain path in a package. +// A package can be the traditional <partition>.img, but isn't limited to those. Other examples could +// be a new filesystem image that is a subset of system.img (e.g. for an Android-like mini OS running +// on a VM), or a zip archive for some of the host tools. +type PackagingSpec struct { + // Path relative to the root of the package + relPathInPackage string + + // The path to the built artifact + srcPath Path + + // If this is not empty, then relPathInPackage should be a symlink to this target. (Then + // srcPath is of course ignored.) + symlinkTarget string + + // Whether relPathInPackage should be marked as executable or not + executable bool +} + +type PackageModule interface { + Module + packagingBase() *PackagingBase + + // AddDeps adds dependencies to the `deps` modules. This should be called in DepsMutator. + AddDeps(ctx BottomUpMutatorContext) + + // CopyDepsToZip zips the built artifacts of the dependencies into the given zip file and + // returns zip entries in it. This is expected to be called in GenerateAndroidBuildActions, + // followed by a build rule that unzips it and creates the final output (img, zip, tar.gz, + // etc.) from the extracted files + CopyDepsToZip(ctx ModuleContext, zipOut OutputPath) []string +} + +// PackagingBase provides basic functionality for packaging dependencies. A module is expected to +// include this struct and call InitPackageModule. +type PackagingBase struct { + properties PackagingProperties + + // Allows this module to skip missing dependencies. In most cases, this + // is not required, but for rare cases like when there's a dependency + // to a module which exists in certain repo checkouts, this is needed. + IgnoreMissingDependencies bool +} + +type depsProperty struct { + // Modules to include in this package + Deps []string `android:"arch_variant"` +} + +type packagingMultilibProperties struct { + First depsProperty `android:"arch_variant"` + Common depsProperty `android:"arch_variant"` + Lib32 depsProperty `android:"arch_variant"` + Lib64 depsProperty `android:"arch_variant"` +} + +type PackagingProperties struct { + Deps []string `android:"arch_variant"` + Multilib packagingMultilibProperties `android:"arch_variant"` +} + +type packagingDependencyTag struct{ blueprint.BaseDependencyTag } + +var depTag = packagingDependencyTag{} + +func InitPackageModule(p PackageModule) { + base := p.packagingBase() + p.AddProperties(&base.properties) +} + +func (p *PackagingBase) packagingBase() *PackagingBase { + return p +} + +// From deps and multilib.*.deps, select the dependencies that are for the given arch +// deps is for the current archicture when this module is not configured for multi target. +// When configured for multi target, deps is selected for each of the targets and is NOT +// selected for the current architecture which would be Common. +func (p *PackagingBase) getDepsForArch(ctx BaseModuleContext, arch ArchType) []string { + var ret []string + if arch == ctx.Target().Arch.ArchType && len(ctx.MultiTargets()) == 0 { + ret = append(ret, p.properties.Deps...) + } else if arch.Multilib == "lib32" { + ret = append(ret, p.properties.Multilib.Lib32.Deps...) + } else if arch.Multilib == "lib64" { + ret = append(ret, p.properties.Multilib.Lib64.Deps...) + } else if arch == Common { + ret = append(ret, p.properties.Multilib.Common.Deps...) + } + for i, t := range ctx.MultiTargets() { + if t.Arch.ArchType == arch { + ret = append(ret, p.properties.Deps...) + if i == 0 { + ret = append(ret, p.properties.Multilib.First.Deps...) + } + } + } + return FirstUniqueStrings(ret) +} + +func (p *PackagingBase) getSupportedTargets(ctx BaseModuleContext) []Target { + var ret []Target + // The current and the common OS targets are always supported + ret = append(ret, ctx.Target()) + if ctx.Arch().ArchType != Common { + ret = append(ret, Target{Os: ctx.Os(), Arch: Arch{ArchType: Common}}) + } + // If this module is configured for multi targets, those should be supported as well + ret = append(ret, ctx.MultiTargets()...) + return ret +} + +// See PackageModule.AddDeps +func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext) { + for _, t := range p.getSupportedTargets(ctx) { + for _, dep := range p.getDepsForArch(ctx, t.Arch.ArchType) { + if p.IgnoreMissingDependencies && !ctx.OtherModuleExists(dep) { + continue + } + ctx.AddFarVariationDependencies(t.Variations(), depTag, dep) + } + } +} + +// See PackageModule.CopyDepsToZip +func (p *PackagingBase) CopyDepsToZip(ctx ModuleContext, zipOut OutputPath) (entries []string) { + var supportedArches []string + for _, t := range p.getSupportedTargets(ctx) { + supportedArches = append(supportedArches, t.Arch.ArchType.String()) + } + m := make(map[string]PackagingSpec) + ctx.WalkDeps(func(child Module, parent Module) bool { + // Don't track modules with unsupported arch + // TODO(jiyong): remove this when aosp/1501613 lands. + if !InList(child.Target().Arch.ArchType.String(), supportedArches) { + return false + } + for _, ps := range child.PackagingSpecs() { + if _, ok := m[ps.relPathInPackage]; !ok { + m[ps.relPathInPackage] = ps + } + } + return true + }) + + builder := NewRuleBuilder() + + dir := PathForModuleOut(ctx, ".zip").OutputPath + builder.Command().Text("rm").Flag("-rf").Text(dir.String()) + builder.Command().Text("mkdir").Flag("-p").Text(dir.String()) + + seenDir := make(map[string]bool) + for _, k := range SortedStringKeys(m) { + ps := m[k] + destPath := dir.Join(ctx, ps.relPathInPackage).String() + destDir := filepath.Dir(destPath) + entries = append(entries, ps.relPathInPackage) + if _, ok := seenDir[destDir]; !ok { + seenDir[destDir] = true + builder.Command().Text("mkdir").Flag("-p").Text(destDir) + } + if ps.symlinkTarget == "" { + builder.Command().Text("cp").Input(ps.srcPath).Text(destPath) + } else { + builder.Command().Text("ln").Flag("-sf").Text(ps.symlinkTarget).Text(destPath) + } + if ps.executable { + builder.Command().Text("chmod").Flag("a+x").Text(destPath) + } + } + + builder.Command(). + BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", zipOut). + FlagWithArg("-C ", dir.String()). + Flag("-L 0"). // no compression because this will be unzipped soon + FlagWithArg("-D ", dir.String()) + builder.Command().Text("rm").Flag("-rf").Text(dir.String()) + + builder.Build(pctx, ctx, "zip_deps", fmt.Sprintf("Zipping deps for %s", ctx.ModuleName())) + return entries +} diff --git a/android/packaging_test.go b/android/packaging_test.go new file mode 100644 index 000000000..7710c7fd8 --- /dev/null +++ b/android/packaging_test.go @@ -0,0 +1,188 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "reflect" + "testing" +) + +// Module to be packaged +type componentTestModule struct { + ModuleBase + props struct { + Deps []string + } +} + +func componentTestModuleFactory() Module { + m := &componentTestModule{} + m.AddProperties(&m.props) + InitAndroidArchModule(m, HostAndDeviceSupported, MultilibBoth) + return m +} + +func (m *componentTestModule) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), nil, m.props.Deps...) +} + +func (m *componentTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + builtFile := PathForModuleOut(ctx, m.Name()) + dir := ctx.Target().Arch.ArchType.Multilib + installDir := PathForModuleInstall(ctx, dir) + ctx.InstallFile(installDir, m.Name(), builtFile) +} + +// Module that itself is a package +type packageTestModule struct { + ModuleBase + PackagingBase + + entries []string +} + +func packageTestModuleFactory() Module { + module := &packageTestModule{} + InitPackageModule(module) + InitAndroidMultiTargetsArchModule(module, DeviceSupported, MultilibCommon) + return module +} + +func (m *packageTestModule) DepsMutator(ctx BottomUpMutatorContext) { + m.AddDeps(ctx) +} + +func (m *packageTestModule) GenerateAndroidBuildActions(ctx ModuleContext) { + zipFile := PathForModuleOut(ctx, "myzip.zip").OutputPath + m.entries = m.CopyDepsToZip(ctx, zipFile) +} + +func runPackagingTest(t *testing.T, bp string, expected []string) { + t.Helper() + + config := TestArchConfig(buildDir, nil, bp, nil) + + ctx := NewTestArchContext(config) + ctx.RegisterModuleType("component", componentTestModuleFactory) + ctx.RegisterModuleType("package_module", packageTestModuleFactory) + ctx.Register() + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + p := ctx.ModuleForTests("package", "android_common").Module().(*packageTestModule) + actual := p.entries + actual = SortedUniqueStrings(actual) + expected = SortedUniqueStrings(expected) + if !reflect.DeepEqual(actual, expected) { + t.Errorf("\ngot: %v\nexpected: %v\n", actual, expected) + } +} + +func TestPackagingBase(t *testing.T) { + runPackagingTest(t, + ` + component { + name: "foo", + } + + package_module { + name: "package", + deps: ["foo"], + } + `, []string{"lib64/foo"}) + + runPackagingTest(t, + ` + component { + name: "foo", + deps: ["bar"], + } + + component { + name: "bar", + } + + package_module { + name: "package", + deps: ["foo"], + } + `, []string{"lib64/foo", "lib64/bar"}) + + runPackagingTest(t, + ` + component { + name: "foo", + deps: ["bar"], + } + + component { + name: "bar", + } + + package_module { + name: "package", + deps: ["foo"], + compile_multilib: "both", + } + `, []string{"lib32/foo", "lib32/bar", "lib64/foo", "lib64/bar"}) + + runPackagingTest(t, + ` + component { + name: "foo", + } + + component { + name: "bar", + compile_multilib: "32", + } + + package_module { + name: "package", + deps: ["foo"], + multilib: { + lib32: { + deps: ["bar"], + }, + }, + compile_multilib: "both", + } + `, []string{"lib32/foo", "lib32/bar", "lib64/foo"}) + + runPackagingTest(t, + ` + component { + name: "foo", + } + + component { + name: "bar", + } + + package_module { + name: "package", + deps: ["foo"], + multilib: { + first: { + deps: ["bar"], + }, + }, + compile_multilib: "both", + } + `, []string{"lib32/foo", "lib64/foo", "lib64/bar"}) +} diff --git a/android/register.go b/android/register.go index bd824c9c1..08e47b330 100644 --- a/android/register.go +++ b/android/register.go @@ -29,7 +29,7 @@ var moduleTypes []moduleType type singleton struct { name string - factory blueprint.SingletonFactory + factory SingletonFactory } var singletons []singleton @@ -57,11 +57,11 @@ type SingletonFactory func() Singleton // SingletonFactoryAdaptor wraps a SingletonFactory into a blueprint.SingletonFactory by converting // a Singleton into a blueprint.Singleton -func SingletonFactoryAdaptor(factory SingletonFactory) blueprint.SingletonFactory { +func SingletonFactoryAdaptor(ctx *Context, factory SingletonFactory) blueprint.SingletonFactory { return func() blueprint.Singleton { singleton := factory() if makevars, ok := singleton.(SingletonMakeVarsProvider); ok { - registerSingletonMakeVarsProvider(makevars) + registerSingletonMakeVarsProvider(ctx.config, makevars) } return &singletonAdaptor{Singleton: singleton} } @@ -72,11 +72,11 @@ func RegisterModuleType(name string, factory ModuleFactory) { } func RegisterSingletonType(name string, factory SingletonFactory) { - singletons = append(singletons, singleton{name, SingletonFactoryAdaptor(factory)}) + singletons = append(singletons, singleton{name, factory}) } func RegisterPreSingletonType(name string, factory SingletonFactory) { - preSingletons = append(preSingletons, singleton{name, SingletonFactoryAdaptor(factory)}) + preSingletons = append(preSingletons, singleton{name, factory}) } type Context struct { @@ -92,7 +92,7 @@ func NewContext(config Config) *Context { func (ctx *Context) Register() { for _, t := range preSingletons { - ctx.RegisterPreSingletonType(t.name, t.factory) + ctx.RegisterPreSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) } for _, t := range moduleTypes { @@ -100,21 +100,23 @@ func (ctx *Context) Register() { } for _, t := range singletons { - ctx.RegisterSingletonType(t.name, t.factory) + ctx.RegisterSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) } registerMutators(ctx.Context, preArch, preDeps, postDeps, finalDeps) - ctx.RegisterSingletonType("bazeldeps", SingletonFactoryAdaptor(BazelSingleton)) + ctx.RegisterSingletonType("bazeldeps", SingletonFactoryAdaptor(ctx, BazelSingleton)) // Register phony just before makevars so it can write out its phony rules as Make rules - ctx.RegisterSingletonType("phony", SingletonFactoryAdaptor(phonySingletonFactory)) + ctx.RegisterSingletonType("phony", SingletonFactoryAdaptor(ctx, phonySingletonFactory)) // Register makevars after other singletons so they can export values through makevars - ctx.RegisterSingletonType("makevars", SingletonFactoryAdaptor(makeVarsSingletonFunc)) + ctx.RegisterSingletonType("makevars", SingletonFactoryAdaptor(ctx, makeVarsSingletonFunc)) - // Register env last so that it can track all used environment variables - ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(EnvSingleton)) + // Register env and ninjadeps last so that they can track all used environment variables and + // Ninja file dependencies stored in the config. + ctx.RegisterSingletonType("env", SingletonFactoryAdaptor(ctx, EnvSingleton)) + ctx.RegisterSingletonType("ninjadeps", SingletonFactoryAdaptor(ctx, ninjaDepsSingletonFactory)) } func ModuleTypeFactories() map[string]ModuleFactory { diff --git a/android/rule_builder.go b/android/rule_builder.go index 8dc9d6a42..86418b2d5 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -15,7 +15,9 @@ package android import ( + "crypto/sha256" "fmt" + "path/filepath" "sort" "strings" @@ -25,6 +27,8 @@ import ( "android/soong/shared" ) +const sboxOutDir = "__SBOX_OUT_DIR__" + // RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build // graph. type RuleBuilder struct { @@ -133,8 +137,8 @@ func (r *RuleBuilder) Install(from Path, to string) { // race with any call to Build. func (r *RuleBuilder) Command() *RuleBuilderCommand { command := &RuleBuilderCommand{ - sbox: r.sbox, - sboxOutDir: r.sboxOutDir, + sbox: r.sbox, + outDir: r.sboxOutDir, } r.commands = append(r.commands, command) return command @@ -163,7 +167,7 @@ func (r *RuleBuilder) DeleteTemporaryFiles() { } // Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take -// input paths, such as RuleBuilderCommand.Input, RuleBuilderComand.Implicit, or +// input paths, such as RuleBuilderCommand.Input, RuleBuilderCommand.Implicit, or // RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command // in the same RuleBuilder are filtered out. The list is sorted and duplicates removed. func (r *RuleBuilder) Inputs() Paths { @@ -362,7 +366,7 @@ func (r *RuleBuilder) Commands() []string { return commands } -// NinjaEscapedCommands returns a slice containin the built command line after ninja escaping for each call to +// NinjaEscapedCommands returns a slice containing the built command line after ninja escaping for each call to // RuleBuilder.Command. func (r *RuleBuilder) NinjaEscapedCommands() []string { var commands []string @@ -427,6 +431,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string tools := r.Tools() commands := r.NinjaEscapedCommands() outputs := r.Outputs() + inputs := r.Inputs() if len(commands) == 0 { return @@ -440,7 +445,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string if r.sbox { sboxOutputs := make([]string, len(outputs)) for i, output := range outputs { - sboxOutputs[i] = "__SBOX_OUT_DIR__/" + Rel(ctx, r.sboxOutDir.String(), output.String()) + sboxOutputs[i] = filepath.Join(sboxOutDir, Rel(ctx, r.sboxOutDir.String(), output.String())) } commandString = proptools.ShellEscape(commandString) @@ -458,10 +463,19 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string sboxCmd.Flag("--depfile-out").Text(depFile.String()) } + // Add a hash of the list of input files to the xbox command line so that ninja reruns + // it when the list of input files changes. + sboxCmd.FlagWithArg("--input-hash ", hashSrcFiles(inputs)) + sboxCmd.Flags(sboxOutputs) commandString = sboxCmd.buf.String() tools = append(tools, sboxCmd.tools...) + } else { + // If not using sbox the rule will run the command directly, put the hash of the + // list of input files in a comment at the end of the command line to ensure ninja + // reruns the rule when the list of input files changes. + commandString += " # hash of input list: " + hashSrcFiles(inputs) } // Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to @@ -499,7 +513,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string Pool: pool, }), Inputs: rspFileInputs, - Implicits: r.Inputs(), + Implicits: inputs, Output: output, ImplicitOutputs: implicitOutputs, SymlinkOutputs: r.SymlinkOutputs(), @@ -527,14 +541,16 @@ type RuleBuilderCommand struct { // spans [start,end) of the command that should not be ninja escaped unescapedSpans [][2]int - sbox bool - sboxOutDir WritablePath + sbox bool + // outDir is the directory that will contain the output files of the rules. sbox will copy + // the output files from the sandbox directory to this directory when it finishes. + outDir WritablePath } func (c *RuleBuilderCommand) addInput(path Path) string { if c.sbox { - if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel { - return "__SBOX_OUT_DIR__/" + rel + if rel, isRel, _ := maybeRelErr(c.outDir.String(), path.String()); isRel { + return filepath.Join(sboxOutDir, rel) } } c.inputs = append(c.inputs, path) @@ -543,8 +559,8 @@ func (c *RuleBuilderCommand) addInput(path Path) string { func (c *RuleBuilderCommand) addImplicit(path Path) string { if c.sbox { - if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel { - return "__SBOX_OUT_DIR__/" + rel + if rel, isRel, _ := maybeRelErr(c.outDir.String(), path.String()); isRel { + return filepath.Join(sboxOutDir, rel) } } c.implicits = append(c.implicits, path) @@ -555,15 +571,22 @@ func (c *RuleBuilderCommand) addOrderOnly(path Path) { c.orderOnlys = append(c.orderOnlys, path) } -func (c *RuleBuilderCommand) outputStr(path Path) string { +func (c *RuleBuilderCommand) outputStr(path WritablePath) string { if c.sbox { - // Errors will be handled in RuleBuilder.Build where we have a context to report them - rel, _, _ := maybeRelErr(c.sboxOutDir.String(), path.String()) - return "__SBOX_OUT_DIR__/" + rel + return SboxPathForOutput(path, c.outDir) } return path.String() } +// SboxPathForOutput takes an output path and the out directory passed to RuleBuilder.Sbox(), +// and returns the corresponding path for the output in the sbox sandbox. This can be used +// on the RuleBuilder command line to reference the output. +func SboxPathForOutput(path WritablePath, outDir WritablePath) string { + // Errors will be handled in RuleBuilder.Build where we have a context to report them + rel, _, _ := maybeRelErr(outDir.String(), path.String()) + return filepath.Join(sboxOutDir, rel) +} + // Text adds the specified raw text to the command line. The text should not contain input or output paths or the // rule will not have them listed in its dependencies or outputs. func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand { @@ -727,7 +750,7 @@ func (c *RuleBuilderCommand) OutputDir() *RuleBuilderCommand { if !c.sbox { panic("OutputDir only valid with Sbox") } - return c.Text("__SBOX_OUT_DIR__") + return c.Text(sboxOutDir) } // DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command @@ -906,3 +929,12 @@ func ninjaNameEscape(s string) string { } return s } + +// hashSrcFiles returns a hash of the list of source files. It is used to ensure the command line +// or the sbox textproto manifest change even if the input files are not listed on the command line. +func hashSrcFiles(srcFiles Paths) string { + h := sha256.New() + srcFileList := strings.Join(srcFiles.Strings(), "\n") + h.Write([]byte(srcFileList)) + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index ca6359d67..c1d552198 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -18,6 +18,7 @@ import ( "fmt" "path/filepath" "reflect" + "regexp" "strings" "testing" @@ -441,7 +442,7 @@ func testRuleBuilderFactory() Module { type testRuleBuilderModule struct { ModuleBase properties struct { - Src string + Srcs []string Restat bool Sbox bool @@ -449,7 +450,7 @@ type testRuleBuilderModule struct { } func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) { - in := PathForSource(ctx, t.properties.Src) + in := PathsForSource(ctx, t.properties.Srcs) out := PathForModuleOut(ctx, ctx.ModuleName()) outDep := PathForModuleOut(ctx, ctx.ModuleName()+".d") outDir := PathForModuleOut(ctx) @@ -468,17 +469,17 @@ func (t *testRuleBuilderSingleton) GenerateBuildActions(ctx SingletonContext) { out := PathForOutput(ctx, "baz") outDep := PathForOutput(ctx, "baz.d") outDir := PathForOutput(ctx) - testRuleBuilder_Build(ctx, in, out, outDep, outDir, true, false) + testRuleBuilder_Build(ctx, Paths{in}, out, outDep, outDir, true, false) } -func testRuleBuilder_Build(ctx BuilderContext, in Path, out, outDep, outDir WritablePath, restat, sbox bool) { +func testRuleBuilder_Build(ctx BuilderContext, in Paths, out, outDep, outDir WritablePath, restat, sbox bool) { rule := NewRuleBuilder() if sbox { rule.Sbox(outDir) } - rule.Command().Tool(PathForSource(ctx, "cp")).Input(in).Output(out).ImplicitDepFile(outDep) + rule.Command().Tool(PathForSource(ctx, "cp")).Inputs(in).Output(out).ImplicitDepFile(outDep) if restat { rule.Restat() @@ -496,12 +497,12 @@ func TestRuleBuilder_Build(t *testing.T) { bp := ` rule_builder_test { name: "foo", - src: "bar", + srcs: ["bar"], restat: true, } rule_builder_test { name: "foo_sbox", - src: "bar", + srcs: ["bar"], sbox: true, } ` @@ -519,7 +520,10 @@ func TestRuleBuilder_Build(t *testing.T) { check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraCmdDeps []string) { t.Helper() - if params.RuleParams.Command != wantCommand { + command := params.RuleParams.Command + re := regexp.MustCompile(" (# hash of input list:|--input-hash) [a-z0-9]*") + command = re.ReplaceAllLiteralString(command, "") + if command != wantCommand { t.Errorf("\nwant RuleParams.Command = %q\n got %q", wantCommand, params.RuleParams.Command) } @@ -651,3 +655,78 @@ func Test_ninjaEscapeExceptForSpans(t *testing.T) { }) } } + +func TestRuleBuilderHashInputs(t *testing.T) { + // The basic idea here is to verify that the command (in the case of a + // non-sbox rule) or the sbox textproto manifest contain a hash of the + // inputs. + + // By including a hash of the inputs, we cause the rule to re-run if + // the list of inputs changes because the command line or a dependency + // changes. + + bp := ` + rule_builder_test { + name: "hash0", + srcs: ["in1.txt", "in2.txt"], + } + rule_builder_test { + name: "hash0_sbox", + srcs: ["in1.txt", "in2.txt"], + sbox: true, + } + rule_builder_test { + name: "hash1", + srcs: ["in1.txt", "in2.txt", "in3.txt"], + } + rule_builder_test { + name: "hash1_sbox", + srcs: ["in1.txt", "in2.txt", "in3.txt"], + sbox: true, + } + ` + testcases := []struct { + name string + expectedHash string + }{ + { + name: "hash0", + // sha256 value obtained from: echo -en 'in1.txt\nin2.txt' | sha256sum + expectedHash: "18da75b9b1cc74b09e365b4ca2e321b5d618f438cc632b387ad9dc2ab4b20e9d", + }, + { + name: "hash1", + // sha256 value obtained from: echo -en 'in1.txt\nin2.txt\nin3.txt' | sha256sum + expectedHash: "a38d432a4b19df93140e1f1fe26c97ff0387dae01fe506412b47208f0595fb45", + }, + } + + config := TestConfig(buildDir, nil, bp, nil) + ctx := NewTestContext(config) + ctx.RegisterModuleType("rule_builder_test", testRuleBuilderFactory) + ctx.Register() + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + FailIfErrored(t, errs) + + for _, test := range testcases { + t.Run(test.name, func(t *testing.T) { + t.Run("sbox", func(t *testing.T) { + gen := ctx.ModuleForTests(test.name+"_sbox", "") + command := gen.Output(test.name + "_sbox").RuleParams.Command + if g, w := command, " --input-hash "+test.expectedHash; !strings.Contains(g, w) { + t.Errorf("Expected command line to end with %q, got %q", w, g) + } + }) + t.Run("", func(t *testing.T) { + gen := ctx.ModuleForTests(test.name+"", "") + command := gen.Output(test.name).RuleParams.Command + if g, w := command, " # hash of input list: "+test.expectedHash; !strings.HasSuffix(g, w) { + t.Errorf("Expected command line to end with %q, got %q", w, g) + } + }) + }) + } +} diff --git a/android/testing.go b/android/testing.go index d83cecc43..1e2ae130a 100644 --- a/android/testing.go +++ b/android/testing.go @@ -104,7 +104,7 @@ func (ctx *TestContext) RegisterModuleType(name string, factory ModuleFactory) { } func (ctx *TestContext) RegisterSingletonType(name string, factory SingletonFactory) { - ctx.Context.RegisterSingletonType(name, SingletonFactoryAdaptor(factory)) + ctx.Context.RegisterSingletonType(name, SingletonFactoryAdaptor(ctx.Context, factory)) } func (ctx *TestContext) ModuleForTests(name, variant string) TestingModule { diff --git a/androidmk/androidmk/androidmk.go b/androidmk/androidmk/androidmk.go index 03cf74d83..f4e5fa066 100644 --- a/androidmk/androidmk/androidmk.go +++ b/androidmk/androidmk/androidmk.go @@ -137,7 +137,14 @@ func ConvertFile(filename string, buffer *bytes.Buffer) (string, []error) { switch x := node.(type) { case *mkparser.Comment: - file.insertComment("//" + x.Comment) + // Split the comment on escaped newlines and then + // add each chunk separately. + chunks := strings.Split(x.Comment, "\\\n") + file.insertComment("//" + chunks[0]) + for i := 1; i < len(chunks); i++ { + file.bpPos.Line++ + file.insertComment("//" + chunks[i]) + } case *mkparser.Assignment: handleAssignment(file, x, assignmentCond) case *mkparser.Directive: diff --git a/androidmk/androidmk/androidmk_test.go b/androidmk/androidmk/androidmk_test.go index 16cb138c9..f32ff2acb 100644 --- a/androidmk/androidmk/androidmk_test.go +++ b/androidmk/androidmk/androidmk_test.go @@ -1260,10 +1260,10 @@ prebuilt_firmware { desc: "comment with ESC", in: ` # Comment line 1 \ -# Comment line 2 + Comment line 2 `, expected: ` -// Comment line 1 \ +// Comment line 1 // Comment line 2 `, }, diff --git a/androidmk/parser/parser.go b/androidmk/parser/parser.go index 86dabf975..c14910a4f 100644 --- a/androidmk/parser/parser.go +++ b/androidmk/parser/parser.go @@ -212,8 +212,21 @@ func (p *parser) parseDirective() bool { expression := SimpleMakeString("", pos) switch d { - case "endif", "endef", "else": + case "endif", "endef": // Nothing + case "else": + p.ignoreSpaces() + if p.tok != '\n' { + d = p.scanner.TokenText() + p.accept(scanner.Ident) + if d == "ifdef" || d == "ifndef" || d == "ifeq" || d == "ifneq" { + d = "el" + d + p.ignoreSpaces() + expression = p.parseExpression() + } else { + p.errorf("expected ifdef/ifndef/ifeq/ifneq, found %s", d) + } + } case "define": expression, endPos = p.parseDefine() default: @@ -484,12 +497,6 @@ loop: switch p.tok { case '\\': p.parseEscape() - if p.tok == '\n' { - // Special case: '\' does not "escape" newline in comment (b/127521510) - comment += "\\" - p.accept(p.tok) - break loop - } comment += "\\" + p.scanner.TokenText() p.accept(p.tok) case '\n': diff --git a/apex/apex.go b/apex/apex.go index 91770f464..ce323ca4d 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -33,692 +33,6 @@ import ( "android/soong/sh" ) -const ( - imageApexSuffix = ".apex" - zipApexSuffix = ".zipapex" - flattenedSuffix = ".flattened" - - imageApexType = "image" - zipApexType = "zip" - flattenedApexType = "flattened" - - ext4FsType = "ext4" - f2fsFsType = "f2fs" -) - -type dependencyTag struct { - blueprint.BaseDependencyTag - name string - - // determines if the dependent will be part of the APEX payload - payload bool -} - -var ( - sharedLibTag = dependencyTag{name: "sharedLib", payload: true} - jniLibTag = dependencyTag{name: "jniLib", payload: true} - executableTag = dependencyTag{name: "executable", payload: true} - javaLibTag = dependencyTag{name: "javaLib", payload: true} - prebuiltTag = dependencyTag{name: "prebuilt", payload: true} - testTag = dependencyTag{name: "test", payload: true} - keyTag = dependencyTag{name: "key"} - certificateTag = dependencyTag{name: "certificate"} - androidAppTag = dependencyTag{name: "androidApp", payload: true} - rroTag = dependencyTag{name: "rro", payload: true} - bpfTag = dependencyTag{name: "bpf", payload: true} - testForTag = dependencyTag{name: "test for"} - - apexAvailBaseline = makeApexAvailableBaseline() - - inverseApexAvailBaseline = invertApexBaseline(apexAvailBaseline) -) - -// Transform the map of apex -> modules to module -> apexes. -func invertApexBaseline(m map[string][]string) map[string][]string { - r := make(map[string][]string) - for apex, modules := range m { - for _, module := range modules { - r[module] = append(r[module], apex) - } - } - return r -} - -// Retrieve the baseline of apexes to which the supplied module belongs. -func BaselineApexAvailable(moduleName string) []string { - return inverseApexAvailBaseline[normalizeModuleName(moduleName)] -} - -// This is a map from apex to modules, which overrides the -// apex_available setting for that particular module to make -// it available for the apex regardless of its setting. -// TODO(b/147364041): remove this -func makeApexAvailableBaseline() map[string][]string { - // The "Module separator"s below are employed to minimize merge conflicts. - m := make(map[string][]string) - // - // Module separator - // - m["com.android.appsearch"] = []string{ - "icing-java-proto-lite", - "libprotobuf-java-lite", - } - // - // Module separator - // - m["com.android.bluetooth.updatable"] = []string{ - "android.hardware.audio.common@5.0", - "android.hardware.bluetooth.a2dp@1.0", - "android.hardware.bluetooth.audio@2.0", - "android.hardware.bluetooth@1.0", - "android.hardware.bluetooth@1.1", - "android.hardware.graphics.bufferqueue@1.0", - "android.hardware.graphics.bufferqueue@2.0", - "android.hardware.graphics.common@1.0", - "android.hardware.graphics.common@1.1", - "android.hardware.graphics.common@1.2", - "android.hardware.media@1.0", - "android.hidl.safe_union@1.0", - "android.hidl.token@1.0", - "android.hidl.token@1.0-utils", - "avrcp-target-service", - "avrcp_headers", - "bluetooth-protos-lite", - "bluetooth.mapsapi", - "com.android.vcard", - "dnsresolver_aidl_interface-V2-java", - "ipmemorystore-aidl-interfaces-V5-java", - "ipmemorystore-aidl-interfaces-java", - "internal_include_headers", - "lib-bt-packets", - "lib-bt-packets-avrcp", - "lib-bt-packets-base", - "libFraunhoferAAC", - "libaudio-a2dp-hw-utils", - "libaudio-hearing-aid-hw-utils", - "libbinder_headers", - "libbluetooth", - "libbluetooth-types", - "libbluetooth-types-header", - "libbluetooth_gd", - "libbluetooth_headers", - "libbluetooth_jni", - "libbt-audio-hal-interface", - "libbt-bta", - "libbt-common", - "libbt-hci", - "libbt-platform-protos-lite", - "libbt-protos-lite", - "libbt-sbc-decoder", - "libbt-sbc-encoder", - "libbt-stack", - "libbt-utils", - "libbtcore", - "libbtdevice", - "libbte", - "libbtif", - "libchrome", - "libevent", - "libfmq", - "libg722codec", - "libgui_headers", - "libmedia_headers", - "libmodpb64", - "libosi", - "libstagefright_foundation_headers", - "libstagefright_headers", - "libstatslog", - "libstatssocket", - "libtinyxml2", - "libudrv-uipc", - "libz", - "media_plugin_headers", - "net-utils-services-common", - "netd_aidl_interface-unstable-java", - "netd_event_listener_interface-java", - "netlink-client", - "networkstack-client", - "sap-api-java-static", - "services.net", - } - // - // Module separator - // - m["com.android.cellbroadcast"] = []string{"CellBroadcastApp", "CellBroadcastServiceModule"} - // - // Module separator - // - m["com.android.extservices"] = []string{ - "error_prone_annotations", - "ExtServices-core", - "ExtServices", - "libtextclassifier-java", - "libz_current", - "textclassifier-statsd", - "TextClassifierNotificationLibNoManifest", - "TextClassifierServiceLibNoManifest", - } - // - // Module separator - // - m["com.android.neuralnetworks"] = []string{ - "android.hardware.neuralnetworks@1.0", - "android.hardware.neuralnetworks@1.1", - "android.hardware.neuralnetworks@1.2", - "android.hardware.neuralnetworks@1.3", - "android.hidl.allocator@1.0", - "android.hidl.memory.token@1.0", - "android.hidl.memory@1.0", - "android.hidl.safe_union@1.0", - "libarect", - "libbuildversion", - "libmath", - "libprocpartition", - "libsync", - } - // - // Module separator - // - m["com.android.media"] = []string{ - "android.frameworks.bufferhub@1.0", - "android.hardware.cas.native@1.0", - "android.hardware.cas@1.0", - "android.hardware.configstore-utils", - "android.hardware.configstore@1.0", - "android.hardware.configstore@1.1", - "android.hardware.graphics.allocator@2.0", - "android.hardware.graphics.allocator@3.0", - "android.hardware.graphics.bufferqueue@1.0", - "android.hardware.graphics.bufferqueue@2.0", - "android.hardware.graphics.common@1.0", - "android.hardware.graphics.common@1.1", - "android.hardware.graphics.common@1.2", - "android.hardware.graphics.mapper@2.0", - "android.hardware.graphics.mapper@2.1", - "android.hardware.graphics.mapper@3.0", - "android.hardware.media.omx@1.0", - "android.hardware.media@1.0", - "android.hidl.allocator@1.0", - "android.hidl.memory.token@1.0", - "android.hidl.memory@1.0", - "android.hidl.token@1.0", - "android.hidl.token@1.0-utils", - "bionic_libc_platform_headers", - "exoplayer2-extractor", - "exoplayer2-extractor-annotation-stubs", - "gl_headers", - "jsr305", - "libEGL", - "libEGL_blobCache", - "libEGL_getProcAddress", - "libFLAC", - "libFLAC-config", - "libFLAC-headers", - "libGLESv2", - "libaacextractor", - "libamrextractor", - "libarect", - "libaudio_system_headers", - "libaudioclient", - "libaudioclient_headers", - "libaudiofoundation", - "libaudiofoundation_headers", - "libaudiomanager", - "libaudiopolicy", - "libaudioutils", - "libaudioutils_fixedfft", - "libbinder_headers", - "libbluetooth-types-header", - "libbufferhub", - "libbufferhub_headers", - "libbufferhubqueue", - "libc_malloc_debug_backtrace", - "libcamera_client", - "libcamera_metadata", - "libdvr_headers", - "libexpat", - "libfifo", - "libflacextractor", - "libgrallocusage", - "libgraphicsenv", - "libgui", - "libgui_headers", - "libhardware_headers", - "libinput", - "liblzma", - "libmath", - "libmedia", - "libmedia_codeclist", - "libmedia_headers", - "libmedia_helper", - "libmedia_helper_headers", - "libmedia_midiiowrapper", - "libmedia_omx", - "libmediautils", - "libmidiextractor", - "libmkvextractor", - "libmp3extractor", - "libmp4extractor", - "libmpeg2extractor", - "libnativebase_headers", - "libnativewindow_headers", - "libnblog", - "liboggextractor", - "libpackagelistparser", - "libpdx", - "libpdx_default_transport", - "libpdx_headers", - "libpdx_uds", - "libprocinfo", - "libspeexresampler", - "libspeexresampler", - "libstagefright_esds", - "libstagefright_flacdec", - "libstagefright_flacdec", - "libstagefright_foundation", - "libstagefright_foundation_headers", - "libstagefright_foundation_without_imemory", - "libstagefright_headers", - "libstagefright_id3", - "libstagefright_metadatautils", - "libstagefright_mpeg2extractor", - "libstagefright_mpeg2support", - "libsync", - "libui", - "libui_headers", - "libunwindstack", - "libvibrator", - "libvorbisidec", - "libwavextractor", - "libwebm", - "media_ndk_headers", - "media_plugin_headers", - "updatable-media", - } - // - // Module separator - // - m["com.android.media.swcodec"] = []string{ - "android.frameworks.bufferhub@1.0", - "android.hardware.common-ndk_platform", - "android.hardware.configstore-utils", - "android.hardware.configstore@1.0", - "android.hardware.configstore@1.1", - "android.hardware.graphics.allocator@2.0", - "android.hardware.graphics.allocator@3.0", - "android.hardware.graphics.allocator@4.0", - "android.hardware.graphics.bufferqueue@1.0", - "android.hardware.graphics.bufferqueue@2.0", - "android.hardware.graphics.common-ndk_platform", - "android.hardware.graphics.common@1.0", - "android.hardware.graphics.common@1.1", - "android.hardware.graphics.common@1.2", - "android.hardware.graphics.mapper@2.0", - "android.hardware.graphics.mapper@2.1", - "android.hardware.graphics.mapper@3.0", - "android.hardware.graphics.mapper@4.0", - "android.hardware.media.bufferpool@2.0", - "android.hardware.media.c2@1.0", - "android.hardware.media.c2@1.1", - "android.hardware.media.omx@1.0", - "android.hardware.media@1.0", - "android.hardware.media@1.0", - "android.hidl.memory.token@1.0", - "android.hidl.memory@1.0", - "android.hidl.safe_union@1.0", - "android.hidl.token@1.0", - "android.hidl.token@1.0-utils", - "libEGL", - "libFLAC", - "libFLAC-config", - "libFLAC-headers", - "libFraunhoferAAC", - "libLibGuiProperties", - "libarect", - "libaudio_system_headers", - "libaudioutils", - "libaudioutils", - "libaudioutils_fixedfft", - "libavcdec", - "libavcenc", - "libavservices_minijail", - "libavservices_minijail", - "libbinder_headers", - "libbinderthreadstateutils", - "libbluetooth-types-header", - "libbufferhub_headers", - "libcodec2", - "libcodec2_headers", - "libcodec2_hidl@1.0", - "libcodec2_hidl@1.1", - "libcodec2_internal", - "libcodec2_soft_aacdec", - "libcodec2_soft_aacenc", - "libcodec2_soft_amrnbdec", - "libcodec2_soft_amrnbenc", - "libcodec2_soft_amrwbdec", - "libcodec2_soft_amrwbenc", - "libcodec2_soft_av1dec_gav1", - "libcodec2_soft_avcdec", - "libcodec2_soft_avcenc", - "libcodec2_soft_common", - "libcodec2_soft_flacdec", - "libcodec2_soft_flacenc", - "libcodec2_soft_g711alawdec", - "libcodec2_soft_g711mlawdec", - "libcodec2_soft_gsmdec", - "libcodec2_soft_h263dec", - "libcodec2_soft_h263enc", - "libcodec2_soft_hevcdec", - "libcodec2_soft_hevcenc", - "libcodec2_soft_mp3dec", - "libcodec2_soft_mpeg2dec", - "libcodec2_soft_mpeg4dec", - "libcodec2_soft_mpeg4enc", - "libcodec2_soft_opusdec", - "libcodec2_soft_opusenc", - "libcodec2_soft_rawdec", - "libcodec2_soft_vorbisdec", - "libcodec2_soft_vp8dec", - "libcodec2_soft_vp8enc", - "libcodec2_soft_vp9dec", - "libcodec2_soft_vp9enc", - "libcodec2_vndk", - "libdvr_headers", - "libfmq", - "libfmq", - "libgav1", - "libgralloctypes", - "libgrallocusage", - "libgraphicsenv", - "libgsm", - "libgui_bufferqueue_static", - "libgui_headers", - "libhardware", - "libhardware_headers", - "libhevcdec", - "libhevcenc", - "libion", - "libjpeg", - "liblzma", - "libmath", - "libmedia_codecserviceregistrant", - "libmedia_headers", - "libmpeg2dec", - "libnativebase_headers", - "libnativewindow_headers", - "libpdx_headers", - "libscudo_wrapper", - "libsfplugin_ccodec_utils", - "libspeexresampler", - "libstagefright_amrnb_common", - "libstagefright_amrnbdec", - "libstagefright_amrnbenc", - "libstagefright_amrwbdec", - "libstagefright_amrwbenc", - "libstagefright_bufferpool@2.0.1", - "libstagefright_bufferqueue_helper", - "libstagefright_enc_common", - "libstagefright_flacdec", - "libstagefright_foundation", - "libstagefright_foundation_headers", - "libstagefright_headers", - "libstagefright_m4vh263dec", - "libstagefright_m4vh263enc", - "libstagefright_mp3dec", - "libsync", - "libui", - "libui_headers", - "libunwindstack", - "libvorbisidec", - "libvpx", - "libyuv", - "libyuv_static", - "media_ndk_headers", - "media_plugin_headers", - "mediaswcodec", - } - // - // Module separator - // - m["com.android.mediaprovider"] = []string{ - "MediaProvider", - "MediaProviderGoogle", - "fmtlib_ndk", - "libbase_ndk", - "libfuse", - "libfuse_jni", - } - // - // Module separator - // - m["com.android.permission"] = []string{ - "car-ui-lib", - "iconloader", - "kotlin-annotations", - "kotlin-stdlib", - "kotlin-stdlib-jdk7", - "kotlin-stdlib-jdk8", - "kotlinx-coroutines-android", - "kotlinx-coroutines-android-nodeps", - "kotlinx-coroutines-core", - "kotlinx-coroutines-core-nodeps", - "permissioncontroller-statsd", - "GooglePermissionController", - "PermissionController", - "SettingsLibActionBarShadow", - "SettingsLibAppPreference", - "SettingsLibBarChartPreference", - "SettingsLibLayoutPreference", - "SettingsLibProgressBar", - "SettingsLibSearchWidget", - "SettingsLibSettingsTheme", - "SettingsLibRestrictedLockUtils", - "SettingsLibHelpUtils", - } - // - // Module separator - // - m["com.android.runtime"] = []string{ - "bionic_libc_platform_headers", - "libarm-optimized-routines-math", - "libc_aeabi", - "libc_bionic", - "libc_bionic_ndk", - "libc_bootstrap", - "libc_common", - "libc_common_shared", - "libc_common_static", - "libc_dns", - "libc_dynamic_dispatch", - "libc_fortify", - "libc_freebsd", - "libc_freebsd_large_stack", - "libc_gdtoa", - "libc_init_dynamic", - "libc_init_static", - "libc_jemalloc_wrapper", - "libc_netbsd", - "libc_nomalloc", - "libc_nopthread", - "libc_openbsd", - "libc_openbsd_large_stack", - "libc_openbsd_ndk", - "libc_pthread", - "libc_static_dispatch", - "libc_syscalls", - "libc_tzcode", - "libc_unwind_static", - "libdebuggerd", - "libdebuggerd_common_headers", - "libdebuggerd_handler_core", - "libdebuggerd_handler_fallback", - "libdl_static", - "libjemalloc5", - "liblinker_main", - "liblinker_malloc", - "liblz4", - "liblzma", - "libprocinfo", - "libpropertyinfoparser", - "libscudo", - "libstdc++", - "libsystemproperties", - "libtombstoned_client_static", - "libunwindstack", - "libz", - "libziparchive", - } - // - // Module separator - // - m["com.android.tethering"] = []string{ - "android.hardware.tetheroffload.config-V1.0-java", - "android.hardware.tetheroffload.control-V1.0-java", - "android.hidl.base-V1.0-java", - "libcgrouprc", - "libcgrouprc_format", - "libtetherutilsjni", - "libvndksupport", - "net-utils-framework-common", - "netd_aidl_interface-V3-java", - "netlink-client", - "networkstack-aidl-interfaces-java", - "tethering-aidl-interfaces-java", - "TetheringApiCurrentLib", - } - // - // Module separator - // - m["com.android.wifi"] = []string{ - "PlatformProperties", - "android.hardware.wifi-V1.0-java", - "android.hardware.wifi-V1.0-java-constants", - "android.hardware.wifi-V1.1-java", - "android.hardware.wifi-V1.2-java", - "android.hardware.wifi-V1.3-java", - "android.hardware.wifi-V1.4-java", - "android.hardware.wifi.hostapd-V1.0-java", - "android.hardware.wifi.hostapd-V1.1-java", - "android.hardware.wifi.hostapd-V1.2-java", - "android.hardware.wifi.supplicant-V1.0-java", - "android.hardware.wifi.supplicant-V1.1-java", - "android.hardware.wifi.supplicant-V1.2-java", - "android.hardware.wifi.supplicant-V1.3-java", - "android.hidl.base-V1.0-java", - "android.hidl.manager-V1.0-java", - "android.hidl.manager-V1.1-java", - "android.hidl.manager-V1.2-java", - "bouncycastle-unbundled", - "dnsresolver_aidl_interface-V2-java", - "error_prone_annotations", - "framework-wifi-pre-jarjar", - "framework-wifi-util-lib", - "ipmemorystore-aidl-interfaces-V3-java", - "ipmemorystore-aidl-interfaces-java", - "ksoap2", - "libnanohttpd", - "libwifi-jni", - "net-utils-services-common", - "netd_aidl_interface-V2-java", - "netd_aidl_interface-unstable-java", - "netd_event_listener_interface-java", - "netlink-client", - "networkstack-client", - "services.net", - "wifi-lite-protos", - "wifi-nano-protos", - "wifi-service-pre-jarjar", - "wifi-service-resources", - } - // - // Module separator - // - m["com.android.sdkext"] = []string{ - "fmtlib_ndk", - "libbase_ndk", - "libprotobuf-cpp-lite-ndk", - } - // - // Module separator - // - m["com.android.os.statsd"] = []string{ - "libstatssocket", - } - // - // Module separator - // - m[android.AvailableToAnyApex] = []string{ - // TODO(b/156996905) Set apex_available/min_sdk_version for androidx/extras support libraries - "androidx", - "androidx-constraintlayout_constraintlayout", - "androidx-constraintlayout_constraintlayout-nodeps", - "androidx-constraintlayout_constraintlayout-solver", - "androidx-constraintlayout_constraintlayout-solver-nodeps", - "com.google.android.material_material", - "com.google.android.material_material-nodeps", - - "libatomic", - "libclang_rt", - "libgcc_stripped", - "libprofile-clang-extras", - "libprofile-clang-extras_ndk", - "libprofile-extras", - "libprofile-extras_ndk", - "libunwind_llvm", - } - return m -} - -// DO NOT EDIT! These are the package prefixes that are exempted from being AOT'ed by ART. -// Adding code to the bootclasspath in new packages will cause issues on module update. -func qModulesPackages() map[string][]string { - return map[string][]string{ - "com.android.conscrypt": []string{ - "android.net.ssl", - "com.android.org.conscrypt", - }, - "com.android.media": []string{ - "android.media", - }, - } -} - -// DO NOT EDIT! These are the package prefixes that are exempted from being AOT'ed by ART. -// Adding code to the bootclasspath in new packages will cause issues on module update. -func rModulesPackages() map[string][]string { - return map[string][]string{ - "com.android.mediaprovider": []string{ - "android.provider", - }, - "com.android.permission": []string{ - "android.permission", - "android.app.role", - "com.android.permission", - "com.android.role", - }, - "com.android.sdkext": []string{ - "android.os.ext", - }, - "com.android.os.statsd": []string{ - "android.app", - "android.os", - "android.util", - "com.android.internal.statsd", - "com.android.server.stats", - }, - "com.android.wifi": []string{ - "com.android.server.wifi", - "com.android.wifi.x", - "android.hardware.wifi", - "android.net.wifi", - }, - "com.android.tethering": []string{ - "android.net", - }, - } -} - func init() { android.RegisterModuleType("apex", BundleFactory) android.RegisterModuleType("apex_test", testApexBundleFactory) @@ -730,24 +44,6 @@ func init() { android.PreDepsMutators(RegisterPreDepsMutators) android.PostDepsMutators(RegisterPostDepsMutators) - - android.AddNeverAllowRules(createApexPermittedPackagesRules(qModulesPackages())...) - android.AddNeverAllowRules(createApexPermittedPackagesRules(rModulesPackages())...) -} - -func createApexPermittedPackagesRules(modules_packages map[string][]string) []android.Rule { - rules := make([]android.Rule, 0, len(modules_packages)) - for module_name, module_packages := range modules_packages { - permitted_packages_rule := android.NeverAllow(). - BootclasspathJar(). - With("apex_available", module_name). - WithMatcher("permitted_packages", android.NotInList(module_packages)). - Because("jars that are part of the " + module_name + - " module may only allow these packages: " + strings.Join(module_packages, ",") + - ". Please jarjar or move code around.") - rules = append(rules, permitted_packages_rule) - } - return rules } func RegisterPreDepsMutators(ctx android.RegisterMutatorsContext) { @@ -766,304 +62,6 @@ func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { ctx.BottomUp("mark_platform_availability", markPlatformAvailability).Parallel() } -// Mark the direct and transitive dependencies of apex bundles so that they -// can be built for the apex bundles. -func apexDepsMutator(mctx android.TopDownMutatorContext) { - if !mctx.Module().Enabled() { - return - } - a, ok := mctx.Module().(*apexBundle) - if !ok || a.vndkApex { - return - } - - useVndk := a.SocSpecific() || a.DeviceSpecific() || (a.ProductSpecific() && mctx.Config().EnforceProductPartitionInterface()) - excludeVndkLibs := useVndk && proptools.Bool(a.properties.Use_vndk_as_stable) - if !useVndk && proptools.Bool(a.properties.Use_vndk_as_stable) { - mctx.PropertyErrorf("use_vndk_as_stable", "not supported for system/system_ext APEXes") - return - } - - contents := make(map[string]android.ApexMembership) - - continueApexDepsWalk := func(child, parent android.Module) bool { - am, ok := child.(android.ApexModule) - if !ok || !am.CanHaveApexVariants() { - return false - } - if !parent.(android.DepIsInSameApex).DepIsInSameApex(mctx, child) { - return false - } - if excludeVndkLibs { - if c, ok := child.(*cc.Module); ok && c.IsVndk() { - return false - } - } - return true - } - - mctx.WalkDeps(func(child, parent android.Module) bool { - if !continueApexDepsWalk(child, parent) { - return false - } - - depName := mctx.OtherModuleName(child) - // If the parent is apexBundle, this child is directly depended. - _, directDep := parent.(*apexBundle) - contents[depName] = contents[depName].Add(directDep) - return true - }) - - apexContents := android.NewApexContents(mctx.ModuleName(), contents) - mctx.SetProvider(ApexBundleInfoProvider, ApexBundleInfo{ - Contents: apexContents, - }) - - apexInfo := android.ApexInfo{ - ApexVariationName: mctx.ModuleName(), - MinSdkVersionStr: a.minSdkVersion(mctx).String(), - RequiredSdks: a.RequiredSdks(), - Updatable: a.Updatable(), - InApexes: []string{mctx.ModuleName()}, - ApexContents: []*android.ApexContents{apexContents}, - } - - mctx.WalkDeps(func(child, parent android.Module) bool { - if !continueApexDepsWalk(child, parent) { - return false - } - - child.(android.ApexModule).BuildForApex(apexInfo) - return true - }) -} - -func apexUniqueVariationsMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - if am, ok := mctx.Module().(android.ApexModule); ok { - // Check if any dependencies use unique apex variations. If so, use unique apex variations - // for this module. - android.UpdateUniqueApexVariationsForDeps(mctx, am) - } -} - -func apexTestForDepsMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - // Check if this module is a test for an apex. If so, add a dependency on the apex - // in order to retrieve its contents later. - if am, ok := mctx.Module().(android.ApexModule); ok { - if testFor := am.TestFor(); len(testFor) > 0 { - mctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "os", Variation: am.Target().OsVariation()}, - {"arch", "common"}, - }, testForTag, testFor...) - } - } -} - -func apexTestForMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - - if _, ok := mctx.Module().(android.ApexModule); ok { - var contents []*android.ApexContents - for _, testFor := range mctx.GetDirectDepsWithTag(testForTag) { - abInfo := mctx.OtherModuleProvider(testFor, ApexBundleInfoProvider).(ApexBundleInfo) - contents = append(contents, abInfo.Contents) - } - mctx.SetProvider(android.ApexTestForInfoProvider, android.ApexTestForInfo{ - ApexContents: contents, - }) - } -} - -// mark if a module cannot be available to platform. A module cannot be available -// to platform if 1) it is explicitly marked as not available (i.e. "//apex_available:platform" -// is absent) or 2) it depends on another module that isn't (or can't be) available to platform -func markPlatformAvailability(mctx android.BottomUpMutatorContext) { - // Host and recovery are not considered as platform - if mctx.Host() || mctx.Module().InstallInRecovery() { - return - } - - if am, ok := mctx.Module().(android.ApexModule); ok { - availableToPlatform := am.AvailableFor(android.AvailableToPlatform) - - // If any of the dep is not available to platform, this module is also considered - // as being not available to platform even if it has "//apex_available:platform" - mctx.VisitDirectDeps(func(child android.Module) { - if !am.DepIsInSameApex(mctx, child) { - // if the dependency crosses apex boundary, don't consider it - return - } - if dep, ok := child.(android.ApexModule); ok && dep.NotAvailableForPlatform() { - availableToPlatform = false - // TODO(b/154889534) trigger an error when 'am' has "//apex_available:platform" - } - }) - - // Exception 1: stub libraries and native bridge libraries are always available to platform - if cc, ok := mctx.Module().(*cc.Module); ok && - (cc.IsStubs() || cc.Target().NativeBridge == android.NativeBridgeEnabled) { - availableToPlatform = true - } - - // Exception 2: bootstrap bionic libraries are also always available to platform - if cc.InstallToBootstrap(mctx.ModuleName(), mctx.Config()) { - availableToPlatform = true - } - - if !availableToPlatform { - am.SetNotAvailableForPlatform() - } - } -} - -// If a module in an APEX depends on a module from an SDK then it needs an APEX -// specific variant created for it. Refer to sdk.sdkDepsReplaceMutator. -func inAnySdk(module android.Module) bool { - if sa, ok := module.(android.SdkAware); ok { - return sa.IsInAnySdk() - } - - return false -} - -// Create apex variations if a module is included in APEX(s). -func apexMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - - if am, ok := mctx.Module().(android.ApexModule); ok && am.CanHaveApexVariants() { - android.CreateApexVariations(mctx, am) - } else if a, ok := mctx.Module().(*apexBundle); ok && !a.vndkApex { - // apex bundle itself is mutated so that it and its modules have same - // apex variant. - apexBundleName := mctx.ModuleName() - mctx.CreateVariations(apexBundleName) - } else if o, ok := mctx.Module().(*OverrideApex); ok { - apexBundleName := o.GetOverriddenModuleName() - if apexBundleName == "" { - mctx.ModuleErrorf("base property is not set") - return - } - mctx.CreateVariations(apexBundleName) - } - -} - -func apexDirectlyInAnyMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - if am, ok := mctx.Module().(android.ApexModule); ok { - android.UpdateDirectlyInAnyApex(mctx, am) - } -} - -func apexFlattenedMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - if ab, ok := mctx.Module().(*apexBundle); ok { - var variants []string - switch proptools.StringDefault(ab.properties.Payload_type, "image") { - case "image": - variants = append(variants, imageApexType, flattenedApexType) - case "zip": - variants = append(variants, zipApexType) - case "both": - variants = append(variants, imageApexType, zipApexType, flattenedApexType) - default: - mctx.PropertyErrorf("type", "%q is not one of \"image\", \"zip\", or \"both\".", *ab.properties.Payload_type) - return - } - - modules := mctx.CreateLocalVariations(variants...) - - for i, v := range variants { - switch v { - case imageApexType: - modules[i].(*apexBundle).properties.ApexType = imageApex - case zipApexType: - modules[i].(*apexBundle).properties.ApexType = zipApex - case flattenedApexType: - modules[i].(*apexBundle).properties.ApexType = flattenedApex - if !mctx.Config().FlattenApex() && ab.Platform() { - modules[i].(*apexBundle).MakeAsSystemExt() - } - } - } - } else if _, ok := mctx.Module().(*OverrideApex); ok { - mctx.CreateVariations(imageApexType, flattenedApexType) - } -} - -var ( - useVendorAllowListKey = android.NewOnceKey("useVendorAllowList") -) - -// useVendorAllowList returns the list of APEXes which are allowed to use_vendor. -// When use_vendor is used, native modules are built with __ANDROID_VNDK__ and __ANDROID_APEX__, -// which may cause compatibility issues. (e.g. libbinder) -// Even though libbinder restricts its availability via 'apex_available' property and relies on -// yet another macro __ANDROID_APEX_<NAME>__, we restrict usage of "use_vendor:" from other APEX modules -// to avoid similar problems. -func useVendorAllowList(config android.Config) []string { - return config.Once(useVendorAllowListKey, func() interface{} { - return []string{ - // swcodec uses "vendor" variants for smaller size - "com.android.media.swcodec", - "test_com.android.media.swcodec", - } - }).([]string) -} - -// setUseVendorAllowListForTest overrides useVendorAllowList and must be -// called before the first call to useVendorAllowList() -func setUseVendorAllowListForTest(config android.Config, allowList []string) { - config.Once(useVendorAllowListKey, func() interface{} { - return allowList - }) -} - -type ApexNativeDependencies struct { - // List of native libraries - Native_shared_libs []string - - // List of JNI libraries - Jni_libs []string - - // List of native executables - Binaries []string - - // List of native tests - Tests []string -} - -type apexMultilibProperties struct { - // Native dependencies whose compile_multilib is "first" - First ApexNativeDependencies - - // Native dependencies whose compile_multilib is "both" - Both ApexNativeDependencies - - // Native dependencies whose compile_multilib is "prefer32" - Prefer32 ApexNativeDependencies - - // Native dependencies whose compile_multilib is "32" - Lib32 ApexNativeDependencies - - // Native dependencies whose compile_multilib is "64" - Lib64 ApexNativeDependencies -} - type apexBundleProperties struct { // Json manifest file describing meta info of this APEX bundle. Default: // "apex_manifest.json" @@ -1164,11 +162,36 @@ type apexBundleProperties struct { Payload_fs_type *string } -type ApexBundleInfo struct { - Contents *android.ApexContents +type ApexNativeDependencies struct { + // List of native libraries + Native_shared_libs []string + + // List of JNI libraries + Jni_libs []string + + // List of native executables + Binaries []string + + // List of native tests + Tests []string } -var ApexBundleInfoProvider = blueprint.NewMutatorProvider(ApexBundleInfo{}, "apex_deps") +type apexMultilibProperties struct { + // Native dependencies whose compile_multilib is "first" + First ApexNativeDependencies + + // Native dependencies whose compile_multilib is "both" + Both ApexNativeDependencies + + // Native dependencies whose compile_multilib is "prefer32" + Prefer32 ApexNativeDependencies + + // Native dependencies whose compile_multilib is "32" + Lib32 ApexNativeDependencies + + // Native dependencies whose compile_multilib is "64" + Lib64 ApexNativeDependencies +} type apexTargetBundleProperties struct { Target struct { @@ -1219,35 +242,75 @@ type overridableProperties struct { Allowed_files *string `android:"path"` } -type apexPackaging int +type apexBundle struct { + android.ModuleBase + android.DefaultableModuleBase + android.OverridableModuleBase + android.SdkBase -const ( - imageApex apexPackaging = iota - zipApex - flattenedApex -) + properties apexBundleProperties + targetProperties apexTargetBundleProperties + overridableProperties overridableProperties -// The suffix for the output "file", not the module -func (a apexPackaging) suffix() string { - switch a { - case imageApex: - return imageApexSuffix - case zipApex: - return zipApexSuffix - default: - panic(fmt.Errorf("unknown APEX type %d", a)) - } -} + // specific to apex_vndk modules + vndkProperties apexVndkProperties -func (a apexPackaging) name() string { - switch a { - case imageApex: - return imageApexType - case zipApex: - return zipApexType - default: - panic(fmt.Errorf("unknown APEX type %d", a)) - } + bundleModuleFile android.WritablePath + outputFile android.WritablePath + installDir android.InstallPath + + prebuiltFileToDelete string + + public_key_file android.Path + private_key_file android.Path + + container_certificate_file android.Path + container_private_key_file android.Path + + fileContexts android.WritablePath + + // list of files to be included in this apex + filesInfo []apexFile + + // list of module names that should be installed along with this APEX + requiredDeps []string + + // list of module names that this APEX is including (to be shown via *-deps-info target) + android.ApexBundleDepsInfo + + testApex bool + vndkApex bool + artApex bool + primaryApexType bool + + manifestJsonOut android.WritablePath + manifestPbOut android.WritablePath + + // list of commands to create symlinks for backward compatibility. + // these commands will be attached as LOCAL_POST_INSTALL_CMD to + // apex package itself(for unflattened build) or apex_manifest(for flattened build) + // so that compat symlinks are always installed regardless of TARGET_FLATTEN_APEX setting. + compatSymlinks []string + + // Suffix of module name in Android.mk + // ".flattened", ".apex", ".zipapex", or "" + suffix string + + installedFilesFile android.WritablePath + + // Whether to create symlink to the system file instead of having a file + // inside the apex or not + linkToSystemLib bool + + // Struct holding the merged notice file paths in different formats + mergedNotices android.NoticeOutputs + + // Optional list of lint report zip files for apexes that contain java or app modules + lintReports android.Paths + + payloadFsType fsType + + distFiles android.TaggedDistFiles } type apexFileClass int @@ -1378,95 +441,6 @@ func (af *apexFile) AvailableToPlatform() bool { return false } -type fsType int - -const ( - ext4 fsType = iota - f2fs -) - -func (f fsType) string() string { - switch f { - case ext4: - return ext4FsType - case f2fs: - return f2fsFsType - default: - panic(fmt.Errorf("unknown APEX payload type %d", f)) - } -} - -type apexBundle struct { - android.ModuleBase - android.DefaultableModuleBase - android.OverridableModuleBase - android.SdkBase - - properties apexBundleProperties - targetProperties apexTargetBundleProperties - overridableProperties overridableProperties - - // specific to apex_vndk modules - vndkProperties apexVndkProperties - - bundleModuleFile android.WritablePath - outputFile android.WritablePath - installDir android.InstallPath - - prebuiltFileToDelete string - - public_key_file android.Path - private_key_file android.Path - - container_certificate_file android.Path - container_private_key_file android.Path - - fileContexts android.WritablePath - - // list of files to be included in this apex - filesInfo []apexFile - - // list of module names that should be installed along with this APEX - requiredDeps []string - - // list of module names that this APEX is including (to be shown via *-deps-info target) - android.ApexBundleDepsInfo - - testApex bool - vndkApex bool - artApex bool - primaryApexType bool - - manifestJsonOut android.WritablePath - manifestPbOut android.WritablePath - - // list of commands to create symlinks for backward compatibility. - // these commands will be attached as LOCAL_POST_INSTALL_CMD to - // apex package itself(for unflattened build) or apex_manifest(for flattened build) - // so that compat symlinks are always installed regardless of TARGET_FLATTEN_APEX setting. - compatSymlinks []string - - // Suffix of module name in Android.mk - // ".flattened", ".apex", ".zipapex", or "" - suffix string - - installedFilesFile android.WritablePath - - // Whether to create symlink to the system file instead of having a file - // inside the apex or not - linkToSystemLib bool - - // Struct holding the merged notice file paths in different formats - mergedNotices android.NoticeOutputs - - // Optional list of lint report zip files for apexes that contain java or app modules - lintReports android.Paths - - payloadFsType fsType - - distFiles android.TaggedDistFiles -} - func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, nativeModules ApexNativeDependencies, target android.Target, imageVariation string) { @@ -1509,6 +483,29 @@ func (a *apexBundle) combineProperties(ctx android.BottomUpMutatorContext) { } } +type dependencyTag struct { + blueprint.BaseDependencyTag + name string + + // determines if the dependent will be part of the APEX payload + payload bool +} + +var ( + sharedLibTag = dependencyTag{name: "sharedLib", payload: true} + jniLibTag = dependencyTag{name: "jniLib", payload: true} + executableTag = dependencyTag{name: "executable", payload: true} + javaLibTag = dependencyTag{name: "javaLib", payload: true} + prebuiltTag = dependencyTag{name: "prebuilt", payload: true} + testTag = dependencyTag{name: "test", payload: true} + keyTag = dependencyTag{name: "key"} + certificateTag = dependencyTag{name: "certificate"} + androidAppTag = dependencyTag{name: "androidApp", payload: true} + rroTag = dependencyTag{name: "rro", payload: true} + bpfTag = dependencyTag{name: "bpf", payload: true} + testForTag = dependencyTag{name: "test for"} +) + func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { if proptools.Bool(a.properties.Use_vendor) && !android.InList(a.Name(), useVendorAllowList(ctx.Config())) { ctx.PropertyErrorf("use_vendor", "not allowed to set use_vendor: true") @@ -1658,6 +655,341 @@ func (a *apexBundle) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato rroTag, a.overridableProperties.Rros...) } +type ApexBundleInfo struct { + Contents *android.ApexContents +} + +var ApexBundleInfoProvider = blueprint.NewMutatorProvider(ApexBundleInfo{}, "apex_deps") + +// Mark the direct and transitive dependencies of apex bundles so that they +// can be built for the apex bundles. +func apexDepsMutator(mctx android.TopDownMutatorContext) { + if !mctx.Module().Enabled() { + return + } + a, ok := mctx.Module().(*apexBundle) + if !ok || a.vndkApex { + return + } + + useVndk := a.SocSpecific() || a.DeviceSpecific() || (a.ProductSpecific() && mctx.Config().EnforceProductPartitionInterface()) + excludeVndkLibs := useVndk && proptools.Bool(a.properties.Use_vndk_as_stable) + if !useVndk && proptools.Bool(a.properties.Use_vndk_as_stable) { + mctx.PropertyErrorf("use_vndk_as_stable", "not supported for system/system_ext APEXes") + return + } + + contents := make(map[string]android.ApexMembership) + + continueApexDepsWalk := func(child, parent android.Module) bool { + am, ok := child.(android.ApexModule) + if !ok || !am.CanHaveApexVariants() { + return false + } + if !parent.(android.DepIsInSameApex).DepIsInSameApex(mctx, child) { + return false + } + if excludeVndkLibs { + if c, ok := child.(*cc.Module); ok && c.IsVndk() { + return false + } + } + return true + } + + mctx.WalkDeps(func(child, parent android.Module) bool { + if !continueApexDepsWalk(child, parent) { + return false + } + + depName := mctx.OtherModuleName(child) + // If the parent is apexBundle, this child is directly depended. + _, directDep := parent.(*apexBundle) + contents[depName] = contents[depName].Add(directDep) + return true + }) + + apexContents := android.NewApexContents(mctx.ModuleName(), contents) + mctx.SetProvider(ApexBundleInfoProvider, ApexBundleInfo{ + Contents: apexContents, + }) + + apexInfo := android.ApexInfo{ + ApexVariationName: mctx.ModuleName(), + MinSdkVersionStr: a.minSdkVersion(mctx).String(), + RequiredSdks: a.RequiredSdks(), + Updatable: a.Updatable(), + InApexes: []string{mctx.ModuleName()}, + ApexContents: []*android.ApexContents{apexContents}, + } + + mctx.WalkDeps(func(child, parent android.Module) bool { + if !continueApexDepsWalk(child, parent) { + return false + } + + child.(android.ApexModule).BuildForApex(apexInfo) + return true + }) +} + +func apexUniqueVariationsMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + if am, ok := mctx.Module().(android.ApexModule); ok { + // Check if any dependencies use unique apex variations. If so, use unique apex variations + // for this module. + android.UpdateUniqueApexVariationsForDeps(mctx, am) + } +} + +func apexTestForDepsMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + // Check if this module is a test for an apex. If so, add a dependency on the apex + // in order to retrieve its contents later. + if am, ok := mctx.Module().(android.ApexModule); ok { + if testFor := am.TestFor(); len(testFor) > 0 { + mctx.AddFarVariationDependencies([]blueprint.Variation{ + {Mutator: "os", Variation: am.Target().OsVariation()}, + {"arch", "common"}, + }, testForTag, testFor...) + } + } +} + +func apexTestForMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + + if _, ok := mctx.Module().(android.ApexModule); ok { + var contents []*android.ApexContents + for _, testFor := range mctx.GetDirectDepsWithTag(testForTag) { + abInfo := mctx.OtherModuleProvider(testFor, ApexBundleInfoProvider).(ApexBundleInfo) + contents = append(contents, abInfo.Contents) + } + mctx.SetProvider(android.ApexTestForInfoProvider, android.ApexTestForInfo{ + ApexContents: contents, + }) + } +} + +// mark if a module cannot be available to platform. A module cannot be available +// to platform if 1) it is explicitly marked as not available (i.e. "//apex_available:platform" +// is absent) or 2) it depends on another module that isn't (or can't be) available to platform +func markPlatformAvailability(mctx android.BottomUpMutatorContext) { + // Host and recovery are not considered as platform + if mctx.Host() || mctx.Module().InstallInRecovery() { + return + } + + if am, ok := mctx.Module().(android.ApexModule); ok { + availableToPlatform := am.AvailableFor(android.AvailableToPlatform) + + // If any of the dep is not available to platform, this module is also considered + // as being not available to platform even if it has "//apex_available:platform" + mctx.VisitDirectDeps(func(child android.Module) { + if !am.DepIsInSameApex(mctx, child) { + // if the dependency crosses apex boundary, don't consider it + return + } + if dep, ok := child.(android.ApexModule); ok && dep.NotAvailableForPlatform() { + availableToPlatform = false + // TODO(b/154889534) trigger an error when 'am' has "//apex_available:platform" + } + }) + + // Exception 1: stub libraries and native bridge libraries are always available to platform + if cc, ok := mctx.Module().(*cc.Module); ok && + (cc.IsStubs() || cc.Target().NativeBridge == android.NativeBridgeEnabled) { + availableToPlatform = true + } + + // Exception 2: bootstrap bionic libraries are also always available to platform + if cc.InstallToBootstrap(mctx.ModuleName(), mctx.Config()) { + availableToPlatform = true + } + + if !availableToPlatform { + am.SetNotAvailableForPlatform() + } + } +} + +// If a module in an APEX depends on a module from an SDK then it needs an APEX +// specific variant created for it. Refer to sdk.sdkDepsReplaceMutator. +func inAnySdk(module android.Module) bool { + if sa, ok := module.(android.SdkAware); ok { + return sa.IsInAnySdk() + } + + return false +} + +// Create apex variations if a module is included in APEX(s). +func apexMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + + if am, ok := mctx.Module().(android.ApexModule); ok && am.CanHaveApexVariants() { + android.CreateApexVariations(mctx, am) + } else if a, ok := mctx.Module().(*apexBundle); ok && !a.vndkApex { + // apex bundle itself is mutated so that it and its modules have same + // apex variant. + apexBundleName := mctx.ModuleName() + mctx.CreateVariations(apexBundleName) + } else if o, ok := mctx.Module().(*OverrideApex); ok { + apexBundleName := o.GetOverriddenModuleName() + if apexBundleName == "" { + mctx.ModuleErrorf("base property is not set") + return + } + mctx.CreateVariations(apexBundleName) + } + +} + +func apexDirectlyInAnyMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + if am, ok := mctx.Module().(android.ApexModule); ok { + android.UpdateDirectlyInAnyApex(mctx, am) + } +} + +type apexPackaging int + +const ( + imageApex apexPackaging = iota + zipApex + flattenedApex +) + +const ( + imageApexSuffix = ".apex" + zipApexSuffix = ".zipapex" + flattenedSuffix = ".flattened" + + imageApexType = "image" + zipApexType = "zip" + flattenedApexType = "flattened" + + ext4FsType = "ext4" + f2fsFsType = "f2fs" +) + +// The suffix for the output "file", not the module +func (a apexPackaging) suffix() string { + switch a { + case imageApex: + return imageApexSuffix + case zipApex: + return zipApexSuffix + default: + panic(fmt.Errorf("unknown APEX type %d", a)) + } +} + +func (a apexPackaging) name() string { + switch a { + case imageApex: + return imageApexType + case zipApex: + return zipApexType + default: + panic(fmt.Errorf("unknown APEX type %d", a)) + } +} + +func apexFlattenedMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Module().Enabled() { + return + } + if ab, ok := mctx.Module().(*apexBundle); ok { + var variants []string + switch proptools.StringDefault(ab.properties.Payload_type, "image") { + case "image": + variants = append(variants, imageApexType, flattenedApexType) + case "zip": + variants = append(variants, zipApexType) + case "both": + variants = append(variants, imageApexType, zipApexType, flattenedApexType) + default: + mctx.PropertyErrorf("type", "%q is not one of \"image\", \"zip\", or \"both\".", *ab.properties.Payload_type) + return + } + + modules := mctx.CreateLocalVariations(variants...) + + for i, v := range variants { + switch v { + case imageApexType: + modules[i].(*apexBundle).properties.ApexType = imageApex + case zipApexType: + modules[i].(*apexBundle).properties.ApexType = zipApex + case flattenedApexType: + modules[i].(*apexBundle).properties.ApexType = flattenedApex + if !mctx.Config().FlattenApex() && ab.Platform() { + modules[i].(*apexBundle).MakeAsSystemExt() + } + } + } + } else if _, ok := mctx.Module().(*OverrideApex); ok { + mctx.CreateVariations(imageApexType, flattenedApexType) + } +} + +var ( + useVendorAllowListKey = android.NewOnceKey("useVendorAllowList") +) + +// useVendorAllowList returns the list of APEXes which are allowed to use_vendor. +// When use_vendor is used, native modules are built with __ANDROID_VNDK__ and __ANDROID_APEX__, +// which may cause compatibility issues. (e.g. libbinder) +// Even though libbinder restricts its availability via 'apex_available' property and relies on +// yet another macro __ANDROID_APEX_<NAME>__, we restrict usage of "use_vendor:" from other APEX modules +// to avoid similar problems. +func useVendorAllowList(config android.Config) []string { + return config.Once(useVendorAllowListKey, func() interface{} { + return []string{ + // swcodec uses "vendor" variants for smaller size + "com.android.media.swcodec", + "test_com.android.media.swcodec", + } + }).([]string) +} + +// setUseVendorAllowListForTest overrides useVendorAllowList and must be +// called before the first call to useVendorAllowList() +func setUseVendorAllowListForTest(config android.Config, allowList []string) { + config.Once(useVendorAllowListKey, func() interface{} { + return allowList + }) +} + +type fsType int + +const ( + ext4 fsType = iota + f2fs +) + +func (f fsType) string() string { + switch f { + case ext4: + return ext4FsType + case f2fs: + return f2fsFsType + default: + panic(fmt.Errorf("unknown APEX payload type %d", f)) + } +} + func (a *apexBundle) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { // direct deps of an APEX bundle are all part of the APEX bundle return true @@ -2615,3 +1947,674 @@ func overrideApexFactory() android.Module { android.InitOverrideModule(m) return m } + +var ( + apexAvailBaseline = makeApexAvailableBaseline() + inverseApexAvailBaseline = invertApexBaseline(apexAvailBaseline) +) + +// Transform the map of apex -> modules to module -> apexes. +func invertApexBaseline(m map[string][]string) map[string][]string { + r := make(map[string][]string) + for apex, modules := range m { + for _, module := range modules { + r[module] = append(r[module], apex) + } + } + return r +} + +// Retrieve the baseline of apexes to which the supplied module belongs. +func BaselineApexAvailable(moduleName string) []string { + return inverseApexAvailBaseline[normalizeModuleName(moduleName)] +} + +// This is a map from apex to modules, which overrides the +// apex_available setting for that particular module to make +// it available for the apex regardless of its setting. +// TODO(b/147364041): remove this +func makeApexAvailableBaseline() map[string][]string { + // The "Module separator"s below are employed to minimize merge conflicts. + m := make(map[string][]string) + // + // Module separator + // + m["com.android.appsearch"] = []string{ + "icing-java-proto-lite", + "libprotobuf-java-lite", + } + // + // Module separator + // + m["com.android.bluetooth.updatable"] = []string{ + "android.hardware.audio.common@5.0", + "android.hardware.bluetooth.a2dp@1.0", + "android.hardware.bluetooth.audio@2.0", + "android.hardware.bluetooth@1.0", + "android.hardware.bluetooth@1.1", + "android.hardware.graphics.bufferqueue@1.0", + "android.hardware.graphics.bufferqueue@2.0", + "android.hardware.graphics.common@1.0", + "android.hardware.graphics.common@1.1", + "android.hardware.graphics.common@1.2", + "android.hardware.media@1.0", + "android.hidl.safe_union@1.0", + "android.hidl.token@1.0", + "android.hidl.token@1.0-utils", + "avrcp-target-service", + "avrcp_headers", + "bluetooth-protos-lite", + "bluetooth.mapsapi", + "com.android.vcard", + "dnsresolver_aidl_interface-V2-java", + "ipmemorystore-aidl-interfaces-V5-java", + "ipmemorystore-aidl-interfaces-java", + "internal_include_headers", + "lib-bt-packets", + "lib-bt-packets-avrcp", + "lib-bt-packets-base", + "libFraunhoferAAC", + "libaudio-a2dp-hw-utils", + "libaudio-hearing-aid-hw-utils", + "libbinder_headers", + "libbluetooth", + "libbluetooth-types", + "libbluetooth-types-header", + "libbluetooth_gd", + "libbluetooth_headers", + "libbluetooth_jni", + "libbt-audio-hal-interface", + "libbt-bta", + "libbt-common", + "libbt-hci", + "libbt-platform-protos-lite", + "libbt-protos-lite", + "libbt-sbc-decoder", + "libbt-sbc-encoder", + "libbt-stack", + "libbt-utils", + "libbtcore", + "libbtdevice", + "libbte", + "libbtif", + "libchrome", + "libevent", + "libfmq", + "libg722codec", + "libgui_headers", + "libmedia_headers", + "libmodpb64", + "libosi", + "libstagefright_foundation_headers", + "libstagefright_headers", + "libstatslog", + "libstatssocket", + "libtinyxml2", + "libudrv-uipc", + "libz", + "media_plugin_headers", + "net-utils-services-common", + "netd_aidl_interface-unstable-java", + "netd_event_listener_interface-java", + "netlink-client", + "networkstack-client", + "sap-api-java-static", + "services.net", + } + // + // Module separator + // + m["com.android.cellbroadcast"] = []string{"CellBroadcastApp", "CellBroadcastServiceModule"} + // + // Module separator + // + m["com.android.extservices"] = []string{ + "error_prone_annotations", + "ExtServices-core", + "ExtServices", + "libtextclassifier-java", + "libz_current", + "textclassifier-statsd", + "TextClassifierNotificationLibNoManifest", + "TextClassifierServiceLibNoManifest", + } + // + // Module separator + // + m["com.android.neuralnetworks"] = []string{ + "android.hardware.neuralnetworks@1.0", + "android.hardware.neuralnetworks@1.1", + "android.hardware.neuralnetworks@1.2", + "android.hardware.neuralnetworks@1.3", + "android.hidl.allocator@1.0", + "android.hidl.memory.token@1.0", + "android.hidl.memory@1.0", + "android.hidl.safe_union@1.0", + "libarect", + "libbuildversion", + "libmath", + "libprocpartition", + "libsync", + } + // + // Module separator + // + m["com.android.media"] = []string{ + "android.frameworks.bufferhub@1.0", + "android.hardware.cas.native@1.0", + "android.hardware.cas@1.0", + "android.hardware.configstore-utils", + "android.hardware.configstore@1.0", + "android.hardware.configstore@1.1", + "android.hardware.graphics.allocator@2.0", + "android.hardware.graphics.allocator@3.0", + "android.hardware.graphics.bufferqueue@1.0", + "android.hardware.graphics.bufferqueue@2.0", + "android.hardware.graphics.common@1.0", + "android.hardware.graphics.common@1.1", + "android.hardware.graphics.common@1.2", + "android.hardware.graphics.mapper@2.0", + "android.hardware.graphics.mapper@2.1", + "android.hardware.graphics.mapper@3.0", + "android.hardware.media.omx@1.0", + "android.hardware.media@1.0", + "android.hidl.allocator@1.0", + "android.hidl.memory.token@1.0", + "android.hidl.memory@1.0", + "android.hidl.token@1.0", + "android.hidl.token@1.0-utils", + "bionic_libc_platform_headers", + "exoplayer2-extractor", + "exoplayer2-extractor-annotation-stubs", + "gl_headers", + "jsr305", + "libEGL", + "libEGL_blobCache", + "libEGL_getProcAddress", + "libFLAC", + "libFLAC-config", + "libFLAC-headers", + "libGLESv2", + "libaacextractor", + "libamrextractor", + "libarect", + "libaudio_system_headers", + "libaudioclient", + "libaudioclient_headers", + "libaudiofoundation", + "libaudiofoundation_headers", + "libaudiomanager", + "libaudiopolicy", + "libaudioutils", + "libaudioutils_fixedfft", + "libbinder_headers", + "libbluetooth-types-header", + "libbufferhub", + "libbufferhub_headers", + "libbufferhubqueue", + "libc_malloc_debug_backtrace", + "libcamera_client", + "libcamera_metadata", + "libdvr_headers", + "libexpat", + "libfifo", + "libflacextractor", + "libgrallocusage", + "libgraphicsenv", + "libgui", + "libgui_headers", + "libhardware_headers", + "libinput", + "liblzma", + "libmath", + "libmedia", + "libmedia_codeclist", + "libmedia_headers", + "libmedia_helper", + "libmedia_helper_headers", + "libmedia_midiiowrapper", + "libmedia_omx", + "libmediautils", + "libmidiextractor", + "libmkvextractor", + "libmp3extractor", + "libmp4extractor", + "libmpeg2extractor", + "libnativebase_headers", + "libnativewindow_headers", + "libnblog", + "liboggextractor", + "libpackagelistparser", + "libpdx", + "libpdx_default_transport", + "libpdx_headers", + "libpdx_uds", + "libprocinfo", + "libspeexresampler", + "libspeexresampler", + "libstagefright_esds", + "libstagefright_flacdec", + "libstagefright_flacdec", + "libstagefright_foundation", + "libstagefright_foundation_headers", + "libstagefright_foundation_without_imemory", + "libstagefright_headers", + "libstagefright_id3", + "libstagefright_metadatautils", + "libstagefright_mpeg2extractor", + "libstagefright_mpeg2support", + "libsync", + "libui", + "libui_headers", + "libunwindstack", + "libvibrator", + "libvorbisidec", + "libwavextractor", + "libwebm", + "media_ndk_headers", + "media_plugin_headers", + "updatable-media", + } + // + // Module separator + // + m["com.android.media.swcodec"] = []string{ + "android.frameworks.bufferhub@1.0", + "android.hardware.common-ndk_platform", + "android.hardware.configstore-utils", + "android.hardware.configstore@1.0", + "android.hardware.configstore@1.1", + "android.hardware.graphics.allocator@2.0", + "android.hardware.graphics.allocator@3.0", + "android.hardware.graphics.allocator@4.0", + "android.hardware.graphics.bufferqueue@1.0", + "android.hardware.graphics.bufferqueue@2.0", + "android.hardware.graphics.common-ndk_platform", + "android.hardware.graphics.common@1.0", + "android.hardware.graphics.common@1.1", + "android.hardware.graphics.common@1.2", + "android.hardware.graphics.mapper@2.0", + "android.hardware.graphics.mapper@2.1", + "android.hardware.graphics.mapper@3.0", + "android.hardware.graphics.mapper@4.0", + "android.hardware.media.bufferpool@2.0", + "android.hardware.media.c2@1.0", + "android.hardware.media.c2@1.1", + "android.hardware.media.omx@1.0", + "android.hardware.media@1.0", + "android.hardware.media@1.0", + "android.hidl.memory.token@1.0", + "android.hidl.memory@1.0", + "android.hidl.safe_union@1.0", + "android.hidl.token@1.0", + "android.hidl.token@1.0-utils", + "libEGL", + "libFLAC", + "libFLAC-config", + "libFLAC-headers", + "libFraunhoferAAC", + "libLibGuiProperties", + "libarect", + "libaudio_system_headers", + "libaudioutils", + "libaudioutils", + "libaudioutils_fixedfft", + "libavcdec", + "libavcenc", + "libavservices_minijail", + "libavservices_minijail", + "libbinder_headers", + "libbinderthreadstateutils", + "libbluetooth-types-header", + "libbufferhub_headers", + "libcodec2", + "libcodec2_headers", + "libcodec2_hidl@1.0", + "libcodec2_hidl@1.1", + "libcodec2_internal", + "libcodec2_soft_aacdec", + "libcodec2_soft_aacenc", + "libcodec2_soft_amrnbdec", + "libcodec2_soft_amrnbenc", + "libcodec2_soft_amrwbdec", + "libcodec2_soft_amrwbenc", + "libcodec2_soft_av1dec_gav1", + "libcodec2_soft_avcdec", + "libcodec2_soft_avcenc", + "libcodec2_soft_common", + "libcodec2_soft_flacdec", + "libcodec2_soft_flacenc", + "libcodec2_soft_g711alawdec", + "libcodec2_soft_g711mlawdec", + "libcodec2_soft_gsmdec", + "libcodec2_soft_h263dec", + "libcodec2_soft_h263enc", + "libcodec2_soft_hevcdec", + "libcodec2_soft_hevcenc", + "libcodec2_soft_mp3dec", + "libcodec2_soft_mpeg2dec", + "libcodec2_soft_mpeg4dec", + "libcodec2_soft_mpeg4enc", + "libcodec2_soft_opusdec", + "libcodec2_soft_opusenc", + "libcodec2_soft_rawdec", + "libcodec2_soft_vorbisdec", + "libcodec2_soft_vp8dec", + "libcodec2_soft_vp8enc", + "libcodec2_soft_vp9dec", + "libcodec2_soft_vp9enc", + "libcodec2_vndk", + "libdvr_headers", + "libfmq", + "libfmq", + "libgav1", + "libgralloctypes", + "libgrallocusage", + "libgraphicsenv", + "libgsm", + "libgui_bufferqueue_static", + "libgui_headers", + "libhardware", + "libhardware_headers", + "libhevcdec", + "libhevcenc", + "libion", + "libjpeg", + "liblzma", + "libmath", + "libmedia_codecserviceregistrant", + "libmedia_headers", + "libmpeg2dec", + "libnativebase_headers", + "libnativewindow_headers", + "libpdx_headers", + "libscudo_wrapper", + "libsfplugin_ccodec_utils", + "libspeexresampler", + "libstagefright_amrnb_common", + "libstagefright_amrnbdec", + "libstagefright_amrnbenc", + "libstagefright_amrwbdec", + "libstagefright_amrwbenc", + "libstagefright_bufferpool@2.0.1", + "libstagefright_bufferqueue_helper", + "libstagefright_enc_common", + "libstagefright_flacdec", + "libstagefright_foundation", + "libstagefright_foundation_headers", + "libstagefright_headers", + "libstagefright_m4vh263dec", + "libstagefright_m4vh263enc", + "libstagefright_mp3dec", + "libsync", + "libui", + "libui_headers", + "libunwindstack", + "libvorbisidec", + "libvpx", + "libyuv", + "libyuv_static", + "media_ndk_headers", + "media_plugin_headers", + "mediaswcodec", + } + // + // Module separator + // + m["com.android.mediaprovider"] = []string{ + "MediaProvider", + "MediaProviderGoogle", + "fmtlib_ndk", + "libbase_ndk", + "libfuse", + "libfuse_jni", + } + // + // Module separator + // + m["com.android.permission"] = []string{ + "car-ui-lib", + "iconloader", + "kotlin-annotations", + "kotlin-stdlib", + "kotlin-stdlib-jdk7", + "kotlin-stdlib-jdk8", + "kotlinx-coroutines-android", + "kotlinx-coroutines-android-nodeps", + "kotlinx-coroutines-core", + "kotlinx-coroutines-core-nodeps", + "permissioncontroller-statsd", + "GooglePermissionController", + "PermissionController", + "SettingsLibActionBarShadow", + "SettingsLibAppPreference", + "SettingsLibBarChartPreference", + "SettingsLibLayoutPreference", + "SettingsLibProgressBar", + "SettingsLibSearchWidget", + "SettingsLibSettingsTheme", + "SettingsLibRestrictedLockUtils", + "SettingsLibHelpUtils", + } + // + // Module separator + // + m["com.android.runtime"] = []string{ + "bionic_libc_platform_headers", + "libarm-optimized-routines-math", + "libc_aeabi", + "libc_bionic", + "libc_bionic_ndk", + "libc_bootstrap", + "libc_common", + "libc_common_shared", + "libc_common_static", + "libc_dns", + "libc_dynamic_dispatch", + "libc_fortify", + "libc_freebsd", + "libc_freebsd_large_stack", + "libc_gdtoa", + "libc_init_dynamic", + "libc_init_static", + "libc_jemalloc_wrapper", + "libc_netbsd", + "libc_nomalloc", + "libc_nopthread", + "libc_openbsd", + "libc_openbsd_large_stack", + "libc_openbsd_ndk", + "libc_pthread", + "libc_static_dispatch", + "libc_syscalls", + "libc_tzcode", + "libc_unwind_static", + "libdebuggerd", + "libdebuggerd_common_headers", + "libdebuggerd_handler_core", + "libdebuggerd_handler_fallback", + "libdl_static", + "libjemalloc5", + "liblinker_main", + "liblinker_malloc", + "liblz4", + "liblzma", + "libprocinfo", + "libpropertyinfoparser", + "libscudo", + "libstdc++", + "libsystemproperties", + "libtombstoned_client_static", + "libunwindstack", + "libz", + "libziparchive", + } + // + // Module separator + // + m["com.android.tethering"] = []string{ + "android.hardware.tetheroffload.config-V1.0-java", + "android.hardware.tetheroffload.control-V1.0-java", + "android.hidl.base-V1.0-java", + "libcgrouprc", + "libcgrouprc_format", + "libtetherutilsjni", + "libvndksupport", + "net-utils-framework-common", + "netd_aidl_interface-V3-java", + "netlink-client", + "networkstack-aidl-interfaces-java", + "tethering-aidl-interfaces-java", + "TetheringApiCurrentLib", + } + // + // Module separator + // + m["com.android.wifi"] = []string{ + "PlatformProperties", + "android.hardware.wifi-V1.0-java", + "android.hardware.wifi-V1.0-java-constants", + "android.hardware.wifi-V1.1-java", + "android.hardware.wifi-V1.2-java", + "android.hardware.wifi-V1.3-java", + "android.hardware.wifi-V1.4-java", + "android.hardware.wifi.hostapd-V1.0-java", + "android.hardware.wifi.hostapd-V1.1-java", + "android.hardware.wifi.hostapd-V1.2-java", + "android.hardware.wifi.supplicant-V1.0-java", + "android.hardware.wifi.supplicant-V1.1-java", + "android.hardware.wifi.supplicant-V1.2-java", + "android.hardware.wifi.supplicant-V1.3-java", + "android.hidl.base-V1.0-java", + "android.hidl.manager-V1.0-java", + "android.hidl.manager-V1.1-java", + "android.hidl.manager-V1.2-java", + "bouncycastle-unbundled", + "dnsresolver_aidl_interface-V2-java", + "error_prone_annotations", + "framework-wifi-pre-jarjar", + "framework-wifi-util-lib", + "ipmemorystore-aidl-interfaces-V3-java", + "ipmemorystore-aidl-interfaces-java", + "ksoap2", + "libnanohttpd", + "libwifi-jni", + "net-utils-services-common", + "netd_aidl_interface-V2-java", + "netd_aidl_interface-unstable-java", + "netd_event_listener_interface-java", + "netlink-client", + "networkstack-client", + "services.net", + "wifi-lite-protos", + "wifi-nano-protos", + "wifi-service-pre-jarjar", + "wifi-service-resources", + } + // + // Module separator + // + m["com.android.sdkext"] = []string{ + "fmtlib_ndk", + "libbase_ndk", + "libprotobuf-cpp-lite-ndk", + } + // + // Module separator + // + m["com.android.os.statsd"] = []string{ + "libstatssocket", + } + // + // Module separator + // + m[android.AvailableToAnyApex] = []string{ + // TODO(b/156996905) Set apex_available/min_sdk_version for androidx/extras support libraries + "androidx", + "androidx-constraintlayout_constraintlayout", + "androidx-constraintlayout_constraintlayout-nodeps", + "androidx-constraintlayout_constraintlayout-solver", + "androidx-constraintlayout_constraintlayout-solver-nodeps", + "com.google.android.material_material", + "com.google.android.material_material-nodeps", + + "libatomic", + "libclang_rt", + "libgcc_stripped", + "libprofile-clang-extras", + "libprofile-clang-extras_ndk", + "libprofile-extras", + "libprofile-extras_ndk", + "libunwind_llvm", + } + return m +} + +func init() { + android.AddNeverAllowRules(createApexPermittedPackagesRules(qModulesPackages())...) + android.AddNeverAllowRules(createApexPermittedPackagesRules(rModulesPackages())...) +} + +func createApexPermittedPackagesRules(modules_packages map[string][]string) []android.Rule { + rules := make([]android.Rule, 0, len(modules_packages)) + for module_name, module_packages := range modules_packages { + permitted_packages_rule := android.NeverAllow(). + BootclasspathJar(). + With("apex_available", module_name). + WithMatcher("permitted_packages", android.NotInList(module_packages)). + Because("jars that are part of the " + module_name + + " module may only allow these packages: " + strings.Join(module_packages, ",") + + ". Please jarjar or move code around.") + rules = append(rules, permitted_packages_rule) + } + return rules +} + +// DO NOT EDIT! These are the package prefixes that are exempted from being AOT'ed by ART. +// Adding code to the bootclasspath in new packages will cause issues on module update. +func qModulesPackages() map[string][]string { + return map[string][]string{ + "com.android.conscrypt": []string{ + "android.net.ssl", + "com.android.org.conscrypt", + }, + "com.android.media": []string{ + "android.media", + }, + } +} + +// DO NOT EDIT! These are the package prefixes that are exempted from being AOT'ed by ART. +// Adding code to the bootclasspath in new packages will cause issues on module update. +func rModulesPackages() map[string][]string { + return map[string][]string{ + "com.android.mediaprovider": []string{ + "android.provider", + }, + "com.android.permission": []string{ + "android.permission", + "android.app.role", + "com.android.permission", + "com.android.role", + }, + "com.android.sdkext": []string{ + "android.os.ext", + }, + "com.android.os.statsd": []string{ + "android.app", + "android.os", + "android.util", + "com.android.internal.statsd", + "com.android.server.stats", + }, + "com.android.wifi": []string{ + "com.android.server.wifi", + "com.android.wifi.x", + "android.hardware.wifi", + "android.net.wifi", + }, + "com.android.tethering": []string{ + "android.net", + }, + } +} @@ -88,7 +88,7 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel() }) - android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory) + ctx.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory) } type Deps struct { @@ -369,7 +369,7 @@ type ModuleContextIntf interface { useSdk() bool sdkVersion() string useVndk() bool - isNdk() bool + isNdk(config android.Config) bool isLlndk(config android.Config) bool isLlndkPublic(config android.Config) bool isVndkPrivate(config android.Config) bool @@ -939,8 +939,8 @@ func (c *Module) isCoverageVariant() bool { return c.coverage.Properties.IsCoverageVariant } -func (c *Module) IsNdk() bool { - return inList(c.BaseModuleName(), ndkKnownLibs) +func (c *Module) IsNdk(config android.Config) bool { + return inList(c.BaseModuleName(), *getNDKKnownLibs(config)) } func (c *Module) isLlndk(config android.Config) bool { @@ -1145,8 +1145,8 @@ func (ctx *moduleContextImpl) useVndk() bool { return ctx.mod.UseVndk() } -func (ctx *moduleContextImpl) isNdk() bool { - return ctx.mod.IsNdk() +func (ctx *moduleContextImpl) isNdk(config android.Config) bool { + return ctx.mod.IsNdk(config) } func (ctx *moduleContextImpl) isLlndk(config android.Config) bool { @@ -1766,7 +1766,7 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { for _, entry := range list { // strip #version suffix out name, _ := StubsLibNameAndVersion(entry) - if ctx.useSdk() && inList(name, ndkKnownLibs) { + if ctx.useSdk() && inList(name, *getNDKKnownLibs(ctx.Config())) { variantLibs = append(variantLibs, name+ndkLibrarySuffix) } else if ctx.useVndk() { nonvariantLibs = append(nonvariantLibs, rewriteVendorLibs(entry)) diff --git a/cc/compiler.go b/cc/compiler.go index 3c86d208b..04ed80db9 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -687,6 +687,9 @@ type RustBindgenClangProperties struct { // list of shared libraries that provide headers for this binding. Shared_libs []string `android:"arch_variant"` + // List of libraries which export include paths required for this module + Header_libs []string `android:"arch_variant,variant_prepend"` + // list of clang flags required to correctly interpret the headers. Cflags []string `android:"arch_variant"` diff --git a/cc/config/vndk.go b/cc/config/vndk.go index eb3d16fd5..563ce7638 100644 --- a/cc/config/vndk.go +++ b/cc/config/vndk.go @@ -22,6 +22,7 @@ var VndkMustUseVendorVariantList = []string{ "android.hardware.light-ndk_platform", "android.hardware.identity-ndk_platform", "android.hardware.keymint-ndk_platform", + "android.hardware.keymint-unstable-ndk_platform", "android.hardware.nfc@1.2", "android.hardware.power-ndk_platform", "android.hardware.rebootescrow-ndk_platform", @@ -75,7 +75,11 @@ func genYacc(ctx android.ModuleContext, rule *android.RuleBuilder, yaccFile andr cmd := rule.Command() // Fix up #line markers to not use the sbox temporary directory - sedCmd := "sed -i.bak 's#__SBOX_OUT_DIR__#" + outDir.String() + "#'" + // android.SboxPathForOutput(outDir, outDir) returns the sbox placeholder for the out + // directory itself, without any filename appended. + // TODO(ccross): make this cmd.PathForOutput(outDir) instead. + sboxOutDir := android.SboxPathForOutput(outDir, outDir) + sedCmd := "sed -i.bak 's#" + sboxOutDir + "#" + outDir.String() + "#'" rule.Command().Text(sedCmd).Input(outFile) rule.Command().Text(sedCmd).Input(headerFile) diff --git a/cc/genrule_test.go b/cc/genrule_test.go index 0c7952b70..fa0c6f209 100644 --- a/cc/genrule_test.go +++ b/cc/genrule_test.go @@ -66,14 +66,14 @@ func TestArchGenruleCmd(t *testing.T) { gen := ctx.ModuleForTests("gen", "android_arm_armv7-a-neon").Output("out_arm") expected := []string{"foo"} - if !reflect.DeepEqual(expected, gen.Inputs.Strings()) { - t.Errorf(`want arm inputs %v, got %v`, expected, gen.Inputs.Strings()) + if !reflect.DeepEqual(expected, gen.Implicits.Strings()[:len(expected)]) { + t.Errorf(`want arm inputs %v, got %v`, expected, gen.Implicits.Strings()) } gen = ctx.ModuleForTests("gen", "android_arm64_armv8-a").Output("out_arm64") expected = []string{"bar"} - if !reflect.DeepEqual(expected, gen.Inputs.Strings()) { - t.Errorf(`want arm64 inputs %v, got %v`, expected, gen.Inputs.Strings()) + if !reflect.DeepEqual(expected, gen.Implicits.Strings()[:len(expected)]) { + t.Errorf(`want arm64 inputs %v, got %v`, expected, gen.Implicits.Strings()) } } @@ -108,10 +108,10 @@ func TestLibraryGenruleCmd(t *testing.T) { gen := ctx.ModuleForTests("gen", "android_arm_armv7-a-neon").Output("out") expected := []string{"libboth.so", "libshared.so", "libstatic.a"} var got []string - for _, input := range gen.Inputs { + for _, input := range gen.Implicits { got = append(got, input.Base()) } - if !reflect.DeepEqual(expected, got) { + if !reflect.DeepEqual(expected, got[:len(expected)]) { t.Errorf(`want inputs %v, got %v`, expected, got) } } diff --git a/cc/library.go b/cc/library.go index eeddd90d6..2127c08da 100644 --- a/cc/library.go +++ b/cc/library.go @@ -567,7 +567,7 @@ func (library *libraryDecorator) classifySourceAbiDump(ctx ModuleContext) string return "" } // Return NDK if the library is both NDK and LLNDK. - if ctx.isNdk() { + if ctx.isNdk(ctx.Config()) { return "NDK" } if ctx.isLlndkPublic(ctx.Config()) { @@ -1099,7 +1099,7 @@ func (library *libraryDecorator) coverageOutputFilePath() android.OptionalPath { func getRefAbiDumpFile(ctx ModuleContext, vndkVersion, fileName string) android.Path { // The logic must be consistent with classifySourceAbiDump. - isNdk := ctx.isNdk() + isNdk := ctx.isNdk(ctx.Config()) isLlndkOrVndk := ctx.isLlndkPublic(ctx.Config()) || (ctx.useVndk() && ctx.isVndk()) refAbiDumpTextFile := android.PathForVndkRefAbiDump(ctx, vndkVersion, fileName, isNdk, isLlndkOrVndk, false) @@ -1153,7 +1153,7 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objec library.sAbiDiff = SourceAbiDiff(ctx, library.sAbiOutputFile.Path(), refAbiDumpFile, fileName, exportedHeaderFlags, Bool(library.Properties.Header_abi_checker.Check_all_apis), - ctx.isLlndk(ctx.Config()), ctx.isNdk(), ctx.isVndkExt()) + ctx.isLlndk(ctx.Config()), ctx.isNdk(ctx.Config()), ctx.isVndkExt()) } } } diff --git a/cc/makevars.go b/cc/makevars.go index dcfd6d8ef..bd8aab532 100644 --- a/cc/makevars.go +++ b/cc/makevars.go @@ -171,6 +171,7 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.StrictRaw("SRC_HEADERS", strings.Join(includes, " ")) ctx.StrictRaw("SRC_SYSTEM_HEADERS", strings.Join(systemIncludes, " ")) + ndkKnownLibs := *getNDKKnownLibs(ctx.Config()) sort.Strings(ndkKnownLibs) ctx.Strict("NDK_KNOWN_LIBS", strings.Join(ndkKnownLibs, " ")) diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 9097e7bbc..a5c43fe21 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -45,8 +45,7 @@ var ( ndkLibrarySuffix = ".ndk" - // Added as a variation dependency via depsMutator. - ndkKnownLibs = []string{} + ndkKnownLibsKey = android.NewOnceKey("ndkKnownLibsKey") // protects ndkKnownLibs writes during parallel BeginMutator. ndkKnownLibsLock sync.Mutex ) @@ -158,6 +157,12 @@ func (this *stubDecorator) initializeProperties(ctx BaseModuleContext) bool { return true } +func getNDKKnownLibs(config android.Config) *[]string { + return config.Once(ndkKnownLibsKey, func() interface{} { + return &[]string{} + }).(*[]string) +} + func (c *stubDecorator) compilerInit(ctx BaseModuleContext) { c.baseCompiler.compilerInit(ctx) @@ -168,12 +173,13 @@ func (c *stubDecorator) compilerInit(ctx BaseModuleContext) { ndkKnownLibsLock.Lock() defer ndkKnownLibsLock.Unlock() - for _, lib := range ndkKnownLibs { + ndkKnownLibs := getNDKKnownLibs(ctx.Config()) + for _, lib := range *ndkKnownLibs { if lib == name { return } } - ndkKnownLibs = append(ndkKnownLibs, name) + *ndkKnownLibs = append(*ndkKnownLibs, name) } func addStubLibraryCompilerFlags(flags Flags) Flags { diff --git a/cmd/soong_env/soong_env.go b/cmd/soong_env/soong_env.go index d305d8388..8020b17a0 100644 --- a/cmd/soong_env/soong_env.go +++ b/cmd/soong_env/soong_env.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// soong_glob is the command line tool that checks if the list of files matching a glob has -// changed, and only updates the output file list if it has changed. It is used to optimize -// out build.ninja regenerations when non-matching files are added. See -// android/soong/android/glob.go for a longer description. +// soong_env determines if the given soong environment file (usually ".soong.environment") is stale +// by comparing its contents to the current corresponding environment variable values. +// It fails if the file cannot be opened or corrupted, or its contents differ from the current +// values. + package main import ( @@ -34,6 +35,7 @@ func usage() { os.Exit(2) } +// This is a simple executable packaging, and the real work happens in env.StaleEnvFile. func main() { flag.Parse() diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index aee8e5a49..29030d69e 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -125,27 +125,35 @@ func main() { os.Exit(1) } + // Create a terminal output that mimics Ninja's. output := terminal.NewStatusOutput(c.stdio().Stdout(), os.Getenv("NINJA_STATUS"), c.simpleOutput, build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")) + // Attach a new logger instance to the terminal output. log := logger.New(output) defer log.Cleanup() + // Create a context to simplify the program termination process. ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // Create a new trace file writer, making it log events to the log instance. trace := tracer.New(log) defer trace.Close() + // Create and start a new metric record. met := metrics.New() met.SetBuildDateTime(buildStarted) met.SetBuildCommand(os.Args) + // Create a new Status instance, which manages action counts and event output channels. stat := &status.Status{} defer stat.Finish() + // Hook up the terminal output and tracer to Status. stat.AddOutput(output) stat.AddOutput(trace.StatusTracer()) + // Set up a cleanup procedure in case the normal termination process doesn't work. build.SetupSignals(log, cancel, func() { trace.Close() log.Cleanup() @@ -165,6 +173,7 @@ func main() { build.SetupOutDir(buildCtx, config) + // Set up files to be outputted in the log directory. logsDir := config.OutDir() if config.Dist() { logsDir = filepath.Join(config.DistDir(), "logs") @@ -192,7 +201,10 @@ func main() { defer met.Dump(soongMetricsFile) defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) + // Read the time at the starting point. if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { + // soong_ui.bash uses the date command's %N (nanosec) flag when getting the start time, + // which Darwin doesn't support. Check if it was executed properly before parsing the value. if !strings.HasSuffix(start, "N") { if start_time, err := strconv.ParseUint(start, 10, 64); err == nil { log.Verbosef("Took %dms to start up.", @@ -211,6 +223,7 @@ func main() { fixBadDanglingLink(buildCtx, "hardware/qcom/sdm710/Android.bp") fixBadDanglingLink(buildCtx, "hardware/qcom/sdm710/Android.mk") + // Create a source finder. f := build.NewSourceFinder(buildCtx, config) defer f.Shutdown() build.FindSources(buildCtx, config, f) @@ -354,6 +367,8 @@ func stdio() terminal.StdioInterface { return terminal.StdioImpl{} } +// dumpvar and dumpvars use stdout to output variable values, so use stderr instead of stdout when +// reporting events to keep stdout clean from noise. func customStdio() terminal.StdioInterface { return terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr) } @@ -493,16 +508,6 @@ func getCommand(args []string) (*command, []string, error) { if c.flag == args[1] { return &c, args[2:], nil } - - // special case for --make-mode: if soong_ui was called from - // build/make/core/main.mk, the makeparallel with --ninja - // option specified puts the -j<num> before --make-mode. - // TODO: Remove this hack once it has been fixed. - if c.flag == makeModeFlagName { - if inList(makeModeFlagName, args) { - return &c, args[1:], nil - } - } } // command not found diff --git a/cmd/zipsync/zipsync.go b/cmd/zipsync/zipsync.go index 294e5ef0a..aecdc3de4 100644 --- a/cmd/zipsync/zipsync.go +++ b/cmd/zipsync/zipsync.go @@ -53,6 +53,16 @@ func writeFile(filename string, in io.Reader, perm os.FileMode) error { return out.Close() } +func writeSymlink(filename string, in io.Reader) error { + b, err := ioutil.ReadAll(in) + if err != nil { + return err + } + dest := string(b) + err = os.Symlink(dest, filename) + return err +} + func main() { flag.Usage = func() { fmt.Fprintln(os.Stderr, "usage: zipsync -d <output dir> [-l <output file>] [-f <pattern>] [zip]...") @@ -122,7 +132,11 @@ func main() { if err != nil { log.Fatal(err) } - must(writeFile(filename, in, f.FileInfo().Mode())) + if f.FileInfo().Mode()&os.ModeSymlink != 0 { + must(writeSymlink(filename, in)) + } else { + must(writeFile(filename, in, f.FileInfo().Mode())) + } in.Close() files = append(files, filename) } diff --git a/env/env.go b/env/env.go index a98e1f6a8..735a38aa4 100644 --- a/env/env.go +++ b/env/env.go @@ -27,6 +27,15 @@ import ( type envFileEntry struct{ Key, Value string } type envFileData []envFileEntry +// Serializes the given environment variable name/value map into JSON formatted bytes by converting +// to envFileEntry values and marshaling them. +// +// e.g. OUT_DIR = "out" +// is converted to: +// { +// "Key": "OUT_DIR", +// "Value": "out", +// }, func EnvFileContents(envDeps map[string]string) ([]byte, error) { contents := make(envFileData, 0, len(envDeps)) for key, value := range envDeps { @@ -45,8 +54,11 @@ func EnvFileContents(envDeps map[string]string) ([]byte, error) { return data, nil } -func StaleEnvFile(filename string) (bool, error) { - data, err := ioutil.ReadFile(filename) +// Reads and deserializes a Soong environment file located at the given file path to determine its +// staleness. If any environment variable values have changed, it prints them out and returns true. +// Failing to read or parse the file also causes it to return true. +func StaleEnvFile(filepath string) (bool, error) { + data, err := ioutil.ReadFile(filepath) if err != nil { return true, err } @@ -79,6 +91,7 @@ func StaleEnvFile(filename string) (bool, error) { return false, nil } +// Implements sort.Interface so that we can use sort.Sort on envFileData arrays. func (e envFileData) Len() int { return len(e) } @@ -90,3 +103,5 @@ func (e envFileData) Less(i, j int) bool { func (e envFileData) Swap(i, j int) { e[i], e[j] = e[j], e[i] } + +var _ sort.Interface = envFileData{} diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index 44b814954..a6d1fcf40 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -14,9 +14,20 @@ package etc -import ( - "strconv" +// This file implements module types that install prebuilt artifacts. +// +// There exist two classes of prebuilt modules in the Android tree. The first class are the ones +// based on `android.Prebuilt`, such as `cc_prebuilt_library` and `java_import`. This kind of +// modules may exist both as prebuilts and source at the same time, though only one would be +// installed and the other would be marked disabled. The `prebuilt_postdeps` mutator would select +// the actual modules to be installed. More details in android/prebuilt.go. +// +// The second class is described in this file. Unlike `android.Prebuilt` based module types, +// `prebuilt_etc` exist only as prebuilts and cannot have a same-named source module counterpart. +// This makes the logic of `prebuilt_etc` to be much simpler as they don't need to go through the +// various `prebuilt_*` mutators. +import ( "github.com/google/blueprint/proptools" "android/soong/android" @@ -42,7 +53,7 @@ func RegisterPrebuiltEtcBuildComponents(ctx android.RegistrationContext) { } type prebuiltEtcProperties struct { - // Source file of this prebuilt. + // Source file of this prebuilt. Can reference a genrule type module with the ":module" syntax. Src *string `android:"path,arch_variant"` // optional subdirectory under which this file is installed into, cannot be specified with relative_install_path, prefer relative_install_path @@ -209,6 +220,11 @@ func (p *PrebuiltEtc) Installable() bool { func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.sourceFilePath = android.PathForModuleSrc(ctx, android.String(p.properties.Src)) + + // Determine the output file basename. + // If Filename is set, use the name specified by the property. + // If Filename_from_src is set, use the source file name. + // Otherwise use the module name. filename := android.String(p.properties.Filename) filename_from_src := android.Bool(p.properties.Filename_from_src) if filename == "" { @@ -274,11 +290,9 @@ func (p *PrebuiltEtc) AndroidMkEntries() []android.AndroidMkEntries { if len(p.properties.Symlinks) > 0 { entries.AddStrings("LOCAL_MODULE_SYMLINKS", p.properties.Symlinks...) } - entries.SetString("LOCAL_UNINSTALLABLE_MODULE", strconv.FormatBool(!p.Installable())) + entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !p.Installable()) if p.additionalDependencies != nil { - for _, path := range *p.additionalDependencies { - entries.AddStrings("LOCAL_ADDITIONAL_DEPENDENCIES", path.String()) - } + entries.AddStrings("LOCAL_ADDITIONAL_DEPENDENCIES", p.additionalDependencies.Strings()...) } }, }, diff --git a/filesystem/Android.bp b/filesystem/Android.bp new file mode 100644 index 000000000..926df6e38 --- /dev/null +++ b/filesystem/Android.bp @@ -0,0 +1,15 @@ +bootstrap_go_package { + name: "soong-filesystem", + pkgPath: "android/soong/filesystem", + deps: [ + "blueprint", + "soong", + "soong-android", + ], + srcs: [ + "filesystem.go", + ], + testSrcs: [ + ], + pluginFor: ["soong_build"], +} diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go new file mode 100644 index 000000000..a1605b449 --- /dev/null +++ b/filesystem/filesystem.go @@ -0,0 +1,76 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystem + +import ( + "fmt" + + "android/soong/android" +) + +func init() { + android.RegisterModuleType("android_filesystem", filesystemFactory) +} + +type filesystem struct { + android.ModuleBase + android.PackagingBase +} + +func filesystemFactory() android.Module { + module := &filesystem{} + android.InitPackageModule(module) + android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) + return module +} + +func (f *filesystem) DepsMutator(ctx android.BottomUpMutatorContext) { + f.AddDeps(ctx) +} + +var pctx = android.NewPackageContext("android/soong/filesystem") + +func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { + zipFile := android.PathForModuleOut(ctx, "temp.zip").OutputPath + f.CopyDepsToZip(ctx, zipFile) + + rootDir := android.PathForModuleOut(ctx, "root").OutputPath + builder := android.NewRuleBuilder() + builder.Command(). + BuiltTool(ctx, "zipsync"). + FlagWithArg("-d ", rootDir.String()). // zipsync wipes this. No need to clear. + Input(zipFile) + + mkuserimg := ctx.Config().HostToolPath(ctx, "mkuserimg_mke2fs") + propFile := android.PathForModuleOut(ctx, "prop").OutputPath + // TODO(jiyong): support more filesystem types other than ext4 + propsText := fmt.Sprintf(`mount_point=system\n`+ + `fs_type=ext4\n`+ + `use_dynamic_partition_size=true\n`+ + `ext_mkuserimg=%s\n`, mkuserimg.String()) + builder.Command().Text("echo").Flag("-e").Flag(`"` + propsText + `"`). + Text(">").Output(propFile). + Implicit(mkuserimg) + + image := android.PathForModuleOut(ctx, "filesystem.img").OutputPath + builder.Command().BuiltTool(ctx, "build_image"). + Text(rootDir.String()). // input directory + Input(propFile). + Output(image). + Text(rootDir.String()) // directory where to find fs_config_files|dirs + + // rootDir is not deleted. Might be useful for quick inspection. + builder.Build(pctx, ctx, "build_filesystem_image", fmt.Sprintf("Creating filesystem %s", f.BaseModuleName())) +} diff --git a/genrule/genrule.go b/genrule/genrule.go index 53b9dbeb2..f85146c6b 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -17,6 +17,7 @@ package genrule import ( "fmt" "io" + "path/filepath" "strconv" "strings" @@ -25,9 +26,6 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" - "android/soong/shared" - "crypto/sha256" - "path/filepath" ) func init() { @@ -156,14 +154,14 @@ type Module struct { type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask type generateTask struct { - in android.Paths - out android.WritablePaths - copyTo android.WritablePaths - genDir android.WritablePath - sandboxOuts []string - cmd string - shard int - shards int + in android.Paths + out android.WritablePaths + depFile android.WritablePath + copyTo android.WritablePaths + genDir android.WritablePath + cmd string + shard int + shards int } func (g *Module) GeneratedSourceFiles() android.Paths { @@ -330,19 +328,23 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { var zipArgs strings.Builder for _, task := range g.taskGenerator(ctx, String(g.properties.Cmd), srcFiles) { + if len(task.out) == 0 { + ctx.ModuleErrorf("must have at least one output file") + return + } + for _, out := range task.out { - addLocationLabel(out.Rel(), []string{filepath.Join("__SBOX_OUT_DIR__", out.Rel())}) + addLocationLabel(out.Rel(), []string{android.SboxPathForOutput(out, task.genDir)}) } - referencedIn := false referencedDepfile := false - rawCommand, err := android.ExpandNinjaEscaped(task.cmd, func(name string) (string, bool, error) { + rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) { // report the error directly without returning an error to android.Expand to catch multiple errors in a // single run - reportError := func(fmt string, args ...interface{}) (string, bool, error) { + reportError := func(fmt string, args ...interface{}) (string, error) { ctx.PropertyErrorf("cmd", fmt, args...) - return "SOONG_ERROR", false, nil + return "SOONG_ERROR", nil } switch name { @@ -357,20 +359,23 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { return reportError("default label %q has multiple files, use $(locations %s) to reference it", firstLabel, firstLabel) } - return locationLabels[firstLabel][0], false, nil + return locationLabels[firstLabel][0], nil case "in": - referencedIn = true - return "${in}", true, nil + return strings.Join(srcFiles.Strings(), " "), nil case "out": - return "__SBOX_OUT_FILES__", false, nil + var sandboxOuts []string + for _, out := range task.out { + sandboxOuts = append(sandboxOuts, android.SboxPathForOutput(out, task.genDir)) + } + return strings.Join(sandboxOuts, " "), nil case "depfile": referencedDepfile = true if !Bool(g.properties.Depfile) { return reportError("$(depfile) used without depfile property") } - return "__SBOX_DEPFILE__", false, nil + return "__SBOX_DEPFILE__", nil case "genDir": - return "__SBOX_OUT_DIR__", false, nil + return android.SboxPathForOutput(task.genDir, task.genDir), nil default: if strings.HasPrefix(name, "location ") { label := strings.TrimSpace(strings.TrimPrefix(name, "location ")) @@ -381,7 +386,7 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { return reportError("label %q has multiple files, use $(locations %s) to reference it", label, label) } - return paths[0], false, nil + return paths[0], nil } else { return reportError("unknown location label %q", label) } @@ -391,7 +396,7 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { if len(paths) == 0 { return reportError("label %q has no files", label) } - return strings.Join(paths, " "), false, nil + return strings.Join(paths, " "), nil } else { return reportError("unknown locations label %q", label) } @@ -410,50 +415,39 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd") return } - - // tell the sbox command which directory to use as its sandbox root - buildDir := android.PathForOutput(ctx).String() - sandboxPath := shared.TempDirForOutDir(buildDir) - - // recall that Sprintf replaces percent sign expressions, whereas dollar signs expressions remain as written, - // to be replaced later by ninja_strings.go - depfilePlaceholder := "" - if Bool(g.properties.Depfile) { - depfilePlaceholder = "$depfileArgs" - } - - // Escape the command for the shell - rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'" g.rawCommands = append(g.rawCommands, rawCommand) - sandboxCommand := fmt.Sprintf("rm -rf %s && $sboxCmd --sandbox-path %s --output-root %s", - task.genDir, sandboxPath, task.genDir) - - if !referencedIn { - sandboxCommand = sandboxCommand + hashSrcFiles(srcFiles) + // Pick a unique rule name and the user-visible description. + desc := "generate" + name := "generator" + if task.shards > 0 { + desc += " " + strconv.Itoa(task.shard) + name += strconv.Itoa(task.shard) + } else if len(task.out) == 1 { + desc += " " + task.out[0].Base() } - sandboxCommand = sandboxCommand + fmt.Sprintf(" -c %s %s $allouts", - rawCommand, depfilePlaceholder) - - ruleParams := blueprint.RuleParams{ - Command: sandboxCommand, - CommandDeps: []string{"$sboxCmd"}, - } - args := []string{"allouts"} + // Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox. + rule := android.NewRuleBuilder().Sbox(task.genDir) + cmd := rule.Command() + cmd.Text(rawCommand) + cmd.ImplicitOutputs(task.out) + cmd.Implicits(task.in) + cmd.Implicits(g.deps) if Bool(g.properties.Depfile) { - ruleParams.Deps = blueprint.DepsGCC - args = append(args, "depfileArgs") + cmd.ImplicitDepFile(task.depFile) } - name := "generator" - if task.shards > 1 { - name += strconv.Itoa(task.shard) - } - rule := ctx.Rule(pctx, name, ruleParams, args...) - g.generateSourceFile(ctx, task, rule) + // Create the rule to run the genrule command inside sbox. + rule.Build(pctx, ctx, name, desc) if len(task.copyTo) > 0 { + // If copyTo is set, multiple shards need to be copied into a single directory. + // task.out contains the per-shard paths, and copyTo contains the corresponding + // final path. The files need to be copied into the final directory by a + // single rule so it can remove the directory before it starts to ensure no + // old files remain. zipsync already does this, so build up zipArgs that + // zip all the per-shard directories into a single zip. outputFiles = append(outputFiles, task.copyTo...) copyFrom = append(copyFrom, task.out.Paths()...) zipArgs.WriteString(" -C " + task.genDir.String()) @@ -464,6 +458,8 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { } if len(copyFrom) > 0 { + // Create a rule that zips all the per-shard directories into a single zip and then + // uses zipsync to unzip it into the final directory. ctx.Build(pctx, android.BuildParams{ Rule: gensrcsMerge, Implicits: copyFrom, @@ -501,51 +497,6 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } } -func hashSrcFiles(srcFiles android.Paths) string { - h := sha256.New() - for _, src := range srcFiles { - h.Write([]byte(src.String())) - } - return fmt.Sprintf(" --input-hash %x", h.Sum(nil)) -} - -func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask, rule blueprint.Rule) { - desc := "generate" - if len(task.out) == 0 { - ctx.ModuleErrorf("must have at least one output file") - return - } - if len(task.out) == 1 { - desc += " " + task.out[0].Base() - } - - var depFile android.ModuleGenPath - if Bool(g.properties.Depfile) { - depFile = android.PathForModuleGen(ctx, task.out[0].Rel()+".d") - } - - if task.shards > 1 { - desc += " " + strconv.Itoa(task.shard) - } - - params := android.BuildParams{ - Rule: rule, - Description: desc, - Output: task.out[0], - ImplicitOutputs: task.out[1:], - Inputs: task.in, - Implicits: g.deps, - Args: map[string]string{ - "allouts": strings.Join(task.sandboxOuts, " "), - }, - } - if Bool(g.properties.Depfile) { - params.Depfile = android.PathForModuleGen(ctx, task.out[0].Rel()+".d") - params.Args["depfileArgs"] = "--depfile-out " + depFile.String() - } - - ctx.Build(pctx, params) -} // Collect information for opening IDE project files in java/jdeps.go. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) { @@ -610,16 +561,6 @@ func (x noopImageInterface) ExtraImageVariations(ctx android.BaseModuleContext) func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, variation string, module android.Module) { } -// replace "out" with "__SBOX_OUT_DIR__/<the value of ${out}>" -func pathToSandboxOut(path android.Path, genDir android.Path) string { - relOut, err := filepath.Rel(genDir.String(), path.String()) - if err != nil { - panic(fmt.Sprintf("Could not make ${out} relative: %v", err)) - } - return filepath.Join("__SBOX_OUT_DIR__", relOut) - -} - func NewGenSrcs() *Module { properties := &genSrcsProperties{} @@ -638,7 +579,7 @@ func NewGenSrcs() *Module { var outFiles android.WritablePaths var copyTo android.WritablePaths var shardDir android.WritablePath - var sandboxOuts []string + var depFile android.WritablePath if len(shards) > 1 { shardDir = android.PathForModuleGen(ctx, strconv.Itoa(i)) @@ -646,9 +587,11 @@ func NewGenSrcs() *Module { shardDir = genDir } - for _, in := range shard { + for j, in := range shard { outFile := android.GenPathWithExt(ctx, "gensrcs", in, String(properties.Output_extension)) - sandboxOutfile := pathToSandboxOut(outFile, genDir) + if j == 0 { + depFile = outFile.ReplaceExtension(ctx, "d") + } if len(shards) > 1 { shardFile := android.GenPathWithExt(ctx, strconv.Itoa(i), in, String(properties.Output_extension)) @@ -657,14 +600,13 @@ func NewGenSrcs() *Module { } outFiles = append(outFiles, outFile) - sandboxOuts = append(sandboxOuts, sandboxOutfile) command, err := android.Expand(rawCommand, func(name string) (string, error) { switch name { case "in": return in.String(), nil case "out": - return sandboxOutfile, nil + return android.SboxPathForOutput(outFile, shardDir), nil default: return "$(" + name + ")", nil } @@ -680,14 +622,14 @@ func NewGenSrcs() *Module { fullCommand := strings.Join(commands, " && ") generateTasks = append(generateTasks, generateTask{ - in: shard, - out: outFiles, - copyTo: copyTo, - genDir: shardDir, - sandboxOuts: sandboxOuts, - cmd: fullCommand, - shard: i, - shards: len(shards), + in: shard, + out: outFiles, + depFile: depFile, + copyTo: copyTo, + genDir: shardDir, + cmd: fullCommand, + shard: i, + shards: len(shards), }) } @@ -720,18 +662,20 @@ func NewGenRule() *Module { taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask { outs := make(android.WritablePaths, len(properties.Out)) - sandboxOuts := make([]string, len(properties.Out)) - genDir := android.PathForModuleGen(ctx) + var depFile android.WritablePath for i, out := range properties.Out { - outs[i] = android.PathForModuleGen(ctx, out) - sandboxOuts[i] = pathToSandboxOut(outs[i], genDir) + outPath := android.PathForModuleGen(ctx, out) + if i == 0 { + depFile = outPath.ReplaceExtension(ctx, "d") + } + outs[i] = outPath } return []generateTask{{ - in: srcFiles, - out: outs, - genDir: android.PathForModuleGen(ctx), - sandboxOuts: sandboxOuts, - cmd: rawCommand, + in: srcFiles, + out: outs, + depFile: depFile, + genDir: android.PathForModuleGen(ctx), + cmd: rawCommand, }} } diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index c19078f0d..6779c73e4 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -141,7 +141,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location) > $(out)", `, - expect: "out/tool > __SBOX_OUT_FILES__", + expect: "out/tool > __SBOX_OUT_DIR__/out", }, { name: "empty location tool2", @@ -150,7 +150,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location) > $(out)", `, - expect: "out/tool > __SBOX_OUT_FILES__", + expect: "out/tool > __SBOX_OUT_DIR__/out", }, { name: "empty location tool file", @@ -159,7 +159,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location) > $(out)", `, - expect: "tool_file1 > __SBOX_OUT_FILES__", + expect: "tool_file1 > __SBOX_OUT_DIR__/out", }, { name: "empty location tool file fg", @@ -168,7 +168,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location) > $(out)", `, - expect: "tool_file1 > __SBOX_OUT_FILES__", + expect: "tool_file1 > __SBOX_OUT_DIR__/out", }, { name: "empty location tool and tool file", @@ -178,7 +178,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location) > $(out)", `, - expect: "out/tool > __SBOX_OUT_FILES__", + expect: "out/tool > __SBOX_OUT_DIR__/out", }, { name: "tool", @@ -187,7 +187,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location tool) > $(out)", `, - expect: "out/tool > __SBOX_OUT_FILES__", + expect: "out/tool > __SBOX_OUT_DIR__/out", }, { name: "tool2", @@ -196,7 +196,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location :tool) > $(out)", `, - expect: "out/tool > __SBOX_OUT_FILES__", + expect: "out/tool > __SBOX_OUT_DIR__/out", }, { name: "tool file", @@ -205,7 +205,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location tool_file1) > $(out)", `, - expect: "tool_file1 > __SBOX_OUT_FILES__", + expect: "tool_file1 > __SBOX_OUT_DIR__/out", }, { name: "tool file fg", @@ -214,7 +214,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(location :1tool_file) > $(out)", `, - expect: "tool_file1 > __SBOX_OUT_FILES__", + expect: "tool_file1 > __SBOX_OUT_DIR__/out", }, { name: "tool files", @@ -223,7 +223,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "$(locations :tool_files) > $(out)", `, - expect: "tool_file1 tool_file2 > __SBOX_OUT_FILES__", + expect: "tool_file1 tool_file2 > __SBOX_OUT_DIR__/out", }, { name: "in1", @@ -232,7 +232,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(in) > $(out)", `, - expect: "cat ${in} > __SBOX_OUT_FILES__", + expect: "cat in1 > __SBOX_OUT_DIR__/out", }, { name: "in1 fg", @@ -241,7 +241,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(in) > $(out)", `, - expect: "cat ${in} > __SBOX_OUT_FILES__", + expect: "cat in1 > __SBOX_OUT_DIR__/out", }, { name: "ins", @@ -250,7 +250,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(in) > $(out)", `, - expect: "cat ${in} > __SBOX_OUT_FILES__", + expect: "cat in1 in2 > __SBOX_OUT_DIR__/out", }, { name: "ins fg", @@ -259,7 +259,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(in) > $(out)", `, - expect: "cat ${in} > __SBOX_OUT_FILES__", + expect: "cat in1 in2 > __SBOX_OUT_DIR__/out", }, { name: "location in1", @@ -268,7 +268,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(location in1) > $(out)", `, - expect: "cat in1 > __SBOX_OUT_FILES__", + expect: "cat in1 > __SBOX_OUT_DIR__/out", }, { name: "location in1 fg", @@ -277,7 +277,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(location :1in) > $(out)", `, - expect: "cat in1 > __SBOX_OUT_FILES__", + expect: "cat in1 > __SBOX_OUT_DIR__/out", }, { name: "location ins", @@ -286,7 +286,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(location in1) > $(out)", `, - expect: "cat in1 > __SBOX_OUT_FILES__", + expect: "cat in1 > __SBOX_OUT_DIR__/out", }, { name: "location ins fg", @@ -295,7 +295,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "cat $(locations :ins) > $(out)", `, - expect: "cat in1 in2 > __SBOX_OUT_FILES__", + expect: "cat in1 in2 > __SBOX_OUT_DIR__/out", }, { name: "outs", @@ -303,7 +303,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out", "out2"], cmd: "echo foo > $(out)", `, - expect: "echo foo > __SBOX_OUT_FILES__", + expect: "echo foo > __SBOX_OUT_DIR__/out __SBOX_OUT_DIR__/out2", }, { name: "location out", @@ -320,7 +320,7 @@ func TestGenruleCmd(t *testing.T) { depfile: true, cmd: "echo foo > $(out) && touch $(depfile)", `, - expect: "echo foo > __SBOX_OUT_FILES__ && touch __SBOX_DEPFILE__", + expect: "echo foo > __SBOX_OUT_DIR__/out && touch __SBOX_DEPFILE__", }, { name: "gendir", @@ -328,7 +328,7 @@ func TestGenruleCmd(t *testing.T) { out: ["out"], cmd: "echo foo > $(genDir)/foo && cp $(genDir)/foo $(out)", `, - expect: "echo foo > __SBOX_OUT_DIR__/foo && cp __SBOX_OUT_DIR__/foo __SBOX_OUT_FILES__", + expect: "echo foo > __SBOX_OUT_DIR__/foo && cp __SBOX_OUT_DIR__/foo __SBOX_OUT_DIR__/out", }, { @@ -443,7 +443,7 @@ func TestGenruleCmd(t *testing.T) { allowMissingDependencies: true, - expect: "cat ***missing srcs :missing*** > __SBOX_OUT_FILES__", + expect: "cat ***missing srcs :missing*** > __SBOX_OUT_DIR__/out", }, { name: "tool allow missing dependencies", @@ -455,7 +455,7 @@ func TestGenruleCmd(t *testing.T) { allowMissingDependencies: true, - expect: "***missing tool :missing*** > __SBOX_OUT_FILES__", + expect: "***missing tool :missing*** > __SBOX_OUT_DIR__/out", }, } @@ -495,7 +495,7 @@ func TestGenruleCmd(t *testing.T) { } gen := ctx.ModuleForTests("gen", "").Module().(*Module) - if g, w := gen.rawCommands[0], "'"+test.expect+"'"; w != g { + if g, w := gen.rawCommands[0], test.expect; w != g { t.Errorf("want %q, got %q", w, g) } }) @@ -542,18 +542,18 @@ func TestGenruleHashInputs(t *testing.T) { }{ { name: "hash0", - // sha256 value obtained from: echo -n 'in1.txtin2.txt' | sha256sum - expectedHash: "031097e11e0a8c822c960eb9742474f46336360a515744000d086d94335a9cb9", + // sha256 value obtained from: echo -en 'in1.txt\nin2.txt' | sha256sum + expectedHash: "18da75b9b1cc74b09e365b4ca2e321b5d618f438cc632b387ad9dc2ab4b20e9d", }, { name: "hash1", - // sha256 value obtained from: echo -n 'in1.txtin2.txtin3.txt' | sha256sum - expectedHash: "de5d22a4a7ab50d250cc59fcdf7a7e0775790d270bfca3a7a9e1f18a70dd996c", + // sha256 value obtained from: echo -en 'in1.txt\nin2.txt\nin3.txt' | sha256sum + expectedHash: "a38d432a4b19df93140e1f1fe26c97ff0387dae01fe506412b47208f0595fb45", }, { name: "hash2", - // $(in) is present, option should not appear - expectedHash: "", + // sha256 value obtained from: echo -en 'in1.txt\nin2.txt\nin3.txt' | sha256sum + expectedHash: "a38d432a4b19df93140e1f1fe26c97ff0387dae01fe506412b47208f0595fb45", }, } @@ -575,7 +575,7 @@ func TestGenruleHashInputs(t *testing.T) { if len(test.expectedHash) > 0 { // We add spaces before and after to make sure that // this option doesn't abutt another sbox option. - expectedInputHashOption := " --input-hash " + test.expectedHash + " " + expectedInputHashOption := " --input-hash " + test.expectedHash if !strings.Contains(command, expectedInputHashOption) { t.Errorf("Expected command \"%s\" to contain \"%s\"", command, expectedInputHashOption) @@ -609,7 +609,7 @@ func TestGenSrcs(t *testing.T) { cmd: "$(location) $(in) > $(out)", `, cmds: []string{ - "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''", + "bash -c 'out/tool in1.txt > __SBOX_OUT_DIR__/in1.h' && bash -c 'out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'", }, deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"}, files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"}, @@ -623,8 +623,8 @@ func TestGenSrcs(t *testing.T) { shard_size: 2, `, cmds: []string{ - "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''", - "'bash -c '\\''out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'\\'''", + "bash -c 'out/tool in1.txt > __SBOX_OUT_DIR__/in1.h' && bash -c 'out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'", + "bash -c 'out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'", }, deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"}, files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"}, @@ -710,7 +710,7 @@ func TestGenruleDefaults(t *testing.T) { } gen := ctx.ModuleForTests("gen", "").Module().(*Module) - expectedCmd := "'cp ${in} __SBOX_OUT_FILES__'" + expectedCmd := "cp in1 __SBOX_OUT_DIR__/out" if gen.rawCommands[0] != expectedCmd { t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommands[0]) } diff --git a/java/app.go b/java/app.go index 17de8b9a5..3446739f3 100755 --- a/java/app.go +++ b/java/app.go @@ -890,7 +890,7 @@ func collectAppDeps(ctx android.ModuleContext, app appDepsInterface, if IsJniDepTag(tag) || cc.IsSharedDepTag(tag) { if dep, ok := module.(*cc.Module); ok { - if dep.IsNdk() || dep.IsStubs() { + if dep.IsNdk(ctx.Config()) || dep.IsStubs() { return false } diff --git a/java/app_test.go b/java/app_test.go index 6429ab836..6efb0dcb3 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -291,7 +291,7 @@ func TestAndroidAppLinkType(t *testing.T) { } `) - testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + testJavaError(t, "consider adjusting sdk_version: OR platform_apis:", ` android_app { name: "foo", srcs: ["a.java"], @@ -335,7 +335,7 @@ func TestAndroidAppLinkType(t *testing.T) { } `) - testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + testJavaError(t, "consider adjusting sdk_version: OR platform_apis:", ` android_app { name: "foo", srcs: ["a.java"], diff --git a/java/droiddoc.go b/java/droiddoc.go index c7a27c2a4..cf7f4fb5d 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -1748,8 +1748,6 @@ type PrebuiltStubsSources struct { properties PrebuiltStubsSourcesProperties - // The source directories containing stubs source files. - srcDirs android.Paths stubsSrcJar android.ModuleOutPath } @@ -1769,21 +1767,29 @@ func (d *PrebuiltStubsSources) StubsSrcJar() android.Path { func (p *PrebuiltStubsSources) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") - p.srcDirs = android.PathsForModuleSrc(ctx, p.properties.Srcs) + if len(p.properties.Srcs) != 1 { + ctx.PropertyErrorf("srcs", "must only specify one directory path, contains %d paths", len(p.properties.Srcs)) + return + } + + localSrcDir := p.properties.Srcs[0] + // Although PathForModuleSrc can return nil if either the path doesn't exist or + // the path components are invalid it won't in this case because no components + // are specified and the module directory must exist in order to get this far. + srcDir := android.PathForModuleSrc(ctx).(android.SourcePath).Join(ctx, localSrcDir) + + // Glob the contents of the directory just in case the directory does not exist. + srcGlob := localSrcDir + "/**/*" + srcPaths := android.PathsForModuleSrc(ctx, []string{srcGlob}) rule := android.NewRuleBuilder() - command := rule.Command(). + rule.Command(). BuiltTool(ctx, "soong_zip"). Flag("-write_if_changed"). Flag("-jar"). - FlagWithOutput("-o ", p.stubsSrcJar) - - for _, d := range p.srcDirs { - dir := d.String() - command. - FlagWithArg("-C ", dir). - FlagWithInput("-D ", d) - } + FlagWithOutput("-o ", p.stubsSrcJar). + FlagWithArg("-C ", srcDir.String()). + FlagWithRspFileInputList("-r ", srcPaths) rule.Restat() diff --git a/java/java.go b/java/java.go index a23c649c1..7cf04fa35 100644 --- a/java/java.go +++ b/java/java.go @@ -248,6 +248,9 @@ type CompilerProperties struct { Errorprone struct { // List of javac flags that should only be used when running errorprone. Javacflags []string + + // List of java_plugin modules that provide extra errorprone checks. + Extra_check_modules []string } Proto struct { @@ -569,6 +572,7 @@ var ( libTag = dependencyTag{name: "javalib"} java9LibTag = dependencyTag{name: "java9lib"} pluginTag = dependencyTag{name: "plugin"} + errorpronePluginTag = dependencyTag{name: "errorprone-plugin"} exportedPluginTag = dependencyTag{name: "exported-plugin"} bootClasspathTag = dependencyTag{name: "bootclasspath"} systemModulesTag = dependencyTag{name: "system modules"} @@ -765,6 +769,7 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { } ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), pluginTag, j.properties.Plugins...) + ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), errorpronePluginTag, j.properties.Errorprone.Extra_check_modules...) ctx.AddFarVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), exportedPluginTag, j.properties.Exported_plugins...) android.ProtoDeps(ctx, &j.protoProperties) @@ -852,21 +857,22 @@ func (j *Module) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.Opt } type deps struct { - classpath classpath - java9Classpath classpath - bootClasspath classpath - processorPath classpath - processorClasses []string - staticJars android.Paths - staticHeaderJars android.Paths - staticResourceJars android.Paths - aidlIncludeDirs android.Paths - srcs android.Paths - srcJars android.Paths - systemModules *systemModules - aidlPreprocess android.OptionalPath - kotlinStdlib android.Paths - kotlinAnnotations android.Paths + classpath classpath + java9Classpath classpath + bootClasspath classpath + processorPath classpath + errorProneProcessorPath classpath + processorClasses []string + staticJars android.Paths + staticHeaderJars android.Paths + staticResourceJars android.Paths + aidlIncludeDirs android.Paths + srcs android.Paths + srcJars android.Paths + systemModules *systemModules + aidlPreprocess android.OptionalPath + kotlinStdlib android.Paths + kotlinAnnotations android.Paths disableTurbine bool } @@ -952,7 +958,9 @@ func checkLinkType(ctx android.ModuleContext, from *Module, to linkTypeContext, return } otherLinkType, _ := to.getLinkType(ctx.OtherModuleName(to)) - commonMessage := "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source." + commonMessage := " In order to fix this, consider adjusting sdk_version: OR platform_apis: " + + "property of the source or target module so that target module is built with the same " + + "or smaller API set when compared to the source." switch myLinkType { case javaCore: @@ -1066,6 +1074,12 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { } else { ctx.PropertyErrorf("plugins", "%q is not a java_plugin module", otherName) } + case errorpronePluginTag: + if plugin, ok := dep.(*Plugin); ok { + deps.errorProneProcessorPath = append(deps.errorProneProcessorPath, plugin.ImplementationAndResourcesJars()...) + } else { + ctx.PropertyErrorf("plugins", "%q is not a java_plugin module", otherName) + } case exportedPluginTag: if plugin, ok := dep.(*Plugin); ok { if plugin.pluginProperties.Generates_api != nil && *plugin.pluginProperties.Generates_api { @@ -1189,7 +1203,7 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB flags.javaVersion = getJavaVersion(ctx, String(j.properties.Java_version), sdkContext(j)) if ctx.Config().RunErrorProne() { - if config.ErrorProneClasspath == nil { + if config.ErrorProneClasspath == nil && ctx.Config().TestProductVariables == nil { ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?") } @@ -1209,6 +1223,7 @@ func (j *Module) collectBuilderFlags(ctx android.ModuleContext, deps deps) javaB flags.classpath = append(flags.classpath, deps.classpath...) flags.java9Classpath = append(flags.java9Classpath, deps.java9Classpath...) flags.processorPath = append(flags.processorPath, deps.processorPath...) + flags.errorProneProcessorPath = append(flags.errorProneProcessorPath, deps.errorProneProcessorPath...) flags.processors = append(flags.processors, deps.processorClasses...) flags.processors = android.FirstUniqueStrings(flags.processors) @@ -2325,7 +2340,7 @@ func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.testProperties.Test_config, j.testProperties.Test_config_template, - j.testProperties.Test_suites, j.testProperties.Auto_gen_config) + j.testProperties.Test_suites, j.testProperties.Auto_gen_config, j.testProperties.Test_options.Unit_test) j.data = android.PathsForModuleSrc(ctx, j.testProperties.Data) @@ -2344,7 +2359,7 @@ func (j *TestHelperLibrary) GenerateAndroidBuildActions(ctx android.ModuleContex func (j *JavaTestImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.testConfig = tradefed.AutoGenJavaTestConfig(ctx, j.prebuiltTestProperties.Test_config, nil, - j.prebuiltTestProperties.Test_suites, nil) + j.prebuiltTestProperties.Test_suites, nil, nil) j.Import.GenerateAndroidBuildActions(ctx) } diff --git a/java/java_test.go b/java/java_test.go index 87d6ebbd4..cf56e66da 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -15,6 +15,7 @@ package java import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -92,8 +93,8 @@ func testContext(config android.Config) *android.TestContext { ctx.PreDepsMutators(python.RegisterPythonPreDepsMutators) ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators) - ctx.RegisterPreSingletonType("overlay", android.SingletonFactoryAdaptor(OverlaySingletonFactory)) - ctx.RegisterPreSingletonType("sdk_versions", android.SingletonFactoryAdaptor(sdkPreSingletonFactory)) + ctx.RegisterPreSingletonType("overlay", android.SingletonFactoryAdaptor(ctx.Context, OverlaySingletonFactory)) + ctx.RegisterPreSingletonType("sdk_versions", android.SingletonFactoryAdaptor(ctx.Context, sdkPreSingletonFactory)) android.RegisterPrebuiltMutators(ctx) @@ -202,7 +203,7 @@ func TestJavaLinkType(t *testing.T) { } `) - testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + testJavaError(t, "consider adjusting sdk_version: OR platform_apis:", ` java_library { name: "foo", srcs: ["a.java"], @@ -246,7 +247,7 @@ func TestJavaLinkType(t *testing.T) { } `) - testJavaError(t, "Adjust sdk_version: property of the source or target module so that target module is built with the same or smaller API set than the source.", ` + testJavaError(t, "consider adjusting sdk_version: OR platform_apis:", ` java_library { name: "foo", srcs: ["a.java"], @@ -622,6 +623,35 @@ func assertDeepEquals(t *testing.T, message string, expected interface{}, actual } } +func TestPrebuiltStubsSources(t *testing.T) { + test := func(t *testing.T, sourcesPath string, expectedInputs []string) { + ctx, _ := testJavaWithFS(t, fmt.Sprintf(` +prebuilt_stubs_sources { + name: "stubs-source", + srcs: ["%s"], +}`, sourcesPath), map[string][]byte{ + "stubs/sources/pkg/A.java": nil, + "stubs/sources/pkg/B.java": nil, + }) + + zipSrc := ctx.ModuleForTests("stubs-source", "android_common").Rule("zip_src") + if expected, actual := expectedInputs, zipSrc.Inputs.Strings(); !reflect.DeepEqual(expected, actual) { + t.Errorf("mismatch of inputs to soong_zip: expected %q, actual %q", expected, actual) + } + } + + t.Run("empty/missing directory", func(t *testing.T) { + test(t, "empty-directory", []string{}) + }) + + t.Run("non-empty set of sources", func(t *testing.T) { + test(t, "stubs/sources", []string{ + "stubs/sources/pkg/A.java", + "stubs/sources/pkg/B.java", + }) + }) +} + func TestJavaSdkLibraryImport(t *testing.T) { ctx, _ := testJava(t, ` java_library { @@ -1379,8 +1409,8 @@ func TestJarGenrules(t *testing.T) { baz := ctx.ModuleForTests("baz", "android_common").Output("javac/baz.jar") barCombined := ctx.ModuleForTests("bar", "android_common").Output("combined/bar.jar") - if len(jargen.Inputs) != 1 || jargen.Inputs[0].String() != foo.Output.String() { - t.Errorf("expected jargen inputs [%q], got %q", foo.Output.String(), jargen.Inputs.Strings()) + if g, w := jargen.Implicits.Strings(), foo.Output.String(); !android.InList(w, g) { + t.Errorf("expected jargen inputs [%q], got %q", w, g) } if !strings.Contains(bar.Args["classpath"], jargen.Output.String()) { diff --git a/java/kotlin_test.go b/java/kotlin_test.go index 60ca1c476..77ef29456 100644 --- a/java/kotlin_test.go +++ b/java/kotlin_test.go @@ -84,11 +84,14 @@ func TestKotlin(t *testing.T) { } func TestKapt(t *testing.T) { - ctx, _ := testJava(t, ` + bp := ` java_library { name: "foo", srcs: ["a.java", "b.kt"], plugins: ["bar", "baz"], + errorprone: { + extra_check_modules: ["my_check"], + }, } java_plugin { @@ -102,64 +105,119 @@ func TestKapt(t *testing.T) { processor_class: "com.baz", srcs: ["b.java"], } - `) - buildOS := android.BuildOs.String() + java_plugin { + name: "my_check", + srcs: ["b.java"], + } + ` + t.Run("", func(t *testing.T) { + ctx, _ := testJava(t, bp) - kapt := ctx.ModuleForTests("foo", "android_common").Rule("kapt") - kotlinc := ctx.ModuleForTests("foo", "android_common").Rule("kotlinc") - javac := ctx.ModuleForTests("foo", "android_common").Rule("javac") + buildOS := android.BuildOs.String() - bar := ctx.ModuleForTests("bar", buildOS+"_common").Rule("javac").Output.String() - baz := ctx.ModuleForTests("baz", buildOS+"_common").Rule("javac").Output.String() + kapt := ctx.ModuleForTests("foo", "android_common").Rule("kapt") + kotlinc := ctx.ModuleForTests("foo", "android_common").Rule("kotlinc") + javac := ctx.ModuleForTests("foo", "android_common").Rule("javac") - // Test that the kotlin and java sources are passed to kapt and kotlinc - if len(kapt.Inputs) != 2 || kapt.Inputs[0].String() != "a.java" || kapt.Inputs[1].String() != "b.kt" { - t.Errorf(`foo kapt inputs %v != ["a.java", "b.kt"]`, kapt.Inputs) - } - if len(kotlinc.Inputs) != 2 || kotlinc.Inputs[0].String() != "a.java" || kotlinc.Inputs[1].String() != "b.kt" { - t.Errorf(`foo kotlinc inputs %v != ["a.java", "b.kt"]`, kotlinc.Inputs) - } + bar := ctx.ModuleForTests("bar", buildOS+"_common").Rule("javac").Output.String() + baz := ctx.ModuleForTests("baz", buildOS+"_common").Rule("javac").Output.String() - // Test that only the java sources are passed to javac - if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" { - t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs) - } + // Test that the kotlin and java sources are passed to kapt and kotlinc + if len(kapt.Inputs) != 2 || kapt.Inputs[0].String() != "a.java" || kapt.Inputs[1].String() != "b.kt" { + t.Errorf(`foo kapt inputs %v != ["a.java", "b.kt"]`, kapt.Inputs) + } + if len(kotlinc.Inputs) != 2 || kotlinc.Inputs[0].String() != "a.java" || kotlinc.Inputs[1].String() != "b.kt" { + t.Errorf(`foo kotlinc inputs %v != ["a.java", "b.kt"]`, kotlinc.Inputs) + } - // Test that the kapt srcjar is a dependency of kotlinc and javac rules - if !inList(kapt.Output.String(), kotlinc.Implicits.Strings()) { - t.Errorf("expected %q in kotlinc implicits %v", kapt.Output.String(), kotlinc.Implicits.Strings()) - } - if !inList(kapt.Output.String(), javac.Implicits.Strings()) { - t.Errorf("expected %q in javac implicits %v", kapt.Output.String(), javac.Implicits.Strings()) - } + // Test that only the java sources are passed to javac + if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" { + t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs) + } - // Test that the kapt srcjar is extracted by the kotlinc and javac rules - if kotlinc.Args["srcJars"] != kapt.Output.String() { - t.Errorf("expected %q in kotlinc srcjars %v", kapt.Output.String(), kotlinc.Args["srcJars"]) - } - if javac.Args["srcJars"] != kapt.Output.String() { - t.Errorf("expected %q in javac srcjars %v", kapt.Output.String(), kotlinc.Args["srcJars"]) - } + // Test that the kapt srcjar is a dependency of kotlinc and javac rules + if !inList(kapt.Output.String(), kotlinc.Implicits.Strings()) { + t.Errorf("expected %q in kotlinc implicits %v", kapt.Output.String(), kotlinc.Implicits.Strings()) + } + if !inList(kapt.Output.String(), javac.Implicits.Strings()) { + t.Errorf("expected %q in javac implicits %v", kapt.Output.String(), javac.Implicits.Strings()) + } - // Test that the processors are passed to kapt - expectedProcessorPath := "-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + bar + - " -P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + baz - if kapt.Args["kaptProcessorPath"] != expectedProcessorPath { - t.Errorf("expected kaptProcessorPath %q, got %q", expectedProcessorPath, kapt.Args["kaptProcessorPath"]) - } - expectedProcessor := "-P plugin:org.jetbrains.kotlin.kapt3:processors=com.bar -P plugin:org.jetbrains.kotlin.kapt3:processors=com.baz" - if kapt.Args["kaptProcessor"] != expectedProcessor { - t.Errorf("expected kaptProcessor %q, got %q", expectedProcessor, kapt.Args["kaptProcessor"]) - } + // Test that the kapt srcjar is extracted by the kotlinc and javac rules + if kotlinc.Args["srcJars"] != kapt.Output.String() { + t.Errorf("expected %q in kotlinc srcjars %v", kapt.Output.String(), kotlinc.Args["srcJars"]) + } + if javac.Args["srcJars"] != kapt.Output.String() { + t.Errorf("expected %q in javac srcjars %v", kapt.Output.String(), kotlinc.Args["srcJars"]) + } - // Test that the processors are not passed to javac - if javac.Args["processorPath"] != "" { - t.Errorf("expected processorPath '', got %q", javac.Args["processorPath"]) - } - if javac.Args["processor"] != "-proc:none" { - t.Errorf("expected processor '-proc:none', got %q", javac.Args["processor"]) - } + // Test that the processors are passed to kapt + expectedProcessorPath := "-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + bar + + " -P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + baz + if kapt.Args["kaptProcessorPath"] != expectedProcessorPath { + t.Errorf("expected kaptProcessorPath %q, got %q", expectedProcessorPath, kapt.Args["kaptProcessorPath"]) + } + expectedProcessor := "-P plugin:org.jetbrains.kotlin.kapt3:processors=com.bar -P plugin:org.jetbrains.kotlin.kapt3:processors=com.baz" + if kapt.Args["kaptProcessor"] != expectedProcessor { + t.Errorf("expected kaptProcessor %q, got %q", expectedProcessor, kapt.Args["kaptProcessor"]) + } + + // Test that the processors are not passed to javac + if javac.Args["processorpath"] != "" { + t.Errorf("expected processorPath '', got %q", javac.Args["processorpath"]) + } + if javac.Args["processor"] != "-proc:none" { + t.Errorf("expected processor '-proc:none', got %q", javac.Args["processor"]) + } + }) + + t.Run("errorprone", func(t *testing.T) { + env := map[string]string{ + "RUN_ERROR_PRONE": "true", + } + config := testConfig(env, bp, nil) + ctx, _ := testJavaWithConfig(t, config) + + buildOS := android.BuildOs.String() + + kapt := ctx.ModuleForTests("foo", "android_common").Rule("kapt") + //kotlinc := ctx.ModuleForTests("foo", "android_common").Rule("kotlinc") + javac := ctx.ModuleForTests("foo", "android_common").Description("javac") + errorprone := ctx.ModuleForTests("foo", "android_common").Description("errorprone") + + bar := ctx.ModuleForTests("bar", buildOS+"_common").Description("javac").Output.String() + baz := ctx.ModuleForTests("baz", buildOS+"_common").Description("javac").Output.String() + myCheck := ctx.ModuleForTests("my_check", buildOS+"_common").Description("javac").Output.String() + + // Test that the errorprone plugins are not passed to kapt + expectedProcessorPath := "-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + bar + + " -P plugin:org.jetbrains.kotlin.kapt3:apclasspath=" + baz + if kapt.Args["kaptProcessorPath"] != expectedProcessorPath { + t.Errorf("expected kaptProcessorPath %q, got %q", expectedProcessorPath, kapt.Args["kaptProcessorPath"]) + } + expectedProcessor := "-P plugin:org.jetbrains.kotlin.kapt3:processors=com.bar -P plugin:org.jetbrains.kotlin.kapt3:processors=com.baz" + if kapt.Args["kaptProcessor"] != expectedProcessor { + t.Errorf("expected kaptProcessor %q, got %q", expectedProcessor, kapt.Args["kaptProcessor"]) + } + + // Test that the errorprone plugins are not passed to javac + if javac.Args["processorpath"] != "" { + t.Errorf("expected processorPath '', got %q", javac.Args["processorpath"]) + } + if javac.Args["processor"] != "-proc:none" { + t.Errorf("expected processor '-proc:none', got %q", javac.Args["processor"]) + } + + // Test that the errorprone plugins are passed to errorprone + expectedProcessorPath = "-processorpath " + myCheck + if errorprone.Args["processorpath"] != expectedProcessorPath { + t.Errorf("expected processorpath %q, got %q", expectedProcessorPath, errorprone.Args["processorpath"]) + } + if errorprone.Args["processor"] != "-proc:none" { + t.Errorf("expected processor '-proc:none', got %q", errorprone.Args["processor"]) + } + }) } func TestKaptEncodeFlags(t *testing.T) { diff --git a/java/sdk_library.go b/java/sdk_library.go index 13cd5f976..4369ffcf7 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -2300,11 +2300,10 @@ func (s *sdkLibrarySdkMemberProperties) AddToPropertySet(ctx android.SdkMemberCo scopeSet.AddProperty("jars", jars) // Merge the stubs source jar into the snapshot zip so that when it is unpacked - // the source files are also unpacked. Use a glob so that if the directory is missing - // (because there are no stubs sources for this scope) it will not fail. + // the source files are also unpacked. snapshotRelativeDir := filepath.Join(scopeDir, ctx.Name()+"_stub_sources") ctx.SnapshotBuilder().UnzipToSnapshot(properties.StubsSrcJar, snapshotRelativeDir) - scopeSet.AddProperty("stub_srcs", []string{snapshotRelativeDir + "/**/*.java"}) + scopeSet.AddProperty("stub_srcs", []string{snapshotRelativeDir}) if properties.CurrentApiFile != nil { currentApiSnapshotPath := filepath.Join(scopeDir, ctx.Name()+".txt") diff --git a/rust/bindgen.go b/rust/bindgen.go index 7cc0fc829..35a807bfd 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -258,5 +258,6 @@ func (b *bindgenDecorator) SourceProviderDeps(ctx DepsContext, deps Deps) Deps { deps.SharedLibs = append(deps.SharedLibs, b.ClangProperties.Shared_libs...) deps.StaticLibs = append(deps.StaticLibs, b.ClangProperties.Static_libs...) + deps.HeaderLibs = append(deps.StaticLibs, b.ClangProperties.Header_libs...) return deps } diff --git a/rust/bindgen_test.go b/rust/bindgen_test.go index c7ce42bb4..af04cfc89 100644 --- a/rust/bindgen_test.go +++ b/rust/bindgen_test.go @@ -32,6 +32,7 @@ func TestRustBindgen(t *testing.T) { cflags: ["--clang-flag()"], shared_libs: ["libfoo_shared"], static_libs: ["libfoo_static"], + header_libs: ["libfoo_header"], } cc_library_shared { name: "libfoo_shared", @@ -41,6 +42,10 @@ func TestRustBindgen(t *testing.T) { name: "libfoo_static", export_include_dirs: ["static_include"], } + cc_library_headers { + name: "libfoo_header", + export_include_dirs: ["header_include"], + } cc_defaults { name: "cc_defaults_flags", cflags: ["--default-flag"], @@ -60,6 +65,9 @@ func TestRustBindgen(t *testing.T) { if !strings.Contains(libbindgen.Args["cflags"], "-Istatic_include") { t.Errorf("missing static_libs exported includes in rust_bindgen rule: cflags %#v", libbindgen.Args["cflags"]) } + if !strings.Contains(libbindgen.Args["cflags"], "-Iheader_include") { + t.Errorf("missing static_libs exported includes in rust_bindgen rule: cflags %#v", libbindgen.Args["cflags"]) + } if !strings.Contains(libbindgen.Args["cflags"], "--default-flag") { t.Errorf("rust_bindgen missing cflags defined in cc_defaults: cflags %#v", libbindgen.Args["cflags"]) } diff --git a/rust/protobuf.go b/rust/protobuf.go index 76fed30fb..7d6e1fd5c 100644 --- a/rust/protobuf.go +++ b/rust/protobuf.go @@ -53,7 +53,7 @@ type ProtobufProperties struct { Proto_flags []string `android:"arch_variant"` // List of libraries which export include paths required for this module - Header_libs []string `android:"arch_variant"` + Header_libs []string `android:"arch_variant,variant_prepend"` } type protobufDecorator struct { diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index ec8ebb3b4..b44f66e85 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -976,21 +976,21 @@ java_sdk_library_import { shared_library: false, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system: { jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], current_api: "sdk_library/system/myjavalib.txt", removed_api: "sdk_library/system/myjavalib-removed.txt", sdk_version: "system_current", }, test: { jars: ["sdk_library/test/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/test/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/test/myjavalib_stub_sources"], current_api: "sdk_library/test/myjavalib.txt", removed_api: "sdk_library/test/myjavalib-removed.txt", sdk_version: "test_current", @@ -1005,21 +1005,21 @@ java_sdk_library_import { shared_library: false, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system: { jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], current_api: "sdk_library/system/myjavalib.txt", removed_api: "sdk_library/system/myjavalib-removed.txt", sdk_version: "system_current", }, test: { jars: ["sdk_library/test/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/test/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/test/myjavalib_stub_sources"], current_api: "sdk_library/test/myjavalib.txt", removed_api: "sdk_library/test/myjavalib-removed.txt", sdk_version: "test_current", @@ -1077,7 +1077,7 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "none", @@ -1092,7 +1092,7 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "none", @@ -1146,7 +1146,7 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "module_current", @@ -1161,7 +1161,7 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "module_current", @@ -1218,14 +1218,14 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system: { jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], current_api: "sdk_library/system/myjavalib.txt", removed_api: "sdk_library/system/myjavalib-removed.txt", sdk_version: "system_current", @@ -1240,14 +1240,14 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system: { jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], current_api: "sdk_library/system/myjavalib.txt", removed_api: "sdk_library/system/myjavalib-removed.txt", sdk_version: "system_current", @@ -1311,21 +1311,21 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system: { jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], current_api: "sdk_library/system/myjavalib.txt", removed_api: "sdk_library/system/myjavalib-removed.txt", sdk_version: "system_current", }, module_lib: { jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], current_api: "sdk_library/module-lib/myjavalib.txt", removed_api: "sdk_library/module-lib/myjavalib-removed.txt", sdk_version: "module_current", @@ -1340,21 +1340,21 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system: { jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], current_api: "sdk_library/system/myjavalib.txt", removed_api: "sdk_library/system/myjavalib-removed.txt", sdk_version: "system_current", }, module_lib: { jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], current_api: "sdk_library/module-lib/myjavalib.txt", removed_api: "sdk_library/module-lib/myjavalib-removed.txt", sdk_version: "module_current", @@ -1419,14 +1419,14 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system_server: { jars: ["sdk_library/system-server/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources"], current_api: "sdk_library/system-server/myjavalib.txt", removed_api: "sdk_library/system-server/myjavalib-removed.txt", sdk_version: "system_server_current", @@ -1441,14 +1441,14 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, system_server: { jars: ["sdk_library/system-server/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources"], current_api: "sdk_library/system-server/myjavalib.txt", removed_api: "sdk_library/system-server/myjavalib-removed.txt", sdk_version: "system_server_current", @@ -1508,7 +1508,7 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", @@ -1524,7 +1524,7 @@ java_sdk_library_import { shared_library: true, public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", @@ -1584,7 +1584,7 @@ java_sdk_library_import { doctag_files: ["doctags/docs/known_doctags"], public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", @@ -1600,7 +1600,7 @@ java_sdk_library_import { doctag_files: ["doctags/docs/known_doctags"], public: { jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources/**/*.java"], + stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], current_api: "sdk_library/public/myjavalib.txt", removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", diff --git a/tradefed/autogen.go b/tradefed/autogen.go index b35f831c7..27d71e828 100644 --- a/tradefed/autogen.go +++ b/tradefed/autogen.go @@ -188,7 +188,7 @@ func AutoGenNativeBenchmarkTestConfig(ctx android.ModuleContext, testConfigProp } func AutoGenJavaTestConfig(ctx android.ModuleContext, testConfigProp *string, testConfigTemplateProp *string, - testSuites []string, autoGenConfig *bool) android.Path { + testSuites []string, autoGenConfig *bool, unitTest *bool) android.Path { path, autogenPath := testConfigPath(ctx, testConfigProp, testSuites, autoGenConfig, testConfigTemplateProp) if autogenPath != nil { templatePath := getTestConfigTemplate(ctx, testConfigTemplateProp) @@ -198,7 +198,11 @@ func AutoGenJavaTestConfig(ctx android.ModuleContext, testConfigProp *string, te if ctx.Device() { autogenTemplate(ctx, autogenPath, "${JavaTestConfigTemplate}", nil, "") } else { - autogenTemplate(ctx, autogenPath, "${JavaHostTestConfigTemplate}", nil, "") + if Bool(unitTest) { + autogenTemplate(ctx, autogenPath, "${JavaHostUnitTestConfigTemplate}", nil, "") + } else { + autogenTemplate(ctx, autogenPath, "${JavaHostTestConfigTemplate}", nil, "") + } } } return autogenPath diff --git a/tradefed/config.go b/tradefed/config.go index f7e834988..f3566a80d 100644 --- a/tradefed/config.go +++ b/tradefed/config.go @@ -27,6 +27,7 @@ func init() { pctx.SourcePathVariable("InstrumentationTestConfigTemplate", "build/make/core/instrumentation_test_config_template.xml") pctx.SourcePathVariable("JavaTestConfigTemplate", "build/make/core/java_test_config_template.xml") pctx.SourcePathVariable("JavaHostTestConfigTemplate", "build/make/core/java_host_test_config_template.xml") + pctx.SourcePathVariable("JavaHostUnitTestConfigTemplate", "build/make/core/java_host_unit_test_config_template.xml") pctx.SourcePathVariable("NativeBenchmarkTestConfigTemplate", "build/make/core/native_benchmark_test_config_template.xml") pctx.SourcePathVariable("NativeHostTestConfigTemplate", "build/make/core/native_host_test_config_template.xml") pctx.SourcePathVariable("NativeTestConfigTemplate", "build/make/core/native_test_config_template.xml") diff --git a/ui/build/finder.go b/ui/build/finder.go index 7a856577e..2eb84ca38 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -27,8 +27,10 @@ import ( "android/soong/ui/metrics" ) -// This file provides an interface to the Finder for use in Soong UI -// This file stores configuration information about which files to find +// This file provides an interface to the Finder type for soong_ui. Finder is +// used to recursively traverse the source tree to gather paths of files, such +// as Android.bp or Android.mk, and store the lists/database of paths in files +// under `$OUT_DIR/.module_paths`. This directory can also be dist'd. // NewSourceFinder returns a new Finder configured to search for source files. // Callers of NewSourceFinder should call <f.Shutdown()> when done @@ -36,14 +38,18 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { ctx.BeginTrace(metrics.RunSetupTool, "find modules") defer ctx.EndTrace() + // Set up the working directory for the Finder. dir, err := os.Getwd() if err != nil { ctx.Fatalf("No working directory for module-finder: %v", err.Error()) } filesystem := fs.OsFs - // if the root dir is ignored, then the subsequent error messages are very confusing, - // so check for that upfront + // .out-dir and .find-ignore are markers for Finder to ignore siblings and + // subdirectories of the directory Finder finds them in, hence stopping the + // search recursively down those branches. It's possible that these files + // are in the root directory, and if they are, then the subsequent error + // messages are very confusing, so check for that here. pruneFiles := []string{".out-dir", ".find-ignore"} for _, name := range pruneFiles { prunePath := filepath.Join(dir, name) @@ -53,22 +59,34 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { } } + // Set up configuration parameters for the Finder cache. cacheParams := finder.CacheParams{ WorkingDirectory: dir, RootDirs: []string{"."}, ExcludeDirs: []string{".git", ".repo"}, PruneFiles: pruneFiles, IncludeFiles: []string{ + // Kati build definitions. "Android.mk", + // Product configuration files. "AndroidProducts.mk", + // General Soong build definitions, using the Blueprint syntax. "Android.bp", + // build/blueprint build definitions, using the Blueprint syntax. "Blueprints", + // Bazel build definitions. "BUILD.bazel", + // Kati clean definitions. "CleanSpec.mk", + // Ownership definition. "OWNERS", + // Test configuration for modules in directories that contain this + // file. "TEST_MAPPING", + // Bazel top-level file to mark a directory as a Bazel workspace. "WORKSPACE", }, + // Bazel Starlark configuration files. IncludeSuffixes: []string{".bzl"}, } dumpDir := config.FileListDir() @@ -80,6 +98,7 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { return f } +// Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree. func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { matches := []string{} for _, foundName := range entries.FileNames { @@ -98,12 +117,21 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { dumpDir := config.FileListDir() os.MkdirAll(dumpDir, 0777) + // Stop searching a subdirectory recursively after finding an Android.mk. androidMks := f.FindFirstNamedAt(".", "Android.mk") err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list")) if err != nil { ctx.Fatalf("Could not export module list: %v", err) } + // Stop searching a subdirectory recursively after finding a CleanSpec.mk. + cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") + err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) + if err != nil { + ctx.Fatalf("Could not export module list: %v", err) + } + + // Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories. androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk") androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...) androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...) @@ -112,31 +140,30 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { ctx.Fatalf("Could not export product list: %v", err) } + // Recursively look for all Bazel related files. bazelFiles := f.FindMatching(".", findBazelFiles) err = dumpListToFile(ctx, config, bazelFiles, filepath.Join(dumpDir, "bazel.list")) if err != nil { ctx.Fatalf("Could not export bazel BUILD list: %v", err) } - cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") - err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) - if err != nil { - ctx.Fatalf("Could not export module list: %v", err) - } - + // Recursively look for all OWNERS files. owners := f.FindNamedAt(".", "OWNERS") err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list")) if err != nil { ctx.Fatalf("Could not find OWNERS: %v", err) } + // Recursively look for all TEST_MAPPING files. testMappings := f.FindNamedAt(".", "TEST_MAPPING") err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list")) if err != nil { ctx.Fatalf("Could not find TEST_MAPPING: %v", err) } + // Recursively look for all Android.bp files androidBps := f.FindNamedAt(".", "Android.bp") + // The files are named "Blueprints" only in the build/blueprint directory. androidBps = append(androidBps, f.FindNamedAt("build/blueprint", "Blueprints")...) if len(androidBps) == 0 { ctx.Fatalf("No Android.bp found") @@ -148,10 +175,12 @@ func FindSources(ctx Context, config Config, f *finder.Finder) { if config.Dist() { f.WaitForDbDump() + // Dist the files.db plain text database. distFile(ctx, config, f.DbPath, "module_paths") } } +// Write the .list files to disk. func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) { desiredText := strings.Join(list, "\n") desiredBytes := []byte(desiredText) diff --git a/zip/cmd/main.go b/zip/cmd/main.go index d603586e6..fc976f689 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -12,6 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +// soong_zip is a utility used during the build to create a zip archive by pulling the entries from +// various sources: +// * explicitly specified files +// * files whose paths are read from a file +// * directories traversed recursively +// It can optionally change the recorded path of an entry. + package main import ( diff --git a/zip/zip.go b/zip/zip.go index e27432cbb..cb85f5ced 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -126,6 +126,7 @@ func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder { return b } +// List reads the file names from the given file and adds them to the source files list. func (b *FileArgsBuilder) List(name string) *FileArgsBuilder { if b.err != nil { return b @@ -150,6 +151,7 @@ func (b *FileArgsBuilder) List(name string) *FileArgsBuilder { return b } +// RspFile reads the file names from given .rsp file and adds them to the source files list. func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder { if b.err != nil { return b @@ -291,7 +293,7 @@ func ReadRespFile(bytes []byte) []string { return args } -func ZipTo(args ZipArgs, w io.Writer) error { +func zipTo(args ZipArgs, w io.Writer) error { if args.EmulateJar { args.AddDirectoryEntriesToZip = true } @@ -392,6 +394,7 @@ func ZipTo(args ZipArgs, w io.Writer) error { return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs) } +// Zip creates an output zip archive from given sources. func Zip(args ZipArgs) error { if args.OutputFilePath == "" { return fmt.Errorf("output file path must be nonempty") @@ -416,7 +419,7 @@ func Zip(args ZipArgs) error { out = f } - err := ZipTo(args, out) + err := zipTo(args, out) if err != nil { return err } @@ -450,7 +453,6 @@ func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping, RelativeRoot: fa.SourcePrefixToStrip, } } - } dest = filepath.Join(fa.PathPrefixInZip, dest) @@ -465,10 +467,9 @@ func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping, } func jarSort(mappings []pathMapping) { - less := func(i int, j int) (smaller bool) { + sort.SliceStable(mappings, func(i int, j int) bool { return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest) - } - sort.SliceStable(mappings, less) + }) } func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool, @@ -709,7 +710,7 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar } } -func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { +func (z *ZipWriter) addManifest(dest string, src string, _ uint16) error { if prev, exists := z.createdDirs[dest]; exists { return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) } @@ -963,7 +964,7 @@ func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) erro dir = filepath.Clean(dir) // discover any uncreated directories in the path - zipDirs := []string{} + var zipDirs []string for dir != "" && dir != "." { if _, exists := z.createdDirs[dir]; exists { break diff --git a/zip/zip_test.go b/zip/zip_test.go index 302a749a3..a16e09286 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -442,7 +442,7 @@ func TestZip(t *testing.T) { args.Stderr = &bytes.Buffer{} buf := &bytes.Buffer{} - err := ZipTo(args, buf) + err := zipTo(args, buf) if (err != nil) != (test.err != nil) { t.Fatalf("want error %v, got %v", test.err, err) @@ -627,7 +627,7 @@ func TestSrcJar(t *testing.T) { args.Stderr = &bytes.Buffer{} buf := &bytes.Buffer{} - err := ZipTo(args, buf) + err := zipTo(args, buf) if err != nil { t.Fatalf("got error %v", err) } |