diff options
Diffstat (limited to 'python/python.go')
| -rw-r--r-- | python/python.go | 524 |
1 files changed, 207 insertions, 317 deletions
diff --git a/python/python.go b/python/python.go index 24e1bb2ec..18e5b68dd 100644 --- a/python/python.go +++ b/python/python.go @@ -22,8 +22,6 @@ import ( "regexp" "strings" - "android/soong/bazel" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -122,26 +120,13 @@ type BaseProperties struct { Embedded_launcher *bool `blueprint:"mutated"` } -type baseAttributes struct { - // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv - //Pkg_path bazel.StringAttribute - // TODO: Related to Pkg_bath and similarLy gated - //Is_internal bazel.BoolAttribute - // Combines Srcs and Exclude_srcs - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - // Combines Data and Java_data (invariant) - Data bazel.LabelListAttribute - Imports bazel.StringListAttribute -} - // Used to store files of current module after expanding dependencies type pathMapping struct { dest string src android.Path } -type Module struct { +type PythonLibraryModule struct { android.ModuleBase android.DefaultableModuleBase android.BazelModuleBase @@ -153,16 +138,6 @@ type Module struct { hod android.HostOrDeviceSupported multilib android.Multilib - // interface used to bootstrap .par executable when embedded_launcher is true - // this should be set by Python modules which are runnable, e.g. binaries and tests - // bootstrapper might be nil (e.g. Python library module). - bootstrapper bootstrapper - - // interface that implements functions required for installation - // this should be set by Python modules which are runnable, e.g. binaries and tests - // installer might be nil (e.g. Python library module). - installer installer - // the Python files of current module after expanding source dependencies. // pathMapping: <dest: runfile_path, src: source_path> srcsPathMappings []pathMapping @@ -171,152 +146,62 @@ type Module struct { // pathMapping: <dest: runfile_path, src: source_path> dataPathMappings []pathMapping - // the zip filepath for zipping current module source/data files. + // The zip file containing the current module's source/data files. srcsZip android.Path - // dependency modules' zip filepath for zipping current module source/data files. - depsSrcsZips android.Paths - - // (.intermediate) module output path as installation source. - installSource android.OptionalPath - - // Map to ensure sub-part of the AndroidMk for this module is only added once - subAndroidMkOnce map[subAndroidMkProvider]bool + // The zip file containing the current module's source/data files, with the + // source files precompiled. + precompiledSrcsZip android.Path } // newModule generates new Python base module -func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { - return &Module{ +func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *PythonLibraryModule { + return &PythonLibraryModule{ hod: hod, multilib: multilib, } } -func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes { - var attrs baseAttributes - archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{}) - for axis, configToProps := range archVariantBaseProps { - for config, props := range configToProps { - if baseProps, ok := props.(*BaseProperties); ok { - attrs.Srcs.SetSelectValue(axis, config, - android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs)) - attrs.Deps.SetSelectValue(axis, config, - android.BazelLabelForModuleDeps(ctx, baseProps.Libs)) - data := android.BazelLabelForModuleSrc(ctx, baseProps.Data) - data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data)) - attrs.Data.SetSelectValue(axis, config, data) - } - } - } - - partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ - "proto": android.ProtoSrcLabelPartition, - "py": bazel.LabelPartition{Keep_remainder: true}, - }) - attrs.Srcs = partitionedSrcs["py"] - - if !partitionedSrcs["proto"].IsEmpty() { - protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) - protoLabel := bazel.Label{Label: ":" + protoInfo.Name} - - pyProtoLibraryName := m.Name() + "_py_proto" - ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ - Rule_class: "py_proto_library", - Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", - }, android.CommonAttributes{ - Name: pyProtoLibraryName, - }, &bazelPythonProtoLibraryAttributes{ - Deps: bazel.MakeSingleLabelListAttribute(protoLabel), - }) - - attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) - } - - // Bazel normally requires `import path.from.top.of.tree` statements in - // python code, but with soong you can directly import modules from libraries. - // Add "imports" attributes to the bazel library so it matches soong's behavior. - imports := "." - if m.properties.Pkg_path != nil { - // TODO(b/215119317) This is a hack to handle the fact that we don't convert - // pkg_path properly right now. If the folder structure that contains this - // Android.bp file matches pkg_path, we can set imports to an appropriate - // number of ../..s to emulate moving the files under a pkg_path folder. - pkg_path := filepath.Clean(*m.properties.Pkg_path) - if strings.HasPrefix(pkg_path, "/") { - ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) - } - - if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { - ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) - } - numFolders := strings.Count(pkg_path, "/") + 1 - dots := make([]string, numFolders) - for i := 0; i < numFolders; i++ { - dots[i] = ".." - } - imports = strings.Join(dots, "/") - } - attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) - - return attrs -} - -// bootstrapper interface should be implemented for runnable modules, e.g. binary and test -type bootstrapper interface { - bootstrapperProps() []interface{} - bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool, - srcsPathMappings []pathMapping, srcsZip android.Path, - depsSrcsZips android.Paths) android.OptionalPath - - autorun() bool -} - -// installer interface should be implemented for installable modules, e.g. binary and test -type installer interface { - install(ctx android.ModuleContext, path android.Path) - setAndroidMkSharedLibs(sharedLibs []string) -} - // interface implemented by Python modules to provide source and data mappings and zip to python // modules that depend on it type pythonDependency interface { getSrcsPathMappings() []pathMapping getDataPathMappings() []pathMapping getSrcsZip() android.Path + getPrecompiledSrcsZip() android.Path } // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination -func (p *Module) getSrcsPathMappings() []pathMapping { +func (p *PythonLibraryModule) getSrcsPathMappings() []pathMapping { return p.srcsPathMappings } // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination -func (p *Module) getDataPathMappings() []pathMapping { +func (p *PythonLibraryModule) getDataPathMappings() []pathMapping { return p.dataPathMappings } // getSrcsZip returns the filepath where the current module's source/data files are zipped. -func (p *Module) getSrcsZip() android.Path { +func (p *PythonLibraryModule) getSrcsZip() android.Path { return p.srcsZip } -var _ pythonDependency = (*Module)(nil) - -var _ android.AndroidMkEntriesProvider = (*Module)(nil) +// getSrcsZip returns the filepath where the current module's source/data files are zipped. +func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path { + return p.precompiledSrcsZip +} -func (p *Module) init(additionalProps ...interface{}) android.Module { - p.AddProperties(&p.properties, &p.protoProperties) +func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { + return &p.properties +} - // Add additional properties for bootstrapping/installation - // This is currently tied to the bootstrapper interface; - // however, these are a combination of properties for the installation and bootstrapping of a module - if p.bootstrapper != nil { - p.AddProperties(p.bootstrapper.bootstrapperProps()...) - } +var _ pythonDependency = (*PythonLibraryModule)(nil) +func (p *PythonLibraryModule) init() android.Module { + p.AddProperties(&p.properties, &p.protoProperties) android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) - + android.InitBazelModule(p) return p } @@ -338,36 +223,48 @@ type installDependencyTag struct { } var ( - pythonLibTag = dependencyTag{name: "pythonLib"} - javaDataTag = dependencyTag{name: "javaData"} + pythonLibTag = dependencyTag{name: "pythonLib"} + javaDataTag = dependencyTag{name: "javaData"} + // The python interpreter, with soong module name "py3-launcher" or "py3-launcher-autorun". launcherTag = dependencyTag{name: "launcher"} launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"} - pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) - pyExt = ".py" - protoExt = ".proto" - pyVersion2 = "PY2" - pyVersion3 = "PY3" - internalPath = "internal" + // The python interpreter built for host so that we can precompile python sources. + // This only works because the precompiled sources don't vary by architecture. + // The soong module name is "py3-launcher". + hostLauncherTag = dependencyTag{name: "hostLauncher"} + hostlauncherSharedLibTag = dependencyTag{name: "hostlauncherSharedLib"} + hostStdLibTag = dependencyTag{name: "hostStdLib"} + pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) + pyExt = ".py" + protoExt = ".proto" + pyVersion2 = "PY2" + pyVersion3 = "PY3" + internalPath = "internal" ) +type basePropertiesProvider interface { + getBaseProperties() *BaseProperties +} + // versionSplitMutator creates version variants for modules and appends the version-specific // properties for a given variant to the properties in the variant module func versionSplitMutator() func(android.BottomUpMutatorContext) { return func(mctx android.BottomUpMutatorContext) { - if base, ok := mctx.Module().(*Module); ok { - versionNames := []string{} + if base, ok := mctx.Module().(basePropertiesProvider); ok { + props := base.getBaseProperties() + var versionNames []string // collect version specific properties, so that we can merge version-specific properties // into the module's overall properties - versionProps := []VersionProperties{} + var versionProps []VersionProperties // PY3 is first so that we alias the PY3 variant rather than PY2 if both // are available - if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) { + if proptools.BoolDefault(props.Version.Py3.Enabled, true) { versionNames = append(versionNames, pyVersion3) - versionProps = append(versionProps, base.properties.Version.Py3) + versionProps = append(versionProps, props.Version.Py3) } - if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) { + if proptools.BoolDefault(props.Version.Py2.Enabled, false) { versionNames = append(versionNames, pyVersion2) - versionProps = append(versionProps, base.properties.Version.Py2) + versionProps = append(versionProps, props.Version.Py2) } modules := mctx.CreateLocalVariations(versionNames...) // Alias module to the first variant @@ -376,9 +273,10 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } for i, v := range versionNames { // set the actual version for Python module. - modules[i].(*Module).properties.Actual_version = v + newProps := modules[i].(basePropertiesProvider).getBaseProperties() + newProps.Actual_version = v // append versioned properties for the Python module to the overall properties - err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil) + err := proptools.AppendMatchingProperties([]interface{}{newProps}, &versionProps[i], nil) if err != nil { panic(err) } @@ -387,38 +285,6 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } } -// HostToolPath returns a path if appropriate such that this module can be used as a host tool, -// fulfilling HostToolProvider interface. -func (p *Module) HostToolPath() android.OptionalPath { - if p.installer != nil { - if bin, ok := p.installer.(*binaryDecorator); ok { - // TODO: This should only be set when building host binaries -- tests built for device would be - // setting this incorrectly. - return android.OptionalPathForPath(bin.path) - } - } - - return android.OptionalPath{} - -} - -// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. -func (p *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if outputFile := p.installSource; outputFile.Valid() { - return android.Paths{outputFile.Path()}, nil - } - return android.Paths{}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - -func (p *Module) isEmbeddedLauncherEnabled() bool { - return p.installer != nil && Bool(p.properties.Embedded_launcher) -} - func anyHasExt(paths []string, ext string) bool { for _, p := range paths { if filepath.Ext(p) == ext { @@ -429,7 +295,7 @@ func anyHasExt(paths []string, ext string) bool { return false } -func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { +func (p *PythonLibraryModule) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { return anyHasExt(p.properties.Srcs, ext) } @@ -437,7 +303,7 @@ func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bo // - handles proto dependencies, // - if required, specifies launcher and adds launcher dependencies, // - applies python version mutations to Python dependencies -func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { +func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) { android.ProtoDeps(ctx, &p.protoProperties) versionVariation := []blueprint.Variation{ @@ -452,111 +318,85 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { // Add python library dependencies for this python version variation ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...) - // If this module will be installed and has an embedded launcher, we need to add dependencies for: - // * standard library - // * launcher - // * shared dependencies of the launcher - if p.installer != nil && p.isEmbeddedLauncherEnabled() { - var stdLib string - var launcherModule string - // Add launcher shared lib dependencies. Ideally, these should be - // derived from the `shared_libs` property of the launcher. However, we - // cannot read the property at this stage and it will be too late to add - // dependencies later. - launcherSharedLibDeps := []string{ - "libsqlite", - } - // Add launcher-specific dependencies for bionic - if ctx.Target().Os.Bionic() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") - } - if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") - } - - switch p.properties.Actual_version { - case pyVersion2: - stdLib = "py2-stdlib" - - launcherModule = "py2-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py2-launcher-autorun" - } - - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") - - case pyVersion3: - stdLib = "py3-stdlib" - - launcherModule = "py3-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py3-launcher-autorun" - } - if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl { - launcherModule += "-static" - } - - if ctx.Device() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") - } - default: - panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", - p.properties.Actual_version, ctx.ModuleName())) - } - ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib) - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...) - } - // Emulate the data property for java_data but with the arch variation overridden to "common" // so that it can point to java modules. javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}} ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...) -} - -func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { - p.generatePythonBuildActions(ctx) - // Only Python binary and test modules have non-empty bootstrapper. - if p.bootstrapper != nil { - // if the module is being installed, we need to collect all transitive dependencies to embed in - // the final par - p.collectPathsFromTransitiveDeps(ctx) - // bootstrap the module, including resolving main file, getting launcher path, and - // registering actions to build the par file - // bootstrap returns the binary output path - p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version, - p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) + p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, hostlauncherSharedLibTag, false, ctx.Config().BuildOSTarget) +} + +// AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib, +// launcher (interpreter), and the launcher's shared libraries. If autorun is true, it will use +// the autorun launcher instead of the regular one. This function acceps a targetForDeps argument +// as the target to use for these dependencies. For embedded launcher python binaries, the launcher +// that will be embedded will be under the same target as the python module itself. But when +// precompiling python code, we need to get the python launcher built for host, even if we're +// compiling the python module for device, so we pass a different target to this function. +func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.BottomUpMutatorContext, + stdLibTag, launcherTag, launcherSharedLibTag blueprint.DependencyTag, + autorun bool, targetForDeps android.Target) { + var stdLib string + var launcherModule string + // Add launcher shared lib dependencies. Ideally, these should be + // derived from the `shared_libs` property of the launcher. TODO: read these from + // the python launcher itself using ctx.OtherModuleProvider() or similar on the result + // of ctx.AddFarVariationDependencies() + launcherSharedLibDeps := []string{ + "libsqlite", } + // Add launcher-specific dependencies for bionic + if targetForDeps.Os.Bionic() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") + } + if targetForDeps.Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") + } + + switch p.properties.Actual_version { + case pyVersion2: + stdLib = "py2-stdlib" - // Only Python binary and test modules have non-empty installer. - if p.installer != nil { - var sharedLibs []string - // if embedded launcher is enabled, we need to collect the shared library depenendencies of the - // launcher - for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { - sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) + launcherModule = "py2-launcher" + if autorun { + launcherModule = "py2-launcher-autorun" } - p.installer.setAndroidMkSharedLibs(sharedLibs) + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") + case pyVersion3: + stdLib = "py3-stdlib" - // Install the par file from installSource - if p.installSource.Valid() { - p.installer.install(ctx, p.installSource.Path()) + launcherModule = "py3-launcher" + if autorun { + launcherModule = "py3-launcher-autorun" } + if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl { + launcherModule += "-static" + } + if ctx.Device() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") + } + default: + panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", + p.properties.Actual_version, ctx.ModuleName())) + } + targetVariations := targetForDeps.Variations() + if ctx.ModuleName() != stdLib { + stdLibVariations := make([]blueprint.Variation, 0, len(targetVariations)+1) + stdLibVariations = append(stdLibVariations, blueprint.Variation{Mutator: "python_version", Variation: p.properties.Actual_version}) + stdLibVariations = append(stdLibVariations, targetVariations...) + // Using AddFarVariationDependencies for all of these because they can be for a different + // platform, like if the python module itself was being compiled for device, we may want + // the python interpreter built for host so that we can precompile python sources. + ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib) } + ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule) + ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...) } -// generatePythonBuildActions performs build actions common to all Python modules -func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) { +// GenerateAndroidBuildActions performs build actions common to all Python modules +func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) - requiresSrcs := true - if p.bootstrapper != nil && !p.bootstrapper.autorun() { - requiresSrcs = false - } - if len(expandedSrcs) == 0 && requiresSrcs { - ctx.ModuleErrorf("doesn't have any source files!") - } // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) @@ -589,6 +429,7 @@ func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) { // generate the zipfile of all source and data files p.srcsZip = p.createSrcsZip(ctx, pkgPath) + p.precompiledSrcsZip = p.precompileSrcs(ctx) } func isValidPythonPath(path string) error { @@ -607,7 +448,7 @@ func isValidPythonPath(path string) error { // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path> // for python/data files expanded from properties. -func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string, +func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, pkgPath string, expandedSrcs, expandedData android.Paths) { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // check current module duplicates. @@ -642,27 +483,28 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string } // createSrcsZip registers build actions to zip current module's sources and data. -func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { +func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { relativeRootMap := make(map[string]android.Paths) - pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) - var protoSrcs android.Paths - // "srcs" or "data" properties may contain filegroup so it might happen that - // the root directory for each source path is different. - for _, path := range pathMappings { + addPathMapping := func(path pathMapping) { // handle proto sources separately if path.src.Ext() == protoExt { protoSrcs = append(protoSrcs, path.src) } else { - var relativeRoot string - relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel()) - if v, found := relativeRootMap[relativeRoot]; found { - relativeRootMap[relativeRoot] = append(v, path.src) - } else { - relativeRootMap[relativeRoot] = android.Paths{path.src} - } + relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel()) + relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src) } } + + // "srcs" or "data" properties may contain filegroups so it might happen that + // the root directory for each source path is different. + for _, path := range p.srcsPathMappings { + addPathMapping(path) + } + for _, path := range p.dataPathMappings { + addPathMapping(path) + } + var zips android.Paths if len(protoSrcs) > 0 { protoFlags := android.GetProtoFlags(ctx, &p.protoProperties) @@ -736,30 +578,79 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi } } -// isPythonLibModule returns whether the given module is a Python library Module or not -func isPythonLibModule(module blueprint.Module) bool { - if m, ok := module.(*Module); ok { - return m.isLibrary() +func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.Path { + // To precompile the python sources, we need a python interpreter and stdlib built + // for host. We then use those to compile the python sources, which may be used on either + // host of device. Python bytecode is architecture agnostic, so we're essentially + // "cross compiling" for device here purely by virtue of host and device python bytecode + // being the same. + var stdLib android.Path + var launcher android.Path + if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" { + stdLib = p.srcsZip + } else { + ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) { + if dep, ok := module.(pythonDependency); ok { + stdLib = dep.getPrecompiledSrcsZip() + } + }) } - return false -} + ctx.VisitDirectDepsWithTag(hostLauncherTag, func(module android.Module) { + if dep, ok := module.(IntermPathProvider); ok { + optionalLauncher := dep.IntermPathForModuleOut() + if optionalLauncher.Valid() { + launcher = optionalLauncher.Path() + } + } + }) + var launcherSharedLibs android.Paths + var ldLibraryPath []string + ctx.VisitDirectDepsWithTag(hostlauncherSharedLibTag, func(module android.Module) { + if dep, ok := module.(IntermPathProvider); ok { + optionalPath := dep.IntermPathForModuleOut() + if optionalPath.Valid() { + launcherSharedLibs = append(launcherSharedLibs, optionalPath.Path()) + ldLibraryPath = append(ldLibraryPath, filepath.Dir(optionalPath.Path().String())) + } + } + }) -// This is distinguished by the fact that Python libraries are not installable, while other Python -// modules are. -func (p *Module) isLibrary() bool { - // Python library has no bootstrapper or installer - return p.bootstrapper == nil && p.installer == nil + out := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszipprecompiled") + if stdLib == nil || launcher == nil { + // This shouldn't happen in a real build because we'll error out when adding dependencies + // on the stdlib and launcher if they don't exist. But some tests set + // AllowMissingDependencies. + return out + } + ctx.Build(pctx, android.BuildParams{ + Rule: precompile, + Input: p.srcsZip, + Output: out, + Implicits: launcherSharedLibs, + Description: "Precompile the python sources of " + ctx.ModuleName(), + Args: map[string]string{ + "stdlibZip": stdLib.String(), + "launcher": launcher.String(), + "ldLibraryPath": strings.Join(ldLibraryPath, ":"), + }, + }) + return out } -func (p *Module) isBinary() bool { - _, ok := p.bootstrapper.(*binaryDecorator) - return ok +// isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not +func isPythonLibModule(module blueprint.Module) bool { + if _, ok := module.(*PythonLibraryModule); ok { + if _, ok := module.(*PythonBinaryModule); !ok { + return true + } + } + return false } // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths // for module and its transitive dependencies and collects list of data/source file // zips for transitive dependencies. -func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { +func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext, precompiled bool) android.Paths { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // check duplicates. destToPySrcs := make(map[string]string) @@ -773,6 +664,8 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { seen := make(map[android.Module]bool) + var result android.Paths + // visit all its dependencies in depth first. ctx.WalkDeps(func(child, parent android.Module) bool { // we only collect dependencies tagged as python library deps @@ -801,10 +694,15 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { checkForDuplicateOutputPath(ctx, destToPyData, path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) } - p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip()) + if precompiled { + result = append(result, dep.getPrecompiledSrcsZip()) + } else { + result = append(result, dep.getSrcsZip()) + } } return true }) + return result } // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which @@ -825,18 +723,10 @@ func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, } // InstallInData returns true as Python is not supported in the system partition -func (p *Module) InstallInData() bool { +func (p *PythonLibraryModule) InstallInData() bool { return true } -func (p *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - if p.isLibrary() { - pythonLibBp2Build(ctx, p) - } else if p.isBinary() { - pythonBinaryBp2Build(ctx, p) - } -} - var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String |