summaryrefslogtreecommitdiff
path: root/python/python.go
diff options
context:
space:
mode:
Diffstat (limited to 'python/python.go')
-rw-r--r--python/python.go350
1 files changed, 217 insertions, 133 deletions
diff --git a/python/python.go b/python/python.go
index ed879eb0d..ad089090b 100644
--- a/python/python.go
+++ b/python/python.go
@@ -43,11 +43,11 @@ type VersionProperties struct {
// non-empty list of .py files under this strict Python version.
// srcs may reference the outputs of other modules that produce source files like genrule
// or filegroup using the syntax ":module".
- Srcs []string `android:"arch_variant"`
+ Srcs []string `android:"path,arch_variant"`
// list of source files that should not be used to build the Python module.
// This is most useful in the arch/multilib variants to remove non-common files
- Exclude_srcs []string `android:"arch_variant"`
+ Exclude_srcs []string `android:"path,arch_variant"`
// list of the Python libraries under this Python version.
Libs []string `android:"arch_variant"`
@@ -63,8 +63,7 @@ type BaseProperties struct {
// files of the current module.
// eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
// (from a.b.c import ...) statement.
- // if left unspecified, all the source/data files of current module are copied to
- // "runfiles/" tree directory directly.
+ // if left unspecified, all the source/data files path is unchanged within zip file.
Pkg_path *string `android:"arch_variant"`
// true, if the Python module is used internally, eg, Python std libs.
@@ -75,15 +74,15 @@ type BaseProperties struct {
// srcs may reference the outputs of other modules that produce source files like genrule
// or filegroup using the syntax ":module".
// Srcs has to be non-empty.
- Srcs []string `android:"arch_variant"`
+ Srcs []string `android:"path,arch_variant"`
// list of source files that should not be used to build the C/C++ module.
// This is most useful in the arch/multilib variants to remove non-common files
- Exclude_srcs []string `android:"arch_variant"`
+ Exclude_srcs []string `android:"path,arch_variant"`
// list of files or filegroup modules that provide data that should be installed alongside
// the test. the file extension can be arbitrary except for (.py).
- Data []string `android:"arch_variant"`
+ Data []string `android:"path,arch_variant"`
// list of the Python libraries compatible both with Python2 and Python3.
Libs []string `android:"arch_variant"`
@@ -111,7 +110,8 @@ type Module struct {
android.ModuleBase
android.DefaultableModuleBase
- properties BaseProperties
+ properties BaseProperties
+ protoProperties android.ProtoProperties
// initialize before calling Init
hod android.HostOrDeviceSupported
@@ -132,18 +132,15 @@ type Module struct {
// pathMapping: <dest: runfile_path, src: source_path>
dataPathMappings []pathMapping
- // soong_zip arguments of all its dependencies.
- depsParSpecs []parSpec
+ // the zip filepath for zipping current module source/data files.
+ srcsZip android.Path
- // Python runfiles paths of all its dependencies.
- depsPyRunfiles []string
+ // dependency modules' zip filepath for zipping current module source/data files.
+ depsSrcsZips android.Paths
// (.intermediate) module output path as installation source.
installSource android.OptionalPath
- // the soong_zip arguments for zipping current module source/data files.
- parSpec parSpec
-
subAndroidMkOnce map[subAndroidMkProvider]bool
}
@@ -156,19 +153,22 @@ func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Mo
type bootstrapper interface {
bootstrapperProps() []interface{}
- bootstrap(ctx android.ModuleContext, Actual_version string, embedded_launcher bool,
- srcsPathMappings []pathMapping, parSpec parSpec,
- depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath
+ bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool,
+ srcsPathMappings []pathMapping, srcsZip android.Path,
+ depsSrcsZips android.Paths) android.OptionalPath
+
+ autorun() bool
}
type installer interface {
install(ctx android.ModuleContext, path android.Path)
+ setAndroidMkSharedLibs(sharedLibs []string)
}
type PythonDependency interface {
GetSrcsPathMappings() []pathMapping
GetDataPathMappings() []pathMapping
- GetParSpec() parSpec
+ GetSrcsZip() android.Path
}
func (p *Module) GetSrcsPathMappings() []pathMapping {
@@ -179,8 +179,8 @@ func (p *Module) GetDataPathMappings() []pathMapping {
return p.dataPathMappings
}
-func (p *Module) GetParSpec() parSpec {
- return p.parSpec
+func (p *Module) GetSrcsZip() android.Path {
+ return p.srcsZip
}
var _ PythonDependency = (*Module)(nil)
@@ -189,7 +189,7 @@ var _ android.AndroidMkDataProvider = (*Module)(nil)
func (p *Module) Init() android.Module {
- p.AddProperties(&p.properties)
+ p.AddProperties(&p.properties, &p.protoProperties)
if p.bootstrapper != nil {
p.AddProperties(p.bootstrapper.bootstrapperProps()...)
}
@@ -206,18 +206,19 @@ type dependencyTag struct {
}
var (
- pythonLibTag = dependencyTag{name: "pythonLib"}
- launcherTag = dependencyTag{name: "launcher"}
- pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`)
- pyExt = ".py"
- pyVersion2 = "PY2"
- pyVersion3 = "PY3"
- initFileName = "__init__.py"
- mainFileName = "__main__.py"
- entryPointFile = "entry_point.txt"
- parFileExt = ".zip"
- runFiles = "runfiles"
- internal = "internal"
+ pythonLibTag = dependencyTag{name: "pythonLib"}
+ launcherTag = dependencyTag{name: "launcher"}
+ launcherSharedLibTag = dependencyTag{name: "launcherSharedLib"}
+ pyIdentifierRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
+ pyExt = ".py"
+ protoExt = ".proto"
+ pyVersion2 = "PY2"
+ pyVersion3 = "PY3"
+ initFileName = "__init__.py"
+ mainFileName = "__main__.py"
+ entryPointFile = "entry_point.txt"
+ parFileExt = ".zip"
+ internal = "internal"
)
// create version variants for modules.
@@ -253,43 +254,78 @@ func (p *Module) HostToolPath() android.OptionalPath {
func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool {
switch actual_version {
case pyVersion2:
- return proptools.Bool(p.properties.Version.Py2.Embedded_launcher)
+ return Bool(p.properties.Version.Py2.Embedded_launcher)
case pyVersion3:
- return proptools.Bool(p.properties.Version.Py3.Embedded_launcher)
+ return Bool(p.properties.Version.Py3.Embedded_launcher)
}
return false
}
-func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
- // deps from "data".
- android.ExtractSourcesDeps(ctx, p.properties.Data)
- // deps from "srcs".
- android.ExtractSourcesDeps(ctx, p.properties.Srcs)
- android.ExtractSourcesDeps(ctx, p.properties.Exclude_srcs)
+func hasSrcExt(srcs []string, ext string) bool {
+ for _, src := range srcs {
+ if filepath.Ext(src) == ext {
+ return true
+ }
+ }
+
+ return false
+}
+func (p *Module) hasSrcExt(ctx android.BottomUpMutatorContext, ext string) bool {
+ if hasSrcExt(p.properties.Srcs, protoExt) {
+ return true
+ }
switch p.properties.Actual_version {
case pyVersion2:
- // deps from "version.py2.srcs" property.
- android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs)
- android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Exclude_srcs)
+ return hasSrcExt(p.properties.Version.Py2.Srcs, protoExt)
+ case pyVersion3:
+ return hasSrcExt(p.properties.Version.Py3.Srcs, protoExt)
+ default:
+ panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+ p.properties.Actual_version, ctx.ModuleName()))
+ }
+}
+
+func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
+ android.ProtoDeps(ctx, &p.protoProperties)
+ if p.hasSrcExt(ctx, protoExt) && p.Name() != "libprotobuf-python" {
+ ctx.AddVariationDependencies(nil, pythonLibTag, "libprotobuf-python")
+ }
+ switch p.properties.Actual_version {
+ case pyVersion2:
ctx.AddVariationDependencies(nil, pythonLibTag,
uniqueLibs(ctx, p.properties.Libs, "version.py2.libs",
p.properties.Version.Py2.Libs)...)
if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) {
ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib")
+
+ launcherModule := "py2-launcher"
+ if p.bootstrapper.autorun() {
+ launcherModule = "py2-launcher-autorun"
+ }
+ ctx.AddFarVariationDependencies([]blueprint.Variation{
+ {Mutator: "arch", Variation: ctx.Target().String()},
+ }, launcherTag, launcherModule)
+
+ // Add py2-launcher shared lib dependencies. Ideally, these should be
+ // derived from the `shared_libs` property of "py2-launcher". However, we
+ // cannot read the property at this stage and it will be too late to add
+ // dependencies later.
ctx.AddFarVariationDependencies([]blueprint.Variation{
- {"arch", ctx.Target().String()},
- }, launcherTag, "py2-launcher")
+ {Mutator: "arch", Variation: ctx.Target().String()},
+ }, launcherSharedLibTag, "libsqlite")
+
+ if ctx.Target().Os.Bionic() {
+ ctx.AddFarVariationDependencies([]blueprint.Variation{
+ {Mutator: "arch", Variation: ctx.Target().String()},
+ }, launcherSharedLibTag, "libc", "libdl", "libm")
+ }
}
case pyVersion3:
- // deps from "version.py3.srcs" property.
- android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs)
- android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Exclude_srcs)
-
ctx.AddVariationDependencies(nil, pythonLibTag,
uniqueLibs(ctx, p.properties.Libs, "version.py3.libs",
p.properties.Version.Py3.Libs)...)
@@ -336,20 +372,31 @@ func uniqueLibs(ctx android.BottomUpMutatorContext,
func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
p.GeneratePythonBuildActions(ctx)
+ // Only Python binaries and test has non-empty bootstrapper.
if p.bootstrapper != nil {
+ p.walkTransitiveDeps(ctx)
// TODO(nanzhang): Since embedded launcher is not supported for Python3 for now,
// so we initialize "embedded_launcher" to false.
- embedded_launcher := false
+ embeddedLauncher := false
if p.properties.Actual_version == pyVersion2 {
- embedded_launcher = p.isEmbeddedLauncherEnabled(pyVersion2)
+ embeddedLauncher = p.isEmbeddedLauncherEnabled(pyVersion2)
}
p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
- embedded_launcher, p.srcsPathMappings, p.parSpec, p.depsPyRunfiles,
- p.depsParSpecs)
+ embeddedLauncher, p.srcsPathMappings, p.srcsZip, p.depsSrcsZips)
}
- if p.installer != nil && p.installSource.Valid() {
- p.installer.install(ctx, p.installSource.Path())
+ if p.installer != nil {
+ var sharedLibs []string
+ ctx.VisitDirectDeps(func(dep android.Module) {
+ if ctx.OtherModuleDependencyTag(dep) == launcherSharedLibTag {
+ sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep))
+ }
+ })
+ p.installer.setAndroidMkSharedLibs(sharedLibs)
+
+ if p.installSource.Valid() {
+ p.installer.install(ctx, p.installSource.Path())
+ }
}
}
@@ -369,65 +416,60 @@ func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) {
panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
p.properties.Actual_version, ctx.ModuleName()))
}
- expandedSrcs := ctx.ExpandSources(srcs, exclude_srcs)
- if len(expandedSrcs) == 0 {
+ expandedSrcs := android.PathsForModuleSrcExcludes(ctx, srcs, exclude_srcs)
+ requiresSrcs := true
+ if p.bootstrapper != nil && !p.bootstrapper.autorun() {
+ requiresSrcs = false
+ }
+ if len(expandedSrcs) == 0 && requiresSrcs {
ctx.ModuleErrorf("doesn't have any source files!")
}
// expand data files from "data" property.
- expandedData := ctx.ExpandSources(p.properties.Data, nil)
+ expandedData := android.PathsForModuleSrc(ctx, p.properties.Data)
// sanitize pkg_path.
- pkg_path := String(p.properties.Pkg_path)
- if pkg_path != "" {
- pkg_path = filepath.Clean(String(p.properties.Pkg_path))
- if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") ||
- strings.HasPrefix(pkg_path, "/") {
+ pkgPath := String(p.properties.Pkg_path)
+ if pkgPath != "" {
+ pkgPath = filepath.Clean(String(p.properties.Pkg_path))
+ if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") ||
+ strings.HasPrefix(pkgPath, "/") {
ctx.PropertyErrorf("pkg_path",
"%q must be a relative path contained in par file.",
String(p.properties.Pkg_path))
return
}
if p.properties.Is_internal != nil && *p.properties.Is_internal {
- // pkg_path starts from "internal/" implicitly.
- pkg_path = filepath.Join(internal, pkg_path)
- } else {
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = filepath.Join(runFiles, pkg_path)
+ pkgPath = filepath.Join(internal, pkgPath)
}
} else {
if p.properties.Is_internal != nil && *p.properties.Is_internal {
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = internal
- } else {
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = runFiles
+ pkgPath = internal
}
}
- p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData)
-
- p.parSpec = p.dumpFileList(ctx, pkg_path)
+ p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData)
- p.uniqWholeRunfilesTree(ctx)
+ p.srcsZip = p.createSrcsZip(ctx, pkgPath)
}
// generate current module unique pathMappings: <dest: runfiles_path, src: source_path>
// for python/data files.
-func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
+func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string,
expandedSrcs, expandedData android.Paths) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
- // check duplicates.
+ // check current module duplicates.
destToPySrcs := make(map[string]string)
destToPyData := make(map[string]string)
for _, s := range expandedSrcs {
- if s.Ext() != pyExt {
- ctx.PropertyErrorf("srcs", "found non (.py) file: %q!", s.String())
+ if s.Ext() != pyExt && s.Ext() != protoExt {
+ ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String())
continue
}
- runfilesPath := filepath.Join(pkg_path, s.Rel())
- identifiers := strings.Split(strings.TrimSuffix(runfilesPath, pyExt), "/")
+ runfilesPath := filepath.Join(pkgPath, s.Rel())
+ identifiers := strings.Split(strings.TrimSuffix(runfilesPath,
+ filepath.Ext(runfilesPath)), "/")
for _, token := range identifiers {
if !pyIdentifierRegexp.MatchString(token) {
ctx.PropertyErrorf("srcs", "the path %q contains invalid token %q.",
@@ -441,56 +483,96 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path strin
}
for _, d := range expandedData {
- if d.Ext() == pyExt {
- ctx.PropertyErrorf("data", "found (.py) file: %q!", d.String())
+ if d.Ext() == pyExt || d.Ext() == protoExt {
+ ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String())
continue
}
- runfilesPath := filepath.Join(pkg_path, d.Rel())
+ runfilesPath := filepath.Join(pkgPath, d.Rel())
if fillInMap(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) {
p.dataPathMappings = append(p.dataPathMappings,
pathMapping{dest: runfilesPath, src: d})
}
}
-
}
-// register build actions to dump filelist to disk.
-func (p *Module) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
+// register build actions to zip current module's sources.
+func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
relativeRootMap := make(map[string]android.Paths)
- // the soong_zip params in order to pack current module's Python/data files.
- ret := parSpec{rootPrefix: pkg_path}
-
pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
+ var protoSrcs android.Paths
// "srcs" or "data" properties may have filegroup so it might happen that
// the relative root for each source path is different.
for _, path := range pathMappings {
- var relativeRoot string
- relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
- if v, found := relativeRootMap[relativeRoot]; found {
- relativeRootMap[relativeRoot] = append(v, path.src)
+ if path.src.Ext() == protoExt {
+ protoSrcs = append(protoSrcs, path.src)
} else {
- relativeRootMap[relativeRoot] = android.Paths{path.src}
+ var relativeRoot string
+ relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
+ if v, found := relativeRootMap[relativeRoot]; found {
+ relativeRootMap[relativeRoot] = append(v, path.src)
+ } else {
+ relativeRootMap[relativeRoot] = android.Paths{path.src}
+ }
}
}
+ var zips android.Paths
+ if len(protoSrcs) > 0 {
+ protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
+ protoFlags.OutTypeFlag = "--python_out"
+
+ for _, srcFile := range protoSrcs {
+ zip := genProto(ctx, srcFile, protoFlags, pkgPath)
+ zips = append(zips, zip)
+ }
+ }
+
+ if len(relativeRootMap) > 0 {
+ var keys []string
- var keys []string
+ // in order to keep stable order of soong_zip params, we sort the keys here.
+ for k := range relativeRootMap {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
- // in order to keep stable order of soong_zip params, we sort the keys here.
- for k := range relativeRootMap {
- keys = append(keys, k)
+ parArgs := []string{}
+ if pkgPath != "" {
+ parArgs = append(parArgs, `-P `+pkgPath)
+ }
+ implicits := android.Paths{}
+ for _, k := range keys {
+ parArgs = append(parArgs, `-C `+k)
+ for _, path := range relativeRootMap[k] {
+ parArgs = append(parArgs, `-f `+path.String())
+ implicits = append(implicits, path)
+ }
+ }
+
+ origSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".py.srcszip")
+ ctx.Build(pctx, android.BuildParams{
+ Rule: zip,
+ Description: "python library archive",
+ Output: origSrcsZip,
+ Implicits: implicits,
+ Args: map[string]string{
+ "args": strings.Join(parArgs, " "),
+ },
+ })
+ zips = append(zips, origSrcsZip)
}
- sort.Strings(keys)
-
- for _, k := range keys {
- // use relative root as filelist name.
- fileListPath := registerBuildActionForModuleFileList(
- ctx, strings.Replace(k, "/", "_", -1), relativeRootMap[k])
- ret.fileListSpecs = append(ret.fileListSpecs,
- fileListSpec{fileList: fileListPath, relativeRoot: k})
+ if len(zips) == 1 {
+ return zips[0]
+ } else {
+ combinedSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszip")
+ ctx.Build(pctx, android.BuildParams{
+ Rule: combineZip,
+ Description: "combine python library archive",
+ Output: combinedSrcsZip,
+ Inputs: zips,
+ })
+ return combinedSrcsZip
}
-
- return ret
}
func isPythonLibModule(module blueprint.Module) bool {
@@ -504,8 +586,9 @@ func isPythonLibModule(module blueprint.Module) bool {
return false
}
-// check Python source/data files duplicates from current module and its whole dependencies.
-func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) {
+// check Python source/data files duplicates for whole runfiles tree since Python binary/test
+// need collect and zip all srcs of whole transitive dependencies to a final par file.
+func (p *Module) walkTransitiveDeps(ctx android.ModuleContext) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates.
destToPySrcs := make(map[string]string)
@@ -518,46 +601,46 @@ func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) {
destToPyData[path.dest] = path.src.String()
}
+ seen := make(map[android.Module]bool)
+
// visit all its dependencies in depth first.
- ctx.VisitDepsDepthFirst(func(module android.Module) {
- if ctx.OtherModuleDependencyTag(module) != pythonLibTag {
- return
+ ctx.WalkDeps(func(child, parent android.Module) bool {
+ if ctx.OtherModuleDependencyTag(child) != pythonLibTag {
+ return false
}
- // Python module cannot depend on modules, except for Python library.
- if !isPythonLibModule(module) {
+ if seen[child] {
+ return false
+ }
+ seen[child] = true
+ // Python modules only can depend on Python libraries.
+ if !isPythonLibModule(child) {
panic(fmt.Errorf(
"the dependency %q of module %q is not Python library!",
- ctx.ModuleName(), ctx.OtherModuleName(module)))
+ ctx.ModuleName(), ctx.OtherModuleName(child)))
}
- if dep, ok := module.(PythonDependency); ok {
+ if dep, ok := child.(PythonDependency); ok {
srcs := dep.GetSrcsPathMappings()
for _, path := range srcs {
if !fillInMap(ctx, destToPySrcs,
- path.dest, path.src.String(), ctx.ModuleName(),
- ctx.OtherModuleName(module)) {
+ path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) {
continue
}
- // binary needs the Python runfiles paths from all its
- // dependencies to fill __init__.py in each runfiles dir.
- p.depsPyRunfiles = append(p.depsPyRunfiles, path.dest)
}
data := dep.GetDataPathMappings()
for _, path := range data {
fillInMap(ctx, destToPyData,
- path.dest, path.src.String(), ctx.ModuleName(),
- ctx.OtherModuleName(module))
+ path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
}
- // binary needs the soong_zip arguments from all its
- // dependencies to generate executable par file.
- p.depsParSpecs = append(p.depsParSpecs, dep.GetParSpec())
+ p.depsSrcsZips = append(p.depsSrcsZips, dep.GetSrcsZip())
}
+ return true
})
}
func fillInMap(ctx android.ModuleContext, m map[string]string,
key, value, curModule, otherModule string) bool {
if oldValue, found := m[key]; found {
- ctx.ModuleErrorf("found two files to be placed at the same runfiles location %q."+
+ ctx.ModuleErrorf("found two files to be placed at the same location within zip %q."+
" First file: in module %s at path %q."+
" Second file: in module %s at path %q.",
key, curModule, oldValue, otherModule, value)
@@ -574,4 +657,5 @@ func (p *Module) InstallInData() bool {
}
var Bool = proptools.Bool
+var BoolDefault = proptools.BoolDefault
var String = proptools.String