diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/Android.bp | 5 | ||||
| -rw-r--r-- | python/androidmk.go | 49 | ||||
| -rw-r--r-- | python/binary.go | 99 | ||||
| -rw-r--r-- | python/builder.go | 4 | ||||
| -rw-r--r-- | python/library.go | 14 | ||||
| -rw-r--r-- | python/proto.go | 8 | ||||
| -rw-r--r-- | python/python.go | 506 | ||||
| -rw-r--r-- | python/python_test.go | 165 | ||||
| -rw-r--r-- | python/scripts/stub_template_host.txt | 4 | ||||
| -rw-r--r-- | python/test.go | 43 | ||||
| -rw-r--r-- | python/testing.go | 24 | ||||
| -rw-r--r-- | python/tests/Android.bp | 14 |
12 files changed, 538 insertions, 397 deletions
diff --git a/python/Android.bp b/python/Android.bp index ffd03fe89..e49fa6a3c 100644 --- a/python/Android.bp +++ b/python/Android.bp @@ -1,3 +1,7 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + bootstrap_go_package { name: "soong-python", pkgPath: "android/soong/python", @@ -16,6 +20,7 @@ bootstrap_go_package { "proto.go", "python.go", "test.go", + "testing.go", ], testSrcs: [ "python_test.go", diff --git a/python/androidmk.go b/python/androidmk.go index 648b14d7f..13b41723e 100644 --- a/python/androidmk.go +++ b/python/androidmk.go @@ -15,9 +15,10 @@ package python import ( - "android/soong/android" "path/filepath" "strings" + + "android/soong/android" ) type subAndroidMkProvider interface { @@ -47,24 +48,29 @@ func (p *Module) AndroidMkEntries() []android.AndroidMkEntries { func (p *binaryDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) { entries.Class = "EXECUTABLES" - entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { - entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) - }) + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) + }) base.subAndroidMk(entries, p.pythonInstaller) } func (p *testDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) { entries.Class = "NATIVE_TESTS" - entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { - entries.AddCompatibilityTestSuites(p.binaryDecorator.binaryProperties.Test_suites...) - if p.testConfig != nil { - entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) - } + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.AddCompatibilityTestSuites(p.binaryDecorator.binaryProperties.Test_suites...) + if p.testConfig != nil { + entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) + } + + entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) - entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) + entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...) - }) + entries.SetBoolIfTrue("LOCAL_IS_UNIT_TEST", Bool(p.testProperties.Test_options.Unit_test)) + }) base.subAndroidMk(entries, p.binaryDecorator.pythonInstaller) } @@ -76,14 +82,15 @@ func (installer *pythonInstaller) AndroidMk(base *Module, entries *android.Andro } entries.Required = append(entries.Required, "libc++") - entries.ExtraEntries = append(entries.ExtraEntries, func(entries *android.AndroidMkEntries) { - path, file := filepath.Split(installer.path.ToMakePath().String()) - stem := strings.TrimSuffix(file, filepath.Ext(file)) - - entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file)) - entries.SetString("LOCAL_MODULE_PATH", path) - entries.SetString("LOCAL_MODULE_STEM", stem) - entries.AddStrings("LOCAL_SHARED_LIBRARIES", installer.androidMkSharedLibs...) - entries.SetBool("LOCAL_CHECK_ELF_FILES", false) - }) + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + path, file := filepath.Split(installer.path.ToMakePath().String()) + stem := strings.TrimSuffix(file, filepath.Ext(file)) + + entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file)) + entries.SetString("LOCAL_MODULE_PATH", path) + entries.SetString("LOCAL_MODULE_STEM", stem) + entries.AddStrings("LOCAL_SHARED_LIBRARIES", installer.androidMkSharedLibs...) + entries.SetBool("LOCAL_CHECK_ELF_FILES", false) + }) } diff --git a/python/binary.go b/python/binary.go index 695fa123b..e955492a6 100644 --- a/python/binary.go +++ b/python/binary.go @@ -20,10 +20,99 @@ import ( "fmt" "android/soong/android" + "android/soong/bazel" + + "github.com/google/blueprint/proptools" ) func init() { - android.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) + registerPythonBinaryComponents(android.InitRegistrationContext) + android.RegisterBp2BuildMutator("python_binary_host", PythonBinaryBp2Build) +} + +func registerPythonBinaryComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) +} + +type bazelPythonBinaryAttributes struct { + Main string + Srcs bazel.LabelListAttribute + Data bazel.LabelListAttribute + Python_version string +} + +type bazelPythonBinary struct { + android.BazelTargetModuleBase + bazelPythonBinaryAttributes +} + +func BazelPythonBinaryFactory() android.Module { + module := &bazelPythonBinary{} + module.AddProperties(&module.bazelPythonBinaryAttributes) + android.InitBazelTargetModule(module) + return module +} + +func (m *bazelPythonBinary) Name() string { + return m.BaseModuleName() +} + +func (m *bazelPythonBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) {} + +func PythonBinaryBp2Build(ctx android.TopDownMutatorContext) { + m, ok := ctx.Module().(*Module) + if !ok || !m.ConvertWithBp2build(ctx) { + return + } + + // a Module can be something other than a python_binary_host + if ctx.ModuleType() != "python_binary_host" { + return + } + + var main string + for _, propIntf := range m.GetProperties() { + if props, ok := propIntf.(*BinaryProperties); ok { + // main is optional. + if props.Main != nil { + main = *props.Main + break + } + } + } + // TODO(b/182306917): this doesn't fully handle all nested props versioned + // by the python version, which would have been handled by the version split + // mutator. This is sufficient for very simple python_binary_host modules + // under Bionic. + py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false) + py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) + var python_version string + if py3Enabled && py2Enabled { + panic(fmt.Errorf( + "error for '%s' module: bp2build's python_binary_host converter does not support "+ + "converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name())) + } else if py2Enabled { + python_version = "PY2" + } else { + // do nothing, since python_version defaults to PY3. + } + + srcs := android.BazelLabelForModuleSrcExcludes(ctx, m.properties.Srcs, m.properties.Exclude_srcs) + data := android.BazelLabelForModuleSrc(ctx, m.properties.Data) + + attrs := &bazelPythonBinaryAttributes{ + Main: main, + Srcs: bazel.MakeLabelListAttribute(srcs), + Data: bazel.MakeLabelListAttribute(data), + Python_version: python_version, + } + + props := bazel.BazelTargetModuleProperties{ + // Use the native py_binary rule. + Rule_class: "py_binary", + } + + ctx.CreateBazelTargetModule(BazelPythonBinaryFactory, m.Name(), props, attrs) } type BinaryProperties struct { @@ -65,7 +154,7 @@ type IntermPathProvider interface { } var ( - stubTemplateHost = "build/soong/python/scripts/stub_template_host.txt" + StubTemplateHost = "build/soong/python/scripts/stub_template_host.txt" ) func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { @@ -79,9 +168,11 @@ func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { } func PythonBinaryHostFactory() android.Module { - module, _ := NewBinary(android.HostSupportedNoCross) + module, _ := NewBinary(android.HostSupported) + + android.InitBazelModule(module) - return module.Init() + return module.init() } func (binary *binaryDecorator) autorun() bool { diff --git a/python/builder.go b/python/builder.go index 36baecdf1..7d7239c55 100644 --- a/python/builder.go +++ b/python/builder.go @@ -45,7 +45,7 @@ var ( hostPar = pctx.AndroidStaticRule("hostPar", blueprint.RuleParams{ Command: `sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` + - `echo "#!/usr/bin/env python" >${out}.prefix &&` + + `echo "#!/usr/bin/env $interp" >${out}.prefix &&` + `$mergeParCmd -p --prefix ${out}.prefix -pm $stub $out $srcsZips && ` + `chmod +x $out && (rm -f $stub; rm -f ${out}.prefix)`, CommandDeps: []string{"$mergeParCmd"}, @@ -91,7 +91,7 @@ func registerBuildActionForParFile(ctx android.ModuleContext, embeddedLauncher b if !embeddedLauncher { // the path of stub_template_host.txt from source tree. - template := android.PathForSource(ctx, stubTemplateHost) + template := android.PathForSource(ctx, StubTemplateHost) implicits = append(implicits, template) // intermediate output path for __main__.py diff --git a/python/library.go b/python/library.go index 65c1352e7..9663b3c75 100644 --- a/python/library.go +++ b/python/library.go @@ -21,18 +21,22 @@ import ( ) func init() { - android.RegisterModuleType("python_library_host", PythonLibraryHostFactory) - android.RegisterModuleType("python_library", PythonLibraryFactory) + registerPythonLibraryComponents(android.InitRegistrationContext) +} + +func registerPythonLibraryComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("python_library_host", PythonLibraryHostFactory) + ctx.RegisterModuleType("python_library", PythonLibraryFactory) } func PythonLibraryHostFactory() android.Module { - module := newModule(android.HostSupportedNoCross, android.MultilibFirst) + module := newModule(android.HostSupported, android.MultilibFirst) - return module.Init() + return module.init() } func PythonLibraryFactory() android.Module { module := newModule(android.HostAndDeviceSupported, android.MultilibBoth) - return module.Init() + return module.init() } diff --git a/python/proto.go b/python/proto.go index b71e047a5..53ebb5895 100644 --- a/python/proto.go +++ b/python/proto.go @@ -24,17 +24,17 @@ func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.P outDir := srcsZipFile.ReplaceExtension(ctx, "tmp") depFile := srcsZipFile.ReplaceExtension(ctx, "srcszip.d") - rule := android.NewRuleBuilder() + rule := android.NewRuleBuilder(pctx, ctx) rule.Command().Text("rm -rf").Flag(outDir.String()) rule.Command().Text("mkdir -p").Flag(outDir.String()) - android.ProtoRule(ctx, rule, protoFile, flags, flags.Deps, outDir, depFile, nil) + android.ProtoRule(rule, protoFile, flags, flags.Deps, outDir, depFile, nil) // Proto generated python files have an unknown package name in the path, so package the entire output directory // into a srcszip. zipCmd := rule.Command(). - BuiltTool(ctx, "soong_zip"). + BuiltTool("soong_zip"). FlagWithOutput("-o ", srcsZipFile) if pkgPath != "" { zipCmd.FlagWithArg("-P ", pkgPath) @@ -44,7 +44,7 @@ func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.P rule.Command().Text("rm -rf").Flag(outDir.String()) - rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel()) + rule.Build("protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel()) return srcsZipFile } diff --git a/python/python.go b/python/python.go index 83ce42d02..0f5b7880e 100644 --- a/python/python.go +++ b/python/python.go @@ -20,7 +20,6 @@ import ( "fmt" "path/filepath" "regexp" - "sort" "strings" "github.com/google/blueprint" @@ -30,34 +29,41 @@ import ( ) func init() { - android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("version_split", versionSplitMutator()).Parallel() - }) + registerPythonMutators(android.InitRegistrationContext) +} + +func registerPythonMutators(ctx android.RegistrationContext) { + ctx.PreDepsMutators(RegisterPythonPreDepsMutators) } -// the version properties that apply to python libraries and binaries. +// Exported to support other packages using Python modules in tests. +func RegisterPythonPreDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("python_version", versionSplitMutator()).Parallel() +} + +// the version-specific properties that apply to python modules. type VersionProperties struct { - // true, if the module is required to be built with this version. + // whether the module is required to be built with this version. + // Defaults to true for Python 3, and false otherwise. Enabled *bool `android:"arch_variant"` - // non-empty list of .py files under this strict Python version. - // srcs may reference the outputs of other modules that produce source files like genrule - // or filegroup using the syntax ":module". + // list of source files specific to this Python version. + // Using the syntax ":module", srcs may reference the outputs of other modules that produce source files, + // e.g. genrule or filegroup. Srcs []string `android:"path,arch_variant"` - // list of source files that should not be used to build the Python module. - // This is most useful in the arch/multilib variants to remove non-common files + // list of source files that should not be used to build the Python module for this version. + // This is most useful to remove files that are not common to all Python versions. Exclude_srcs []string `android:"path,arch_variant"` - // list of the Python libraries under this Python version. + // list of the Python libraries used only for this Python version. Libs []string `android:"arch_variant"` - // true, if the binary is required to be built with embedded launcher. - // TODO(nanzhang): Remove this flag when embedded Python3 is supported later. - Embedded_launcher *bool `android:"arch_variant"` + // whether the binary is required to be built with embedded launcher for this version, defaults to false. + Embedded_launcher *bool `android:"arch_variant"` // TODO(b/174041232): Remove this property } -// properties that apply to python libraries and binaries. +// properties that apply to all python modules type BaseProperties struct { // the package path prefix within the output artifact at which to place the source/data // files of the current module. @@ -84,14 +90,19 @@ type BaseProperties struct { // the test. the file extension can be arbitrary except for (.py). Data []string `android:"path,arch_variant"` + // list of java modules that provide data that should be installed alongside the test. + Java_data []string + // list of the Python libraries compatible both with Python2 and Python3. Libs []string `android:"arch_variant"` Version struct { - // all the "srcs" or Python dependencies that are to be used only for Python2. + // Python2-specific properties, including whether Python2 is supported for this module + // and version-specific sources, exclusions and dependencies. Py2 VersionProperties `android:"arch_variant"` - // all the "srcs" or Python dependencies that are to be used only for Python3. + // Python3-specific properties, including whether Python3 is supported for this module + // and version-specific sources, exclusions and dependencies. Py3 VersionProperties `android:"arch_variant"` } `android:"arch_variant"` @@ -99,8 +110,17 @@ type BaseProperties struct { // this property name is hidden from users' perspectives, and soong will populate it during // runtime. Actual_version string `blueprint:"mutated"` + + // whether the module is required to be built with actual_version. + // this is set by the python version mutator based on version-specific properties + Enabled *bool `blueprint:"mutated"` + + // whether the binary is required to be built with embedded launcher for this actual_version. + // this is set by the python version mutator based on version-specific properties + Embedded_launcher *bool `blueprint:"mutated"` } +// Used to store files of current module after expanding dependencies type pathMapping struct { dest string src android.Path @@ -109,6 +129,7 @@ type pathMapping struct { type Module struct { android.ModuleBase android.DefaultableModuleBase + android.BazelModuleBase properties BaseProperties protoProperties android.ProtoProperties @@ -117,11 +138,14 @@ type Module struct { hod android.HostOrDeviceSupported multilib android.Multilib - // the bootstrapper is used to bootstrap .par executable. - // bootstrapper might be nil (Python library module). + // interface used to bootstrap .par executable when embedded_launcher is true + // this should be set by Python modules which are runnable, e.g. binaries and tests + // bootstrapper might be nil (e.g. Python library module). bootstrapper bootstrapper - // the installer might be nil. + // interface that implements functions required for installation + // this should be set by Python modules which are runnable, e.g. binaries and tests + // installer might be nil (e.g. Python library module). installer installer // the Python files of current module after expanding source dependencies. @@ -141,9 +165,11 @@ type Module struct { // (.intermediate) module output path as installation source. installSource android.OptionalPath + // Map to ensure sub-part of the AndroidMk for this module is only added once subAndroidMkOnce map[subAndroidMkProvider]bool } +// newModule generates new Python base module func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { return &Module{ hod: hod, @@ -151,6 +177,7 @@ func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Mo } } +// bootstrapper interface should be implemented for runnable modules, e.g. binary and test type bootstrapper interface { bootstrapperProps() []interface{} bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool, @@ -160,36 +187,45 @@ type bootstrapper interface { autorun() bool } +// installer interface should be implemented for installable modules, e.g. binary and test type installer interface { install(ctx android.ModuleContext, path android.Path) setAndroidMkSharedLibs(sharedLibs []string) } -type PythonDependency interface { - GetSrcsPathMappings() []pathMapping - GetDataPathMappings() []pathMapping - GetSrcsZip() android.Path +// interface implemented by Python modules to provide source and data mappings and zip to python +// modules that depend on it +type pythonDependency interface { + getSrcsPathMappings() []pathMapping + getDataPathMappings() []pathMapping + getSrcsZip() android.Path } -func (p *Module) GetSrcsPathMappings() []pathMapping { +// getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination +func (p *Module) getSrcsPathMappings() []pathMapping { return p.srcsPathMappings } -func (p *Module) GetDataPathMappings() []pathMapping { +// getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination +func (p *Module) getDataPathMappings() []pathMapping { return p.dataPathMappings } -func (p *Module) GetSrcsZip() android.Path { +// getSrcsZip returns the filepath where the current module's source/data files are zipped. +func (p *Module) getSrcsZip() android.Path { return p.srcsZip } -var _ PythonDependency = (*Module)(nil) +var _ pythonDependency = (*Module)(nil) var _ android.AndroidMkEntriesProvider = (*Module)(nil) -func (p *Module) Init() android.Module { - +func (p *Module) init(additionalProps ...interface{}) android.Module { p.AddProperties(&p.properties, &p.protoProperties) + + // Add additional properties for bootstrapping/installation + // This is currently tied to the bootstrapper interface; + // however, these are a combination of properties for the installation and bootstrapping of a module if p.bootstrapper != nil { p.AddProperties(p.bootstrapper.bootstrapperProps()...) } @@ -200,16 +236,29 @@ func (p *Module) Init() android.Module { return p } +// Python-specific tag to transfer information on the purpose of a dependency. +// This is used when adding a dependency on a module, which can later be accessed when visiting +// dependencies. type dependencyTag struct { blueprint.BaseDependencyTag name string } +// Python-specific tag that indicates that installed files of this module should depend on installed +// files of the dependency +type installDependencyTag struct { + blueprint.BaseDependencyTag + // embedding this struct provides the installation dependency requirement + android.InstallAlwaysNeededDependencyTag + name string +} + var ( pythonLibTag = dependencyTag{name: "pythonLib"} + javaDataTag = dependencyTag{name: "javaData"} launcherTag = dependencyTag{name: "launcher"} - launcherSharedLibTag = dependencyTag{name: "launcherSharedLib"} - pyIdentifierRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) + launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"} + pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) pyExt = ".py" protoExt = ".proto" pyVersion2 = "PY2" @@ -218,39 +267,59 @@ var ( mainFileName = "__main__.py" entryPointFile = "entry_point.txt" parFileExt = ".zip" - internal = "internal" + internalPath = "internal" ) -// create version variants for modules. +// versionSplitMutator creates version variants for modules and appends the version-specific +// properties for a given variant to the properties in the variant module func versionSplitMutator() func(android.BottomUpMutatorContext) { return func(mctx android.BottomUpMutatorContext) { if base, ok := mctx.Module().(*Module); ok { versionNames := []string{} - if base.properties.Version.Py2.Enabled != nil && - *(base.properties.Version.Py2.Enabled) == true { + // collect version specific properties, so that we can merge version-specific properties + // into the module's overall properties + versionProps := []VersionProperties{} + // PY3 is first so that we alias the PY3 variant rather than PY2 if both + // are available + if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) { + versionNames = append(versionNames, pyVersion3) + versionProps = append(versionProps, base.properties.Version.Py3) + } + if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) { versionNames = append(versionNames, pyVersion2) + versionProps = append(versionProps, base.properties.Version.Py2) } - if !(base.properties.Version.Py3.Enabled != nil && - *(base.properties.Version.Py3.Enabled) == false) { - versionNames = append(versionNames, pyVersion3) + modules := mctx.CreateLocalVariations(versionNames...) + // Alias module to the first variant + if len(versionNames) > 0 { + mctx.AliasVariation(versionNames[0]) } - modules := mctx.CreateVariations(versionNames...) for i, v := range versionNames { // set the actual version for Python module. modules[i].(*Module).properties.Actual_version = v + // append versioned properties for the Python module to the overall properties + err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil) + if err != nil { + panic(err) + } } } } } +// HostToolPath returns a path if appropriate such that this module can be used as a host tool, +// fulfilling HostToolProvider interface. func (p *Module) HostToolPath() android.OptionalPath { if p.installer == nil { // python_library is just meta module, and doesn't have any installer. return android.OptionalPath{} } + // TODO: This should only be set when building host binaries -- tests built for device would be + // setting this incorrectly. return android.OptionalPathForPath(p.installer.(*binaryDecorator).path) } +// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. func (p *Module) OutputFiles(tag string) (android.Paths, error) { switch tag { case "": @@ -263,20 +332,13 @@ func (p *Module) OutputFiles(tag string) (android.Paths, error) { } } -func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool { - switch actual_version { - case pyVersion2: - return Bool(p.properties.Version.Py2.Embedded_launcher) - case pyVersion3: - return Bool(p.properties.Version.Py3.Embedded_launcher) - } - - return false +func (p *Module) isEmbeddedLauncherEnabled() bool { + return p.installer != nil && Bool(p.properties.Embedded_launcher) } -func hasSrcExt(srcs []string, ext string) bool { - for _, src := range srcs { - if filepath.Ext(src) == ext { +func anyHasExt(paths []string, ext string) bool { + for _, p := range paths { + if filepath.Ext(p) == ext { return true } } @@ -284,166 +346,120 @@ func hasSrcExt(srcs []string, ext string) bool { return false } -func (p *Module) hasSrcExt(ctx android.BottomUpMutatorContext, ext string) bool { - if hasSrcExt(p.properties.Srcs, protoExt) { - return true - } - switch p.properties.Actual_version { - case pyVersion2: - return hasSrcExt(p.properties.Version.Py2.Srcs, protoExt) - case pyVersion3: - return hasSrcExt(p.properties.Version.Py3.Srcs, protoExt) - default: - panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", - p.properties.Actual_version, ctx.ModuleName())) - } +func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { + return anyHasExt(p.properties.Srcs, ext) } +// DepsMutator mutates dependencies for this module: +// * handles proto dependencies, +// * if required, specifies launcher and adds launcher dependencies, +// * applies python version mutations to Python dependencies func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { android.ProtoDeps(ctx, &p.protoProperties) - if p.hasSrcExt(ctx, protoExt) && p.Name() != "libprotobuf-python" { - ctx.AddVariationDependencies(nil, pythonLibTag, "libprotobuf-python") + versionVariation := []blueprint.Variation{ + {"python_version", p.properties.Actual_version}, } - switch p.properties.Actual_version { - case pyVersion2: - ctx.AddVariationDependencies(nil, pythonLibTag, - uniqueLibs(ctx, p.properties.Libs, "version.py2.libs", - p.properties.Version.Py2.Libs)...) - if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) { - ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib") + // If sources contain a proto file, add dependency on libprotobuf-python + if p.anySrcHasExt(ctx, protoExt) && p.Name() != "libprotobuf-python" { + ctx.AddVariationDependencies(versionVariation, pythonLibTag, "libprotobuf-python") + } - launcherModule := "py2-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py2-launcher-autorun" - } - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) + // Add python library dependencies for this python version variation + ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...) + + // If this module will be installed and has an embedded launcher, we need to add dependencies for: + // * standard library + // * launcher + // * shared dependencies of the launcher + if p.installer != nil && p.isEmbeddedLauncherEnabled() { + var stdLib string + var launcherModule string + // Add launcher shared lib dependencies. Ideally, these should be + // derived from the `shared_libs` property of the launcher. However, we + // cannot read the property at this stage and it will be too late to add + // dependencies later. + launcherSharedLibDeps := []string{ + "libsqlite", + } + // Add launcher-specific dependencies for bionic + if ctx.Target().Os.Bionic() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") + } - // Add py2-launcher shared lib dependencies. Ideally, these should be - // derived from the `shared_libs` property of "py2-launcher". However, we - // cannot read the property at this stage and it will be too late to add - // dependencies later. - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, "libsqlite") + switch p.properties.Actual_version { + case pyVersion2: + stdLib = "py2-stdlib" - if ctx.Target().Os.Bionic() { - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, - "libc", "libdl", "libm") + launcherModule = "py2-launcher" + if p.bootstrapper.autorun() { + launcherModule = "py2-launcher-autorun" } - } + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") - case pyVersion3: - ctx.AddVariationDependencies(nil, pythonLibTag, - uniqueLibs(ctx, p.properties.Libs, "version.py3.libs", - p.properties.Version.Py3.Libs)...) + case pyVersion3: + stdLib = "py3-stdlib" - if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion3) { - ctx.AddVariationDependencies(nil, pythonLibTag, "py3-stdlib") - - launcherModule := "py3-launcher" + launcherModule = "py3-launcher" if p.bootstrapper.autorun() { launcherModule = "py3-launcher-autorun" } - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) - - // Add py3-launcher shared lib dependencies. Ideally, these should be - // derived from the `shared_libs` property of "py3-launcher". However, we - // cannot read the property at this stage and it will be too late to add - // dependencies later. - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, "libsqlite") if ctx.Device() { - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, - "liblog") - } - - if ctx.Target().Os.Bionic() { - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, - "libc", "libdl", "libm") + launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") } + default: + panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", + p.properties.Actual_version, ctx.ModuleName())) } - default: - panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", - p.properties.Actual_version, ctx.ModuleName())) + ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...) } -} - -// check "libs" duplicates from current module dependencies. -func uniqueLibs(ctx android.BottomUpMutatorContext, - commonLibs []string, versionProp string, versionLibs []string) []string { - set := make(map[string]string) - ret := []string{} - // deps from "libs" property. - for _, l := range commonLibs { - if _, found := set[l]; found { - ctx.PropertyErrorf("libs", "%q has duplicates within libs.", l) - } else { - set[l] = "libs" - ret = append(ret, l) - } - } - // deps from "version.pyX.libs" property. - for _, l := range versionLibs { - if _, found := set[l]; found { - ctx.PropertyErrorf(versionProp, "%q has duplicates within %q.", set[l]) - } else { - set[l] = versionProp - ret = append(ret, l) - } - } - - return ret + // Emulate the data property for java_data but with the arch variation overridden to "common" + // so that it can point to java modules. + javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}} + ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...) } func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { - p.GeneratePythonBuildActions(ctx) + p.generatePythonBuildActions(ctx) - // Only Python binaries and test has non-empty bootstrapper. + // Only Python binary and test modules have non-empty bootstrapper. if p.bootstrapper != nil { - p.walkTransitiveDeps(ctx) - embeddedLauncher := false - if p.properties.Actual_version == pyVersion2 { - embeddedLauncher = p.isEmbeddedLauncherEnabled(pyVersion2) - } else { - embeddedLauncher = p.isEmbeddedLauncherEnabled(pyVersion3) - } + // if the module is being installed, we need to collect all transitive dependencies to embed in + // the final par + p.collectPathsFromTransitiveDeps(ctx) + // bootstrap the module, including resolving main file, getting launcher path, and + // registering actions to build the par file + // bootstrap returns the binary output path p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version, - embeddedLauncher, p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) + p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) } + // Only Python binary and test modules have non-empty installer. if p.installer != nil { var sharedLibs []string - ctx.VisitDirectDeps(func(dep android.Module) { - if ctx.OtherModuleDependencyTag(dep) == launcherSharedLibTag { - sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) - } - }) + // if embedded launcher is enabled, we need to collect the shared library depenendencies of the + // launcher + for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { + sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) + } + p.installer.setAndroidMkSharedLibs(sharedLibs) + // Install the par file from installSource if p.installSource.Valid() { p.installer.install(ctx, p.installSource.Path()) } } - } -func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) { - // expand python files from "srcs" property. - srcs := p.properties.Srcs - exclude_srcs := p.properties.Exclude_srcs - switch p.properties.Actual_version { - case pyVersion2: - srcs = append(srcs, p.properties.Version.Py2.Srcs...) - exclude_srcs = append(exclude_srcs, p.properties.Version.Py2.Exclude_srcs...) - case pyVersion3: - srcs = append(srcs, p.properties.Version.Py3.Srcs...) - exclude_srcs = append(exclude_srcs, p.properties.Version.Py3.Exclude_srcs...) - default: - panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", - p.properties.Actual_version, ctx.ModuleName())) - } - expandedSrcs := android.PathsForModuleSrcExcludes(ctx, srcs, exclude_srcs) +// generatePythonBuildActions performs build actions common to all Python modules +func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) { + expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) requiresSrcs := true if p.bootstrapper != nil && !p.bootstrapper.autorun() { requiresSrcs = false @@ -455,9 +471,15 @@ func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) { // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) - // sanitize pkg_path. + // Emulate the data property for java_data dependencies. + for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { + expandedData = append(expandedData, android.OutputFilesForModule(ctx, javaData, "")...) + } + + // Validate pkg_path property pkgPath := String(p.properties.Pkg_path) if pkgPath != "" { + // TODO: export validation from android/paths.go handling to replace this duplicated functionality pkgPath = filepath.Clean(String(p.properties.Pkg_path)) if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") || strings.HasPrefix(pkgPath, "/") { @@ -466,22 +488,35 @@ func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) { String(p.properties.Pkg_path)) return } - if p.properties.Is_internal != nil && *p.properties.Is_internal { - pkgPath = filepath.Join(internal, pkgPath) - } - } else { - if p.properties.Is_internal != nil && *p.properties.Is_internal { - pkgPath = internal - } + } + // If property Is_internal is set, prepend pkgPath with internalPath + if proptools.BoolDefault(p.properties.Is_internal, false) { + pkgPath = filepath.Join(internalPath, pkgPath) } + // generate src:destination path mappings for this module p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData) + // generate the zipfile of all source and data files p.srcsZip = p.createSrcsZip(ctx, pkgPath) } -// generate current module unique pathMappings: <dest: runfiles_path, src: source_path> -// for python/data files. +func isValidPythonPath(path string) error { + identifiers := strings.Split(strings.TrimSuffix(path, filepath.Ext(path)), "/") + for _, token := range identifiers { + if !pathComponentRegexp.MatchString(token) { + return fmt.Errorf("the path %q contains invalid subpath %q. "+ + "Subpaths must be at least one character long. "+ + "The first character must an underscore or letter. "+ + "Following characters may be any of: letter, digit, underscore, hyphen.", + path, token) + } + } + return nil +} + +// For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path> +// for python/data files expanded from properties. func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string, expandedSrcs, expandedData android.Paths) { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to @@ -495,17 +530,11 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string continue } runfilesPath := filepath.Join(pkgPath, s.Rel()) - identifiers := strings.Split(strings.TrimSuffix(runfilesPath, - filepath.Ext(runfilesPath)), "/") - for _, token := range identifiers { - if !pyIdentifierRegexp.MatchString(token) { - ctx.PropertyErrorf("srcs", "the path %q contains invalid token %q.", - runfilesPath, token) - } + if err := isValidPythonPath(runfilesPath); err != nil { + ctx.PropertyErrorf("srcs", err.Error()) } - if fillInMap(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) { - p.srcsPathMappings = append(p.srcsPathMappings, - pathMapping{dest: runfilesPath, src: s}) + if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) { + p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s}) } } @@ -515,22 +544,23 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string continue } runfilesPath := filepath.Join(pkgPath, d.Rel()) - if fillInMap(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) { + if !checkForDuplicateOutputPath(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) { p.dataPathMappings = append(p.dataPathMappings, pathMapping{dest: runfilesPath, src: d}) } } } -// register build actions to zip current module's sources. +// createSrcsZip registers build actions to zip current module's sources and data. func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { relativeRootMap := make(map[string]android.Paths) pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) var protoSrcs android.Paths - // "srcs" or "data" properties may have filegroup so it might happen that - // the relative root for each source path is different. + // "srcs" or "data" properties may contain filegroup so it might happen that + // the root directory for each source path is different. for _, path := range pathMappings { + // handle proto sources separately if path.src.Ext() == protoExt { protoSrcs = append(protoSrcs, path.src) } else { @@ -555,24 +585,21 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi } if len(relativeRootMap) > 0 { - var keys []string - // in order to keep stable order of soong_zip params, we sort the keys here. - for k := range relativeRootMap { - keys = append(keys, k) - } - sort.Strings(keys) + roots := android.SortedStringKeys(relativeRootMap) parArgs := []string{} if pkgPath != "" { + // use package path as path prefix parArgs = append(parArgs, `-P `+pkgPath) } - implicits := android.Paths{} - for _, k := range keys { - parArgs = append(parArgs, `-C `+k) - for _, path := range relativeRootMap[k] { + paths := android.Paths{} + for _, root := range roots { + // specify relative root of file in following -f arguments + parArgs = append(parArgs, `-C `+root) + for _, path := range relativeRootMap[root] { parArgs = append(parArgs, `-f `+path.String()) - implicits = append(implicits, path) + paths = append(paths, path) } } @@ -581,13 +608,15 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi Rule: zip, Description: "python library archive", Output: origSrcsZip, - Implicits: implicits, + // as zip rule does not use $in, there is no real need to distinguish between Inputs and Implicits + Implicits: paths, Args: map[string]string{ "args": strings.Join(parArgs, " "), }, }) zips = append(zips, origSrcsZip) } + // we may have multiple zips due to separate handling of proto source files if len(zips) == 1 { return zips[0] } else { @@ -602,25 +631,27 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi } } +// isPythonLibModule returns whether the given module is a Python library Module or not +// This is distinguished by the fact that Python libraries are not installable, while other Python +// modules are. func isPythonLibModule(module blueprint.Module) bool { if m, ok := module.(*Module); ok { - // Python library has no bootstrapper or installer. - if m.bootstrapper != nil || m.installer != nil { - return false + // Python library has no bootstrapper or installer + if m.bootstrapper == nil && m.installer == nil { + return true } - return true } return false } -// check Python source/data files duplicates for whole runfiles tree since Python binary/test -// need collect and zip all srcs of whole transitive dependencies to a final par file. -func (p *Module) walkTransitiveDeps(ctx android.ModuleContext) { +// collectPathsFromTransitiveDeps checks for source/data files for duplicate paths +// for module and its transitive dependencies and collects list of data/source file +// zips for transitive dependencies. +func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // check duplicates. destToPySrcs := make(map[string]string) destToPyData := make(map[string]string) - for _, path := range p.srcsPathMappings { destToPySrcs[path.dest] = path.src.String() } @@ -632,6 +663,7 @@ func (p *Module) walkTransitiveDeps(ctx android.ModuleContext) { // visit all its dependencies in depth first. ctx.WalkDeps(func(child, parent android.Module) bool { + // we only collect dependencies tagged as python library deps if ctx.OtherModuleDependencyTag(child) != pythonLibTag { return false } @@ -641,44 +673,46 @@ func (p *Module) walkTransitiveDeps(ctx android.ModuleContext) { seen[child] = true // Python modules only can depend on Python libraries. if !isPythonLibModule(child) { - panic(fmt.Errorf( + ctx.PropertyErrorf("libs", "the dependency %q of module %q is not Python library!", - ctx.ModuleName(), ctx.OtherModuleName(child))) + ctx.ModuleName(), ctx.OtherModuleName(child)) } - if dep, ok := child.(PythonDependency); ok { - srcs := dep.GetSrcsPathMappings() + // collect source and data paths, checking that there are no duplicate output file conflicts + if dep, ok := child.(pythonDependency); ok { + srcs := dep.getSrcsPathMappings() for _, path := range srcs { - if !fillInMap(ctx, destToPySrcs, - path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) { - continue - } + checkForDuplicateOutputPath(ctx, destToPySrcs, + path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) } - data := dep.GetDataPathMappings() + data := dep.getDataPathMappings() for _, path := range data { - fillInMap(ctx, destToPyData, + checkForDuplicateOutputPath(ctx, destToPyData, path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) } - p.depsSrcsZips = append(p.depsSrcsZips, dep.GetSrcsZip()) + p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip()) } return true }) } -func fillInMap(ctx android.ModuleContext, m map[string]string, - key, value, curModule, otherModule string) bool { - if oldValue, found := m[key]; found { +// chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which +// would result in two files being placed in the same location. +// If there is a duplicate path, an error is thrown and true is returned +// Otherwise, outputPath: srcPath is added to m and returns false +func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, outputPath, srcPath, curModule, otherModule string) bool { + if oldSrcPath, found := m[outputPath]; found { ctx.ModuleErrorf("found two files to be placed at the same location within zip %q."+ " First file: in module %s at path %q."+ " Second file: in module %s at path %q.", - key, curModule, oldValue, otherModule, value) - return false - } else { - m[key] = value + outputPath, curModule, oldSrcPath, otherModule, srcPath) + return true } + m[outputPath] = srcPath - return true + return false } +// InstallInData returns true as Python is not supported in the system partition func (p *Module) InstallInData() bool { return true } diff --git a/python/python_test.go b/python/python_test.go index 1245ca184..f57f504d7 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -15,21 +15,15 @@ package python import ( - "errors" "fmt" - "io/ioutil" "os" "path/filepath" - "reflect" - "sort" - "strings" + "regexp" "testing" "android/soong/android" ) -var buildDir string - type pyModule struct { name string actualVersion string @@ -44,7 +38,7 @@ var ( pkgPathErrTemplate = moduleVariantErrTemplate + "pkg_path: %q must be a relative path contained in par file." badIdentifierErrTemplate = moduleVariantErrTemplate + - "srcs: the path %q contains invalid token %q." + "srcs: the path %q contains invalid subpath %q." dupRunfileErrTemplate = moduleVariantErrTemplate + "found two files to be placed at the same location within zip %q." + " First file: in module %s at path %q." + @@ -56,7 +50,7 @@ var ( data = []struct { desc string - mockFiles map[string][]byte + mockFiles android.MockFS errors []string expectedBinaries []pyModule @@ -64,7 +58,6 @@ var ( { desc: "module without any src files", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_library_host { name: "lib1", @@ -79,7 +72,6 @@ var ( { desc: "module with bad src file ext", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_library_host { name: "lib1", @@ -98,7 +90,6 @@ var ( { desc: "module with bad data file ext", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_library_host { name: "lib1", @@ -121,7 +112,6 @@ var ( { desc: "module with bad pkg_path format", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_library_host { name: "lib1", @@ -159,7 +149,6 @@ var ( { desc: "module with bad runfile src path format", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_library_host { name: "lib1", @@ -187,7 +176,6 @@ var ( { desc: "module with duplicate runfile path", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_library_host { name: "lib1", @@ -207,21 +195,32 @@ var ( "lib1", ], } + + python_binary_host { + name: "bin", + pkg_path: "e/", + srcs: [ + "bin.py", + ], + libs: [ + "lib2", + ], + } `, ), "dir/c/file1.py": nil, "dir/file1.py": nil, + "dir/bin.py": nil, }, errors: []string{ - fmt.Sprintf(dupRunfileErrTemplate, "dir/Android.bp:9:6", - "lib2", "PY3", "a/b/c/file1.py", "lib2", "dir/file1.py", + fmt.Sprintf(dupRunfileErrTemplate, "dir/Android.bp:20:6", + "bin", "PY3", "a/b/c/file1.py", "bin", "dir/file1.py", "lib1", "dir/c/file1.py"), }, }, { desc: "module for testing dependencies", mockFiles: map[string][]byte{ - bpFile: []byte(`subdirs = ["dir"]`), filepath.Join("dir", bpFile): []byte( `python_defaults { name: "default_lib", @@ -301,7 +300,7 @@ var ( filepath.Join("dir", "file2.py"): nil, filepath.Join("dir", "bin.py"): nil, filepath.Join("dir", "file4.py"): nil, - stubTemplateHost: []byte(`PYTHON_BINARY = '%interpreter%' + StubTemplateHost: []byte(`PYTHON_BINARY = '%interpreter%' MAIN_FILE = '%main%'`), }, expectedBinaries: []pyModule{ @@ -314,10 +313,10 @@ var ( "e/default_py3.py", "e/file4.py", }, - srcsZip: "@prefix@/.intermediates/dir/bin/PY3/bin.py.srcszip", + srcsZip: "out/soong/.intermediates/dir/bin/PY3/bin.py.srcszip", depsSrcsZips: []string{ - "@prefix@/.intermediates/dir/lib5/PY3/lib5.py.srcszip", - "@prefix@/.intermediates/dir/lib6/PY3/lib6.py.srcszip", + "out/soong/.intermediates/dir/lib5/PY3/lib5.py.srcszip", + "out/soong/.intermediates/dir/lib6/PY3/lib6.py.srcszip", }, }, }, @@ -327,62 +326,36 @@ var ( func TestPythonModule(t *testing.T) { for _, d := range data { + if d.desc != "module with duplicate runfile path" { + continue + } + errorPatterns := make([]string, len(d.errors)) + for i, s := range d.errors { + errorPatterns[i] = regexp.QuoteMeta(s) + } + t.Run(d.desc, func(t *testing.T) { - config := android.TestConfig(buildDir, nil, "", d.mockFiles) - ctx := android.NewTestContext() - ctx.PreDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("version_split", versionSplitMutator()).Parallel() - }) - ctx.RegisterModuleType("python_library_host", PythonLibraryHostFactory) - ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) - ctx.RegisterModuleType("python_defaults", defaultsFactory) - ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) - ctx.Register(config) - _, testErrs := ctx.ParseBlueprintsFiles(bpFile) - android.FailIfErrored(t, testErrs) - _, actErrs := ctx.PrepareBuildActions(config) - if len(actErrs) > 0 { - testErrs = append(testErrs, expectErrors(t, actErrs, d.errors)...) - } else { - for _, e := range d.expectedBinaries { - testErrs = append(testErrs, - expectModule(t, ctx, buildDir, e.name, - e.actualVersion, - e.srcsZip, - e.pyRunfiles, - e.depsSrcsZips)...) - } + result := android.GroupFixturePreparers( + android.PrepareForTestWithDefaults, + PrepareForTestWithPythonBuildComponents, + d.mockFiles.AddToFixture(), + ).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(errorPatterns)). + RunTest(t) + + if len(result.Errs) > 0 { + return } - android.FailIfErrored(t, testErrs) - }) - } -} -func expectErrors(t *testing.T, actErrs []error, expErrs []string) (testErrs []error) { - actErrStrs := []string{} - for _, v := range actErrs { - actErrStrs = append(actErrStrs, v.Error()) - } - sort.Strings(actErrStrs) - if len(actErrStrs) != len(expErrs) { - t.Errorf("got (%d) errors, expected (%d) errors!", len(actErrStrs), len(expErrs)) - for _, v := range actErrStrs { - testErrs = append(testErrs, errors.New(v)) - } - } else { - sort.Strings(expErrs) - for i, v := range actErrStrs { - if v != expErrs[i] { - testErrs = append(testErrs, errors.New(v)) + for _, e := range d.expectedBinaries { + t.Run(e.name, func(t *testing.T) { + expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles, e.depsSrcsZips) + }) } - } + }) } - - return } -func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, variant, expectedSrcsZip string, - expectedPyRunfiles, expectedDepsSrcsZips []string) (testErrs []error) { +func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles, expectedDepsSrcsZips []string) { module := ctx.ModuleForTests(name, variant) base, baseOk := module.Module().(*Module) @@ -395,55 +368,13 @@ func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, varian actualPyRunfiles = append(actualPyRunfiles, path.dest) } - if !reflect.DeepEqual(actualPyRunfiles, expectedPyRunfiles) { - testErrs = append(testErrs, errors.New(fmt.Sprintf( - `binary "%s" variant "%s" has unexpected pyRunfiles: %q!`, - base.Name(), - base.properties.Actual_version, - actualPyRunfiles))) - } - - if base.srcsZip.String() != strings.Replace(expectedSrcsZip, "@prefix@", buildDir, 1) { - testErrs = append(testErrs, errors.New(fmt.Sprintf( - `binary "%s" variant "%s" has unexpected srcsZip: %q!`, - base.Name(), - base.properties.Actual_version, - base.srcsZip))) - } - - for i, _ := range expectedDepsSrcsZips { - expectedDepsSrcsZips[i] = strings.Replace(expectedDepsSrcsZips[i], "@prefix@", buildDir, 1) - } - if !reflect.DeepEqual(base.depsSrcsZips.Strings(), expectedDepsSrcsZips) { - testErrs = append(testErrs, errors.New(fmt.Sprintf( - `binary "%s" variant "%s" has unexpected depsSrcsZips: %q!`, - base.Name(), - base.properties.Actual_version, - base.depsSrcsZips))) - } + android.AssertDeepEquals(t, "pyRunfiles", expectedPyRunfiles, actualPyRunfiles) - return -} + android.AssertPathRelativeToTopEquals(t, "srcsZip", expectedSrcsZip, base.srcsZip) -func setUp() { - var err error - buildDir, err = ioutil.TempDir("", "soong_python_test") - if err != nil { - panic(err) - } -} - -func tearDown() { - os.RemoveAll(buildDir) + android.AssertPathsRelativeToTopEquals(t, "depsSrcsZips", expectedDepsSrcsZips, base.depsSrcsZips) } func TestMain(m *testing.M) { - run := func() int { - setUp() - defer tearDown() - - return m.Run() - } - - os.Exit(run()) + os.Exit(m.Run()) } diff --git a/python/scripts/stub_template_host.txt b/python/scripts/stub_template_host.txt index a48a86f51..138404bf3 100644 --- a/python/scripts/stub_template_host.txt +++ b/python/scripts/stub_template_host.txt @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env '%interpreter%' import os import re @@ -82,7 +82,7 @@ def Main(): sys.stdout.flush() retCode = subprocess.call(args) - exit(retCode) + sys.exit(retCode) except: raise finally: diff --git a/python/test.go b/python/test.go index a669c73a6..6713189fd 100644 --- a/python/test.go +++ b/python/test.go @@ -22,8 +22,18 @@ import ( // This file contains the module types for building Python test. func init() { - android.RegisterModuleType("python_test_host", PythonTestHostFactory) - android.RegisterModuleType("python_test", PythonTestFactory) + registerPythonTestComponents(android.InitRegistrationContext) +} + +func registerPythonTestComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("python_test_host", PythonTestHostFactory) + ctx.RegisterModuleType("python_test", PythonTestFactory) +} + +// Test option struct. +type TestOptions struct { + // If the test is a hostside(no device required) unittest that shall be run during presubmit check. + Unit_test *bool } type TestProperties struct { @@ -34,6 +44,16 @@ type TestProperties struct { // the name of the test configuration template (for example "AndroidTestTemplate.xml") that // should be installed with the module. Test_config_template *string `android:"path,arch_variant"` + + // list of files or filegroup modules that provide data that should be installed alongside + // the test + Data []string `android:"path,arch_variant"` + + // list of java modules that provide data that should be installed alongside the test. + Java_data []string + + // Test options. + Test_options TestOptions } type testDecorator struct { @@ -42,6 +62,8 @@ type testDecorator struct { testProperties TestProperties testConfig android.Path + + data []android.DataPath } func (test *testDecorator) bootstrapperProps() []interface{} { @@ -59,6 +81,19 @@ func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) test.binaryDecorator.pythonInstaller.relative = ctx.ModuleName() test.binaryDecorator.pythonInstaller.install(ctx, file) + + dataSrcPaths := android.PathsForModuleSrc(ctx, test.testProperties.Data) + + for _, dataSrcPath := range dataSrcPaths { + test.data = append(test.data, android.DataPath{SrcPath: dataSrcPath}) + } + + // Emulate the data property for java_data dependencies. + for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { + for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") { + test.data = append(test.data, android.DataPath{SrcPath: javaDataSrcPath}) + } + } } func NewTest(hod android.HostOrDeviceSupported) *Module { @@ -77,12 +112,12 @@ func NewTest(hod android.HostOrDeviceSupported) *Module { func PythonTestHostFactory() android.Module { module := NewTest(android.HostSupportedNoCross) - return module.Init() + return module.init() } func PythonTestFactory() android.Module { module := NewTest(android.HostAndDeviceSupported) module.multilib = android.MultilibBoth - return module.Init() + return module.init() } diff --git a/python/testing.go b/python/testing.go new file mode 100644 index 000000000..ce1a5ab27 --- /dev/null +++ b/python/testing.go @@ -0,0 +1,24 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import "android/soong/android" + +var PrepareForTestWithPythonBuildComponents = android.GroupFixturePreparers( + android.FixtureRegisterWithContext(registerPythonBinaryComponents), + android.FixtureRegisterWithContext(registerPythonLibraryComponents), + android.FixtureRegisterWithContext(registerPythonTestComponents), + android.FixtureRegisterWithContext(registerPythonMutators), +) diff --git a/python/tests/Android.bp b/python/tests/Android.bp index c8bf42023..a65685960 100644 --- a/python/tests/Android.bp +++ b/python/tests/Android.bp @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + python_test_host { name: "par_test", main: "par_test.py", @@ -19,7 +23,10 @@ python_test_host { "par_test.py", "testpkg/par_test.py", ], - + // Is not implemented as a python unittest + test_options: { + unit_test: false, + }, version: { py2: { enabled: true, @@ -39,7 +46,10 @@ python_test_host { "par_test.py", "testpkg/par_test.py", ], - + // Is not implemented as a python unittest + test_options: { + unit_test: false, + }, version: { py3: { embedded_launcher: true, |