diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/Android.bp | 2 | ||||
| -rw-r--r-- | python/binary.go | 14 | ||||
| -rw-r--r-- | python/bp2build.go | 233 | ||||
| -rw-r--r-- | python/builder.go | 4 | ||||
| -rw-r--r-- | python/proto.go | 3 | ||||
| -rw-r--r-- | python/python.go | 72 | ||||
| -rw-r--r-- | python/python_test.go | 75 | ||||
| -rw-r--r-- | python/scripts/precompile_python.py | 41 | ||||
| -rw-r--r-- | python/test.go | 115 | ||||
| -rw-r--r-- | python/tests/Android.bp | 28 | ||||
| -rw-r--r-- | python/tests/dont_import_folder_of_entrypoint/Android.bp | 14 | ||||
| -rw-r--r-- | python/tests/par_test.py | 9 | ||||
| -rw-r--r-- | python/tests/py-cmd_test.py | 18 | ||||
| -rwxr-xr-x | python/tests/runtest.sh | 26 | ||||
| -rw-r--r-- | python/tests/testpkg/par_test.py | 6 |
15 files changed, 278 insertions, 382 deletions
diff --git a/python/Android.bp b/python/Android.bp index 75786733a..14e83c184 100644 --- a/python/Android.bp +++ b/python/Android.bp @@ -10,10 +10,10 @@ bootstrap_go_package { "soong-android", "soong-tradefed", "soong-cc", + "soong-testing", ], srcs: [ "binary.go", - "bp2build.go", "builder.go", "defaults.go", "library.go", diff --git a/python/binary.go b/python/binary.go index a5db2f6ef..5f60761be 100644 --- a/python/binary.go +++ b/python/binary.go @@ -95,7 +95,6 @@ func (p *PythonBinaryModule) init() android.Module { p.AddProperties(&p.binaryProperties) android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) - android.InitBazelModule(p) return p } @@ -104,6 +103,7 @@ func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleConte p.buildBinary(ctx) p.installedDest = ctx.InstallFile(installDir(ctx, "bin", "", ""), p.installSource.Base(), p.installSource) + ctx.SetOutputFiles(android.Paths{p.installSource}, "") } func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { @@ -188,18 +188,8 @@ func (p *PythonBinaryModule) HostToolPath() android.OptionalPath { return android.OptionalPathForPath(p.installedDest) } -// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. -func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - return android.Paths{p.installSource}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool { - return Bool(p.properties.Embedded_launcher) + return BoolDefault(p.properties.Embedded_launcher, true) } func (b *PythonBinaryModule) autorun() bool { diff --git a/python/bp2build.go b/python/bp2build.go deleted file mode 100644 index cd3f2a1a8..000000000 --- a/python/bp2build.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2023 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "path/filepath" - "strings" - - "github.com/google/blueprint/proptools" - - "android/soong/android" - "android/soong/bazel" -) - -type bazelPythonLibraryAttributes struct { - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Imports bazel.StringListAttribute - Srcs_version *string -} - -type bazelPythonProtoLibraryAttributes struct { - Deps bazel.LabelListAttribute -} - -type baseAttributes struct { - // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv - //Pkg_path bazel.StringAttribute - // TODO: Related to Pkg_bath and similarLy gated - //Is_internal bazel.BoolAttribute - // Combines Srcs and Exclude_srcs - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - // Combines Data and Java_data (invariant) - Data bazel.LabelListAttribute - Imports bazel.StringListAttribute -} - -func (m *PythonLibraryModule) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes { - var attrs baseAttributes - archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{}) - for axis, configToProps := range archVariantBaseProps { - for config, props := range configToProps { - if baseProps, ok := props.(*BaseProperties); ok { - attrs.Srcs.SetSelectValue(axis, config, - android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs)) - attrs.Deps.SetSelectValue(axis, config, - android.BazelLabelForModuleDeps(ctx, baseProps.Libs)) - data := android.BazelLabelForModuleSrc(ctx, baseProps.Data) - data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data)) - attrs.Data.SetSelectValue(axis, config, data) - } - } - } - - partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ - "proto": android.ProtoSrcLabelPartition, - "py": bazel.LabelPartition{Keep_remainder: true}, - }) - attrs.Srcs = partitionedSrcs["py"] - - if !partitionedSrcs["proto"].IsEmpty() { - protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) - protoLabel := bazel.Label{Label: ":" + protoInfo.Name} - - pyProtoLibraryName := m.Name() + "_py_proto" - ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ - Rule_class: "py_proto_library", - Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", - }, android.CommonAttributes{ - Name: pyProtoLibraryName, - }, &bazelPythonProtoLibraryAttributes{ - Deps: bazel.MakeSingleLabelListAttribute(protoLabel), - }) - - attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) - } - - // Bazel normally requires `import path.from.top.of.tree` statements in - // python code, but with soong you can directly import modules from libraries. - // Add "imports" attributes to the bazel library so it matches soong's behavior. - imports := "." - if m.properties.Pkg_path != nil { - // TODO(b/215119317) This is a hack to handle the fact that we don't convert - // pkg_path properly right now. If the folder structure that contains this - // Android.bp file matches pkg_path, we can set imports to an appropriate - // number of ../..s to emulate moving the files under a pkg_path folder. - pkg_path := filepath.Clean(*m.properties.Pkg_path) - if strings.HasPrefix(pkg_path, "/") { - ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) - } - - if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { - ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) - } - numFolders := strings.Count(pkg_path, "/") + 1 - dots := make([]string, numFolders) - for i := 0; i < numFolders; i++ { - dots[i] = ".." - } - imports = strings.Join(dots, "/") - } - attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) - - return attrs -} - -func (m *PythonLibraryModule) bp2buildPythonVersion(ctx android.TopDownMutatorContext) *string { - py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true) - py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) - if py2Enabled && !py3Enabled { - return &pyVersion2 - } else if !py2Enabled && py3Enabled { - return &pyVersion3 - } else if !py2Enabled && !py3Enabled { - ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled") - return &pyVersion3 - } else { - return &pyVersion2And3 - } -} - -type bazelPythonBinaryAttributes struct { - Main *bazel.Label - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Python_version *string - Imports bazel.StringListAttribute -} - -func (p *PythonLibraryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - // TODO(b/182306917): this doesn't fully handle all nested props versioned - // by the python version, which would have been handled by the version split - // mutator. This is sufficient for very simple python_library modules under - // Bionic. - baseAttrs := p.makeArchVariantBaseAttributes(ctx) - pyVersion := p.bp2buildPythonVersion(ctx) - if *pyVersion == pyVersion2And3 { - // Libraries default to python 2 and 3 - pyVersion = nil - } - - attrs := &bazelPythonLibraryAttributes{ - Srcs: baseAttrs.Srcs, - Deps: baseAttrs.Deps, - Srcs_version: pyVersion, - Imports: baseAttrs.Imports, - } - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_library rule. - Rule_class: "py_library", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: p.Name(), - Data: baseAttrs.Data, - }, attrs) -} - -func (p *PythonBinaryModule) bp2buildBinaryProperties(ctx android.TopDownMutatorContext) (*bazelPythonBinaryAttributes, bazel.LabelListAttribute) { - // TODO(b/182306917): this doesn't fully handle all nested props versioned - // by the python version, which would have been handled by the version split - // mutator. This is sufficient for very simple python_binary_host modules - // under Bionic. - - baseAttrs := p.makeArchVariantBaseAttributes(ctx) - pyVersion := p.bp2buildPythonVersion(ctx) - if *pyVersion == pyVersion3 { - // Binaries default to python 3 - pyVersion = nil - } else if *pyVersion == pyVersion2And3 { - ctx.ModuleErrorf("error for '%s' module: bp2build's python_binary_host converter "+ - "does not support converting a module that is enabled for both Python 2 and 3 at the "+ - "same time.", p.Name()) - } - - attrs := &bazelPythonBinaryAttributes{ - Main: nil, - Srcs: baseAttrs.Srcs, - Deps: baseAttrs.Deps, - Python_version: pyVersion, - Imports: baseAttrs.Imports, - } - - // main is optional. - if p.binaryProperties.Main != nil { - main := android.BazelLabelForModuleSrcSingle(ctx, *p.binaryProperties.Main) - attrs.Main = &main - } - return attrs, baseAttrs.Data -} - -func (p *PythonBinaryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - attrs, data := p.bp2buildBinaryProperties(ctx) - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_binary rule. - Rule_class: "py_binary", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: p.Name(), - Data: data, - }, attrs) -} - -func (p *PythonTestModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - // Python tests are currently exactly the same as binaries, but with a different module type - attrs, data := p.bp2buildBinaryProperties(ctx) - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_binary rule. - Rule_class: "py_test", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: p.Name(), - Data: data, - }, attrs) -} diff --git a/python/builder.go b/python/builder.go index 106649398..2553a7714 100644 --- a/python/builder.go +++ b/python/builder.go @@ -73,14 +73,14 @@ var ( precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{ Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` + - `PYTHONPATH=$stdlibZip/internal/stdlib ` + + `PYTHONPATH=$stdlibZip/internal/$stdlibPkg ` + `$launcher build/soong/python/scripts/precompile_python.py $in $out`, CommandDeps: []string{ "$stdlibZip", "$launcher", "build/soong/python/scripts/precompile_python.py", }, - }, "stdlibZip", "launcher", "ldLibraryPath") + }, "stdlibZip", "stdlibPkg", "launcher", "ldLibraryPath") ) func init() { diff --git a/python/proto.go b/python/proto.go index 400e72c99..ad2b786e2 100644 --- a/python/proto.go +++ b/python/proto.go @@ -19,7 +19,8 @@ import ( ) func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.ProtoFlags) android.Path { - srcsZipFile := android.PathForModuleGen(ctx, protoFile.Base()+".srcszip") + // Using protoFile.Base() would generate duplicate source errors in some cases, so we use Rel() instead + srcsZipFile := android.PathForModuleGen(ctx, protoFile.Rel()+".srcszip") outDir := srcsZipFile.ReplaceExtension(ctx, "tmp") depFile := srcsZipFile.ReplaceExtension(ctx, "srcszip.d") diff --git a/python/python.go b/python/python.go index 1a129737a..1ee533fa8 100644 --- a/python/python.go +++ b/python/python.go @@ -59,7 +59,7 @@ type VersionProperties struct { // list of the Python libraries used only for this Python version. Libs []string `android:"arch_variant"` - // whether the binary is required to be built with embedded launcher for this version, defaults to false. + // whether the binary is required to be built with embedded launcher for this version, defaults to true. Embedded_launcher *bool // TODO(b/174041232): Remove this property } @@ -129,7 +129,6 @@ type pathMapping struct { type PythonLibraryModule struct { android.ModuleBase android.DefaultableModuleBase - android.BazelModuleBase properties BaseProperties protoProperties android.ProtoProperties @@ -152,6 +151,8 @@ type PythonLibraryModule struct { // The zip file containing the current module's source/data files, with the // source files precompiled. precompiledSrcsZip android.Path + + sourceProperties android.SourceProperties } // newModule generates new Python base module @@ -169,6 +170,7 @@ type pythonDependency interface { getDataPathMappings() []pathMapping getSrcsZip() android.Path getPrecompiledSrcsZip() android.Path + getPkgPath() string } // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination @@ -191,6 +193,11 @@ func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path { return p.precompiledSrcsZip } +// getPkgPath returns the pkg_path value +func (p *PythonLibraryModule) getPkgPath() string { + return String(p.properties.Pkg_path) +} + func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { return &p.properties } @@ -198,10 +205,9 @@ func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { var _ pythonDependency = (*PythonLibraryModule)(nil) func (p *PythonLibraryModule) init() android.Module { - p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.properties, &p.protoProperties, &p.sourceProperties) android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) - android.InitBazelModule(p) return p } @@ -265,7 +271,6 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } if proptools.BoolDefault(props.Version.Py2.Enabled, false) { if !mctx.DeviceConfig().BuildBrokenUsesSoongPython2Modules() && - mctx.ModuleName() != "par_test" && mctx.ModuleName() != "py2-cmd" && mctx.ModuleName() != "py2-stdlib" { mctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3. This error can be temporarily overridden by setting BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES := true in the product configuration") @@ -371,7 +376,20 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") case pyVersion3: - stdLib = "py3-stdlib" + var prebuiltStdLib bool + if targetForDeps.Os.Bionic() { + prebuiltStdLib = false + } else if ctx.Config().VendorConfig("cpython3").Bool("force_build_host") { + prebuiltStdLib = false + } else { + prebuiltStdLib = true + } + + if prebuiltStdLib { + stdLib = "py3-stdlib-prebuilt" + } else { + stdLib = "py3-stdlib" + } launcherModule = "py3-launcher" if autorun { @@ -404,6 +422,12 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto // GenerateAndroidBuildActions performs build actions common to all Python modules func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) + android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: expandedSrcs.Strings()}) + // Keep before any early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: Bool(p.sourceProperties.Test_only), + TopLevelTarget: p.sourceProperties.Top_level_test_target, + }) // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) @@ -462,14 +486,19 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p destToPySrcs := make(map[string]string) destToPyData := make(map[string]string) + // Disable path checks for the stdlib, as it includes a "." in the version string + isInternal := proptools.BoolDefault(p.properties.Is_internal, false) + for _, s := range expandedSrcs { if s.Ext() != pyExt && s.Ext() != protoExt { ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String()) continue } runfilesPath := filepath.Join(pkgPath, s.Rel()) - if err := isValidPythonPath(runfilesPath); err != nil { - ctx.PropertyErrorf("srcs", err.Error()) + if !isInternal { + if err := isValidPythonPath(runfilesPath); err != nil { + ctx.PropertyErrorf("srcs", err.Error()) + } } if !checkForDuplicateOutputPath(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) { p.srcsPathMappings = append(p.srcsPathMappings, pathMapping{dest: runfilesPath, src: s}) @@ -477,8 +506,8 @@ func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, p } for _, d := range expandedData { - if d.Ext() == pyExt || d.Ext() == protoExt { - ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String()) + if d.Ext() == pyExt { + ctx.PropertyErrorf("data", "found (.py) file: %q!", d.String()) continue } runfilesPath := filepath.Join(pkgPath, d.Rel()) @@ -494,19 +523,19 @@ func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath s relativeRootMap := make(map[string]android.Paths) var protoSrcs android.Paths addPathMapping := func(path pathMapping) { - // handle proto sources separately - if path.src.Ext() == protoExt { - protoSrcs = append(protoSrcs, path.src) - } else { - relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel()) - relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src) - } + relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel()) + relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src) } // "srcs" or "data" properties may contain filegroups so it might happen that // the root directory for each source path is different. for _, path := range p.srcsPathMappings { - addPathMapping(path) + // handle proto sources separately + if path.src.Ext() == protoExt { + protoSrcs = append(protoSrcs, path.src) + } else { + addPathMapping(path) + } } for _, path := range p.dataPathMappings { addPathMapping(path) @@ -523,7 +552,6 @@ func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath s var stagedProtoSrcs android.Paths for _, srcFile := range protoSrcs { stagedProtoSrc := pkgPathStagingDir.Join(ctx, pkgPath, srcFile.Rel()) - rule.Command().Text("mkdir -p").Flag(filepath.Base(stagedProtoSrc.String())) rule.Command().Text("cp -f").Input(srcFile).Output(stagedProtoSrc) stagedProtoSrcs = append(stagedProtoSrcs, stagedProtoSrc) } @@ -592,13 +620,16 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android. // "cross compiling" for device here purely by virtue of host and device python bytecode // being the same. var stdLib android.Path + var stdLibPkg string var launcher android.Path - if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" { + if proptools.BoolDefault(p.properties.Is_internal, false) { stdLib = p.srcsZip + stdLibPkg = p.getPkgPath() } else { ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) { if dep, ok := module.(pythonDependency); ok { stdLib = dep.getPrecompiledSrcsZip() + stdLibPkg = dep.getPkgPath() } }) } @@ -637,6 +668,7 @@ func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android. Description: "Precompile the python sources of " + ctx.ModuleName(), Args: map[string]string{ "stdlibZip": stdLib.String(), + "stdlibPkg": stdLibPkg, "launcher": launcher.String(), "ldLibraryPath": strings.Join(ldLibraryPath, ":"), }, diff --git a/python/python_test.go b/python/python_test.go index 75a6a899b..6a6bd1d91 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -18,10 +18,13 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "android/soong/android" "android/soong/cc" + + "github.com/google/blueprint" ) type pyModule struct { @@ -47,7 +50,7 @@ var ( " Second file: in module %s at path %q." noSrcFileErr = moduleVariantErrTemplate + "doesn't have any source files!" badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py|.proto) file: %q!" - badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py|.proto) file: %q!" + badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py) file: %q!" bpFile = "Android.bp" data = []struct { @@ -360,6 +363,76 @@ cc_binary { } } +func TestTestOnlyProvider(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithPythonBuildComponents, + android.PrepareForTestWithAllowMissingDependencies, + ).RunTestWithBp(t, ` + // These should be test-only + python_library { name: "py-lib-test", test_only: true } + python_library { name: "py-lib-test-host", test_only: true, host_supported: true } + python_test { name: "py-test", srcs: ["py-test.py"] } + python_test_host { name: "py-test-host", srcs: ["py-test-host.py"] } + python_binary_host { name: "py-bin-test", srcs: ["py-bin-test.py"] } + + // These should not be. + python_library { name: "py-lib" } + python_binary_host { name: "py-bin", srcs: ["py-bin.py"] } + `) + + // Visit all modules and ensure only the ones that should + // marked as test-only are marked as test-only. + + actualTestOnly := []string{} + ctx.VisitAllModules(func(m blueprint.Module) { + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + actualTestOnly = append(actualTestOnly, m.Name()) + } + } + }) + expectedTestOnlyModules := []string{ + "py-lib-test", + "py-lib-test-host", + "py-test", + "py-test-host", + } + + notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) + if notEqual { + t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) + } +} + +// Don't allow setting test-only on things that are always tests or never tests. +func TestInvalidTestOnlyTargets(t *testing.T) { + testCases := []string{ + ` python_test { name: "py-test", test_only: true, srcs: ["py-test.py"] } `, + ` python_test_host { name: "py-test-host", test_only: true, srcs: ["py-test-host.py"] } `, + ` python_defaults { name: "py-defaults", test_only: true, srcs: ["foo.py"] } `, + } + + for i, bp := range testCases { + ctx := android.GroupFixturePreparers( + PrepareForTestWithPythonBuildComponents, + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + + ctx.RegisterModuleType("python_defaults", DefaultsFactory) + }), + android.PrepareForTestWithAllowMissingDependencies). + ExtendWithErrorHandler(android.FixtureIgnoreErrors). + RunTestWithBp(t, bp) + if len(ctx.Errs) != 1 { + t.Errorf("Expected err setting test_only in testcase #%d: %d errs", i, len(ctx.Errs)) + continue + } + if !strings.Contains(ctx.Errs[0].Error(), "unrecognized property \"test_only\"") { + t.Errorf("ERR: %s bad bp: %s", ctx.Errs[0], bp) + } + } +} + func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles []string) { module := ctx.ModuleForTests(name, variant) diff --git a/python/scripts/precompile_python.py b/python/scripts/precompile_python.py index e12e7d24c..b3cf950d9 100644 --- a/python/scripts/precompile_python.py +++ b/python/scripts/precompile_python.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function import argparse import py_compile import os +import sys import shutil import tempfile import zipfile @@ -23,22 +25,35 @@ import zipfile # This file needs to support both python 2 and 3. -def process_one_file(name, inf, outzip): - if not name.endswith('.py'): - outzip.writestr(name, inf.read()) +def process_one_file(name, infile, outzip): + # Create a ZipInfo instance with a fixed date to ensure a deterministic output. + # Date was chosen to be the same as + # https://cs.android.com/android/platform/superproject/main/+/main:build/soong/jar/jar.go;l=36;drc=2863e4535eb65e15f955dc8ed48fa99b1d2a1db5 + info = zipfile.ZipInfo(filename=name, date_time=(2008, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + + if not info.filename.endswith('.py'): + outzip.writestr(info, infile.read()) return # Unfortunately py_compile requires the input/output files to be written # out to disk. with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp: - shutil.copyfileobj(inf, tmp) + shutil.copyfileobj(infile, tmp) in_name = tmp.name with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp: out_name = tmp.name try: - py_compile.compile(in_name, out_name, name, doraise=True) + # Ensure a deterministic .pyc output by using the hash rather than the timestamp. + # Only works on Python 3.7+ + # See https://docs.python.org/3/library/py_compile.html#py_compile.PycInvalidationMode + if sys.version_info >= (3, 7): + py_compile.compile(in_name, out_name, info.filename, doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH) + else: + py_compile.compile(in_name, out_name, info.filename, doraise=True) with open(out_name, 'rb') as f: - outzip.writestr(name + 'c', f.read()) + info.filename = info.filename + 'c' + outzip.writestr(info, f.read()) finally: os.remove(in_name) os.remove(out_name) @@ -50,11 +65,23 @@ def main(): parser.add_argument('dst_zip') args = parser.parse_args() + errors = [] with open(args.dst_zip, 'wb') as outf, open(args.src_zip, 'rb') as inf: with zipfile.ZipFile(outf, mode='w') as outzip, zipfile.ZipFile(inf, mode='r') as inzip: for name in inzip.namelist(): with inzip.open(name, mode='r') as inzipf: - process_one_file(name, inzipf, outzip) + try: + process_one_file(name, inzipf, outzip) + except py_compile.PyCompileError as e: + errors.append(e) + + if errors: + for i, error in enumerate(errors): + # Print an empty line in between each error + if i > 0: + print(file=sys.stderr) + print(str(error).strip(), file=sys.stderr) + sys.exit(1) if __name__ == "__main__": diff --git a/python/test.go b/python/test.go index 31da17e61..85decf931 100644 --- a/python/test.go +++ b/python/test.go @@ -17,6 +17,8 @@ package python import ( "fmt" + "android/soong/testing" + "github.com/google/blueprint/proptools" "android/soong/android" @@ -35,11 +37,13 @@ func registerPythonTestComponents(ctx android.RegistrationContext) { } func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule { - return &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} + p := &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} + p.sourceProperties = android.SourceProperties{Test_only: proptools.BoolPtr(true), Top_level_test_target: true} + return p } func PythonTestHostFactory() android.Module { - return NewTest(android.HostSupportedNoCross).init() + return NewTest(android.HostSupported).init() } func PythonTestFactory() android.Module { @@ -66,6 +70,10 @@ type TestProperties struct { // Test options. Test_options TestOptions + + // list of device binary modules that should be installed alongside the test + // This property adds 64bit AND 32bit variants of the dependency + Data_device_bins_both []string `android:"arch_variant"` } type TestOptions struct { @@ -97,13 +105,48 @@ func (p *PythonTestModule) init() android.Module { p.AddProperties(&p.testProperties) android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) - android.InitBazelModule(p) - if p.hod == android.HostSupportedNoCross && p.testProperties.Test_options.Unit_test == nil { + if p.isTestHost() && p.testProperties.Test_options.Unit_test == nil { p.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) } return p } +func (p *PythonTestModule) isTestHost() bool { + return p.hod == android.HostSupported +} + +var dataDeviceBinsTag = dependencyTag{name: "dataDeviceBins"} + +// python_test_host DepsMutator uses this method to add multilib dependencies of +// data_device_bin_both +func (p *PythonTestModule) addDataDeviceBinsDeps(ctx android.BottomUpMutatorContext, filter string) { + if len(p.testProperties.Data_device_bins_both) < 1 { + return + } + + var maybeAndroidTarget *android.Target + androidTargetList := android.FirstTarget(ctx.Config().Targets[android.Android], filter) + if len(androidTargetList) > 0 { + maybeAndroidTarget = &androidTargetList[0] + } + + if maybeAndroidTarget != nil { + ctx.AddFarVariationDependencies( + maybeAndroidTarget.Variations(), + dataDeviceBinsTag, + p.testProperties.Data_device_bins_both..., + ) + } +} + +func (p *PythonTestModule) DepsMutator(ctx android.BottomUpMutatorContext) { + p.PythonBinaryModule.DepsMutator(ctx) + if p.isTestHost() { + p.addDataDeviceBinsDeps(ctx, "lib32") + p.addDataDeviceBinsDeps(ctx, "lib64") + } +} + func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { // We inherit from only the library's GenerateAndroidBuildActions, and then // just use buildBinary() so that the binary is not installed into the location @@ -117,48 +160,48 @@ func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext } runner := proptools.StringDefault(p.testProperties.Test_options.Runner, "tradefed") - if runner == "tradefed" { - p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ - TestConfigProp: p.testProperties.Test_config, - TestConfigTemplateProp: p.testProperties.Test_config_template, - TestSuites: p.binaryProperties.Test_suites, - OptionsForAutogenerated: configs, - AutoGenConfig: p.binaryProperties.Auto_gen_config, - DeviceTemplate: "${PythonBinaryHostTestConfigTemplate}", - HostTemplate: "${PythonBinaryHostTestConfigTemplate}", - }) - } else if runner == "mobly" { - if p.testProperties.Test_config != nil || p.testProperties.Test_config_template != nil || p.binaryProperties.Auto_gen_config != nil { - panic(fmt.Errorf("cannot set test_config, test_config_template or auto_gen_config for mobly test")) - } - - for _, testSuite := range p.binaryProperties.Test_suites { - if testSuite == "cts" { - configs = append(configs, tradefed.Option{Name: "test-suite-tag", Value: "cts"}) - break - } + template := "${PythonBinaryHostTestConfigTemplate}" + if runner == "mobly" { + // Add tag to enable Atest mobly runner + if !android.InList("mobly", p.testProperties.Test_options.Tags) { + p.testProperties.Test_options.Tags = append(p.testProperties.Test_options.Tags, "mobly") } - p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ - OptionsForAutogenerated: configs, - DeviceTemplate: "${PythonBinaryHostMoblyTestConfigTemplate}", - HostTemplate: "${PythonBinaryHostMoblyTestConfigTemplate}", - }) - } else { + template = "${PythonBinaryHostMoblyTestConfigTemplate}" + } else if runner != "tradefed" { panic(fmt.Errorf("unknown python test runner '%s', should be 'tradefed' or 'mobly'", runner)) } - - p.installedDest = ctx.InstallFile(installDir(ctx, "nativetest", "nativetest64", ctx.ModuleName()), p.installSource.Base(), p.installSource) + p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ + TestConfigProp: p.testProperties.Test_config, + TestConfigTemplateProp: p.testProperties.Test_config_template, + TestSuites: p.binaryProperties.Test_suites, + OptionsForAutogenerated: configs, + AutoGenConfig: p.binaryProperties.Auto_gen_config, + DeviceTemplate: template, + HostTemplate: template, + }) for _, dataSrcPath := range android.PathsForModuleSrc(ctx, p.testProperties.Data) { p.data = append(p.data, android.DataPath{SrcPath: dataSrcPath}) } + if p.isTestHost() && len(p.testProperties.Data_device_bins_both) > 0 { + ctx.VisitDirectDepsWithTag(dataDeviceBinsTag, func(dep android.Module) { + p.data = append(p.data, android.DataPath{SrcPath: android.OutputFileForModule(ctx, dep, "")}) + }) + } + // Emulate the data property for java_data dependencies. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") { p.data = append(p.data, android.DataPath{SrcPath: javaDataSrcPath}) } } + + installDir := installDir(ctx, "nativetest", "nativetest64", ctx.ModuleName()) + installedData := ctx.InstallTestData(installDir, p.data) + p.installedDest = ctx.InstallFile(installDir, p.installSource.Base(), p.installSource, installedData...) + + android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries { @@ -177,9 +220,13 @@ func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries { entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) } - entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) + // ATS 2.0 is the test harness for mobly tests and the test config is for ATS 2.0. + // Add "v2" suffix to test config name to distinguish it from the config for TF. + if proptools.String(p.testProperties.Test_options.Runner) == "mobly" { + entries.SetString("LOCAL_TEST_CONFIG_SUFFIX", "v2") + } - entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...) + entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) p.testProperties.Test_options.SetAndroidMkEntries(entries) }) diff --git a/python/tests/Android.bp b/python/tests/Android.bp index a65685960..056f7eddd 100644 --- a/python/tests/Android.bp +++ b/python/tests/Android.bp @@ -27,32 +27,4 @@ python_test_host { test_options: { unit_test: false, }, - version: { - py2: { - enabled: true, - embedded_launcher: true, - }, - py3: { - enabled: false, - embedded_launcher: true, - }, - }, -} - -python_test_host { - name: "par_test3", - main: "par_test.py", - srcs: [ - "par_test.py", - "testpkg/par_test.py", - ], - // Is not implemented as a python unittest - test_options: { - unit_test: false, - }, - version: { - py3: { - embedded_launcher: true, - }, - }, } diff --git a/python/tests/dont_import_folder_of_entrypoint/Android.bp b/python/tests/dont_import_folder_of_entrypoint/Android.bp index e54e9b2c0..ab2e314be 100644 --- a/python/tests/dont_import_folder_of_entrypoint/Android.bp +++ b/python/tests/dont_import_folder_of_entrypoint/Android.bp @@ -10,17 +10,3 @@ python_test_host { "mypkg/mymodule.py", ], } - -python_test_host { - name: "py_dont_import_folder_of_entrypoint_test_embedded_launcher", - main: "mypkg/main.py", - srcs: [ - "mypkg/main.py", - "mypkg/mymodule.py", - ], - version: { - py3: { - embedded_launcher: true, - }, - }, -} diff --git a/python/tests/par_test.py b/python/tests/par_test.py index 1e03f1669..96b42ae83 100644 --- a/python/tests/par_test.py +++ b/python/tests/par_test.py @@ -33,6 +33,8 @@ if fileName.endswith('.pyc'): assert_equal("os.path.basename(__file__)", fileName, "par_test.py") archive = os.path.dirname(__file__) +major = sys.version_info.major +minor = sys.version_info.minor assert_equal("__package__", __package__, "") assert_equal("sys.argv[0]", sys.argv[0], archive) @@ -42,10 +44,11 @@ assert_equal("sys.prefix", sys.prefix, archive) assert_equal("__loader__.archive", __loader__.archive, archive) assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None) -assert_equal("len(sys.path)", len(sys.path), 3) +assert_equal("len(sys.path)", len(sys.path), 4) assert_equal("sys.path[0]", sys.path[0], archive) -assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal")) -assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", "stdlib")) +assert_equal("sys.path[1]", sys.path[1], os.path.join(archive, "internal", f"python{major}{minor}.zip")) +assert_equal("sys.path[2]", sys.path[2], os.path.join(archive, "internal", f"python{major}.{minor}")) +assert_equal("sys.path[3]", sys.path[3], os.path.join(archive, "internal", f"python{major}.{minor}", "lib-dynload")) if os.getenv('ARGTEST', False): assert_equal("len(sys.argv)", len(sys.argv), 3) diff --git a/python/tests/py-cmd_test.py b/python/tests/py-cmd_test.py index acda2d742..8aed78289 100644 --- a/python/tests/py-cmd_test.py +++ b/python/tests/py-cmd_test.py @@ -55,22 +55,22 @@ assert_equal("sys.exec_prefix", sys.exec_prefix, sys.executable) assert_equal("sys.prefix", sys.prefix, sys.executable) assert_equal("site.ENABLE_USER_SITE", site.ENABLE_USER_SITE, None) -if sys.version_info[0] == 2: +major = sys.version_info.major +minor = sys.version_info.minor + +if major == 2: assert_equal("len(sys.path)", len(sys.path), 4) - assert_equal("sys.path[0]", sys.path[0], os.path.dirname(__file__)) + assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__))) assert_equal("sys.path[1]", sys.path[1], "/extra") assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, "internal")) assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, "internal", "stdlib")) else: - assert_equal("len(sys.path)", len(sys.path), 8) + assert_equal("len(sys.path)", len(sys.path), 5) assert_equal("sys.path[0]", sys.path[0], os.path.abspath(os.path.dirname(__file__))) assert_equal("sys.path[1]", sys.path[1], "/extra") - assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip')) - assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), '..')) - assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]))) - assert_equal("sys.path[5]", sys.path[5], os.path.join(sys.executable, 'lib', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload')) - assert_equal("sys.path[6]", sys.path[6], os.path.join(sys.executable, "internal")) - assert_equal("sys.path[7]", sys.path[7], os.path.join(sys.executable, "internal", "stdlib")) + assert_equal("sys.path[2]", sys.path[2], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + str(sys.version_info[1]) + '.zip')) + assert_equal("sys.path[3]", sys.path[3], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]))) + assert_equal("sys.path[4]", sys.path[4], os.path.join(sys.executable, 'internal', 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]), 'lib-dynload')) if failed: sys.exit(1) diff --git a/python/tests/runtest.sh b/python/tests/runtest.sh index 35941dc88..c44ec582a 100755 --- a/python/tests/runtest.sh +++ b/python/tests/runtest.sh @@ -24,12 +24,16 @@ if [ -z $ANDROID_HOST_OUT ]; then fi if [[ ( ! -f $ANDROID_HOST_OUT/nativetest64/par_test/par_test ) || - ( ! -f $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 ) || - ( ! -f $ANDROID_HOST_OUT/bin/py2-cmd ) || ( ! -f $ANDROID_HOST_OUT/bin/py3-cmd )]]; then - echo "Run 'm par_test par_test3 py2-cmd py3-cmd' first" + echo "Run 'm par_test py2-cmd py3-cmd' first" exit 1 fi +if [ $(uname -s) = Linux ]; then + if [[ ! -f $ANDROID_HOST_OUT/bin/py2-cmd ]]; then + echo "Run 'm par_test py2-cmd py3-cmd' first" + exit 1 + fi +fi export LD_LIBRARY_PATH=$ANDROID_HOST_OUT/lib64 @@ -41,19 +45,17 @@ PYTHONPATH=/usr $ANDROID_HOST_OUT/nativetest64/par_test/par_test ARGTEST=true $ANDROID_HOST_OUT/nativetest64/par_test/par_test --arg1 arg2 -PYTHONHOME= PYTHONPATH= $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 -PYTHONHOME=/usr $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 -PYTHONPATH=/usr $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 - -ARGTEST=true $ANDROID_HOST_OUT/nativetest64/par_test3/par_test3 --arg1 arg2 - cd $(dirname ${BASH_SOURCE[0]}) -PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py +if [ $(uname -s) = Linux ]; then + PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py +fi PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py3-cmd py-cmd_test.py -ARGTEST=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py arg1 arg2 -ARGTEST2=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py --arg1 arg2 +if [ $(uname -s) = Linux ]; then + ARGTEST=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py arg1 arg2 + ARGTEST2=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py2-cmd py-cmd_test.py --arg1 arg2 +fi ARGTEST=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py3-cmd py-cmd_test.py arg1 arg2 ARGTEST2=true PYTHONPATH=/extra $ANDROID_HOST_OUT/bin/py3-cmd py-cmd_test.py --arg1 arg2 diff --git a/python/tests/testpkg/par_test.py b/python/tests/testpkg/par_test.py index b51340907..e12c527ad 100644 --- a/python/tests/testpkg/par_test.py +++ b/python/tests/testpkg/par_test.py @@ -33,11 +33,7 @@ if fileName.endswith('.pyc'): fileName = fileName[:-1] assert_equal("__file__", fileName, os.path.join(archive, "testpkg/par_test.py")) -# Python3 is returning None here for me, and I haven't found any problems caused by this. -if sys.version_info[0] == 2: - assert_equal("__package__", __package__, "testpkg") -else: - assert_equal("__package__", __package__, None) +assert_equal("__package__", __package__, "testpkg") assert_equal("__loader__.archive", __loader__.archive, archive) assert_equal("__loader__.prefix", __loader__.prefix, "testpkg/") |