diff options
34 files changed, 906 insertions, 574 deletions
diff --git a/Android.bp b/Android.bp index 00db8f2da..916292633 100644 --- a/Android.bp +++ b/Android.bp @@ -267,6 +267,7 @@ bootstrap_go_package { "python/defaults.go", "python/installer.go", "python/library.go", + "python/proto.go", "python/python.go", "python/test.go", ], diff --git a/android/arch.go b/android/arch.go index 6ab184f36..fd80eecdb 100644 --- a/android/arch.go +++ b/android/arch.go @@ -784,14 +784,18 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) { a.appendProperties(ctx, genProps, targetProp, field, prefix) } - if arch.ArchType == X86 && (hasArmAbi(arch) || - hasArmAndroidArch(ctx.Config().Targets[Device])) { + if (arch.ArchType == X86 && (hasArmAbi(arch) || + hasArmAndroidArch(ctx.Config().Targets[Device]))) || + (arch.ArchType == Arm && + hasX86AndroidArch(ctx.Config().Targets[Device])) { field := "Arm_on_x86" prefix := "target.arm_on_x86" a.appendProperties(ctx, genProps, targetProp, field, prefix) } - if arch.ArchType == X86_64 && (hasArmAbi(arch) || - hasArmAndroidArch(ctx.Config().Targets[Device])) { + if (arch.ArchType == X86_64 && (hasArmAbi(arch) || + hasArmAndroidArch(ctx.Config().Targets[Device]))) || + (arch.ArchType == Arm && + hasX8664AndroidArch(ctx.Config().Targets[Device])) { field := "Arm_on_x86_64" prefix := "target.arm_on_x86_64" a.appendProperties(ctx, genProps, targetProp, field, prefix) @@ -914,6 +918,26 @@ func hasArmAndroidArch(targets []Target) bool { return false } +// hasX86Arch returns true if targets has at least x86 Android arch +func hasX86AndroidArch(targets []Target) bool { + for _, target := range targets { + if target.Os == Android && target.Arch.ArchType == X86 { + return true + } + } + return false +} + +// hasX8664Arch returns true if targets has at least x86_64 Android arch +func hasX8664AndroidArch(targets []Target) bool { + for _, target := range targets { + if target.Os == Android && target.Arch.ArchType == X86_64 { + return true + } + } + return false +} + type archConfig struct { arch string archVariant string diff --git a/android/neverallow.go b/android/neverallow.go index 8fba4b902..ec440d5cc 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -56,7 +56,7 @@ var neverallows = []*rule{ without("vendor", "true"). without("owner", ""). because("a VNDK module can never have an owner."), - neverallow().notIn("libcore").with("no_standard_libs", "true"), + neverallow().notIn("libcore", "development").with("no_standard_libs", "true"), // TODO(b/67974785): always enforce the manifest neverallow(). diff --git a/android/package_ctx.go b/android/package_ctx.go index 849eb72a8..e228bba56 100644 --- a/android/package_ctx.go +++ b/android/package_ctx.go @@ -47,6 +47,9 @@ type configErrorWrapper struct { var _ PathContext = &configErrorWrapper{} var _ errorfContext = &configErrorWrapper{} +var _ PackageVarContext = &configErrorWrapper{} +var _ PackagePoolContext = &configErrorWrapper{} +var _ PackageRuleContext = &configErrorWrapper{} func (e *configErrorWrapper) Config() Config { return e.config @@ -62,33 +65,56 @@ func (e *configErrorWrapper) Fs() pathtools.FileSystem { return nil } +type PackageVarContext interface { + PathContext + errorfContext +} + +type PackagePoolContext PackageVarContext +type PackageRuleContext PackageVarContext + // VariableFunc wraps blueprint.PackageContext.VariableFunc, converting the interface{} config -// argument to a Config. +// argument to a PackageVarContext. func (p PackageContext) VariableFunc(name string, - f func(Config) (string, error)) blueprint.Variable { + f func(PackageVarContext) string) blueprint.Variable { return p.PackageContext.VariableFunc(name, func(config interface{}) (string, error) { - return f(config.(Config)) + ctx := &configErrorWrapper{p, config.(Config), nil} + ret := f(ctx) + if len(ctx.errors) > 0 { + return "", ctx.errors[0] + } + return ret, nil }) } // PoolFunc wraps blueprint.PackageContext.PoolFunc, converting the interface{} config -// argument to a Config. +// argument to a Context that supports Config(). func (p PackageContext) PoolFunc(name string, - f func(Config) (blueprint.PoolParams, error)) blueprint.Pool { + f func(PackagePoolContext) blueprint.PoolParams) blueprint.Pool { return p.PackageContext.PoolFunc(name, func(config interface{}) (blueprint.PoolParams, error) { - return f(config.(Config)) + ctx := &configErrorWrapper{p, config.(Config), nil} + params := f(ctx) + if len(ctx.errors) > 0 { + return params, ctx.errors[0] + } + return params, nil }) } // RuleFunc wraps blueprint.PackageContext.RuleFunc, converting the interface{} config -// argument to a Config. +// argument to a Context that supports Config(). func (p PackageContext) RuleFunc(name string, - f func(Config) (blueprint.RuleParams, error), argNames ...string) blueprint.Rule { + f func(PackageRuleContext) blueprint.RuleParams, argNames ...string) blueprint.Rule { return p.PackageContext.RuleFunc(name, func(config interface{}) (blueprint.RuleParams, error) { - return f(config.(Config)) + ctx := &configErrorWrapper{p, config.(Config), nil} + params := f(ctx) + if len(ctx.errors) > 0 { + return params, ctx.errors[0] + } + return params, nil }, argNames...) } @@ -97,13 +123,8 @@ func (p PackageContext) RuleFunc(name string, // initialization - either from the init() function or as part of a // package-scoped variable's initialization. func (p PackageContext) SourcePathVariable(name, path string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - ctx := &configErrorWrapper{p, config, []error{}} - p := safePathForSource(ctx, path) - if len(ctx.errors) > 0 { - return "", ctx.errors[0] - } - return p.String(), nil + return p.VariableFunc(name, func(ctx PackageVarContext) string { + return safePathForSource(ctx, path).String() }) } @@ -112,17 +133,13 @@ func (p PackageContext) SourcePathVariable(name, path string) blueprint.Variable // called during a Go package's initialization - either from the init() // function or as part of a package-scoped variable's initialization. func (p PackageContext) SourcePathsVariable(name, separator string, paths ...string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - ctx := &configErrorWrapper{p, config, []error{}} + return p.VariableFunc(name, func(ctx PackageVarContext) string { var ret []string for _, path := range paths { p := safePathForSource(ctx, path) - if len(ctx.errors) > 0 { - return "", ctx.errors[0] - } ret = append(ret, p.String()) } - return strings.Join(ret, separator), nil + return strings.Join(ret, separator) }) } @@ -132,13 +149,9 @@ func (p PackageContext) SourcePathsVariable(name, separator string, paths ...str // It may only be called during a Go package's initialization - either from the init() function or // as part of a package-scoped variable's initialization. func (p PackageContext) SourcePathVariableWithEnvOverride(name, path, env string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - ctx := &configErrorWrapper{p, config, []error{}} + return p.VariableFunc(name, func(ctx PackageVarContext) string { p := safePathForSource(ctx, path) - if len(ctx.errors) > 0 { - return "", ctx.errors[0] - } - return config.GetenvWithDefault(env, p.String()), nil + return ctx.Config().GetenvWithDefault(env, p.String()) }) } @@ -147,22 +160,13 @@ func (p PackageContext) SourcePathVariableWithEnvOverride(name, path, env string // package's initialization - either from the init() function or as part of a // package-scoped variable's initialization. func (p PackageContext) HostBinToolVariable(name, path string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - po, err := p.HostBinToolPath(config, path) - if err != nil { - return "", err - } - return po.String(), nil + return p.VariableFunc(name, func(ctx PackageVarContext) string { + return p.HostBinToolPath(ctx, path).String() }) } -func (p PackageContext) HostBinToolPath(config Config, path string) (Path, error) { - ctx := &configErrorWrapper{p, config, []error{}} - pa := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "bin", path) - if len(ctx.errors) > 0 { - return nil, ctx.errors[0] - } - return pa, nil +func (p PackageContext) HostBinToolPath(ctx PackageVarContext, path string) Path { + return PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "bin", path) } // HostJNIToolVariable returns a Variable whose value is the path to a host tool @@ -170,26 +174,17 @@ func (p PackageContext) HostBinToolPath(config Config, path string) (Path, error // package's initialization - either from the init() function or as part of a // package-scoped variable's initialization. func (p PackageContext) HostJNIToolVariable(name, path string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - po, err := p.HostJNIToolPath(config, path) - if err != nil { - return "", err - } - return po.String(), nil + return p.VariableFunc(name, func(ctx PackageVarContext) string { + return p.HostJNIToolPath(ctx, path).String() }) } -func (p PackageContext) HostJNIToolPath(config Config, path string) (Path, error) { - ctx := &configErrorWrapper{p, config, []error{}} +func (p PackageContext) HostJNIToolPath(ctx PackageVarContext, path string) Path { ext := ".so" if runtime.GOOS == "darwin" { ext = ".dylib" } - pa := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "lib64", path+ext) - if len(ctx.errors) > 0 { - return nil, ctx.errors[0] - } - return pa, nil + return PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "lib64", path+ext) } // HostJavaToolVariable returns a Variable whose value is the path to a host @@ -197,23 +192,13 @@ func (p PackageContext) HostJNIToolPath(config Config, path string) (Path, error // during a Go package's initialization - either from the init() function or as // part of a package-scoped variable's initialization. func (p PackageContext) HostJavaToolVariable(name, path string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - ctx := &configErrorWrapper{p, config, []error{}} - p := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "framework", path) - if len(ctx.errors) > 0 { - return "", ctx.errors[0] - } - return p.String(), nil + return p.VariableFunc(name, func(ctx PackageVarContext) string { + return p.HostJavaToolPath(ctx, path).String() }) } -func (p PackageContext) HostJavaToolPath(config Config, path string) (Path, error) { - ctx := &configErrorWrapper{p, config, []error{}} - pa := PathForOutput(ctx, "host", ctx.config.PrebuiltOS(), "framework", path) - if len(ctx.errors) > 0 { - return nil, ctx.errors[0] - } - return pa, nil +func (p PackageContext) HostJavaToolPath(ctx PackageVarContext, path string) Path { + return PathForOutput(ctx, "host", ctx.Config().PrebuiltOS(), "framework", path) } // IntermediatesPathVariable returns a Variable whose value is the intermediate @@ -221,13 +206,8 @@ func (p PackageContext) HostJavaToolPath(config Config, path string) (Path, erro // package's initialization - either from the init() function or as part of a // package-scoped variable's initialization. func (p PackageContext) IntermediatesPathVariable(name, path string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - ctx := &configErrorWrapper{p, config, []error{}} - p := PathForIntermediates(ctx, path) - if len(ctx.errors) > 0 { - return "", ctx.errors[0] - } - return p.String(), nil + return p.VariableFunc(name, func(ctx PackageVarContext) string { + return PathForIntermediates(ctx, path).String() }) } @@ -238,21 +218,17 @@ func (p PackageContext) IntermediatesPathVariable(name, path string) blueprint.V func (p PackageContext) PrefixedExistentPathsForSourcesVariable( name, prefix string, paths []string) blueprint.Variable { - return p.VariableFunc(name, func(config Config) (string, error) { - ctx := &configErrorWrapper{p, config, []error{}} + return p.VariableFunc(name, func(ctx PackageVarContext) string { paths := ExistentPathsForSources(ctx, paths) - if len(ctx.errors) > 0 { - return "", ctx.errors[0] - } - return JoinWithPrefix(paths.Strings(), prefix), nil + return JoinWithPrefix(paths.Strings(), prefix) }) } // AndroidStaticRule wraps blueprint.StaticRule and provides a default Pool if none is specified func (p PackageContext) AndroidStaticRule(name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule { - return p.AndroidRuleFunc(name, func(Config) (blueprint.RuleParams, error) { - return params, nil + return p.AndroidRuleFunc(name, func(PackageRuleContext) blueprint.RuleParams { + return params }, argNames...) } @@ -263,14 +239,14 @@ func (p PackageContext) AndroidGomaStaticRule(name string, params blueprint.Rule } func (p PackageContext) AndroidRuleFunc(name string, - f func(Config) (blueprint.RuleParams, error), argNames ...string) blueprint.Rule { - return p.RuleFunc(name, func(config Config) (blueprint.RuleParams, error) { - params, err := f(config) - if config.UseGoma() && params.Pool == nil { + f func(PackageRuleContext) blueprint.RuleParams, argNames ...string) blueprint.Rule { + return p.RuleFunc(name, func(ctx PackageRuleContext) blueprint.RuleParams { + params := f(ctx) + if ctx.Config().UseGoma() && params.Pool == nil { // When USE_GOMA=true is set and the rule is not supported by goma, restrict jobs to the // local parallelism value params.Pool = localPool } - return params, err + return params }, argNames...) } diff --git a/android/paths.go b/android/paths.go index 3605dcf42..dc89bfdbb 100644 --- a/android/paths.go +++ b/android/paths.go @@ -693,6 +693,46 @@ func PathForIntermediates(ctx PathContext, paths ...string) OutputPath { return PathForOutput(ctx, ".intermediates", path) } +// DistPath is a Path representing a file path rooted from the dist directory +type DistPath struct { + basePath +} + +func (p DistPath) withRel(rel string) DistPath { + p.basePath = p.basePath.withRel(rel) + return p +} + +var _ Path = DistPath{} + +// PathForDist joins the provided paths and returns a DistPath that is +// validated to not escape the dist dir. +// On error, it will return a usable, but invalid DistPath, and report a ModuleError. +func PathForDist(ctx PathContext, pathComponents ...string) DistPath { + path, err := validatePath(pathComponents...) + if err != nil { + reportPathError(ctx, err) + } + return DistPath{basePath{path, ctx.Config(), ""}} +} + +func (p DistPath) writablePath() {} + +func (p DistPath) Valid() bool { + return p.config.ProductVariables.DistDir != nil && *p.config.ProductVariables.DistDir != "" +} + +func (p DistPath) String() string { + if !p.Valid() { + panic("Requesting an invalid path") + } + return filepath.Join(*p.config.ProductVariables.DistDir, p.path) +} + +func (p DistPath) RelPathString() string { + return p.path +} + // ModuleSrcPath is a Path representing a file rooted from a module's local source dir type ModuleSrcPath struct { SourcePath diff --git a/android/variable.go b/android/variable.go index 01a8122d3..880bc0840 100644 --- a/android/variable.go +++ b/android/variable.go @@ -42,11 +42,6 @@ type variableProperties struct { Enabled *bool `android:"arch_variant"` } `android:"arch_variant"` - Brillo struct { - Cflags []string - Version_script *string `android:"arch_variant"` - } `android:"arch_variant"` - Malloc_not_svelte struct { Cflags []string } @@ -169,7 +164,6 @@ type productVariables struct { Allow_missing_dependencies *bool `json:",omitempty"` Unbundled_build *bool `json:",omitempty"` - Brillo *bool `json:",omitempty"` Malloc_not_svelte *bool `json:",omitempty"` Safestack *bool `json:",omitempty"` HostStaticBinaries *bool `json:",omitempty"` diff --git a/cc/builder.go b/cc/builder.go index 59a8cc8af..754391566 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -109,13 +109,14 @@ var ( "objcopyCmd", "prefix") _ = pctx.SourcePathVariable("stripPath", "build/soong/scripts/strip.sh") + _ = pctx.SourcePathVariable("xzCmd", "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/xz") strip = pctx.AndroidStaticRule("strip", blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, - Command: "CROSS_COMPILE=$crossCompile $stripPath ${args} -i ${in} -o ${out} -d ${out}.d", - CommandDeps: []string{"$stripPath"}, + Command: "CROSS_COMPILE=$crossCompile XZ=$xzCmd $stripPath ${args} -i ${in} -o ${out} -d ${out}.d", + CommandDeps: []string{"$stripPath", "$xzCmd"}, }, "args", "crossCompile") @@ -196,18 +197,17 @@ var ( _ = pctx.SourcePathVariable("sAbiDiffer", "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/header-abi-diff") sAbiDiff = pctx.AndroidRuleFunc("sAbiDiff", - func(config android.Config) (blueprint.RuleParams, error) { + func(ctx android.PackageRuleContext) blueprint.RuleParams { commandStr := "($sAbiDiffer $allowFlags -lib $libName -arch $arch -check-all-apis -o ${out} -new $in -old $referenceDump)" - distDir := config.ProductVariables.DistDir - if distDir != nil && *distDir != "" { - distAbiDiffDir := *distDir + "/abidiffs/" - commandStr += " || (mkdir -p " + distAbiDiffDir + " && cp ${out} " + distAbiDiffDir + " && exit 1)" + distAbiDiffDir := android.PathForDist(ctx, "abidiffs") + if distAbiDiffDir.Valid() { + commandStr += " || (mkdir -p " + distAbiDiffDir.String() + " && cp ${out} " + distAbiDiffDir.String() + " && exit 1)" } return blueprint.RuleParams{ Command: commandStr, CommandDeps: []string{"$sAbiDiffer"}, - }, nil + } }, "allowFlags", "referenceDump", "libName", "arch") diff --git a/cc/compiler.go b/cc/compiler.go index ef2228546..b2095a6bc 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -106,6 +106,9 @@ type BaseCompilerProperties struct { // list of directories relative to the Blueprints file that will // be added to the aidl include paths. Local_include_dirs []string + + // whether to generate traces (for systrace) for this interface + Generate_traces *bool } Renderscript struct { @@ -283,7 +286,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps // behavior here, and the NDK always has all the NDK headers available. flags.SystemIncludeFlags = append(flags.SystemIncludeFlags, "-isystem "+getCurrentIncludePath(ctx).String(), - "-isystem "+getCurrentIncludePath(ctx).Join(ctx, tc.ClangTriple()).String()) + "-isystem "+getCurrentIncludePath(ctx).Join(ctx, config.NDKTriple(tc)).String()) // Traditionally this has come from android/api-level.h, but with the // libc headers unified it must be set by the build system since we @@ -369,10 +372,6 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps fmt.Sprintf("${config.%sGlobalCflags}", hod)) } - if Bool(ctx.Config().ProductVariables.Brillo) { - flags.GlobalFlags = append(flags.GlobalFlags, "-D__BRILLO__") - } - if ctx.Device() { if Bool(compiler.Properties.Rtti) { flags.CppFlags = append(flags.CppFlags, "-frtti") @@ -477,6 +476,10 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps flags.aidlFlags = append(flags.aidlFlags, includeDirsToFlags(rootAidlIncludeDirs)) } + if Bool(compiler.Properties.Aidl.Generate_traces) { + flags.aidlFlags = append(flags.aidlFlags, "-t") + } + flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.PathForModuleGen(ctx, "aidl").String()) } diff --git a/cc/config/arm_device.go b/cc/config/arm_device.go index 0d3750e3c..66b3b3829 100644 --- a/cc/config/arm_device.go +++ b/cc/config/arm_device.go @@ -325,6 +325,12 @@ func (t *toolchainArm) InstructionSetFlags(isa string) (string, error) { } func (t *toolchainArm) ClangTriple() string { + // http://b/72619014 work around llvm LTO bug. + return "armv7a-linux-androideabi" +} + +func (t *toolchainArm) ndkTriple() string { + // Use current NDK include path, while ClangTriple is changed. return t.GccTriple() } diff --git a/cc/config/clang.go b/cc/config/clang.go index 92019897f..7de6f063d 100644 --- a/cc/config/clang.go +++ b/cc/config/clang.go @@ -116,10 +116,6 @@ func init() { // http://b/68236239 Allow 0/NULL instead of using nullptr everywhere. "-Wno-zero-as-null-pointer-constant", - - // http://b/36463318 Clang executes with an absolute path, so clang-provided - // headers are now absolute. - "-fdebug-prefix-map=$$PWD/=", }, " ")) pctx.StaticVariable("ClangExtraCppflags", strings.Join([]string{ diff --git a/cc/config/global.go b/cc/config/global.go index 5e5d457f7..bb58b3d2a 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -180,26 +180,26 @@ func init() { []string{"libnativehelper/include_jni"}) pctx.SourcePathVariable("ClangDefaultBase", ClangDefaultBase) - pctx.VariableFunc("ClangBase", func(config android.Config) (string, error) { - if override := config.Getenv("LLVM_PREBUILTS_BASE"); override != "" { - return override, nil + pctx.VariableFunc("ClangBase", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("LLVM_PREBUILTS_BASE"); override != "" { + return override } - return "${ClangDefaultBase}", nil + return "${ClangDefaultBase}" }) - pctx.VariableFunc("ClangVersion", func(config android.Config) (string, error) { - if override := config.Getenv("LLVM_PREBUILTS_VERSION"); override != "" { - return override, nil + pctx.VariableFunc("ClangVersion", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("LLVM_PREBUILTS_VERSION"); override != "" { + return override } - return ClangDefaultVersion, nil + return ClangDefaultVersion }) pctx.StaticVariable("ClangPath", "${ClangBase}/${HostPrebuiltTag}/${ClangVersion}") pctx.StaticVariable("ClangBin", "${ClangPath}/bin") - pctx.VariableFunc("ClangShortVersion", func(config android.Config) (string, error) { - if override := config.Getenv("LLVM_RELEASE_VERSION"); override != "" { - return override, nil + pctx.VariableFunc("ClangShortVersion", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("LLVM_RELEASE_VERSION"); override != "" { + return override } - return ClangDefaultShortVersion, nil + return ClangDefaultShortVersion }) pctx.StaticVariable("ClangAsanLibDir", "${ClangBase}/linux-x86/${ClangVersion}/lib64/clang/${ClangShortVersion}/lib/linux") if runtime.GOOS == "darwin" { @@ -222,11 +222,11 @@ func init() { "frameworks/rs/script_api/include", }) - pctx.VariableFunc("CcWrapper", func(config android.Config) (string, error) { - if override := config.Getenv("CC_WRAPPER"); override != "" { - return override + " ", nil + pctx.VariableFunc("CcWrapper", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("CC_WRAPPER"); override != "" { + return override + " " } - return "", nil + return "" }) } diff --git a/cc/config/tidy.go b/cc/config/tidy.go index 76a5f9ee9..a20d55606 100644 --- a/cc/config/tidy.go +++ b/cc/config/tidy.go @@ -15,9 +15,8 @@ package config import ( - "strings" - "android/soong/android" + "strings" ) func init() { @@ -25,9 +24,9 @@ func init() { // Global tidy checks include only google*, performance*, // and misc-macro-parentheses, but not google-readability* // or google-runtime-references. - pctx.VariableFunc("TidyDefaultGlobalChecks", func(config android.Config) (string, error) { - if override := config.Getenv("DEFAULT_GLOBAL_TIDY_CHECKS"); override != "" { - return override, nil + pctx.VariableFunc("TidyDefaultGlobalChecks", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("DEFAULT_GLOBAL_TIDY_CHECKS"); override != "" { + return override } return strings.Join([]string{ "-*", @@ -36,14 +35,14 @@ func init() { "performance*", "-google-readability*", "-google-runtime-references", - }, ","), nil + }, ",") }) // There are too many clang-tidy warnings in external and vendor projects. // Enable only some google checks for these projects. - pctx.VariableFunc("TidyExternalVendorChecks", func(config android.Config) (string, error) { - if override := config.Getenv("DEFAULT_EXTERNAL_VENDOR_TIDY_CHECKS"); override != "" { - return override, nil + pctx.VariableFunc("TidyExternalVendorChecks", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("DEFAULT_EXTERNAL_VENDOR_TIDY_CHECKS"); override != "" { + return override } return strings.Join([]string{ "-*", @@ -54,7 +53,7 @@ func init() { "-google-readability*", "-google-runtime-int", "-google-runtime-references", - }, ","), nil + }, ",") }) // Give warnings to header files only in selected directories. diff --git a/cc/config/toolchain.go b/cc/config/toolchain.go index 279ceef03..7961575fc 100644 --- a/cc/config/toolchain.go +++ b/cc/config/toolchain.go @@ -68,6 +68,8 @@ type Toolchain interface { ClangLdflags() string ClangInstructionSetFlags(string) (string, error) + ndkTriple() string + YasmFlags() string WindresFlags() string @@ -87,6 +89,19 @@ type Toolchain interface { type toolchainBase struct { } +func (t *toolchainBase) ndkTriple() string { + return "" +} + +func NDKTriple(toolchain Toolchain) string { + triple := toolchain.ndkTriple() + if triple == "" { + // Use the clang triple if there is no explicit NDK triple + triple = toolchain.ClangTriple() + } + return triple +} + func (toolchainBase) InstructionSetFlags(s string) (string, error) { if s != "" { return "", fmt.Errorf("instruction_set: %s is not a supported instruction set", s) diff --git a/cc/config/x86_darwin_host.go b/cc/config/x86_darwin_host.go index dbaa6fa6a..cae9757fa 100644 --- a/cc/config/x86_darwin_host.go +++ b/cc/config/x86_darwin_host.go @@ -15,7 +15,6 @@ package config import ( - "fmt" "os/exec" "path/filepath" "strings" @@ -107,26 +106,28 @@ const ( ) func init() { - pctx.VariableFunc("macSdkPath", func(config android.Config) (string, error) { - xcodeselect := config.HostSystemTool("xcode-select") + pctx.VariableFunc("macSdkPath", func(ctx android.PackageVarContext) string { + xcodeselect := ctx.Config().HostSystemTool("xcode-select") bytes, err := exec.Command(xcodeselect, "--print-path").Output() - return strings.TrimSpace(string(bytes)), err + if err != nil { + ctx.Errorf("xcode-select failed with: %q", err.Error()) + } + return strings.TrimSpace(string(bytes)) }) - pctx.VariableFunc("macSdkRoot", func(config android.Config) (string, error) { - return xcrunSdk(config, "--show-sdk-path") + pctx.VariableFunc("macSdkRoot", func(ctx android.PackageVarContext) string { + return xcrunSdk(ctx, "--show-sdk-path") }) pctx.StaticVariable("macMinVersion", "10.8") - pctx.VariableFunc("MacArPath", func(config android.Config) (string, error) { - return xcrun(config, "--find", "ar") + pctx.VariableFunc("MacArPath", func(ctx android.PackageVarContext) string { + return xcrun(ctx, "--find", "ar") }) - pctx.VariableFunc("MacStripPath", func(config android.Config) (string, error) { - return xcrun(config, "--find", "strip") + pctx.VariableFunc("MacStripPath", func(ctx android.PackageVarContext) string { + return xcrun(ctx, "--find", "strip") }) - pctx.VariableFunc("MacToolPath", func(config android.Config) (string, error) { - path, err := xcrun(config, "--find", "ld") - return filepath.Dir(path), err + pctx.VariableFunc("MacToolPath", func(ctx android.PackageVarContext) string { + return filepath.Dir(xcrun(ctx, "--find", "ld")) }) pctx.StaticVariable("DarwinGccVersion", darwinGccVersion) @@ -156,33 +157,38 @@ func init() { pctx.StaticVariable("DarwinX8664YasmFlags", "-f macho -m amd64") } -func xcrun(config android.Config, args ...string) (string, error) { - xcrun := config.HostSystemTool("xcrun") +func xcrun(ctx android.PackageVarContext, args ...string) string { + xcrun := ctx.Config().HostSystemTool("xcrun") bytes, err := exec.Command(xcrun, args...).Output() - return strings.TrimSpace(string(bytes)), err + if err != nil { + ctx.Errorf("xcrun failed with: %q", err.Error()) + } + return strings.TrimSpace(string(bytes)) } -func xcrunSdk(config android.Config, arg string) (string, error) { - xcrun := config.HostSystemTool("xcrun") - if selected := config.Getenv("MAC_SDK_VERSION"); selected != "" { +func xcrunSdk(ctx android.PackageVarContext, arg string) string { + xcrun := ctx.Config().HostSystemTool("xcrun") + if selected := ctx.Config().Getenv("MAC_SDK_VERSION"); selected != "" { if !inList(selected, darwinSupportedSdkVersions) { - return "", fmt.Errorf("MAC_SDK_VERSION %s isn't supported: %q", selected, darwinSupportedSdkVersions) + ctx.Errorf("MAC_SDK_VERSION %s isn't supported: %q", selected, darwinSupportedSdkVersions) + return "" } bytes, err := exec.Command(xcrun, "--sdk", "macosx"+selected, arg).Output() - if err == nil { - return strings.TrimSpace(string(bytes)), err + if err != nil { + ctx.Errorf("MAC_SDK_VERSION %s is not installed", selected) } - return "", fmt.Errorf("MAC_SDK_VERSION %s is not installed", selected) + return strings.TrimSpace(string(bytes)) } for _, sdk := range darwinSupportedSdkVersions { bytes, err := exec.Command(xcrun, "--sdk", "macosx"+sdk, arg).Output() if err == nil { - return strings.TrimSpace(string(bytes)), err + return strings.TrimSpace(string(bytes)) } } - return "", fmt.Errorf("Could not find a supported mac sdk: %q", darwinSupportedSdkVersions) + ctx.Errorf("Could not find a supported mac sdk: %q", darwinSupportedSdkVersions) + return "" } type toolchainDarwin struct { @@ -74,12 +74,27 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { var ltoFlag string if Bool(lto.Properties.Lto.Thin) { ltoFlag = "-flto=thin" + } else { ltoFlag = "-flto" } flags.CFlags = append(flags.CFlags, ltoFlag) flags.LdFlags = append(flags.LdFlags, ltoFlag) + + if ctx.Config().IsEnvTrue("USE_THINLTO_CACHE") && Bool(lto.Properties.Lto.Thin) { + // Set appropriate ThinLTO cache policy + cacheDirFormat := "-Wl,-plugin-opt,cache-dir=" + cacheDir := android.PathForOutput(ctx, "thinlto-cache").String() + flags.LdFlags = append(flags.LdFlags, cacheDirFormat+cacheDir) + + // Limit the size of the ThinLTO cache to the lesser of 10% of available + // disk space and 10GB. + cachePolicyFormat := "-Wl,-plugin-opt,cache-policy=" + policy := "cache_size=10%:cache_size_bytes=10g" + flags.LdFlags = append(flags.LdFlags, cachePolicyFormat+policy) + } + if ctx.Device() { // Work around bug in Clang that doesn't pass correct emulated // TLS option to target. See b/72706604 or diff --git a/cc/makevars.go b/cc/makevars.go index 0386f9321..042af2a14 100644 --- a/cc/makevars.go +++ b/cc/makevars.go @@ -196,9 +196,6 @@ func makeVarsToolchain(ctx android.MakeVarsContext, secondPrefix string, hod = "Device" } - if target.Os.Class == android.Device && Bool(ctx.Config().ProductVariables.Brillo) { - productExtraCflags += "-D__BRILLO__" - } if target.Os.Class == android.Host && Bool(ctx.Config().ProductVariables.HostStaticBinaries) { productExtraLdflags += "-static" } @@ -310,7 +307,7 @@ func makeVarsToolchain(ctx android.MakeVarsContext, secondPrefix string, ctx.Strict(makePrefix+"STRIP", gccCmd(toolchain, "strip")) ctx.Strict(makePrefix+"GCC_VERSION", toolchain.GccVersion()) ctx.Strict(makePrefix+"NDK_GCC_VERSION", toolchain.GccVersion()) - ctx.Strict(makePrefix+"NDK_TRIPLE", toolchain.ClangTriple()) + ctx.Strict(makePrefix+"NDK_TRIPLE", config.NDKTriple(toolchain)) } if target.Os.Class == android.Host || target.Os.Class == android.HostCross { @@ -49,6 +49,11 @@ func rsGeneratedCppFile(ctx android.ModuleContext, rsFile android.Path) android. return android.PathForModuleGen(ctx, "rs", "ScriptC_"+fileName+".cpp") } +func rsGeneratedHFile(ctx android.ModuleContext, rsFile android.Path) android.WritablePath { + fileName := strings.TrimSuffix(rsFile.Base(), rsFile.Ext()) + return android.PathForModuleGen(ctx, "rs", "ScriptC_"+fileName+".h") +} + func rsGeneratedDepFile(ctx android.ModuleContext, rsFile android.Path) android.WritablePath { fileName := strings.TrimSuffix(rsFile.Base(), rsFile.Ext()) return android.PathForModuleGen(ctx, "rs", fileName+".d") @@ -56,18 +61,20 @@ func rsGeneratedDepFile(ctx android.ModuleContext, rsFile android.Path) android. func rsGenerateCpp(ctx android.ModuleContext, rsFiles android.Paths, rsFlags string) android.Paths { stampFile := android.PathForModuleGen(ctx, "rs", "rs.stamp") - depFiles := make(android.WritablePaths, len(rsFiles)) - cppFiles := make(android.WritablePaths, len(rsFiles)) - for i, rsFile := range rsFiles { - depFiles[i] = rsGeneratedDepFile(ctx, rsFile) - cppFiles[i] = rsGeneratedCppFile(ctx, rsFile) + depFiles := make(android.WritablePaths, 0, len(rsFiles)) + genFiles := make(android.WritablePaths, 0, 2*len(rsFiles)) + for _, rsFile := range rsFiles { + depFiles = append(depFiles, rsGeneratedDepFile(ctx, rsFile)) + genFiles = append(genFiles, + rsGeneratedCppFile(ctx, rsFile), + rsGeneratedHFile(ctx, rsFile)) } ctx.Build(pctx, android.BuildParams{ Rule: rsCpp, Description: "llvm-rs-cc", Output: stampFile, - ImplicitOutputs: cppFiles, + ImplicitOutputs: genFiles, Inputs: rsFiles, Args: map[string]string{ "rsFlags": rsFlags, diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index df04358b8..105765521 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -63,6 +63,7 @@ var ( zipsToNotStrip = make(zipsToNotStripSet) stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file") manifest = flag.String("m", "", "manifest file to insert in jar") + pyMain = flag.String("pm", "", "__main__.py file to insert in par") entrypoint = flag.String("e", "", "par entrypoint file to insert in par") ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn") ) @@ -75,7 +76,7 @@ func init() { func main() { flag.Usage = func() { - fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [-e entrypoint] output [inputs...]") + fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [-e entrypoint] [-pm __main__.py] output [inputs...]") flag.PrintDefaults() } @@ -125,8 +126,12 @@ func main() { log.Fatal(errors.New("must specify -p when specifying a entrypoint via -e")) } + if *pyMain != "" && !*emulatePar { + log.Fatal(errors.New("must specify -p when specifying a Python __main__.py via -pm")) + } + // do merge - err = mergeZips(readers, writer, *manifest, *entrypoint, *sortEntries, *emulateJar, *emulatePar, + err = mergeZips(readers, writer, *manifest, *entrypoint, *pyMain, *sortEntries, *emulateJar, *emulatePar, *stripDirEntries, *ignoreDuplicates) if err != nil { log.Fatal(err) @@ -218,7 +223,7 @@ type fileMapping struct { source zipSource } -func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoint string, +func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoint, pyMain string, sortEntries, emulateJar, emulatePar, stripDirEntries, ignoreDuplicates bool) error { sourceByDest := make(map[string]zipSource, 0) @@ -268,6 +273,22 @@ func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoin addMapping("entry_point.txt", fileSource) } + if pyMain != "" { + buf, err := ioutil.ReadFile(pyMain) + if err != nil { + return err + } + fh := &zip.FileHeader{ + Name: "__main__.py", + Method: zip.Store, + UncompressedSize64: uint64(len(buf)), + } + fh.SetMode(0700) + fh.SetModTime(jar.DefaultTime) + fileSource := bufferEntry{fh, buf} + addMapping("__main__.py", fileSource) + } + if emulatePar { // the runfiles packages needs to be populated with "__init__.py". newPyPkgs := []string{} diff --git a/cmd/zipsync/zipsync.go b/cmd/zipsync/zipsync.go index 035a1455f..ed8a06f5a 100644 --- a/cmd/zipsync/zipsync.go +++ b/cmd/zipsync/zipsync.go @@ -118,7 +118,10 @@ func main() { } if *outputFile != "" { - data := strings.Join(files, "\n") + "\n" + data := strings.Join(files, "\n") + if len(files) > 0 { + data += "\n" + } must(ioutil.WriteFile(*outputFile, []byte(data), 0666)) } } diff --git a/java/androidmk.go b/java/androidmk.go index 3658636f8..1e1577909 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -218,15 +218,15 @@ func (app *AndroidApp) AndroidMk() android.AndroidMkData { func (jd *Javadoc) AndroidMk() android.AndroidMkData { return android.AndroidMkData{ Class: "JAVA_LIBRARIES", - OutputFile: android.OptionalPathForPath(jd.stubsJar), + OutputFile: android.OptionalPathForPath(jd.stubsSrcJar), Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", Extra: []android.AndroidMkExtraFunc{ func(w io.Writer, outputFile android.Path) { if jd.properties.Installable == nil || *jd.properties.Installable == true { fmt.Fprintln(w, "LOCAL_DROIDDOC_DOC_ZIP := ", jd.docZip.String()) } - if jd.stubsJar != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_JAR := ", jd.stubsJar.String()) + if jd.stubsSrcJar != nil { + fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_SRCJAR := ", jd.stubsSrcJar.String()) } }, }, @@ -236,15 +236,34 @@ func (jd *Javadoc) AndroidMk() android.AndroidMkData { func (ddoc *Droiddoc) AndroidMk() android.AndroidMkData { return android.AndroidMkData{ Class: "JAVA_LIBRARIES", - OutputFile: android.OptionalPathForPath(ddoc.stubsJar), + OutputFile: android.OptionalPathForPath(ddoc.stubsSrcJar), Include: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", Extra: []android.AndroidMkExtraFunc{ func(w io.Writer, outputFile android.Path) { if ddoc.Javadoc.properties.Installable == nil || *ddoc.Javadoc.properties.Installable == true { fmt.Fprintln(w, "LOCAL_DROIDDOC_DOC_ZIP := ", ddoc.Javadoc.docZip.String()) } - if ddoc.Javadoc.stubsJar != nil { - fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_JAR := ", ddoc.Javadoc.stubsJar.String()) + if ddoc.Javadoc.stubsSrcJar != nil { + fmt.Fprintln(w, "LOCAL_DROIDDOC_STUBS_SRCJAR := ", ddoc.Javadoc.stubsSrcJar.String()) + } + apiFilePrefix := "INTERNAL_PLATFORM_" + if String(ddoc.properties.Api_tag_name) != "" { + apiFilePrefix += String(ddoc.properties.Api_tag_name) + "_" + } + if String(ddoc.properties.Api_filename) != "" { + fmt.Fprintln(w, apiFilePrefix+"API_FILE := ", ddoc.apiFile.String()) + } + if String(ddoc.properties.Private_api_filename) != "" { + fmt.Fprintln(w, apiFilePrefix+"PRIVATE_API_FILE := ", ddoc.privateApiFile.String()) + } + if String(ddoc.properties.Private_dex_api_filename) != "" { + fmt.Fprintln(w, apiFilePrefix+"PRIVATE_DEX_API_FILE := ", ddoc.privateDexApiFile.String()) + } + if String(ddoc.properties.Removed_api_filename) != "" { + fmt.Fprintln(w, apiFilePrefix+"REMOVED_API_FILE := ", ddoc.removedApiFile.String()) + } + if String(ddoc.properties.Exact_api_filename) != "" { + fmt.Fprintln(w, apiFilePrefix+"EXACT_API_FILE := ", ddoc.exactApiFile.String()) } }, }, diff --git a/java/builder.go b/java/builder.go index bc9559955..d36e0dc0f 100644 --- a/java/builder.go +++ b/java/builder.go @@ -66,6 +66,7 @@ var ( `${config.GenKotlinBuildFileCmd} $classpath $outDir $out.rsp $srcJarDir/list > $outDir/kotlinc-build.xml &&` + `${config.KotlincCmd} $kotlincFlags ` + `-jvm-target $kotlinJvmTarget -Xbuild-file=$outDir/kotlinc-build.xml && ` + + `rm $outDir/kotlinc-build.xml && ` + `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`, CommandDeps: []string{ "${config.KotlincCmd}", @@ -108,7 +109,7 @@ var ( `${config.JavaCmd} -jar ${config.TurbineJar} --output $out.tmp ` + `--temp_dir "$outDir" --sources @$out.rsp --source_jars $srcJars ` + `--javacopts ${config.CommonJdkFlags} ` + - `$javacFlags -source $javaVersion -target $javaVersion $bootClasspath $classpath && ` + + `$javacFlags -source $javaVersion -target $javaVersion -- $bootClasspath $classpath && ` + `${config.Ziptime} $out.tmp && ` + `(if cmp -s $out.tmp $out ; then rm $out.tmp ; else mv $out.tmp $out ; fi )`, CommandDeps: []string{ @@ -242,7 +243,7 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android. // ensure java does not fall back to the default bootclasspath. bootClasspath = `--bootclasspath ""` } else { - bootClasspath = flags.bootClasspath.FormJavaClassPath("--bootclasspath") + bootClasspath = strings.Join(flags.bootClasspath.FormDesugarClasspath("--bootclasspath"), " ") } ctx.Build(pctx, android.BuildParams{ @@ -255,7 +256,7 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android. "javacFlags": flags.javacFlags, "bootClasspath": bootClasspath, "srcJars": strings.Join(srcJars.Strings(), " "), - "classpath": flags.classpath.FormJavaClassPath("--classpath"), + "classpath": strings.Join(flags.classpath.FormDesugarClasspath("--classpath"), " "), "outDir": android.PathForModuleOut(ctx, "turbine", "classes").String(), "javaVersion": flags.javaVersion, }, @@ -358,6 +359,11 @@ func TransformJarsToJar(ctx android.ModuleContext, outputFile android.WritablePa // for downstream tools like desugar. jarArgs = append(jarArgs, "-stripFile module-info.class") + // Remove any kotlin-reflect related files + // TODO(pszczepaniak): Support kotlin-reflect + jarArgs = append(jarArgs, "-stripFile \"*.kotlin_module\"") + jarArgs = append(jarArgs, "-stripFile \"*.kotlin_builtin\"") + if stripDirs { jarArgs = append(jarArgs, "-D") } diff --git a/java/config/config.go b/java/config/config.go index 37f652821..5587a16dc 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -68,9 +68,9 @@ func init() { pctx.VariableConfigMethod("hostPrebuiltTag", android.Config.PrebuiltOS) - pctx.VariableFunc("JavaHome", func(config android.Config) (string, error) { + pctx.VariableFunc("JavaHome", func(ctx android.PackageVarContext) string { // This is set up and guaranteed by soong_ui - return config.Getenv("ANDROID_JAVA_HOME"), nil + return ctx.Config().Getenv("ANDROID_JAVA_HOME") }) pctx.SourcePathVariable("JavaToolchain", "${JavaHome}/bin") @@ -91,38 +91,27 @@ func init() { pctx.HostBinToolVariable("MergeZipsCmd", "merge_zips") pctx.HostBinToolVariable("Zip2ZipCmd", "zip2zip") pctx.HostBinToolVariable("ZipSyncCmd", "zipsync") - pctx.VariableFunc("DxCmd", func(config android.Config) (string, error) { + pctx.VariableFunc("DxCmd", func(ctx android.PackageVarContext) string { + config := ctx.Config() if config.IsEnvFalse("USE_D8") { if config.UnbundledBuild() || config.IsPdkBuild() { - return "prebuilts/build-tools/common/bin/dx", nil + return "prebuilts/build-tools/common/bin/dx" } else { - path, err := pctx.HostBinToolPath(config, "dx") - if err != nil { - return "", err - } - return path.String(), nil + return pctx.HostBinToolPath(ctx, "dx").String() } } else { - path, err := pctx.HostBinToolPath(config, "d8-compat-dx") - if err != nil { - return "", err - } - return path.String(), nil + return pctx.HostBinToolPath(ctx, "d8-compat-dx").String() } }) pctx.HostBinToolVariable("D8Cmd", "d8") pctx.HostBinToolVariable("R8Cmd", "r8-compat-proguard") - pctx.VariableFunc("TurbineJar", func(config android.Config) (string, error) { + pctx.VariableFunc("TurbineJar", func(ctx android.PackageVarContext) string { turbine := "turbine.jar" - if config.UnbundledBuild() { - return "prebuilts/build-tools/common/framework/" + turbine, nil + if ctx.Config().UnbundledBuild() { + return "prebuilts/build-tools/common/framework/" + turbine } else { - path, err := pctx.HostJavaToolPath(config, turbine) - if err != nil { - return "", err - } - return path.String(), nil + return pctx.HostJavaToolPath(ctx, turbine).String() } }) @@ -133,25 +122,21 @@ func init() { pctx.HostBinToolVariable("SoongJavacWrapper", "soong_javac_wrapper") - pctx.VariableFunc("JavacWrapper", func(config android.Config) (string, error) { - if override := config.Getenv("JAVAC_WRAPPER"); override != "" { - return override + " ", nil + pctx.VariableFunc("JavacWrapper", func(ctx android.PackageVarContext) string { + if override := ctx.Config().Getenv("JAVAC_WRAPPER"); override != "" { + return override + " " } - return "", nil + return "" }) pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) { - pctx.VariableFunc(name, func(config android.Config) (string, error) { - if config.UnbundledBuild() || config.IsPdkBuild() { - return filepath.Join(prebuiltDir, runtime.GOOS, "bin", tool), nil + pctx.VariableFunc(name, func(ctx android.PackageVarContext) string { + if ctx.Config().UnbundledBuild() || ctx.Config().IsPdkBuild() { + return filepath.Join(prebuiltDir, runtime.GOOS, "bin", tool) } else { - if path, err := pctx.HostBinToolPath(config, tool); err != nil { - return "", err - } else { - return path.String(), nil - } + return pctx.HostBinToolPath(ctx, tool).String() } }) } diff --git a/java/config/error_prone.go b/java/config/error_prone.go index 862217f77..f2032342b 100644 --- a/java/config/error_prone.go +++ b/java/config/error_prone.go @@ -27,8 +27,8 @@ var ( // Wrapper that grabs value of val late so it can be initialized by a later module's init function func errorProneVar(name string, val *string) { - pctx.VariableFunc(name, func(config android.Config) (string, error) { - return *val, nil + pctx.VariableFunc(name, func(android.PackageVarContext) string { + return *val }) } diff --git a/java/droiddoc.go b/java/droiddoc.go index 7171c9186..c65911ce3 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -18,6 +18,7 @@ import ( "android/soong/android" "android/soong/java/config" "fmt" + "path/filepath" "strings" "github.com/google/blueprint" @@ -49,6 +50,8 @@ var ( ) func init() { + android.RegisterModuleType("doc_defaults", DocDefaultsFactory) + android.RegisterModuleType("droiddoc", DroiddocFactory) android.RegisterModuleType("droiddoc_host", DroiddocHostFactory) android.RegisterModuleType("droiddoc_template", DroiddocTemplateFactory) @@ -63,18 +66,31 @@ type JavadocProperties struct { // list of directories rooted at the Android.bp file that will // be added to the search paths for finding source files when passing package names. - Local_sourcepaths []string `android:"arch_variant"` + Local_sourcepaths []string // list of source files that should not be used to build the Java module. // This is most useful in the arch/multilib variants to remove non-common files // filegroup or genrule can be included within this property. Exclude_srcs []string `android:"arch_variant"` - // list of of java libraries that will be in the classpath. + // list of java libraries that will be in the classpath. Libs []string `android:"arch_variant"` + // don't build against the framework libraries (legacy-test, core-junit, + // ext, and framework for device targets) + No_framework_libs *bool + + // the java library (in classpath) for documentation that provides java srcs and srcjars. + Srcs_lib *string + + // the base dirs under srcs_lib will be scanned for java srcs. + Srcs_lib_whitelist_dirs []string + + // the sub dirs under srcs_lib_whitelist_dirs will be scanned for java srcs. + Srcs_lib_whitelist_pkgs []string + // If set to false, don't allow this module(-docs.zip) to be exported. Defaults to true. - Installable *bool `android:"arch_variant"` + Installable *bool // if not blank, set to the version of the sdk to compile against Sdk_version *string `android:"arch_variant"` @@ -82,37 +98,61 @@ type JavadocProperties struct { type DroiddocProperties struct { // directory relative to top of the source tree that contains doc templates files. - Custom_template *string `android:"arch_variant"` + Custom_template *string // directories relative to top of the source tree which contains html/jd files. - Html_dirs []string `android:"arch_variant"` + Html_dirs []string // set a value in the Clearsilver hdf namespace. - Hdf []string `android:"arch_variant"` + Hdf []string // proofread file contains all of the text content of the javadocs concatenated into one file, // suitable for spell-checking and other goodness. - Proofread_file *string `android:"arch_variant"` + Proofread_file *string // a todo file lists the program elements that are missing documentation. // At some point, this might be improved to show more warnings. - Todo_file *string `android:"arch_variant"` + Todo_file *string + + // directory under current module source that provide additional resources (images). + Resourcesdir *string + + // resources output directory under out/soong/.intermediates. + Resourcesoutdir *string // local files that are used within user customized droiddoc options. - Arg_files []string `android:"arch_variant"` + Arg_files []string // user customized droiddoc args. // Available variables for substitution: // // $(location <label>): the path to the arg_files with name <label> - Args *string `android:"arch_variant"` + Args *string // names of the output files used in args that will be generated - Out []string `android:"arch_variant"` + Out []string // a list of files under current module source dir which contains known tags in Java sources. // filegroup or genrule can be included within this property. - Knowntags []string `android:"arch_variant"` + Knowntags []string + + // the tag name used to distinguish if the API files belong to public/system/test. + Api_tag_name *string + + // the generated public API filename by Doclava. + Api_filename *string + + // the generated private API filename by Doclava. + Private_api_filename *string + + // the generated private Dex API filename by Doclava. + Private_dex_api_filename *string + + // the generated removed API filename by Doclava. + Removed_api_filename *string + + // the generated exact API filename by Doclava. + Exact_api_filename *string } type Javadoc struct { @@ -125,14 +165,25 @@ type Javadoc struct { srcFiles android.Paths sourcepaths android.Paths - docZip android.WritablePath - stubsJar android.WritablePath + docZip android.WritablePath + stubsSrcJar android.WritablePath +} + +func (j *Javadoc) Srcs() android.Paths { + return android.Paths{j.stubsSrcJar} } +var _ android.SourceFileProducer = (*Javadoc)(nil) + type Droiddoc struct { Javadoc - properties DroiddocProperties + properties DroiddocProperties + apiFile android.WritablePath + privateApiFile android.WritablePath + privateDexApiFile android.WritablePath + removedApiFile android.WritablePath + exactApiFile android.WritablePath } func InitDroiddocModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) { @@ -183,7 +234,9 @@ func (j *Javadoc) addDeps(ctx android.BottomUpMutatorContext) { sdkDep := decodeSdkDep(ctx, String(j.properties.Sdk_version)) if sdkDep.useDefaultLibs { ctx.AddDependency(ctx.Module(), bootClasspathTag, config.DefaultBootclasspathLibraries...) - ctx.AddDependency(ctx.Module(), libTag, []string{"ext", "framework"}...) + if !Bool(j.properties.No_framework_libs) { + ctx.AddDependency(ctx.Module(), libTag, []string{"ext", "framework"}...) + } } else if sdkDep.useModule { ctx.AddDependency(ctx.Module(), bootClasspathTag, sdkDep.module) } @@ -197,6 +250,17 @@ func (j *Javadoc) addDeps(ctx android.BottomUpMutatorContext) { android.ExtractSourcesDeps(ctx, j.properties.Exclude_srcs) } +func (j *Javadoc) genWhitelistPathPrefixes(whitelistPathPrefixes map[string]bool) { + for _, dir := range j.properties.Srcs_lib_whitelist_dirs { + for _, pkg := range j.properties.Srcs_lib_whitelist_pkgs { + prefix := filepath.Join(dir, pkg) + if _, found := whitelistPathPrefixes[prefix]; !found { + whitelistPathPrefixes[prefix] = true + } + } + } +} + func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { var deps deps @@ -218,6 +282,24 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { deps.bootClasspath = append(deps.bootClasspath, dep.ImplementationJars()...) case libTag: deps.classpath = append(deps.classpath, dep.ImplementationJars()...) + if otherName == String(j.properties.Srcs_lib) { + srcs := dep.(SrcDependency).CompiledSrcs() + whitelistPathPrefixes := make(map[string]bool) + j.genWhitelistPathPrefixes(whitelistPathPrefixes) + for _, src := range srcs { + if _, ok := src.(android.WritablePath); ok { // generated sources + deps.srcs = append(deps.srcs, src) + } else { // select source path for documentation based on whitelist path prefixs. + for k, _ := range whitelistPathPrefixes { + if strings.HasPrefix(src.Rel(), k) { + deps.srcs = append(deps.srcs, src) + break + } + } + } + } + deps.srcJars = append(deps.srcJars, dep.(SrcDependency).CompiledSrcJars()...) + } default: panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName())) } @@ -246,10 +328,13 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { // srcs may depend on some genrule output. j.srcJars = srcFiles.FilterByExt(".srcjar") + j.srcJars = append(j.srcJars, deps.srcJars...) + j.srcFiles = srcFiles.FilterOutByExt(".srcjar") + j.srcFiles = append(j.srcFiles, deps.srcs...) j.docZip = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"docs.zip") - j.stubsJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") + j.stubsSrcJar = android.PathForModuleOut(ctx, ctx.ModuleName()+"-"+"stubs.srcjar") if j.properties.Local_sourcepaths == nil { j.properties.Local_sourcepaths = append(j.properties.Local_sourcepaths, ".") @@ -298,7 +383,7 @@ func (j *Javadoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.Build(pctx, android.BuildParams{ Rule: javadoc, Description: "Javadoc", - Output: j.stubsJar, + Output: j.stubsSrcJar, ImplicitOutput: j.docZip, Inputs: j.srcFiles, Implicits: implicits, @@ -422,6 +507,7 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { proofreadFile := android.PathForModuleOut(ctx, String(d.properties.Proofread_file)) args = args + " -proofread " + proofreadFile.String() } + if String(d.properties.Todo_file) != "" { // tricky part: // we should not compute full path for todo_file through PathForModuleOut(). @@ -429,6 +515,49 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { args = args + " -todo " + String(d.properties.Todo_file) } + if String(d.properties.Resourcesdir) != "" { + // TODO: should we add files under resourcesDir to the implicits? It seems that + // resourcesDir is one sub dir of htmlDir + resourcesDir := android.PathForModuleSrc(ctx, String(d.properties.Resourcesdir)) + args = args + " -resourcesdir " + resourcesDir.String() + } + + if String(d.properties.Resourcesoutdir) != "" { + // TODO: it seems -resourceoutdir reference/android/images/ didn't get generated anywhere. + args = args + " -resourcesoutdir " + String(d.properties.Resourcesoutdir) + } + + var implicitOutputs android.WritablePaths + if String(d.properties.Api_filename) != "" { + d.apiFile = android.PathForModuleOut(ctx, String(d.properties.Api_filename)) + args = args + " -api " + d.apiFile.String() + implicitOutputs = append(implicitOutputs, d.apiFile) + } + + if String(d.properties.Private_api_filename) != "" { + d.privateApiFile = android.PathForModuleOut(ctx, String(d.properties.Private_api_filename)) + args = args + " -privateApi " + d.privateApiFile.String() + implicitOutputs = append(implicitOutputs, d.privateApiFile) + } + + if String(d.properties.Private_dex_api_filename) != "" { + d.privateDexApiFile = android.PathForModuleOut(ctx, String(d.properties.Private_dex_api_filename)) + args = args + " -privateDexApi " + d.privateDexApiFile.String() + implicitOutputs = append(implicitOutputs, d.privateDexApiFile) + } + + if String(d.properties.Removed_api_filename) != "" { + d.removedApiFile = android.PathForModuleOut(ctx, String(d.properties.Removed_api_filename)) + args = args + " -removedApi " + d.removedApiFile.String() + implicitOutputs = append(implicitOutputs, d.removedApiFile) + } + + if String(d.properties.Exact_api_filename) != "" { + d.exactApiFile = android.PathForModuleOut(ctx, String(d.properties.Exact_api_filename)) + args = args + " -exactApi " + d.exactApiFile.String() + implicitOutputs = append(implicitOutputs, d.exactApiFile) + } + implicits = append(implicits, d.Javadoc.srcJars...) opts := "-source 1.8 -J-Xmx1600m -J-XX:-OmitStackTraceInFastThrow -XDignore.symbol.file " + @@ -438,7 +567,6 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { "-hdf page.now " + `"$$(date -d @$$(cat ` + ctx.Config().Getenv("BUILD_DATETIME_FILE") + `) "+%d %b %Y %k:%M")"` + " " + args + " -stubs " + android.PathForModuleOut(ctx, "docs", "stubsDir").String() - var implicitOutputs android.WritablePaths implicitOutputs = append(implicitOutputs, d.Javadoc.docZip) for _, o := range d.properties.Out { implicitOutputs = append(implicitOutputs, android.PathForModuleGen(ctx, o)) @@ -447,7 +575,7 @@ func (d *Droiddoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.Build(pctx, android.BuildParams{ Rule: javadoc, Description: "Droiddoc", - Output: d.Javadoc.stubsJar, + Output: d.Javadoc.stubsSrcJar, Inputs: d.Javadoc.srcFiles, Implicits: implicits, ImplicitOutputs: implicitOutputs, @@ -497,3 +625,30 @@ func (d *DroiddocTemplate) GenerateAndroidBuildActions(ctx android.ModuleContext d.dir = path d.deps = ctx.Glob(path.Join(ctx, "**/*").String(), nil) } + +// +// Defaults +// +type DocDefaults struct { + android.ModuleBase + android.DefaultsModuleBase +} + +func (*DocDefaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { +} + +func (d *DocDefaults) DepsMutator(ctx android.BottomUpMutatorContext) { +} + +func DocDefaultsFactory() android.Module { + module := &DocDefaults{} + + module.AddProperties( + &JavadocProperties{}, + &DroiddocProperties{}, + ) + + android.InitDefaultsModule(module) + + return module +} diff --git a/java/java.go b/java/java.go index b04f5be01..06dce052a 100644 --- a/java/java.go +++ b/java/java.go @@ -19,7 +19,6 @@ package java // is handled in builder.go import ( - "fmt" "path/filepath" "strconv" "strings" @@ -83,6 +82,10 @@ type CompilerProperties struct { // ext, and framework for device targets) No_framework_libs *bool + // Use renamed kotlin stdlib (com.android.kotlin.*). This allows kotlin usage without colliding + // with app-provided kotlin stdlib. + Renamed_kotlin_stdlib *bool + // list of module-specific flags that will be used for javac compiles Javacflags []string `android:"arch_variant"` @@ -175,6 +178,9 @@ type CompilerDeviceProperties struct { // directories that should be added as include directories for any aidl sources of modules // that depend on this module, as well as to aidl for this module. Export_include_dirs []string + + // whether to generate traces (for systrace) for this interface + Generate_traces *bool } // If true, export a copy of the module as a -hostdex module for host testing. @@ -283,6 +289,21 @@ type Dependency interface { AidlIncludeDirs() android.Paths } +type SrcDependency interface { + CompiledSrcs() android.Paths + CompiledSrcJars() android.Paths +} + +func (j *Module) CompiledSrcs() android.Paths { + return j.compiledJavaSrcs +} + +func (j *Module) CompiledSrcJars() android.Paths { + return j.compiledSrcJars +} + +var _ SrcDependency = (*Module)(nil) + func InitJavaModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) { android.InitAndroidArchModule(module, hod, android.MultilibCommon) android.InitDefaultableModule(module) @@ -299,6 +320,7 @@ var ( bootClasspathTag = dependencyTag{name: "bootclasspath"} systemModulesTag = dependencyTag{name: "system modules"} frameworkResTag = dependencyTag{name: "framework-res"} + frameworkApkTag = dependencyTag{name: "framework-apk"} kotlinStdlibTag = dependencyTag{name: "kotlin-stdlib"} proguardRaiseTag = dependencyTag{name: "proguard-raise"} ) @@ -459,6 +481,11 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { if ctx.ModuleName() == "framework" { ctx.AddDependency(ctx.Module(), frameworkResTag, "framework-res") } + if ctx.ModuleName() == "android_stubs_current" || + ctx.ModuleName() == "android_system_stubs_current" || + ctx.ModuleName() == "android_test_stubs_current" { + ctx.AddDependency(ctx.Module(), frameworkApkTag, "framework-res") + } } ctx.AddDependency(ctx.Module(), libTag, j.properties.Libs...) @@ -534,6 +561,10 @@ func (j *Module) aidlFlags(ctx android.ModuleContext, aidlPreprocess android.Opt flags = append(flags, "-I"+src.String()) } + if Bool(j.deviceProperties.Aidl.Generate_traces) { + flags = append(flags, "-t") + } + return flags } @@ -544,6 +575,7 @@ type deps struct { staticHeaderJars android.Paths staticJarResources android.Paths aidlIncludeDirs android.Paths + srcs android.Paths srcJars android.Paths systemModules android.Path aidlPreprocess android.OptionalPath @@ -571,13 +603,15 @@ func checkLinkType(ctx android.ModuleContext, from *Module, to *Library, tag dep func (j *Module) collectDeps(ctx android.ModuleContext) deps { var deps deps - sdkDep := decodeSdkDep(ctx, String(j.deviceProperties.Sdk_version)) - if sdkDep.invalidVersion { - ctx.AddMissingDependencies([]string{sdkDep.module}) - } else if sdkDep.useFiles { - // sdkDep.jar is actually equivalent to turbine header.jar. - deps.classpath = append(deps.classpath, sdkDep.jar) - deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, sdkDep.aidl) + if ctx.Device() { + sdkDep := decodeSdkDep(ctx, String(j.deviceProperties.Sdk_version)) + if sdkDep.invalidVersion { + ctx.AddMissingDependencies([]string{sdkDep.module}) + } else if sdkDep.useFiles { + // sdkDep.jar is actually equivalent to turbine header.jar. + deps.classpath = append(deps.classpath, sdkDep.jar) + deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, sdkDep.aidl) + } } ctx.VisitDirectDeps(func(module android.Module) { @@ -604,10 +638,20 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { // generated by framework-res.apk deps.srcJars = append(deps.srcJars, dep.(*AndroidApp).aaptSrcJar) } + case frameworkApkTag: + if ctx.ModuleName() == "android_stubs_current" || + ctx.ModuleName() == "android_system_stubs_current" || + ctx.ModuleName() == "android_test_stubs_current" { + // framework stubs.jar need to depend on framework-res.apk, in order to pull the + // resource files out of there for aapt. + // + // Normally the package rule runs aapt, which includes the resource, + // but we're not running that in our package rule so just copy in the + // resource files here. + deps.staticJarResources = append(deps.staticJarResources, dep.(*AndroidApp).exportPackage) + } case kotlinStdlibTag: deps.kotlinStdlib = dep.HeaderJars() - default: - panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName())) } deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...) @@ -774,6 +818,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path // won't emit any classes for them. flags.kotlincFlags = "-no-stdlib" + if ctx.Device() { flags.kotlincFlags += " -no-jdk" } @@ -782,6 +827,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path kotlinSrcFiles = append(kotlinSrcFiles, uniqueSrcFiles...) kotlinSrcFiles = append(kotlinSrcFiles, srcFiles.FilterByExt(".kt")...) + flags.kotlincClasspath = append(flags.kotlincClasspath, deps.bootClasspath...) flags.kotlincClasspath = append(flags.kotlincClasspath, deps.kotlinStdlib...) flags.kotlincClasspath = append(flags.kotlincClasspath, deps.classpath...) @@ -793,9 +839,15 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path // Make javac rule depend on the kotlinc rule flags.classpath = append(flags.classpath, kotlinJar) + // Jar kotlin classes into the final jar after javac jars = append(jars, kotlinJar) - jars = append(jars, deps.kotlinStdlib...) + + // Don't add kotlin-stdlib if using (on-device) renamed stdlib + // (it's expected to be on device bootclasspath) + if !proptools.Bool(j.properties.Renamed_kotlin_stdlib) { + jars = append(jars, deps.kotlinStdlib...) + } } // Store the list of .java files that was passed to javac @@ -892,6 +944,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path // static classpath jars have the resources in them, so the resource jars aren't necessary here jars = append(jars, deps.staticJars...) + jars = append(jars, deps.staticJarResources...) var manifest android.OptionalPath if j.properties.Manifest != nil { @@ -914,6 +967,17 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars ...android.Path outputFile = combinedJar } + // Use renamed kotlin standard library? + if srcFiles.HasExt(".kt") && proptools.Bool(j.properties.Renamed_kotlin_stdlib) { + jarjarFile := android.PathForModuleOut(ctx, "kotlin-renamed", jarName) + TransformJarJar(ctx, jarjarFile, outputFile, + android.PathForSource(ctx, "external/kotlinc/jarjar-rules.txt")) + outputFile = jarjarFile + if ctx.Failed() { + return + } + } + if j.properties.Jarjar_rules != nil { jarjar_rules := android.PathForModuleSrc(ctx, *j.properties.Jarjar_rules) // Transform classes.jar into classes-jarjar.jar diff --git a/java/java_test.go b/java/java_test.go index 6ef406f76..6e3ada424 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -176,6 +176,8 @@ func testContext(config android.Config, bp string, "bar-doc/b.java": nil, "bar-doc/known_oj_tags.txt": nil, "external/doclava/templates-sdk": nil, + + "external/kotlinc/jarjar-rules.txt": nil, } for k, v := range fs { @@ -755,6 +757,12 @@ func TestKotlin(t *testing.T) { name: "baz", srcs: ["c.java"], } + + java_library { + name: "blorg", + renamed_kotlin_stdlib: true, + srcs: ["b.kt"], + } `) fooKotlinc := ctx.ModuleForTests("foo", "android_common").Rule("kotlinc") @@ -797,6 +805,12 @@ func TestKotlin(t *testing.T) { t.Errorf(`expected %q in bar implicits %v`, bazHeaderJar.Output.String(), barKotlinc.Implicits.Strings()) } + + blorgRenamedJar := ctx.ModuleForTests("blorg", "android_common").Output("kotlin-renamed/blorg.jar") + if blorgRenamedJar.Implicit.String() != "external/kotlinc/jarjar-rules.txt" { + t.Errorf(`expected external/kotlinc/jarjar-rules.txt in blorg implicit %q`, + blorgRenamedJar.Implicit.String()) + } } func TestTurbine(t *testing.T) { diff --git a/python/binary.go b/python/binary.go index 0314edb8c..4135dfef9 100644 --- a/python/binary.go +++ b/python/binary.go @@ -18,8 +18,6 @@ package python import ( "fmt" - "path/filepath" - "strings" "android/soong/android" ) @@ -80,89 +78,44 @@ func (binary *binaryDecorator) bootstrapperProps() []interface{} { return []interface{}{&binary.binaryProperties} } -func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actual_version string, - embedded_launcher bool, srcsPathMappings []pathMapping, parSpec parSpec, - depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath { - // no Python source file for compiling .par file. - if len(srcsPathMappings) == 0 { - return android.OptionalPath{} - } - - // the runfiles packages needs to be populated with "__init__.py". - newPyPkgs := []string{} - // the set to de-duplicate the new Python packages above. - newPyPkgSet := make(map[string]bool) - // the runfiles dirs have been treated as packages. - existingPyPkgSet := make(map[string]bool) - - wholePyRunfiles := []string{} - for _, path := range srcsPathMappings { - wholePyRunfiles = append(wholePyRunfiles, path.dest) - } - wholePyRunfiles = append(wholePyRunfiles, depsPyRunfiles...) - - // find all the runfiles dirs which have been treated as packages. - for _, path := range wholePyRunfiles { - if filepath.Base(path) != initFileName { - continue - } - existingPyPkg := PathBeforeLastSlash(path) - if _, found := existingPyPkgSet[existingPyPkg]; found { - panic(fmt.Errorf("found init file path duplicates: %q for module: %q.", - path, ctx.ModuleName())) - } else { - existingPyPkgSet[existingPyPkg] = true - } - parentPath := PathBeforeLastSlash(existingPyPkg) - populateNewPyPkgs(parentPath, existingPyPkgSet, newPyPkgSet, &newPyPkgs) - } - - // create new packages under runfiles tree. - for _, path := range wholePyRunfiles { - if filepath.Base(path) == initFileName { - continue - } - parentPath := PathBeforeLastSlash(path) - populateNewPyPkgs(parentPath, existingPyPkgSet, newPyPkgSet, &newPyPkgs) - } +func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersion string, + embeddedLauncher bool, srcsPathMappings []pathMapping, srcsZip android.Path, + depsSrcsZips android.Paths) android.OptionalPath { main := binary.getPyMainFile(ctx, srcsPathMappings) - if main == "" { - return android.OptionalPath{} - } - var launcher_path android.Path - if embedded_launcher { + var launcherPath android.Path + if embeddedLauncher { ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) { if provider, ok := m.(IntermPathProvider); ok { - if launcher_path != nil { + if launcherPath != nil { panic(fmt.Errorf("launcher path was found before: %q", - launcher_path)) + launcherPath)) } - launcher_path = provider.IntermPathForModuleOut().Path() + launcherPath = provider.IntermPathForModuleOut().Path() } }) } - binFile := registerBuildActionForParFile(ctx, embedded_launcher, launcher_path, - binary.getHostInterpreterName(ctx, actual_version), - main, binary.getStem(ctx), newPyPkgs, append(depsParSpecs, parSpec)) + binFile := registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, + binary.getHostInterpreterName(ctx, actualVersion), + main, binary.getStem(ctx), append(android.Paths{srcsZip}, depsSrcsZips...)) return android.OptionalPathForPath(binFile) } // get host interpreter name. func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext, - actual_version string) string { + actualVersion string) string { var interp string - switch actual_version { + switch actualVersion { case pyVersion2: interp = "python2.7" case pyVersion3: interp = "python3" default: panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.", - actual_version, ctx.ModuleName())) + actualVersion, ctx.ModuleName())) } return interp @@ -196,30 +149,3 @@ func (binary *binaryDecorator) getStem(ctx android.ModuleContext) string { return stem + String(binary.binaryProperties.Suffix) } - -// Sets the given directory and all its ancestor directories as Python packages. -func populateNewPyPkgs(pkgPath string, existingPyPkgSet, - newPyPkgSet map[string]bool, newPyPkgs *[]string) { - for pkgPath != "" { - if _, found := existingPyPkgSet[pkgPath]; found { - break - } - if _, found := newPyPkgSet[pkgPath]; !found { - newPyPkgSet[pkgPath] = true - *newPyPkgs = append(*newPyPkgs, pkgPath) - // Gets its ancestor directory by trimming last slash. - pkgPath = PathBeforeLastSlash(pkgPath) - } else { - break - } - } -} - -// filepath.Dir("abc") -> "." and filepath.Dir("/abc") -> "/". However, -// the PathBeforeLastSlash() will return "" for both cases above. -func PathBeforeLastSlash(path string) string { - if idx := strings.LastIndex(path, "/"); idx != -1 { - return path[:idx] - } - return "" -} diff --git a/python/builder.go b/python/builder.go index f19435435..969f9efde 100644 --- a/python/builder.go +++ b/python/builder.go @@ -17,7 +17,6 @@ package python // This file contains Ninja build actions for building Python program. import ( - "fmt" "strings" "android/soong/android" @@ -29,25 +28,37 @@ import ( var ( pctx = android.NewPackageContext("android/soong/python") - host_par = pctx.AndroidStaticRule("host_par", + zip = pctx.AndroidStaticRule("zip", blueprint.RuleParams{ - Command: `touch $initFile && ` + - `sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` + - `$parCmd -o $parFile $parArgs && echo '#!/usr/bin/env python' | cat - $parFile > $out && ` + - `chmod +x $out && (rm -f $initFile; rm -f $stub; rm -f $parFile)`, - CommandDeps: []string{"$parCmd", "$template"}, + Command: `$parCmd -o $out $args`, + CommandDeps: []string{"$parCmd"}, }, - "initFile", "interp", "main", "template", "stub", "parCmd", "parFile", "parArgs") + "args") - embedded_par = pctx.AndroidStaticRule("embedded_par", + combineZip = pctx.AndroidStaticRule("combineZip", blueprint.RuleParams{ - Command: `touch $initFile && ` + - `echo '$main' > $entry_point && ` + - `$parCmd -o $parFile $parArgs && cat $launcher | cat - $parFile > $out && ` + - `chmod +x $out && (rm -f $initFile; rm -f $entry_point; rm -f $parFile)`, - CommandDeps: []string{"$parCmd"}, + Command: `$mergeParCmd $out $in`, + CommandDeps: []string{"$mergeParCmd"}, + }, + ) + + hostPar = pctx.AndroidStaticRule("hostPar", + blueprint.RuleParams{ + Command: `sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` + + `$mergeParCmd -p -pm $stub $mergedZip $srcsZips && echo '#!/usr/bin/env python' | cat - $mergedZip > $out && ` + + `chmod +x $out && (rm -f $stub; rm -f $mergedZip)`, + CommandDeps: []string{"$mergeParCmd"}, + }, + "interp", "main", "template", "stub", "mergedZip", "srcsZips") + + embeddedPar = pctx.AndroidStaticRule("embeddedPar", + blueprint.RuleParams{ + Command: `echo '$main' > $entryPoint &&` + + `$mergeParCmd -p -e $entryPoint $mergedZip $srcsZips && cat $launcher | cat - $mergedZip > $out && ` + + `chmod +x $out && (rm -f $entryPoint; rm -f $mergedZip)`, + CommandDeps: []string{"$mergeParCmd"}, }, - "initFile", "main", "entry_point", "parCmd", "parFile", "parArgs", "launcher") + "main", "entryPoint", "mergedZip", "srcsZips", "launcher") ) func init() { @@ -55,132 +66,65 @@ func init() { pctx.Import("android/soong/common") pctx.HostBinToolVariable("parCmd", "soong_zip") + pctx.HostBinToolVariable("mergeParCmd", "merge_zips") } -type fileListSpec struct { - fileList android.Path - relativeRoot string -} - -type parSpec struct { - rootPrefix string - - fileListSpecs []fileListSpec -} - -func (p parSpec) soongParArgs() string { - ret := `-P ` + p.rootPrefix - - for _, spec := range p.fileListSpecs { - ret += ` -C ` + spec.relativeRoot + ` -l ` + spec.fileList.String() - } - - return ret -} - -func registerBuildActionForModuleFileList(ctx android.ModuleContext, - name string, files android.Paths) android.Path { - fileList := android.PathForModuleOut(ctx, name+".list") - - content := []string{} - for _, file := range files { - content = append(content, file.String()) - } - - ctx.Build(pctx, android.BuildParams{ - Rule: android.WriteFile, - Description: "generate " + fileList.Rel(), - Output: fileList, - Implicits: files, - Args: map[string]string{ - "content": strings.Join(content, `\n`), - }, - }) - - return fileList -} - -func registerBuildActionForParFile(ctx android.ModuleContext, embedded_launcher bool, - launcher_path android.Path, interpreter, main, binName string, - newPyPkgs []string, parSpecs []parSpec) android.Path { - - // .intermediate output path for __init__.py - initFile := android.PathForModuleOut(ctx, initFileName).String() +func registerBuildActionForParFile(ctx android.ModuleContext, embeddedLauncher bool, + launcherPath android.Path, interpreter, main, binName string, + srcsZips android.Paths) android.Path { - // .intermediate output path for par file. - parFile := android.PathForModuleOut(ctx, binName+parFileExt) + // .intermediate output path for merged zip file. + mergedZip := android.PathForModuleOut(ctx, binName+".mergedzip") // .intermediate output path for bin executable. binFile := android.PathForModuleOut(ctx, binName) // implicit dependency for parFile build action. - implicits := android.Paths{} - for _, p := range parSpecs { - for _, f := range p.fileListSpecs { - implicits = append(implicits, f.fileList) - } - } - - parArgs := []string{} - parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(initFile, initFileName)+` -f `+initFile) - for _, pkg := range newPyPkgs { - parArgs = append(parArgs, `-P `+pkg+` -f `+initFile) - } - for _, p := range parSpecs { - parArgs = append(parArgs, p.soongParArgs()) - } + implicits := srcsZips - if !embedded_launcher { + if !embeddedLauncher { // the path of stub_template_host.txt from source tree. template := android.PathForSource(ctx, stubTemplateHost) + implicits = append(implicits, template) // intermediate output path for __main__.py stub := android.PathForModuleOut(ctx, mainFileName).String() - // added stub file to the soong_zip args. - parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(stub, mainFileName)+` -f `+stub) - ctx.Build(pctx, android.BuildParams{ - Rule: host_par, + Rule: hostPar, Description: "host python archive", Output: binFile, Implicits: implicits, Args: map[string]string{ - "initFile": initFile, - "interp": strings.Replace(interpreter, "/", `\/`, -1), + "interp": strings.Replace(interpreter, "/", `\/`, -1), // we need remove "runfiles/" suffix since stub script starts // searching for main file in each sub-dir of "runfiles" directory tree. "main": strings.Replace(strings.TrimPrefix(main, runFiles+"/"), "/", `\/`, -1), - "template": template.String(), - "stub": stub, - "parFile": parFile.String(), - "parArgs": strings.Join(parArgs, " "), + "template": template.String(), + "stub": stub, + "mergedZip": mergedZip.String(), + "srcsZips": strings.Join(srcsZips.Strings(), " "), }, }) } else { - // added launcher_path to the implicits Ninja dependencies. - implicits = append(implicits, launcher_path) + // added launcherPath to the implicits Ninja dependencies. + implicits = append(implicits, launcherPath) // .intermediate output path for entry_point.txt entryPoint := android.PathForModuleOut(ctx, entryPointFile).String() - // added entry_point file to the soong_zip args. - parArgs = append(parArgs, `-P "" `+`-C `+fmt.Sprintf( - "%q", strings.TrimSuffix(entryPoint, entryPointFile))+` -f `+entryPoint) - ctx.Build(pctx, android.BuildParams{ - Rule: embedded_par, + Rule: embeddedPar, Description: "embedded python archive", Output: binFile, Implicits: implicits, Args: map[string]string{ - "initFile": initFile, - "main": main, - "entry_point": entryPoint, - "parFile": parFile.String(), - "parArgs": strings.Join(parArgs, " "), - "launcher": launcher_path.String(), + "main": main, + "entryPoint": entryPoint, + "mergedZip": mergedZip.String(), + "srcsZips": strings.Join(srcsZips.Strings(), " "), + "launcher": launcherPath.String(), }, }) } diff --git a/python/proto.go b/python/proto.go new file mode 100644 index 000000000..82ee3cb0d --- /dev/null +++ b/python/proto.go @@ -0,0 +1,67 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "android/soong/android" + "strings" + + "github.com/google/blueprint" +) + +func init() { + pctx.HostBinToolVariable("protocCmd", "aprotoc") +} + +var ( + proto = pctx.AndroidStaticRule("protoc", + blueprint.RuleParams{ + Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` + + `$protocCmd --python_out=$out.tmp -I $protoBase $protoFlags $in && ` + + `$parCmd -o $out -P $pkgPath -C $out.tmp -D $out.tmp && rm -rf $out.tmp`, + CommandDeps: []string{ + "$protocCmd", + "$parCmd", + }, + }, "protoBase", "protoFlags", "pkgPath") +) + +func genProto(ctx android.ModuleContext, p *android.ProtoProperties, + protoFile android.Path, protoFlags []string, pkgPath string) android.Path { + srcJarFile := android.PathForModuleGen(ctx, protoFile.Base()+".srcszip") + + protoRoot := android.ProtoCanonicalPathFromRoot(ctx, p) + + var protoBase string + if protoRoot { + protoBase = "." + } else { + protoBase = strings.TrimSuffix(protoFile.String(), protoFile.Rel()) + } + + ctx.Build(pctx, android.BuildParams{ + Rule: proto, + Description: "protoc " + protoFile.Rel(), + Output: srcJarFile, + Input: protoFile, + Args: map[string]string{ + "protoBase": protoBase, + "protoFlags": strings.Join(protoFlags, " "), + "pkgPath": pkgPath, + }, + }) + + return srcJarFile +} diff --git a/python/python.go b/python/python.go index ed879eb0d..4eb496a03 100644 --- a/python/python.go +++ b/python/python.go @@ -111,7 +111,8 @@ type Module struct { android.ModuleBase android.DefaultableModuleBase - properties BaseProperties + properties BaseProperties + protoProperties android.ProtoProperties // initialize before calling Init hod android.HostOrDeviceSupported @@ -132,18 +133,15 @@ type Module struct { // pathMapping: <dest: runfile_path, src: source_path> dataPathMappings []pathMapping - // soong_zip arguments of all its dependencies. - depsParSpecs []parSpec + // the zip filepath for zipping current module source/data files. + srcsZip android.Path - // Python runfiles paths of all its dependencies. - depsPyRunfiles []string + // dependency modules' zip filepath for zipping current module source/data files. + depsSrcsZips android.Paths // (.intermediate) module output path as installation source. installSource android.OptionalPath - // the soong_zip arguments for zipping current module source/data files. - parSpec parSpec - subAndroidMkOnce map[subAndroidMkProvider]bool } @@ -156,9 +154,9 @@ func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Mo type bootstrapper interface { bootstrapperProps() []interface{} - bootstrap(ctx android.ModuleContext, Actual_version string, embedded_launcher bool, - srcsPathMappings []pathMapping, parSpec parSpec, - depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath + bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool, + srcsPathMappings []pathMapping, srcsZip android.Path, + depsSrcsZips android.Paths) android.OptionalPath } type installer interface { @@ -168,7 +166,7 @@ type installer interface { type PythonDependency interface { GetSrcsPathMappings() []pathMapping GetDataPathMappings() []pathMapping - GetParSpec() parSpec + GetSrcsZip() android.Path } func (p *Module) GetSrcsPathMappings() []pathMapping { @@ -179,8 +177,8 @@ func (p *Module) GetDataPathMappings() []pathMapping { return p.dataPathMappings } -func (p *Module) GetParSpec() parSpec { - return p.parSpec +func (p *Module) GetSrcsZip() android.Path { + return p.srcsZip } var _ PythonDependency = (*Module)(nil) @@ -189,7 +187,7 @@ var _ android.AndroidMkDataProvider = (*Module)(nil) func (p *Module) Init() android.Module { - p.AddProperties(&p.properties) + p.AddProperties(&p.properties, &p.protoProperties) if p.bootstrapper != nil { p.AddProperties(p.bootstrapper.bootstrapperProps()...) } @@ -210,6 +208,7 @@ var ( launcherTag = dependencyTag{name: "launcher"} pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`) pyExt = ".py" + protoExt = ".proto" pyVersion2 = "PY2" pyVersion3 = "PY3" initFileName = "__init__.py" @@ -261,6 +260,31 @@ func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool { return false } +func hasSrcExt(srcs []string, ext string) bool { + for _, src := range srcs { + if filepath.Ext(src) == ext { + return true + } + } + + return false +} + +func (p *Module) hasSrcExt(ctx android.BottomUpMutatorContext, ext string) bool { + if hasSrcExt(p.properties.Srcs, protoExt) { + return true + } + switch p.properties.Actual_version { + case pyVersion2: + return hasSrcExt(p.properties.Version.Py2.Srcs, protoExt) + case pyVersion3: + return hasSrcExt(p.properties.Version.Py3.Srcs, protoExt) + default: + panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", + p.properties.Actual_version, ctx.ModuleName())) + } +} + func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { // deps from "data". android.ExtractSourcesDeps(ctx, p.properties.Data) @@ -268,6 +292,9 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { android.ExtractSourcesDeps(ctx, p.properties.Srcs) android.ExtractSourcesDeps(ctx, p.properties.Exclude_srcs) + if p.hasSrcExt(ctx, protoExt) && p.Name() != "libprotobuf-python" { + ctx.AddVariationDependencies(nil, pythonLibTag, "libprotobuf-python") + } switch p.properties.Actual_version { case pyVersion2: // deps from "version.py2.srcs" property. @@ -336,16 +363,17 @@ func uniqueLibs(ctx android.BottomUpMutatorContext, func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { p.GeneratePythonBuildActions(ctx) + // Only Python binaries and test has non-empty bootstrapper. if p.bootstrapper != nil { + p.walkTransitiveDeps(ctx) // TODO(nanzhang): Since embedded launcher is not supported for Python3 for now, // so we initialize "embedded_launcher" to false. - embedded_launcher := false + embeddedLauncher := false if p.properties.Actual_version == pyVersion2 { - embedded_launcher = p.isEmbeddedLauncherEnabled(pyVersion2) + embeddedLauncher = p.isEmbeddedLauncherEnabled(pyVersion2) } p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version, - embedded_launcher, p.srcsPathMappings, p.parSpec, p.depsPyRunfiles, - p.depsParSpecs) + embeddedLauncher, p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) } if p.installer != nil && p.installSource.Valid() { @@ -378,11 +406,11 @@ func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) { expandedData := ctx.ExpandSources(p.properties.Data, nil) // sanitize pkg_path. - pkg_path := String(p.properties.Pkg_path) - if pkg_path != "" { - pkg_path = filepath.Clean(String(p.properties.Pkg_path)) - if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") || - strings.HasPrefix(pkg_path, "/") { + pkgPath := String(p.properties.Pkg_path) + if pkgPath != "" { + pkgPath = filepath.Clean(String(p.properties.Pkg_path)) + if pkgPath == ".." || strings.HasPrefix(pkgPath, "../") || + strings.HasPrefix(pkgPath, "/") { ctx.PropertyErrorf("pkg_path", "%q must be a relative path contained in par file.", String(p.properties.Pkg_path)) @@ -390,44 +418,43 @@ func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) { } if p.properties.Is_internal != nil && *p.properties.Is_internal { // pkg_path starts from "internal/" implicitly. - pkg_path = filepath.Join(internal, pkg_path) + pkgPath = filepath.Join(internal, pkgPath) } else { // pkg_path starts from "runfiles/" implicitly. - pkg_path = filepath.Join(runFiles, pkg_path) + pkgPath = filepath.Join(runFiles, pkgPath) } } else { if p.properties.Is_internal != nil && *p.properties.Is_internal { // pkg_path starts from "runfiles/" implicitly. - pkg_path = internal + pkgPath = internal } else { // pkg_path starts from "runfiles/" implicitly. - pkg_path = runFiles + pkgPath = runFiles } } - p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData) + p.genModulePathMappings(ctx, pkgPath, expandedSrcs, expandedData) - p.parSpec = p.dumpFileList(ctx, pkg_path) - - p.uniqWholeRunfilesTree(ctx) + p.srcsZip = p.createSrcsZip(ctx, pkgPath) } // generate current module unique pathMappings: <dest: runfiles_path, src: source_path> // for python/data files. -func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path string, +func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string, expandedSrcs, expandedData android.Paths) { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to - // check duplicates. + // check current module duplicates. destToPySrcs := make(map[string]string) destToPyData := make(map[string]string) for _, s := range expandedSrcs { - if s.Ext() != pyExt { - ctx.PropertyErrorf("srcs", "found non (.py) file: %q!", s.String()) + if s.Ext() != pyExt && s.Ext() != protoExt { + ctx.PropertyErrorf("srcs", "found non (.py|.proto) file: %q!", s.String()) continue } - runfilesPath := filepath.Join(pkg_path, s.Rel()) - identifiers := strings.Split(strings.TrimSuffix(runfilesPath, pyExt), "/") + runfilesPath := filepath.Join(pkgPath, s.Rel()) + identifiers := strings.Split(strings.TrimSuffix(runfilesPath, + filepath.Ext(runfilesPath)), "/") for _, token := range identifiers { if !pyIdentifierRegexp.MatchString(token) { ctx.PropertyErrorf("srcs", "the path %q contains invalid token %q.", @@ -441,56 +468,92 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path strin } for _, d := range expandedData { - if d.Ext() == pyExt { - ctx.PropertyErrorf("data", "found (.py) file: %q!", d.String()) + if d.Ext() == pyExt || d.Ext() == protoExt { + ctx.PropertyErrorf("data", "found (.py|.proto) file: %q!", d.String()) continue } - runfilesPath := filepath.Join(pkg_path, d.Rel()) + runfilesPath := filepath.Join(pkgPath, d.Rel()) if fillInMap(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) { p.dataPathMappings = append(p.dataPathMappings, pathMapping{dest: runfilesPath, src: d}) } } - } -// register build actions to dump filelist to disk. -func (p *Module) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec { +// register build actions to zip current module's sources. +func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { relativeRootMap := make(map[string]android.Paths) - // the soong_zip params in order to pack current module's Python/data files. - ret := parSpec{rootPrefix: pkg_path} - pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) + var protoSrcs android.Paths // "srcs" or "data" properties may have filegroup so it might happen that // the relative root for each source path is different. for _, path := range pathMappings { - var relativeRoot string - relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel()) - if v, found := relativeRootMap[relativeRoot]; found { - relativeRootMap[relativeRoot] = append(v, path.src) + if path.src.Ext() == protoExt { + protoSrcs = append(protoSrcs, path.src) } else { - relativeRootMap[relativeRoot] = android.Paths{path.src} + var relativeRoot string + relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel()) + if v, found := relativeRootMap[relativeRoot]; found { + relativeRootMap[relativeRoot] = append(v, path.src) + } else { + relativeRootMap[relativeRoot] = android.Paths{path.src} + } + } + } + var zips android.Paths + if len(protoSrcs) > 0 { + for _, srcFile := range protoSrcs { + zip := genProto(ctx, &p.protoProperties, srcFile, + android.ProtoFlags(ctx, &p.protoProperties), pkgPath) + zips = append(zips, zip) } } - var keys []string + if len(relativeRootMap) > 0 { + var keys []string + + // in order to keep stable order of soong_zip params, we sort the keys here. + for k := range relativeRootMap { + keys = append(keys, k) + } + sort.Strings(keys) + + parArgs := []string{} + parArgs = append(parArgs, `-P `+pkgPath) + implicits := android.Paths{} + for _, k := range keys { + parArgs = append(parArgs, `-C `+k) + for _, path := range relativeRootMap[k] { + parArgs = append(parArgs, `-f `+path.String()) + implicits = append(implicits, path) + } + } - // in order to keep stable order of soong_zip params, we sort the keys here. - for k := range relativeRootMap { - keys = append(keys, k) + origSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".py.srcszip") + ctx.Build(pctx, android.BuildParams{ + Rule: zip, + Description: "python library archive", + Output: origSrcsZip, + Implicits: implicits, + Args: map[string]string{ + "args": strings.Join(parArgs, " "), + }, + }) + zips = append(zips, origSrcsZip) } - sort.Strings(keys) - - for _, k := range keys { - // use relative root as filelist name. - fileListPath := registerBuildActionForModuleFileList( - ctx, strings.Replace(k, "/", "_", -1), relativeRootMap[k]) - ret.fileListSpecs = append(ret.fileListSpecs, - fileListSpec{fileList: fileListPath, relativeRoot: k}) + if len(zips) == 1 { + return zips[0] + } else { + combinedSrcsZip := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszip") + ctx.Build(pctx, android.BuildParams{ + Rule: combineZip, + Description: "combine python library archive", + Output: combinedSrcsZip, + Inputs: zips, + }) + return combinedSrcsZip } - - return ret } func isPythonLibModule(module blueprint.Module) bool { @@ -504,8 +567,9 @@ func isPythonLibModule(module blueprint.Module) bool { return false } -// check Python source/data files duplicates from current module and its whole dependencies. -func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) { +// check Python source/data files duplicates for whole runfiles tree since Python binary/test +// need collect and zip all srcs of whole transitive dependencies to a final par file. +func (p *Module) walkTransitiveDeps(ctx android.ModuleContext) { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // check duplicates. destToPySrcs := make(map[string]string) @@ -523,7 +587,7 @@ func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) { if ctx.OtherModuleDependencyTag(module) != pythonLibTag { return } - // Python module cannot depend on modules, except for Python library. + // Python modules only can depend on Python libraries. if !isPythonLibModule(module) { panic(fmt.Errorf( "the dependency %q of module %q is not Python library!", @@ -533,23 +597,16 @@ func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) { srcs := dep.GetSrcsPathMappings() for _, path := range srcs { if !fillInMap(ctx, destToPySrcs, - path.dest, path.src.String(), ctx.ModuleName(), - ctx.OtherModuleName(module)) { + path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(module)) { continue } - // binary needs the Python runfiles paths from all its - // dependencies to fill __init__.py in each runfiles dir. - p.depsPyRunfiles = append(p.depsPyRunfiles, path.dest) } data := dep.GetDataPathMappings() for _, path := range data { fillInMap(ctx, destToPyData, - path.dest, path.src.String(), ctx.ModuleName(), - ctx.OtherModuleName(module)) + path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(module)) } - // binary needs the soong_zip arguments from all its - // dependencies to generate executable par file. - p.depsParSpecs = append(p.depsParSpecs, dep.GetParSpec()) + p.depsSrcsZips = append(p.depsSrcsZips, dep.GetSrcsZip()) } }) } diff --git a/python/python_test.go b/python/python_test.go index ad80167e7..60a1c82a9 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -29,12 +29,11 @@ import ( ) type pyModule struct { - name string - actualVersion string - pyRunfiles []string - depsPyRunfiles []string - parSpec string - depsParSpecs []string + name string + actualVersion string + pyRunfiles []string + srcsZip string + depsSrcsZips []string } var ( @@ -49,8 +48,8 @@ var ( " First file: in module %s at path %q." + " Second file: in module %s at path %q." noSrcFileErr = moduleVariantErrTemplate + "doesn't have any source files!" - badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py) file: %q!" - badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py) file: %q!" + badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py|.proto) file: %q!" + badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py|.proto) file: %q!" bpFile = "Blueprints" data = []struct { @@ -313,14 +312,10 @@ var ( "runfiles/e/default_py3.py", "runfiles/e/file4.py", }, - depsPyRunfiles: []string{ - "runfiles/a/b/file1.py", - "runfiles/c/d/file2.py", - }, - parSpec: "-P runfiles/e -C dir/ -l @prefix@/.intermediates/dir/bin/PY3/dir_.list", - depsParSpecs: []string{ - "-P runfiles/a/b -C dir/ -l @prefix@/.intermediates/dir/lib5/PY3/dir_.list", - "-P runfiles/c/d -C dir/ -l @prefix@/.intermediates/dir/lib6/PY3/dir_.list", + srcsZip: "@prefix@/.intermediates/dir/bin/PY3/bin.py.srcszip", + depsSrcsZips: []string{ + "@prefix@/.intermediates/dir/lib5/PY3/lib5.py.srcszip", + "@prefix@/.intermediates/dir/lib6/PY3/lib6.py.srcszip", }, }, }, @@ -356,8 +351,9 @@ func TestPythonModule(t *testing.T) { testErrs = append(testErrs, expectModule(t, ctx, buildDir, e.name, e.actualVersion, - e.pyRunfiles, e.depsPyRunfiles, - e.parSpec, e.depsParSpecs)...) + e.srcsZip, + e.pyRunfiles, + e.depsSrcsZips)...) } } android.FailIfErrored(t, testErrs) @@ -388,9 +384,8 @@ func expectErrors(t *testing.T, actErrs []error, expErrs []string) (testErrs []e return } -func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, variant string, - expPyRunfiles, expDepsPyRunfiles []string, - expParSpec string, expDepsParSpecs []string) (testErrs []error) { +func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, variant, expectedSrcsZip string, + expectedPyRunfiles, expectedDepsSrcsZips []string) (testErrs []error) { module := ctx.ModuleForTests(name, variant) base, baseOk := module.Module().(*Module) @@ -398,47 +393,36 @@ func expectModule(t *testing.T, ctx *android.TestContext, buildDir, name, varian t.Fatalf("%s is not Python module!", name) } - actPyRunfiles := []string{} + actualPyRunfiles := []string{} for _, path := range base.srcsPathMappings { - actPyRunfiles = append(actPyRunfiles, path.dest) + actualPyRunfiles = append(actualPyRunfiles, path.dest) } - if !reflect.DeepEqual(actPyRunfiles, expPyRunfiles) { + if !reflect.DeepEqual(actualPyRunfiles, expectedPyRunfiles) { testErrs = append(testErrs, errors.New(fmt.Sprintf( `binary "%s" variant "%s" has unexpected pyRunfiles: %q!`, base.Name(), base.properties.Actual_version, - actPyRunfiles))) + actualPyRunfiles))) } - if !reflect.DeepEqual(base.depsPyRunfiles, expDepsPyRunfiles) { + if base.srcsZip.String() != strings.Replace(expectedSrcsZip, "@prefix@", buildDir, 1) { testErrs = append(testErrs, errors.New(fmt.Sprintf( - `binary "%s" variant "%s" has unexpected depsPyRunfiles: %q!`, + `binary "%s" variant "%s" has unexpected srcsZip: %q!`, base.Name(), base.properties.Actual_version, - base.depsPyRunfiles))) + base.srcsZip))) } - if base.parSpec.soongParArgs() != strings.Replace(expParSpec, "@prefix@", buildDir, 1) { - testErrs = append(testErrs, errors.New(fmt.Sprintf( - `binary "%s" variant "%s" has unexpected parSpec: %q!`, - base.Name(), - base.properties.Actual_version, - base.parSpec.soongParArgs()))) + for i, _ := range expectedDepsSrcsZips { + expectedDepsSrcsZips[i] = strings.Replace(expectedDepsSrcsZips[i], "@prefix@", buildDir, 1) } - - actDepsParSpecs := []string{} - for i, p := range base.depsParSpecs { - actDepsParSpecs = append(actDepsParSpecs, p.soongParArgs()) - expDepsParSpecs[i] = strings.Replace(expDepsParSpecs[i], "@prefix@", buildDir, 1) - } - - if !reflect.DeepEqual(actDepsParSpecs, expDepsParSpecs) { + if !reflect.DeepEqual(base.depsSrcsZips.Strings(), expectedDepsSrcsZips) { testErrs = append(testErrs, errors.New(fmt.Sprintf( - `binary "%s" variant "%s" has unexpected depsParSpecs: %q!`, + `binary "%s" variant "%s" has unexpected depsSrcsZips: %q!`, base.Name(), base.properties.Actual_version, - actDepsParSpecs))) + base.depsSrcsZips))) } return diff --git a/scripts/strip.sh b/scripts/strip.sh index 848fe8d42..ff8aaa03e 100755 --- a/scripts/strip.sh +++ b/scripts/strip.sh @@ -18,6 +18,7 @@ # Inputs: # Environment: # CROSS_COMPILE: prefix added to readelf, objcopy tools +# XZ: path to the xz binary # Arguments: # -i ${file}: input file (required) # -o ${file}: output file (required) @@ -58,7 +59,7 @@ do_strip_keep_mini_debug_info() { "${CROSS_COMPILE}objcopy" --rename-section .debug_frame=saved_debug_frame "${outfile}.debug" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" -S --remove-section .gdb_index --remove-section .comment --keep-symbols="${outfile}.keep_symbols" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" --rename-section saved_debug_frame=.debug_frame "${outfile}.mini_debuginfo" - xz "${outfile}.mini_debuginfo" + "${XZ}" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" else cp -f "${infile}" "${outfile}.tmp" diff --git a/ui/build/config.go b/ui/build/config.go index 27ed8e9fe..a1b5bd4a9 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -115,6 +115,8 @@ func NewConfig(ctx Context, args ...string) Config { // Tell python not to spam the source tree with .pyc files. ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") + ret.environ.Set("TMPDIR", absPath(ctx, ret.TempDir())) + // Precondition: the current directory is the top of the source tree if _, err := os.Stat(srcDirFileCheck); err != nil { if os.IsNotExist(err) { diff --git a/ui/build/kati.go b/ui/build/kati.go index e4715bb9a..dde5ecce0 100644 --- a/ui/build/kati.go +++ b/ui/build/kati.go @@ -169,6 +169,11 @@ func katiRewriteOutput(ctx Context, pipe io.ReadCloser) { if !haveBlankLine { fmt.Fprintln(ctx.Stdout()) } + + if err := scanner.Err(); err != nil { + ctx.Println("Error from kati parser:", err) + io.Copy(ctx.Stderr(), pipe) + } } func runKatiCleanSpec(ctx Context, config Config) { |