diff options
73 files changed, 3030 insertions, 1394 deletions
diff --git a/android/Android.bp b/android/Android.bp index 4bd272da1..f8c1d5567 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -23,7 +23,8 @@ bootstrap_go_package { "csuite_config.go", "defaults.go", "defs.go", - "depset.go", + "depset_generic.go", + "depset_paths.go", "deptag.go", "expand.go", "filegroup.go", diff --git a/android/androidmk.go b/android/androidmk.go index 42c5d0083..b32048aab 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -141,7 +141,7 @@ type AndroidMkEntries struct { } type AndroidMkExtraEntriesFunc func(entries *AndroidMkEntries) -type AndroidMkExtraFootersFunc func(w io.Writer, name, prefix, moduleDir string, entries *AndroidMkEntries) +type AndroidMkExtraFootersFunc func(w io.Writer, name, prefix, moduleDir string) // Utility funcs to manipulate Android.mk variable entries. @@ -554,7 +554,7 @@ func (a *AndroidMkEntries) fillInEntries(config Config, bpPath string, mod bluep fmt.Fprintln(&a.footer, "include "+a.Include) blueprintDir := filepath.Dir(bpPath) for _, footerFunc := range a.ExtraFooters { - footerFunc(&a.footer, name, prefix, blueprintDir, a) + footerFunc(&a.footer, name, prefix, blueprintDir) } } diff --git a/android/config.go b/android/config.go index 9882d5508..453e074c6 100644 --- a/android/config.go +++ b/android/config.go @@ -1272,6 +1272,10 @@ func (c *config) FlattenApex() bool { return Bool(c.productVariables.Flatten_apex) } +func (c *config) CompressedApex() bool { + return Bool(c.productVariables.CompressedApex) +} + func (c *config) EnforceSystemCertificate() bool { return Bool(c.productVariables.EnforceSystemCertificate) } diff --git a/android/depset_generic.go b/android/depset_generic.go new file mode 100644 index 000000000..f00e462c2 --- /dev/null +++ b/android/depset_generic.go @@ -0,0 +1,351 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "reflect" +) + +// depSet is designed to be conceptually compatible with Bazel's depsets: +// https://docs.bazel.build/versions/master/skylark/depsets.html + +type DepSetOrder int + +const ( + PREORDER DepSetOrder = iota + POSTORDER + TOPOLOGICAL +) + +func (o DepSetOrder) String() string { + switch o { + case PREORDER: + return "PREORDER" + case POSTORDER: + return "POSTORDER" + case TOPOLOGICAL: + return "TOPOLOGICAL" + default: + panic(fmt.Errorf("Invalid DepSetOrder %d", o)) + } +} + +// A depSet efficiently stores a slice of an arbitrary type from transitive dependencies without +// copying. It is stored as a DAG of depSet nodes, each of which has some direct contents and a list +// of dependency depSet nodes. +// +// A depSet has an order that will be used to walk the DAG when ToList() is called. The order +// can be POSTORDER, PREORDER, or TOPOLOGICAL. POSTORDER and PREORDER orders return a postordered +// or preordered left to right flattened list. TOPOLOGICAL returns a list that guarantees that +// elements of children are listed after all of their parents (unless there are duplicate direct +// elements in the depSet or any of its transitive dependencies, in which case the ordering of the +// duplicated element is not guaranteed). +// +// A depSet is created by newDepSet or newDepSetBuilder.Build from the slice for direct contents +// and the *depSets of dependencies. A depSet is immutable once created. +// +// This object uses reflection to remain agnostic to the type it contains. It should be replaced +// with generics once those exist in Go. Callers should generally use a thin wrapper around depSet +// that provides type-safe methods like DepSet for Paths. +type depSet struct { + preorder bool + reverse bool + order DepSetOrder + direct interface{} + transitive []*depSet +} + +type depSetInterface interface { + embeddedDepSet() *depSet +} + +func (d *depSet) embeddedDepSet() *depSet { + return d +} + +var _ depSetInterface = (*depSet)(nil) + +// newDepSet returns an immutable depSet with the given order, direct and transitive contents. +// direct must be a slice, but is not type-safe due to the lack of generics in Go. It can be a +// nil slice, but not a nil interface{}, i.e. []string(nil) but not nil. +func newDepSet(order DepSetOrder, direct interface{}, transitive interface{}) *depSet { + var directCopy interface{} + transitiveDepSet := sliceToDepSets(transitive, order) + + if order == TOPOLOGICAL { + directCopy = reverseSlice(direct) + reverseSliceInPlace(transitiveDepSet) + } else { + directCopy = copySlice(direct) + } + + return &depSet{ + preorder: order == PREORDER, + reverse: order == TOPOLOGICAL, + order: order, + direct: directCopy, + transitive: transitiveDepSet, + } +} + +// depSetBuilder is used to create an immutable depSet. +type depSetBuilder struct { + order DepSetOrder + direct reflect.Value + transitive []*depSet +} + +// newDepSetBuilder returns a depSetBuilder to create an immutable depSet with the given order and +// type, represented by a slice of type that will be in the depSet. +func newDepSetBuilder(order DepSetOrder, typ interface{}) *depSetBuilder { + empty := reflect.Zero(reflect.TypeOf(typ)) + return &depSetBuilder{ + order: order, + direct: empty, + } +} + +// sliceToDepSets converts a slice of any type that implements depSetInterface (by having a depSet +// embedded in it) into a []*depSet. +func sliceToDepSets(in interface{}, order DepSetOrder) []*depSet { + slice := reflect.ValueOf(in) + length := slice.Len() + out := make([]*depSet, length) + for i := 0; i < length; i++ { + vi := slice.Index(i) + depSetIntf, ok := vi.Interface().(depSetInterface) + if !ok { + panic(fmt.Errorf("element %d is a %s, not a depSetInterface", i, vi.Type())) + } + depSet := depSetIntf.embeddedDepSet() + if depSet.order != order { + panic(fmt.Errorf("incompatible order, new depSet is %s but transitive depSet is %s", + order, depSet.order)) + } + out[i] = depSet + } + return out +} + +// DirectSlice adds direct contents to the depSet being built by a depSetBuilder. Newly added direct +// contents are to the right of any existing direct contents. The argument must be a slice, but +// is not type-safe due to the lack of generics in Go. +func (b *depSetBuilder) DirectSlice(direct interface{}) *depSetBuilder { + b.direct = reflect.AppendSlice(b.direct, reflect.ValueOf(direct)) + return b +} + +// Direct adds direct contents to the depSet being built by a depSetBuilder. Newly added direct +// contents are to the right of any existing direct contents. The argument must be the same type +// as the element of the slice passed to newDepSetBuilder, but is not type-safe due to the lack of +// generics in Go. +func (b *depSetBuilder) Direct(direct interface{}) *depSetBuilder { + b.direct = reflect.Append(b.direct, reflect.ValueOf(direct)) + return b +} + +// Transitive adds transitive contents to the DepSet being built by a DepSetBuilder. Newly added +// transitive contents are to the right of any existing transitive contents. The argument can +// be any slice of type that has depSet embedded in it. +func (b *depSetBuilder) Transitive(transitive interface{}) *depSetBuilder { + depSets := sliceToDepSets(transitive, b.order) + b.transitive = append(b.transitive, depSets...) + return b +} + +// Returns the depSet being built by this depSetBuilder. The depSetBuilder retains its contents +// for creating more depSets. +func (b *depSetBuilder) Build() *depSet { + return newDepSet(b.order, b.direct.Interface(), b.transitive) +} + +// walk calls the visit method in depth-first order on a DepSet, preordered if d.preorder is set, +// otherwise postordered. +func (d *depSet) walk(visit func(interface{})) { + visited := make(map[*depSet]bool) + + var dfs func(d *depSet) + dfs = func(d *depSet) { + visited[d] = true + if d.preorder { + visit(d.direct) + } + for _, dep := range d.transitive { + if !visited[dep] { + dfs(dep) + } + } + + if !d.preorder { + visit(d.direct) + } + } + + dfs(d) +} + +// ToList returns the depSet flattened to a list. The order in the list is based on the order +// of the depSet. POSTORDER and PREORDER orders return a postordered or preordered left to right +// flattened list. TOPOLOGICAL returns a list that guarantees that elements of children are listed +// after all of their parents (unless there are duplicate direct elements in the DepSet or any of +// its transitive dependencies, in which case the ordering of the duplicated element is not +// guaranteed). +// +// This method uses a reflection-based implementation to find the unique elements in slice, which +// is around 3x slower than a concrete implementation. Type-safe wrappers around depSet can +// provide their own implementation of ToList that calls depSet.toList with a method that +// uses a concrete implementation. +func (d *depSet) ToList() interface{} { + return d.toList(firstUnique) +} + +// toList returns the depSet flattened to a list. The order in the list is based on the order +// of the depSet. POSTORDER and PREORDER orders return a postordered or preordered left to right +// flattened list. TOPOLOGICAL returns a list that guarantees that elements of children are listed +// after all of their parents (unless there are duplicate direct elements in the DepSet or any of +// its transitive dependencies, in which case the ordering of the duplicated element is not +// guaranteed). The firstUniqueFunc is used to remove duplicates from the list. +func (d *depSet) toList(firstUniqueFunc func(interface{}) interface{}) interface{} { + if d == nil { + return nil + } + slice := reflect.Zero(reflect.TypeOf(d.direct)) + d.walk(func(paths interface{}) { + slice = reflect.AppendSlice(slice, reflect.ValueOf(paths)) + }) + list := slice.Interface() + list = firstUniqueFunc(list) + if d.reverse { + reverseSliceInPlace(list) + } + return list +} + +// firstUnique returns all unique elements of a slice, keeping the first copy of each. It +// modifies the slice contents in place, and returns a subslice of the original slice. The +// argument must be a slice, but is not type-safe due to the lack of reflection in Go. +// +// Performance of the reflection-based firstUnique is up to 3x slower than a concrete type +// version such as FirstUniqueStrings. +func firstUnique(slice interface{}) interface{} { + // 4 was chosen based on Benchmark_firstUnique results. + if reflect.ValueOf(slice).Len() > 4 { + return firstUniqueMap(slice) + } + return firstUniqueList(slice) +} + +// firstUniqueList is an implementation of firstUnique using an O(N^2) list comparison to look for +// duplicates. +func firstUniqueList(in interface{}) interface{} { + writeIndex := 0 + slice := reflect.ValueOf(in) + length := slice.Len() +outer: + for readIndex := 0; readIndex < length; readIndex++ { + readValue := slice.Index(readIndex) + for compareIndex := 0; compareIndex < writeIndex; compareIndex++ { + compareValue := slice.Index(compareIndex) + // These two Interface() calls seem to cause an allocation and significantly + // slow down this list-based implementation. The map implementation below doesn't + // have this issue because reflect.Value.MapIndex takes a Value and appears to be + // able to do the map lookup without an allocation. + if readValue.Interface() == compareValue.Interface() { + // The value at readIndex already exists somewhere in the output region + // of the slice before writeIndex, skip it. + continue outer + } + } + if readIndex != writeIndex { + writeValue := slice.Index(writeIndex) + writeValue.Set(readValue) + } + writeIndex++ + } + return slice.Slice(0, writeIndex).Interface() +} + +var trueValue = reflect.ValueOf(true) + +// firstUniqueList is an implementation of firstUnique using an O(N) hash set lookup to look for +// duplicates. +func firstUniqueMap(in interface{}) interface{} { + writeIndex := 0 + slice := reflect.ValueOf(in) + length := slice.Len() + seen := reflect.MakeMapWithSize(reflect.MapOf(slice.Type().Elem(), trueValue.Type()), slice.Len()) + for readIndex := 0; readIndex < length; readIndex++ { + readValue := slice.Index(readIndex) + if seen.MapIndex(readValue).IsValid() { + continue + } + seen.SetMapIndex(readValue, trueValue) + if readIndex != writeIndex { + writeValue := slice.Index(writeIndex) + writeValue.Set(readValue) + } + writeIndex++ + } + return slice.Slice(0, writeIndex).Interface() +} + +// reverseSliceInPlace reverses the elements of a slice in place. The argument must be a slice, but +// is not type-safe due to the lack of reflection in Go. +func reverseSliceInPlace(in interface{}) { + swapper := reflect.Swapper(in) + slice := reflect.ValueOf(in) + length := slice.Len() + for i, j := 0, length-1; i < j; i, j = i+1, j-1 { + swapper(i, j) + } +} + +// reverseSlice returns a copy of a slice in reverse order. The argument must be a slice, but is +// not type-safe due to the lack of reflection in Go. +func reverseSlice(in interface{}) interface{} { + slice := reflect.ValueOf(in) + if !slice.IsValid() || slice.IsNil() { + return in + } + if slice.Kind() != reflect.Slice { + panic(fmt.Errorf("%t is not a slice", in)) + } + length := slice.Len() + if length == 0 { + return in + } + out := reflect.MakeSlice(slice.Type(), length, length) + for i := 0; i < length; i++ { + out.Index(i).Set(slice.Index(length - 1 - i)) + } + return out.Interface() +} + +// copySlice returns a copy of a slice. The argument must be a slice, but is not type-safe due to +// the lack of reflection in Go. +func copySlice(in interface{}) interface{} { + slice := reflect.ValueOf(in) + if !slice.IsValid() || slice.IsNil() { + return in + } + length := slice.Len() + if length == 0 { + return in + } + out := reflect.MakeSlice(slice.Type(), length, length) + reflect.Copy(out, slice) + return out.Interface() +} diff --git a/android/depset.go b/android/depset_paths.go index 60ebcacc8..ed561ba24 100644 --- a/android/depset.go +++ b/android/depset_paths.go @@ -14,10 +14,7 @@ package android -import "fmt" - -// DepSet is designed to be conceptually compatible with Bazel's depsets: -// https://docs.bazel.build/versions/master/skylark/depsets.html +// This file implements DepSet, a thin type-safe wrapper around depSet that contains Paths. // A DepSet efficiently stores Paths from transitive dependencies without copying. It is stored // as a DAG of DepSet nodes, each of which has some direct contents and a list of dependency @@ -33,123 +30,42 @@ import "fmt" // A DepSet is created by NewDepSet or NewDepSetBuilder.Build from the Paths for direct contents // and the *DepSets of dependencies. A DepSet is immutable once created. type DepSet struct { - preorder bool - reverse bool - order DepSetOrder - direct Paths - transitive []*DepSet + depSet } // DepSetBuilder is used to create an immutable DepSet. type DepSetBuilder struct { - order DepSetOrder - direct Paths - transitive []*DepSet -} - -type DepSetOrder int - -const ( - PREORDER DepSetOrder = iota - POSTORDER - TOPOLOGICAL -) - -func (o DepSetOrder) String() string { - switch o { - case PREORDER: - return "PREORDER" - case POSTORDER: - return "POSTORDER" - case TOPOLOGICAL: - return "TOPOLOGICAL" - default: - panic(fmt.Errorf("Invalid DepSetOrder %d", o)) - } + depSetBuilder } // NewDepSet returns an immutable DepSet with the given order, direct and transitive contents. func NewDepSet(order DepSetOrder, direct Paths, transitive []*DepSet) *DepSet { - var directCopy Paths - transitiveCopy := make([]*DepSet, 0, len(transitive)) - - for _, dep := range transitive { - if dep != nil { - if dep.order != order { - panic(fmt.Errorf("incompatible order, new DepSet is %s but transitive DepSet is %s", - order, dep.order)) - } - transitiveCopy = append(transitiveCopy, dep) - } - } - - if order == TOPOLOGICAL { - directCopy = ReversePaths(direct) - reverseDepSetsInPlace(transitiveCopy) - } else { - // Use copy instead of append(nil, ...) to make a slice that is exactly the size of the input - // slice. The DepSet is immutable, there is no need for additional capacity. - directCopy = make(Paths, len(direct)) - copy(directCopy, direct) - } - - return &DepSet{ - preorder: order == PREORDER, - reverse: order == TOPOLOGICAL, - order: order, - direct: directCopy, - transitive: transitiveCopy, - } + return &DepSet{*newDepSet(order, direct, transitive)} } // NewDepSetBuilder returns a DepSetBuilder to create an immutable DepSet with the given order. func NewDepSetBuilder(order DepSetOrder) *DepSetBuilder { - return &DepSetBuilder{order: order} + return &DepSetBuilder{*newDepSetBuilder(order, Paths(nil))} } // Direct adds direct contents to the DepSet being built by a DepSetBuilder. Newly added direct // contents are to the right of any existing direct contents. func (b *DepSetBuilder) Direct(direct ...Path) *DepSetBuilder { - b.direct = append(b.direct, direct...) + b.depSetBuilder.DirectSlice(direct) return b } // Transitive adds transitive contents to the DepSet being built by a DepSetBuilder. Newly added // transitive contents are to the right of any existing transitive contents. func (b *DepSetBuilder) Transitive(transitive ...*DepSet) *DepSetBuilder { - b.transitive = append(b.transitive, transitive...) + b.depSetBuilder.Transitive(transitive) return b } // Returns the DepSet being built by this DepSetBuilder. The DepSetBuilder retains its contents // for creating more DepSets. func (b *DepSetBuilder) Build() *DepSet { - return NewDepSet(b.order, b.direct, b.transitive) -} - -// walk calls the visit method in depth-first order on a DepSet, preordered if d.preorder is set, -// otherwise postordered. -func (d *DepSet) walk(visit func(Paths)) { - visited := make(map[*DepSet]bool) - - var dfs func(d *DepSet) - dfs = func(d *DepSet) { - visited[d] = true - if d.preorder { - visit(d.direct) - } - for _, dep := range d.transitive { - if !visited[dep] { - dfs(dep) - } - } - - if !d.preorder { - visit(d.direct) - } - } - - dfs(d) + return &DepSet{*b.depSetBuilder.Build()} } // ToList returns the DepSet flattened to a list. The order in the list is based on the order @@ -162,33 +78,17 @@ func (d *DepSet) ToList() Paths { if d == nil { return nil } - var list Paths - d.walk(func(paths Paths) { - list = append(list, paths...) - }) - list = FirstUniquePaths(list) - if d.reverse { - reversePathsInPlace(list) - } - return list + return d.toList(func(paths interface{}) interface{} { + return FirstUniquePaths(paths.(Paths)) + }).(Paths) } // ToSortedList returns the direct and transitive contents of a DepSet in lexically sorted order // with duplicates removed. func (d *DepSet) ToSortedList() Paths { - list := d.ToList() - return SortedUniquePaths(list) -} - -func reversePathsInPlace(list Paths) { - for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 { - list[i], list[j] = list[j], list[i] - } -} - -func reverseDepSetsInPlace(list []*DepSet) { - for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 { - list[i], list[j] = list[j], list[i] + if d == nil { + return nil } - + paths := d.ToList() + return SortedUniquePaths(paths) } diff --git a/android/depset_test.go b/android/depset_test.go index c328127b1..955ccb001 100644 --- a/android/depset_test.go +++ b/android/depset_test.go @@ -17,6 +17,7 @@ package android import ( "fmt" "reflect" + "strconv" "strings" "testing" ) @@ -108,6 +109,7 @@ func TestDepSet(t *testing.T) { name: "builderReuse", depSet: func(t *testing.T, order DepSetOrder) *DepSet { assertEquals := func(t *testing.T, w, g Paths) { + t.Helper() if !reflect.DeepEqual(w, g) { t.Errorf("want %q, got %q", w, g) } @@ -302,3 +304,87 @@ func TestDepSetInvalidOrder(t *testing.T) { }) } } + +func Test_firstUnique(t *testing.T) { + f := func(t *testing.T, imp func([]string) []string, in, want []string) { + t.Helper() + out := imp(in) + if !reflect.DeepEqual(out, want) { + t.Errorf("incorrect output:") + t.Errorf(" input: %#v", in) + t.Errorf(" expected: %#v", want) + t.Errorf(" got: %#v", out) + } + } + + for _, testCase := range firstUniqueStringsTestCases { + t.Run("list", func(t *testing.T) { + f(t, func(s []string) []string { + return firstUniqueList(s).([]string) + }, testCase.in, testCase.out) + }) + t.Run("map", func(t *testing.T) { + f(t, func(s []string) []string { + return firstUniqueMap(s).([]string) + }, testCase.in, testCase.out) + }) + } +} + +func Benchmark_firstUnique(b *testing.B) { + implementations := []struct { + name string + f func([]string) []string + }{ + { + name: "list", + f: func(slice []string) []string { + return firstUniqueList(slice).([]string) + }, + }, + { + name: "map", + f: func(slice []string) []string { + return firstUniqueMap(slice).([]string) + }, + }, + { + name: "optimal", + f: func(slice []string) []string { + return firstUnique(slice).([]string) + }, + }, + } + const maxSize = 1024 + uniqueStrings := make([]string, maxSize) + for i := range uniqueStrings { + uniqueStrings[i] = strconv.Itoa(i) + } + sameString := make([]string, maxSize) + for i := range sameString { + sameString[i] = uniqueStrings[0] + } + + f := func(b *testing.B, imp func([]string) []string, s []string) { + for i := 0; i < b.N; i++ { + b.ReportAllocs() + s = append([]string(nil), s...) + imp(s) + } + } + + for n := 1; n <= maxSize; n <<= 1 { + b.Run(strconv.Itoa(n), func(b *testing.B) { + for _, implementation := range implementations { + b.Run(implementation.name, func(b *testing.B) { + b.Run("same", func(b *testing.B) { + f(b, implementation.f, sameString[:n]) + }) + b.Run("unique", func(b *testing.B) { + f(b, implementation.f, uniqueStrings[:n]) + }) + }) + } + }) + } +} diff --git a/android/makefile_goal.go b/android/makefile_goal.go index b5d9d691e..07354a648 100644 --- a/android/makefile_goal.go +++ b/android/makefile_goal.go @@ -80,7 +80,7 @@ func (p *makefileGoal) AndroidMkEntries() []AndroidMkEntries { Class: "ETC", OutputFile: OptionalPathForPath(p.outputFilePath), ExtraFooters: []AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { // Can't use Cp because inputPath() is not a valid Path. fmt.Fprintf(w, "$(eval $(call copy-one-file,%s,%s))\n", proptools.String(p.inputPath()), p.outputFilePath) }, diff --git a/android/module.go b/android/module.go index b6729c0b7..bbdeb2733 100644 --- a/android/module.go +++ b/android/module.go @@ -441,6 +441,10 @@ type Module interface { FilesToInstall() InstallPaths PackagingSpecs() []PackagingSpec + + // TransitivePackagingSpecs returns the PackagingSpecs for this module and any transitive + // dependencies with dependency tags for which IsInstallDepNeeded() returns true. + TransitivePackagingSpecs() []PackagingSpec } // Qualified id for a module @@ -1003,12 +1007,14 @@ type ModuleBase struct { // The primary visibility property, may be nil, that controls access to the module. primaryVisibilityProperty visibilityProperty - noAddressSanitizer bool - installFiles InstallPaths - checkbuildFiles Paths - packagingSpecs []PackagingSpec - noticeFiles Paths - phonies map[string]Paths + noAddressSanitizer bool + installFiles InstallPaths + installFilesDepSet *installPathsDepSet + checkbuildFiles Paths + packagingSpecs []PackagingSpec + packagingSpecsDepSet *packagingSpecsDepSet + noticeFiles Paths + phonies map[string]Paths // The files to copy to the dist as explicitly specified in the .bp file. distFiles TaggedDistFiles @@ -1338,19 +1344,17 @@ func (m *ModuleBase) ExportedToMake() bool { // computeInstallDeps finds the installed paths of all dependencies that have a dependency // tag that is annotated as needing installation via the IsInstallDepNeeded method. -func (m *ModuleBase) computeInstallDeps(ctx blueprint.ModuleContext) InstallPaths { - var result InstallPaths - ctx.WalkDeps(func(child, parent blueprint.Module) bool { - if a, ok := child.(Module); ok { - if IsInstallDepNeeded(ctx.OtherModuleDependencyTag(child)) { - result = append(result, a.FilesToInstall()...) - return true - } +func (m *ModuleBase) computeInstallDeps(ctx ModuleContext) ([]*installPathsDepSet, []*packagingSpecsDepSet) { + var installDeps []*installPathsDepSet + var packagingSpecs []*packagingSpecsDepSet + ctx.VisitDirectDeps(func(dep Module) { + if IsInstallDepNeeded(ctx.OtherModuleDependencyTag(dep)) { + installDeps = append(installDeps, dep.base().installFilesDepSet) + packagingSpecs = append(packagingSpecs, dep.base().packagingSpecsDepSet) } - return false }) - return result + return installDeps, packagingSpecs } func (m *ModuleBase) FilesToInstall() InstallPaths { @@ -1361,6 +1365,10 @@ func (m *ModuleBase) PackagingSpecs() []PackagingSpec { return m.packagingSpecs } +func (m *ModuleBase) TransitivePackagingSpecs() []PackagingSpec { + return m.packagingSpecsDepSet.ToList() +} + func (m *ModuleBase) NoAddressSanitizer() bool { return m.noAddressSanitizer } @@ -1587,11 +1595,15 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) module: m.module, bp: blueprintCtx, baseModuleContext: m.baseModuleContextFactory(blueprintCtx), - installDeps: m.computeInstallDeps(blueprintCtx), - installFiles: m.installFiles, variables: make(map[string]string), } + dependencyInstallFiles, dependencyPackagingSpecs := m.computeInstallDeps(ctx) + // set m.installFilesDepSet to only the transitive dependencies to be used as the dependencies + // of installed files of this module. It will be replaced by a depset including the installed + // files of this module at the end for use by modules that depend on this one. + m.installFilesDepSet = newInstallPathsDepSet(nil, dependencyInstallFiles) + // Temporarily continue to call blueprintCtx.GetMissingDependencies() to maintain the previous behavior of never // reporting missing dependency errors in Blueprint when AllowMissingDependencies == true. // TODO: This will be removed once defaults modules handle missing dependency errors @@ -1700,6 +1712,9 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) } } + m.installFilesDepSet = newInstallPathsDepSet(m.installFiles, dependencyInstallFiles) + m.packagingSpecsDepSet = newPackagingSpecsDepSet(m.packagingSpecs, dependencyPackagingSpecs) + m.buildParams = ctx.buildParams m.ruleParams = ctx.ruleParams m.variables = ctx.variables @@ -1874,7 +1889,6 @@ type moduleContext struct { bp blueprint.ModuleContext baseModuleContext packagingSpecs []PackagingSpec - installDeps InstallPaths installFiles InstallPaths checkbuildFiles Paths module Module @@ -2420,8 +2434,7 @@ func (m *moduleContext) installFile(installPath InstallPath, name string, srcPat m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, false) if !m.skipInstall(fullInstallPath) { - - deps = append(deps, m.installDeps.Paths()...) + deps = append(deps, m.module.base().installFilesDepSet.ToList().Paths()...) var implicitDeps, orderOnlyDeps Paths @@ -2858,3 +2871,23 @@ func CheckBlueprintSyntax(ctx BaseModuleContext, filename string, contents strin bpctx := ctx.blueprintBaseModuleContext() return blueprint.CheckBlueprintSyntax(bpctx.ModuleFactories(), filename, contents) } + +// installPathsDepSet is a thin type-safe wrapper around the generic depSet. It always uses +// topological order. +type installPathsDepSet struct { + depSet +} + +// newInstallPathsDepSet returns an immutable packagingSpecsDepSet with the given direct and +// transitive contents. +func newInstallPathsDepSet(direct InstallPaths, transitive []*installPathsDepSet) *installPathsDepSet { + return &installPathsDepSet{*newDepSet(TOPOLOGICAL, direct, transitive)} +} + +// ToList returns the installPathsDepSet flattened to a list in topological order. +func (d *installPathsDepSet) ToList() InstallPaths { + if d == nil { + return nil + } + return d.depSet.ToList().(InstallPaths) +} diff --git a/android/packaging.go b/android/packaging.go index 09432e65d..da745ff19 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -203,3 +203,23 @@ func (p *PackagingBase) CopyDepsToZip(ctx ModuleContext, zipOut OutputPath) (ent builder.Build("zip_deps", fmt.Sprintf("Zipping deps for %s", ctx.ModuleName())) return entries } + +// packagingSpecsDepSet is a thin type-safe wrapper around the generic depSet. It always uses +// topological order. +type packagingSpecsDepSet struct { + depSet +} + +// newPackagingSpecsDepSet returns an immutable packagingSpecsDepSet with the given direct and +// transitive contents. +func newPackagingSpecsDepSet(direct []PackagingSpec, transitive []*packagingSpecsDepSet) *packagingSpecsDepSet { + return &packagingSpecsDepSet{*newDepSet(TOPOLOGICAL, direct, transitive)} +} + +// ToList returns the packagingSpecsDepSet flattened to a list in topological order. +func (d *packagingSpecsDepSet) ToList() []PackagingSpec { + if d == nil { + return nil + } + return d.depSet.ToList().([]PackagingSpec) +} diff --git a/android/paths.go b/android/paths.go index 5a41cf16f..0238a3fcf 100644 --- a/android/paths.go +++ b/android/paths.go @@ -542,6 +542,45 @@ func firstUniquePathsMap(list Paths) Paths { return list[:k] } +// FirstUniqueInstallPaths returns all unique elements of an InstallPaths, keeping the first copy of each. It +// modifies the InstallPaths slice contents in place, and returns a subslice of the original slice. +func FirstUniqueInstallPaths(list InstallPaths) InstallPaths { + // 128 was chosen based on BenchmarkFirstUniquePaths results. + if len(list) > 128 { + return firstUniqueInstallPathsMap(list) + } + return firstUniqueInstallPathsList(list) +} + +func firstUniqueInstallPathsList(list InstallPaths) InstallPaths { + k := 0 +outer: + for i := 0; i < len(list); i++ { + for j := 0; j < k; j++ { + if list[i] == list[j] { + continue outer + } + } + list[k] = list[i] + k++ + } + return list[:k] +} + +func firstUniqueInstallPathsMap(list InstallPaths) InstallPaths { + k := 0 + seen := make(map[InstallPath]bool, len(list)) + for i := 0; i < len(list); i++ { + if seen[list[i]] { + continue + } + seen[list[i]] = true + list[k] = list[i] + k++ + } + return list[:k] +} + // LastUniquePaths returns all unique elements of a Paths, keeping the last copy of each. It // modifies the Paths slice contents in place, and returns a subslice of the original slice. func LastUniquePaths(list Paths) Paths { diff --git a/android/prebuilt.go b/android/prebuilt.go index 294a6e080..bb98ed438 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -178,6 +178,9 @@ func InitSingleSourcePrebuiltModule(module PrebuiltInterface, srcProps interface srcPropertyName := proptools.PropertyNameForField(srcField) srcsSupplier := func(ctx BaseModuleContext) []string { + if !module.Enabled() { + return nil + } value := srcPropsValue.FieldByIndex(srcFieldIndex) if value.Kind() == reflect.Ptr { value = value.Elem() diff --git a/android/util_test.go b/android/util_test.go index 25b52ca03..fa26c77ac 100644 --- a/android/util_test.go +++ b/android/util_test.go @@ -593,6 +593,10 @@ func BenchmarkFirstUniqueStrings(b *testing.B) { name: "map", f: firstUniqueStringsMap, }, + { + name: "optimal", + f: FirstUniqueStrings, + }, } const maxSize = 1024 uniqueStrings := make([]string, maxSize) diff --git a/android/variable.go b/android/variable.go index 0df5272c0..aed145c96 100644 --- a/android/variable.go +++ b/android/variable.go @@ -319,8 +319,9 @@ type productVariables struct { Ndk_abis *bool `json:",omitempty"` Exclude_draft_ndk_apis *bool `json:",omitempty"` - Flatten_apex *bool `json:",omitempty"` - Aml_abis *bool `json:",omitempty"` + Flatten_apex *bool `json:",omitempty"` + CompressedApex *bool `json:",omitempty"` + Aml_abis *bool `json:",omitempty"` DexpreoptGlobalConfig *string `json:",omitempty"` diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go index 4b782a23c..3c4815ea5 100644 --- a/androidmk/parser/make_strings.go +++ b/androidmk/parser/make_strings.go @@ -15,8 +15,10 @@ package parser import ( + "fmt" "strings" "unicode" + "unicode/utf8" ) // A MakeString is a string that may contain variable substitutions in it. @@ -130,8 +132,85 @@ func (ms *MakeString) SplitN(sep string, n int) []*MakeString { }) } +// Words splits MakeString into multiple makeStrings separated by whitespace. +// Thus, " a $(X)b c " will be split into ["a", "$(X)b", "c"]. +// Splitting a MakeString consisting solely of whitespace yields empty array. func (ms *MakeString) Words() []*MakeString { - return ms.splitNFunc(-1, splitWords) + var ch rune // current character + const EOF = -1 // no more characters + const EOS = -2 // at the end of a string chunk + + // Next character's chunk and position + iString := 0 + iChar := 0 + + var words []*MakeString + word := SimpleMakeString("", ms.Pos()) + + nextChar := func() { + if iString >= len(ms.Strings) { + ch = EOF + } else if iChar >= len(ms.Strings[iString]) { + iString++ + iChar = 0 + ch = EOS + } else { + var w int + ch, w = utf8.DecodeRuneInString(ms.Strings[iString][iChar:]) + iChar += w + } + } + + appendVariableAndAdvance := func() { + if iString-1 < len(ms.Variables) { + word.appendVariable(ms.Variables[iString-1]) + } + nextChar() + } + + appendCharAndAdvance := func(c rune) { + if c != EOF { + word.appendString(string(c)) + } + nextChar() + } + + nextChar() + for ch != EOF { + // Skip whitespace + for ch == ' ' || ch == '\t' { + nextChar() + } + if ch == EOS { + // "... $(X)... " case. The current word should be empty. + if !word.Empty() { + panic(fmt.Errorf("%q: EOS while current word %q is not empty, iString=%d", + ms.Dump(), word.Dump(), iString)) + } + appendVariableAndAdvance() + } + // Copy word + for ch != EOF { + if ch == ' ' || ch == '\t' { + words = append(words, word) + word = SimpleMakeString("", ms.Pos()) + break + } + if ch == EOS { + // "...a$(X)..." case. Append variable to the current word + appendVariableAndAdvance() + } else { + if ch == '\\' { + appendCharAndAdvance('\\') + } + appendCharAndAdvance(ch) + } + } + } + if !word.Empty() { + words = append(words, word) + } + return words } func (ms *MakeString) splitNFunc(n int, splitFunc func(s string, n int) []string) []*MakeString { @@ -166,9 +245,7 @@ func (ms *MakeString) splitNFunc(n int, splitFunc func(s string, n int) []string } } - if !curMs.Empty() { - ret = append(ret, curMs) - } + ret = append(ret, curMs) return ret } @@ -219,44 +296,6 @@ func splitAnyN(s, sep string, n int) []string { return ret } -func splitWords(s string, n int) []string { - ret := []string{} - preserve := "" - for n == -1 || n > 1 { - index := strings.IndexAny(s, " \t") - if index == 0 && len(preserve) == 0 { - s = s[1:] - } else if index >= 0 { - escapeCount := 0 - for i := index - 1; i >= 0; i-- { - if s[i] != '\\' { - break - } - escapeCount += 1 - } - - if escapeCount%2 == 1 { - preserve += s[0 : index+1] - s = s[index+1:] - continue - } - - ret = append(ret, preserve+s[0:index]) - s = s[index+1:] - preserve = "" - if n > 0 { - n-- - } - } else { - break - } - } - if preserve != "" || s != "" || len(ret) == 0 { - ret = append(ret, preserve+s) - } - return ret -} - func unescape(s string) string { ret := "" for { diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go index 6995e8913..fbb289bb7 100644 --- a/androidmk/parser/make_strings_test.go +++ b/androidmk/parser/make_strings_test.go @@ -26,64 +26,53 @@ var splitNTestCases = []struct { n int }{ { - in: &MakeString{ - Strings: []string{ - "a b c", - "d e f", - " h i j", - }, - Variables: []Variable{ - Variable{Name: SimpleMakeString("var1", NoPos)}, - Variable{Name: SimpleMakeString("var2", NoPos)}, - }, - }, + // "a b c$(var1)d e f$(var2) h i j" + in: genMakeString("a b c", "var1", "d e f", "var2", " h i j"), sep: " ", n: -1, expected: []*MakeString{ - SimpleMakeString("a", NoPos), - SimpleMakeString("b", NoPos), - &MakeString{ - Strings: []string{"c", "d"}, - Variables: []Variable{ - Variable{Name: SimpleMakeString("var1", NoPos)}, - }, - }, - SimpleMakeString("e", NoPos), - &MakeString{ - Strings: []string{"f", ""}, - Variables: []Variable{ - Variable{Name: SimpleMakeString("var2", NoPos)}, - }, - }, - SimpleMakeString("h", NoPos), - SimpleMakeString("i", NoPos), - SimpleMakeString("j", NoPos), + genMakeString("a"), + genMakeString("b"), + genMakeString("c", "var1", "d"), + genMakeString("e"), + genMakeString("f", "var2", ""), + genMakeString("h"), + genMakeString("i"), + genMakeString("j"), }, }, { - in: &MakeString{ - Strings: []string{ - "a b c", - "d e f", - " h i j", - }, - Variables: []Variable{ - Variable{Name: SimpleMakeString("var1", NoPos)}, - Variable{Name: SimpleMakeString("var2", NoPos)}, - }, - }, + // "a b c$(var1)d e f$(var2) h i j" + in: genMakeString("a b c", "var1", "d e f", "var2", " h i j"), sep: " ", n: 3, expected: []*MakeString{ - SimpleMakeString("a", NoPos), - SimpleMakeString("b", NoPos), - &MakeString{ - Strings: []string{"c", "d e f", " h i j"}, - Variables: []Variable{ - Variable{Name: SimpleMakeString("var1", NoPos)}, - Variable{Name: SimpleMakeString("var2", NoPos)}, - }, - }, + genMakeString("a"), + genMakeString("b"), + genMakeString("c", "var1", "d e f", "var2", " h i j"), + }, + }, + { + // "$(var1) $(var2)" + in: genMakeString("", "var1", " ", "var2", ""), + sep: " ", + n: -1, + expected: []*MakeString{ + genMakeString("", "var1", ""), + genMakeString("", "var2", ""), + }, + }, + { + // "a,,b,c," + in: genMakeString("a,,b,c,"), + sep: ",", + n: -1, + expected: []*MakeString{ + genMakeString("a"), + genMakeString(""), + genMakeString("b"), + genMakeString("c"), + genMakeString(""), }, }, } @@ -104,15 +93,15 @@ var valueTestCases = []struct { expected string }{ { - in: SimpleMakeString("a b", NoPos), + in: genMakeString("a b"), expected: "a b", }, { - in: SimpleMakeString("a\\ \\\tb\\\\", NoPos), + in: genMakeString("a\\ \\\tb\\\\"), expected: "a \tb\\", }, { - in: SimpleMakeString("a\\b\\", NoPos), + in: genMakeString("a\\b\\"), expected: "a\\b\\", }, } @@ -131,31 +120,88 @@ var splitWordsTestCases = []struct { expected []*MakeString }{ { - in: SimpleMakeString("", NoPos), + in: genMakeString(""), expected: []*MakeString{}, }, { - in: SimpleMakeString(" a b\\ c d", NoPos), + in: genMakeString(` a b\ c d`), + expected: []*MakeString{ + genMakeString("a"), + genMakeString(`b\ c`), + genMakeString("d"), + }, + }, + { + in: SimpleMakeString(" a\tb"+`\`+"\t"+`\ c d `, NoPos), expected: []*MakeString{ - SimpleMakeString("a", NoPos), - SimpleMakeString("b\\ c", NoPos), - SimpleMakeString("d", NoPos), + genMakeString("a"), + genMakeString("b" + `\` + "\t" + `\ c`), + genMakeString("d"), }, }, { - in: SimpleMakeString(" a\tb\\\t\\ c d ", NoPos), + in: genMakeString(`a\\ b\\\ c d`), expected: []*MakeString{ - SimpleMakeString("a", NoPos), - SimpleMakeString("b\\\t\\ c", NoPos), - SimpleMakeString("d", NoPos), + genMakeString(`a\\`), + genMakeString(`b\\\ c`), + genMakeString("d"), }, }, { - in: SimpleMakeString(`a\\ b\\\ c d`, NoPos), + in: genMakeString(`\\ a`), expected: []*MakeString{ - SimpleMakeString(`a\\`, NoPos), - SimpleMakeString(`b\\\ c`, NoPos), - SimpleMakeString("d", NoPos), + genMakeString(`\\`), + genMakeString("a"), + }, + }, + { + // " " + in: &MakeString{ + Strings: []string{" \t \t"}, + Variables: nil, + }, + expected: []*MakeString{}, + }, + { + // " a $(X)b c " + in: genMakeString(" a ", "X", "b c "), + expected: []*MakeString{ + genMakeString("a"), + genMakeString("", "X", "b"), + genMakeString("c"), + }, + }, + { + // " a b$(X)c d" + in: genMakeString(" a b", "X", "c d"), + expected: []*MakeString{ + genMakeString("a"), + genMakeString("b", "X", "c"), + genMakeString("d"), + }, + }, + { + // "$(X) $(Y)" + in: genMakeString("", "X", " ", "Y", ""), + expected: []*MakeString{ + genMakeString("", "X", ""), + genMakeString("", "Y", ""), + }, + }, + { + // " a$(X) b" + in: genMakeString(" a", "X", " b"), + expected: []*MakeString{ + genMakeString("a", "X", ""), + genMakeString("b"), + }, + }, + { + // "a$(X) b$(Y) " + in: genMakeString("a", "X", " b", "Y", " "), + expected: []*MakeString{ + genMakeString("a", "X", ""), + genMakeString("b", "Y", ""), }, }, } @@ -180,3 +226,20 @@ func dumpArray(a []*MakeString) string { return strings.Join(ret, "|||") } + +// generates MakeString from alternating string chunks and variable names, +// e.g., genMakeString("a", "X", "b") returns MakeString for "a$(X)b" +func genMakeString(items ...string) *MakeString { + n := len(items) / 2 + if len(items) != (2*n + 1) { + panic("genMakeString expects odd number of arguments") + } + + ms := &MakeString{Strings: make([]string, n+1), Variables: make([]Variable, n)} + ms.Strings[0] = items[0] + for i := 1; i <= n; i++ { + ms.Variables[i-1] = Variable{Name: SimpleMakeString(items[2*i-1], NoPos)} + ms.Strings[i] = items[2*i] + } + return ms +} diff --git a/androidmk/parser/parser.go b/androidmk/parser/parser.go index c14910a4f..5afef652a 100644 --- a/androidmk/parser/parser.go +++ b/androidmk/parser/parser.go @@ -553,12 +553,14 @@ var directives = [...]string{ "else", "endef", "endif", + "export", "ifdef", "ifeq", "ifndef", "ifneq", "include", "-include", + "unexport", } var functions = [...]string{ diff --git a/apex/allowed_deps.txt b/apex/allowed_deps.txt index 5b8563dc2..a2f87978e 100644 --- a/apex/allowed_deps.txt +++ b/apex/allowed_deps.txt @@ -132,6 +132,7 @@ bouncycastle_ike_digests(minSdkVersion:current) brotli-java(minSdkVersion:current) captiveportal-lib(minSdkVersion:29) car-ui-lib(minSdkVersion:28) +car-ui-lib-overlayable(minSdkVersion:28) CellBroadcastApp(minSdkVersion:29) CellBroadcastServiceModule(minSdkVersion:29) codecs_g711dec(minSdkVersion:29) @@ -333,6 +334,7 @@ libminijail_gen_syscall(minSdkVersion:(no version)) libminijail_gen_syscall_obj(minSdkVersion:29) libminijail_generated(minSdkVersion:29) libmkvextractor(minSdkVersion:29) +libmodules-utils-build(minSdkVersion:29) libmp3extractor(minSdkVersion:29) libmp4extractor(minSdkVersion:29) libmpeg2dec(minSdkVersion:29) @@ -446,6 +448,7 @@ neuralnetworks_utils_hal_1_1(minSdkVersion:30) neuralnetworks_utils_hal_1_2(minSdkVersion:30) neuralnetworks_utils_hal_1_3(minSdkVersion:30) neuralnetworks_utils_hal_common(minSdkVersion:30) +neuralnetworks_utils_hal_service(minSdkVersion:30) PermissionController(minSdkVersion:28) permissioncontroller-statsd(minSdkVersion:current) philox_random(minSdkVersion:(no version)) diff --git a/apex/androidmk.go b/apex/androidmk.go index da38c2ac8..6c76ad3f9 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -360,7 +360,11 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC") // do we need a new class? fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", a.outputFile.String()) fmt.Fprintln(w, "LOCAL_MODULE_PATH :=", a.installDir.ToMakePath().String()) - fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+apexType.suffix()) + stemSuffix := apexType.suffix() + if a.isCompressed { + stemSuffix = ".capex" + } + fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", name+stemSuffix) fmt.Fprintln(w, "LOCAL_UNINSTALLABLE_MODULE :=", !a.installable()) // Because apex writes .mk with Custom(), we need to write manually some common properties diff --git a/apex/apex.go b/apex/apex.go index b906b28f3..12b5bc0c1 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -120,6 +120,12 @@ type apexBundleProperties struct { // Default: true. Installable *bool + // Whether this APEX can be compressed or not. Setting this property to false means this + // APEX will never be compressed. When set to true, APEX will be compressed if other + // conditions, e.g, target device needs to support APEX compression, are also fulfilled. + // Default: true. + Compressible *bool + // For native libraries and binaries, use the vendor variant instead of the core (platform) // variant. Default is false. DO NOT use this for APEXes that are installed to the system or // system_ext partition. @@ -354,6 +360,8 @@ type apexBundle struct { prebuiltFileToDelete string + isCompressed bool + // Path of API coverage generate file coverageOutputPath android.ModuleOutPath } @@ -522,11 +530,8 @@ func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext, nativeM if ctx.Device() { binVariations = append(binVariations, blueprint.Variation{Mutator: "image", Variation: imageVariation}) - libVariations = append(libVariations, - blueprint.Variation{Mutator: "image", Variation: imageVariation}, - blueprint.Variation{Mutator: "version", Variation: ""}) // "" is the non-stub variant - rustLibVariations = append(rustLibVariations, - blueprint.Variation{Mutator: "image", Variation: imageVariation}) + libVariations = append(libVariations, blueprint.Variation{Mutator: "image", Variation: imageVariation}) + rustLibVariations = append(rustLibVariations, blueprint.Variation{Mutator: "image", Variation: imageVariation}) } // Use *FarVariation* to be able to depend on modules having conflicting variations with @@ -1522,6 +1527,9 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { provideNativeLibs = append(provideNativeLibs, fi.stem()) } return true // track transitive dependencies + } else if r, ok := child.(*rust.Module); ok { + fi := apexFileForRustLibrary(ctx, r) + filesInfo = append(filesInfo, fi) } else { propertyName := "native_shared_libs" if isJniLib { @@ -1681,6 +1689,24 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Don't track further return false } + + // If the dep is not considered to be in the same + // apex, don't add it to filesInfo so that it is not + // included in this APEX. + // TODO(jiyong): move this to at the top of the + // else-if clause for the indirect dependencies. + // Currently, that's impossible because we would + // like to record requiredNativeLibs even when + // DepIsInSameAPex is false. + if !am.DepIsInSameApex(ctx, am) { + return false + } + + filesInfo = append(filesInfo, af) + return true // track transitive dependencies + } else if rm, ok := child.(*rust.Module); ok { + af := apexFileForRustLibrary(ctx, rm) + af.transitiveDep = true filesInfo = append(filesInfo, af) return true // track transitive dependencies } diff --git a/apex/apex_test.go b/apex/apex_test.go index 0b67ef577..5b841fc8c 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -346,6 +346,13 @@ func ensureListEmpty(t *testing.T, result []string) { } } +func ensureListNotEmpty(t *testing.T, result []string) { + t.Helper() + if len(result) == 0 { + t.Errorf("%q is expected to be not empty", result) + } +} + // Minimal test func TestBasicApex(t *testing.T) { ctx, config := testApex(t, ` @@ -355,7 +362,10 @@ func TestBasicApex(t *testing.T) { androidManifest: ":myapex.androidmanifest", key: "myapex.key", binaries: ["foo.rust"], - native_shared_libs: ["mylib"], + native_shared_libs: [ + "mylib", + "libfoo.ffi", + ], rust_dyn_libs: ["libfoo.dylib.rust"], multilib: { both: { @@ -392,7 +402,10 @@ func TestBasicApex(t *testing.T) { cc_library { name: "mylib", srcs: ["mylib.cpp"], - shared_libs: ["mylib2"], + shared_libs: [ + "mylib2", + "libbar.ffi", + ], system_shared_libs: [], stl: "none", // TODO: remove //apex_available:platform @@ -444,6 +457,20 @@ func TestBasicApex(t *testing.T) { apex_available: ["myapex"], } + rust_ffi_shared { + name: "libfoo.ffi", + srcs: ["foo.rs"], + crate_name: "foo", + apex_available: ["myapex"], + } + + rust_ffi_shared { + name: "libbar.ffi", + srcs: ["foo.rs"], + crate_name: "bar", + apex_available: ["myapex"], + } + apex { name: "com.android.gki.fake", binaries: ["foo"], @@ -559,12 +586,14 @@ func TestBasicApex(t *testing.T) { ensureListContains(t, ctx.ModuleVariantsForTests("myjar"), "android_common_apex10000") ensureListContains(t, ctx.ModuleVariantsForTests("myjar_dex"), "android_common_apex10000") ensureListContains(t, ctx.ModuleVariantsForTests("foo.rust"), "android_arm64_armv8-a_apex10000") + ensureListContains(t, ctx.ModuleVariantsForTests("libfoo.ffi"), "android_arm64_armv8-a_shared_apex10000") // Ensure that apex variant is created for the indirect dep ensureListContains(t, ctx.ModuleVariantsForTests("mylib2"), "android_arm64_armv8-a_shared_apex10000") ensureListContains(t, ctx.ModuleVariantsForTests("myotherjar"), "android_common_apex10000") ensureListContains(t, ctx.ModuleVariantsForTests("libfoo.rlib.rust"), "android_arm64_armv8-a_rlib_dylib-std_apex10000") ensureListContains(t, ctx.ModuleVariantsForTests("libfoo.dylib.rust"), "android_arm64_armv8-a_dylib_apex10000") + ensureListContains(t, ctx.ModuleVariantsForTests("libbar.ffi"), "android_arm64_armv8-a_shared_apex10000") // Ensure that both direct and indirect deps are copied into apex ensureContains(t, copyCmds, "image.apex/lib64/mylib.so") @@ -572,6 +601,8 @@ func TestBasicApex(t *testing.T) { ensureContains(t, copyCmds, "image.apex/javalib/myjar_stem.jar") ensureContains(t, copyCmds, "image.apex/javalib/myjar_dex.jar") ensureContains(t, copyCmds, "image.apex/lib64/libfoo.dylib.rust.dylib.so") + ensureContains(t, copyCmds, "image.apex/lib64/libfoo.ffi.so") + ensureContains(t, copyCmds, "image.apex/lib64/libbar.ffi.so") // .. but not for java libs ensureNotContains(t, copyCmds, "image.apex/javalib/myotherjar.jar") ensureNotContains(t, copyCmds, "image.apex/javalib/msharedjar.jar") @@ -1782,6 +1813,31 @@ func TestApexMinSdkVersion_ErrorIfIncompatibleVersion(t *testing.T) { min_sdk_version: "30", } `) + + testApexError(t, `module "libfoo.ffi".*: should support min_sdk_version\(29\)`, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["libfoo.ffi"], + min_sdk_version: "29", + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + rust_ffi_shared { + name: "libfoo.ffi", + srcs: ["foo.rs"], + crate_name: "foo", + apex_available: [ + "myapex", + ], + min_sdk_version: "30", + } + `) } func TestApexMinSdkVersion_Okay(t *testing.T) { @@ -5964,9 +6020,27 @@ func TestTestFor(t *testing.T) { srcs: ["mylib.cpp"], system_shared_libs: [], stl: "none", - shared_libs: ["mylib", "myprivlib"], + shared_libs: ["mylib", "myprivlib", "mytestlib"], test_for: ["myapex"] } + + cc_library { + name: "mytestlib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + shared_libs: ["mylib", "myprivlib"], + stl: "none", + test_for: ["myapex"], + } + + cc_benchmark { + name: "mybench", + srcs: ["mylib.cpp"], + system_shared_libs: [], + shared_libs: ["mylib", "myprivlib"], + stl: "none", + test_for: ["myapex"], + } `) // the test 'mytest' is a test for the apex, therefore is linked to the @@ -5974,6 +6048,16 @@ func TestTestFor(t *testing.T) { ldFlags := ctx.ModuleForTests("mytest", "android_arm64_armv8-a").Rule("ld").Args["libFlags"] ensureContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared/mylib.so") ensureNotContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared_1/mylib.so") + + // The same should be true for cc_library + ldFlags = ctx.ModuleForTests("mytestlib", "android_arm64_armv8-a_shared").Rule("ld").Args["libFlags"] + ensureContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared/mylib.so") + ensureNotContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared_1/mylib.so") + + // ... and for cc_benchmark + ldFlags = ctx.ModuleForTests("mybench", "android_arm64_armv8-a").Rule("ld").Args["libFlags"] + ensureContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared/mylib.so") + ensureNotContains(t, ldFlags, "mylib/android_arm64_armv8-a_shared_1/mylib.so") } // TODO(jungjw): Move this to proptools @@ -6186,6 +6270,40 @@ func TestNonPreferredPrebuiltDependency(t *testing.T) { `) } +func TestCompressedApex(t *testing.T) { + ctx, config := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + compressible: true, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `, func(fs map[string][]byte, config android.Config) { + config.TestProductVariables.CompressedApex = proptools.BoolPtr(true) + }) + + compressRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("compressRule") + ensureContains(t, compressRule.Output.String(), "myapex.capex.unsigned") + + signApkRule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Description("sign compressedApex") + ensureEquals(t, signApkRule.Input.String(), compressRule.Output.String()) + + // Make sure output of bundle is .capex + ab := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) + ensureContains(t, ab.outputFile.String(), "myapex.capex") + + // Verify android.mk rules + data := android.AndroidMkDataForTest(t, config, "", ab) + var builder strings.Builder + data.Custom(&builder, ab.BaseModuleName(), "TARGET_", "", data) + androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE_STEM := myapex.capex\n") +} + func TestPreferredPrebuiltSharedLibDep(t *testing.T) { ctx, config := testApex(t, ` apex { @@ -6237,6 +6355,55 @@ func TestPreferredPrebuiltSharedLibDep(t *testing.T) { ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES += otherlib\n") } +func TestExcludeDependency(t *testing.T) { + ctx, _ := testApex(t, ` + apex { + name: "myapex", + key: "myapex.key", + native_shared_libs: ["mylib"], + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + cc_library { + name: "mylib", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + apex_available: ["myapex"], + shared_libs: ["mylib2"], + target: { + apex: { + exclude_shared_libs: ["mylib2"], + }, + }, + } + + cc_library { + name: "mylib2", + srcs: ["mylib.cpp"], + system_shared_libs: [], + stl: "none", + } + `) + + // Check if mylib is linked to mylib2 for the non-apex target + ldFlags := ctx.ModuleForTests("mylib", "android_arm64_armv8-a_shared").Rule("ld").Args["libFlags"] + ensureContains(t, ldFlags, "mylib2/android_arm64_armv8-a_shared/mylib2.so") + + // Make sure that the link doesn't occur for the apex target + ldFlags = ctx.ModuleForTests("mylib", "android_arm64_armv8-a_shared_apex10000").Rule("ld").Args["libFlags"] + ensureNotContains(t, ldFlags, "mylib2/android_arm64_armv8-a_shared_apex10000/mylib2.so") + + // It shouldn't appear in the copy cmd as well. + copyCmds := ctx.ModuleForTests("myapex", "android_common_myapex_image").Rule("apexRule").Args["copy_commands"] + ensureNotContains(t, copyCmds, "image.apex/lib64/mylib2.so") +} + func TestMain(m *testing.M) { run := func() int { setUp() diff --git a/apex/builder.go b/apex/builder.go index 66eaff1d7..9db8e5929 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -66,6 +66,7 @@ func init() { pctx.HostBinToolVariable("extract_apks", "extract_apks") pctx.HostBinToolVariable("make_f2fs", "make_f2fs") pctx.HostBinToolVariable("sload_f2fs", "sload_f2fs") + pctx.HostBinToolVariable("apex_compression_tool", "apex_compression_tool") pctx.SourcePathVariable("genNdkUsedbyApexPath", "build/soong/scripts/gen_ndk_usedby_apex.sh") } @@ -738,7 +739,7 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { //////////////////////////////////////////////////////////////////////////////////// // Step 4: Sign the APEX using signapk - a.outputFile = android.PathForModuleOut(ctx, a.Name()+suffix) + signedOutputFile := android.PathForModuleOut(ctx, a.Name()+suffix) pem, key := a.getCertificateAndPrivateKey(ctx) rule := java.Signapk @@ -750,16 +751,47 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_SIGNAPK") { rule = java.SignapkRE args["implicits"] = strings.Join(implicits.Strings(), ",") - args["outCommaList"] = a.outputFile.String() + args["outCommaList"] = signedOutputFile.String() } ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "signapk", - Output: a.outputFile, + Output: signedOutputFile, Input: unsignedOutputFile, Implicits: implicits, Args: args, }) + a.outputFile = signedOutputFile + + // Process APEX compression if enabled + compressionEnabled := ctx.Config().CompressedApex() && proptools.BoolDefault(a.properties.Compressible, true) + if compressionEnabled && apexType == imageApex { + a.isCompressed = true + unsignedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+".capex.unsigned") + + compressRule := android.NewRuleBuilder(pctx, ctx) + compressRule.Command(). + Text("rm"). + FlagWithOutput("-f ", unsignedCompressedOutputFile) + compressRule.Command(). + BuiltTool("apex_compression_tool"). + Flag("compress"). + FlagWithArg("--apex_compression_tool ", outHostBinDir+":"+prebuiltSdkToolsBinDir). + FlagWithInput("--input ", signedOutputFile). + FlagWithOutput("--output ", unsignedCompressedOutputFile) + compressRule.Build("compressRule", "Generate unsigned compressed APEX file") + + signedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+".capex") + ctx.Build(pctx, android.BuildParams{ + Rule: rule, + Description: "sign compressedApex", + Output: signedCompressedOutputFile, + Input: unsignedCompressedOutputFile, + Implicits: implicits, + Args: args, + }) + a.outputFile = signedCompressedOutputFile + } // Install to $OUT/soong/{target,host}/.../apex if a.installable() { diff --git a/cc/Android.bp b/cc/Android.bp index 88104e295..33f3db22a 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -31,6 +31,7 @@ bootstrap_go_package { "sanitize.go", "sabi.go", "sdk.go", + "snapshot_prebuilt.go", "snapshot_utils.go", "stl.go", "strip.go", diff --git a/cc/androidmk.go b/cc/androidmk.go index 320e69b4f..9b61e55e9 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -130,7 +130,7 @@ func (c *Module) AndroidMkEntries() []android.AndroidMkEntries { }, }, ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { if c.Properties.IsSdkVariant && c.Properties.SdkAndPlatformVariantVisibleToMake && c.CcLibraryInterface() && c.Shared() { // Using the SDK variant as a JNI library needs a copy of the .so that @@ -296,7 +296,7 @@ func (library *libraryDecorator) AndroidMkEntries(ctx AndroidMkContext, entries func (object *objectLinker) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { entries.Class = "STATIC_LIBRARIES" entries.ExtraFooters = append(entries.ExtraFooters, - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { out := entries.OutputFile.Path() varname := fmt.Sprintf("SOONG_%sOBJECT_%s%s", prefix, name, entries.SubName) @@ -574,7 +574,7 @@ func (c *snapshotObjectLinker) AndroidMkEntries(ctx AndroidMkContext, entries *a } entries.ExtraFooters = append(entries.ExtraFooters, - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { out := entries.OutputFile.Path() varname := fmt.Sprintf("SOONG_%sOBJECT_%s%s", prefix, name, entries.SubName) diff --git a/cc/builder.go b/cc/builder.go index 5545a5b05..9cd78d59a 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -235,6 +235,7 @@ var ( }, &remoteexec.REParams{ Labels: map[string]string{"type": "abi-dump", "tool": "header-abi-dumper"}, ExecStrategy: "${config.REAbiDumperExecStrategy}", + Inputs: []string{"$sAbiLinkerLibs"}, Platform: map[string]string{ remoteexec.PoolKey: "${config.RECXXPool}", }, @@ -129,6 +129,9 @@ type Deps struct { // Used for host bionic LinkerFlagsFile string DynamicLinker string + + // List of libs that need to be excluded for APEX variant + ExcludeLibsForApex []string } // PathDeps is a struct containing file paths to dependencies of a module. @@ -332,6 +335,11 @@ type BaseProperties struct { // framework module from a snapshot. Exclude_from_vendor_snapshot *bool Exclude_from_recovery_snapshot *bool + + // List of APEXes that this module has private access to for testing purpose. The module + // can depend on libraries that are not exported by the APEXes and use private symbols + // from the exported libraries. + Test_for []string } type VendorProperties struct { @@ -572,6 +580,9 @@ type libraryDependencyTag struct { staticUnwinder bool makeSuffix string + + // Whether or not this dependency has to be followed for the apex variants + excludeInApex bool } // header returns true if the libraryDependencyTag is tagging a header lib dependency. @@ -1590,7 +1601,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { // glob exported headers for snapshot, if BOARD_VNDK_VERSION is current. if i, ok := c.linker.(snapshotLibraryInterface); ok && ctx.DeviceConfig().VndkVersion() == "current" { - if isSnapshotAware(ctx, c, apexInfo) { + if shouldCollectHeadersForSnapshot(ctx, c, apexInfo) { i.collectHeadersForSnapshot(ctx) } } @@ -1937,6 +1948,9 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { if inList(lib, deps.ReexportStaticLibHeaders) { depTag.reexportFlags = true } + if inList(lib, deps.ExcludeLibsForApex) { + depTag.excludeInApex = true + } if impl, ok := syspropImplLibraries[lib]; ok { lib = impl @@ -1974,6 +1988,9 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { if inList(lib, deps.ReexportSharedLibHeaders) { depTag.reexportFlags = true } + if inList(lib, deps.ExcludeLibsForApex) { + depTag.excludeInApex = true + } if impl, ok := syspropImplLibraries[lib]; ok { lib = impl @@ -2262,8 +2279,8 @@ func checkDoubleLoadableLibraries(ctx android.TopDownMutatorContext) { // For example, with maxSdkVersion is 10 and versionList is [9,11] // it returns 9 as string. The list of stubs must be in order from // oldest to newest. -func (c *Module) chooseSdkVersion(ctx android.PathContext, stubsInfo []SharedLibraryStubsInfo, - maxSdkVersion android.ApiLevel) (SharedLibraryStubsInfo, error) { +func (c *Module) chooseSdkVersion(ctx android.PathContext, stubsInfo []SharedStubLibrary, + maxSdkVersion android.ApiLevel) (SharedStubLibrary, error) { for i := range stubsInfo { stubInfo := stubsInfo[len(stubsInfo)-i-1] @@ -2274,7 +2291,7 @@ func (c *Module) chooseSdkVersion(ctx android.PathContext, stubsInfo []SharedLib var err error ver, err = android.ApiLevelFromUser(ctx, stubInfo.Version) if err != nil { - return SharedLibraryStubsInfo{}, err + return SharedStubLibrary{}, err } } if ver.LessThanOrEqualTo(maxSdkVersion) { @@ -2285,7 +2302,7 @@ func (c *Module) chooseSdkVersion(ctx android.PathContext, stubsInfo []SharedLib for _, stubInfo := range stubsInfo { versionList = append(versionList, stubInfo.Version) } - return SharedLibraryStubsInfo{}, fmt.Errorf("not found a version(<=%s) in versionList: %v", maxSdkVersion.String(), versionList) + return SharedStubLibrary{}, fmt.Errorf("not found a version(<=%s) in versionList: %v", maxSdkVersion.String(), versionList) } // Convert dependencies to paths. Returns a PathDeps containing paths @@ -2408,6 +2425,10 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { return } + if !apexInfo.IsForPlatform() && libDepTag.excludeInApex { + return + } + depExporterInfo := ctx.OtherModuleProvider(dep, FlagExporterInfoProvider).(FlagExporterInfo) var ptr *android.Paths @@ -2427,10 +2448,11 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { } return } + sharedLibraryInfo := ctx.OtherModuleProvider(dep, SharedLibraryInfoProvider).(SharedLibraryInfo) - sharedLibraryStubsInfo := ctx.OtherModuleProvider(dep, SharedLibraryImplementationStubsInfoProvider).(SharedLibraryImplementationStubsInfo) + sharedLibraryStubsInfo := ctx.OtherModuleProvider(dep, SharedLibraryStubsProvider).(SharedLibraryStubsInfo) - if !libDepTag.explicitlyVersioned && len(sharedLibraryStubsInfo.SharedLibraryStubsInfos) > 0 { + if !libDepTag.explicitlyVersioned && len(sharedLibraryStubsInfo.SharedStubLibraries) > 0 { useStubs := false if lib := moduleLibraryInterface(dep); lib.buildStubs() && c.UseVndk() { // LLNDK @@ -2465,7 +2487,7 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // when to use (unspecified) stubs, check min_sdk_version and choose the right one if useStubs { sharedLibraryStubsInfo, err := - c.chooseSdkVersion(ctx, sharedLibraryStubsInfo.SharedLibraryStubsInfos, c.apexSdkVersion) + c.chooseSdkVersion(ctx, sharedLibraryStubsInfo.SharedStubLibraries, c.apexSdkVersion) if err != nil { ctx.OtherModuleErrorf(dep, err.Error()) return @@ -2942,13 +2964,7 @@ func (c *Module) AvailableFor(what string) bool { } func (c *Module) TestFor() []string { - if test, ok := c.linker.(interface { - testFor() []string - }); ok { - return test.testFor() - } else { - return c.ApexModuleBase.TestFor() - } + return c.Properties.Test_for } func (c *Module) UniqueApexVariations() bool { @@ -3022,6 +3038,10 @@ func (c *Module) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Modu // linked; the dependency is used only during the compilation phase. return false } + + if isLibDepTag && libDepTag.excludeInApex { + return false + } } if depTag == stubImplDepTag || depTag == llndkImplDep { // We don't track beyond LLNDK or from an implementation library to its stubs. diff --git a/cc/library.go b/cc/library.go index c626b03f6..b796aafe3 100644 --- a/cc/library.go +++ b/cc/library.go @@ -28,6 +28,7 @@ import ( "android/soong/cc/config" ) +// LibraryProperties is a collection of properties shared by cc library rules. type LibraryProperties struct { // local file name to pass to the linker as -unexported_symbols_list Unexported_symbols_list *string `android:"path,arch_variant"` @@ -115,14 +116,23 @@ type LibraryProperties struct { Llndk_stubs *string } +// StaticProperties is a properties stanza to affect only attributes of the "static" variants of a +// library module. type StaticProperties struct { Static StaticOrSharedProperties `android:"arch_variant"` } +// SharedProperties is a properties stanza to affect only attributes of the "shared" variants of a +// library module. type SharedProperties struct { Shared StaticOrSharedProperties `android:"arch_variant"` } +// StaticOrSharedProperties is an embedded struct representing properties to affect attributes of +// either only the "static" variants or only the "shared" variants of a library module. These override +// the base properties of the same name. +// Use `StaticProperties` or `SharedProperties`, depending on which variant is needed. +// `StaticOrSharedProperties` exists only to avoid duplication. type StaticOrSharedProperties struct { Srcs []string `android:"path,arch_variant"` @@ -242,16 +252,23 @@ func LibraryHostSharedFactory() android.Module { return module.Init() } +// flagExporter is a separated portion of libraryDecorator pertaining to exported +// include paths and flags. Keeping this dependency-related information separate +// from the rest of library information is helpful in keeping data more structured +// and explicit. type flagExporter struct { Properties FlagExporterProperties - dirs android.Paths - systemDirs android.Paths - flags []string + dirs android.Paths // Include directories to be included with -I + systemDirs android.Paths // System include directories to be included with -isystem + flags []string // Exported raw flags. deps android.Paths headers android.Paths } +// exportedIncludes returns the effective include paths for this module and +// any module that links against this module. This is obtained from +// the export_include_dirs property in the appropriate target stanza. func (f *flagExporter) exportedIncludes(ctx ModuleContext) android.Paths { // TODO(b/150902910): product variant must use Target.Product if ctx.useVndk() && f.Properties.Target.Vendor.Override_export_include_dirs != nil { @@ -261,25 +278,35 @@ func (f *flagExporter) exportedIncludes(ctx ModuleContext) android.Paths { } } +// exportIncludes registers the include directories and system include directories to be exported +// transitively to modules depending on this module. func (f *flagExporter) exportIncludes(ctx ModuleContext) { f.dirs = append(f.dirs, f.exportedIncludes(ctx)...) f.systemDirs = append(f.systemDirs, android.PathsForModuleSrc(ctx, f.Properties.Export_system_include_dirs)...) } +// exportIncludesAsSystem registers the include directories and system include directories to be +// exported transitively both as system include directories to modules depending on this module. func (f *flagExporter) exportIncludesAsSystem(ctx ModuleContext) { // all dirs are force exported as system f.systemDirs = append(f.systemDirs, f.exportedIncludes(ctx)...) f.systemDirs = append(f.systemDirs, android.PathsForModuleSrc(ctx, f.Properties.Export_system_include_dirs)...) } +// reexportDirs registers the given directories as include directories to be exported transitively +// to modules depending on this module. func (f *flagExporter) reexportDirs(dirs ...android.Path) { f.dirs = append(f.dirs, dirs...) } +// reexportSystemDirs registers the given directories as system include directories +// to be exported transitively to modules depending on this module. func (f *flagExporter) reexportSystemDirs(dirs ...android.Path) { f.systemDirs = append(f.systemDirs, dirs...) } +// reexportFlags registers the flags to be exported transitively to modules depending on this +// module. func (f *flagExporter) reexportFlags(flags ...string) { if android.PrefixInList(flags, "-I") || android.PrefixInList(flags, "-isystem") { panic(fmt.Errorf("Exporting invalid flag %q: "+ @@ -457,6 +484,8 @@ func (l *libraryDecorator) snapshotHeaders() android.Paths { return l.collectedSnapshotHeaders } +// linkerProps returns the list of properties structs relevant for this library. (For example, if +// the library is cc_shared_library, then static-library properties are omitted.) func (library *libraryDecorator) linkerProps() []interface{} { var props []interface{} props = append(props, library.baseLinker.linkerProps()...) @@ -476,6 +505,9 @@ func (library *libraryDecorator) linkerProps() []interface{} { return props } +// linkerFlags takes a Flags struct and augments it to contain linker flags that are defined by this +// library, or that are implied by attributes of this library (such as whether this library is a +// shared library). func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { flags = library.baseLinker.linkerFlags(ctx, flags) @@ -526,6 +558,9 @@ func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Fla return flags } +// compilerFlags takes a Flags and augments it to contain compile flags from global values, +// per-target values, module type values, per-module Blueprints properties, extra flags from +// `flags`, and generated sources from `deps`. func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags, deps PathDeps) Flags { exportIncludeDirs := library.flagExporter.exportedIncludes(ctx) if len(exportIncludeDirs) > 0 { @@ -727,6 +762,8 @@ func (library *libraryDecorator) getLibNameHelper(baseModuleName string, useVndk return name + suffix } +// getLibName returns the actual canonical name of the library (the name which +// should be passed to the linker via linker flags). func (library *libraryDecorator) getLibName(ctx BaseModuleContext) string { name := library.getLibNameHelper(ctx.baseModuleName(), ctx.useVndk()) @@ -1058,18 +1095,18 @@ func (library *libraryDecorator) linkShared(ctx ModuleContext, stubs := ctx.GetDirectDepsWithTag(stubImplDepTag) if len(stubs) > 0 { - var stubsInfo []SharedLibraryStubsInfo + var stubsInfo []SharedStubLibrary for _, stub := range stubs { stubInfo := ctx.OtherModuleProvider(stub, SharedLibraryInfoProvider).(SharedLibraryInfo) flagInfo := ctx.OtherModuleProvider(stub, FlagExporterInfoProvider).(FlagExporterInfo) - stubsInfo = append(stubsInfo, SharedLibraryStubsInfo{ + stubsInfo = append(stubsInfo, SharedStubLibrary{ Version: moduleLibraryInterface(stub).stubsVersion(), SharedLibraryInfo: stubInfo, FlagExporterInfo: flagInfo, }) } - ctx.SetProvider(SharedLibraryImplementationStubsInfoProvider, SharedLibraryImplementationStubsInfo{ - SharedLibraryStubsInfos: stubsInfo, + ctx.SetProvider(SharedLibraryStubsProvider, SharedLibraryStubsInfo{ + SharedStubLibraries: stubsInfo, IsLLNDK: ctx.isLlndk(ctx.Config()), }) @@ -1158,9 +1195,15 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objec } } +// link registers actions to link this library, and sets various fields +// on this library to reflect information that should be exported up the build +// tree (for example, exported flags and include paths). func (library *libraryDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { + // Linking this library consists of linking `deps.Objs` (.o files in dependencies + // of this library), together with `objs` (.o files created by compiling this + // library). objs = deps.Objs.Copy().Append(objs) var out android.Path if library.static() || library.header() { @@ -1169,6 +1212,7 @@ func (library *libraryDecorator) link(ctx ModuleContext, out = library.linkShared(ctx, flags, deps, objs) } + // Export include paths and flags to be propagated up the tree. library.exportIncludes(ctx) library.reexportDirs(deps.ReexportedDirs...) library.reexportSystemDirs(deps.ReexportedSystemDirs...) @@ -1176,6 +1220,7 @@ func (library *libraryDecorator) link(ctx ModuleContext, library.reexportDeps(deps.ReexportedDeps...) library.addExportedGeneratedHeaders(deps.ReexportedGeneratedHeaders...) + // Optionally export aidl headers. if Bool(library.Properties.Aidl.Export_aidl_headers) { if library.baseCompiler.hasSrcExt(".aidl") { dir := android.PathForModuleGen(ctx, "aidl") @@ -1187,6 +1232,7 @@ func (library *libraryDecorator) link(ctx ModuleContext, } } + // Optionally export proto headers. if Bool(library.Properties.Proto.Export_proto_headers) { if library.baseCompiler.hasSrcExt(".proto") { var includes android.Paths @@ -1221,25 +1267,30 @@ func (library *libraryDecorator) link(ctx ModuleContext, } } + // Add sysprop-related directories to the exported directories of this library. library.reexportDirs(dir) library.reexportDeps(library.baseCompiler.pathDeps...) library.addExportedGeneratedHeaders(library.baseCompiler.pathDeps...) } + // Add stub-related flags if this library is a stub library. if library.buildStubs() && !library.skipAPIDefine { library.reexportFlags("-D" + versioningMacroName(ctx.Module().(*Module).ImplementationModuleName(ctx)) + "=" + library.stubsVersion()) } + // Propagate a Provider containing information about exported flags, deps, and include paths. library.flagExporter.setProvider(ctx) return out } +// buildStatic returns true if this library should be built as a static library. func (library *libraryDecorator) buildStatic() bool { return library.MutatedProperties.BuildStatic && BoolDefault(library.StaticProperties.Static.Enabled, true) } +// buildShared returns true if this library should be built as a shared library. func (library *libraryDecorator) buildShared() bool { return library.MutatedProperties.BuildShared && BoolDefault(library.SharedProperties.Shared.Enabled, true) @@ -1346,36 +1397,46 @@ func (library *libraryDecorator) everInstallable() bool { return library.shared() || library.static() } +// static returns true if this library is for a "static' variant. func (library *libraryDecorator) static() bool { return library.MutatedProperties.VariantIsStatic } +// shared returns true if this library is for a "shared' variant. func (library *libraryDecorator) shared() bool { return library.MutatedProperties.VariantIsShared } +// header returns true if this library is for a header-only variant. func (library *libraryDecorator) header() bool { + // Neither "static" nor "shared" implies this library is header-only. return !library.static() && !library.shared() } +// setStatic marks the library variant as "static". func (library *libraryDecorator) setStatic() { library.MutatedProperties.VariantIsStatic = true library.MutatedProperties.VariantIsShared = false } +// setShared marks the library variant as "shared". func (library *libraryDecorator) setShared() { library.MutatedProperties.VariantIsStatic = false library.MutatedProperties.VariantIsShared = true } +// BuildOnlyStatic disables building this library as a shared library. func (library *libraryDecorator) BuildOnlyStatic() { library.MutatedProperties.BuildShared = false } +// BuildOnlyShared disables building this library as a static library. func (library *libraryDecorator) BuildOnlyShared() { library.MutatedProperties.BuildStatic = false } +// HeaderOnly disables building this library as a shared or static library; +// the library only exists to propagate header file dependencies up the build graph. func (library *libraryDecorator) HeaderOnly() { library.MutatedProperties.BuildShared = false library.MutatedProperties.BuildStatic = false @@ -1458,6 +1519,17 @@ func (library *libraryDecorator) makeUninstallable(mod *Module) { var versioningMacroNamesListKey = android.NewOnceKey("versioningMacroNamesList") +// versioningMacroNamesList returns a singleton map, where keys are "version macro names", +// and values are the module name responsible for registering the version macro name. +// +// Version macros are used when building against stubs, to provide version information about +// the stub. Only stub libraries should have an entry in this list. +// +// For example, when building against libFoo#ver, __LIBFOO_API__ macro is set to ver so +// that headers from libFoo can be conditionally compiled (this may hide APIs +// that are not available for the version). +// +// This map is used to ensure that there aren't conflicts between these version macro names. func versioningMacroNamesList(config android.Config) *map[string]string { return config.Once(versioningMacroNamesListKey, func() interface{} { m := make(map[string]string) @@ -1469,12 +1541,17 @@ func versioningMacroNamesList(config android.Config) *map[string]string { // other characters are all converted to _ var charsNotForMacro = regexp.MustCompile("[^a-zA-Z0-9_]+") +// versioningMacroName returns the canonical version macro name for the given module. func versioningMacroName(moduleName string) string { macroName := charsNotForMacro.ReplaceAllString(moduleName, "_") macroName = strings.ToUpper(macroName) return "__" + macroName + "_API__" } +// NewLibrary builds and returns a new Module corresponding to a C++ library. +// Individual module implementations which comprise a C++ library (or something like +// a C++ library) should call this function, set some fields on the result, and +// then call the Init function. func NewLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { module := newModule(hod, android.MultilibBoth) @@ -1530,6 +1607,8 @@ func reuseStaticLibrary(mctx android.BottomUpMutatorContext, static, shared *Mod } } +// LinkageMutator adds "static" or "shared" variants for modules depending +// on whether the module can be built as a static library or a shared library. func LinkageMutator(mctx android.BottomUpMutatorContext) { cc_prebuilt := false if m, ok := mctx.Module().(*Module); ok && m.linker != nil { @@ -1607,6 +1686,9 @@ func LinkageMutator(mctx android.BottomUpMutatorContext) { } } +// normalizeVersions modifies `versions` in place, so that each raw version +// string becomes its normalized canonical form. +// Validates that the versions in `versions` are specified in least to greatest order. func normalizeVersions(ctx android.BaseModuleContext, versions []string) { var previous android.ApiLevel for i, v := range versions { diff --git a/cc/linkable.go b/cc/linkable.go index 0609b288d..ddf395009 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -6,6 +6,7 @@ import ( "github.com/google/blueprint" ) +// LinkableInterface is an interface for a type of module that is linkable in a C++ library. type LinkableInterface interface { Module() android.Module CcLibrary() bool @@ -51,23 +52,30 @@ type LinkableInterface interface { } var ( + // Dependency tag for crtbegin, an object file responsible for initialization. CrtBeginDepTag = dependencyTag{name: "crtbegin"} - CrtEndDepTag = dependencyTag{name: "crtend"} + // Dependency tag for crtend, an object file responsible for program termination. + CrtEndDepTag = dependencyTag{name: "crtend"} + // Dependency tag for coverage library. CoverageDepTag = dependencyTag{name: "coverage"} ) +// SharedDepTag returns the dependency tag for any C++ shared libraries. func SharedDepTag() blueprint.DependencyTag { return libraryDependencyTag{Kind: sharedLibraryDependency} } +// StaticDepTag returns the dependency tag for any C++ static libraries. func StaticDepTag() blueprint.DependencyTag { return libraryDependencyTag{Kind: staticLibraryDependency} } +// HeaderDepTag returns the dependency tag for any C++ "header-only" libraries. func HeaderDepTag() blueprint.DependencyTag { return libraryDependencyTag{Kind: headerLibraryDependency} } +// SharedLibraryInfo is a provider to propagate information about a shared C++ library. type SharedLibraryInfo struct { SharedLibrary android.Path UnstrippedSharedLibrary android.Path @@ -80,22 +88,30 @@ type SharedLibraryInfo struct { var SharedLibraryInfoProvider = blueprint.NewProvider(SharedLibraryInfo{}) -type SharedLibraryImplementationStubsInfo struct { - SharedLibraryStubsInfos []SharedLibraryStubsInfo - - IsLLNDK bool -} - -var SharedLibraryImplementationStubsInfoProvider = blueprint.NewProvider(SharedLibraryImplementationStubsInfo{}) - -type SharedLibraryStubsInfo struct { +// SharedStubLibrary is a struct containing information about a stub shared library. +// Stub libraries are used for cross-APEX dependencies; when a library is to depend on a shared +// library in another APEX, it must depend on the stub version of that library. +type SharedStubLibrary struct { + // The version of the stub (corresponding to the stable version of the shared library being + // stubbed). Version string SharedLibraryInfo SharedLibraryInfo FlagExporterInfo FlagExporterInfo } -var SharedLibraryStubsInfoProvider = blueprint.NewProvider(SharedLibraryStubsInfo{}) +// SharedLibraryStubsInfo is a provider to propagate information about all shared library stubs +// which are dependencies of a library. +// Stub libraries are used for cross-APEX dependencies; when a library is to depend on a shared +// library in another APEX, it must depend on the stub version of that library. +type SharedLibraryStubsInfo struct { + SharedStubLibraries []SharedStubLibrary + + IsLLNDK bool +} + +var SharedLibraryStubsProvider = blueprint.NewProvider(SharedLibraryStubsInfo{}) +// StaticLibraryInfo is a provider to propagate information about a static C++ library. type StaticLibraryInfo struct { StaticLibrary android.Path Objects Objects @@ -109,10 +125,12 @@ type StaticLibraryInfo struct { var StaticLibraryInfoProvider = blueprint.NewProvider(StaticLibraryInfo{}) +// FlagExporterInfo is a provider to propagate transitive library information +// pertaining to exported include paths and flags. type FlagExporterInfo struct { - IncludeDirs android.Paths - SystemIncludeDirs android.Paths - Flags []string + IncludeDirs android.Paths // Include directories to be included with -I + SystemIncludeDirs android.Paths // System include directories to be included with -isystem + Flags []string // Exported raw flags. Deps android.Paths GeneratedHeaders android.Paths } diff --git a/cc/linker.go b/cc/linker.go index 9d4a643d2..7bc4105c9 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -174,6 +174,15 @@ type BaseLinkerProperties struct { // variants. Shared_libs []string } + Apex struct { + // list of shared libs that should not be used to build the apex variant of + // the C/C++ module. + Exclude_shared_libs []string + + // list of static libs that should not be used to build the apex ramdisk + // variant of the C/C++ module. + Exclude_static_libs []string + } } // make android::build:GetBuildNumber() available containing the build ID. @@ -240,6 +249,16 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { deps.StaticLibs = removeListFromList(deps.StaticLibs, linker.Properties.Exclude_static_libs) deps.WholeStaticLibs = removeListFromList(deps.WholeStaticLibs, linker.Properties.Exclude_static_libs) + // Record the libraries that need to be excluded when building for APEX. Unlike other + // target.*.exclude_* properties, SharedLibs and StaticLibs are not modified here because + // this module hasn't yet passed the apexMutator. Therefore, we can't tell whether this is + // an apex variant of not. Record the exclude list in the deps struct for now. The info is + // used to mark the dependency tag when adding dependencies to the deps. Then inside + // GenerateAndroidBuildActions, the marked dependencies are ignored (i.e. not used) for APEX + // variants. + deps.ExcludeLibsForApex = append(deps.ExcludeLibsForApex, linker.Properties.Target.Apex.Exclude_shared_libs...) + deps.ExcludeLibsForApex = append(deps.ExcludeLibsForApex, linker.Properties.Target.Apex.Exclude_static_libs...) + if Bool(linker.Properties.Use_version_lib) { deps.WholeStaticLibs = append(deps.WholeStaticLibs, "libbuildversion") } diff --git a/cc/snapshot_prebuilt.go b/cc/snapshot_prebuilt.go new file mode 100644 index 000000000..b291bd0d3 --- /dev/null +++ b/cc/snapshot_prebuilt.go @@ -0,0 +1,844 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cc + +// This file defines snapshot prebuilt modules, e.g. vendor snapshot and recovery snapshot. Such +// snapshot modules will override original source modules with setting BOARD_VNDK_VERSION, with +// snapshot mutators and snapshot information maps which are also defined in this file. + +import ( + "strings" + "sync" + + "android/soong/android" +) + +// Defines the specifics of different images to which the snapshot process is applicable, e.g., +// vendor, recovery, ramdisk. +type snapshotImage interface { + // Used to register callbacks with the build system. + init() + + // Function that returns true if the module is included in this image. + // Using a function return instead of a value to prevent early + // evalution of a function that may be not be defined. + inImage(m *Module) func() bool + + // Returns the value of the "available" property for a given module for + // and snapshot, e.g., "vendor_available", "recovery_available", etc. + // or nil if the property is not defined. + available(m *Module) *bool + + // Returns true if a dir under source tree is an SoC-owned proprietary + // directory, such as device/, vendor/, etc. + // + // For a given snapshot (e.g., vendor, recovery, etc.) if + // isProprietaryPath(dir) returns true, then the module in dir will be + // built from sources. + isProprietaryPath(dir string) bool + + // Whether to include VNDK in the snapshot for this image. + includeVndk() bool + + // Whether a given module has been explicitly excluded from the + // snapshot, e.g., using the exclude_from_vendor_snapshot or + // exclude_from_recovery_snapshot properties. + excludeFromSnapshot(m *Module) bool +} + +type vendorSnapshotImage struct{} +type recoverySnapshotImage struct{} + +func (vendorSnapshotImage) init() { + android.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton) + android.RegisterModuleType("vendor_snapshot_shared", VendorSnapshotSharedFactory) + android.RegisterModuleType("vendor_snapshot_static", VendorSnapshotStaticFactory) + android.RegisterModuleType("vendor_snapshot_header", VendorSnapshotHeaderFactory) + android.RegisterModuleType("vendor_snapshot_binary", VendorSnapshotBinaryFactory) + android.RegisterModuleType("vendor_snapshot_object", VendorSnapshotObjectFactory) +} + +func (vendorSnapshotImage) inImage(m *Module) func() bool { + return m.inVendor +} + +func (vendorSnapshotImage) available(m *Module) *bool { + return m.VendorProperties.Vendor_available +} + +func (vendorSnapshotImage) isProprietaryPath(dir string) bool { + return isVendorProprietaryPath(dir) +} + +// vendor snapshot includes static/header libraries with vndk: {enabled: true}. +func (vendorSnapshotImage) includeVndk() bool { + return true +} + +func (vendorSnapshotImage) excludeFromSnapshot(m *Module) bool { + return m.ExcludeFromVendorSnapshot() +} + +func (recoverySnapshotImage) init() { + android.RegisterSingletonType("recovery-snapshot", RecoverySnapshotSingleton) + android.RegisterModuleType("recovery_snapshot_shared", RecoverySnapshotSharedFactory) + android.RegisterModuleType("recovery_snapshot_static", RecoverySnapshotStaticFactory) + android.RegisterModuleType("recovery_snapshot_header", RecoverySnapshotHeaderFactory) + android.RegisterModuleType("recovery_snapshot_binary", RecoverySnapshotBinaryFactory) + android.RegisterModuleType("recovery_snapshot_object", RecoverySnapshotObjectFactory) +} + +func (recoverySnapshotImage) inImage(m *Module) func() bool { + return m.InRecovery +} + +func (recoverySnapshotImage) available(m *Module) *bool { + return m.Properties.Recovery_available +} + +func (recoverySnapshotImage) isProprietaryPath(dir string) bool { + return isRecoveryProprietaryPath(dir) +} + +// recovery snapshot does NOT treat vndk specially. +func (recoverySnapshotImage) includeVndk() bool { + return false +} + +func (recoverySnapshotImage) excludeFromSnapshot(m *Module) bool { + return m.ExcludeFromRecoverySnapshot() +} + +var vendorSnapshotImageSingleton vendorSnapshotImage +var recoverySnapshotImageSingleton recoverySnapshotImage + +func init() { + vendorSnapshotImageSingleton.init() + recoverySnapshotImageSingleton.init() +} + +const ( + vendorSnapshotHeaderSuffix = ".vendor_header." + vendorSnapshotSharedSuffix = ".vendor_shared." + vendorSnapshotStaticSuffix = ".vendor_static." + vendorSnapshotBinarySuffix = ".vendor_binary." + vendorSnapshotObjectSuffix = ".vendor_object." +) + +const ( + recoverySnapshotHeaderSuffix = ".recovery_header." + recoverySnapshotSharedSuffix = ".recovery_shared." + recoverySnapshotStaticSuffix = ".recovery_static." + recoverySnapshotBinarySuffix = ".recovery_binary." + recoverySnapshotObjectSuffix = ".recovery_object." +) + +var ( + vendorSnapshotsLock sync.Mutex + vendorSuffixModulesKey = android.NewOnceKey("vendorSuffixModules") + vendorSnapshotHeaderLibsKey = android.NewOnceKey("vendorSnapshotHeaderLibs") + vendorSnapshotStaticLibsKey = android.NewOnceKey("vendorSnapshotStaticLibs") + vendorSnapshotSharedLibsKey = android.NewOnceKey("vendorSnapshotSharedLibs") + vendorSnapshotBinariesKey = android.NewOnceKey("vendorSnapshotBinaries") + vendorSnapshotObjectsKey = android.NewOnceKey("vendorSnapshotObjects") +) + +// vendorSuffixModules holds names of modules whose vendor variants should have the vendor suffix. +// This is determined by source modules, and then this will be used when exporting snapshot modules +// to Makefile. +// +// For example, if libbase has "vendor_available: true", the name of core variant will be "libbase" +// while the name of vendor variant will be "libbase.vendor". In such cases, the vendor snapshot of +// "libbase" should be exported with the name "libbase.vendor". +// +// Refer to VendorSnapshotSourceMutator and makeLibName which use this. +func vendorSuffixModules(config android.Config) map[string]bool { + return config.Once(vendorSuffixModulesKey, func() interface{} { + return make(map[string]bool) + }).(map[string]bool) +} + +// these are vendor snapshot maps holding names of vendor snapshot modules +func vendorSnapshotHeaderLibs(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotHeaderLibsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotSharedLibs(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotSharedLibsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotStaticLibs(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotStaticLibsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotBinaries(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotBinariesKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +func vendorSnapshotObjects(config android.Config) *snapshotMap { + return config.Once(vendorSnapshotObjectsKey, func() interface{} { + return newSnapshotMap() + }).(*snapshotMap) +} + +type baseSnapshotDecoratorProperties struct { + // snapshot version. + Version string + + // Target arch name of the snapshot (e.g. 'arm64' for variant 'aosp_arm64') + Target_arch string +} + +// baseSnapshotDecorator provides common basic functions for all snapshot modules, such as snapshot +// version, snapshot arch, etc. It also adds a special suffix to Soong module name, so it doesn't +// collide with source modules. e.g. the following example module, +// +// vendor_snapshot_static { +// name: "libbase", +// arch: "arm64", +// version: 30, +// ... +// } +// +// will be seen as "libbase.vendor_static.30.arm64" by Soong. +type baseSnapshotDecorator struct { + baseProperties baseSnapshotDecoratorProperties + moduleSuffix string +} + +func (p *baseSnapshotDecorator) Name(name string) string { + return name + p.NameSuffix() +} + +func (p *baseSnapshotDecorator) NameSuffix() string { + versionSuffix := p.version() + if p.arch() != "" { + versionSuffix += "." + p.arch() + } + + return p.moduleSuffix + versionSuffix +} + +func (p *baseSnapshotDecorator) version() string { + return p.baseProperties.Version +} + +func (p *baseSnapshotDecorator) arch() string { + return p.baseProperties.Target_arch +} + +func (p *baseSnapshotDecorator) isSnapshotPrebuilt() bool { + return true +} + +// Call this with a module suffix after creating a snapshot module, such as +// vendorSnapshotSharedSuffix, recoverySnapshotBinarySuffix, etc. +func (p *baseSnapshotDecorator) init(m *Module, suffix string) { + p.moduleSuffix = suffix + m.AddProperties(&p.baseProperties) + android.AddLoadHook(m, func(ctx android.LoadHookContext) { + vendorSnapshotLoadHook(ctx, p) + }) +} + +// vendorSnapshotLoadHook disables snapshots if it's not BOARD_VNDK_VERSION. +// As vendor snapshot is only for vendor, such modules won't be used at all. +func vendorSnapshotLoadHook(ctx android.LoadHookContext, p *baseSnapshotDecorator) { + if p.version() != ctx.DeviceConfig().VndkVersion() { + ctx.Module().Disable() + return + } +} + +// +// Module definitions for snapshots of libraries (shared, static, header). +// +// Modules (vendor|recovery)_snapshot_(shared|static|header) are defined here. Shared libraries and +// static libraries have their prebuilt library files (.so for shared, .a for static) as their src, +// which can be installed or linked against. Also they export flags needed when linked, such as +// include directories, c flags, sanitize dependency information, etc. +// +// These modules are auto-generated by development/vendor_snapshot/update.py. +type snapshotLibraryProperties struct { + // Prebuilt file for each arch. + Src *string `android:"arch_variant"` + + // list of directories that will be added to the include path (using -I). + Export_include_dirs []string `android:"arch_variant"` + + // list of directories that will be added to the system path (using -isystem). + Export_system_include_dirs []string `android:"arch_variant"` + + // list of flags that will be used for any module that links against this module. + Export_flags []string `android:"arch_variant"` + + // Whether this prebuilt needs to depend on sanitize ubsan runtime or not. + Sanitize_ubsan_dep *bool `android:"arch_variant"` + + // Whether this prebuilt needs to depend on sanitize minimal runtime or not. + Sanitize_minimal_dep *bool `android:"arch_variant"` +} + +type snapshotSanitizer interface { + isSanitizerEnabled(t sanitizerType) bool + setSanitizerVariation(t sanitizerType, enabled bool) +} + +type snapshotLibraryDecorator struct { + baseSnapshotDecorator + *libraryDecorator + properties snapshotLibraryProperties + sanitizerProperties struct { + CfiEnabled bool `blueprint:"mutated"` + + // Library flags for cfi variant. + Cfi snapshotLibraryProperties `android:"arch_variant"` + } + androidMkVendorSuffix bool +} + +func (p *snapshotLibraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { + p.libraryDecorator.libName = strings.TrimSuffix(ctx.ModuleName(), p.NameSuffix()) + return p.libraryDecorator.linkerFlags(ctx, flags) +} + +func (p *snapshotLibraryDecorator) matchesWithDevice(config android.DeviceConfig) bool { + arches := config.Arches() + if len(arches) == 0 || arches[0].ArchType.String() != p.arch() { + return false + } + if !p.header() && p.properties.Src == nil { + return false + } + return true +} + +// cc modules' link functions are to link compiled objects into final binaries. +// As snapshots are prebuilts, this just returns the prebuilt binary after doing things which are +// done by normal library decorator, e.g. exporting flags. +func (p *snapshotLibraryDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { + m := ctx.Module().(*Module) + p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] + + if p.header() { + return p.libraryDecorator.link(ctx, flags, deps, objs) + } + + if p.sanitizerProperties.CfiEnabled { + p.properties = p.sanitizerProperties.Cfi + } + + if !p.matchesWithDevice(ctx.DeviceConfig()) { + return nil + } + + p.libraryDecorator.reexportDirs(android.PathsForModuleSrc(ctx, p.properties.Export_include_dirs)...) + p.libraryDecorator.reexportSystemDirs(android.PathsForModuleSrc(ctx, p.properties.Export_system_include_dirs)...) + p.libraryDecorator.reexportFlags(p.properties.Export_flags...) + + in := android.PathForModuleSrc(ctx, *p.properties.Src) + p.unstrippedOutputFile = in + + if p.shared() { + libName := in.Base() + builderFlags := flagsToBuilderFlags(flags) + + // Optimize out relinking against shared libraries whose interface hasn't changed by + // depending on a table of contents file instead of the library itself. + tocFile := android.PathForModuleOut(ctx, libName+".toc") + p.tocFile = android.OptionalPathForPath(tocFile) + transformSharedObjectToToc(ctx, in, tocFile, builderFlags) + + ctx.SetProvider(SharedLibraryInfoProvider, SharedLibraryInfo{ + SharedLibrary: in, + UnstrippedSharedLibrary: p.unstrippedOutputFile, + + TableOfContents: p.tocFile, + }) + } + + if p.static() { + depSet := android.NewDepSetBuilder(android.TOPOLOGICAL).Direct(in).Build() + ctx.SetProvider(StaticLibraryInfoProvider, StaticLibraryInfo{ + StaticLibrary: in, + + TransitiveStaticLibrariesForOrdering: depSet, + }) + } + + p.libraryDecorator.flagExporter.setProvider(ctx) + + return in +} + +func (p *snapshotLibraryDecorator) install(ctx ModuleContext, file android.Path) { + if p.matchesWithDevice(ctx.DeviceConfig()) && (p.shared() || p.static()) { + p.baseInstaller.install(ctx, file) + } +} + +func (p *snapshotLibraryDecorator) nativeCoverage() bool { + return false +} + +func (p *snapshotLibraryDecorator) isSanitizerEnabled(t sanitizerType) bool { + switch t { + case cfi: + return p.sanitizerProperties.Cfi.Src != nil + default: + return false + } +} + +func (p *snapshotLibraryDecorator) setSanitizerVariation(t sanitizerType, enabled bool) { + if !enabled { + return + } + switch t { + case cfi: + p.sanitizerProperties.CfiEnabled = true + default: + return + } +} + +func snapshotLibraryFactory(suffix string) (*Module, *snapshotLibraryDecorator) { + module, library := NewLibrary(android.DeviceSupported) + + module.stl = nil + module.sanitize = nil + library.disableStripping() + + prebuilt := &snapshotLibraryDecorator{ + libraryDecorator: library, + } + + prebuilt.baseLinker.Properties.No_libcrt = BoolPtr(true) + prebuilt.baseLinker.Properties.Nocrt = BoolPtr(true) + + // Prevent default system libs (libc, libm, and libdl) from being linked + if prebuilt.baseLinker.Properties.System_shared_libs == nil { + prebuilt.baseLinker.Properties.System_shared_libs = []string{} + } + + module.compiler = nil + module.linker = prebuilt + module.installer = prebuilt + + prebuilt.init(module, suffix) + module.AddProperties( + &prebuilt.properties, + &prebuilt.sanitizerProperties, + ) + + return module, prebuilt +} + +// vendor_snapshot_shared is a special prebuilt shared library which is auto-generated by +// development/vendor_snapshot/update.py. As a part of vendor snapshot, vendor_snapshot_shared +// overrides the vendor variant of the cc shared library with the same name, if BOARD_VNDK_VERSION +// is set. +func VendorSnapshotSharedFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(vendorSnapshotSharedSuffix) + prebuilt.libraryDecorator.BuildOnlyShared() + return module.Init() +} + +// recovery_snapshot_shared is a special prebuilt shared library which is auto-generated by +// development/vendor_snapshot/update.py. As a part of recovery snapshot, recovery_snapshot_shared +// overrides the recovery variant of the cc shared library with the same name, if BOARD_VNDK_VERSION +// is set. +func RecoverySnapshotSharedFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(recoverySnapshotSharedSuffix) + prebuilt.libraryDecorator.BuildOnlyShared() + return module.Init() +} + +// vendor_snapshot_static is a special prebuilt static library which is auto-generated by +// development/vendor_snapshot/update.py. As a part of vendor snapshot, vendor_snapshot_static +// overrides the vendor variant of the cc static library with the same name, if BOARD_VNDK_VERSION +// is set. +func VendorSnapshotStaticFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(vendorSnapshotStaticSuffix) + prebuilt.libraryDecorator.BuildOnlyStatic() + return module.Init() +} + +// recovery_snapshot_static is a special prebuilt static library which is auto-generated by +// development/vendor_snapshot/update.py. As a part of recovery snapshot, recovery_snapshot_static +// overrides the recovery variant of the cc static library with the same name, if BOARD_VNDK_VERSION +// is set. +func RecoverySnapshotStaticFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(recoverySnapshotStaticSuffix) + prebuilt.libraryDecorator.BuildOnlyStatic() + return module.Init() +} + +// vendor_snapshot_header is a special header library which is auto-generated by +// development/vendor_snapshot/update.py. As a part of vendor snapshot, vendor_snapshot_header +// overrides the vendor variant of the cc header library with the same name, if BOARD_VNDK_VERSION +// is set. +func VendorSnapshotHeaderFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(vendorSnapshotHeaderSuffix) + prebuilt.libraryDecorator.HeaderOnly() + return module.Init() +} + +// recovery_snapshot_header is a special header library which is auto-generated by +// development/vendor_snapshot/update.py. As a part of recovery snapshot, recovery_snapshot_header +// overrides the recovery variant of the cc header library with the same name, if BOARD_VNDK_VERSION +// is set. +func RecoverySnapshotHeaderFactory() android.Module { + module, prebuilt := snapshotLibraryFactory(recoverySnapshotHeaderSuffix) + prebuilt.libraryDecorator.HeaderOnly() + return module.Init() +} + +var _ snapshotSanitizer = (*snapshotLibraryDecorator)(nil) + +// +// Module definitions for snapshots of executable binaries. +// +// Modules (vendor|recovery)_snapshot_binary are defined here. They have their prebuilt executable +// binaries (e.g. toybox, sh) as their src, which can be installed. +// +// These modules are auto-generated by development/vendor_snapshot/update.py. +type snapshotBinaryProperties struct { + // Prebuilt file for each arch. + Src *string `android:"arch_variant"` +} + +type snapshotBinaryDecorator struct { + baseSnapshotDecorator + *binaryDecorator + properties snapshotBinaryProperties + androidMkVendorSuffix bool +} + +func (p *snapshotBinaryDecorator) matchesWithDevice(config android.DeviceConfig) bool { + if config.DeviceArch() != p.arch() { + return false + } + if p.properties.Src == nil { + return false + } + return true +} + +// cc modules' link functions are to link compiled objects into final binaries. +// As snapshots are prebuilts, this just returns the prebuilt binary +func (p *snapshotBinaryDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { + if !p.matchesWithDevice(ctx.DeviceConfig()) { + return nil + } + + in := android.PathForModuleSrc(ctx, *p.properties.Src) + p.unstrippedOutputFile = in + binName := in.Base() + + m := ctx.Module().(*Module) + p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] + + // use cpExecutable to make it executable + outputFile := android.PathForModuleOut(ctx, binName) + ctx.Build(pctx, android.BuildParams{ + Rule: android.CpExecutable, + Description: "prebuilt", + Output: outputFile, + Input: in, + }) + + return outputFile +} + +func (p *snapshotBinaryDecorator) nativeCoverage() bool { + return false +} + +// vendor_snapshot_binary is a special prebuilt executable binary which is auto-generated by +// development/vendor_snapshot/update.py. As a part of vendor snapshot, vendor_snapshot_binary +// overrides the vendor variant of the cc binary with the same name, if BOARD_VNDK_VERSION is set. +func VendorSnapshotBinaryFactory() android.Module { + return snapshotBinaryFactory(vendorSnapshotBinarySuffix) +} + +// recovery_snapshot_binary is a special prebuilt executable binary which is auto-generated by +// development/vendor_snapshot/update.py. As a part of recovery snapshot, recovery_snapshot_binary +// overrides the recovery variant of the cc binary with the same name, if BOARD_VNDK_VERSION is set. +func RecoverySnapshotBinaryFactory() android.Module { + return snapshotBinaryFactory(recoverySnapshotBinarySuffix) +} + +func snapshotBinaryFactory(suffix string) android.Module { + module, binary := NewBinary(android.DeviceSupported) + binary.baseLinker.Properties.No_libcrt = BoolPtr(true) + binary.baseLinker.Properties.Nocrt = BoolPtr(true) + + // Prevent default system libs (libc, libm, and libdl) from being linked + if binary.baseLinker.Properties.System_shared_libs == nil { + binary.baseLinker.Properties.System_shared_libs = []string{} + } + + prebuilt := &snapshotBinaryDecorator{ + binaryDecorator: binary, + } + + module.compiler = nil + module.sanitize = nil + module.stl = nil + module.linker = prebuilt + + prebuilt.init(module, suffix) + module.AddProperties(&prebuilt.properties) + return module.Init() +} + +// +// Module definitions for snapshots of object files (*.o). +// +// Modules (vendor|recovery)_snapshot_object are defined here. They have their prebuilt object +// files (*.o) as their src. +// +// These modules are auto-generated by development/vendor_snapshot/update.py. +type vendorSnapshotObjectProperties struct { + // Prebuilt file for each arch. + Src *string `android:"arch_variant"` +} + +type snapshotObjectLinker struct { + baseSnapshotDecorator + objectLinker + properties vendorSnapshotObjectProperties + androidMkVendorSuffix bool +} + +func (p *snapshotObjectLinker) matchesWithDevice(config android.DeviceConfig) bool { + if config.DeviceArch() != p.arch() { + return false + } + if p.properties.Src == nil { + return false + } + return true +} + +// cc modules' link functions are to link compiled objects into final binaries. +// As snapshots are prebuilts, this just returns the prebuilt binary +func (p *snapshotObjectLinker) link(ctx ModuleContext, flags Flags, deps PathDeps, objs Objects) android.Path { + if !p.matchesWithDevice(ctx.DeviceConfig()) { + return nil + } + + m := ctx.Module().(*Module) + p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] + + return android.PathForModuleSrc(ctx, *p.properties.Src) +} + +func (p *snapshotObjectLinker) nativeCoverage() bool { + return false +} + +// vendor_snapshot_object is a special prebuilt compiled object file which is auto-generated by +// development/vendor_snapshot/update.py. As a part of vendor snapshot, vendor_snapshot_object +// overrides the vendor variant of the cc object with the same name, if BOARD_VNDK_VERSION is set. +func VendorSnapshotObjectFactory() android.Module { + module := newObject() + + prebuilt := &snapshotObjectLinker{ + objectLinker: objectLinker{ + baseLinker: NewBaseLinker(nil), + }, + } + module.linker = prebuilt + + prebuilt.init(module, vendorSnapshotObjectSuffix) + module.AddProperties(&prebuilt.properties) + return module.Init() +} + +// recovery_snapshot_object is a special prebuilt compiled object file which is auto-generated by +// development/vendor_snapshot/update.py. As a part of recovery snapshot, recovery_snapshot_object +// overrides the recovery variant of the cc object with the same name, if BOARD_VNDK_VERSION is set. +func RecoverySnapshotObjectFactory() android.Module { + module := newObject() + + prebuilt := &snapshotObjectLinker{ + objectLinker: objectLinker{ + baseLinker: NewBaseLinker(nil), + }, + } + module.linker = prebuilt + + prebuilt.init(module, recoverySnapshotObjectSuffix) + module.AddProperties(&prebuilt.properties) + return module.Init() +} + +type snapshotInterface interface { + matchesWithDevice(config android.DeviceConfig) bool +} + +var _ snapshotInterface = (*vndkPrebuiltLibraryDecorator)(nil) +var _ snapshotInterface = (*snapshotLibraryDecorator)(nil) +var _ snapshotInterface = (*snapshotBinaryDecorator)(nil) +var _ snapshotInterface = (*snapshotObjectLinker)(nil) + +// +// Mutators that helps vendor snapshot modules override source modules. +// + +// VendorSnapshotMutator gathers all snapshots for vendor, and disable all snapshots which don't +// match with device, e.g. +// - snapshot version is different with BOARD_VNDK_VERSION +// - snapshot arch is different with device's arch (e.g. arm vs x86) +// +// This also handles vndk_prebuilt_shared, except for they won't be disabled in any cases, given +// that any versions of VNDK might be packed into vndk APEX. +// +// TODO(b/145966707): remove mutator and utilize android.Prebuilt to override source modules +func VendorSnapshotMutator(ctx android.BottomUpMutatorContext) { + vndkVersion := ctx.DeviceConfig().VndkVersion() + // don't need snapshot if current + if vndkVersion == "current" || vndkVersion == "" { + return + } + + module, ok := ctx.Module().(*Module) + if !ok || !module.Enabled() || module.VndkVersion() != vndkVersion { + return + } + + if !module.isSnapshotPrebuilt() { + return + } + + // isSnapshotPrebuilt ensures snapshotInterface + if !module.linker.(snapshotInterface).matchesWithDevice(ctx.DeviceConfig()) { + // Disable unnecessary snapshot module, but do not disable + // vndk_prebuilt_shared because they might be packed into vndk APEX + if !module.IsVndk() { + module.Disable() + } + return + } + + var snapshotMap *snapshotMap + + if lib, ok := module.linker.(libraryInterface); ok { + if lib.static() { + snapshotMap = vendorSnapshotStaticLibs(ctx.Config()) + } else if lib.shared() { + snapshotMap = vendorSnapshotSharedLibs(ctx.Config()) + } else { + // header + snapshotMap = vendorSnapshotHeaderLibs(ctx.Config()) + } + } else if _, ok := module.linker.(*snapshotBinaryDecorator); ok { + snapshotMap = vendorSnapshotBinaries(ctx.Config()) + } else if _, ok := module.linker.(*snapshotObjectLinker); ok { + snapshotMap = vendorSnapshotObjects(ctx.Config()) + } else { + return + } + + vendorSnapshotsLock.Lock() + defer vendorSnapshotsLock.Unlock() + snapshotMap.add(module.BaseModuleName(), ctx.Arch().ArchType, ctx.ModuleName()) +} + +// VendorSnapshotSourceMutator disables source modules which have corresponding snapshots. +func VendorSnapshotSourceMutator(ctx android.BottomUpMutatorContext) { + if !ctx.Device() { + return + } + + vndkVersion := ctx.DeviceConfig().VndkVersion() + // don't need snapshot if current + if vndkVersion == "current" || vndkVersion == "" { + return + } + + module, ok := ctx.Module().(*Module) + if !ok { + return + } + + // vendor suffix should be added to snapshots if the source module isn't vendor: true. + if !module.SocSpecific() { + // But we can't just check SocSpecific() since we already passed the image mutator. + // Check ramdisk and recovery to see if we are real "vendor: true" module. + ramdisk_available := module.InRamdisk() && !module.OnlyInRamdisk() + vendor_ramdisk_available := module.InVendorRamdisk() && !module.OnlyInVendorRamdisk() + recovery_available := module.InRecovery() && !module.OnlyInRecovery() + + if !ramdisk_available && !recovery_available && !vendor_ramdisk_available { + vendorSnapshotsLock.Lock() + defer vendorSnapshotsLock.Unlock() + + vendorSuffixModules(ctx.Config())[ctx.ModuleName()] = true + } + } + + if module.isSnapshotPrebuilt() || module.VndkVersion() != ctx.DeviceConfig().VndkVersion() { + // only non-snapshot modules with BOARD_VNDK_VERSION + return + } + + // .. and also filter out llndk library + if module.isLlndk(ctx.Config()) { + return + } + + var snapshotMap *snapshotMap + + if lib, ok := module.linker.(libraryInterface); ok { + if lib.static() { + snapshotMap = vendorSnapshotStaticLibs(ctx.Config()) + } else if lib.shared() { + snapshotMap = vendorSnapshotSharedLibs(ctx.Config()) + } else { + // header + snapshotMap = vendorSnapshotHeaderLibs(ctx.Config()) + } + } else if module.binary() { + snapshotMap = vendorSnapshotBinaries(ctx.Config()) + } else if module.object() { + snapshotMap = vendorSnapshotObjects(ctx.Config()) + } else { + return + } + + if _, ok := snapshotMap.get(ctx.ModuleName(), ctx.Arch().ArchType); !ok { + // Corresponding snapshot doesn't exist + return + } + + // Disables source modules if corresponding snapshot exists. + if lib, ok := module.linker.(libraryInterface); ok && lib.buildStatic() && lib.buildShared() { + // But do not disable because the shared variant depends on the static variant. + module.SkipInstall() + module.Properties.HideFromMake = true + } else { + module.Disable() + } +} diff --git a/cc/snapshot_utils.go b/cc/snapshot_utils.go index a3d52e621..e841a547b 100644 --- a/cc/snapshot_utils.go +++ b/cc/snapshot_utils.go @@ -13,6 +13,8 @@ // limitations under the License. package cc +// This file contains utility types and functions for VNDK / vendor snapshot. + import ( "android/soong/android" ) @@ -21,15 +23,24 @@ var ( headerExts = []string{".h", ".hh", ".hpp", ".hxx", ".h++", ".inl", ".inc", ".ipp", ".h.generic"} ) +// snapshotLibraryInterface is an interface for libraries captured to VNDK / vendor snapshots. type snapshotLibraryInterface interface { libraryInterface + + // collectHeadersForSnapshot is called in GenerateAndroidBuildActions for snapshot aware + // modules (See isSnapshotAware below). + // This function should gather all headers needed for snapshot. collectHeadersForSnapshot(ctx android.ModuleContext) + + // snapshotHeaders should return collected headers by collectHeadersForSnapshot. + // Calling snapshotHeaders before collectHeadersForSnapshot is an error. snapshotHeaders() android.Paths } var _ snapshotLibraryInterface = (*prebuiltLibraryLinker)(nil) var _ snapshotLibraryInterface = (*libraryDecorator)(nil) +// snapshotMap is a helper wrapper to a map from base module name to snapshot module name. type snapshotMap struct { snapshots map[string]string } @@ -57,43 +68,14 @@ func (s *snapshotMap) get(name string, arch android.ArchType) (snapshot string, return snapshot, found } -func isSnapshotAware(ctx android.ModuleContext, m *Module, apexInfo android.ApexInfo) bool { - if _, _, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m, apexInfo); ok { +// shouldCollectHeadersForSnapshot determines if the module is a possible candidate for snapshot. +// If it's true, collectHeadersForSnapshot will be called in GenerateAndroidBuildActions. +func shouldCollectHeadersForSnapshot(ctx android.ModuleContext, m *Module, apexInfo android.ApexInfo) bool { + if _, _, ok := isVndkSnapshotAware(ctx.DeviceConfig(), m, apexInfo); ok { return ctx.Config().VndkSnapshotBuildArtifacts() - } else if isVendorSnapshotModule(m, isVendorProprietaryPath(ctx.ModuleDir()), apexInfo) || - isRecoverySnapshotModule(m, isVendorProprietaryPath(ctx.ModuleDir()), apexInfo) { + } else if isVendorSnapshotAware(m, isVendorProprietaryPath(ctx.ModuleDir()), apexInfo) || + isRecoverySnapshotAware(m, isRecoveryProprietaryPath(ctx.ModuleDir()), apexInfo) { return true } return false } - -func copyFile(ctx android.SingletonContext, path android.Path, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: path, - Output: outPath, - Description: "Cp " + out, - Args: map[string]string{ - "cpFlags": "-f -L", - }, - }) - return outPath -} - -func combineNotices(ctx android.SingletonContext, paths android.Paths, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cat, - Inputs: paths, - Output: outPath, - Description: "combine notices for " + out, - }) - return outPath -} - -func writeStringToFile(ctx android.SingletonContext, content, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - android.WriteFileRule(ctx, outPath, content) - return outPath -} diff --git a/cc/test.go b/cc/test.go index 37726914b..a9be6f9e4 100644 --- a/cc/test.go +++ b/cc/test.go @@ -29,11 +29,6 @@ type TestProperties struct { // if set, use the isolated gtest runner. Defaults to false. Isolated *bool - - // List of APEXes that this module tests. The module has access to - // the private part of the listed APEXes even when it is not included in the - // APEXes. - Test_for []string } // Test option struct. @@ -241,10 +236,6 @@ func (test *testDecorator) gtest() bool { return BoolDefault(test.Properties.Gtest, true) } -func (test *testDecorator) testFor() []string { - return test.Properties.Test_for -} - func (test *testDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { if !test.gtest() { return flags diff --git a/cc/testing.go b/cc/testing.go index 95a93a0ed..fc5b030c7 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -29,6 +29,7 @@ func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.RegisterModuleType("toolchain_library", ToolchainLibraryFactory) ctx.RegisterModuleType("llndk_library", LlndkLibraryFactory) + ctx.RegisterModuleType("cc_benchmark", BenchmarkFactory) ctx.RegisterModuleType("cc_object", ObjectFactory) ctx.RegisterModuleType("cc_genrule", genRuleFactory) ctx.RegisterModuleType("ndk_prebuilt_shared_stl", NdkPrebuiltSharedStlFactory) @@ -334,7 +335,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { }, apex_available: [ "//apex_available:platform", - "myapex" + "//apex_available:anyapex", ], } cc_library { @@ -437,6 +438,13 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { ndk_prebuilt_shared_stl { name: "ndk_libc++_shared", } + + cc_library_static { + name: "libgoogle-benchmark", + sdk_version: "current", + stl: "none", + system_shared_libs: [], + } ` supportLinuxBionic := false diff --git a/cc/util.go b/cc/util.go index 40374bff7..1220d8432 100644 --- a/cc/util.go +++ b/cc/util.go @@ -125,3 +125,52 @@ func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) stri return "mkdir -p " + dir + " && " + "ln -sf " + target + " " + filepath.Join(dir, linkName) } + +func copyFileRule(ctx android.SingletonContext, path android.Path, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: path, + Output: outPath, + Description: "copy " + path.String() + " -> " + out, + Args: map[string]string{ + "cpFlags": "-f -L", + }, + }) + return outPath +} + +func combineNoticesRule(ctx android.SingletonContext, paths android.Paths, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cat, + Inputs: paths, + Output: outPath, + Description: "combine notices for " + out, + }) + return outPath +} + +func writeStringToFileRule(ctx android.SingletonContext, content, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + android.WriteFileRule(ctx, outPath, content) + return outPath +} + +// Dump a map to a list file as: +// +// {key1} {value1} +// {key2} {value2} +// ... +func installMapListFileRule(ctx android.SingletonContext, m map[string]string, path string) android.OutputPath { + var txtBuilder strings.Builder + for idx, k := range android.SortedStringKeys(m) { + if idx > 0 { + txtBuilder.WriteString("\n") + } + txtBuilder.WriteString(k) + txtBuilder.WriteString(" ") + txtBuilder.WriteString(m[k]) + } + return writeStringToFileRule(ctx, txtBuilder.String(), path) +} diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go index 3ef0b62b8..25960ab96 100644 --- a/cc/vendor_snapshot.go +++ b/cc/vendor_snapshot.go @@ -13,625 +13,27 @@ // limitations under the License. package cc +// This file contains singletons to capture vendor and recovery snapshot. They consist of prebuilt +// modules under AOSP so older vendor and recovery can be built with a newer system in a single +// source tree. + import ( "encoding/json" "path/filepath" "sort" "strings" - "sync" "github.com/google/blueprint/proptools" "android/soong/android" ) -// Defines the specifics of different images to which the snapshot process is -// applicable, e.g., vendor, recovery, ramdisk. -type image interface { - // Used to register callbacks with the build system. - init() - - // Function that returns true if the module is included in this image. - // Using a function return instead of a value to prevent early - // evalution of a function that may be not be defined. - inImage(m *Module) func() bool - - // Returns the value of the "available" property for a given module for - // and snapshot, e.g., "vendor_available", "recovery_available", etc. - // or nil if the property is not defined. - available(m *Module) *bool - - // Returns true if a dir under source tree is an SoC-owned proprietary - // directory, such as device/, vendor/, etc. - // - // For a given snapshot (e.g., vendor, recovery, etc.) if - // isProprietaryPath(dir) returns true, then the module in dir will be - // built from sources. - isProprietaryPath(dir string) bool - - // Whether to include VNDK in the snapshot for this image. - includeVndk() bool - - // Whether a given module has been explicitly excluded from the - // snapshot, e.g., using the exclude_from_vendor_snapshot or - // exclude_from_recovery_snapshot properties. - excludeFromSnapshot(m *Module) bool -} - -type vendorImage struct{} -type recoveryImage struct{} - -func (vendorImage) init() { - android.RegisterSingletonType( - "vendor-snapshot", VendorSnapshotSingleton) - android.RegisterModuleType( - "vendor_snapshot_shared", VendorSnapshotSharedFactory) - android.RegisterModuleType( - "vendor_snapshot_static", VendorSnapshotStaticFactory) - android.RegisterModuleType( - "vendor_snapshot_header", VendorSnapshotHeaderFactory) - android.RegisterModuleType( - "vendor_snapshot_binary", VendorSnapshotBinaryFactory) - android.RegisterModuleType( - "vendor_snapshot_object", VendorSnapshotObjectFactory) -} - -func (vendorImage) inImage(m *Module) func() bool { - return m.inVendor -} - -func (vendorImage) available(m *Module) *bool { - return m.VendorProperties.Vendor_available -} - -func (vendorImage) isProprietaryPath(dir string) bool { - return isVendorProprietaryPath(dir) -} - -func (vendorImage) includeVndk() bool { - return true -} - -func (vendorImage) excludeFromSnapshot(m *Module) bool { - return m.ExcludeFromVendorSnapshot() -} - -func (recoveryImage) init() { - android.RegisterSingletonType( - "recovery-snapshot", RecoverySnapshotSingleton) - android.RegisterModuleType( - "recovery_snapshot_shared", RecoverySnapshotSharedFactory) - android.RegisterModuleType( - "recovery_snapshot_static", RecoverySnapshotStaticFactory) - android.RegisterModuleType( - "recovery_snapshot_header", RecoverySnapshotHeaderFactory) - android.RegisterModuleType( - "recovery_snapshot_binary", RecoverySnapshotBinaryFactory) - android.RegisterModuleType( - "recovery_snapshot_object", RecoverySnapshotObjectFactory) -} - -func (recoveryImage) inImage(m *Module) func() bool { - return m.InRecovery -} - -func (recoveryImage) available(m *Module) *bool { - return m.Properties.Recovery_available -} - -func (recoveryImage) isProprietaryPath(dir string) bool { - return isRecoveryProprietaryPath(dir) -} - -func (recoveryImage) includeVndk() bool { - return false -} - -func (recoveryImage) excludeFromSnapshot(m *Module) bool { - return m.ExcludeFromRecoverySnapshot() -} - -var vendorImageSingleton vendorImage -var recoveryImageSingleton recoveryImage - -const ( - vendorSnapshotHeaderSuffix = ".vendor_header." - vendorSnapshotSharedSuffix = ".vendor_shared." - vendorSnapshotStaticSuffix = ".vendor_static." - vendorSnapshotBinarySuffix = ".vendor_binary." - vendorSnapshotObjectSuffix = ".vendor_object." -) - -const ( - recoverySnapshotHeaderSuffix = ".recovery_header." - recoverySnapshotSharedSuffix = ".recovery_shared." - recoverySnapshotStaticSuffix = ".recovery_static." - recoverySnapshotBinarySuffix = ".recovery_binary." - recoverySnapshotObjectSuffix = ".recovery_object." -) - -var ( - vendorSnapshotsLock sync.Mutex - vendorSuffixModulesKey = android.NewOnceKey("vendorSuffixModules") - vendorSnapshotHeaderLibsKey = android.NewOnceKey("vendorSnapshotHeaderLibs") - vendorSnapshotStaticLibsKey = android.NewOnceKey("vendorSnapshotStaticLibs") - vendorSnapshotSharedLibsKey = android.NewOnceKey("vendorSnapshotSharedLibs") - vendorSnapshotBinariesKey = android.NewOnceKey("vendorSnapshotBinaries") - vendorSnapshotObjectsKey = android.NewOnceKey("vendorSnapshotObjects") -) - -// vendor snapshot maps hold names of vendor snapshot modules per arch -func vendorSuffixModules(config android.Config) map[string]bool { - return config.Once(vendorSuffixModulesKey, func() interface{} { - return make(map[string]bool) - }).(map[string]bool) -} - -func vendorSnapshotHeaderLibs(config android.Config) *snapshotMap { - return config.Once(vendorSnapshotHeaderLibsKey, func() interface{} { - return newSnapshotMap() - }).(*snapshotMap) -} - -func vendorSnapshotSharedLibs(config android.Config) *snapshotMap { - return config.Once(vendorSnapshotSharedLibsKey, func() interface{} { - return newSnapshotMap() - }).(*snapshotMap) -} - -func vendorSnapshotStaticLibs(config android.Config) *snapshotMap { - return config.Once(vendorSnapshotStaticLibsKey, func() interface{} { - return newSnapshotMap() - }).(*snapshotMap) -} - -func vendorSnapshotBinaries(config android.Config) *snapshotMap { - return config.Once(vendorSnapshotBinariesKey, func() interface{} { - return newSnapshotMap() - }).(*snapshotMap) -} - -func vendorSnapshotObjects(config android.Config) *snapshotMap { - return config.Once(vendorSnapshotObjectsKey, func() interface{} { - return newSnapshotMap() - }).(*snapshotMap) -} - -type vendorSnapshotBaseProperties struct { - // snapshot version. - Version string - - // Target arch name of the snapshot (e.g. 'arm64' for variant 'aosp_arm64') - Target_arch string -} - -// vendorSnapshotModuleBase provides common basic functions for all snapshot modules. -type vendorSnapshotModuleBase struct { - baseProperties vendorSnapshotBaseProperties - moduleSuffix string -} - -func (p *vendorSnapshotModuleBase) Name(name string) string { - return name + p.NameSuffix() -} - -func (p *vendorSnapshotModuleBase) NameSuffix() string { - versionSuffix := p.version() - if p.arch() != "" { - versionSuffix += "." + p.arch() - } - - return p.moduleSuffix + versionSuffix -} - -func (p *vendorSnapshotModuleBase) version() string { - return p.baseProperties.Version -} - -func (p *vendorSnapshotModuleBase) arch() string { - return p.baseProperties.Target_arch -} - -func (p *vendorSnapshotModuleBase) isSnapshotPrebuilt() bool { - return true -} - -// Call this after creating a snapshot module with module suffix -// such as vendorSnapshotSharedSuffix -func (p *vendorSnapshotModuleBase) init(m *Module, suffix string) { - p.moduleSuffix = suffix - m.AddProperties(&p.baseProperties) - android.AddLoadHook(m, func(ctx android.LoadHookContext) { - vendorSnapshotLoadHook(ctx, p) - }) -} - -func vendorSnapshotLoadHook(ctx android.LoadHookContext, p *vendorSnapshotModuleBase) { - if p.version() != ctx.DeviceConfig().VndkVersion() { - ctx.Module().Disable() - return - } -} - -type snapshotLibraryProperties struct { - // Prebuilt file for each arch. - Src *string `android:"arch_variant"` - - // list of directories that will be added to the include path (using -I). - Export_include_dirs []string `android:"arch_variant"` - - // list of directories that will be added to the system path (using -isystem). - Export_system_include_dirs []string `android:"arch_variant"` - - // list of flags that will be used for any module that links against this module. - Export_flags []string `android:"arch_variant"` - - // Whether this prebuilt needs to depend on sanitize ubsan runtime or not. - Sanitize_ubsan_dep *bool `android:"arch_variant"` - - // Whether this prebuilt needs to depend on sanitize minimal runtime or not. - Sanitize_minimal_dep *bool `android:"arch_variant"` -} - -type snapshotSanitizer interface { - isSanitizerEnabled(t sanitizerType) bool - setSanitizerVariation(t sanitizerType, enabled bool) -} - -type snapshotLibraryDecorator struct { - vendorSnapshotModuleBase - *libraryDecorator - properties snapshotLibraryProperties - sanitizerProperties struct { - CfiEnabled bool `blueprint:"mutated"` - - // Library flags for cfi variant. - Cfi snapshotLibraryProperties `android:"arch_variant"` - } - androidMkVendorSuffix bool -} - -func (p *snapshotLibraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Flags { - p.libraryDecorator.libName = strings.TrimSuffix(ctx.ModuleName(), p.NameSuffix()) - return p.libraryDecorator.linkerFlags(ctx, flags) -} - -func (p *snapshotLibraryDecorator) matchesWithDevice(config android.DeviceConfig) bool { - arches := config.Arches() - if len(arches) == 0 || arches[0].ArchType.String() != p.arch() { - return false - } - if !p.header() && p.properties.Src == nil { - return false - } - return true -} - -func (p *snapshotLibraryDecorator) link(ctx ModuleContext, - flags Flags, deps PathDeps, objs Objects) android.Path { - m := ctx.Module().(*Module) - p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] - - if p.header() { - return p.libraryDecorator.link(ctx, flags, deps, objs) - } - - if p.sanitizerProperties.CfiEnabled { - p.properties = p.sanitizerProperties.Cfi - } - - if !p.matchesWithDevice(ctx.DeviceConfig()) { - return nil - } - - p.libraryDecorator.reexportDirs(android.PathsForModuleSrc(ctx, p.properties.Export_include_dirs)...) - p.libraryDecorator.reexportSystemDirs(android.PathsForModuleSrc(ctx, p.properties.Export_system_include_dirs)...) - p.libraryDecorator.reexportFlags(p.properties.Export_flags...) - - in := android.PathForModuleSrc(ctx, *p.properties.Src) - p.unstrippedOutputFile = in - - if p.shared() { - libName := in.Base() - builderFlags := flagsToBuilderFlags(flags) - - // Optimize out relinking against shared libraries whose interface hasn't changed by - // depending on a table of contents file instead of the library itself. - tocFile := android.PathForModuleOut(ctx, libName+".toc") - p.tocFile = android.OptionalPathForPath(tocFile) - transformSharedObjectToToc(ctx, in, tocFile, builderFlags) - - ctx.SetProvider(SharedLibraryInfoProvider, SharedLibraryInfo{ - SharedLibrary: in, - UnstrippedSharedLibrary: p.unstrippedOutputFile, - - TableOfContents: p.tocFile, - }) - } - - if p.static() { - depSet := android.NewDepSetBuilder(android.TOPOLOGICAL).Direct(in).Build() - ctx.SetProvider(StaticLibraryInfoProvider, StaticLibraryInfo{ - StaticLibrary: in, - - TransitiveStaticLibrariesForOrdering: depSet, - }) - } - - p.libraryDecorator.flagExporter.setProvider(ctx) - - return in -} - -func (p *snapshotLibraryDecorator) install(ctx ModuleContext, file android.Path) { - if p.matchesWithDevice(ctx.DeviceConfig()) && (p.shared() || p.static()) { - p.baseInstaller.install(ctx, file) - } -} - -func (p *snapshotLibraryDecorator) nativeCoverage() bool { - return false -} - -func (p *snapshotLibraryDecorator) isSanitizerEnabled(t sanitizerType) bool { - switch t { - case cfi: - return p.sanitizerProperties.Cfi.Src != nil - default: - return false - } -} - -func (p *snapshotLibraryDecorator) setSanitizerVariation(t sanitizerType, enabled bool) { - if !enabled { - return - } - switch t { - case cfi: - p.sanitizerProperties.CfiEnabled = true - default: - return - } -} - -func snapshotLibrary(suffix string) (*Module, *snapshotLibraryDecorator) { - module, library := NewLibrary(android.DeviceSupported) - - module.stl = nil - module.sanitize = nil - library.disableStripping() - - prebuilt := &snapshotLibraryDecorator{ - libraryDecorator: library, - } - - prebuilt.baseLinker.Properties.No_libcrt = BoolPtr(true) - prebuilt.baseLinker.Properties.Nocrt = BoolPtr(true) - - // Prevent default system libs (libc, libm, and libdl) from being linked - if prebuilt.baseLinker.Properties.System_shared_libs == nil { - prebuilt.baseLinker.Properties.System_shared_libs = []string{} - } - - module.compiler = nil - module.linker = prebuilt - module.installer = prebuilt - - prebuilt.init(module, suffix) - module.AddProperties( - &prebuilt.properties, - &prebuilt.sanitizerProperties, - ) - - return module, prebuilt -} - -func VendorSnapshotSharedFactory() android.Module { - module, prebuilt := snapshotLibrary(vendorSnapshotSharedSuffix) - prebuilt.libraryDecorator.BuildOnlyShared() - return module.Init() -} - -func RecoverySnapshotSharedFactory() android.Module { - module, prebuilt := snapshotLibrary(recoverySnapshotSharedSuffix) - prebuilt.libraryDecorator.BuildOnlyShared() - return module.Init() -} - -func VendorSnapshotStaticFactory() android.Module { - module, prebuilt := snapshotLibrary(vendorSnapshotStaticSuffix) - prebuilt.libraryDecorator.BuildOnlyStatic() - return module.Init() -} - -func RecoverySnapshotStaticFactory() android.Module { - module, prebuilt := snapshotLibrary(recoverySnapshotStaticSuffix) - prebuilt.libraryDecorator.BuildOnlyStatic() - return module.Init() -} - -func VendorSnapshotHeaderFactory() android.Module { - module, prebuilt := snapshotLibrary(vendorSnapshotHeaderSuffix) - prebuilt.libraryDecorator.HeaderOnly() - return module.Init() -} - -func RecoverySnapshotHeaderFactory() android.Module { - module, prebuilt := snapshotLibrary(recoverySnapshotHeaderSuffix) - prebuilt.libraryDecorator.HeaderOnly() - return module.Init() -} - -var _ snapshotSanitizer = (*snapshotLibraryDecorator)(nil) - -type snapshotBinaryProperties struct { - // Prebuilt file for each arch. - Src *string `android:"arch_variant"` -} - -type snapshotBinaryDecorator struct { - vendorSnapshotModuleBase - *binaryDecorator - properties snapshotBinaryProperties - androidMkVendorSuffix bool -} - -func (p *snapshotBinaryDecorator) matchesWithDevice(config android.DeviceConfig) bool { - if config.DeviceArch() != p.arch() { - return false - } - if p.properties.Src == nil { - return false - } - return true -} - -func (p *snapshotBinaryDecorator) link(ctx ModuleContext, - flags Flags, deps PathDeps, objs Objects) android.Path { - if !p.matchesWithDevice(ctx.DeviceConfig()) { - return nil - } - - in := android.PathForModuleSrc(ctx, *p.properties.Src) - stripFlags := flagsToStripFlags(flags) - p.unstrippedOutputFile = in - binName := in.Base() - if p.stripper.NeedsStrip(ctx) { - stripped := android.PathForModuleOut(ctx, "stripped", binName) - p.stripper.StripExecutableOrSharedLib(ctx, in, stripped, stripFlags) - in = stripped - } - - m := ctx.Module().(*Module) - p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] - - // use cpExecutable to make it executable - outputFile := android.PathForModuleOut(ctx, binName) - ctx.Build(pctx, android.BuildParams{ - Rule: android.CpExecutable, - Description: "prebuilt", - Output: outputFile, - Input: in, - }) - - return outputFile -} - -func (p *snapshotBinaryDecorator) nativeCoverage() bool { - return false -} - -func VendorSnapshotBinaryFactory() android.Module { - return snapshotBinaryFactory(vendorSnapshotBinarySuffix) -} - -func RecoverySnapshotBinaryFactory() android.Module { - return snapshotBinaryFactory(recoverySnapshotBinarySuffix) -} - -func snapshotBinaryFactory(suffix string) android.Module { - module, binary := NewBinary(android.DeviceSupported) - binary.baseLinker.Properties.No_libcrt = BoolPtr(true) - binary.baseLinker.Properties.Nocrt = BoolPtr(true) - - // Prevent default system libs (libc, libm, and libdl) from being linked - if binary.baseLinker.Properties.System_shared_libs == nil { - binary.baseLinker.Properties.System_shared_libs = []string{} - } - - prebuilt := &snapshotBinaryDecorator{ - binaryDecorator: binary, - } - - module.compiler = nil - module.sanitize = nil - module.stl = nil - module.linker = prebuilt - - prebuilt.init(module, suffix) - module.AddProperties(&prebuilt.properties) - return module.Init() -} - -type vendorSnapshotObjectProperties struct { - // Prebuilt file for each arch. - Src *string `android:"arch_variant"` -} - -type snapshotObjectLinker struct { - vendorSnapshotModuleBase - objectLinker - properties vendorSnapshotObjectProperties - androidMkVendorSuffix bool -} - -func (p *snapshotObjectLinker) matchesWithDevice(config android.DeviceConfig) bool { - if config.DeviceArch() != p.arch() { - return false - } - if p.properties.Src == nil { - return false - } - return true -} - -func (p *snapshotObjectLinker) link(ctx ModuleContext, - flags Flags, deps PathDeps, objs Objects) android.Path { - if !p.matchesWithDevice(ctx.DeviceConfig()) { - return nil - } - - m := ctx.Module().(*Module) - p.androidMkVendorSuffix = vendorSuffixModules(ctx.Config())[m.BaseModuleName()] - - return android.PathForModuleSrc(ctx, *p.properties.Src) -} - -func (p *snapshotObjectLinker) nativeCoverage() bool { - return false -} - -func VendorSnapshotObjectFactory() android.Module { - module := newObject() - - prebuilt := &snapshotObjectLinker{ - objectLinker: objectLinker{ - baseLinker: NewBaseLinker(nil), - }, - } - module.linker = prebuilt - - prebuilt.init(module, vendorSnapshotObjectSuffix) - module.AddProperties(&prebuilt.properties) - return module.Init() -} - -func RecoverySnapshotObjectFactory() android.Module { - module := newObject() - - prebuilt := &snapshotObjectLinker{ - objectLinker: objectLinker{ - baseLinker: NewBaseLinker(nil), - }, - } - module.linker = prebuilt - - prebuilt.init(module, recoverySnapshotObjectSuffix) - module.AddProperties(&prebuilt.properties) - return module.Init() -} - -func init() { - vendorImageSingleton.init() - recoveryImageSingleton.init() -} - var vendorSnapshotSingleton = snapshotSingleton{ "vendor", "SOONG_VENDOR_SNAPSHOT_ZIP", android.OptionalPath{}, true, - vendorImageSingleton, + vendorSnapshotImageSingleton, } var recoverySnapshotSingleton = snapshotSingleton{ @@ -639,7 +41,7 @@ var recoverySnapshotSingleton = snapshotSingleton{ "SOONG_RECOVERY_SNAPSHOT_ZIP", android.OptionalPath{}, false, - recoveryImageSingleton, + recoverySnapshotImageSingleton, } func VendorSnapshotSingleton() android.Singleton { @@ -667,13 +69,12 @@ type snapshotSingleton struct { // Implementation of the image interface specific to the image // associated with this snapshot (e.g., specific to the vendor image, // recovery image, etc.). - image image + image snapshotImage } var ( // Modules under following directories are ignored. They are OEM's and vendor's // proprietary modules(device/, kernel/, vendor/, and hardware/). - // TODO(b/65377115): Clean up these with more maintainable way vendorProprietaryDirs = []string{ "device", "kernel", @@ -683,7 +84,6 @@ var ( // Modules under following directories are ignored. They are OEM's and vendor's // proprietary modules(device/, kernel/, vendor/, and hardware/). - // TODO(b/65377115): Clean up these with more maintainable way recoveryProprietaryDirs = []string{ "bootable/recovery", "device", @@ -694,7 +94,6 @@ var ( // Modules under following directories are included as they are in AOSP, // although hardware/ and kernel/ are normally for vendor's own. - // TODO(b/65377115): Clean up these with more maintainable way aospDirsUnderProprietary = []string{ "kernel/configs", "kernel/prebuilts", @@ -738,10 +137,8 @@ func isProprietaryPath(dir string, proprietaryDirs []string) bool { } func isVendorProprietaryModule(ctx android.BaseModuleContext) bool { - // Any module in a vendor proprietary path is a vendor proprietary // module. - if isVendorProprietaryPath(ctx.ModuleDir()) { return true } @@ -750,7 +147,6 @@ func isVendorProprietaryModule(ctx android.BaseModuleContext) bool { // still be a vendor proprietary module. This happens for cc modules // that are excluded from the vendor snapshot, and it means that the // vendor has assumed control of the framework-provided module. - if c, ok := ctx.Module().(*Module); ok { if c.ExcludeFromVendorSnapshot() { return true @@ -766,15 +162,21 @@ func isVendorProprietaryModule(ctx android.BaseModuleContext) bool { // AOSP. They are not guaranteed to be compatible with older vendor images. (e.g. might // depend on newer VNDK) So they are captured as vendor snapshot To build older vendor // image and newer system image altogether. -func isVendorSnapshotModule(m *Module, inVendorProprietaryPath bool, apexInfo android.ApexInfo) bool { - return isSnapshotModule(m, inVendorProprietaryPath, apexInfo, vendorImageSingleton) +func isVendorSnapshotAware(m *Module, inVendorProprietaryPath bool, apexInfo android.ApexInfo) bool { + return isSnapshotAware(m, inVendorProprietaryPath, apexInfo, vendorSnapshotImageSingleton) } -func isRecoverySnapshotModule(m *Module, inRecoveryProprietaryPath bool, apexInfo android.ApexInfo) bool { - return isSnapshotModule(m, inRecoveryProprietaryPath, apexInfo, recoveryImageSingleton) +// Determine if a module is going to be included in recovery snapshot or not. +// +// Targets of recovery snapshot are "recovery: true" or "recovery_available: true" +// modules in AOSP. They are not guaranteed to be compatible with older recovery images. +// So they are captured as recovery snapshot To build older recovery image. +func isRecoverySnapshotAware(m *Module, inRecoveryProprietaryPath bool, apexInfo android.ApexInfo) bool { + return isSnapshotAware(m, inRecoveryProprietaryPath, apexInfo, recoverySnapshotImageSingleton) } -func isSnapshotModule(m *Module, inProprietaryPath bool, apexInfo android.ApexInfo, image image) bool { +// Determines if the module is a candidate for snapshot. +func isSnapshotAware(m *Module, inProprietaryPath bool, apexInfo android.ApexInfo, image snapshotImage) bool { if !m.Enabled() || m.Properties.HideFromMake { return false } @@ -799,7 +201,7 @@ func isSnapshotModule(m *Module, inProprietaryPath bool, apexInfo android.ApexIn if m.Target().NativeBridge == android.NativeBridgeEnabled { return false } - // the module must be installed in /vendor + // the module must be installed in target image if !apexInfo.IsForPlatform() || m.isSnapshotPrebuilt() || !image.inImage(m)() { return false } @@ -817,7 +219,6 @@ func isSnapshotModule(m *Module, inProprietaryPath bool, apexInfo android.ApexIn // Libraries if l, ok := m.linker.(snapshotLibraryInterface); ok { - // TODO(b/65377115): add full support for sanitizer if m.sanitize != nil { // scs and hwasan export both sanitized and unsanitized variants for static and header // Always use unsanitized variants of them. @@ -827,6 +228,8 @@ func isSnapshotModule(m *Module, inProprietaryPath bool, apexInfo android.ApexIn } } // cfi also exports both variants. But for static, we capture both. + // This is because cfi static libraries can't be linked from non-cfi modules, + // and vice versa. This isn't the case for scs and hwasan sanitizers. if !l.static() && !l.shared() && m.sanitize.isSanitizerEnabled(cfi) { return false } @@ -856,6 +259,33 @@ func isSnapshotModule(m *Module, inProprietaryPath bool, apexInfo android.ApexIn return false } +// This is to be saved as .json files, which is for development/vendor_snapshot/update.py. +// These flags become Android.bp snapshot module properties. +type snapshotJsonFlags struct { + ModuleName string `json:",omitempty"` + RelativeInstallPath string `json:",omitempty"` + + // library flags + ExportedDirs []string `json:",omitempty"` + ExportedSystemDirs []string `json:",omitempty"` + ExportedFlags []string `json:",omitempty"` + Sanitize string `json:",omitempty"` + SanitizeMinimalDep bool `json:",omitempty"` + SanitizeUbsanDep bool `json:",omitempty"` + + // binary flags + Symlinks []string `json:",omitempty"` + + // dependencies + SharedLibs []string `json:",omitempty"` + RuntimeLibs []string `json:",omitempty"` + Required []string `json:",omitempty"` + + // extra config files + InitRc []string `json:",omitempty"` + VintfFragments []string `json:",omitempty"` +} + func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { // BOARD_VNDK_VERSION must be set to 'current' in order to generate a vendor snapshot. if ctx.DeviceConfig().VndkVersion() != "current" { @@ -909,6 +339,8 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { var headers android.Paths + // installSnapshot function copies prebuilt file (.so, .a, or executable) and json flag file. + // For executables, init_rc and vintf_fragments files are also copied. installSnapshot := func(m *Module) android.Paths { targetArch := "arch-" + m.Target().Arch.ArchType.String() if m.Target().Arch.ArchVariant != "" { @@ -917,30 +349,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { var ret android.Paths - prop := struct { - ModuleName string `json:",omitempty"` - RelativeInstallPath string `json:",omitempty"` - - // library flags - ExportedDirs []string `json:",omitempty"` - ExportedSystemDirs []string `json:",omitempty"` - ExportedFlags []string `json:",omitempty"` - Sanitize string `json:",omitempty"` - SanitizeMinimalDep bool `json:",omitempty"` - SanitizeUbsanDep bool `json:",omitempty"` - - // binary flags - Symlinks []string `json:",omitempty"` - - // dependencies - SharedLibs []string `json:",omitempty"` - RuntimeLibs []string `json:",omitempty"` - Required []string `json:",omitempty"` - - // extra config files - InitRc []string `json:",omitempty"` - VintfFragments []string `json:",omitempty"` - }{} + prop := snapshotJsonFlags{} // Common properties among snapshots. prop.ModuleName = ctx.ModuleName(m) @@ -968,7 +377,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { out := filepath.Join(configsDir, path.Base()) if !installedConfigs[out] { installedConfigs[out] = true - ret = append(ret, copyFile(ctx, path, out)) + ret = append(ret, copyFileRule(ctx, path, out)) } } @@ -1019,7 +428,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { prop.ModuleName += ".cfi" } snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, libType, stem) - ret = append(ret, copyFile(ctx, libPath, snapshotLibOut)) + ret = append(ret, copyFileRule(ctx, libPath, snapshotLibOut)) } else { stem = ctx.ModuleName(m) } @@ -1033,7 +442,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { // install bin binPath := m.outputFile.Path() snapshotBinOut := filepath.Join(snapshotArchDir, targetArch, "binary", binPath.Base()) - ret = append(ret, copyFile(ctx, binPath, snapshotBinOut)) + ret = append(ret, copyFileRule(ctx, binPath, snapshotBinOut)) propOut = snapshotBinOut + ".json" } else if m.object() { // object files aren't installed to the device, so their names can conflict. @@ -1041,7 +450,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { objPath := m.outputFile.Path() snapshotObjOut := filepath.Join(snapshotArchDir, targetArch, "object", ctx.ModuleName(m)+filepath.Ext(objPath.Base())) - ret = append(ret, copyFile(ctx, objPath, snapshotObjOut)) + ret = append(ret, copyFileRule(ctx, objPath, snapshotObjOut)) propOut = snapshotObjOut + ".json" } else { ctx.Errorf("unknown module %q in vendor snapshot", m.String()) @@ -1053,7 +462,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { ctx.Errorf("json marshal to %q failed: %#v", propOut, err) return nil } - ret = append(ret, writeStringToFile(ctx, string(j), propOut)) + ret = append(ret, writeStringToFileRule(ctx, string(j), propOut)) return ret } @@ -1088,11 +497,14 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { } } - if !isSnapshotModule(m, inProprietaryPath, apexInfo, c.image) { + if !isSnapshotAware(m, inProprietaryPath, apexInfo, c.image) { return } + // installSnapshot installs prebuilts and json flag files snapshotOutputs = append(snapshotOutputs, installSnapshot(m)...) + + // just gather headers and notice files here, because they are to be deduplicated if l, ok := m.linker.(snapshotLibraryInterface); ok { headers = append(headers, l.snapshotHeaders()...) } @@ -1103,7 +515,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { // skip already copied notice file if !installedNotices[noticeOut] { installedNotices[noticeOut] = true - snapshotOutputs = append(snapshotOutputs, combineNotices( + snapshotOutputs = append(snapshotOutputs, combineNoticesRule( ctx, m.NoticeFiles(), noticeOut)) } } @@ -1111,7 +523,7 @@ func (c *snapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { // install all headers after removing duplicates for _, header := range android.FirstUniquePaths(headers) { - snapshotOutputs = append(snapshotOutputs, copyFile( + snapshotOutputs = append(snapshotOutputs, copyFileRule( ctx, header, filepath.Join(includeDir, header.String()))) } @@ -1155,141 +567,3 @@ func (c *snapshotSingleton) MakeVars(ctx android.MakeVarsContext) { c.makeVar, c.snapshotZipFile.String()) } - -type snapshotInterface interface { - matchesWithDevice(config android.DeviceConfig) bool -} - -var _ snapshotInterface = (*vndkPrebuiltLibraryDecorator)(nil) -var _ snapshotInterface = (*snapshotLibraryDecorator)(nil) -var _ snapshotInterface = (*snapshotBinaryDecorator)(nil) -var _ snapshotInterface = (*snapshotObjectLinker)(nil) - -// gathers all snapshot modules for vendor, and disable unnecessary snapshots -// TODO(b/145966707): remove mutator and utilize android.Prebuilt to override source modules -func VendorSnapshotMutator(ctx android.BottomUpMutatorContext) { - vndkVersion := ctx.DeviceConfig().VndkVersion() - // don't need snapshot if current - if vndkVersion == "current" || vndkVersion == "" { - return - } - - module, ok := ctx.Module().(*Module) - if !ok || !module.Enabled() || module.VndkVersion() != vndkVersion { - return - } - - if !module.isSnapshotPrebuilt() { - return - } - - // isSnapshotPrebuilt ensures snapshotInterface - if !module.linker.(snapshotInterface).matchesWithDevice(ctx.DeviceConfig()) { - // Disable unnecessary snapshot module, but do not disable - // vndk_prebuilt_shared because they might be packed into vndk APEX - if !module.IsVndk() { - module.Disable() - } - return - } - - var snapshotMap *snapshotMap - - if lib, ok := module.linker.(libraryInterface); ok { - if lib.static() { - snapshotMap = vendorSnapshotStaticLibs(ctx.Config()) - } else if lib.shared() { - snapshotMap = vendorSnapshotSharedLibs(ctx.Config()) - } else { - // header - snapshotMap = vendorSnapshotHeaderLibs(ctx.Config()) - } - } else if _, ok := module.linker.(*snapshotBinaryDecorator); ok { - snapshotMap = vendorSnapshotBinaries(ctx.Config()) - } else if _, ok := module.linker.(*snapshotObjectLinker); ok { - snapshotMap = vendorSnapshotObjects(ctx.Config()) - } else { - return - } - - vendorSnapshotsLock.Lock() - defer vendorSnapshotsLock.Unlock() - snapshotMap.add(module.BaseModuleName(), ctx.Arch().ArchType, ctx.ModuleName()) -} - -// Disables source modules which have snapshots -func VendorSnapshotSourceMutator(ctx android.BottomUpMutatorContext) { - if !ctx.Device() { - return - } - - vndkVersion := ctx.DeviceConfig().VndkVersion() - // don't need snapshot if current - if vndkVersion == "current" || vndkVersion == "" { - return - } - - module, ok := ctx.Module().(*Module) - if !ok { - return - } - - // vendor suffix should be added to snapshots if the source module isn't vendor: true. - if !module.SocSpecific() { - // But we can't just check SocSpecific() since we already passed the image mutator. - // Check ramdisk and recovery to see if we are real "vendor: true" module. - ramdisk_available := module.InRamdisk() && !module.OnlyInRamdisk() - vendor_ramdisk_available := module.InVendorRamdisk() && !module.OnlyInVendorRamdisk() - recovery_available := module.InRecovery() && !module.OnlyInRecovery() - - if !ramdisk_available && !recovery_available && !vendor_ramdisk_available { - vendorSnapshotsLock.Lock() - defer vendorSnapshotsLock.Unlock() - - vendorSuffixModules(ctx.Config())[ctx.ModuleName()] = true - } - } - - if module.isSnapshotPrebuilt() || module.VndkVersion() != ctx.DeviceConfig().VndkVersion() { - // only non-snapshot modules with BOARD_VNDK_VERSION - return - } - - // .. and also filter out llndk library - if module.isLlndk(ctx.Config()) { - return - } - - var snapshotMap *snapshotMap - - if lib, ok := module.linker.(libraryInterface); ok { - if lib.static() { - snapshotMap = vendorSnapshotStaticLibs(ctx.Config()) - } else if lib.shared() { - snapshotMap = vendorSnapshotSharedLibs(ctx.Config()) - } else { - // header - snapshotMap = vendorSnapshotHeaderLibs(ctx.Config()) - } - } else if module.binary() { - snapshotMap = vendorSnapshotBinaries(ctx.Config()) - } else if module.object() { - snapshotMap = vendorSnapshotObjects(ctx.Config()) - } else { - return - } - - if _, ok := snapshotMap.get(ctx.ModuleName(), ctx.Arch().ArchType); !ok { - // Corresponding snapshot doesn't exist - return - } - - // Disables source modules if corresponding snapshot exists. - if lib, ok := module.linker.(libraryInterface); ok && lib.buildStatic() && lib.buildShared() { - // But do not disable because the shared variant depends on the static variant. - module.SkipInstall() - module.Properties.HideFromMake = true - } else { - module.Disable() - } -} diff --git a/cc/vndk.go b/cc/vndk.go index d57cdf79d..eca1cdfdb 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -533,7 +533,7 @@ type vndkSnapshotSingleton struct { vndkSnapshotZipFile android.OptionalPath } -func isVndkSnapshotLibrary(config android.DeviceConfig, m *Module, +func isVndkSnapshotAware(config android.DeviceConfig, m *Module, apexInfo android.ApexInfo) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) { if m.Target().NativeBridge == android.NativeBridgeEnabled { @@ -622,6 +622,9 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex var headers android.Paths + // installVndkSnapshotLib copies built .so file from the module. + // Also, if the build artifacts is on, write a json file which contains all exported flags + // with FlagExporterInfo. installVndkSnapshotLib := func(m *Module, vndkType string) (android.Paths, bool) { var ret android.Paths @@ -632,7 +635,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex libPath := m.outputFile.Path() snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "shared", vndkType, libPath.Base()) - ret = append(ret, copyFile(ctx, libPath, snapshotLibOut)) + ret = append(ret, copyFileRule(ctx, libPath, snapshotLibOut)) if ctx.Config().VndkSnapshotBuildArtifacts() { prop := struct { @@ -654,7 +657,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex ctx.Errorf("json marshal to %q failed: %#v", propOut, err) return nil, false } - ret = append(ret, writeStringToFile(ctx, string(j), propOut)) + ret = append(ret, writeStringToFileRule(ctx, string(j), propOut)) } return ret, true } @@ -667,11 +670,21 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo) - l, vndkType, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m, apexInfo) + l, vndkType, ok := isVndkSnapshotAware(ctx.DeviceConfig(), m, apexInfo) if !ok { return } + // For all snapshot candidates, the followings are captured. + // - .so files + // - notice files + // + // The followings are also captured if VNDK_SNAPSHOT_BUILD_ARTIFACTS. + // - .json files containing exported flags + // - exported headers from collectHeadersForSnapshot() + // + // Headers are deduplicated after visiting all modules. + // install .so files for appropriate modules. // Also install .json files if VNDK_SNAPSHOT_BUILD_ARTIFACTS libs, ok := installVndkSnapshotLib(m, vndkType) @@ -690,7 +703,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex // skip already copied notice file if _, ok := noticeBuilt[noticeName]; !ok { noticeBuilt[noticeName] = true - snapshotOutputs = append(snapshotOutputs, combineNotices( + snapshotOutputs = append(snapshotOutputs, combineNoticesRule( ctx, m.NoticeFiles(), filepath.Join(noticeDir, noticeName))) } } @@ -702,7 +715,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex // install all headers after removing duplicates for _, header := range android.FirstUniquePaths(headers) { - snapshotOutputs = append(snapshotOutputs, copyFile( + snapshotOutputs = append(snapshotOutputs, copyFileRule( ctx, header, filepath.Join(includeDir, header.String()))) } @@ -712,38 +725,18 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex if !ok || !m.Enabled() || m.Name() == vndkUsingCoreVariantLibrariesTxt { return } - snapshotOutputs = append(snapshotOutputs, copyFile( + snapshotOutputs = append(snapshotOutputs, copyFileRule( ctx, m.OutputFile(), filepath.Join(configsDir, m.Name()))) }) /* - Dump a map to a list file as: - - {key1} {value1} - {key2} {value2} - ... - */ - installMapListFile := func(m map[string]string, path string) android.OutputPath { - var txtBuilder strings.Builder - for idx, k := range android.SortedStringKeys(m) { - if idx > 0 { - txtBuilder.WriteString("\n") - } - txtBuilder.WriteString(k) - txtBuilder.WriteString(" ") - txtBuilder.WriteString(m[k]) - } - return writeStringToFile(ctx, txtBuilder.String(), path) - } - - /* module_paths.txt contains paths on which VNDK modules are defined. e.g., libbase.so system/libbase libc.so bionic/libc ... */ - snapshotOutputs = append(snapshotOutputs, installMapListFile(modulePaths, filepath.Join(configsDir, "module_paths.txt"))) + snapshotOutputs = append(snapshotOutputs, installMapListFileRule(ctx, modulePaths, filepath.Join(configsDir, "module_paths.txt"))) /* module_names.txt contains names as which VNDK modules are defined, @@ -754,7 +747,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex libprotobuf-cpp-full-3.9.2.so libprotobuf-cpp-full ... */ - snapshotOutputs = append(snapshotOutputs, installMapListFile(moduleNames, filepath.Join(configsDir, "module_names.txt"))) + snapshotOutputs = append(snapshotOutputs, installMapListFileRule(ctx, moduleNames, filepath.Join(configsDir, "module_names.txt"))) // All artifacts are ready. Sort them to normalize ninja and then zip. sort.Slice(snapshotOutputs, func(i, j int) bool { @@ -764,7 +757,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex zipPath := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+".zip") zipRule := android.NewRuleBuilder(pctx, ctx) - // filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with xargs + // filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with tr snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+"_list") zipRule.Command(). Text("tr"). diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 0a9b15671..c079e8388 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -185,7 +185,11 @@ func main() { Status: stat, }} - config := build.NewConfig(buildCtx) + args := "" + if *alternateResultDir { + args = "dist" + } + config := build.NewConfig(buildCtx, args) if *outDir == "" { name := "multiproduct" if !*incremental { @@ -212,15 +216,10 @@ func main() { os.MkdirAll(logsDir, 0777) build.SetupOutDir(buildCtx, config) - if *alternateResultDir { - distLogsDir := filepath.Join(config.DistDir(), "logs") - os.MkdirAll(distLogsDir, 0777) - log.SetOutput(filepath.Join(distLogsDir, "soong.log")) - trace.SetOutput(filepath.Join(distLogsDir, "build.trace")) - } else { - log.SetOutput(filepath.Join(config.OutDir(), "soong.log")) - trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) - } + + os.MkdirAll(config.LogsDir(), 0777) + log.SetOutput(filepath.Join(config.LogsDir(), "soong.log")) + trace.SetOutput(filepath.Join(config.LogsDir(), "build.trace")) var jobs = *numJobs if jobs < 1 { @@ -344,7 +343,7 @@ func main() { FileArgs: []zip.FileArg{ {GlobDir: logsDir, SourcePrefixToStrip: logsDir}, }, - OutputFilePath: filepath.Join(config.DistDir(), "logs.zip"), + OutputFilePath: filepath.Join(config.RealDistDir(), "logs.zip"), NumParallelJobs: runtime.NumCPU(), CompressionLevel: 5, } diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go index a4f57ea1f..f8919a400 100644 --- a/cmd/sbox/sbox.go +++ b/cmd/sbox/sbox.go @@ -337,7 +337,7 @@ func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error { for _, copyPair := range copies { fromPath := joinPath(fromDir, copyPair.GetFrom()) toPath := joinPath(toDir, copyPair.GetTo()) - err := copyOneFile(fromPath, toPath) + err := copyOneFile(fromPath, toPath, copyPair.GetExecutable()) if err != nil { return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err) } @@ -346,7 +346,7 @@ func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error { } // copyOneFile copies a file. -func copyOneFile(from string, to string) error { +func copyOneFile(from string, to string, executable bool) error { err := os.MkdirAll(filepath.Dir(to), 0777) if err != nil { return err @@ -358,6 +358,9 @@ func copyOneFile(from string, to string) error { } perm := stat.Mode() + if executable { + perm = perm | 0100 // u+x + } in, err := os.Open(from) if err != nil { diff --git a/cmd/sbox/sbox_proto/sbox.pb.go b/cmd/sbox/sbox_proto/sbox.pb.go index 6584bdf0b..79bb90c4c 100644 --- a/cmd/sbox/sbox_proto/sbox.pb.go +++ b/cmd/sbox/sbox_proto/sbox.pb.go @@ -156,8 +156,10 @@ func (m *Command) GetInputHash() string { // are relative to is specific to the context the Copy is used in and will be different for // from and to. type Copy struct { - From *string `protobuf:"bytes,1,req,name=from" json:"from,omitempty"` - To *string `protobuf:"bytes,2,req,name=to" json:"to,omitempty"` + From *string `protobuf:"bytes,1,req,name=from" json:"from,omitempty"` + To *string `protobuf:"bytes,2,req,name=to" json:"to,omitempty"` + // If true, make the file executable after copying it. + Executable *bool `protobuf:"varint,3,opt,name=executable" json:"executable,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -202,6 +204,13 @@ func (m *Copy) GetTo() string { return "" } +func (m *Copy) GetExecutable() bool { + if m != nil && m.Executable != nil { + return *m.Executable + } + return false +} + func init() { proto.RegisterType((*Manifest)(nil), "sbox.Manifest") proto.RegisterType((*Command)(nil), "sbox.Command") @@ -213,21 +222,22 @@ func init() { } var fileDescriptor_9d0425bf0de86ed1 = []byte{ - // 252 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x41, 0x4b, 0xc3, 0x40, - 0x10, 0x85, 0x49, 0x9a, 0xd2, 0x66, 0x6a, 0x7b, 0x18, 0x3c, 0xec, 0x45, 0x08, 0x01, 0x21, 0x55, - 0xe8, 0xc1, 0x7f, 0x60, 0xf5, 0xe0, 0xc5, 0xcb, 0x1e, 0x45, 0x08, 0xdb, 0x64, 0x97, 0x04, 0x4c, - 0x66, 0xd9, 0xdd, 0x82, 0xfd, 0x57, 0xfe, 0x44, 0xd9, 0x49, 0xea, 0xc5, 0xdb, 0xcc, 0xfb, 0x78, - 0xf3, 0x1e, 0x03, 0xe0, 0x4f, 0xf4, 0x7d, 0xb0, 0x8e, 0x02, 0x61, 0x16, 0xe7, 0xf2, 0x13, 0xd6, - 0xef, 0x6a, 0xec, 0x8d, 0xf6, 0x01, 0xf7, 0xb0, 0x6e, 0x68, 0x18, 0xd4, 0xd8, 0x7a, 0x91, 0x14, - 0x8b, 0x6a, 0xf3, 0xb4, 0x3d, 0xb0, 0xe1, 0x65, 0x52, 0xe5, 0x1f, 0xc6, 0x7b, 0xd8, 0xd1, 0x39, - 0xd8, 0x73, 0xa8, 0x5b, 0x6d, 0x4d, 0xff, 0xa5, 0x45, 0x5a, 0x24, 0x55, 0x2e, 0xb7, 0x93, 0xfa, - 0x3a, 0x89, 0xe5, 0x4f, 0x02, 0xab, 0xd9, 0x8c, 0x8f, 0xb0, 0x69, 0xc8, 0x5e, 0xea, 0x93, 0x36, - 0xe4, 0xf4, 0x1c, 0x00, 0xd7, 0x00, 0x7b, 0x91, 0x10, 0xf1, 0x91, 0x29, 0xde, 0xc2, 0xb2, 0xe9, - 0xda, 0xde, 0xf1, 0xd9, 0xb5, 0x9c, 0x16, 0x14, 0xb0, 0x9a, 0x1b, 0x88, 0x45, 0x91, 0x56, 0xb9, - 0xbc, 0xae, 0xb8, 0x07, 0x76, 0xd7, 0xca, 0x04, 0xed, 0x44, 0xf6, 0xef, 0x76, 0x1e, 0xe9, 0x73, - 0x84, 0x78, 0x07, 0xd0, 0x8f, 0xb1, 0x79, 0xa7, 0x7c, 0x27, 0x96, 0x5c, 0x3b, 0x67, 0xe5, 0x4d, - 0xf9, 0xae, 0x7c, 0x80, 0x2c, 0x3a, 0x10, 0x21, 0x33, 0x8e, 0x06, 0x91, 0x70, 0x10, 0xcf, 0xb8, - 0x83, 0x34, 0x90, 0x48, 0x59, 0x49, 0x03, 0x1d, 0x6f, 0x3e, 0xf8, 0xa1, 0x35, 0x3f, 0xf4, 0x37, - 0x00, 0x00, 0xff, 0xff, 0x95, 0x4d, 0xee, 0x7d, 0x5d, 0x01, 0x00, 0x00, + // 268 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x4f, 0x4b, 0xc3, 0x40, + 0x10, 0xc5, 0xc9, 0x9f, 0xd2, 0x64, 0x6a, 0x7b, 0x18, 0x3c, 0xec, 0x45, 0x09, 0x01, 0x21, 0x45, + 0xe8, 0xc1, 0x6f, 0x60, 0xf5, 0x20, 0x82, 0x97, 0x1c, 0x45, 0x08, 0x9b, 0x64, 0x43, 0x02, 0x4d, + 0x26, 0xec, 0x6e, 0xa0, 0xfd, 0x56, 0x7e, 0x44, 0xd9, 0x49, 0x2a, 0x82, 0xb7, 0x99, 0xdf, 0xe3, + 0xcd, 0x7b, 0x0c, 0x80, 0x29, 0xe9, 0x7c, 0x18, 0x35, 0x59, 0xc2, 0xd0, 0xcd, 0xe9, 0x17, 0x44, + 0x1f, 0x72, 0xe8, 0x1a, 0x65, 0x2c, 0xee, 0x21, 0xaa, 0xa8, 0xef, 0xe5, 0x50, 0x1b, 0xe1, 0x25, + 0x41, 0xb6, 0x79, 0xda, 0x1e, 0xd8, 0xf0, 0x32, 0xd3, 0xfc, 0x57, 0xc6, 0x07, 0xd8, 0xd1, 0x64, + 0xc7, 0xc9, 0x16, 0xb5, 0x1a, 0x9b, 0xee, 0xa4, 0x84, 0x9f, 0x78, 0x59, 0x9c, 0x6f, 0x67, 0xfa, + 0x3a, 0xc3, 0xf4, 0xdb, 0x83, 0xf5, 0x62, 0xc6, 0x47, 0xd8, 0x54, 0x34, 0x5e, 0x8a, 0x52, 0x35, + 0xa4, 0xd5, 0x12, 0x00, 0xd7, 0x80, 0xf1, 0x92, 0x83, 0x93, 0x8f, 0xac, 0xe2, 0x2d, 0xac, 0xaa, + 0xb6, 0xee, 0x34, 0x9f, 0x8d, 0xf2, 0x79, 0x41, 0x01, 0xeb, 0xa5, 0x81, 0x08, 0x12, 0x3f, 0x8b, + 0xf3, 0xeb, 0x8a, 0x7b, 0x60, 0x77, 0x21, 0x1b, 0xab, 0xb4, 0x08, 0xff, 0xdd, 0x8e, 0x9d, 0xfa, + 0xec, 0x44, 0xbc, 0x03, 0xe8, 0x06, 0xd7, 0xbc, 0x95, 0xa6, 0x15, 0x2b, 0xae, 0x1d, 0x33, 0x79, + 0x93, 0xa6, 0x4d, 0xdf, 0x21, 0x74, 0x0e, 0x44, 0x08, 0x1b, 0x4d, 0xbd, 0xf0, 0x38, 0x88, 0x67, + 0xdc, 0x81, 0x6f, 0x49, 0xf8, 0x4c, 0x7c, 0x4b, 0x78, 0x0f, 0xa0, 0xce, 0xaa, 0x9a, 0xac, 0x2c, + 0x4f, 0x4a, 0x04, 0x5c, 0xf5, 0x0f, 0x39, 0xde, 0x7c, 0xf2, 0xc3, 0x0b, 0x7e, 0xf8, 0x4f, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x78, 0x37, 0x3e, 0x6a, 0x7d, 0x01, 0x00, 0x00, } diff --git a/cmd/sbox/sbox_proto/sbox.proto b/cmd/sbox/sbox_proto/sbox.proto index ab95545a5..695b0e86e 100644 --- a/cmd/sbox/sbox_proto/sbox.proto +++ b/cmd/sbox/sbox_proto/sbox.proto @@ -55,4 +55,7 @@ message Command { message Copy { required string from = 1; required string to = 2; + + // If true, make the file executable after copying it. + optional bool executable = 3; }
\ No newline at end of file diff --git a/cmd/soong_build/writedocs.go b/cmd/soong_build/writedocs.go index 5fb6e6ba2..253979e41 100644 --- a/cmd/soong_build/writedocs.go +++ b/cmd/soong_build/writedocs.go @@ -44,9 +44,10 @@ var propertyRank = map[string]int{ "name": 0, "src": 1, "srcs": 2, - "defaults": 3, - "host_supported": 4, - "device_supported": 5, + "exclude_srcs": 3, + "defaults": 4, + "host_supported": 5, + "device_supported": 6, } // For each module type, extract its documentation and convert it to the template data. diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 29030d69e..bd1d4507b 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -18,6 +18,7 @@ import ( "context" "flag" "fmt" + "io/ioutil" "os" "path/filepath" "strconv" @@ -173,16 +174,18 @@ func main() { build.SetupOutDir(buildCtx, config) - // Set up files to be outputted in the log directory. - logsDir := config.OutDir() - if config.Dist() { - logsDir = filepath.Join(config.DistDir(), "logs") + if config.UseBazel() { + defer populateExternalDistDir(buildCtx, config) } + // Set up files to be outputted in the log directory. + logsDir := config.LogsDir() + + // Common list of metric file definition. buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error") rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb") soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics") - defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, buildErrorFile, rbeMetricsFile, soongMetricsFile) + build.PrintOutDirWarning(buildCtx, config) os.MkdirAll(logsDir, 0777) @@ -198,8 +201,22 @@ func main() { buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v", config.Parallel(), config.RemoteParallel(), config.HighmemParallel()) - defer met.Dump(soongMetricsFile) - defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) + { + // The order of the function calls is important. The last defer function call + // is the first one that is executed to save the rbe metrics to a protobuf + // file. The soong metrics file is then next. Bazel profiles are written + // before the uploadMetrics is invoked. The written files are then uploaded + // if the uploading of the metrics is enabled. + files := []string{ + buildErrorFile, // build error strings + rbeMetricsFile, // high level metrics related to remote build execution. + soongMetricsFile, // high level metrics related to this build system. + config.BazelMetricsDir(), // directory that contains a set of bazel metrics. + } + defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...) + defer met.Dump(soongMetricsFile) + defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile) + } // Read the time at the starting point. if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { @@ -513,3 +530,72 @@ func getCommand(args []string) (*command, []string, error) { // command not found return nil, nil, fmt.Errorf("Command not found: %q", args) } + +// For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary. +func populateExternalDistDir(ctx build.Context, config build.Config) { + // Make sure that internalDistDirPath and externalDistDirPath are both absolute paths, so we can compare them + var err error + var internalDistDirPath string + var externalDistDirPath string + if internalDistDirPath, err = filepath.Abs(config.DistDir()); err != nil { + ctx.Fatalf("Unable to find absolute path of %s: %s", internalDistDirPath, err) + } + if externalDistDirPath, err = filepath.Abs(config.RealDistDir()); err != nil { + ctx.Fatalf("Unable to find absolute path of %s: %s", externalDistDirPath, err) + } + if externalDistDirPath == internalDistDirPath { + return + } + + // Make sure the external DIST_DIR actually exists before trying to write to it + if err = os.MkdirAll(externalDistDirPath, 0755); err != nil { + ctx.Fatalf("Unable to make directory %s: %s", externalDistDirPath, err) + } + + ctx.Println("Populating external DIST_DIR...") + + populateExternalDistDirHelper(ctx, config, internalDistDirPath, externalDistDirPath) +} + +func populateExternalDistDirHelper(ctx build.Context, config build.Config, internalDistDirPath string, externalDistDirPath string) { + files, err := ioutil.ReadDir(internalDistDirPath) + if err != nil { + ctx.Fatalf("Can't read internal distdir %s: %s", internalDistDirPath, err) + } + for _, f := range files { + internalFilePath := filepath.Join(internalDistDirPath, f.Name()) + externalFilePath := filepath.Join(externalDistDirPath, f.Name()) + + if f.IsDir() { + // Moving a directory - check if there is an existing directory to merge with + externalLstat, err := os.Lstat(externalFilePath) + if err != nil { + if !os.IsNotExist(err) { + ctx.Fatalf("Can't lstat external %s: %s", externalDistDirPath, err) + } + // Otherwise, if the error was os.IsNotExist, that's fine and we fall through to the rename at the bottom + } else { + if externalLstat.IsDir() { + // Existing dir - try to merge the directories? + populateExternalDistDirHelper(ctx, config, internalFilePath, externalFilePath) + continue + } else { + // Existing file being replaced with a directory. Delete the existing file... + if err := os.RemoveAll(externalFilePath); err != nil { + ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err) + } + } + } + } else { + // Moving a file (not a dir) - delete any existing file or directory + if err := os.RemoveAll(externalFilePath); err != nil { + ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err) + } + } + + // The actual move - do a rename instead of a copy in order to save disk space. + if err := os.Rename(internalFilePath, externalFilePath); err != nil { + ctx.Fatalf("Unable to rename %s -> %s due to error %s", internalFilePath, externalFilePath, err) + } + } +} diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 375921729..deaf77fe4 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -437,7 +437,11 @@ func validateClassLoaderContextRec(sdkVer int, clcs []*ClassLoaderContext) (bool if sdkVer == AnySdkVersion { // Return error if dexpreopt doesn't know paths to one of the <uses-library> // dependencies. In the future we may need to relax this and just disable dexpreopt. - return false, fmt.Errorf("invalid path for <uses-library> \"%s\"", clc.Name) + if clc.Host == nil { + return false, fmt.Errorf("invalid build path for <uses-library> \"%s\"", clc.Name) + } else { + return false, fmt.Errorf("invalid install path for <uses-library> \"%s\"", clc.Name) + } } else { // No error for compatibility libraries, as Soong doesn't know if they are needed // (this depends on the targetSdkVersion in the manifest), but the CLC is invalid. diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index df6856390..be7d4c63b 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -195,7 +195,7 @@ func TestCLCMaybeAdd(t *testing.T) { // But class loader context in such cases should raise an error on validation. t.Run("validate", func(t *testing.T) { _, err := validateClassLoaderContext(m) - checkError(t, err, "invalid path for <uses-library> \"a\"") + checkError(t, err, "invalid build path for <uses-library> \"a\"") }) } diff --git a/java/aar.go b/java/aar.go index 1940d7f7b..3b6b34e27 100644 --- a/java/aar.go +++ b/java/aar.go @@ -407,6 +407,7 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext, classLoaderConte ctx.VisitDirectDeps(func(module android.Module) { depName := ctx.OtherModuleName(module) + depTag := ctx.OtherModuleDependencyTag(module) var exportPackage android.Path aarDep, _ := module.(AndroidLibraryDependency) @@ -414,7 +415,7 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext, classLoaderConte exportPackage = aarDep.ExportPackage() } - switch ctx.OtherModuleDependencyTag(module) { + switch depTag { case instrumentationForTag: // Nothing, instrumentationForTag is treated as libTag for javac but not for aapt2. case libTag: @@ -439,7 +440,6 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext, classLoaderConte transitiveStaticLibs = append(transitiveStaticLibs, aarDep.ExportedStaticPackages()...) transitiveStaticLibs = append(transitiveStaticLibs, exportPackage) transitiveStaticLibManifests = append(transitiveStaticLibManifests, aarDep.ExportedManifests()...) - classLoaderContexts.AddContextMap(aarDep.ClassLoaderContexts(), depName) if aarDep.ExportedAssets().Valid() { assets = append(assets, aarDep.ExportedAssets().Path()) } @@ -458,11 +458,8 @@ func aaptLibs(ctx android.ModuleContext, sdkContext sdkContext, classLoaderConte } } - // Add nested dependencies after processing the direct dependency: if it is a <uses-library>, - // nested context is added as its subcontext, and should not be re-added at the top-level. - if dep, ok := module.(Dependency); ok { - classLoaderContexts.AddContextMap(dep.ClassLoaderContexts(), depName) - } + // Merge dep's CLC after processing the dep itself (which may add its own <uses-library>). + maybeAddCLCFromDep(module, depTag, depName, classLoaderContexts) }) deps = append(deps, sharedLibs...) diff --git a/java/androidmk.go b/java/androidmk.go index fc573c8b6..aaad44f72 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -274,7 +274,7 @@ func (binary *Binary) AndroidMkEntries() []android.AndroidMkEntries { }, }, ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { fmt.Fprintln(w, "jar_installed_module := $(LOCAL_INSTALLED_MODULE)") }, }, @@ -289,7 +289,7 @@ func (binary *Binary) AndroidMkEntries() []android.AndroidMkEntries { }, }, ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { // Ensure that the wrapper script timestamp is always updated when the jar is updated fmt.Fprintln(w, "$(LOCAL_INSTALLED_MODULE): $(jar_installed_module)") fmt.Fprintln(w, "jar_installed_module :=") @@ -393,7 +393,7 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { }, }, ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { if app.noticeOutputs.Merged.Valid() { fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", app.installApkName, app.noticeOutputs.Merged.String(), app.installApkName+"_NOTICE") @@ -548,7 +548,7 @@ func (dstubs *Droidstubs) AndroidMkEntries() []android.AndroidMkEntries { }, }, ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { if dstubs.apiFile != nil { fmt.Fprintf(w, ".PHONY: %s %s.txt\n", dstubs.Name(), dstubs.Name()) fmt.Fprintf(w, "%s %s.txt: %s\n", dstubs.Name(), dstubs.Name(), dstubs.apiFile) diff --git a/java/app.go b/java/app.go index 4bf9d33ea..e6d9550ec 100755 --- a/java/app.go +++ b/java/app.go @@ -1400,6 +1400,13 @@ func (a *AndroidAppImport) processVariants(ctx android.LoadHookContext) { archProps := reflect.ValueOf(a.archVariants).Elem().FieldByName("Arch") archType := ctx.Config().AndroidFirstDeviceTarget.Arch.ArchType MergePropertiesFromVariant(ctx, &a.properties, archProps, archType.Name) + + if String(a.properties.Apk) == "" { + // Disable this module since the apk property is still empty after processing all matching + // variants. This likely means there is no matching variant, and the default variant doesn't + // have an apk property value either. + a.Disable() + } } func MergePropertiesFromVariant(ctx android.EarlyModuleContext, diff --git a/java/app_builder.go b/java/app_builder.go index 69e462c3d..b53c15ab4 100644 --- a/java/app_builder.go +++ b/java/app_builder.go @@ -32,7 +32,7 @@ import ( var ( Signapk, SignapkRE = remoteexec.StaticRules(pctx, "signapk", blueprint.RuleParams{ - Command: `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -Djava.library.path=$$(dirname ${config.SignapkJniLibrary}) ` + + Command: `rm -f $out && $reTemplate${config.JavaCmd} ${config.JavaVmFlags} -Djava.library.path=$$(dirname ${config.SignapkJniLibrary}) ` + `-jar ${config.SignapkCmd} $flags $certificates $in $out`, CommandDeps: []string{"${config.SignapkCmd}", "${config.SignapkJniLibrary}"}, }, diff --git a/java/app_test.go b/java/app_test.go index 6efb0dcb3..e13c6b9f0 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -2524,6 +2524,24 @@ func TestAndroidAppImport_ArchVariants(t *testing.T) { `, expected: "prebuilts/apk/app.apk", }, + { + name: "no matching arch without default", + bp: ` + android_app_import { + name: "foo", + arch: { + arm: { + apk: "prebuilts/apk/app_arm.apk", + }, + }, + presigned: true, + dex_preopt: { + enabled: true, + }, + } + `, + expected: "", + }, } jniRuleRe := regexp.MustCompile("^if \\(zipinfo (\\S+)") @@ -2531,6 +2549,12 @@ func TestAndroidAppImport_ArchVariants(t *testing.T) { ctx, _ := testJava(t, test.bp) variant := ctx.ModuleForTests("foo", "android_common") + if test.expected == "" { + if variant.Module().Enabled() { + t.Error("module should have been disabled, but wasn't") + } + continue + } jniRuleCommand := variant.Output("jnis-uncompressed/foo.apk").RuleParams.Command matches := jniRuleRe.FindStringSubmatch(jniRuleCommand) if len(matches) != 2 { @@ -2542,6 +2566,34 @@ func TestAndroidAppImport_ArchVariants(t *testing.T) { } } +func TestAndroidAppImport_overridesDisabledAndroidApp(t *testing.T) { + ctx, _ := testJava(t, ` + android_app { + name: "foo", + srcs: ["a.java"], + enabled: false, + } + + android_app_import { + name: "foo", + apk: "prebuilts/apk/app.apk", + certificate: "platform", + prefer: true, + } + `) + + variant := ctx.ModuleForTests("prebuilt_foo", "android_common") + a := variant.Module().(*AndroidAppImport) + // The prebuilt module should still be enabled and active even if the source-based counterpart + // is disabled. + if !a.prebuilt.UsePrebuilt() { + t.Errorf("prebuilt foo module is not active") + } + if !a.Enabled() { + t.Errorf("prebuilt foo module is disabled") + } +} + func TestAndroidTestImport(t *testing.T) { ctx, config := testJava(t, ` android_test_import { @@ -2730,6 +2782,13 @@ func TestUsesLibraries(t *testing.T) { } java_sdk_library { + name: "fred", + srcs: ["a.java"], + api_packages: ["fred"], + sdk_version: "current", + } + + java_sdk_library { name: "bar", srcs: ["a.java"], api_packages: ["bar"], @@ -2753,7 +2812,12 @@ func TestUsesLibraries(t *testing.T) { name: "app", srcs: ["a.java"], libs: ["qux", "quuz.stubs"], - static_libs: ["static-runtime-helper"], + static_libs: [ + "static-runtime-helper", + // statically linked component libraries should not pull their SDK libraries, + // so "fred" should not be added to class loader context + "fred.stubs", + ], uses_libs: ["foo"], sdk_version: "current", optional_uses_libs: [ diff --git a/java/builder.go b/java/builder.go index cd3524542..995160d0e 100644 --- a/java/builder.go +++ b/java/builder.go @@ -42,7 +42,7 @@ var ( // TODO(b/143658984): goma can't handle the --system argument to javac. javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac", blueprint.RuleParams{ - Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + + Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" "$out" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` + `${config.SoongJavacWrapper} $javaTemplate${config.JavacCmd} ` + diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index f16ddf1df..062005b5b 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -418,43 +418,53 @@ func (d *dexpreoptBootJars) GenerateBuildActions(ctx android.SingletonContext) { dumpOatRules(ctx, d.defaultBootImage) } -func isHostdex(module android.Module) bool { - if lib, ok := module.(*Library); ok { - return Bool(lib.deviceProperties.Hostdex) - } - return false -} - // Inspect this module to see if it contains a bootclasspath dex jar. // Note that the same jar may occur in multiple modules. // This logic is tested in the apex package to avoid import cycle apex <-> java. func getBootImageJar(ctx android.SingletonContext, image *bootImageConfig, module android.Module) (int, android.Path) { - // All apex Java libraries have non-installable platform variants, skip them. - if module.IsSkipInstall() { + // Ignore any module that is not listed in the boot image configuration. + name := ctx.ModuleName(module) + index := image.modules.IndexOfJar(name) + if index == -1 { return -1, nil } + // It is an error if a module configured in the boot image does not support + // accessing the dex jar. This is safe because every module that has the same + // name has to have the same module type. jar, hasJar := module.(interface{ DexJarBuildPath() android.Path }) if !hasJar { + ctx.Errorf("module %q configured in boot image %q does not support accessing dex jar", module, image.name) return -1, nil } - name := ctx.ModuleName(module) - index := image.modules.IndexOfJar(name) - if index == -1 { + // It is also an error if the module is not an ApexModule. + if _, ok := module.(android.ApexModule); !ok { + ctx.Errorf("module %q configured in boot image %q does not support being added to an apex", module, image.name) return -1, nil } - // Check that this module satisfies constraints for a particular boot image. - _, isApexModule := module.(android.ApexModule) apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo) - fromUpdatableApex := isApexModule && apexInfo.Updatable - if image.name == artBootImageName { - if isApexModule && len(apexInfo.InApexes) > 0 && allHavePrefix(apexInfo.InApexes, "com.android.art") { - // ok: found the jar in the ART apex - } else if isApexModule && apexInfo.IsForPlatform() && isHostdex(module) { - // exception (skip and continue): special "hostdex" platform variant + + // Now match the apex part of the boot image configuration. + requiredApex := image.modules.Apex(index) + if requiredApex == "platform" { + if len(apexInfo.InApexes) != 0 { + // A platform variant is required but this is for an apex so ignore it. return -1, nil + } + } else if !android.InList(requiredApex, apexInfo.InApexes) { + // An apex variant for a specific apex is required but this is the wrong apex. + return -1, nil + } + + // Check that this module satisfies any boot image specific constraints. + fromUpdatableApex := apexInfo.Updatable + + switch image.name { + case artBootImageName: + if len(apexInfo.InApexes) > 0 && allHavePrefix(apexInfo.InApexes, "com.android.art") { + // ok: found the jar in the ART apex } else if name == "jacocoagent" && ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { // exception (skip and continue): Jacoco platform variant for a coverage build return -1, nil @@ -465,14 +475,15 @@ func getBootImageJar(ctx android.SingletonContext, image *bootImageConfig, modul // error: this jar is part of the platform or a non-updatable apex ctx.Errorf("module %q is not allowed in the ART boot image", name) } - } else if image.name == frameworkBootImageName { + + case frameworkBootImageName: if !fromUpdatableApex { // ok: this jar is part of the platform or a non-updatable apex } else { // error: this jar is part of an updatable apex ctx.Errorf("module %q from updatable apexes %q is not allowed in the framework boot image", name, apexInfo.InApexes) } - } else { + default: panic("unknown boot image: " + image.name) } @@ -495,6 +506,12 @@ func buildBootImage(ctx android.SingletonContext, image *bootImageConfig) *bootI bootDexJars := make(android.Paths, image.modules.Len()) ctx.VisitAllModules(func(module android.Module) { if i, j := getBootImageJar(ctx, image, module); i != -1 { + if existing := bootDexJars[i]; existing != nil { + ctx.Errorf("Multiple dex jars found for %s:%s - %s and %s", + image.modules.Apex(i), image.modules.Jar(i), existing, j) + return + } + bootDexJars[i] = j } }) @@ -506,7 +523,7 @@ func buildBootImage(ctx android.SingletonContext, image *bootImageConfig) *bootI m := image.modules.Jar(i) if ctx.Config().AllowMissingDependencies() { missingDeps = append(missingDeps, m) - bootDexJars[i] = android.PathForOutput(ctx, "missing") + bootDexJars[i] = android.PathForOutput(ctx, "missing/module", m, "from/apex", image.modules.Apex(i)) } else { ctx.Errorf("failed to find a dex jar path for module '%s'"+ ", note that some jars may be filtered out by module constraints", m) @@ -779,7 +796,7 @@ func bootFrameworkProfileRule(ctx android.SingletonContext, image *bootImageConf bootFrameworkProfile = path.Path() } else { missingDeps = append(missingDeps, defaultProfile) - bootFrameworkProfile = android.PathForOutput(ctx, "missing") + bootFrameworkProfile = android.PathForOutput(ctx, "missing", defaultProfile) } profile := image.dir.Join(ctx, "boot.bprof") diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index ce8410ed4..419dc3424 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -177,12 +177,13 @@ func stubFlagsRule(ctx android.SingletonContext) { for moduleList, pathList := range moduleListToPathList { for i := range pathList { if pathList[i] == nil { - pathList[i] = android.PathForOutput(ctx, "missing") + moduleName := (*moduleList)[i] + pathList[i] = android.PathForOutput(ctx, "missing/module", moduleName) if ctx.Config().AllowMissingDependencies() { - missingDeps = append(missingDeps, (*moduleList)[i]) + missingDeps = append(missingDeps, moduleName) } else { ctx.Errorf("failed to find dex jar path for module %q", - (*moduleList)[i]) + moduleName) } } } diff --git a/java/java.go b/java/java.go index 3d121ccea..d44719e99 100644 --- a/java/java.go +++ b/java/java.go @@ -1081,7 +1081,6 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { switch tag { case libTag: deps.classpath = append(deps.classpath, dep.SdkHeaderJars(ctx, j.sdkVersion())...) - // names of sdk libs that are directly depended are exported j.classLoaderContexts.MaybeAddContext(ctx, dep.OptionalImplicitSdkLibrary(), dep.DexJarBuildPath(), dep.DexJarInstallPath()) case staticLibTag: @@ -1093,7 +1092,6 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { deps.bootClasspath = append(deps.bootClasspath, dep.HeaderJars()...) case libTag, instrumentationForTag: deps.classpath = append(deps.classpath, dep.HeaderJars()...) - // sdk lib names from dependencies are re-exported j.classLoaderContexts.AddContextMap(dep.ClassLoaderContexts(), otherName) deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...) pluginJars, pluginClasses, disableTurbine := dep.ExportedPlugins() @@ -1106,8 +1104,6 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { deps.staticJars = append(deps.staticJars, dep.ImplementationJars()...) deps.staticHeaderJars = append(deps.staticHeaderJars, dep.HeaderJars()...) deps.staticResourceJars = append(deps.staticResourceJars, dep.ResourceJars()...) - // sdk lib names from dependencies are re-exported - j.classLoaderContexts.AddContextMap(dep.ClassLoaderContexts(), otherName) deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs()...) pluginJars, pluginClasses, disableTurbine := dep.ExportedPlugins() addPlugins(&deps, pluginJars, pluginClasses...) @@ -1182,6 +1178,9 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { deps.systemModules = &systemModules{outputDir, outputDeps} } } + + // Merge dep's CLC after processing the dep itself (which may add its own <uses-library>). + maybeAddCLCFromDep(module, tag, otherName, j.classLoaderContexts) }) return deps @@ -2815,8 +2814,6 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { switch tag { case libTag, staticLibTag: flags.classpath = append(flags.classpath, dep.HeaderJars()...) - // sdk lib names from dependencies are re-exported - j.classLoaderContexts.AddContextMap(dep.ClassLoaderContexts(), otherName) case bootClasspathTag: flags.bootClasspath = append(flags.bootClasspath, dep.HeaderJars()...) } @@ -2824,10 +2821,12 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { switch tag { case libTag: flags.classpath = append(flags.classpath, dep.SdkHeaderJars(ctx, j.sdkVersion())...) - // names of sdk libs that are directly depended are exported j.classLoaderContexts.AddContext(ctx, otherName, dep.DexJarBuildPath(), dep.DexJarInstallPath()) } } + + // Merge dep's CLC after processing the dep itself (which may add its own <uses-library>). + maybeAddCLCFromDep(module, tag, otherName, j.classLoaderContexts) }) var installFile android.Path @@ -3248,3 +3247,31 @@ var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String var inList = android.InList + +// Add class loader context of a given dependency to the given class loader context, provided that +// all the necessary conditions are met. +func maybeAddCLCFromDep(depModule android.Module, depTag blueprint.DependencyTag, + depName string, clcMap dexpreopt.ClassLoaderContextMap) { + + if dep, ok := depModule.(Dependency); ok { + if depTag == libTag { + // Ok, propagate <uses-library> through non-static library dependencies. + } else if depTag == staticLibTag { + // Propagate <uses-library> through static library dependencies, unless it is a + // component library (such as stubs). Component libraries have a dependency on their + // SDK library, which should not be pulled just because of a static component library. + if comp, isComp := depModule.(SdkLibraryComponentDependency); isComp { + if compName := comp.OptionalImplicitSdkLibrary(); compName != nil { + dep = nil + } + } + } else { + // Don't propagate <uses-library> for other dependency tags. + dep = nil + } + + if dep != nil { + clcMap.AddContextMap(dep.ClassLoaderContexts(), depName) + } + } +} diff --git a/java/robolectric.go b/java/robolectric.go index 419efda88..c821e5bd3 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -260,7 +260,7 @@ func (r *robolectricTest) AndroidMkEntries() []android.AndroidMkEntries { entries := &entriesList[0] entries.ExtraFooters = []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { if s := r.robolectricProperties.Test_options.Shards; s != nil && *s > 1 { numShards := int(*s) shardSize := (len(r.tests) + numShards - 1) / numShards diff --git a/rust/binary.go b/rust/binary.go index af39d383d..c2d97f3ab 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -24,11 +24,6 @@ func init() { } type BinaryCompilerProperties struct { - // Change the rustlibs linkage to select rlib linkage by default for device targets. - // Also link libstd as an rlib as well on device targets. - // Note: This is the default behavior for host targets. - Prefer_rlib *bool `android:"arch_variant"` - // Builds this binary as a static binary. Implies prefer_rlib true. // // Static executables currently only support for bionic targets. Non-bionic targets will not produce a fully static @@ -115,7 +110,7 @@ func (binary *binaryDecorator) nativeCoverage() bool { } func (binary *binaryDecorator) preferRlib() bool { - return Bool(binary.Properties.Prefer_rlib) || Bool(binary.Properties.Static_executable) + return Bool(binary.baseCompiler.Properties.Prefer_rlib) || Bool(binary.Properties.Static_executable) } func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { @@ -156,8 +151,7 @@ func (binary *binaryDecorator) autoDep(ctx BaseModuleContext) autoDep { // Binaries default to dylib dependencies for device, rlib for host. if binary.preferRlib() { return rlibAutoDep - } - if ctx.Device() { + } else if ctx.Device() { return dylibAutoDep } else { return rlibAutoDep diff --git a/rust/compiler.go b/rust/compiler.go index 8d2f09c2b..4312452d2 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -122,6 +122,17 @@ type BaseCompilerProperties struct { // whether to suppress inclusion of standard crates - defaults to false No_stdlibs *bool + + // Change the rustlibs linkage to select rlib linkage by default for device targets. + // Also link libstd as an rlib as well on device targets. + // Note: This is the default behavior for host targets. + // + // This is primarily meant for rust_binary and rust_ffi modules where the default + // linkage of libstd might need to be overridden in some use cases. This should + // generally be avoided with other module types since it may cause collisions at + // linkage if all dependencies of the root binary module do not link against libstd\ + // the same way. + Prefer_rlib *bool `android:"arch_variant"` } type baseCompiler struct { @@ -154,9 +165,15 @@ func (compiler *baseCompiler) coverageOutputZipPath() android.OptionalPath { panic("baseCompiler does not implement coverageOutputZipPath()") } +func (compiler *baseCompiler) preferRlib() bool { + return Bool(compiler.Properties.Prefer_rlib) +} + func (compiler *baseCompiler) stdLinkage(ctx *depsContext) RustLinkage { // For devices, we always link stdlibs in as dylibs by default. - if ctx.Device() { + if compiler.preferRlib() { + return RlibLinkage + } else if ctx.Device() { return DylibLinkage } else { return RlibLinkage diff --git a/rust/config/Android.bp b/rust/config/Android.bp index e0cc4ce07..1f0109f0d 100644 --- a/rust/config/Android.bp +++ b/rust/config/Android.bp @@ -13,6 +13,7 @@ bootstrap_go_package { "toolchain.go", "allowed_list.go", "x86_darwin_host.go", + "x86_linux_bionic_host.go", "x86_linux_host.go", "x86_device.go", "x86_64_device.go", diff --git a/rust/config/x86_linux_bionic_host.go b/rust/config/x86_linux_bionic_host.go new file mode 100644 index 000000000..b1a2c178e --- /dev/null +++ b/rust/config/x86_linux_bionic_host.go @@ -0,0 +1,73 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + LinuxBionicRustFlags = []string{} + LinuxBionicRustLinkFlags = []string{ + "-B${cc_config.ClangBin}", + "-fuse-ld=lld", + "-Wl,--undefined-version", + "-nostdlib", + } +) + +func init() { + registerToolchainFactory(android.LinuxBionic, android.X86_64, linuxBionicX8664ToolchainFactory) + + pctx.StaticVariable("LinuxBionicToolchainRustFlags", strings.Join(LinuxBionicRustFlags, " ")) + pctx.StaticVariable("LinuxBionicToolchainLinkFlags", strings.Join(LinuxBionicRustLinkFlags, " ")) +} + +type toolchainLinuxBionicX8664 struct { + toolchain64Bit +} + +func (toolchainLinuxBionicX8664) Supported() bool { + return true +} + +func (toolchainLinuxBionicX8664) Bionic() bool { + return true +} + +func (t *toolchainLinuxBionicX8664) Name() string { + return "x86_64" +} + +func (t *toolchainLinuxBionicX8664) RustTriple() string { + return "x86_64-linux-android" +} + +func (t *toolchainLinuxBionicX8664) ToolchainLinkFlags() string { + // Prepend the lld flags from cc_config so we stay in sync with cc + return "${cc_config.LinuxBionicLldflags} ${config.LinuxBionicToolchainLinkFlags}" +} + +func (t *toolchainLinuxBionicX8664) ToolchainRustFlags() string { + return "${config.LinuxBionicToolchainRustFlags}" +} + +func linuxBionicX8664ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxBionicX8664Singleton +} + +var toolchainLinuxBionicX8664Singleton Toolchain = &toolchainLinuxBionicX8664{} diff --git a/rust/library.go b/rust/library.go index 971588d8a..9d731e68f 100644 --- a/rust/library.go +++ b/rust/library.go @@ -159,14 +159,6 @@ func (library *libraryDecorator) static() bool { return library.MutatedProperties.VariantIsStatic } -func (library *libraryDecorator) stdLinkage(ctx *depsContext) RustLinkage { - // libraries should only request the RlibLinkage when building a static FFI or when variant is StaticStd - if library.static() || library.MutatedProperties.VariantIsStaticStd { - return RlibLinkage - } - return DefaultLinkage -} - func (library *libraryDecorator) source() bool { return library.MutatedProperties.VariantIsSource } @@ -228,7 +220,9 @@ func (library *libraryDecorator) setSource() { } func (library *libraryDecorator) autoDep(ctx BaseModuleContext) autoDep { - if library.rlib() || library.static() { + if library.preferRlib() { + return rlibAutoDep + } else if library.rlib() || library.static() { return rlibAutoDep } else if library.dylib() || library.shared() { return dylibAutoDep @@ -237,6 +231,15 @@ func (library *libraryDecorator) autoDep(ctx BaseModuleContext) autoDep { } } +func (library *libraryDecorator) stdLinkage(ctx *depsContext) RustLinkage { + if library.static() || library.MutatedProperties.VariantIsStaticStd { + return RlibLinkage + } else if library.baseCompiler.preferRlib() { + return RlibLinkage + } + return DefaultLinkage +} + var _ compiler = (*libraryDecorator)(nil) var _ libraryInterface = (*libraryDecorator)(nil) var _ exportedFlagsProducer = (*libraryDecorator)(nil) diff --git a/rust/library_test.go b/rust/library_test.go index fec3992aa..54cd2a5b3 100644 --- a/rust/library_test.go +++ b/rust/library_test.go @@ -251,6 +251,13 @@ func TestLibstdLinkage(t *testing.T) { srcs: ["foo.rs"], crate_name: "bar", rustlibs: ["libfoo"], + } + rust_ffi { + name: "libbar.prefer_rlib", + srcs: ["foo.rs"], + crate_name: "bar", + rustlibs: ["libfoo"], + prefer_rlib: true, }`) libfooDylib := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_dylib").Module().(*Module) @@ -260,6 +267,9 @@ func TestLibstdLinkage(t *testing.T) { libbarShared := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_shared").Module().(*Module) libbarStatic := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_static").Module().(*Module) + // prefer_rlib works the same for both rust_library and rust_ffi, so a single check is sufficient here. + libbarRlibStd := ctx.ModuleForTests("libbar.prefer_rlib", "android_arm64_armv8-a_shared").Module().(*Module) + if !android.InList("libstd", libfooRlibStatic.Properties.AndroidMkRlibs) { t.Errorf("rlib-std variant for device rust_library_rlib does not link libstd as an rlib") } @@ -279,4 +289,8 @@ func TestLibstdLinkage(t *testing.T) { if !android.InList("libfoo.rlib-std", libbarStatic.Properties.AndroidMkRlibs) { t.Errorf("Device rust_ffi_static does not link dependent rustlib rlib-std variant") } + if !android.InList("libstd", libbarRlibStd.Properties.AndroidMkRlibs) { + t.Errorf("rust_ffi with prefer_rlib does not link libstd as an rlib") + } + } diff --git a/rust/project_json.go b/rust/project_json.go index c4d60ad53..32ce6f4b0 100644 --- a/rust/project_json.go +++ b/rust/project_json.go @@ -45,10 +45,11 @@ type rustProjectDep struct { } type rustProjectCrate struct { - RootModule string `json:"root_module"` - Edition string `json:"edition,omitempty"` - Deps []rustProjectDep `json:"deps"` - Cfgs []string `json:"cfgs"` + DisplayName string `json:"display_name"` + RootModule string `json:"root_module"` + Edition string `json:"edition,omitempty"` + Deps []rustProjectDep `json:"deps"` + Cfgs []string `json:"cfgs"` } type rustProjectJson struct { @@ -58,13 +59,13 @@ type rustProjectJson struct { // crateInfo is used during the processing to keep track of the known crates. type crateInfo struct { - ID int - Deps map[string]int + Idx int // Index of the crate in rustProjectJson.Crates slice. + Deps map[string]int // The keys are the module names and not the crate names. } type projectGeneratorSingleton struct { project rustProjectJson - knownCrates map[string]crateInfo + knownCrates map[string]crateInfo // Keys are module names. } func rustProjectGeneratorSingleton() android.Singleton { @@ -75,66 +76,129 @@ func init() { android.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton) } -// crateSource finds the main source file (.rs) for a crate. -func crateSource(ctx android.SingletonContext, rModule *Module, comp *baseCompiler) (string, bool) { - srcs := comp.Properties.Srcs - if len(srcs) != 0 { - return path.Join(ctx.ModuleDir(rModule), srcs[0]), true - } +// sourceProviderVariantSource returns the path to the source file if this +// module variant should be used as a priority. +// +// SourceProvider modules may have multiple variants considered as source +// (e.g., x86_64 and armv8). For a module available on device, use the source +// generated for the target. For a host-only module, use the source generated +// for the host. +func sourceProviderVariantSource(ctx android.SingletonContext, rModule *Module) (string, bool) { rustLib, ok := rModule.compiler.(*libraryDecorator) if !ok { return "", false } - if !rustLib.source() { + if rustLib.source() { + switch rModule.hod { + case android.HostSupported, android.HostSupportedNoCross: + if rModule.Target().String() == ctx.Config().BuildOSTarget.String() { + src := rustLib.sourceProvider.Srcs()[0] + return src.String(), true + } + default: + if rModule.Target().String() == ctx.Config().AndroidFirstDeviceTarget.String() { + src := rustLib.sourceProvider.Srcs()[0] + return src.String(), true + } + } + } + return "", false +} + +// sourceProviderSource finds the main source file of a source-provider crate. +func sourceProviderSource(ctx android.SingletonContext, rModule *Module) (string, bool) { + rustLib, ok := rModule.compiler.(*libraryDecorator) + if !ok { return "", false } - // It is a SourceProvider module. If this module is host only, uses the variation for the host. - // Otherwise, use the variation for the primary target. - switch rModule.hod { - case android.HostSupported: - case android.HostSupportedNoCross: - if rModule.Target().String() != ctx.Config().BuildOSTarget.String() { - return "", false + if rustLib.source() { + // This is a source-variant, check if we are the right variant + // depending on the module configuration. + if src, ok := sourceProviderVariantSource(ctx, rModule); ok { + return src, true } - default: - if rModule.Target().String() != ctx.Config().AndroidFirstDeviceTarget.String() { - return "", false + } + foundSource := false + sourceSrc := "" + // Find the variant with the source and return its. + ctx.VisitAllModuleVariants(rModule, func(variant android.Module) { + if foundSource { + return + } + // All variants of a source provider library are libraries. + rVariant, _ := variant.(*Module) + variantLib, _ := rVariant.compiler.(*libraryDecorator) + if variantLib.source() { + sourceSrc, ok = sourceProviderVariantSource(ctx, rVariant) + if ok { + foundSource = true + } } + }) + if !foundSource { + fmt.Errorf("No valid source for source provider found: %v\n", rModule) + } + return sourceSrc, foundSource +} + +// crateSource finds the main source file (.rs) for a crate. +func crateSource(ctx android.SingletonContext, rModule *Module, comp *baseCompiler) (string, bool) { + // Basic libraries, executables and tests. + srcs := comp.Properties.Srcs + if len(srcs) != 0 { + return path.Join(ctx.ModuleDir(rModule), srcs[0]), true + } + // SourceProvider libraries. + if rModule.sourceProvider != nil { + return sourceProviderSource(ctx, rModule) } - src := rustLib.sourceProvider.Srcs()[0] - return src.String(), true + return "", false } +// mergeDependencies visits all the dependencies for module and updates crate and deps +// with any new dependency. func (singleton *projectGeneratorSingleton) mergeDependencies(ctx android.SingletonContext, - module android.Module, crate *rustProjectCrate, deps map[string]int) { + module *Module, crate *rustProjectCrate, deps map[string]int) { ctx.VisitDirectDeps(module, func(child android.Module) { - childId, childCrateName, ok := singleton.appendLibraryAndDeps(ctx, child) - if !ok { - return - } // Skip intra-module dependencies (i.e., generated-source library depending on the source variant). if module.Name() == child.Name() { return } - if _, ok = deps[ctx.ModuleName(child)]; ok { + // Skip unsupported modules. + rChild, compChild, ok := isModuleSupported(ctx, child) + if !ok { + return + } + // For unknown dependency, add it first. + var childId int + cInfo, known := singleton.knownCrates[rChild.Name()] + if !known { + childId, ok = singleton.addCrate(ctx, rChild, compChild) + if !ok { + return + } + } else { + childId = cInfo.Idx + } + // Is this dependency known already? + if _, ok = deps[child.Name()]; ok { return } - crate.Deps = append(crate.Deps, rustProjectDep{Crate: childId, Name: childCrateName}) - deps[ctx.ModuleName(child)] = childId + crate.Deps = append(crate.Deps, rustProjectDep{Crate: childId, Name: rChild.CrateName()}) + deps[child.Name()] = childId }) } -// appendLibraryAndDeps creates a rustProjectCrate for the module argument and appends it to singleton.project. -// It visits the dependencies of the module depth-first so the dependency ID can be added to the current module. If the -// current module is already in singleton.knownCrates, its dependencies are merged. Returns a tuple (id, crate_name, ok). -func (singleton *projectGeneratorSingleton) appendLibraryAndDeps(ctx android.SingletonContext, module android.Module) (int, string, bool) { +// isModuleSupported returns the RustModule and baseCompiler if the module +// should be considered for inclusion in rust-project.json. +func isModuleSupported(ctx android.SingletonContext, module android.Module) (*Module, *baseCompiler, bool) { rModule, ok := module.(*Module) if !ok { - return 0, "", false + return nil, nil, false } if rModule.compiler == nil { - return 0, "", false + return nil, nil, false } var comp *baseCompiler switch c := rModule.compiler.(type) { @@ -145,35 +209,57 @@ func (singleton *projectGeneratorSingleton) appendLibraryAndDeps(ctx android.Sin case *testDecorator: comp = c.binaryDecorator.baseCompiler default: - return 0, "", false - } - moduleName := ctx.ModuleName(module) - crateName := rModule.CrateName() - if cInfo, ok := singleton.knownCrates[moduleName]; ok { - // We have seen this crate already; merge any new dependencies. - crate := singleton.project.Crates[cInfo.ID] - singleton.mergeDependencies(ctx, module, &crate, cInfo.Deps) - singleton.project.Crates[cInfo.ID] = crate - return cInfo.ID, crateName, true - } - crate := rustProjectCrate{Deps: make([]rustProjectDep, 0), Cfgs: make([]string, 0)} + return nil, nil, false + } + return rModule, comp, true +} + +// addCrate adds a crate to singleton.project.Crates ensuring that required +// dependencies are also added. It returns the index of the new crate in +// singleton.project.Crates +func (singleton *projectGeneratorSingleton) addCrate(ctx android.SingletonContext, rModule *Module, comp *baseCompiler) (int, bool) { rootModule, ok := crateSource(ctx, rModule, comp) if !ok { - return 0, "", false + fmt.Errorf("Unable to find source for valid module: %v", rModule) + return 0, false + } + + crate := rustProjectCrate{ + DisplayName: rModule.Name(), + RootModule: rootModule, + Edition: comp.edition(), + Deps: make([]rustProjectDep, 0), + Cfgs: make([]string, 0), } - crate.RootModule = rootModule - crate.Edition = comp.edition() deps := make(map[string]int) - singleton.mergeDependencies(ctx, module, &crate, deps) + singleton.mergeDependencies(ctx, rModule, &crate, deps) - id := len(singleton.project.Crates) - singleton.knownCrates[moduleName] = crateInfo{ID: id, Deps: deps} + idx := len(singleton.project.Crates) + singleton.knownCrates[rModule.Name()] = crateInfo{Idx: idx, Deps: deps} singleton.project.Crates = append(singleton.project.Crates, crate) // rust-analyzer requires that all crates belong to at least one root: // https://github.com/rust-analyzer/rust-analyzer/issues/4735. singleton.project.Roots = append(singleton.project.Roots, path.Dir(crate.RootModule)) - return id, crateName, true + return idx, true +} + +// appendCrateAndDependencies creates a rustProjectCrate for the module argument and appends it to singleton.project. +// It visits the dependencies of the module depth-first so the dependency ID can be added to the current module. If the +// current module is already in singleton.knownCrates, its dependencies are merged. +func (singleton *projectGeneratorSingleton) appendCrateAndDependencies(ctx android.SingletonContext, module android.Module) { + rModule, comp, ok := isModuleSupported(ctx, module) + if !ok { + return + } + // If we have seen this crate already; merge any new dependencies. + if cInfo, ok := singleton.knownCrates[module.Name()]; ok { + crate := singleton.project.Crates[cInfo.Idx] + singleton.mergeDependencies(ctx, rModule, &crate, cInfo.Deps) + singleton.project.Crates[cInfo.Idx] = crate + return + } + singleton.addCrate(ctx, rModule, comp) } func (singleton *projectGeneratorSingleton) GenerateBuildActions(ctx android.SingletonContext) { @@ -183,7 +269,7 @@ func (singleton *projectGeneratorSingleton) GenerateBuildActions(ctx android.Sin singleton.knownCrates = make(map[string]crateInfo) ctx.VisitAllModules(func(module android.Module) { - singleton.appendLibraryAndDeps(ctx, module) + singleton.appendCrateAndDependencies(ctx, module) }) path := android.PathForOutput(ctx, rustProjectJsonFileName) diff --git a/rust/project_json_test.go b/rust/project_json_test.go index aff16978b..ba66215c5 100644 --- a/rust/project_json_test.go +++ b/rust/project_json_test.go @@ -119,9 +119,9 @@ func TestProjectJsonDep(t *testing.T) { func TestProjectJsonBinary(t *testing.T) { bp := ` rust_binary { - name: "liba", - srcs: ["a/src/lib.rs"], - crate_name: "a" + name: "libz", + srcs: ["z/src/lib.rs"], + crate_name: "z" } ` jsonContent := testProjectJson(t, bp) @@ -132,7 +132,7 @@ func TestProjectJsonBinary(t *testing.T) { if !ok { t.Fatalf("Unexpected type for root_module: %v", crate["root_module"]) } - if rootModule == "a/src/lib.rs" { + if rootModule == "z/src/lib.rs" { return } } @@ -142,10 +142,10 @@ func TestProjectJsonBinary(t *testing.T) { func TestProjectJsonBindGen(t *testing.T) { bp := ` rust_library { - name: "liba", - srcs: ["src/lib.rs"], + name: "libd", + srcs: ["d/src/lib.rs"], rlibs: ["libbindings1"], - crate_name: "a" + crate_name: "d" } rust_bindgen { name: "libbindings1", @@ -155,10 +155,10 @@ func TestProjectJsonBindGen(t *testing.T) { wrapper_src: "src/any.h", } rust_library_host { - name: "libb", - srcs: ["src/lib.rs"], + name: "libe", + srcs: ["e/src/lib.rs"], rustlibs: ["libbindings2"], - crate_name: "b" + crate_name: "e" } rust_bindgen_host { name: "libbindings2", @@ -190,6 +190,19 @@ func TestProjectJsonBindGen(t *testing.T) { } } } + // Check that liba depends on libbindings1 + if strings.Contains(rootModule, "d/src/lib.rs") { + found := false + for _, depName := range validateDependencies(t, crate) { + if depName == "bindings1" { + found = true + break + } + } + if !found { + t.Errorf("liba does not depend on libbindings1: %v", crate) + } + } } } diff --git a/rust/rust.go b/rust/rust.go index 38caad373..1ceedf397 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -67,6 +67,9 @@ type BaseProperties struct { SubName string `blueprint:"mutated"` + // Minimum sdk version that the artifact should support when it runs as part of mainline modules(APEX). + Min_sdk_version *string + PreventInstall bool HideFromMake bool } @@ -1076,7 +1079,29 @@ func (mod *Module) HostToolPath() android.OptionalPath { var _ android.ApexModule = (*Module)(nil) +func (mod *Module) minSdkVersion() string { + return String(mod.Properties.Min_sdk_version) +} + func (mod *Module) ShouldSupportSdkVersion(ctx android.BaseModuleContext, sdkVersion android.ApiLevel) error { + minSdkVersion := mod.minSdkVersion() + if minSdkVersion == "apex_inherit" { + return nil + } + if minSdkVersion == "" { + return fmt.Errorf("min_sdk_version is not specificed") + } + + // Not using nativeApiLevelFromUser because the context here is not + // necessarily a native context. + ver, err := android.ApiLevelFromUser(ctx, minSdkVersion) + if err != nil { + return err + } + + if ver.GreaterThan(sdkVersion) { + return fmt.Errorf("newer SDK(%v)", ver) + } return nil } diff --git a/rust/testing.go b/rust/testing.go index a8496d983..8648dc39a 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -78,6 +78,7 @@ func GatherRequiredDepsForTest() string { nocrt: true, system_shared_libs: [], apex_available: ["//apex_available:platform", "//apex_available:anyapex"], + min_sdk_version: "29", } cc_library { name: "libprotobuf-cpp-full", @@ -95,6 +96,7 @@ func GatherRequiredDepsForTest() string { native_coverage: false, sysroot: true, apex_available: ["//apex_available:platform", "//apex_available:anyapex"], + min_sdk_version: "29", } rust_library { name: "libtest", @@ -105,6 +107,7 @@ func GatherRequiredDepsForTest() string { native_coverage: false, sysroot: true, apex_available: ["//apex_available:platform", "//apex_available:anyapex"], + min_sdk_version: "29", } rust_library { name: "libprotobuf", diff --git a/scripts/build-aml-prebuilts.sh b/scripts/build-aml-prebuilts.sh index 0c868ea11..de22c459f 100755 --- a/scripts/build-aml-prebuilts.sh +++ b/scripts/build-aml-prebuilts.sh @@ -94,7 +94,13 @@ cat > ${SOONG_VARS}.new << EOF "Allow_missing_dependencies": ${SOONG_ALLOW_MISSING_DEPENDENCIES:-false}, "Unbundled_build": ${TARGET_BUILD_UNBUNDLED:-false}, - "UseGoma": ${USE_GOMA} + "UseGoma": ${USE_GOMA}, + + "VendorVars": { + "art_module": { + "source_build": "${ENABLE_ART_SOURCE_BUILD:-false}" + } + } } EOF diff --git a/sdk/sdk.go b/sdk/sdk.go index 50b0886d4..f3d075022 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -311,7 +311,7 @@ func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { DistFiles: android.MakeDefaultDistFiles(s.snapshotFile.Path()), Include: "$(BUILD_PHONY_PACKAGE)", ExtraFooters: []android.AndroidMkExtraFootersFunc{ - func(w io.Writer, name, prefix, moduleDir string, entries *android.AndroidMkEntries) { + func(w io.Writer, name, prefix, moduleDir string) { // Allow the sdk to be built by simply passing its name on the command line. fmt.Fprintln(w, ".PHONY:", s.Name()) fmt.Fprintln(w, s.Name()+":", s.snapshotFile.String()) diff --git a/shared/paths.go b/shared/paths.go index 24ba057aa..f5dc5dd2e 100644 --- a/shared/paths.go +++ b/shared/paths.go @@ -20,21 +20,23 @@ import ( "path/filepath" ) +// A SharedPaths represents a list of paths that are shared between +// soong_ui and soong. +type SharedPaths interface { + // BazelMetricsDir returns the path where a set of bazel profile + // files are stored for later processed by the metrics pipeline. + BazelMetricsDir() string +} + // Given the out directory, returns the root of the temp directory (to be cleared at the start of each execution of Soong) func TempDirForOutDir(outDir string) (tempPath string) { return filepath.Join(outDir, ".temp") } -// BazelMetricsDir returns the path where a set of bazel profile -// files are stored for later processed by the metrics pipeline. -func BazelMetricsDir(outDir string) string { - return filepath.Join(outDir, "bazel_metrics") -} - // BazelMetricsFilename returns the bazel profile filename based // on the action name. This is to help to store a set of bazel // profiles since bazel may execute multiple times during a single // build. -func BazelMetricsFilename(outDir, actionName string) string { - return filepath.Join(BazelMetricsDir(outDir), actionName+"_bazel_profile.gz") +func BazelMetricsFilename(s SharedPaths, actionName string) string { + return filepath.Join(s.BazelMetricsDir(), actionName+"_bazel_profile.gz") } diff --git a/ui/build/bazel.go b/ui/build/bazel.go index 1cab8f8a7..4f2d6455c 100644 --- a/ui/build/bazel.go +++ b/ui/build/bazel.go @@ -78,7 +78,10 @@ func runBazel(ctx Context, config Config) { bazelEnv["PACKAGE_NINJA"] = config.KatiPackageNinjaFile() bazelEnv["SOONG_NINJA"] = config.SoongNinjaFile() + // NOTE: When Bazel is used, config.DistDir() is rigged to return a fake distdir under config.OutDir() + // This is to ensure that Bazel can actually write there. See config.go for more details. bazelEnv["DIST_DIR"] = config.DistDir() + bazelEnv["SHELL"] = "/bin/bash" // `tools/bazel` is the default entry point for executing Bazel in the AOSP @@ -101,7 +104,7 @@ func runBazel(ctx Context, config Config) { // ninja_build target. "--output_groups="+outputGroups, // Generate a performance profile - "--profile="+filepath.Join(shared.BazelMetricsFilename(config.OutDir(), actionName)), + "--profile="+filepath.Join(shared.BazelMetricsFilename(config, actionName)), "--slim_profile=true", ) @@ -156,9 +159,6 @@ func runBazel(ctx Context, config Config) { "//:"+config.TargetProduct()+"-"+config.TargetBuildVariant(), ) - // Print the full command line for debugging purposes. - ctx.Println(cmd.Cmd) - // Execute the command at the root of the directory. cmd.Dir = filepath.Join(config.OutDir(), "..") @@ -172,8 +172,11 @@ func runBazel(ctx Context, config Config) { fmt.Fprintf(bazelEnvStringBuffer, "%s=%s ", k, v) } - // Print the full command line (including environment variables) for debugging purposes. - ctx.Println("Bazel command line: " + bazelEnvStringBuffer.String() + cmd.Cmd.String() + "\n") + // Print the implicit command line + ctx.Println("Bazel implicit command line: " + strings.Join(cmd.Environment.Environ(), " ") + " " + cmd.Cmd.String() + "\n") + + // Print the explicit command line too + ctx.Println("Bazel explicit command line: " + bazelEnvStringBuffer.String() + cmd.Cmd.String() + "\n") // Execute the build command. cmd.RunAndStreamOrFatal() @@ -189,13 +192,14 @@ func runBazel(ctx Context, config Config) { // currently hardcoded as ninja_build.output_root. bazelNinjaBuildOutputRoot := filepath.Join(outputBasePath, "..", "out") - ctx.Println("Creating output symlinks..") - symlinkOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".") + ctx.Println("Populating output directory...") + populateOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".") } // For all files F recursively under rootPath/relativePath, creates symlinks // such that OutDir/F resolves to rootPath/F via symlinks. -func symlinkOutdir(ctx Context, config Config, rootPath string, relativePath string) { +// NOTE: For distdir paths we rename files instead of creating symlinks, so that the distdir is independent. +func populateOutdir(ctx Context, config Config, rootPath string, relativePath string) { destDir := filepath.Join(rootPath, relativePath) os.MkdirAll(destDir, 0755) files, err := ioutil.ReadDir(destDir) @@ -220,7 +224,7 @@ func symlinkOutdir(ctx Context, config Config, rootPath string, relativePath str if srcLstatErr == nil { if srcLstatResult.IsDir() && destLstatResult.IsDir() { // src and dest are both existing dirs - recurse on the dest dir contents... - symlinkOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name())) + populateOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name())) } else { // Ignore other pre-existing src files (could be pre-existing files, directories, symlinks, ...) // This can arise for files which are generated under OutDir outside of soong_build, such as .bootstrap files. @@ -231,9 +235,17 @@ func symlinkOutdir(ctx Context, config Config, rootPath string, relativePath str ctx.Fatalf("Unable to Lstat src %s: %s", srcPath, srcLstatErr) } - // src does not exist, so try to create a src -> dest symlink (i.e. a Soong path -> Bazel path symlink) - if symlinkErr := os.Symlink(destPath, srcPath); symlinkErr != nil { - ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, symlinkErr) + if strings.Contains(destDir, config.DistDir()) { + // We need to make a "real" file/dir instead of making a symlink (because the distdir can't have symlinks) + // Rename instead of copy in order to save disk space. + if err := os.Rename(destPath, srcPath); err != nil { + ctx.Fatalf("Unable to rename %s -> %s due to error %s", srcPath, destPath, err) + } + } else { + // src does not exist, so try to create a src -> dest symlink (i.e. a Soong path -> Bazel path symlink) + if err := os.Symlink(destPath, srcPath); err != nil { + ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, err) + } } } } diff --git a/ui/build/build.go b/ui/build/build.go index e8f0fc49d..926da3137 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -302,7 +302,7 @@ func distGzipFile(ctx Context, config Config, src string, subDirs ...string) { } subDir := filepath.Join(subDirs...) - destDir := filepath.Join(config.DistDir(), "soong_ui", subDir) + destDir := filepath.Join(config.RealDistDir(), "soong_ui", subDir) if err := os.MkdirAll(destDir, 0777); err != nil { // a+rwx ctx.Printf("failed to mkdir %s: %s", destDir, err.Error()) @@ -321,7 +321,7 @@ func distFile(ctx Context, config Config, src string, subDirs ...string) { } subDir := filepath.Join(subDirs...) - destDir := filepath.Join(config.DistDir(), "soong_ui", subDir) + destDir := filepath.Join(config.RealDistDir(), "soong_ui", subDir) if err := os.MkdirAll(destDir, 0777); err != nil { // a+rwx ctx.Printf("failed to mkdir %s: %s", destDir, err.Error()) diff --git a/ui/build/config.go b/ui/build/config.go index c9911f3d1..ecca4de0d 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -65,6 +65,12 @@ type configImpl struct { brokenNinjaEnvVars []string pathReplaced bool + + useBazel bool + + // During Bazel execution, Bazel cannot write outside OUT_DIR. + // So if DIST_DIR is set to an external dir (outside of OUT_DIR), we need to rig it temporarily and then migrate files at the end of the build. + riggedDistDirForBazel string } const srcDirFileCheck = "build/soong/root.bp" @@ -221,7 +227,7 @@ func NewConfig(ctx Context, args ...string) Config { ctx.Fatalln("Directory names containing spaces are not supported") } - if distDir := ret.DistDir(); strings.ContainsRune(distDir, ' ') { + if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") ctx.Println() ctx.Printf("%q\n", distDir) @@ -275,16 +281,26 @@ func NewConfig(ctx Context, args ...string) Config { } } - bpd := shared.BazelMetricsDir(ret.OutDir()) + bpd := ret.BazelMetricsDir() if err := os.RemoveAll(bpd); err != nil { ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err) } + + ret.useBazel = ret.environ.IsEnvTrue("USE_BAZEL") + if ret.UseBazel() { if err := os.MkdirAll(bpd, 0777); err != nil { ctx.Fatalf("Failed to create bazel profile directory %q: %v", bpd, err) } } + if ret.UseBazel() { + ret.riggedDistDirForBazel = filepath.Join(ret.OutDir(), "dist") + } else { + // Not rigged + ret.riggedDistDirForBazel = ret.distDir + } + c := Config{ret} storeConfigMetrics(ctx, c) return c @@ -697,6 +713,14 @@ func (c *configImpl) OutDir() string { } func (c *configImpl) DistDir() string { + if c.UseBazel() { + return c.riggedDistDirForBazel + } else { + return c.distDir + } +} + +func (c *configImpl) RealDistDir() string { return c.distDir } @@ -863,13 +887,7 @@ func (c *configImpl) UseRBE() bool { } func (c *configImpl) UseBazel() bool { - if v, ok := c.environ.Get("USE_BAZEL"); ok { - v = strings.TrimSpace(v) - if v != "" && v != "false" { - return true - } - } - return false + return c.useBazel } func (c *configImpl) StartRBE() bool { @@ -886,14 +904,14 @@ func (c *configImpl) StartRBE() bool { return true } -func (c *configImpl) logDir() string { +func (c *configImpl) rbeLogDir() string { for _, f := range []string{"RBE_log_dir", "FLAG_log_dir"} { if v, ok := c.environ.Get(f); ok { return v } } if c.Dist() { - return filepath.Join(c.DistDir(), "logs") + return c.LogsDir() } return c.OutDir() } @@ -904,7 +922,7 @@ func (c *configImpl) rbeStatsOutputDir() string { return v } } - return c.logDir() + return c.rbeLogDir() } func (c *configImpl) rbeLogPath() string { @@ -913,7 +931,7 @@ func (c *configImpl) rbeLogPath() string { return v } } - return fmt.Sprintf("text://%v/reproxy_log.txt", c.logDir()) + return fmt.Sprintf("text://%v/reproxy_log.txt", c.rbeLogDir()) } func (c *configImpl) rbeExecRoot() string { @@ -1121,3 +1139,21 @@ func (c *configImpl) MetricsUploaderApp() string { } return "" } + +// LogsDir returns the logs directory where build log and metrics +// files are located. By default, the logs directory is the out +// directory. If the argument dist is specified, the logs directory +// is <dist_dir>/logs. +func (c *configImpl) LogsDir() string { + if c.Dist() { + // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files + return filepath.Join(c.RealDistDir(), "logs") + } + return c.OutDir() +} + +// BazelMetricsDir returns the <logs dir>/bazel_metrics directory +// where the bazel profiles are located. +func (c *configImpl) BazelMetricsDir() string { + return filepath.Join(c.LogsDir(), "bazel_metrics") +} diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 64f3d4c38..d9c33f61e 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -74,7 +74,7 @@ func sockAddr(dir string) (string, error) { func getRBEVars(ctx Context, config Config) map[string]string { vars := map[string]string{ "RBE_log_path": config.rbeLogPath(), - "RBE_log_dir": config.logDir(), + "RBE_log_dir": config.rbeLogDir(), "RBE_re_proxy": config.rbeReproxy(), "RBE_exec_root": config.rbeExecRoot(), "RBE_output_dir": config.rbeStatsOutputDir(), diff --git a/ui/build/test_build.go b/ui/build/test_build.go index 31646802e..41acc2618 100644 --- a/ui/build/test_build.go +++ b/ui/build/test_build.go @@ -68,6 +68,7 @@ func testForDanglingRules(ctx Context, config Config) { miniBootstrapDir := filepath.Join(outDir, "soong", ".minibootstrap") modulePathsDir := filepath.Join(outDir, ".module_paths") variablesFilePath := filepath.Join(outDir, "soong", "soong.variables") + // dexpreopt.config is an input to the soong_docs action, which runs the // soong_build primary builder. However, this file is created from $(shell) // invocation at Kati parse time, so it's not an explicit output of any @@ -75,6 +76,9 @@ func testForDanglingRules(ctx Context, config Config) { // treated as an source file. dexpreoptConfigFilePath := filepath.Join(outDir, "soong", "dexpreopt.config") + // out/build_date.txt is considered a "source file" + buildDatetimeFilePath := filepath.Join(outDir, "build_date.txt") + danglingRules := make(map[string]bool) scanner := bufio.NewScanner(stdout) @@ -88,7 +92,8 @@ func testForDanglingRules(ctx Context, config Config) { strings.HasPrefix(line, miniBootstrapDir) || strings.HasPrefix(line, modulePathsDir) || line == variablesFilePath || - line == dexpreoptConfigFilePath { + line == dexpreoptConfigFilePath || + line == buildDatetimeFilePath { // Leaf node is in one of Soong's bootstrap directories, which do not have // full build rules in the primary build.ninja file. continue diff --git a/ui/build/upload.go b/ui/build/upload.go index 4f30136b9..55ca800b5 100644 --- a/ui/build/upload.go +++ b/ui/build/upload.go @@ -40,13 +40,42 @@ var ( tmpDir = ioutil.TempDir ) +// pruneMetricsFiles iterates the list of paths, checking if a path exist. +// If a path is a file, it is added to the return list. If the path is a +// directory, a recursive call is made to add the children files of the +// path. +func pruneMetricsFiles(paths []string) []string { + var metricsFiles []string + for _, p := range paths { + fi, err := os.Stat(p) + // Some paths passed may not exist. For example, build errors protobuf + // file may not exist since the build was successful. + if err != nil { + continue + } + + if fi.IsDir() { + if l, err := ioutil.ReadDir(p); err == nil { + files := make([]string, 0, len(l)) + for _, fi := range l { + files = append(files, filepath.Join(p, fi.Name())) + } + metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...) + } + } else { + metricsFiles = append(metricsFiles, p) + } + } + return metricsFiles +} + // UploadMetrics uploads a set of metrics files to a server for analysis. An // uploader full path is specified in ANDROID_ENABLE_METRICS_UPLOAD environment // variable in order to upload the set of metrics files. The metrics files are // first copied to a temporary directory and the uploader is then executed in // the background to allow the user/system to continue working. Soong communicates // to the uploader through the upload_proto raw protobuf file. -func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, files ...string) { +func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) { ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") defer ctx.EndTrace() @@ -56,15 +85,8 @@ func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted t return } - // Some files passed in to this function may not exist. For example, - // build errors protobuf file may not exist since the build was successful. - var metricsFiles []string - for _, f := range files { - if _, err := os.Stat(f); err == nil { - metricsFiles = append(metricsFiles, f) - } - } - + // Several of the files might be directories. + metricsFiles := pruneMetricsFiles(paths) if len(metricsFiles) == 0 { return } diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go index 768b03112..b740c1120 100644 --- a/ui/build/upload_test.go +++ b/ui/build/upload_test.go @@ -19,6 +19,8 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" + "sort" "strconv" "strings" "testing" @@ -27,6 +29,49 @@ import ( "android/soong/ui/logger" ) +func TestPruneMetricsFiles(t *testing.T) { + rootDir := t.TempDir() + + dirs := []string{ + filepath.Join(rootDir, "d1"), + filepath.Join(rootDir, "d1", "d2"), + filepath.Join(rootDir, "d1", "d2", "d3"), + } + + files := []string{ + filepath.Join(rootDir, "d1", "f1"), + filepath.Join(rootDir, "d1", "d2", "f1"), + filepath.Join(rootDir, "d1", "d2", "d3", "f1"), + } + + for _, d := range dirs { + if err := os.MkdirAll(d, 0777); err != nil { + t.Fatalf("got %v, expecting nil error for making directory %q", err, d) + } + } + + for _, f := range files { + if err := ioutil.WriteFile(f, []byte{}, 0777); err != nil { + t.Fatalf("got %v, expecting nil error on writing file %q", err, f) + } + } + + want := []string{ + filepath.Join(rootDir, "d1", "f1"), + filepath.Join(rootDir, "d1", "d2", "f1"), + filepath.Join(rootDir, "d1", "d2", "d3", "f1"), + } + + got := pruneMetricsFiles([]string{rootDir}) + + sort.Strings(got) + sort.Strings(want) + + if !reflect.DeepEqual(got, want) { + t.Errorf("got %q, want %q after pruning metrics files", got, want) + } +} + func TestUploadMetrics(t *testing.T) { ctx := testContext() tests := []struct { |