blob: a413ebef3126623de82a571b74a66ce6ff2d6b79 [file] [log] [blame]
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compliance
import (
"fmt"
"io"
"io/fs"
"os"
"strings"
"sync"
"android/soong/compliance/license_metadata_proto"
"google.golang.org/protobuf/encoding/prototext"
)
var (
// ConcurrentReaders is the size of the task pool for limiting resource usage e.g. open files.
ConcurrentReaders = 5
)
type globalFS struct{}
var _ fs.FS = globalFS{}
var _ fs.StatFS = globalFS{}
func (s globalFS) Open(name string) (fs.File, error) {
return os.Open(name)
}
func (s globalFS) Stat(name string) (fs.FileInfo, error) {
return os.Stat(name)
}
var FS globalFS
// GetFS returns a filesystem for accessing files under the OUT_DIR environment variable.
func GetFS(outDir string) fs.FS {
if len(outDir) > 0 {
return os.DirFS(outDir)
}
return os.DirFS(".")
}
// result describes the outcome of reading and parsing a single license metadata file.
type result struct {
// file identifies the path to the license metadata file
file string
// target contains the parsed metadata or nil if an error
target *TargetNode
// err is nil unless an error occurs
err error
}
// receiver coordinates the tasks for reading and parsing license metadata files.
type receiver struct {
// lg accumulates the read metadata and becomes the final resulting LicenseGraph.
lg *LicenseGraph
// rootFS locates the root of the file system from which to read the files.
rootFS fs.FS
// stderr identifies the error output writer.
stderr io.Writer
// task provides a fixed-size task pool to limit concurrent open files etc.
task chan bool
// results returns one license metadata file result at a time.
results chan *result
// wg detects when done
wg sync.WaitGroup
}
// ReadLicenseGraph reads and parses `files` and their dependencies into a LicenseGraph.
//
// `files` become the root files of the graph for top-down walks of the graph.
func ReadLicenseGraph(rootFS fs.FS, stderr io.Writer, files []string) (*LicenseGraph, error) {
if len(files) == 0 {
return nil, fmt.Errorf("no license metadata to analyze")
}
if ConcurrentReaders < 1 {
return nil, fmt.Errorf("need at least one task in pool")
}
lg := newLicenseGraph()
for _, f := range files {
if strings.HasSuffix(f, "meta_lic") {
lg.rootFiles = append(lg.rootFiles, f)
} else {
lg.rootFiles = append(lg.rootFiles, f+".meta_lic")
}
}
recv := &receiver{
lg: lg,
rootFS: rootFS,
stderr: stderr,
task: make(chan bool, ConcurrentReaders),
results: make(chan *result, ConcurrentReaders),
wg: sync.WaitGroup{},
}
for i := 0; i < ConcurrentReaders; i++ {
recv.task <- true
}
readFiles := func() {
lg.mu.Lock()
// identify the metadata files to schedule reading tasks for
for _, f := range lg.rootFiles {
lg.targets[f] = nil
}
lg.mu.Unlock()
// schedule tasks to read the files
for _, f := range lg.rootFiles {
readFile(recv, f)
}
// schedule a task to wait until finished and close the channel.
go func() {
recv.wg.Wait()
close(recv.task)
close(recv.results)
}()
}
go readFiles()
// tasks to read license metadata files are scheduled; read and process results from channel
var err error
for recv.results != nil {
select {
case r, ok := <-recv.results:
if ok {
// handle errors by nil'ing ls, setting err, and clobbering results channel
if r.err != nil {
err = r.err
fmt.Fprintf(recv.stderr, "%s\n", err.Error())
lg = nil
recv.results = nil
continue
}
// record the parsed metadata (guarded by mutex)
recv.lg.mu.Lock()
lg.targets[r.target.name] = r.target
recv.lg.mu.Unlock()
} else {
// finished -- nil the results channel
recv.results = nil
}
}
}
if lg != nil {
esize := 0
for _, tn := range lg.targets {
esize += len(tn.proto.Deps)
}
lg.edges = make(TargetEdgeList, 0, esize)
for _, tn := range lg.targets {
tn.licenseConditions = LicenseConditionSetFromNames(tn.proto.LicenseConditions...)
err = addDependencies(lg, tn)
if err != nil {
return nil, fmt.Errorf("error indexing dependencies for %q: %w", tn.name, err)
}
tn.proto.Deps = []*license_metadata_proto.AnnotatedDependency{}
}
}
return lg, err
}
// targetNode contains the license metadata for a node in the license graph.
type targetNode struct {
proto license_metadata_proto.LicenseMetadata
// name is the path to the metadata file.
name string
// lg is the license graph the node belongs to.
lg *LicenseGraph
// edges identifies the dependencies of the target.
edges TargetEdgeList
// licenseConditions identifies the set of license conditions originating at the target node.
licenseConditions LicenseConditionSet
// resolution identifies the set of conditions resolved by acting on the target node.
resolution LicenseConditionSet
// pure indicates whether to treat the node as a pure aggregate (no internal linkage)
pure bool
}
// addDependencies converts the proto AnnotatedDependencies into `edges`
func addDependencies(lg *LicenseGraph, tn *TargetNode) error {
tn.edges = make(TargetEdgeList, 0, len(tn.proto.Deps))
for _, ad := range tn.proto.Deps {
dependency := ad.GetFile()
if len(dependency) == 0 {
return fmt.Errorf("missing dependency name")
}
dtn, ok := lg.targets[dependency]
if !ok {
return fmt.Errorf("unknown dependency name %q", dependency)
}
if dtn == nil {
return fmt.Errorf("nil dependency for name %q", dependency)
}
annotations := newEdgeAnnotations()
for _, a := range ad.Annotations {
// look up a common constant annotation string from a small map
// instead of creating 1000's of copies of the same 3 strings.
if ann, ok := RecognizedAnnotations[a]; ok {
annotations.annotations[ann] = struct{}{}
}
}
edge := &TargetEdge{tn, dtn, annotations}
lg.edges = append(lg.edges, edge)
tn.edges = append(tn.edges, edge)
}
return nil
}
// readFile is a task to read and parse a single license metadata file, and to schedule
// additional tasks for reading and parsing dependencies as necessary.
func readFile(recv *receiver, file string) {
recv.wg.Add(1)
<-recv.task
go func() {
f, err := recv.rootFS.Open(file)
if err != nil {
recv.results <- &result{file, nil, fmt.Errorf("error opening license metadata %q: %w", file, err)}
return
}
// read the file
data, err := io.ReadAll(f)
if err != nil {
recv.results <- &result{file, nil, fmt.Errorf("error reading license metadata %q: %w", file, err)}
return
}
f.Close()
tn := &TargetNode{lg: recv.lg, name: file}
err = prototext.Unmarshal(data, &tn.proto)
if err != nil {
recv.results <- &result{file, nil, fmt.Errorf("error license metadata %q: %w", file, err)}
return
}
// send result for this file and release task before scheduling dependencies,
// but do not signal done to WaitGroup until dependencies are scheduled.
recv.results <- &result{file, tn, nil}
recv.task <- true
// schedule tasks as necessary to read dependencies
for _, ad := range tn.proto.Deps {
dependency := ad.GetFile()
// decide, signal and record whether to schedule task in critical section
recv.lg.mu.Lock()
_, alreadyScheduled := recv.lg.targets[dependency]
if !alreadyScheduled {
recv.lg.targets[dependency] = nil
}
recv.lg.mu.Unlock()
// schedule task to read dependency file outside critical section
if !alreadyScheduled {
readFile(recv, dependency)
}
}
// signal task done after scheduling dependencies
recv.wg.Done()
}()
}