diff options
| -rw-r--r-- | tools/Android.bp | 16 | ||||
| -rw-r--r-- | tools/sbom/Android.bp | 53 | ||||
| -rwxr-xr-x | tools/sbom/generate-sbom.py (renamed from tools/generate-sbom.py) | 384 | ||||
| -rw-r--r-- | tools/sbom/sbom_data.py | 120 | ||||
| -rw-r--r-- | tools/sbom/sbom_writers.py | 365 | ||||
| -rw-r--r-- | tools/sbom/sbom_writers_test.py | 153 | ||||
| -rw-r--r-- | tools/sbom/testdata/expected_json_sbom.spdx.json | 137 | ||||
| -rw-r--r-- | tools/sbom/testdata/expected_tagvalue_sbom.spdx | 65 | ||||
| -rw-r--r-- | tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx | 12 |
9 files changed, 1020 insertions, 285 deletions
diff --git a/tools/Android.bp b/tools/Android.bp index e325f6bb21..bea0602f59 100644 --- a/tools/Android.bp +++ b/tools/Android.bp @@ -71,22 +71,6 @@ python_binary_host { } python_binary_host { - name: "generate-sbom", - srcs: [ - "generate-sbom.py", - ], - version: { - py3: { - embedded_launcher: true, - }, - }, - libs: [ - "metadata_file_proto_py", - "libprotobuf-python", - ], -} - -python_binary_host { name: "list_files", main: "list_files.py", srcs: [ diff --git a/tools/sbom/Android.bp b/tools/sbom/Android.bp new file mode 100644 index 0000000000..f6c01900c5 --- /dev/null +++ b/tools/sbom/Android.bp @@ -0,0 +1,53 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +python_binary_host { + name: "generate-sbom", + srcs: [ + "generate-sbom.py", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, + libs: [ + "metadata_file_proto_py", + "libprotobuf-python", + "sbom_lib", + ], +} + +python_library_host { + name: "sbom_lib", + srcs: [ + "sbom_data.py", + "sbom_writers.py", + ], +} + +python_test_host { + name: "sbom_writers_test", + main: "sbom_writers_test.py", + srcs: [ + "sbom_writers_test.py", + ], + data: [ + "testdata/*", + ], + libs: [ + "sbom_lib", + ], + test_suites: ["general-tests"], +} diff --git a/tools/generate-sbom.py b/tools/sbom/generate-sbom.py index 9583395a7b..0c5deb2868 100755 --- a/tools/generate-sbom.py +++ b/tools/sbom/generate-sbom.py @@ -29,50 +29,11 @@ import csv import datetime import google.protobuf.text_format as text_format import hashlib -import json import os import metadata_file_pb2 +import sbom_data +import sbom_writers -# Common -SPDXID = 'SPDXID' -SPDX_VERSION = 'SPDXVersion' -DATA_LICENSE = 'DataLicense' -DOCUMENT_NAME = 'DocumentName' -DOCUMENT_NAMESPACE = 'DocumentNamespace' -CREATED = 'Created' -CREATOR = 'Creator' -EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef' - -# Package -PACKAGE_NAME = 'PackageName' -PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation' -PACKAGE_VERSION = 'PackageVersion' -PACKAGE_SUPPLIER = 'PackageSupplier' -FILES_ANALYZED = 'FilesAnalyzed' -PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode' -PACKAGE_EXTERNAL_REF = 'ExternalRef' -# Package license -PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded' -PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles' -PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared' -PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments' - -# File -FILE_NAME = 'FileName' -FILE_CHECKSUM = 'FileChecksum' -# File license -FILE_LICENSE_CONCLUDED = 'LicenseConcluded' -FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile' -FILE_LICENSE_COMMENTS = 'LicenseComments' -FILE_COPYRIGHT_TEXT = 'FileCopyrightText' -FILE_NOTICE = 'FileNotice' -FILE_ATTRIBUTION_TEXT = 'FileAttributionText' - -# Relationship -RELATIONSHIP = 'Relationship' -REL_DESCRIBES = 'DESCRIBES' -REL_VARIANT_OF = 'VARIANT_OF' -REL_GENERATED_FROM = 'GENERATED_FROM' # Package type PKG_SOURCE = 'SOURCE' @@ -111,44 +72,6 @@ def log(*info): print(i) -def new_doc_header(doc_id): - return { - SPDX_VERSION: 'SPDX-2.3', - DATA_LICENSE: 'CC0-1.0', - SPDXID: doc_id, - DOCUMENT_NAME: args.build_version, - DOCUMENT_NAMESPACE: f'https://www.google.com/sbom/spdx/android/{args.build_version}', - CREATOR: 'Organization: Google, LLC', - CREATED: '<timestamp>', - EXTERNAL_DOCUMENT_REF: [], - } - - -def new_package_record(id, name, version, supplier, download_location=None, files_analyzed='false', external_refs=[]): - package = { - PACKAGE_NAME: name, - SPDXID: id, - PACKAGE_DOWNLOAD_LOCATION: download_location if download_location else 'NONE', - FILES_ANALYZED: files_analyzed, - } - if version: - package[PACKAGE_VERSION] = version - if supplier: - package[PACKAGE_SUPPLIER] = f'Organization: {supplier}' - if external_refs: - package[PACKAGE_EXTERNAL_REF] = external_refs - - return package - - -def new_file_record(id, name, checksum): - return { - FILE_NAME: name, - SPDXID: id, - FILE_CHECKSUM: checksum - } - - def encode_for_spdxid(s): """Simple encode for string values used in SPDXID which uses the charset of A-Za-Z0-9.-""" result = '' @@ -167,19 +90,10 @@ def new_package_id(package_name, type): return f'SPDXRef-{type}-{encode_for_spdxid(package_name)}' -def new_external_doc_ref(package_name, sbom_url, sbom_checksum): - doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(package_name)}' - return f'{EXTERNAL_DOCUMENT_REF}: {doc_ref_id} {sbom_url} {sbom_checksum}', doc_ref_id - - def new_file_id(file_path): return f'SPDXRef-{encode_for_spdxid(file_path)}' -def new_relationship_record(id1, relationship, id2): - return f'{RELATIONSHIP}: {id1} {relationship} {id2}' - - def checksum(file_path): file_path = args.product_out_dir + '/' + file_path h = hashlib.sha1() @@ -243,6 +157,11 @@ def is_prebuilt_package(file_metadata): def get_source_package_info(file_metadata, metadata_file_path): + """Return source package info exists in its METADATA file, currently including name, security tag + and external SBOM reference. + + See go/android-spdx and go/android-sbom-gen for more details. + """ if not metadata_file_path: return file_metadata['module_path'], [] @@ -250,9 +169,15 @@ def get_source_package_info(file_metadata, metadata_file_path): external_refs = [] for tag in metadata_proto.third_party.security.tag: if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()): - external_refs.append(f'{PACKAGE_EXTERNAL_REF}: SECURITY cpe23Type {tag.removeprefix(NVD_CPE23)}') + external_refs.append( + sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe23Type, + locator=tag.removeprefix(NVD_CPE23))) elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()): - external_refs.append(f'{PACKAGE_EXTERNAL_REF}: SECURITY cpe22Type {tag.removeprefix(NVD_CPE23)}') + external_refs.append( + sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe22Type, + locator=tag.removeprefix(NVD_CPE23))) if metadata_proto.name: return metadata_proto.name, external_refs @@ -261,6 +186,11 @@ def get_source_package_info(file_metadata, metadata_file_path): def get_prebuilt_package_name(file_metadata, metadata_file_path): + """Return name of a prebuilt package, which can be from the METADATA file, metadata file path, + module path or kernel module's source path if the installed file is a kernel module. + + See go/android-spdx and go/android-sbom-gen for more details. + """ name = None if metadata_file_path: metadata_proto = metadata_file_protos[metadata_file_path] @@ -278,6 +208,7 @@ def get_prebuilt_package_name(file_metadata, metadata_file_path): def get_metadata_file_path(file_metadata): + """Search for METADATA file of a package and return its path.""" metadata_path = '' if file_metadata['module_path']: metadata_path = file_metadata['module_path'] @@ -291,6 +222,7 @@ def get_metadata_file_path(file_metadata): def get_package_version(metadata_file_path): + """Return a package's version in its METADATA file.""" if not metadata_file_path: return None metadata_proto = metadata_file_protos[metadata_file_path] @@ -298,6 +230,7 @@ def get_package_version(metadata_file_path): def get_package_homepage(metadata_file_path): + """Return a package's homepage URL in its METADATA file.""" if not metadata_file_path: return None metadata_proto = metadata_file_protos[metadata_file_path] @@ -311,6 +244,7 @@ def get_package_homepage(metadata_file_path): def get_package_download_location(metadata_file_path): + """Return a package's code repository URL in its METADATA file.""" if not metadata_file_path: return None metadata_proto = metadata_file_protos[metadata_file_path] @@ -325,6 +259,12 @@ def get_package_download_location(metadata_file_path): def get_sbom_fragments(installed_file_metadata, metadata_file_path): + """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT + package, a UPSTREAM package if it's a source package and a external SBOM document reference if + it's a prebuilt package with sbom_ref defined in its METADATA file. + + See go/android-spdx and go/android-sbom-gen for more details. + """ external_doc_ref = None packages = [] relationships = [] @@ -338,18 +278,26 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path): # Source fork packages name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path) source_package_id = new_package_id(name, PKG_SOURCE) - source_package = new_package_record(source_package_id, name, args.build_version, args.product_mfr, - external_refs=external_refs) + source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version, + supplier='Organization: ' + args.product_mfr, + external_refs=external_refs) upstream_package_id = new_package_id(name, PKG_UPSTREAM) - upstream_package = new_package_record(upstream_package_id, name, version, homepage, download_location) + upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version, + supplier='Organization: ' + homepage if homepage else None, + download_location=download_location) packages += [source_package, upstream_package] - relationships.append(new_relationship_record(source_package_id, REL_VARIANT_OF, upstream_package_id)) + relationships.append(sbom_data.Relationship(id1=source_package_id, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=upstream_package_id)) elif is_prebuilt_package(installed_file_metadata): # Prebuilt fork packages name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path) prebuilt_package_id = new_package_id(name, PKG_PREBUILT) - prebuilt_package = new_package_record(prebuilt_package_id, name, args.build_version, args.product_mfr) + prebuilt_package = sbom_data.Package(id=prebuilt_package_id, + name=name, + version=args.build_version, + supplier='Organization: ' + args.product_mfr) packages.append(prebuilt_package) if metadata_file_path: @@ -359,136 +307,26 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path): sbom_checksum = metadata_proto.third_party.sbom_ref.checksum upstream_element_id = metadata_proto.third_party.sbom_ref.element_id if sbom_url and sbom_checksum and upstream_element_id: - external_doc_ref, doc_ref_id = new_external_doc_ref(name, sbom_url, sbom_checksum) + doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}' + external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id, + uri=sbom_url, + checksum=sbom_checksum) relationships.append( - new_relationship_record(prebuilt_package_id, REL_VARIANT_OF, doc_ref_id + ':' + upstream_element_id)) + sbom_data.Relationship(id1=prebuilt_package_id, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=doc_ref_id + ':' + upstream_element_id)) return external_doc_ref, packages, relationships def generate_package_verification_code(files): - checksums = [file[FILE_CHECKSUM] for file in files] + checksums = [file.checksum for file in files] checksums.sort() h = hashlib.sha1() h.update(''.join(checksums).encode(encoding='utf-8')) return h.hexdigest() -def write_record(f, record): - if record.__class__.__name__ == 'dict': - for k, v in record.items(): - if k == EXTERNAL_DOCUMENT_REF or k == PACKAGE_EXTERNAL_REF: - for ref in v: - f.write(ref + '\n') - else: - f.write('{}: {}\n'.format(k, v)) - elif record.__class__.__name__ == 'str': - f.write(record + '\n') - f.write('\n') - - -def write_tagvalue_sbom(all_records): - with open(args.output_file, 'w', encoding="utf-8") as output_file: - for rec in all_records: - write_record(output_file, rec) - - -def write_json_sbom(all_records, product_package_id): - doc = {} - product_package = None - for r in all_records: - if r.__class__.__name__ == 'dict': - if DOCUMENT_NAME in r: # Doc header - doc['spdxVersion'] = r[SPDX_VERSION] - doc['dataLicense'] = r[DATA_LICENSE] - doc[SPDXID] = r[SPDXID] - doc['name'] = r[DOCUMENT_NAME] - doc['documentNamespace'] = r[DOCUMENT_NAMESPACE] - doc['creationInfo'] = { - 'creators': [r[CREATOR]], - 'created': r[CREATED], - } - doc['externalDocumentRefs'] = [] - for ref in r[EXTERNAL_DOCUMENT_REF]: - # ref is 'ExternalDocumentRef: <doc id> <doc url> SHA1: xxxxx' - fields = ref.split(' ') - doc_ref = { - 'externalDocumentId': fields[1], - 'spdxDocument': fields[2], - 'checksum': { - 'algorithm': fields[3][:-1], - 'checksumValue': fields[4] - } - } - doc['externalDocumentRefs'].append(doc_ref) - doc['documentDescribes'] = [] - doc['packages'] = [] - doc['files'] = [] - doc['relationships'] = [] - - elif PACKAGE_NAME in r: # packages - package = { - 'name': r[PACKAGE_NAME], - SPDXID: r[SPDXID], - 'downloadLocation': r[PACKAGE_DOWNLOAD_LOCATION], - 'filesAnalyzed': r[FILES_ANALYZED] == "true" - } - if PACKAGE_VERSION in r: - package['versionInfo'] = r[PACKAGE_VERSION] - if PACKAGE_SUPPLIER in r: - package['supplier'] = r[PACKAGE_SUPPLIER] - if PACKAGE_VERIFICATION_CODE in r: - package['packageVerificationCode'] = { - 'packageVerificationCodeValue': r[PACKAGE_VERIFICATION_CODE] - } - if PACKAGE_EXTERNAL_REF in r: - package['externalRefs'] = [] - for ref in r[PACKAGE_EXTERNAL_REF]: - # ref is 'ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4' - fields = ref.split(' ') - ext_ref = { - 'referenceCategory': fields[1], - 'referenceType': fields[2], - 'referenceLocator': fields[3], - } - package['externalRefs'].append(ext_ref) - - doc['packages'].append(package) - if r[SPDXID] == product_package_id: - product_package = package - product_package['hasFiles'] = [] - - elif FILE_NAME in r: # files - file = { - 'fileName': r[FILE_NAME], - SPDXID: r[SPDXID] - } - checksum = r[FILE_CHECKSUM].split(': ') - file['checksums'] = [{ - 'algorithm': checksum[0], - 'checksumValue': checksum[1], - }] - doc['files'].append(file) - product_package['hasFiles'].append(r[SPDXID]) - - elif r.__class__.__name__ == 'str': - if r.startswith(RELATIONSHIP): - # r is 'Relationship: <spdxid> <relationship> <spdxid>' - fields = r.split(' ') - rel = { - 'spdxElementId': fields[1], - 'relatedSpdxElement': fields[3], - 'relationshipType': fields[2], - } - if fields[2] == REL_DESCRIBES: - doc['documentDescribes'].append(fields[3]) - else: - doc['relationships'].append(rel) - - with open(args.output_file + '.json', 'w', encoding="utf-8") as output_file: - output_file.write(json.dumps(doc, indent=4)) - - def save_report(report): prefix, _ = os.path.splitext(args.output_file) with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file: @@ -499,12 +337,6 @@ def save_report(report): report_file.write('\n') -def sort_rels(rel): - # rel = 'Relationship file_id GENERATED_FROM package_id' - fields = rel.split(' ') - return fields[3] + fields[1] - - # Validate the metadata generated by Make for installed files and report if there is no metadata. def installed_file_has_metadata(installed_file_metadata, report): installed_file = installed_file_metadata['installed_file'] @@ -555,24 +387,38 @@ def report_metadata_file(metadata_file_path, installed_file_metadata, report): installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) -def generate_fragment(): +def generate_sbom_for_unbundled(): with open(args.metadata, newline='') as sbom_metadata_file: reader = csv.DictReader(sbom_metadata_file) + doc = sbom_data.Document(name=args.build_version, + namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', + creators=['Organization: ' + args.product_mfr]) for installed_file_metadata in reader: installed_file = installed_file_metadata['installed_file'] if args.output_file != args.product_out_dir + installed_file + ".spdx": continue module_path = installed_file_metadata['module_path'] - package_id = new_package_id(encode_for_spdxid(module_path), PKG_PREBUILT) - package = new_package_record(package_id, module_path, args.build_version, args.product_mfr) + package_id = new_package_id(module_path, PKG_PREBUILT) + package = sbom_data.Package(id=package_id, + name=module_path, + version=args.build_version, + supplier='Organization: ' + args.product_mfr) file_id = new_file_id(installed_file) - file = new_file_record(file_id, installed_file, checksum(installed_file)) - relationship = new_relationship_record(file_id, REL_GENERATED_FROM, package_id) - records = [package, file, relationship] - write_tagvalue_sbom(records) + file = sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file)) + relationship = sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=package_id) + doc.add_package(package) + doc.files.append(file) + doc.describes = file_id + doc.add_relationship(relationship) + doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') break + with open(args.output_file, 'w', encoding="utf-8") as file: + sbom_writers.TagValueWriter.write(doc, file, fragment=True) + def main(): global args @@ -580,21 +426,27 @@ def main(): log('Args:', vars(args)) if args.unbundled: - generate_fragment() + generate_sbom_for_unbundled() return global metadata_file_protos metadata_file_protos = {} - doc_id = 'SPDXRef-DOCUMENT' - doc_header = new_doc_header(doc_id) + doc = sbom_data.Document(name=args.build_version, + namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}', + creators=['Organization: ' + args.product_mfr]) - product_package_id = 'SPDXRef-PRODUCT' - product_package = new_package_record(product_package_id, 'PRODUCT', args.build_version, args.product_mfr, - files_analyzed='true') + product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT, + name=sbom_data.PACKAGE_NAME_PRODUCT, + version=args.build_version, + supplier='Organization: ' + args.product_mfr, + files_analyzed=True) + doc.packages.append(product_package) - platform_package_id = 'SPDXRef-PLATFORM' - platform_package = new_package_record(platform_package_id, 'PLATFORM', args.build_version, args.product_mfr) + doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, + name=sbom_data.PACKAGE_NAME_PLATFORM, + version=args.build_version, + supplier='Organization: ' + args.product_mfr)) # Report on some issues and information report = { @@ -607,10 +459,6 @@ def main(): } # Scan the metadata in CSV file and create the corresponding package and file records in SPDX - product_files = [] - package_ids = [] - package_records = [] - rels_file_gen_from = [] with open(args.metadata, newline='') as sbom_metadata_file: reader = csv.DictReader(sbom_metadata_file) for installed_file_metadata in reader: @@ -627,7 +475,9 @@ def main(): continue file_id = new_file_id(installed_file) - product_files.append(new_file_record(file_id, installed_file, checksum(installed_file))) + doc.files.append( + sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file))) + product_package.file_ids.append(file_id) if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): metadata_file_path = get_metadata_file_path(installed_file_metadata) @@ -636,54 +486,50 @@ def main(): # File from source fork packages or prebuilt fork packages external_doc_ref, pkgs, rels = get_sbom_fragments(installed_file_metadata, metadata_file_path) if len(pkgs) > 0: - if external_doc_ref and external_doc_ref not in doc_header[EXTERNAL_DOCUMENT_REF]: - doc_header[EXTERNAL_DOCUMENT_REF].append(external_doc_ref) + if external_doc_ref: + doc.add_external_ref(external_doc_ref) for p in pkgs: - if not p[SPDXID] in package_ids: - package_ids.append(p[SPDXID]) - package_records.append(p) + doc.add_package(p) for rel in rels: - if not rel in package_records: - package_records.append(rel) - fork_package_id = pkgs[0][SPDXID] # The first package should be the source/prebuilt fork package - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, fork_package_id)) + doc.add_relationship(rel) + fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=fork_package_id)) elif module_path or installed_file_metadata['is_platform_generated']: # File from PLATFORM package - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) elif product_copy_files: # Format of product_copy_files: <source path>:<dest path> src_path = product_copy_files.split(':')[0] # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device, # so process them as files from PLATFORM package - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) elif installed_file.endswith('.fsv_meta'): # See build/make/core/Makefile:2988 - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) elif kernel_module_copy_files.startswith('ANDROID-GEN'): # For the four files generated for _dlkm, _ramdisk partitions # See build/make/core/Makefile:323 - rels_file_gen_from.append(new_relationship_record(file_id, REL_GENERATED_FROM, platform_package_id)) + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) - product_package[PACKAGE_VERIFICATION_CODE] = generate_package_verification_code(product_files) - - all_records = [ - doc_header, - product_package, - new_relationship_record(doc_id, REL_DESCRIBES, product_package_id), - ] - all_records += product_files - all_records.append(platform_package) - all_records += package_records - rels_file_gen_from.sort(key=sort_rels) - all_records += rels_file_gen_from + product_package.verification_code = generate_package_verification_code(doc.files) # Save SBOM records to output file - doc_header[CREATED] = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') - write_tagvalue_sbom(all_records) + doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + with open(args.output_file, 'w', encoding="utf-8") as file: + sbom_writers.TagValueWriter.write(doc, file) if args.json: - write_json_sbom(all_records, product_package_id) - - save_report(report) + with open(args.output_file+'.json', 'w', encoding="utf-8") as file: + sbom_writers.JSONWriter.write(doc, file) if __name__ == '__main__': diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py new file mode 100644 index 0000000000..0c380f60d4 --- /dev/null +++ b/tools/sbom/sbom_data.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Define data classes that model SBOMs defined by SPDX. The data classes could be +written out to different formats (tagvalue, JSON, etc) of SPDX with corresponding +writer utilities. + +Rrefer to SPDX 2.3 spec: https://spdx.github.io/spdx-spec/v2.3/ and go/android-spdx for details of +fields in each data class. +""" + +from dataclasses import dataclass, field +from typing import List + +SPDXID_DOC = 'SPDXRef-DOCUMENT' +SPDXID_PRODUCT = 'SPDXRef-PRODUCT' +SPDXID_PLATFORM = 'SPDXRef-PLATFORM' + +PACKAGE_NAME_PRODUCT = 'PRODUCT' +PACKAGE_NAME_PLATFORM = 'PLATFORM' + + +class PackageExternalRefCategory: + SECURITY = 'SECURITY' + PACKAGE_MANAGER = 'PACKAGE-MANAGER' + PERSISTENT_ID = 'PERSISTENT-ID' + OTHER = 'OTHER' + + +class PackageExternalRefType: + cpe22Type = 'cpe22Type' + cpe23Type = 'cpe23Type' + + +@dataclass +class PackageExternalRef: + category: PackageExternalRefCategory + type: PackageExternalRefType + locator: str + + +@dataclass +class Package: + name: str + id: str + version: str = None + supplier: str = None + download_location: str = None + files_analyzed: bool = False + verification_code: str = None + file_ids: List[str] = field(default_factory=list) + external_refs: List[PackageExternalRef] = field(default_factory=list) + + +@dataclass +class File: + id: str + name: str + checksum: str + + +class RelationshipType: + DESCRIBES = 'DESCRIBES' + VARIANT_OF = 'VARIANT_OF' + GENERATED_FROM = 'GENERATED_FROM' + + +@dataclass +class Relationship: + id1: str + relationship: RelationshipType + id2: str + + +@dataclass +class DocumentExternalReference: + id: str + uri: str + checksum: str + + +@dataclass +class Document: + name: str + namespace: str + id: str = SPDXID_DOC + describes: str = SPDXID_PRODUCT + creators: List[str] = field(default_factory=list) + created: str = None + external_refs: List[DocumentExternalReference] = field(default_factory=list) + packages: List[Package] = field(default_factory=list) + files: List[File] = field(default_factory=list) + relationships: List[Relationship] = field(default_factory=list) + + def add_external_ref(self, external_ref): + if not any(external_ref.uri == ref.uri for ref in self.external_refs): + self.external_refs.append(external_ref) + + def add_package(self, package): + if not any(package.id == p.id for p in self.packages): + self.packages.append(package) + + def add_relationship(self, rel): + if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship + for r in self.relationships): + self.relationships.append(rel) diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py new file mode 100644 index 0000000000..66aa6b4a2f --- /dev/null +++ b/tools/sbom/sbom_writers.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Serialize objects defined in package sbom_data to SPDX format: tagvalue, JSON. +""" + +import json +import sbom_data + +SPDX_VER = 'SPDX-2.3' +DATA_LIC = 'CC0-1.0' + + +class Tags: + # Common + SPDXID = 'SPDXID' + SPDX_VERSION = 'SPDXVersion' + DATA_LICENSE = 'DataLicense' + DOCUMENT_NAME = 'DocumentName' + DOCUMENT_NAMESPACE = 'DocumentNamespace' + CREATED = 'Created' + CREATOR = 'Creator' + EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef' + + # Package + PACKAGE_NAME = 'PackageName' + PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation' + PACKAGE_VERSION = 'PackageVersion' + PACKAGE_SUPPLIER = 'PackageSupplier' + FILES_ANALYZED = 'FilesAnalyzed' + PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode' + PACKAGE_EXTERNAL_REF = 'ExternalRef' + # Package license + PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded' + PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles' + PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared' + PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments' + + # File + FILE_NAME = 'FileName' + FILE_CHECKSUM = 'FileChecksum' + # File license + FILE_LICENSE_CONCLUDED = 'LicenseConcluded' + FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile' + FILE_LICENSE_COMMENTS = 'LicenseComments' + FILE_COPYRIGHT_TEXT = 'FileCopyrightText' + FILE_NOTICE = 'FileNotice' + FILE_ATTRIBUTION_TEXT = 'FileAttributionText' + + # Relationship + RELATIONSHIP = 'Relationship' + + +class TagValueWriter: + @staticmethod + def marshal_doc_headers(sbom_doc): + headers = [ + f'{Tags.SPDX_VERSION}: {SPDX_VER}', + f'{Tags.DATA_LICENSE}: {DATA_LIC}', + f'{Tags.SPDXID}: {sbom_doc.id}', + f'{Tags.DOCUMENT_NAME}: {sbom_doc.name}', + f'{Tags.DOCUMENT_NAMESPACE}: {sbom_doc.namespace}', + ] + for creator in sbom_doc.creators: + headers.append(f'{Tags.CREATOR}: {creator}') + headers.append(f'{Tags.CREATED}: {sbom_doc.created}') + for doc_ref in sbom_doc.external_refs: + headers.append( + f'{Tags.EXTERNAL_DOCUMENT_REF}: {doc_ref.id} {doc_ref.uri} {doc_ref.checksum}') + headers.append('') + return headers + + @staticmethod + def marshal_package(package): + download_location = 'NONE' + if package.download_location: + download_location = package.download_location + tagvalues = [ + f'{Tags.PACKAGE_NAME}: {package.name}', + f'{Tags.SPDXID}: {package.id}', + f'{Tags.PACKAGE_DOWNLOAD_LOCATION}: {download_location}', + f'{Tags.FILES_ANALYZED}: {str(package.files_analyzed).lower()}', + ] + if package.version: + tagvalues.append(f'{Tags.PACKAGE_VERSION}: {package.version}') + if package.supplier: + tagvalues.append(f'{Tags.PACKAGE_SUPPLIER}: {package.supplier}') + if package.verification_code: + tagvalues.append(f'{Tags.PACKAGE_VERIFICATION_CODE}: {package.verification_code}') + if package.external_refs: + for external_ref in package.external_refs: + tagvalues.append( + f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}') + + tagvalues.append('') + return tagvalues + + @staticmethod + def marshal_described_element(sbom_doc): + if not sbom_doc.describes: + return None + + product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes] + if product_package: + tagvalues = TagValueWriter.marshal_package(product_package[0]) + tagvalues.append( + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}') + + tagvalues.append('') + return tagvalues + + file = [f for f in sbom_doc.files if f.id == sbom_doc.describes] + if file: + tagvalues = [ + f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}' + ] + + return tagvalues + + return None + + @staticmethod + def marshal_packages(sbom_doc): + tagvalues = [] + marshaled_relationships = [] + i = 0 + packages = sbom_doc.packages + while i < len(packages): + if packages[i].id == sbom_doc.describes: + i += 1 + continue + + if i + 1 < len(packages) \ + and packages[i].id.startswith('SPDXRef-SOURCE-') \ + and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'): + tagvalues += TagValueWriter.marshal_package(packages[i]) + tagvalues += TagValueWriter.marshal_package(packages[i + 1]) + rel = next((r for r in sbom_doc.relationships if + r.id1 == packages[i].id and + r.id2 == packages[i + 1].id and + r.relationship == sbom_data.RelationshipType.VARIANT_OF), None) + if rel: + marshaled_relationships.append(rel) + tagvalues.append(TagValueWriter.marshal_relationship(rel)) + tagvalues.append('') + + i += 2 + else: + tagvalues += TagValueWriter.marshal_package(packages[i]) + i += 1 + + return tagvalues, marshaled_relationships + + @staticmethod + def marshal_file(file): + tagvalues = [ + f'{Tags.FILE_NAME}: {file.name}', + f'{Tags.SPDXID}: {file.id}', + f'{Tags.FILE_CHECKSUM}: {file.checksum}', + '', + ] + + return tagvalues + + @staticmethod + def marshal_files(sbom_doc): + tagvalues = [] + for file in sbom_doc.files: + tagvalues += TagValueWriter.marshal_file(file) + return tagvalues + + @staticmethod + def marshal_relationship(rel): + return f'{Tags.RELATIONSHIP}: {rel.id1} {rel.relationship} {rel.id2}' + + @staticmethod + def marshal_relationships(sbom_doc, marshaled_rels): + tagvalues = [] + sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.id2 + r.id1) + for rel in sorted_rels: + if any(r.id1 == rel.id1 and r.id2 == rel.id2 and r.relationship == rel.relationship + for r in marshaled_rels): + continue + tagvalues.append(TagValueWriter.marshal_relationship(rel)) + tagvalues.append('') + return tagvalues + + @staticmethod + def write(sbom_doc, file, fragment=False): + content = [] + if not fragment: + content += TagValueWriter.marshal_doc_headers(sbom_doc) + described_element = TagValueWriter.marshal_described_element(sbom_doc) + if described_element: + content += described_element + content += TagValueWriter.marshal_files(sbom_doc) + tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc) + content += tagvalues + content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships) + file.write('\n'.join(content)) + + +class PropNames: + # Common + SPDXID = 'SPDXID' + SPDX_VERSION = 'spdxVersion' + DATA_LICENSE = 'dataLicense' + NAME = 'name' + DOCUMENT_NAMESPACE = 'documentNamespace' + CREATION_INFO = 'creationInfo' + CREATORS = 'creators' + CREATED = 'created' + EXTERNAL_DOCUMENT_REF = 'externalDocumentRefs' + DOCUMENT_DESCRIBES = 'documentDescribes' + EXTERNAL_DOCUMENT_ID = 'externalDocumentId' + EXTERNAL_DOCUMENT_URI = 'spdxDocument' + EXTERNAL_DOCUMENT_CHECKSUM = 'checksum' + ALGORITHM = 'algorithm' + CHECKSUM_VALUE = 'checksumValue' + + # Package + PACKAGES = 'packages' + PACKAGE_DOWNLOAD_LOCATION = 'downloadLocation' + PACKAGE_VERSION = 'versionInfo' + PACKAGE_SUPPLIER = 'supplier' + FILES_ANALYZED = 'filesAnalyzed' + PACKAGE_VERIFICATION_CODE = 'packageVerificationCode' + PACKAGE_VERIFICATION_CODE_VALUE = 'packageVerificationCodeValue' + PACKAGE_EXTERNAL_REFS = 'externalRefs' + PACKAGE_EXTERNAL_REF_CATEGORY = 'referenceCategory' + PACKAGE_EXTERNAL_REF_TYPE = 'referenceType' + PACKAGE_EXTERNAL_REF_LOCATOR = 'referenceLocator' + PACKAGE_HAS_FILES = 'hasFiles' + + # File + FILES = 'files' + FILE_NAME = 'fileName' + FILE_CHECKSUMS = 'checksums' + + # Relationship + RELATIONSHIPS = 'relationships' + REL_ELEMENT_ID = 'spdxElementId' + REL_RELATED_ELEMENT_ID = 'relatedSpdxElement' + REL_TYPE = 'relationshipType' + + +class JSONWriter: + @staticmethod + def marshal_doc_headers(sbom_doc): + headers = { + PropNames.SPDX_VERSION: SPDX_VER, + PropNames.DATA_LICENSE: DATA_LIC, + PropNames.SPDXID: sbom_doc.id, + PropNames.NAME: sbom_doc.name, + PropNames.DOCUMENT_NAMESPACE: sbom_doc.namespace, + PropNames.CREATION_INFO: {} + } + creators = [creator for creator in sbom_doc.creators] + headers[PropNames.CREATION_INFO][PropNames.CREATORS] = creators + headers[PropNames.CREATION_INFO][PropNames.CREATED] = sbom_doc.created + external_refs = [] + for doc_ref in sbom_doc.external_refs: + checksum = doc_ref.checksum.split(': ') + external_refs.append({ + PropNames.EXTERNAL_DOCUMENT_ID: f'{doc_ref.id}', + PropNames.EXTERNAL_DOCUMENT_URI: doc_ref.uri, + PropNames.EXTERNAL_DOCUMENT_CHECKSUM: { + PropNames.ALGORITHM: checksum[0], + PropNames.CHECKSUM_VALUE: checksum[1] + } + }) + if external_refs: + headers[PropNames.EXTERNAL_DOCUMENT_REF] = external_refs + headers[PropNames.DOCUMENT_DESCRIBES] = [sbom_doc.describes] + + return headers + + @staticmethod + def marshal_packages(sbom_doc): + packages = [] + for p in sbom_doc.packages: + package = { + PropNames.NAME: p.name, + PropNames.SPDXID: p.id, + PropNames.PACKAGE_DOWNLOAD_LOCATION: p.download_location if p.download_location else 'NONE', + PropNames.FILES_ANALYZED: p.files_analyzed + } + if p.version: + package[PropNames.PACKAGE_VERSION] = p.version + if p.supplier: + package[PropNames.PACKAGE_SUPPLIER] = p.supplier + if p.verification_code: + package[PropNames.PACKAGE_VERIFICATION_CODE] = { + PropNames.PACKAGE_VERIFICATION_CODE_VALUE: p.verification_code + } + if p.external_refs: + package[PropNames.PACKAGE_EXTERNAL_REFS] = [] + for ref in p.external_refs: + ext_ref = { + PropNames.PACKAGE_EXTERNAL_REF_CATEGORY: ref.category, + PropNames.PACKAGE_EXTERNAL_REF_TYPE: ref.type, + PropNames.PACKAGE_EXTERNAL_REF_LOCATOR: ref.locator, + } + package[PropNames.PACKAGE_EXTERNAL_REFS].append(ext_ref) + if p.file_ids: + package[PropNames.PACKAGE_HAS_FILES] = [] + for file_id in p.file_ids: + package[PropNames.PACKAGE_HAS_FILES].append(file_id) + + packages.append(package) + + return {PropNames.PACKAGES: packages} + + @staticmethod + def marshal_files(sbom_doc): + files = [] + for f in sbom_doc.files: + file = { + PropNames.FILE_NAME: f.name, + PropNames.SPDXID: f.id + } + checksum = f.checksum.split(': ') + file[PropNames.FILE_CHECKSUMS] = [{ + PropNames.ALGORITHM: checksum[0], + PropNames.CHECKSUM_VALUE: checksum[1], + }] + files.append(file) + return {PropNames.FILES: files} + + @staticmethod + def marshal_relationships(sbom_doc): + relationships = [] + sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.relationship + r.id2 + r.id1) + for r in sorted_rels: + rel = { + PropNames.REL_ELEMENT_ID: r.id1, + PropNames.REL_RELATED_ELEMENT_ID: r.id2, + PropNames.REL_TYPE: r.relationship, + } + relationships.append(rel) + + return {PropNames.RELATIONSHIPS: relationships} + + @staticmethod + def write(sbom_doc, file): + doc = {} + doc.update(JSONWriter.marshal_doc_headers(sbom_doc)) + doc.update(JSONWriter.marshal_packages(sbom_doc)) + doc.update(JSONWriter.marshal_files(sbom_doc)) + doc.update(JSONWriter.marshal_relationships(sbom_doc)) + file.write(json.dumps(doc, indent=4)) diff --git a/tools/sbom/sbom_writers_test.py b/tools/sbom/sbom_writers_test.py new file mode 100644 index 0000000000..4db2bb7601 --- /dev/null +++ b/tools/sbom/sbom_writers_test.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import pathlib +import unittest +import sbom_data +import sbom_writers + +BUILD_FINGER_PRINT = 'build_finger_print' +SUPPLIER_GOOGLE = 'Organization: Google' +SUPPLIER_UPSTREAM = 'Organization: upstream' + +SPDXID_PREBUILT_PACKAGE1 = 'SPDXRef-PREBUILT-package1' +SPDXID_SOURCE_PACKAGE1 = 'SPDXRef-SOURCE-package1' +SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1' + +SPDXID_FILE1 = 'SPDXRef-file1' +SPDXID_FILE2 = 'SPDXRef-file2' +SPDXID_FILE3 = 'SPDXRef-file3' + + +class SBOMWritersTest(unittest.TestCase): + + def setUp(self): + # SBOM of a product + self.sbom_doc = sbom_data.Document(name='test doc', + namespace='http://www.google.com/sbom/spdx/android', + creators=[SUPPLIER_GOOGLE], + created='2023-03-31T22:17:58Z', + describes=sbom_data.SPDXID_PRODUCT) + self.sbom_doc.add_external_ref( + sbom_data.DocumentExternalReference(id='DocumentRef-external_doc_ref', + uri='external_doc_uri', + checksum='SHA1: 1234567890')) + self.sbom_doc.add_package( + sbom_data.Package(id=sbom_data.SPDXID_PRODUCT, + name=sbom_data.PACKAGE_NAME_PRODUCT, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + files_analyzed=True, + verification_code='123456', + file_ids=[SPDXID_FILE1, SPDXID_FILE2, SPDXID_FILE3])) + + self.sbom_doc.add_package( + sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, + name=sbom_data.PACKAGE_NAME_PLATFORM, + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE1, + name='Prebuilt package1', + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1, + name='Source package1', + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT, + external_refs=[sbom_data.PackageExternalRef( + category=sbom_data.PackageExternalRefCategory.SECURITY, + type=sbom_data.PackageExternalRefType.cpe22Type, + locator='cpe:/a:jsoncpp_project:jsoncpp:1.9.4')] + )) + + self.sbom_doc.add_package( + sbom_data.Package(id=SPDXID_UPSTREAM_PACKAGE1, + name='Upstream package1', + supplier=SUPPLIER_UPSTREAM, + version='1.1', + )) + + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_SOURCE_PACKAGE1, + relationship=sbom_data.RelationshipType.VARIANT_OF, + id2=SPDXID_UPSTREAM_PACKAGE1)) + + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', checksum='SHA1: 11111')) + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222')) + self.sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333')) + + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE2, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_PREBUILT_PACKAGE1)) + self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE3, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_SOURCE_PACKAGE1 + )) + + # SBOM fragment of a APK + self.unbundled_sbom_doc = sbom_data.Document(name='test doc', + namespace='http://www.google.com/sbom/spdx/android', + creators=[SUPPLIER_GOOGLE], + created='2023-03-31T22:17:58Z', + describes=SPDXID_FILE1) + + self.unbundled_sbom_doc.files.append( + sbom_data.File(id=SPDXID_FILE1, name='/bin/file1.apk', checksum='SHA1: 11111')) + self.unbundled_sbom_doc.add_package( + sbom_data.Package(id=SPDXID_SOURCE_PACKAGE1, + name='Unbundled apk package', + supplier=SUPPLIER_GOOGLE, + version=BUILD_FINGER_PRINT)) + self.unbundled_sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_FILE1, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=SPDXID_SOURCE_PACKAGE1)) + + def test_tagvalue_writer(self): + with io.StringIO() as output: + sbom_writers.TagValueWriter.write(self.sbom_doc, output) + expected_output = pathlib.Path('testdata/expected_tagvalue_sbom.spdx').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + def test_tagvalue_writer_unbundled(self): + with io.StringIO() as output: + sbom_writers.TagValueWriter.write(self.unbundled_sbom_doc, output, fragment=True) + expected_output = pathlib.Path('testdata/expected_tagvalue_sbom_unbundled.spdx').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + def test_json_writer(self): + with io.StringIO() as output: + sbom_writers.JSONWriter.write(self.sbom_doc, output) + expected_output = pathlib.Path('testdata/expected_json_sbom.spdx.json').read_text() + self.maxDiff = None + self.assertEqual(expected_output, output.getvalue()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tools/sbom/testdata/expected_json_sbom.spdx.json b/tools/sbom/testdata/expected_json_sbom.spdx.json new file mode 100644 index 0000000000..628615fe26 --- /dev/null +++ b/tools/sbom/testdata/expected_json_sbom.spdx.json @@ -0,0 +1,137 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "test doc", + "documentNamespace": "http://www.google.com/sbom/spdx/android", + "creationInfo": { + "creators": [ + "Organization: Google" + ], + "created": "2023-03-31T22:17:58Z" + }, + "externalDocumentRefs": [ + { + "externalDocumentId": "DocumentRef-external_doc_ref", + "spdxDocument": "external_doc_uri", + "checksum": { + "algorithm": "SHA1", + "checksumValue": "1234567890" + } + } + ], + "documentDescribes": [ + "SPDXRef-PRODUCT" + ], + "packages": [ + { + "name": "PRODUCT", + "SPDXID": "SPDXRef-PRODUCT", + "downloadLocation": "NONE", + "filesAnalyzed": true, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google", + "packageVerificationCode": { + "packageVerificationCodeValue": "123456" + }, + "hasFiles": [ + "SPDXRef-file1", + "SPDXRef-file2", + "SPDXRef-file3" + ] + }, + { + "name": "PLATFORM", + "SPDXID": "SPDXRef-PLATFORM", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google" + }, + { + "name": "Prebuilt package1", + "SPDXID": "SPDXRef-PREBUILT-package1", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google" + }, + { + "name": "Source package1", + "SPDXID": "SPDXRef-SOURCE-package1", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "build_finger_print", + "supplier": "Organization: Google", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceType": "cpe22Type", + "referenceLocator": "cpe:/a:jsoncpp_project:jsoncpp:1.9.4" + } + ] + }, + { + "name": "Upstream package1", + "SPDXID": "SPDXRef-UPSTREAM-package1", + "downloadLocation": "NONE", + "filesAnalyzed": false, + "versionInfo": "1.1", + "supplier": "Organization: upstream" + } + ], + "files": [ + { + "fileName": "/bin/file1", + "SPDXID": "SPDXRef-file1", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "11111" + } + ] + }, + { + "fileName": "/bin/file2", + "SPDXID": "SPDXRef-file2", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "22222" + } + ] + }, + { + "fileName": "/bin/file3", + "SPDXID": "SPDXRef-file3", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "33333" + } + ] + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-file1", + "relatedSpdxElement": "SPDXRef-PLATFORM", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-file2", + "relatedSpdxElement": "SPDXRef-PREBUILT-package1", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-file3", + "relatedSpdxElement": "SPDXRef-SOURCE-package1", + "relationshipType": "GENERATED_FROM" + }, + { + "spdxElementId": "SPDXRef-SOURCE-package1", + "relatedSpdxElement": "SPDXRef-UPSTREAM-package1", + "relationshipType": "VARIANT_OF" + } + ] +}
\ No newline at end of file diff --git a/tools/sbom/testdata/expected_tagvalue_sbom.spdx b/tools/sbom/testdata/expected_tagvalue_sbom.spdx new file mode 100644 index 0000000000..0f1c6f8ec8 --- /dev/null +++ b/tools/sbom/testdata/expected_tagvalue_sbom.spdx @@ -0,0 +1,65 @@ +SPDXVersion: SPDX-2.3 +DataLicense: CC0-1.0 +SPDXID: SPDXRef-DOCUMENT +DocumentName: test doc +DocumentNamespace: http://www.google.com/sbom/spdx/android +Creator: Organization: Google +Created: 2023-03-31T22:17:58Z +ExternalDocumentRef: DocumentRef-external_doc_ref external_doc_uri SHA1: 1234567890 + +PackageName: PRODUCT +SPDXID: SPDXRef-PRODUCT +PackageDownloadLocation: NONE +FilesAnalyzed: true +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +PackageVerificationCode: 123456 + +Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-PRODUCT + +FileName: /bin/file1 +SPDXID: SPDXRef-file1 +FileChecksum: SHA1: 11111 + +FileName: /bin/file2 +SPDXID: SPDXRef-file2 +FileChecksum: SHA1: 22222 + +FileName: /bin/file3 +SPDXID: SPDXRef-file3 +FileChecksum: SHA1: 33333 + +PackageName: PLATFORM +SPDXID: SPDXRef-PLATFORM +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Prebuilt package1 +SPDXID: SPDXRef-PREBUILT-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +PackageName: Source package1 +SPDXID: SPDXRef-SOURCE-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google +ExternalRef: SECURITY cpe22Type cpe:/a:jsoncpp_project:jsoncpp:1.9.4 + +PackageName: Upstream package1 +SPDXID: SPDXRef-UPSTREAM-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: 1.1 +PackageSupplier: Organization: upstream + +Relationship: SPDXRef-SOURCE-package1 VARIANT_OF SPDXRef-UPSTREAM-package1 + +Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-PLATFORM +Relationship: SPDXRef-file2 GENERATED_FROM SPDXRef-PREBUILT-package1 +Relationship: SPDXRef-file3 GENERATED_FROM SPDXRef-SOURCE-package1 diff --git a/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx b/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx new file mode 100644 index 0000000000..a00c291ad7 --- /dev/null +++ b/tools/sbom/testdata/expected_tagvalue_sbom_unbundled.spdx @@ -0,0 +1,12 @@ +FileName: /bin/file1.apk +SPDXID: SPDXRef-file1 +FileChecksum: SHA1: 11111 + +PackageName: Unbundled apk package +SPDXID: SPDXRef-SOURCE-package1 +PackageDownloadLocation: NONE +FilesAnalyzed: false +PackageVersion: build_finger_print +PackageSupplier: Organization: Google + +Relationship: SPDXRef-file1 GENERATED_FROM SPDXRef-SOURCE-package1 |