diff options
Diffstat (limited to 'system/build.py')
| -rwxr-xr-x | system/build.py | 720 |
1 files changed, 720 insertions, 0 deletions
diff --git a/system/build.py b/system/build.py new file mode 100755 index 0000000000..3fcb2fb14a --- /dev/null +++ b/system/build.py @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Build BT targets on the host system. + +For building, you will first have to stage a platform directory that has the +following structure: +|-common-mk +|-bt +|-external +|-|-rust +|-|-|-vendor + +The simplest way to do this is to check out platform2 to another directory (that +is not a subdir of this bt directory), symlink bt there and symlink the rust +vendor repository as well. +""" +import argparse +import multiprocessing +import os +import shutil +import six +import subprocess +import sys +import time + +# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {}) +COMMON_MK_USES = [ + 'asan', + 'coverage', + 'cros_host', + 'fuzzer', + 'fuzzer', + 'msan', + 'profiling', + 'tcmalloc', + 'test', + 'ubsan', +] + +# Default use flags. +USE_DEFAULTS = { + 'android': False, + 'bt_nonstandard_codecs': False, + 'test': False, +} + +VALID_TARGETS = [ + 'prepare', # Prepare the output directory (gn gen + rust setup) + 'tools', # Build the host tools (i.e. packetgen) + 'rust', # Build only the rust components + copy artifacts to output dir + 'main', # Build the main C++ codebase + 'test', # Run the unit tests + 'clean', # Clean up output directory + 'all', # All targets except test and clean +] + +# TODO(b/190750167) - Host tests are disabled until we are full bazel build +HOST_TESTS = [ + # 'bluetooth_test_common', + # 'bluetoothtbd_test', + # 'net_test_avrcp', + # 'net_test_btcore', + # 'net_test_types', + # 'net_test_btm_iso', + # 'net_test_btpackets', +] + +BOOTSTRAP_GIT_REPOS = { + 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2', + 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates', + 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging' +} + +# List of packages required for linux build +REQUIRED_APT_PACKAGES = [ + 'bison', + 'build-essential', + 'curl', + 'debmake', + 'flatbuffers-compiler', + 'flex', + 'g++-multilib', + 'gcc-multilib', + 'generate-ninja', + 'gnupg', + 'gperf', + 'libc++-dev', + 'libdbus-1-dev', + 'libevent-dev', + 'libevent-dev', + 'libflatbuffers-dev', + 'libflatbuffers1', + 'libgl1-mesa-dev', + 'libglib2.0-dev', + 'liblz4-tool', + 'libncurses5', + 'libnss3-dev', + 'libprotobuf-dev', + 'libre2-9', + 'libssl-dev', + 'libtinyxml2-dev', + 'libx11-dev', + 'libxml2-utils', + 'ninja-build', + 'openssl', + 'protobuf-compiler', + 'unzip', + 'x11proto-core-dev', + 'xsltproc', + 'zip', + 'zlib1g-dev', +] + +# List of cargo packages required for linux build +REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd'] + +APT_PKG_LIST = ['apt', '-qq', 'list'] +CARGO_PKG_LIST = ['cargo', 'install', '--list'] + + +class UseFlags(): + + def __init__(self, use_flags): + """ Construct the use flags. + + Args: + use_flags: List of use flags parsed from the command. + """ + self.flags = {} + + # Import use flags required by common-mk + for use in COMMON_MK_USES: + self.set_flag(use, False) + + # Set our defaults + for use, value in USE_DEFAULTS.items(): + self.set_flag(use, value) + + # Set use flags - value is set to True unless the use starts with - + # All given use flags always override the defaults + for use in use_flags: + value = not use.startswith('-') + self.set_flag(use, value) + + def set_flag(self, key, value=True): + setattr(self, key, value) + self.flags[key] = value + + +class HostBuild(): + + def __init__(self, args): + """ Construct the builder. + + Args: + args: Parsed arguments from ArgumentParser + """ + self.args = args + + # Set jobs to number of cpus unless explicitly set + self.jobs = self.args.jobs + if not self.jobs: + self.jobs = multiprocessing.cpu_count() + print("Number of jobs = {}".format(self.jobs)) + + # Normalize bootstrap dir and make sure it exists + self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir) + os.makedirs(self.bootstrap_dir, exist_ok=True) + + # Output and platform directories are based on bootstrap + self.output_dir = os.path.join(self.bootstrap_dir, 'output') + self.platform_dir = os.path.join(self.bootstrap_dir, 'staging') + self.sysroot = self.args.sysroot + self.libdir = self.args.libdir + + # If default target isn't set, build everything + self.target = 'all' + if hasattr(self.args, 'target') and self.args.target: + self.target = self.args.target + + target_use = self.args.use if self.args.use else [] + + # Unless set, always build test code + if not self.args.notest: + target_use.append('test') + + self.use = UseFlags(target_use) + + # Validate platform directory + assert os.path.isdir(self.platform_dir), 'Platform dir does not exist' + assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root' + + # Make sure output directory exists (or create it) + os.makedirs(self.output_dir, exist_ok=True) + + # Set some default attributes + self.libbase_ver = None + + self.configure_environ() + + def _generate_rustflags(self): + """ Rustflags to include for the build. + """ + rust_flags = [ + '-L', + '{}/out/Default'.format(self.output_dir), + '-C', + 'link-arg=-Wl,--allow-multiple-definition', + ] + + return ' '.join(rust_flags) + + def configure_environ(self): + """ Configure environment variables for GN and Cargo. + """ + self.env = os.environ.copy() + + # Make sure cargo home dir exists and has a bin directory + cargo_home = os.path.join(self.output_dir, 'cargo_home') + os.makedirs(cargo_home, exist_ok=True) + os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True) + + # Configure Rust env variables + self.env['CARGO_TARGET_DIR'] = self.output_dir + self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home') + self.env['RUSTFLAGS'] = self._generate_rustflags() + self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt') + + def run_command(self, target, args, cwd=None, env=None): + """ Run command and stream the output. + """ + # Set some defaults + if not cwd: + cwd = self.platform_dir + if not env: + env = self.env + + log_file = os.path.join(self.output_dir, '{}.log'.format(target)) + with open(log_file, 'wb') as lf: + rc = 0 + process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE) + while True: + line = process.stdout.readline() + print(line.decode('utf-8'), end="") + lf.write(line) + if not line: + rc = process.poll() + if rc is not None: + break + + time.sleep(0.1) + + if rc != 0: + raise Exception("Return code is {}".format(rc)) + + def _get_basever(self): + if self.libbase_ver: + return self.libbase_ver + + self.libbase_ver = os.environ.get('BASE_VER', '') + if not self.libbase_ver: + base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER') + try: + with open(base_file, 'r') as f: + self.libbase_ver = f.read().strip('\n') + except: + self.libbase_ver = 'NOT-INSTALLED' + + return self.libbase_ver + + def _gn_default_output(self): + return os.path.join(self.output_dir, 'out/Default') + + def _gn_configure(self): + """ Configure all required parameters for platform2. + + Mostly copied from //common-mk/platform2.py + """ + clang = not self.args.no_clang + + def to_gn_string(s): + return '"%s"' % s.replace('"', '\\"') + + def to_gn_list(strs): + return '[%s]' % ','.join([to_gn_string(s) for s in strs]) + + def to_gn_args_args(gn_args): + for k, v in gn_args.items(): + if isinstance(v, bool): + v = str(v).lower() + elif isinstance(v, list): + v = to_gn_list(v) + elif isinstance(v, six.string_types): + v = to_gn_string(v) + else: + raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v)) + yield '%s=%s' % (k.replace('-', '_'), v) + + gn_args = { + 'platform_subdir': 'bt', + 'cc': 'clang' if clang else 'gcc', + 'cxx': 'clang++' if clang else 'g++', + 'ar': 'llvm-ar' if clang else 'ar', + 'pkg-config': 'pkg-config', + 'clang_cc': clang, + 'clang_cxx': clang, + 'OS': 'linux', + 'sysroot': self.sysroot, + 'libdir': os.path.join(self.sysroot, self.libdir), + 'build_root': self.output_dir, + 'platform2_root': self.platform_dir, + 'libbase_ver': self._get_basever(), + 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1', + 'external_cflags': [], + 'external_cxxflags': [], + 'enable_werror': False, + } + + if clang: + # Make sure to mark the clang use flag as true + self.use.set_flag('clang', True) + gn_args['external_cxxflags'] += ['-I/usr/include/'] + + gn_args_args = list(to_gn_args_args(gn_args)) + use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()] + gn_args_args += ['use={%s}' % (' '.join(use_args))] + + gn_args = [ + 'gn', + 'gen', + ] + + if self.args.verbose: + gn_args.append('-v') + + gn_args += [ + '--root=%s' % self.platform_dir, + '--args=%s' % ' '.join(gn_args_args), + self._gn_default_output(), + ] + + if 'PKG_CONFIG_PATH' in self.env: + print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH']) + + self.run_command('configure', gn_args) + + def _gn_build(self, target): + """ Generate the ninja command for the target and run it. + """ + args = ['%s:%s' % ('bt', target)] + ninja_args = ['ninja', '-C', self._gn_default_output()] + if self.jobs: + ninja_args += ['-j', str(self.jobs)] + ninja_args += args + + if self.args.verbose: + ninja_args.append('-v') + + self.run_command('build', ninja_args) + + def _rust_configure(self): + """ Generate config file at cargo_home so we use vendored crates. + """ + template = """ + [source.systembt] + directory = "{}/external/rust/vendor" + + [source.crates-io] + replace-with = "systembt" + local-registry = "/nonexistent" + """ + + if not self.args.no_vendored_rust: + contents = template.format(self.platform_dir) + with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f: + f.write(contents) + + def _rust_build(self): + """ Run `cargo build` from platform2/bt directory. + """ + self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) + + def _target_prepare(self): + """ Target to prepare the output directory for building. + + This runs gn gen to generate all rquired files and set up the Rust + config properly. This will be run + """ + self._gn_configure() + self._rust_configure() + + def _target_tools(self): + """ Build the tools target in an already prepared environment. + """ + self._gn_build('tools') + + # Also copy bluetooth_packetgen to CARGO_HOME so it's available + shutil.copy( + os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin')) + + def _target_rust(self): + """ Build rust artifacts in an already prepared environment. + """ + self._rust_build() + rust_dir = os.path.join(self._gn_default_output(), 'rust') + if os.path.exists(rust_dir): + shutil.rmtree(rust_dir) + shutil.copytree(os.path.join(self.output_dir, 'debug'), rust_dir) + + def _target_main(self): + """ Build the main GN artifacts in an already prepared environment. + """ + self._gn_build('all') + + def _target_test(self): + """ Runs the host tests. + """ + # Rust tests first + self.run_command('test', ['cargo', 'test'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) + + # Host tests second based on host test list + for t in HOST_TESTS: + self.run_command( + 'test', [os.path.join(self.output_dir, 'out/Default', t)], + cwd=os.path.join(self.output_dir), + env=self.env) + + def _target_clean(self): + """ Delete the output directory entirely. + """ + shutil.rmtree(self.output_dir) + # Remove Cargo.lock that may have become generated + os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock')) + + def _target_all(self): + """ Build all common targets (skipping test and clean). + """ + self._target_prepare() + self._target_tools() + self._target_main() + self._target_rust() + + def build(self): + """ Builds according to self.target + """ + print('Building target ', self.target) + + if self.target == 'prepare': + self._target_prepare() + elif self.target == 'tools': + self._target_tools() + elif self.target == 'rust': + self._target_rust() + elif self.target == 'main': + self._target_main() + elif self.target == 'test': + self._target_test() + elif self.target == 'clean': + self._target_clean() + elif self.target == 'all': + self._target_all() + + +class Bootstrap(): + + def __init__(self, base_dir, bt_dir): + """ Construct bootstrapper. + + Args: + base_dir: Where to stage everything. + bt_dir: Where bluetooth source is kept (will be symlinked) + """ + self.base_dir = os.path.abspath(base_dir) + self.bt_dir = os.path.abspath(bt_dir) + + # Create base directory if it doesn't already exist + os.makedirs(self.base_dir, exist_ok=True) + + if not os.path.isdir(self.bt_dir): + raise Exception('{} is not a valid directory'.format(self.bt_dir)) + + self.git_dir = os.path.join(self.base_dir, 'repos') + self.staging_dir = os.path.join(self.base_dir, 'staging') + self.output_dir = os.path.join(self.base_dir, 'output') + self.external_dir = os.path.join(self.base_dir, 'staging', 'external') + + self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete') + + def _update_platform2(self): + """Updates repositories used for build.""" + for repo in BOOTSTRAP_GIT_REPOS.keys(): + cwd = os.path.join(self.git_dir, repo) + subprocess.check_call(['git', 'pull'], cwd=cwd) + + def _setup_platform2(self): + """ Set up platform2. + + This will check out all the git repos and symlink everything correctly. + """ + + # If already set up, exit early + if os.path.isfile(self.dir_setup_complete): + print('{} already set-up. Updating instead.'.format(self.base_dir)) + self._update_platform2() + return + + # Create all directories we will need to use + for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]: + os.makedirs(dirpath) + + # Check out all repos in git directory + for repo in BOOTSTRAP_GIT_REPOS.values(): + subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir) + + # Symlink things + symlinks = [ + (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')), + (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')), + (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')), + (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')), + (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')), + ] + + # Create symlinks + for pairs in symlinks: + (src, dst) = pairs + os.symlink(src, dst) + + # Write to setup complete file so we don't repeat this step + with open(self.dir_setup_complete, 'w') as f: + f.write('Setup complete.') + + def _pretty_print_install(self, install_cmd, packages, line_limit=80): + """ Pretty print an install command. + + Args: + install_cmd: Prefixed install command. + packages: Enumerate packages and append them to install command. + line_limit: Number of characters per line. + + Return: + Array of lines to join and print. + """ + install = [install_cmd] + line = ' ' + # Remainder needed = space + len(pkg) + space + \ + # Assuming 80 character lines, that's 80 - 3 = 77 + line_limit = line_limit - 3 + for pkg in packages: + if len(line) + len(pkg) < line_limit: + line = '{}{} '.format(line, pkg) + else: + install.append(line) + line = ' {} '.format(pkg) + + if len(line) > 0: + install.append(line) + + return install + + def _check_package_installed(self, package, cmd, predicate): + """Check that the given package is installed. + + Args: + package: Check that this package is installed. + cmd: Command prefix to check if installed (package appended to end) + predicate: Function/lambda to check if package is installed based + on output. Takes string output and returns boolean. + + Return: + True if package is installed. + """ + try: + output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT) + is_installed = predicate(output.decode('utf-8')) + print(' {} is {}'.format(package, 'installed' if is_installed else 'missing')) + + return is_installed + except Exception as e: + print(e) + return False + + def _get_command_output(self, cmd): + """Runs the command and gets the output. + + Args: + cmd: Command to run. + + Return: + Tuple (Success, Output). Success represents if the command ran ok. + """ + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return (True, output.decode('utf-8').split('\n')) + except Exception as e: + print(e) + return (False, "") + + def _print_missing_packages(self): + """Print any missing packages found via apt. + + This will find any missing packages necessary for build using apt and + print it out as an apt-get install printf. + """ + print('Checking for any missing packages...') + + (success, output) = self._get_command_output(APT_PKG_LIST) + if not success: + raise Exception("Could not query apt for packages.") + + packages_installed = {} + for line in output: + if 'installed' in line: + split = line.split('/', 2) + packages_installed[split[0]] = True + + need_packages = [] + for pkg in REQUIRED_APT_PACKAGES: + if pkg not in packages_installed: + need_packages.append(pkg) + + # No packages need to be installed + if len(need_packages) == 0: + print('+ All required packages are installed') + return + + install = self._pretty_print_install('sudo apt-get install', need_packages) + + # Print all lines so they can be run in cmdline + print('Missing system packages. Run the following command: ') + print(' \\\n'.join(install)) + + def _print_missing_rust_packages(self): + """Print any missing packages found via cargo. + + This will find any missing packages necessary for build using cargo and + print it out as a cargo-install printf. + """ + print('Checking for any missing cargo packages...') + + (success, output) = self._get_command_output(CARGO_PKG_LIST) + if not success: + raise Exception("Could not query cargo for packages.") + + packages_installed = {} + for line in output: + # Cargo installed packages have this format + # <crate name> <version>: + # <binary name> + # We only care about the crates themselves + if ':' not in line: + continue + + split = line.split(' ', 2) + packages_installed[split[0]] = True + + need_packages = [] + for pkg in REQUIRED_CARGO_PACKAGES: + if pkg not in packages_installed: + need_packages.append(pkg) + + # No packages to be installed + if len(need_packages) == 0: + print('+ All required cargo packages are installed') + return + + install = self._pretty_print_install('cargo install', need_packages) + print('Missing cargo packages. Run the following command: ') + print(' \\\n'.join(install)) + + def bootstrap(self): + """ Bootstrap the Linux build.""" + self._setup_platform2() + self._print_missing_packages() + self._print_missing_rust_packages() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Simple build for host.') + parser.add_argument( + '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss") + parser.add_argument( + '--run-bootstrap', + help='Run bootstrap code to verify build env is ok to build.', + default=False, + action='store_true') + parser.add_argument('--no-clang', help='Use clang compiler.', default=False, action='store_true') + parser.add_argument('--use', help='Set a specific use flag.') + parser.add_argument('--notest', help="Don't compile test code.", default=False, action='store_true') + parser.add_argument('--target', help='Run specific build target') + parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/') + parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib') + parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int) + parser.add_argument( + '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true') + parser.add_argument('--verbose', help='Verbose logs for build.') + args = parser.parse_args() + + # Make sure we get absolute path + expanded path for bootstrap directory + args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir)) + + if args.run_bootstrap: + bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__)) + bootstrap.bootstrap() + else: + build = HostBuild(args) + build.build() |