diff options
Diffstat (limited to 'ci/optimized_targets.py')
| -rw-r--r-- | ci/optimized_targets.py | 451 |
1 files changed, 432 insertions, 19 deletions
diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py index 224c8c0221..688bdd8370 100644 --- a/ci/optimized_targets.py +++ b/ci/optimized_targets.py @@ -14,6 +14,16 @@ # limitations under the License. from abc import ABC +import argparse +import functools +import json +import logging +import os +import pathlib +import subprocess + +from build_context import BuildContext +import test_mapping_module_retriever class OptimizedBuildTarget(ABC): @@ -24,15 +34,132 @@ class OptimizedBuildTarget(ABC): build. """ - def __init__(self, build_context, args): + _SOONG_UI_BASH_PATH = 'build/soong/soong_ui.bash' + _PREBUILT_SOONG_ZIP_PATH = 'prebuilts/build-tools/linux-x86/bin/soong_zip' + + def __init__( + self, + target: str, + build_context: BuildContext, + args: argparse.Namespace, + ): + self.target = target self.build_context = build_context self.args = args - def get_build_targets(self): - pass + def get_build_targets(self) -> set[str]: + features = self.build_context.enabled_build_features + if self.get_enabled_flag() in features: + self.modules_to_build = self.get_build_targets_impl() + return self.modules_to_build + + self.modules_to_build = {self.target} + return {self.target} + + def get_package_outputs_commands(self) -> list[list[str]]: + features = self.build_context.enabled_build_features + if self.get_enabled_flag() in features: + return self.get_package_outputs_commands_impl() + + return [] + + def get_package_outputs_commands_impl(self) -> list[list[str]]: + raise NotImplementedError( + 'get_package_outputs_commands_impl not implemented in' + f' {type(self).__name__}' + ) - def package_outputs(self): - pass + def get_enabled_flag(self): + raise NotImplementedError( + f'get_enabled_flag not implemented in {type(self).__name__}' + ) + + def get_build_targets_impl(self) -> set[str]: + raise NotImplementedError( + f'get_build_targets_impl not implemented in {type(self).__name__}' + ) + + def _generate_zip_options_for_items( + self, + prefix: str = '', + relative_root: str = '', + list_files: list[str] | None = None, + files: list[str] | None = None, + directories: list[str] | None = None, + ) -> list[str]: + if not list_files and not files and not directories: + raise RuntimeError( + f'No items specified to be added to zip! Prefix: {prefix}, Relative' + f' root: {relative_root}' + ) + command_segment = [] + # These are all soong_zip options so consult soong_zip --help for specifics. + if prefix: + command_segment.append('-P') + command_segment.append(prefix) + if relative_root: + command_segment.append('-C') + command_segment.append(relative_root) + if list_files: + for list_file in list_files: + command_segment.append('-l') + command_segment.append(list_file) + if files: + for file in files: + command_segment.append('-f') + command_segment.append(file) + if directories: + for directory in directories: + command_segment.append('-D') + command_segment.append(directory) + + return command_segment + + def _query_soong_vars( + self, src_top: pathlib.Path, soong_vars: list[str] + ) -> dict[str, str]: + process_result = subprocess.run( + args=[ + f'{src_top / self._SOONG_UI_BASH_PATH}', + '--dumpvars-mode', + f'--abs-vars={" ".join(soong_vars)}', + ], + env=os.environ, + check=False, + capture_output=True, + text=True, + ) + if not process_result.returncode == 0: + logging.error('soong dumpvars command failed! stderr:') + logging.error(process_result.stderr) + raise RuntimeError('Soong dumpvars failed! See log for stderr.') + + if not process_result.stdout: + raise RuntimeError( + 'Necessary soong variables ' + soong_vars + ' not found.' + ) + + try: + return { + line.split('=')[0]: line.split('=')[1].strip("'") + for line in process_result.stdout.strip().split('\n') + } + except IndexError as e: + raise RuntimeError( + 'Error parsing soong dumpvars output! See output here:' + f' {process_result.stdout}', + e, + ) + + def _base_zip_command( + self, src_top: pathlib.Path, dist_dir: pathlib.Path, name: str + ) -> list[str]: + return [ + f'{src_top / self._PREBUILT_SOONG_ZIP_PATH }', + '-d', + '-o', + f'{dist_dir / name}', + ] class NullOptimizer(OptimizedBuildTarget): @@ -48,22 +175,308 @@ class NullOptimizer(OptimizedBuildTarget): def get_build_targets(self): return {self.target} - def package_outputs(self): - pass + def get_package_outputs_commands(self): + return [] + + +class ChangeInfo: + + def __init__(self, change_info_file_path): + try: + with open(change_info_file_path) as change_info_file: + change_info_contents = json.load(change_info_file) + except json.decoder.JSONDecodeError: + logging.error(f'Failed to load CHANGE_INFO: {change_info_file_path}') + raise + + self._change_info_contents = change_info_contents + + def find_changed_files(self) -> set[str]: + changed_files = set() + + for change in self._change_info_contents['changes']: + project_path = change.get('projectPath') + '/' + + for revision in change.get('revisions'): + for file_info in revision.get('fileInfos'): + changed_files.add(project_path + file_info.get('path')) + + return changed_files + + +class GeneralTestsOptimizer(OptimizedBuildTarget): + """general-tests optimizer + + This optimizer reads in the list of changed files from the file located in + env[CHANGE_INFO] and uses this list alongside the normal TEST MAPPING logic to + determine what test mapping modules will run for the given changes. It then + builds those modules and packages them in the same way general-tests.zip is + normally built. + """ + + # List of modules that are built alongside general-tests as dependencies. + _REQUIRED_MODULES = frozenset([ + 'cts-tradefed', + 'vts-tradefed', + 'compatibility-host-util', + 'general-tests-shared-libs', + ]) + + def get_build_targets_impl(self) -> set[str]: + change_info_file_path = os.environ.get('CHANGE_INFO') + if not change_info_file_path: + logging.info( + 'No CHANGE_INFO env var found, general-tests optimization disabled.' + ) + return {'general-tests'} + + test_infos = self.build_context.test_infos + test_mapping_test_groups = set() + for test_info in test_infos: + is_test_mapping = test_info.is_test_mapping + current_test_mapping_test_groups = test_info.test_mapping_test_groups + uses_general_tests = test_info.build_target_used('general-tests') + + if uses_general_tests and not is_test_mapping: + logging.info( + 'Test uses general-tests.zip but is not test-mapping, general-tests' + ' optimization disabled.' + ) + return {'general-tests'} + + if is_test_mapping: + test_mapping_test_groups.update(current_test_mapping_test_groups) + + change_info = ChangeInfo(change_info_file_path) + changed_files = change_info.find_changed_files() + + test_mappings = test_mapping_module_retriever.GetTestMappings( + changed_files, set() + ) + + modules_to_build = set(self._REQUIRED_MODULES) + + modules_to_build.update( + test_mapping_module_retriever.FindAffectedModules( + test_mappings, changed_files, test_mapping_test_groups + ) + ) + + return modules_to_build + + def get_package_outputs_commands_impl(self): + src_top = pathlib.Path(os.environ.get('TOP', os.getcwd())) + dist_dir = pathlib.Path(os.environ.get('DIST_DIR')) + + soong_vars = self._query_soong_vars( + src_top, + [ + 'HOST_OUT_TESTCASES', + 'TARGET_OUT_TESTCASES', + 'PRODUCT_OUT', + 'SOONG_HOST_OUT', + 'HOST_OUT', + ], + ) + host_out_testcases = pathlib.Path(soong_vars.get('HOST_OUT_TESTCASES')) + target_out_testcases = pathlib.Path(soong_vars.get('TARGET_OUT_TESTCASES')) + product_out = pathlib.Path(soong_vars.get('PRODUCT_OUT')) + soong_host_out = pathlib.Path(soong_vars.get('SOONG_HOST_OUT')) + host_out = pathlib.Path(soong_vars.get('HOST_OUT')) + + host_paths = [] + target_paths = [] + host_config_files = [] + target_config_files = [] + for module in self.modules_to_build: + # The required modules are handled separately, no need to package. + if module in self._REQUIRED_MODULES: + continue + + host_path = host_out_testcases / module + if os.path.exists(host_path): + host_paths.append(host_path) + self._collect_config_files(src_top, host_path, host_config_files) + + target_path = target_out_testcases / module + if os.path.exists(target_path): + target_paths.append(target_path) + self._collect_config_files(src_top, target_path, target_config_files) + + if not os.path.exists(host_path) and not os.path.exists(target_path): + logging.info(f'No host or target build outputs found for {module}.') + + zip_commands = [] + + zip_commands.extend( + self._get_zip_test_configs_zips_commands( + src_top, + dist_dir, + host_out, + product_out, + host_config_files, + target_config_files, + ) + ) + + zip_command = self._base_zip_command(src_top, dist_dir, 'general-tests.zip') + + # Add host testcases. + if host_paths: + zip_command.extend( + self._generate_zip_options_for_items( + prefix='host', + relative_root=f'{src_top / soong_host_out}', + directories=host_paths, + ) + ) + + # Add target testcases. + if target_paths: + zip_command.extend( + self._generate_zip_options_for_items( + prefix='target', + relative_root=f'{src_top / product_out}', + directories=target_paths, + ) + ) + + # TODO(lucafarsi): Push this logic into a general-tests-minimal build command + # Add necessary tools. These are also hardcoded in general-tests.mk. + framework_path = soong_host_out / 'framework' + + zip_command.extend( + self._generate_zip_options_for_items( + prefix='host/tools', + relative_root=str(framework_path), + files=[ + f"{framework_path / 'cts-tradefed.jar'}", + f"{framework_path / 'compatibility-host-util.jar'}", + f"{framework_path / 'vts-tradefed.jar'}", + ], + ) + ) + + zip_commands.append(zip_command) + return zip_commands + + def _collect_config_files( + self, + src_top: pathlib.Path, + root_dir: pathlib.Path, + config_files: list[str], + ): + for root, dirs, files in os.walk(src_top / root_dir): + for file in files: + if file.endswith('.config'): + config_files.append(root_dir / file) + + def _get_zip_test_configs_zips_commands( + self, + src_top: pathlib.Path, + dist_dir: pathlib.Path, + host_out: pathlib.Path, + product_out: pathlib.Path, + host_config_files: list[str], + target_config_files: list[str], + ) -> tuple[list[str], list[str]]: + """Generate general-tests_configs.zip and general-tests_list.zip. + + general-tests_configs.zip contains all of the .config files that were + built and general-tests_list.zip contains a text file which lists + all of the .config files that are in general-tests_configs.zip. + + general-tests_configs.zip is organized as follows: + / + host/ + testcases/ + test_1.config + test_2.config + ... + target/ + testcases/ + test_1.config + test_2.config + ... + + So the process is we write out the paths to all the host config files into + one + file and all the paths to the target config files in another. We also write + the paths to all the config files into a third file to use for + general-tests_list.zip. + + Args: + dist_dir: dist directory. + host_out: host out directory. + product_out: product out directory. + host_config_files: list of all host config files. + target_config_files: list of all target config files. + + Returns: + The commands to generate general-tests_configs.zip and + general-tests_list.zip + """ + with open( + f"{host_out / 'host_general-tests_list'}", 'w' + ) as host_list_file, open( + f"{product_out / 'target_general-tests_list'}", 'w' + ) as target_list_file, open( + f"{host_out / 'general-tests_list'}", 'w' + ) as list_file: + + for config_file in host_config_files: + host_list_file.write(f'{config_file}' + '\n') + list_file.write('host/' + os.path.relpath(config_file, host_out) + '\n') + + for config_file in target_config_files: + target_list_file.write(f'{config_file}' + '\n') + list_file.write( + 'target/' + os.path.relpath(config_file, product_out) + '\n' + ) + + zip_commands = [] + + tests_config_zip_command = self._base_zip_command( + src_top, dist_dir, 'general-tests_configs.zip' + ) + tests_config_zip_command.extend( + self._generate_zip_options_for_items( + prefix='host', + relative_root=str(host_out), + list_files=[f"{host_out / 'host_general-tests_list'}"], + ) + ) + + tests_config_zip_command.extend( + self._generate_zip_options_for_items( + prefix='target', + relative_root=str(product_out), + list_files=[f"{product_out / 'target_general-tests_list'}"], + ), + ) + + zip_commands.append(tests_config_zip_command) + + tests_list_zip_command = self._base_zip_command( + src_top, dist_dir, 'general-tests_list.zip' + ) + tests_list_zip_command.extend( + self._generate_zip_options_for_items( + relative_root=str(host_out), + files=[f"{host_out / 'general-tests_list'}"], + ) + ) + zip_commands.append(tests_list_zip_command) + return zip_commands -def get_target_optimizer(target, enabled_flag, build_context, optimizer): - if enabled_flag in build_context['enabled_build_features']: - return optimizer + def get_enabled_flag(self): + return 'general_tests_optimized' - return NullOptimizer(target) + @classmethod + def get_optimized_targets(cls) -> dict[str, OptimizedBuildTarget]: + return {'general-tests': functools.partial(cls)} -# To be written as: -# 'target': lambda target, build_context, args: get_target_optimizer( -# target, -# 'target_enabled_flag', -# build_context, -# TargetOptimizer(build_context, args), -# ) -OPTIMIZED_BUILD_TARGETS = dict() +OPTIMIZED_BUILD_TARGETS = {} +OPTIMIZED_BUILD_TARGETS.update(GeneralTestsOptimizer.get_optimized_targets()) |