Split the huge merge_target_files script into multiple files.

Bug: 221858722
Test: m otatools; Use to create merged builds
Test: atest --host releasetools_test
Change-Id: I5f932f160d3f6405b41a7721b1c75cc96749e77b

1 files changed, 322 insertions, 0 deletions

diff --git a/tools/releasetools/merge/merge_dexopt.py b/tools/releasetools/merge/merge_dexopt.py
new file mode 100644
index 0000000000..dd6e2f906f
--- /dev/null
+++ b/tools/releasetools/merge/merge_dexopt.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+"""Generates dexopt files for vendor apps, from a merged target_files.
+
+Expects items in OPTIONS prepared by merge_target_files.py.
+"""
+
+import glob
+import json
+import logging
+import os
+import shutil
+import subprocess
+
+import common
+
+logger = logging.getLogger(__name__)
+OPTIONS = common.OPTIONS
+
+
+def MergeDexopt(temp_dir, output_target_files_dir):
+  """If needed, generates dexopt files for vendor apps.
+
+  Args:
+    temp_dir: Location containing an 'output' directory where target files have
+      been extracted, e.g. <temp_dir>/output/SYSTEM, <temp_dir>/output/IMAGES,
+      etc.
+    output_target_files_dir: The name of a directory that will be used to create
+      the output target files package after all the special cases are processed.
+  """
+  # Load vendor and framework META/misc_info.txt.
+  if (OPTIONS.vendor_misc_info.get('building_with_vsdk') != 'true' or
+      OPTIONS.framework_dexpreopt_tools is None or
+      OPTIONS.framework_dexpreopt_config is None or
+      OPTIONS.vendor_dexpreopt_config is None):
+    return
+
+  logger.info('applying dexpreopt')
+
+  # The directory structure to apply dexpreopt is:
+  #
+  # <temp_dir>/
+  #     framework_meta/
+  #         META/
+  #     vendor_meta/
+  #         META/
+  #     output/
+  #         SYSTEM/
+  #         VENDOR/
+  #         IMAGES/
+  #         <other items extracted from system and vendor target files>
+  #     tools/
+  #         <contents of dexpreopt_tools.zip>
+  #     system_config/
+  #         <contents of system dexpreopt_config.zip>
+  #     vendor_config/
+  #         <contents of vendor dexpreopt_config.zip>
+  #     system -> output/SYSTEM
+  #     vendor -> output/VENDOR
+  #     apex -> output/SYSTEM/apex (only for flattened APEX builds)
+  #     apex/ (extracted updatable APEX)
+  #         <apex 1>/
+  #             ...
+  #         <apex 2>/
+  #             ...
+  #         ...
+  #     out/dex2oat_result/vendor/
+  #         <app>
+  #             oat/arm64/
+  #                 package.vdex
+  #                 package.odex
+  #         <priv-app>
+  #             oat/arm64/
+  #                 package.vdex
+  #                 package.odex
+  dexpreopt_tools_files_temp_dir = os.path.join(temp_dir, 'tools')
+  dexpreopt_framework_config_files_temp_dir = os.path.join(
+      temp_dir, 'system_config')
+  dexpreopt_vendor_config_files_temp_dir = os.path.join(temp_dir,
+                                                        'vendor_config')
+
+  extract_items(
+      input_zip=OPTIONS.framework_dexpreopt_tools,
+      output_dir=dexpreopt_tools_files_temp_dir,
+      extract_item_list=('*',))
+  extract_items(
+      input_zip=OPTIONS.framework_dexpreopt_config,
+      output_dir=dexpreopt_framework_config_files_temp_dir,
+      extract_item_list=('*',))
+  extract_items(
+      input_zip=OPTIONS.vendor_dexpreopt_config,
+      output_dir=dexpreopt_vendor_config_files_temp_dir,
+      extract_item_list=('*',))
+
+  os.symlink(
+      os.path.join(output_target_files_dir, 'SYSTEM'),
+      os.path.join(temp_dir, 'system'))
+  os.symlink(
+      os.path.join(output_target_files_dir, 'VENDOR'),
+      os.path.join(temp_dir, 'vendor'))
+
+  # The directory structure for flatteded APEXes is:
+  #
+  # SYSTEM
+  #     apex
+  #         <APEX name, e.g., com.android.wifi>
+  #             apex_manifest.pb
+  #             apex_pubkey
+  #             etc/
+  #             javalib/
+  #             lib/
+  #             lib64/
+  #             priv-app/
+  #
+  # The directory structure for updatable APEXes is:
+  #
+  # SYSTEM
+  #     apex
+  #         com.android.adbd.apex
+  #         com.android.appsearch.apex
+  #         com.android.art.apex
+  #         ...
+  apex_root = os.path.join(output_target_files_dir, 'SYSTEM', 'apex')
+
+  # Check for flattended versus updatable APEX.
+  if OPTIONS.framework_misc_info.get('target_flatten_apex') == 'false':
+    # Extract APEX.
+    logging.info('extracting APEX')
+
+    apex_extract_root_dir = os.path.join(temp_dir, 'apex')
+    os.makedirs(apex_extract_root_dir)
+
+    for apex in (glob.glob(os.path.join(apex_root, '*.apex')) +
+                 glob.glob(os.path.join(apex_root, '*.capex'))):
+      logging.info('  apex: %s', apex)
+      # deapexer is in the same directory as the merge_target_files binary extracted
+      # from otatools.zip.
+      apex_json_info = subprocess.check_output(['deapexer', 'info', apex])
+      logging.info('    info: %s', apex_json_info)
+      apex_info = json.loads(apex_json_info)
+      apex_name = apex_info['name']
+      logging.info('    name: %s', apex_name)
+
+      apex_extract_dir = os.path.join(apex_extract_root_dir, apex_name)
+      os.makedirs(apex_extract_dir)
+
+      # deapexer uses debugfs_static, which is part of otatools.zip.
+      command = [
+          'deapexer',
+          '--debugfs_path',
+          'debugfs_static',
+          'extract',
+          apex,
+          apex_extract_dir,
+      ]
+      logging.info('    running %s', command)
+      subprocess.check_call(command)
+  else:
+    # Flattened APEXes don't need to be extracted since they have the necessary
+    # directory structure.
+    os.symlink(os.path.join(apex_root), os.path.join(temp_dir, 'apex'))
+
+  # Modify system config to point to the tools that have been extracted.
+  # Absolute or .. paths are not allowed  by the dexpreopt_gen tool in
+  # dexpreopt_soong.config.
+  dexpreopt_framework_soon_config = os.path.join(
+      dexpreopt_framework_config_files_temp_dir, 'dexpreopt_soong.config')
+  with open(dexpreopt_framework_soon_config, 'w') as f:
+    dexpreopt_soong_config = {
+        'Profman': 'tools/profman',
+        'Dex2oat': 'tools/dex2oatd',
+        'Aapt': 'tools/aapt2',
+        'SoongZip': 'tools/soong_zip',
+        'Zip2zip': 'tools/zip2zip',
+        'ManifestCheck': 'tools/manifest_check',
+        'ConstructContext': 'tools/construct_context',
+    }
+    json.dump(dexpreopt_soong_config, f)
+
+  # TODO(b/188179859): Make *dex location configurable to vendor or system_other.
+  use_system_other_odex = False
+
+  if use_system_other_odex:
+    dex_img = 'SYSTEM_OTHER'
+  else:
+    dex_img = 'VENDOR'
+    # Open vendor_filesystem_config to append the items generated by dexopt.
+    vendor_file_system_config = open(
+        os.path.join(temp_dir, 'output', 'META',
+                     'vendor_filesystem_config.txt'), 'a')
+
+  # Dexpreopt vendor apps.
+  dexpreopt_config_suffix = '_dexpreopt.config'
+  for config in glob.glob(
+      os.path.join(dexpreopt_vendor_config_files_temp_dir,
+                   '*' + dexpreopt_config_suffix)):
+    app = os.path.basename(config)[:-len(dexpreopt_config_suffix)]
+    logging.info('dexpreopt config: %s %s', config, app)
+
+    apk_dir = 'app'
+    apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
+    if not os.path.exists(apk_path):
+      apk_dir = 'priv-app'
+      apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
+      if not os.path.exists(apk_path):
+        logging.warning(
+            'skipping dexpreopt for %s, no apk found in vendor/app '
+            'or vendor/priv-app', app)
+        continue
+
+    # Generate dexpreopting script. Note 'out_dir' is not the output directory
+    # where the script is generated, but the OUT_DIR at build time referenced
+    # in the dexpreot config files, e.g., "out/.../core-oj.jar", so the tool knows
+    # how to adjust the path.
+    command = [
+        os.path.join(dexpreopt_tools_files_temp_dir, 'dexpreopt_gen'),
+        '-global',
+        os.path.join(dexpreopt_framework_config_files_temp_dir,
+                     'dexpreopt.config'),
+        '-global_soong',
+        os.path.join(dexpreopt_framework_config_files_temp_dir,
+                     'dexpreopt_soong.config'),
+        '-module',
+        config,
+        '-dexpreopt_script',
+        'dexpreopt_app.sh',
+        '-out_dir',
+        'out',
+        '-base_path',
+        '.',
+        '--uses_target_files',
+    ]
+
+    # Run the command from temp_dir so all tool paths are its descendants.
+    logging.info('running %s', command)
+    subprocess.check_call(command, cwd=temp_dir)
+
+    # Call the generated script.
+    command = ['sh', 'dexpreopt_app.sh', apk_path]
+    logging.info('running %s', command)
+    subprocess.check_call(command, cwd=temp_dir)
+
+    # Output files are in:
+    #
+    # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.vdex
+    # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.odex
+    # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.vdex
+    # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.odex
+    #
+    # Copy the files to their destination. The structure of system_other is:
+    #
+    # system_other/
+    #     system-other-odex-marker
+    #     system/
+    #         app/
+    #             <app>/oat/arm64/
+    #                 <app>.odex
+    #                 <app>.vdex
+    #             ...
+    #         priv-app/
+    #             <app>/oat/arm64/
+    #                 <app>.odex
+    #                 <app>.vdex
+    #             ...
+
+    # TODO(b/188179859): Support for other architectures.
+    arch = 'arm64'
+
+    dex_destination = os.path.join(temp_dir, 'output', dex_img, apk_dir, app,
+                                   'oat', arch)
+    os.makedirs(dex_destination)
+    dex2oat_path = os.path.join(temp_dir, 'out', 'dex2oat_result', 'vendor',
+                                apk_dir, app, 'oat', arch)
+    shutil.copy(
+        os.path.join(dex2oat_path, 'package.vdex'),
+        os.path.join(dex_destination, app + '.vdex'))
+    shutil.copy(
+        os.path.join(dex2oat_path, 'package.odex'),
+        os.path.join(dex_destination, app + '.odex'))
+
+    # Append entries to vendor_file_system_config.txt, such as:
+    #
+    # vendor/app/<app>/oat 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+    # vendor/app/<app>/oat/arm64 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+    # vendor/app/<app>/oat/arm64/<app>.odex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+    # vendor/app/<app>/oat/arm64/<app>.vdex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+    if not use_system_other_odex:
+      vendor_app_prefix = 'vendor/' + apk_dir + '/' + app + '/oat'
+      selabel = 'selabel=u:object_r:vendor_app_file:s0 capabilities=0x0'
+      vendor_file_system_config.writelines([
+          vendor_app_prefix + ' 0 2000 755 ' + selabel + '\n',
+          vendor_app_prefix + '/' + arch + ' 0 2000 755 ' + selabel + '\n',
+          vendor_app_prefix + '/' + arch + '/' + app + '.odex 0 0 644 ' +
+          selabel + '\n',
+          vendor_app_prefix + '/' + arch + '/' + app + '.vdex 0 0 644 ' +
+          selabel + '\n',
+      ])
+
+  if not use_system_other_odex:
+    vendor_file_system_config.close()
+    # Delete vendor.img so that it will be regenerated.
+    # TODO(b/188179859): Rebuilding a vendor image in GRF mode (e.g., T(framework)
+    #                    and S(vendor) may require logic similar to that in
+    #                    rebuild_image_with_sepolicy.
+    vendor_img = os.path.join(output_target_files_dir, 'IMAGES', 'vendor.img')
+    if os.path.exists(vendor_img):
+      logging.info('Deleting %s', vendor_img)
+      os.remove(vendor_img)