Add merging of apexkeys and apkcerts metadata

Bug: 131437873
Test: Built system-only and vendor builds for merge. Compared
resulting apkcertx and apexkeys text files to that of a monolithic
build,
Test: Created colliding entries in both apexkeys and apkcerts text
files and ensure the script exited with an appropriate error message.
Test: Created unit tests to cover both non-colliding and colliding
entries
Change-Id: I6e42ce682ffa9059344e8cd63ba3a720c1f93452

7 files changed, 149 insertions, 13 deletions

diff --git a/core/Makefile b/core/Makefile
index 57a05d4b1a..5ba3091063 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -695,7 +695,7 @@ $(APKCERTS_FILE):
 	@echo APK certs list: $@
 	@mkdir -p $(dir $@)
 	@rm -f $@
-	$(foreach p,$(PACKAGES),\
+	$(foreach p,$(sort $(PACKAGES)),\
 	  $(if $(PACKAGES.$(p).EXTERNAL_KEY),\
 	    $(call _apkcerts_write_line,$(p),"EXTERNAL","",$(PACKAGES.$(p).COMPRESSED),$@),\
 	    $(call _apkcerts_write_line,$(p),$(PACKAGES.$(p).CERTIFICATE),$(PACKAGES.$(p).PRIVATE_KEY),$(PACKAGES.$(p).COMPRESSED),$@)))
diff --git a/tools/releasetools/merge_target_files.py b/tools/releasetools/merge_target_files.py
index f03cc1e96d..2a722c5246 100755
--- a/tools/releasetools/merge_target_files.py
+++ b/tools/releasetools/merge_target_files.py
@@ -185,6 +185,26 @@ other_extract_special_item_list = [
 ]
 
 
+def write_sorted_data(data, path):
+  """Write the sorted contents of either a list or dict to file.
+
+  This function sorts the contents of the list or dict and then
+  writes the resulting sorted contents to a file specified by path.
+
+  Args:
+    data: The list or dict to sort and write.
+    path: Path to the file to write the sorted values to. The file at path will
+      be overridden if it exists.
+  """
+  with open(path, 'w') as output:
+    sorted_data = sorted(data.keys()) if isinstance(data,
+                                                    dict) else sorted(data)
+    for entry in sorted_data:
+      out_str = '{}={}\n'.format(entry, data[entry]) if isinstance(
+          data, dict) else '{}\n'.format(entry)
+      output.write(out_str)
+
+
 def extract_items(target_files, target_files_temp_dir, extract_item_list):
   """Extract items from target files to temporary directory.
 
@@ -341,9 +361,7 @@ def process_ab_partitions_txt(system_target_files_temp_dir,
   output_ab_partitions_txt = os.path.join(output_target_files_temp_dir, 'META',
                                           'ab_partitions.txt')
 
-  with open(output_ab_partitions_txt, 'w') as output:
-    for partition in sorted(output_ab_partitions):
-      output.write('%s\n' % partition)
+  write_sorted_data(data=output_ab_partitions, path=output_ab_partitions_txt)
 
 
 def append_recovery_to_filesystem_config(output_target_files_temp_dir):
@@ -485,10 +503,7 @@ def process_misc_info_txt(system_target_files_temp_dir,
 
   output_misc_info_txt = os.path.join(output_target_files_temp_dir, 'META',
                                       'misc_info.txt')
-  with open(output_misc_info_txt, 'w') as output:
-    sorted_keys = sorted(merged_info_dict.keys())
-    for key in sorted_keys:
-      output.write('{}={}\n'.format(key, merged_info_dict[key]))
+  write_sorted_data(data=merged_info_dict, path=output_misc_info_txt)
 
 
 def process_dynamic_partitions_info_txt(system_target_files_dir,
@@ -538,10 +553,56 @@ def process_dynamic_partitions_info_txt(system_target_files_dir,
 
   output_dynamic_partitions_info_txt = os.path.join(
       output_target_files_dir, 'META', 'dynamic_partitions_info.txt')
-  with open(output_dynamic_partitions_info_txt, 'w') as output:
-    sorted_keys = sorted(merged_dynamic_partitions_dict.keys())
-    for key in sorted_keys:
-      output.write('{}={}\n'.format(key, merged_dynamic_partitions_dict[key]))
+  write_sorted_data(
+      data=merged_dynamic_partitions_dict,
+      path=output_dynamic_partitions_info_txt)
+
+
+def process_apex_keys_apk_certs_common(system_target_files_dir,
+                                       other_target_files_dir,
+                                       output_target_files_dir, file_name):
+  """Perform special processing for META/apexkeys.txt or META/apkcerts.txt.
+
+  This function merges the contents of the META/apexkeys.txt or
+  META/apkcerts.txt
+  files from the system directory and the other directory, placing the merged
+  result in the output directory. The precondition in that the files are already
+  extracted.
+  The post condition is that the output META/apexkeys.txt or META/apkcerts.txt
+  contains the merged content.
+
+  Args:
+    system_target_files_dir: The name of a directory containing the special
+      items extracted from the system target files package.
+    other_target_files_dir: The name of a directory containing the special items
+      extracted from the other target files package.
+    output_target_files_dir: The name of a directory that will be used to create
+      the output target files package after all the special cases are processed.
+    file_name: The name of the file to merge. One of apkcerts.txt or
+      apexkeys.txt.
+  """
+
+  def read_helper(d):
+    temp = {}
+    file_path = os.path.join(d, 'META', file_name)
+    with open(file_path) as f:
+      for line in f:
+        if line.strip():
+          temp[line.split()[0]] = line.strip()
+    return temp
+
+  system_dict = read_helper(system_target_files_dir)
+  other_dict = read_helper(other_target_files_dir)
+
+  for key in system_dict:
+    if key in other_dict and other_dict[key] != system_dict[key]:
+      raise ValueError('Conflicting entries found in %s:\n %s and\n %s' %
+                       (file_name, system_dict[key], other_dict[key]))
+    other_dict[key] = system_dict[key]
+
+  output_file = os.path.join(output_target_files_dir, 'META', file_name)
+
+  write_sorted_data(data=other_dict.values(), path=output_file)
 
 
 def process_special_cases(system_target_files_temp_dir,
@@ -588,6 +649,18 @@ def process_special_cases(system_target_files_temp_dir,
       other_target_files_dir=other_target_files_temp_dir,
       output_target_files_dir=output_target_files_temp_dir)
 
+  process_apex_keys_apk_certs_common(
+      system_target_files_dir=system_target_files_temp_dir,
+      other_target_files_dir=other_target_files_temp_dir,
+      output_target_files_dir=output_target_files_temp_dir,
+      file_name='apkcerts.txt')
+
+  process_apex_keys_apk_certs_common(
+      system_target_files_dir=system_target_files_temp_dir,
+      other_target_files_dir=other_target_files_temp_dir,
+      output_target_files_dir=output_target_files_temp_dir,
+      file_name='apexkeys.txt')
+
 
 def merge_target_files(temp_dir, system_target_files, system_item_list,
                        system_misc_info_keys, other_target_files,
diff --git a/tools/releasetools/test_merge_target_files.py b/tools/releasetools/test_merge_target_files.py
index 3f15d8f61f..1e29fde95e 100644
--- a/tools/releasetools/test_merge_target_files.py
+++ b/tools/releasetools/test_merge_target_files.py
@@ -22,7 +22,8 @@ from merge_target_files import (read_config_list, validate_config_lists,
                                 default_system_item_list,
                                 default_other_item_list,
                                 default_system_misc_info_keys, copy_items,
-                                merge_dynamic_partition_info_dicts)
+                                merge_dynamic_partition_info_dicts,
+                                process_apex_keys_apk_certs_common)
 
 
 class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
@@ -160,3 +161,55 @@ class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
         'super_group_b_size': '2000',
     }
     self.assertEqual(merged_dict, expected_merged_dict)
+
+  def test_process_apex_keys_apk_certs_ReturnsTrueIfNoConflicts(self):
+    output_dir = common.MakeTempDir()
+    os.makedirs(os.path.join(output_dir, 'META'))
+
+    system_dir = common.MakeTempDir()
+    os.makedirs(os.path.join(system_dir, 'META'))
+    os.symlink(
+        os.path.join(self.testdata_dir, 'apexkeys_system.txt'),
+        os.path.join(system_dir, 'META', 'apexkeys.txt'))
+
+    other_dir = common.MakeTempDir()
+    os.makedirs(os.path.join(other_dir, 'META'))
+    os.symlink(
+        os.path.join(self.testdata_dir, 'apexkeys_other.txt'),
+        os.path.join(other_dir, 'META', 'apexkeys.txt'))
+
+    process_apex_keys_apk_certs_common(system_dir, other_dir, output_dir,
+                                       'apexkeys.txt')
+
+    merged_entries = []
+    merged_path = os.path.join(self.testdata_dir, 'apexkeys_merge.txt')
+
+    with open(merged_path) as f:
+      merged_entries = f.read().split('\n')
+
+    output_entries = []
+    output_path = os.path.join(output_dir, 'META', 'apexkeys.txt')
+
+    with open(output_path) as f:
+      output_entries = f.read().split('\n')
+
+    return self.assertEqual(merged_entries, output_entries)
+
+  def test_process_apex_keys_apk_certs_ReturnsFalseIfConflictsPresent(self):
+    output_dir = common.MakeTempDir()
+    os.makedirs(os.path.join(output_dir, 'META'))
+
+    system_dir = common.MakeTempDir()
+    os.makedirs(os.path.join(system_dir, 'META'))
+    os.symlink(
+        os.path.join(self.testdata_dir, 'apexkeys_system.txt'),
+        os.path.join(system_dir, 'META', 'apexkeys.txt'))
+
+    conflict_dir = common.MakeTempDir()
+    os.makedirs(os.path.join(conflict_dir, 'META'))
+    os.symlink(
+        os.path.join(self.testdata_dir, 'apexkeys_system_conflict.txt'),
+        os.path.join(conflict_dir, 'META', 'apexkeys.txt'))
+
+    self.assertRaises(ValueError, process_apex_keys_apk_certs_common,
+                      system_dir, conflict_dir, output_dir, 'apexkeys.txt')
diff --git a/tools/releasetools/testdata/apexkeys_merge.txt b/tools/releasetools/testdata/apexkeys_merge.txt
new file mode 100644
index 0000000000..48e789f43d
--- /dev/null
+++ b/tools/releasetools/testdata/apexkeys_merge.txt
@@ -0,0 +1,4 @@
+name="com.android.conscrypt.apex" public_key="external/conscrypt/apex/com.android.conscrypt.avbpubkey" private_key="external/conscrypt/apex/com.android.conscrypt.pem" container_certificate="external/conscrypt/apex/com.android.conscrypt.x509.pem" container_private_key="external/conscrypt/apex/com.android.conscrypt.pk8"
+name="com.android.runtime.debug.apex" public_key="art/build/apex/com.android.runtime.avbpubkey" private_key="art/build/apex/com.android.runtime.pem" container_certificate="art/build/apex/com.android.runtime.debug.x509.pem" container_private_key="art/build/apex/com.android.runtime.debug.pk8"
+name="com.android.runtime.release.apex" public_key="art/build/apex/com.android.runtime.avbpubkey" private_key="art/build/apex/com.android.runtime.pem" container_certificate="art/build/apex/com.android.runtime.release.x509.pem" container_private_key="art/build/apex/com.android.runtime.release.pk8"
+name="com.android.support.apexer.apex" public_key="system/apex/apexer/etc/com.android.support.apexer.avbpubkey" private_key="system/apex/apexer/etc/com.android.support.apexer.pem" container_certificate="build/target/product/security/testkey.x509.pem" container_private_key="build/target/product/security/testkey.pk8"
diff --git a/tools/releasetools/testdata/apexkeys_other.txt b/tools/releasetools/testdata/apexkeys_other.txt
new file mode 100644
index 0000000000..b751227aca
--- /dev/null
+++ b/tools/releasetools/testdata/apexkeys_other.txt
@@ -0,0 +1,3 @@
+name="com.android.runtime.release.apex" public_key="art/build/apex/com.android.runtime.avbpubkey" private_key="art/build/apex/com.android.runtime.pem" container_certificate="art/build/apex/com.android.runtime.release.x509.pem" container_private_key="art/build/apex/com.android.runtime.release.pk8"
+name="com.android.support.apexer.apex" public_key="system/apex/apexer/etc/com.android.support.apexer.avbpubkey" private_key="system/apex/apexer/etc/com.android.support.apexer.pem" container_certificate="build/target/product/security/testkey.x509.pem" container_private_key="build/target/product/security/testkey.pk8"
+name="com.android.runtime.debug.apex" public_key="art/build/apex/com.android.runtime.avbpubkey" private_key="art/build/apex/com.android.runtime.pem" container_certificate="art/build/apex/com.android.runtime.debug.x509.pem" container_private_key="art/build/apex/com.android.runtime.debug.pk8"
diff --git a/tools/releasetools/testdata/apexkeys_system.txt b/tools/releasetools/testdata/apexkeys_system.txt
new file mode 100644
index 0000000000..2346668708
--- /dev/null
+++ b/tools/releasetools/testdata/apexkeys_system.txt
@@ -0,0 +1,2 @@
+name="com.android.runtime.debug.apex" public_key="art/build/apex/com.android.runtime.avbpubkey" private_key="art/build/apex/com.android.runtime.pem" container_certificate="art/build/apex/com.android.runtime.debug.x509.pem" container_private_key="art/build/apex/com.android.runtime.debug.pk8"
+name="com.android.conscrypt.apex" public_key="external/conscrypt/apex/com.android.conscrypt.avbpubkey" private_key="external/conscrypt/apex/com.android.conscrypt.pem" container_certificate="external/conscrypt/apex/com.android.conscrypt.x509.pem" container_private_key="external/conscrypt/apex/com.android.conscrypt.pk8"
diff --git a/tools/releasetools/testdata/apexkeys_system_conflict.txt b/tools/releasetools/testdata/apexkeys_system_conflict.txt
new file mode 100644
index 0000000000..caa21c2fab
--- /dev/null
+++ b/tools/releasetools/testdata/apexkeys_system_conflict.txt
@@ -0,0 +1 @@
+name="com.android.runtime.debug.apex" public_key="art/build/apex/com.android.runtime.avbpubkey" private_key="art/build/apex/com.android.runtime.pem" container_certificate="art/build/apex/com.android.runtime.release.x509.pem" container_private_key="art/build/apex/com.android.runtime.debug.pk8"