summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xci/build_test_suites22
-rw-r--r--ci/build_test_suites.py282
-rw-r--r--ci/test_mapping_module_retriever.py125
-rw-r--r--core/main.mk6
-rw-r--r--target/product/base_system.mk2
-rw-r--r--tools/aconfig/printflags/src/main.rs6
-rw-r--r--tools/aconfig/src/commands.rs28
-rw-r--r--tools/aconfig/src/main.rs2
-rw-r--r--tools/aconfig/src/storage/flag_table.rs55
-rw-r--r--tools/aconfig/src/storage/flag_value.rs181
-rw-r--r--tools/aconfig/src/storage/mod.rs38
-rw-r--r--tools/aconfig/src/storage/package_table.rs10
-rw-r--r--tools/metadata/generator.go29
-rw-r--r--tools/metadata/testdata/generatedEmptyOutputFile.txt1
-rwxr-xr-xtools/releasetools/ota_from_target_files.py25
15 files changed, 737 insertions, 75 deletions
diff --git a/ci/build_test_suites b/ci/build_test_suites
new file mode 100755
index 0000000000..861065a73a
--- /dev/null
+++ b/ci/build_test_suites
@@ -0,0 +1,22 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import build_test_suites
+
+if __name__ == '__main__':
+ sys.dont_write_bytecode = True
+
+ build_test_suites.main(sys.argv)
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
new file mode 100644
index 0000000000..e88b420d3f
--- /dev/null
+++ b/ci/build_test_suites.py
@@ -0,0 +1,282 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to build only the necessary modules for general-tests along
+
+with whatever other targets are passed in.
+"""
+
+import argparse
+from collections.abc import Sequence
+import json
+import os
+import pathlib
+import re
+import subprocess
+import sys
+from typing import Any, Dict, Set, Text
+
+import test_mapping_module_retriever
+
+
+# List of modules that are always required to be in general-tests.zip
+REQUIRED_MODULES = frozenset(
+ ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip']
+)
+
+
+def build_test_suites(argv):
+ args = parse_args(argv)
+
+ if not args.change_info:
+ build_everything(args)
+ return
+
+ # Call the class to map changed files to modules to build.
+ # TODO(lucafarsi): Move this into a replaceable class.
+ build_affected_modules(args)
+
+
+def parse_args(argv):
+ argparser = argparse.ArgumentParser()
+ argparser.add_argument(
+ 'extra_targets', nargs='*', help='Extra test suites to build.'
+ )
+ argparser.add_argument('--target_product')
+ argparser.add_argument('--target_release')
+ argparser.add_argument(
+ '--with_dexpreopt_boot_img_and_system_server_only', action='store_true'
+ )
+ argparser.add_argument('--dist_dir')
+ argparser.add_argument('--change_info', nargs='?')
+ argparser.add_argument('--extra_required_modules', nargs='*')
+
+ return argparser.parse_args()
+
+
+def build_everything(args: argparse.Namespace):
+ build_command = base_build_command(args)
+ build_command.append('general-tests')
+
+ run_command(build_command)
+
+
+def build_affected_modules(args: argparse.Namespace):
+ modules_to_build = find_modules_to_build(
+ pathlib.Path(args.change_info), args.extra_required_modules
+ )
+
+ # Call the build command with everything.
+ build_command = base_build_command(args)
+ build_command.extend(modules_to_build)
+
+ run_command(build_command)
+
+ zip_build_outputs(modules_to_build, args.dist_dir)
+
+
+def base_build_command(args: argparse.Namespace) -> list:
+ build_command = []
+ build_command.append('time')
+ build_command.append('./build/soong/soong_ui.bash')
+ build_command.append('--make-mode')
+ build_command.append('dist')
+ build_command.append('DIST_DIR=' + args.dist_dir)
+ build_command.append('TARGET_PRODUCT=' + args.target_product)
+ build_command.append('TARGET_RELEASE=' + args.target_release)
+ build_command.extend(args.extra_targets)
+
+ return build_command
+
+
+def run_command(args: list[str]) -> str:
+ result = subprocess.run(
+ args=args,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ # If the process failed, print its stdout and propagate the exception.
+ if not result.returncode == 0:
+ print('Build command failed! output:')
+ print('stdout: ' + result.stdout)
+ print('stderr: ' + result.stderr)
+
+ result.check_returncode()
+ return result.stdout
+
+
+def find_modules_to_build(
+ change_info: pathlib.Path, extra_required_modules: list[Text]
+) -> Set[Text]:
+ changed_files = find_changed_files(change_info)
+
+ test_mappings = test_mapping_module_retriever.GetTestMappings(
+ changed_files, set()
+ )
+
+ # Soong_zip is required to generate the output zip so always build it.
+ modules_to_build = set(REQUIRED_MODULES)
+ if extra_required_modules:
+ modules_to_build.update(extra_required_modules)
+
+ modules_to_build.update(find_affected_modules(test_mappings, changed_files))
+
+ return modules_to_build
+
+
+def find_changed_files(change_info: pathlib.Path) -> Set[Text]:
+ with open(change_info) as change_info_file:
+ change_info_contents = json.load(change_info_file)
+
+ changed_files = set()
+
+ for change in change_info_contents['changes']:
+ project_path = change.get('projectPath') + '/'
+
+ for revision in change.get('revisions'):
+ for file_info in revision.get('fileInfos'):
+ changed_files.add(project_path + file_info.get('path'))
+
+ return changed_files
+
+
+def find_affected_modules(
+ test_mappings: Dict[str, Any], changed_files: Set[Text]
+) -> Set[Text]:
+ modules = set()
+
+ # The test_mappings object returned by GetTestMappings is organized as
+ # follows:
+ # {
+ # 'test_mapping_file_path': {
+ # 'group_name' : [
+ # 'name': 'module_name',
+ # ],
+ # }
+ # }
+ for test_mapping in test_mappings.values():
+ for group in test_mapping.values():
+ for entry in group:
+ module_name = entry.get('name', None)
+
+ if not module_name:
+ continue
+
+ file_patterns = entry.get('file_patterns')
+ if not file_patterns:
+ modules.add(module_name)
+ continue
+
+ if matches_file_patterns(file_patterns, changed_files):
+ modules.add(module_name)
+ continue
+
+ return modules
+
+
+# TODO(lucafarsi): Share this logic with the original logic in
+# test_mapping_test_retriever.py
+def matches_file_patterns(
+ file_patterns: list[Text], changed_files: Set[Text]
+) -> bool:
+ for changed_file in changed_files:
+ for pattern in file_patterns:
+ if re.search(pattern, changed_file):
+ return True
+
+ return False
+
+
+def zip_build_outputs(modules_to_build: Set[Text], dist_dir: Text):
+ src_top = os.environ.get('TOP', os.getcwd())
+
+ # Call dumpvars to get the necessary things.
+ # TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can
+ # do it but it requires parsing.
+ host_out_testcases = get_soong_var('HOST_OUT_TESTCASES')
+ target_out_testcases = get_soong_var('TARGET_OUT_TESTCASES')
+ product_out = get_soong_var('PRODUCT_OUT')
+ soong_host_out = get_soong_var('SOONG_HOST_OUT')
+ host_out = get_soong_var('HOST_OUT')
+
+ # Call the class to package the outputs.
+ # TODO(lucafarsi): Move this code into a replaceable class.
+ host_paths = []
+ target_paths = []
+ for module in modules_to_build:
+ host_path = os.path.join(host_out_testcases, module)
+ if os.path.exists(host_path):
+ host_paths.append(host_path)
+
+ target_path = os.path.join(target_out_testcases, module)
+ if os.path.exists(target_path):
+ target_paths.append(target_path)
+
+ zip_command = ['time', os.path.join(host_out, 'bin', 'soong_zip')]
+
+ # Add host testcases.
+ zip_command.append('-C')
+ zip_command.append(os.path.join(src_top, soong_host_out))
+ zip_command.append('-P')
+ zip_command.append('host/')
+ for path in host_paths:
+ zip_command.append('-D')
+ zip_command.append(path)
+
+ # Add target testcases.
+ zip_command.append('-C')
+ zip_command.append(os.path.join(src_top, product_out))
+ zip_command.append('-P')
+ zip_command.append('target')
+ for path in target_paths:
+ zip_command.append('-D')
+ zip_command.append(path)
+
+ # TODO(lucafarsi): Push this logic into a general-tests-minimal build command
+ # Add necessary tools. These are also hardcoded in general-tests.mk.
+ framework_path = os.path.join(soong_host_out, 'framework')
+
+ zip_command.append('-C')
+ zip_command.append(framework_path)
+ zip_command.append('-P')
+ zip_command.append('host/tools')
+ zip_command.append('-f')
+ zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar'))
+ zip_command.append('-f')
+ zip_command.append(
+ os.path.join(framework_path, 'compatibility-host-util.jar')
+ )
+ zip_command.append('-f')
+ zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar'))
+
+ # Zip to the DIST dir.
+ zip_command.append('-o')
+ zip_command.append(os.path.join(dist_dir, 'general-tests.zip'))
+
+ run_command(zip_command)
+
+
+def get_soong_var(var: str) -> str:
+ value = run_command(
+ ['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var]
+ ).strip()
+ if not value:
+ raise RuntimeError('Necessary soong variable ' + var + ' not found.')
+
+ return value
+
+
+def main(argv):
+ build_test_suites(sys.argv)
diff --git a/ci/test_mapping_module_retriever.py b/ci/test_mapping_module_retriever.py
new file mode 100644
index 0000000000..d2c13c0e7d
--- /dev/null
+++ b/ci/test_mapping_module_retriever.py
@@ -0,0 +1,125 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Simple parsing code to scan test_mapping files and determine which
+modules are needed to build for the given list of changed files.
+TODO(lucafarsi): Deduplicate from artifact_helper.py
+"""
+
+from typing import Any, Dict, Set, Text
+import json
+import os
+import re
+
+# Regex to extra test name from the path of test config file.
+TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config'
+
+# Key name for TEST_MAPPING imports
+KEY_IMPORTS = 'imports'
+KEY_IMPORT_PATH = 'path'
+
+# Name of TEST_MAPPING file.
+TEST_MAPPING = 'TEST_MAPPING'
+
+# Pattern used to identify double-quoted strings and '//'-format comments in
+# TEST_MAPPING file, but only double-quoted strings are included within the
+# matching group.
+_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?')
+
+
+def FilterComments(test_mapping_file: Text) -> Text:
+ """Remove comments in TEST_MAPPING file to valid format.
+
+ Only '//' is regarded as comments.
+
+ Args:
+ test_mapping_file: Path to a TEST_MAPPING file.
+
+ Returns:
+ Valid json string without comments.
+ """
+ return re.sub(_COMMENTS_RE, r'\1', test_mapping_file)
+
+def GetTestMappings(paths: Set[Text],
+ checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]:
+ """Get the affected TEST_MAPPING files.
+
+ TEST_MAPPING files in source code are packaged into a build artifact
+ `test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file
+ is preserved. From all TEST_MAPPING files in the source code, this method
+ locates the affected TEST_MAPPING files based on the given paths list.
+
+ A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files
+ from a different location, e.g.,
+ "imports": [
+ {
+ "path": "../folder2"
+ }
+ ]
+ In that example, TEST_MAPPING files inside ../folder2 (relative to the
+ TEST_MAPPING file containing that imports section) and its parent directories
+ will also be included.
+
+ Args:
+ paths: A set of paths with related TEST_MAPPING files for given changes.
+ checked_paths: A set of paths that have been checked for TEST_MAPPING file
+ already. The set is updated after processing each TEST_MAPPING file. It's
+ used to prevent infinite loop when the method is called recursively.
+
+ Returns:
+ A dictionary of Test Mapping containing the content of the affected
+ TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file.
+ """
+ test_mappings = {}
+
+ # Search for TEST_MAPPING files in each modified path and its parent
+ # directories.
+ all_paths = set()
+ for path in paths:
+ dir_names = path.split(os.path.sep)
+ all_paths |= set(
+ [os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))])
+ # Add root directory to the paths to search for TEST_MAPPING file.
+ all_paths.add('')
+
+ all_paths.difference_update(checked_paths)
+ checked_paths |= all_paths
+ # Try to load TEST_MAPPING file in each possible path.
+ for path in all_paths:
+ try:
+ test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING')
+ # Read content of TEST_MAPPING file.
+ content = FilterComments(open(test_mapping_file, "r").read())
+ test_mapping = json.loads(content)
+ test_mappings[path] = test_mapping
+
+ import_paths = set()
+ for import_detail in test_mapping.get(KEY_IMPORTS, []):
+ import_path = import_detail[KEY_IMPORT_PATH]
+ # Try the import path as absolute path.
+ import_paths.add(import_path)
+ # Try the import path as relative path based on the test mapping file
+ # containing the import.
+ norm_import_path = os.path.normpath(os.path.join(path, import_path))
+ import_paths.add(norm_import_path)
+ import_paths.difference_update(checked_paths)
+ if import_paths:
+ import_test_mappings = GetTestMappings(import_paths, checked_paths)
+ test_mappings.update(import_test_mappings)
+ except (KeyError, FileNotFoundError, NotADirectoryError):
+ # TEST_MAPPING file doesn't exist in path
+ pass
+
+ return test_mappings
diff --git a/core/main.mk b/core/main.mk
index 348a964e0a..649c75c1a2 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -1721,10 +1721,8 @@ droidcore: droidcore-unbundled
# dist_files only for putting your library into the dist directory with a full build.
.PHONY: dist_files
-ifeq ($(SOONG_COLLECT_JAVA_DEPS), true)
- $(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json)
- $(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json)
-endif
+$(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json)
+$(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json)
.PHONY: apps_only
ifeq ($(HOST_OS),darwin)
diff --git a/target/product/base_system.mk b/target/product/base_system.mk
index 098ed2764f..6a101da9d4 100644
--- a/target/product/base_system.mk
+++ b/target/product/base_system.mk
@@ -70,7 +70,7 @@ PRODUCT_PACKAGES += \
com.android.scheduling \
com.android.sdkext \
com.android.tethering \
- com.android.tzdata \
+ $(RELEASE_PACKAGE_TZDATA_MODULE) \
com.android.uwb \
com.android.virt \
com.android.wifi \
diff --git a/tools/aconfig/printflags/src/main.rs b/tools/aconfig/printflags/src/main.rs
index 4110317238..ae9b83aae6 100644
--- a/tools/aconfig/printflags/src/main.rs
+++ b/tools/aconfig/printflags/src/main.rs
@@ -20,6 +20,7 @@ use aconfig_protos::aconfig::Flag_state as State;
use aconfig_protos::aconfig::Parsed_flags as ProtoParsedFlags;
use anyhow::{bail, Context, Result};
use regex::Regex;
+use std::collections::BTreeMap;
use std::collections::HashMap;
use std::process::Command;
use std::{fs, str};
@@ -66,7 +67,7 @@ fn main() -> Result<()> {
let device_config_flags = parse_device_config(dc_stdout);
// read aconfig_flags.pb files
- let mut flags: HashMap<String, Vec<String>> = HashMap::new();
+ let mut flags: BTreeMap<String, Vec<String>> = BTreeMap::new();
for partition in ["system", "system_ext", "product", "vendor"] {
let path = format!("/{}/etc/aconfig_flags.pb", partition);
let Ok(bytes) = fs::read(&path) else {
@@ -86,11 +87,10 @@ fn main() -> Result<()> {
// print flags
for (key, mut value) in flags {
- let (_, package_and_name) = key.split_once('/').unwrap();
if let Some(dc_value) = device_config_flags.get(&key) {
value.push(dc_value.to_string());
}
- println!("{}: {}", package_and_name, value.join(", "));
+ println!("{}: {}", key, value.join(", "));
}
Ok(())
diff --git a/tools/aconfig/src/commands.rs b/tools/aconfig/src/commands.rs
index 1a8872bc3e..f7a641776c 100644
--- a/tools/aconfig/src/commands.rs
+++ b/tools/aconfig/src/commands.rs
@@ -361,7 +361,7 @@ pub fn modify_parsed_flags_based_on_mode(
Ok(modified_parsed_flags)
}
-pub fn assign_flag_ids<'a, I>(package: &str, parsed_flags_iter: I) -> Result<HashMap<String, u32>>
+pub fn assign_flag_ids<'a, I>(package: &str, parsed_flags_iter: I) -> Result<HashMap<String, u16>>
where
I: Iterator<Item = &'a ProtoParsedFlag> + Clone,
{
@@ -371,7 +371,13 @@ where
if package != pf.package() {
return Err(anyhow::anyhow!("encountered a flag not in current package"));
}
- flag_ids.insert(pf.name().to_string(), id_to_assign);
+
+ // put a cap on how many flags a package can contain to 65535
+ if id_to_assign > u16::MAX as u32 {
+ return Err(anyhow::anyhow!("the number of flags in a package cannot exceed 65535"));
+ }
+
+ flag_ids.insert(pf.name().to_string(), id_to_assign as u16);
}
Ok(flag_ids)
}
@@ -693,15 +699,15 @@ mod tests {
let package = find_unique_package(&parsed_flags.parsed_flag).unwrap().to_string();
let flag_ids = assign_flag_ids(&package, parsed_flags.parsed_flag.iter()).unwrap();
let expected_flag_ids = HashMap::from([
- (String::from("disabled_ro"), 0_u32),
- (String::from("disabled_rw"), 1_u32),
- (String::from("disabled_rw_exported"), 2_u32),
- (String::from("disabled_rw_in_other_namespace"), 3_u32),
- (String::from("enabled_fixed_ro"), 4_u32),
- (String::from("enabled_fixed_ro_exported"), 5_u32),
- (String::from("enabled_ro"), 6_u32),
- (String::from("enabled_ro_exported"), 7_u32),
- (String::from("enabled_rw"), 8_u32),
+ (String::from("disabled_ro"), 0_u16),
+ (String::from("disabled_rw"), 1_u16),
+ (String::from("disabled_rw_exported"), 2_u16),
+ (String::from("disabled_rw_in_other_namespace"), 3_u16),
+ (String::from("enabled_fixed_ro"), 4_u16),
+ (String::from("enabled_fixed_ro_exported"), 5_u16),
+ (String::from("enabled_ro"), 6_u16),
+ (String::from("enabled_ro_exported"), 7_u16),
+ (String::from("enabled_rw"), 8_u16),
]);
assert_eq!(flag_ids, expected_flag_ids);
}
diff --git a/tools/aconfig/src/main.rs b/tools/aconfig/src/main.rs
index 6c4e2416a1..7d719f0c4d 100644
--- a/tools/aconfig/src/main.rs
+++ b/tools/aconfig/src/main.rs
@@ -135,7 +135,7 @@ fn cli() -> Command {
.required(true)
.help("The target container for the generated storage file."),
)
- .arg(Arg::new("cache").long("cache").required(true))
+ .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true))
.arg(Arg::new("out").long("out").required(true)),
)
}
diff --git a/tools/aconfig/src/storage/flag_table.rs b/tools/aconfig/src/storage/flag_table.rs
index 46753f0ad6..3545700b6a 100644
--- a/tools/aconfig/src/storage/flag_table.rs
+++ b/tools/aconfig/src/storage/flag_table.rs
@@ -58,18 +58,26 @@ impl FlagTableHeader {
pub struct FlagTableNode {
pub package_id: u32,
pub flag_name: String,
- pub flag_id: u32,
+ pub flag_type: u16,
+ pub flag_id: u16,
pub next_offset: Option<u32>,
pub bucket_index: u32,
}
impl FlagTableNode {
- fn new(package_id: u32, flag_name: &str, flag_id: u32, num_buckets: u32) -> Self {
+ fn new(
+ package_id: u32,
+ flag_name: &str,
+ flag_type: u16,
+ flag_id: u16,
+ num_buckets: u32,
+ ) -> Self {
let full_flag_name = package_id.to_string() + "/" + flag_name;
let bucket_index = storage::get_bucket_index(&full_flag_name, num_buckets);
Self {
package_id,
flag_name: flag_name.to_string(),
+ flag_type,
flag_id,
next_offset: None,
bucket_index,
@@ -82,6 +90,7 @@ impl FlagTableNode {
let name_bytes = self.flag_name.as_bytes();
result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes());
result.extend_from_slice(name_bytes);
+ result.extend_from_slice(&self.flag_type.to_le_bytes());
result.extend_from_slice(&self.flag_id.to_le_bytes());
result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes());
result
@@ -97,8 +106,6 @@ pub struct FlagTable {
impl FlagTable {
fn create_nodes(package: &FlagPackage, num_buckets: u32) -> Result<Vec<FlagTableNode>> {
- let flag_names = package.boolean_flags.iter().map(|pf| pf.name()).collect::<Vec<_>>();
- println!("{:?}", flag_names);
let flag_ids =
assign_flag_ids(package.package_name, package.boolean_flags.iter().copied())?;
package
@@ -108,7 +115,11 @@ impl FlagTable {
let fid = flag_ids
.get(pf.name())
.ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?;
- Ok(FlagTableNode::new(package.package_id, pf.name(), *fid, num_buckets))
+ // all flags are boolean value at the moment, thus using the last bit. When more
+ // flag value types are supported, flag value type information should come from the
+ // parsed flag, and we will set the flag_type bit mask properly.
+ let flag_type = 1;
+ Ok(FlagTableNode::new(package.package_id, pf.name(), flag_type, *fid, num_buckets))
})
.collect::<Result<Vec<_>>>()
}
@@ -177,7 +188,7 @@ mod tests {
use super::*;
use crate::storage::{
group_flags_by_package, tests::parse_all_test_flags, tests::read_str_from_bytes,
- tests::read_u32_from_bytes,
+ tests::read_u16_from_bytes, tests::read_u32_from_bytes,
};
impl FlagTableHeader {
@@ -202,7 +213,8 @@ mod tests {
let mut node = Self {
package_id: read_u32_from_bytes(bytes, &mut head)?,
flag_name: read_str_from_bytes(bytes, &mut head)?,
- flag_id: read_u32_from_bytes(bytes, &mut head)?,
+ flag_type: read_u16_from_bytes(bytes, &mut head)?,
+ flag_id: read_u16_from_bytes(bytes, &mut head)?,
next_offset: match read_u32_from_bytes(bytes, &mut head)? {
0 => None,
val => Some(val),
@@ -218,13 +230,15 @@ mod tests {
fn new_expected(
package_id: u32,
flag_name: &str,
- flag_id: u32,
+ flag_type: u16,
+ flag_id: u16,
next_offset: Option<u32>,
bucket_index: u32,
) -> Self {
Self {
package_id,
flag_name: flag_name.to_string(),
+ flag_type,
flag_id,
next_offset,
bucket_index,
@@ -281,8 +295,6 @@ mod tests {
};
assert_eq!(header, &expected_header);
- println!("{:?}", &flag_table.as_ref().unwrap().nodes);
-
let buckets: &Vec<Option<u32>> = &flag_table.as_ref().unwrap().buckets;
let expected_bucket: Vec<Option<u32>> = vec![
Some(98),
@@ -308,22 +320,23 @@ mod tests {
let nodes: &Vec<FlagTableNode> = &flag_table.as_ref().unwrap().nodes;
assert_eq!(nodes.len(), 8);
- assert_eq!(nodes[0], FlagTableNode::new_expected(0, "enabled_ro", 1, None, 0));
- assert_eq!(nodes[1], FlagTableNode::new_expected(0, "enabled_rw", 2, Some(150), 1));
- assert_eq!(nodes[2], FlagTableNode::new_expected(1, "disabled_ro", 0, None, 1));
- assert_eq!(nodes[3], FlagTableNode::new_expected(2, "enabled_ro", 1, None, 5));
- assert_eq!(nodes[4], FlagTableNode::new_expected(1, "enabled_fixed_ro", 1, Some(235), 7));
- assert_eq!(nodes[5], FlagTableNode::new_expected(1, "enabled_ro", 2, None, 7));
- assert_eq!(nodes[6], FlagTableNode::new_expected(2, "enabled_fixed_ro", 0, None, 9));
- assert_eq!(nodes[7], FlagTableNode::new_expected(0, "disabled_rw", 0, None, 15));
+ assert_eq!(nodes[0], FlagTableNode::new_expected(0, "enabled_ro", 1, 1, None, 0));
+ assert_eq!(nodes[1], FlagTableNode::new_expected(0, "enabled_rw", 1, 2, Some(150), 1));
+ assert_eq!(nodes[2], FlagTableNode::new_expected(1, "disabled_ro", 1, 0, None, 1));
+ assert_eq!(nodes[3], FlagTableNode::new_expected(2, "enabled_ro", 1, 1, None, 5));
+ assert_eq!(
+ nodes[4],
+ FlagTableNode::new_expected(1, "enabled_fixed_ro", 1, 1, Some(235), 7)
+ );
+ assert_eq!(nodes[5], FlagTableNode::new_expected(1, "enabled_ro", 1, 2, None, 7));
+ assert_eq!(nodes[6], FlagTableNode::new_expected(2, "enabled_fixed_ro", 1, 0, None, 9));
+ assert_eq!(nodes[7], FlagTableNode::new_expected(0, "disabled_rw", 1, 0, None, 15));
}
#[test]
// this test point locks down the table serialization
fn test_serialization() {
- let flag_table = create_test_flag_table();
- assert!(flag_table.is_ok());
- let flag_table = flag_table.unwrap();
+ let flag_table = create_test_flag_table().unwrap();
let header: &FlagTableHeader = &flag_table.header;
let reinterpreted_header = FlagTableHeader::from_bytes(&header.as_bytes());
diff --git a/tools/aconfig/src/storage/flag_value.rs b/tools/aconfig/src/storage/flag_value.rs
new file mode 100644
index 0000000000..45f5ec0c08
--- /dev/null
+++ b/tools/aconfig/src/storage/flag_value.rs
@@ -0,0 +1,181 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use crate::commands::assign_flag_ids;
+use crate::protos::ProtoFlagState;
+use crate::storage::{self, FlagPackage};
+use anyhow::{anyhow, Result};
+
+#[derive(PartialEq, Debug)]
+pub struct FlagValueHeader {
+ pub version: u32,
+ pub container: String,
+ pub file_size: u32,
+ pub num_flags: u32,
+ pub boolean_value_offset: u32,
+}
+
+impl FlagValueHeader {
+ fn new(container: &str, num_flags: u32) -> Self {
+ Self {
+ version: storage::FILE_VERSION,
+ container: String::from(container),
+ file_size: 0,
+ num_flags,
+ boolean_value_offset: 0,
+ }
+ }
+
+ fn as_bytes(&self) -> Vec<u8> {
+ let mut result = Vec::new();
+ result.extend_from_slice(&self.version.to_le_bytes());
+ let container_bytes = self.container.as_bytes();
+ result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes());
+ result.extend_from_slice(container_bytes);
+ result.extend_from_slice(&self.file_size.to_le_bytes());
+ result.extend_from_slice(&self.num_flags.to_le_bytes());
+ result.extend_from_slice(&self.boolean_value_offset.to_le_bytes());
+ result
+ }
+}
+
+#[derive(PartialEq, Debug)]
+pub struct FlagValueList {
+ pub header: FlagValueHeader,
+ pub booleans: Vec<bool>,
+}
+
+impl FlagValueList {
+ pub fn new(container: &str, packages: &[FlagPackage]) -> Result<Self> {
+ // create list
+ let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum();
+
+ let mut list = Self {
+ header: FlagValueHeader::new(container, num_flags),
+ booleans: vec![false; num_flags as usize],
+ };
+
+ for pkg in packages.iter() {
+ let start_offset = pkg.boolean_offset as usize;
+ let flag_ids = assign_flag_ids(pkg.package_name, pkg.boolean_flags.iter().copied())?;
+ for pf in pkg.boolean_flags.iter() {
+ let fid = flag_ids
+ .get(pf.name())
+ .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?;
+
+ list.booleans[start_offset + (*fid as usize)] =
+ pf.state() == ProtoFlagState::ENABLED;
+ }
+ }
+
+ // initialize all header fields
+ list.header.boolean_value_offset = list.header.as_bytes().len() as u32;
+ list.header.file_size = list.header.boolean_value_offset + num_flags;
+
+ Ok(list)
+ }
+
+ pub fn as_bytes(&self) -> Vec<u8> {
+ [
+ self.header.as_bytes(),
+ self.booleans
+ .iter()
+ .map(|&v| u8::from(v).to_le_bytes())
+ .collect::<Vec<_>>()
+ .concat(),
+ ]
+ .concat()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::storage::{
+ group_flags_by_package, tests::parse_all_test_flags, tests::read_str_from_bytes,
+ tests::read_u32_from_bytes, tests::read_u8_from_bytes,
+ };
+
+ impl FlagValueHeader {
+ // test only method to deserialize back into the header struct
+ fn from_bytes(bytes: &[u8]) -> Result<Self> {
+ let mut head = 0;
+ Ok(Self {
+ version: read_u32_from_bytes(bytes, &mut head)?,
+ container: read_str_from_bytes(bytes, &mut head)?,
+ file_size: read_u32_from_bytes(bytes, &mut head)?,
+ num_flags: read_u32_from_bytes(bytes, &mut head)?,
+ boolean_value_offset: read_u32_from_bytes(bytes, &mut head)?,
+ })
+ }
+ }
+
+ impl FlagValueList {
+ // test only method to deserialize back into the flag value struct
+ fn from_bytes(bytes: &[u8]) -> Result<Self> {
+ let header = FlagValueHeader::from_bytes(bytes)?;
+ let num_flags = header.num_flags;
+ let mut head = header.as_bytes().len();
+ let booleans = (0..num_flags)
+ .map(|_| read_u8_from_bytes(bytes, &mut head).unwrap() == 1)
+ .collect();
+ let list = Self { header, booleans };
+ Ok(list)
+ }
+ }
+
+ pub fn create_test_flag_value_list() -> Result<FlagValueList> {
+ let caches = parse_all_test_flags();
+ let packages = group_flags_by_package(caches.iter());
+ FlagValueList::new("system", &packages)
+ }
+
+ #[test]
+ // this test point locks down the flag value creation and each field
+ fn test_list_contents() {
+ let flag_value_list = create_test_flag_value_list();
+ assert!(flag_value_list.is_ok());
+
+ let header: &FlagValueHeader = &flag_value_list.as_ref().unwrap().header;
+ let expected_header = FlagValueHeader {
+ version: storage::FILE_VERSION,
+ container: String::from("system"),
+ file_size: 34,
+ num_flags: 8,
+ boolean_value_offset: 26,
+ };
+ assert_eq!(header, &expected_header);
+
+ let booleans: &Vec<bool> = &flag_value_list.as_ref().unwrap().booleans;
+ let expected_booleans: Vec<bool> = vec![false; header.num_flags as usize];
+ assert_eq!(booleans, &expected_booleans);
+ }
+
+ #[test]
+ // this test point locks down the value list serialization
+ fn test_serialization() {
+ let flag_value_list = create_test_flag_value_list().unwrap();
+
+ let header: &FlagValueHeader = &flag_value_list.header;
+ let reinterpreted_header = FlagValueHeader::from_bytes(&header.as_bytes());
+ assert!(reinterpreted_header.is_ok());
+ assert_eq!(header, &reinterpreted_header.unwrap());
+
+ let reinterpreted_value_list = FlagValueList::from_bytes(&flag_value_list.as_bytes());
+ assert!(reinterpreted_value_list.is_ok());
+ assert_eq!(&flag_value_list, &reinterpreted_value_list.unwrap());
+ }
+}
diff --git a/tools/aconfig/src/storage/mod.rs b/tools/aconfig/src/storage/mod.rs
index 76835e015d..36ea3094d3 100644
--- a/tools/aconfig/src/storage/mod.rs
+++ b/tools/aconfig/src/storage/mod.rs
@@ -15,6 +15,7 @@
*/
pub mod flag_table;
+pub mod flag_value;
pub mod package_table;
use anyhow::{anyhow, Result};
@@ -24,7 +25,9 @@ use std::path::PathBuf;
use crate::commands::OutputFile;
use crate::protos::{ProtoParsedFlag, ProtoParsedFlags};
-use crate::storage::{flag_table::FlagTable, package_table::PackageTable};
+use crate::storage::{
+ flag_table::FlagTable, flag_value::FlagValueList, package_table::PackageTable,
+};
pub const FILE_VERSION: u32 = 1;
@@ -56,6 +59,8 @@ pub struct FlagPackage<'a> {
pub package_id: u32,
pub flag_names: HashSet<&'a str>,
pub boolean_flags: Vec<&'a ProtoParsedFlag>,
+ // offset of the first boolean flag in this flag package with respect to the start of
+ // boolean flag value array in the flag value file
pub boolean_offset: u32,
}
@@ -95,12 +100,11 @@ where
}
// calculate package flag value start offset, in flag value file, each boolean
- // is stored as two bytes, the first byte will be the flag value. the second
- // byte is flag info byte, which is a bitmask to indicate the status of a flag
+ // is stored as a single byte
let mut boolean_offset = 0;
for p in packages.iter_mut() {
p.boolean_offset = boolean_offset;
- boolean_offset += 2 * p.boolean_flags.len() as u32;
+ boolean_offset += p.boolean_flags.len() as u32;
}
packages
@@ -127,7 +131,13 @@ where
let flag_table_file =
OutputFile { contents: flag_table.as_bytes(), path: flag_table_file_path };
- Ok(vec![package_table_file, flag_table_file])
+ // create and serialize flag value
+ let flag_value = FlagValueList::new(container, &packages)?;
+ let flag_value_file_path = PathBuf::from("flag.val");
+ let flag_value_file =
+ OutputFile { contents: flag_value.as_bytes(), path: flag_value_file_path };
+
+ Ok(vec![package_table_file, flag_table_file, flag_value_file])
}
#[cfg(test)]
@@ -135,6 +145,20 @@ mod tests {
use super::*;
use crate::Input;
+ /// Read and parse bytes as u8
+ pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result<u8> {
+ let val = u8::from_le_bytes(buf[*head..*head + 1].try_into()?);
+ *head += 1;
+ Ok(val)
+ }
+
+ /// Read and parse bytes as u16
+ pub fn read_u16_from_bytes(buf: &[u8], head: &mut usize) -> Result<u16> {
+ let val = u16::from_le_bytes(buf[*head..*head + 2].try_into()?);
+ *head += 2;
+ Ok(val)
+ }
+
/// Read and parse bytes as u32
pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result<u32> {
let val = u32::from_le_bytes(buf[*head..*head + 4].try_into()?);
@@ -218,13 +242,13 @@ mod tests {
assert!(packages[1].flag_names.contains("enabled_ro"));
assert!(packages[1].flag_names.contains("disabled_ro"));
assert!(packages[1].flag_names.contains("enabled_fixed_ro"));
- assert_eq!(packages[1].boolean_offset, 6);
+ assert_eq!(packages[1].boolean_offset, 3);
assert_eq!(packages[2].package_name, "com.android.aconfig.storage.test_4");
assert_eq!(packages[2].package_id, 2);
assert_eq!(packages[2].flag_names.len(), 2);
assert!(packages[2].flag_names.contains("enabled_ro"));
assert!(packages[2].flag_names.contains("enabled_fixed_ro"));
- assert_eq!(packages[2].boolean_offset, 12);
+ assert_eq!(packages[2].boolean_offset, 6);
}
}
diff --git a/tools/aconfig/src/storage/package_table.rs b/tools/aconfig/src/storage/package_table.rs
index 1a3bbc3eb0..40362340e0 100644
--- a/tools/aconfig/src/storage/package_table.rs
+++ b/tools/aconfig/src/storage/package_table.rs
@@ -57,6 +57,8 @@ impl PackageTableHeader {
pub struct PackageTableNode {
pub package_name: String,
pub package_id: u32,
+ // offset of the first boolean flag in this flag package with respect to the start of
+ // boolean flag value array in the flag value file
pub boolean_offset: u32,
pub next_offset: Option<u32>,
pub bucket_index: u32,
@@ -249,7 +251,7 @@ mod tests {
let first_node_expected = PackageTableNode {
package_name: String::from("com.android.aconfig.storage.test_2"),
package_id: 1,
- boolean_offset: 6,
+ boolean_offset: 3,
next_offset: None,
bucket_index: 0,
};
@@ -265,7 +267,7 @@ mod tests {
let third_node_expected = PackageTableNode {
package_name: String::from("com.android.aconfig.storage.test_4"),
package_id: 2,
- boolean_offset: 12,
+ boolean_offset: 6,
next_offset: None,
bucket_index: 3,
};
@@ -275,9 +277,7 @@ mod tests {
#[test]
// this test point locks down the table serialization
fn test_serialization() {
- let package_table = create_test_package_table();
- assert!(package_table.is_ok());
- let package_table = package_table.unwrap();
+ let package_table = create_test_package_table().unwrap();
let header: &PackageTableHeader = &package_table.header;
let reinterpreted_header = PackageTableHeader::from_bytes(&header.as_bytes());
diff --git a/tools/metadata/generator.go b/tools/metadata/generator.go
index d328876ed2..b7668be44f 100644
--- a/tools/metadata/generator.go
+++ b/tools/metadata/generator.go
@@ -77,9 +77,18 @@ func readFileToString(filePath string) string {
return string(data)
}
-func writeNewlineToOutputFile(outputFile string) {
+func writeEmptyOutputProto(outputFile string, metadataRule string) {
file, err := os.Create(outputFile)
- data := "\n"
+ if err != nil {
+ log.Fatal(err)
+ }
+ var message proto.Message
+ if metadataRule == "test_spec" {
+ message = &test_spec_proto.TestSpec{}
+ } else if metadataRule == "code_metadata" {
+ message = &code_metadata_proto.CodeMetadata{}
+ }
+ data, err := proto.Marshal(message)
if err != nil {
log.Fatal(err)
}
@@ -92,8 +101,8 @@ func writeNewlineToOutputFile(outputFile string) {
}
func processTestSpecProtobuf(
- filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap,
- errCh chan error, wg *sync.WaitGroup,
+ filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap,
+ errCh chan error, wg *sync.WaitGroup,
) {
defer wg.Done()
@@ -121,7 +130,7 @@ func processTestSpecProtobuf(
if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() {
errCh <- fmt.Errorf(
"Conflicting trendy team IDs found for %s at:\n%s with teamId"+
- ": %s,\n%s with teamId: %s",
+ ": %s,\n%s with teamId: %s",
key,
metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(),
existing.GetTrendyTeamId(),
@@ -147,8 +156,8 @@ func processTestSpecProtobuf(
// processCodeMetadataProtobuf processes CodeMetadata protobuf files
func processCodeMetadataProtobuf(
- filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap,
- errCh chan error, wg *sync.WaitGroup,
+ filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap,
+ errCh chan error, wg *sync.WaitGroup,
) {
defer wg.Done()
@@ -182,8 +191,8 @@ func processCodeMetadataProtobuf(
if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership || !existing.MultiOwnership) {
errCh <- fmt.Errorf(
"Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+
- " If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+
- "Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.",
+ " If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+
+ "Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.",
srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path,
)
srcFileLock.Unlock()
@@ -235,7 +244,7 @@ func main() {
inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n")
filePaths := strings.Split(inputFileData, " ")
if len(filePaths) == 1 && filePaths[0] == "" {
- writeNewlineToOutputFile(*outputFile)
+ writeEmptyOutputProto(*outputFile, *rule)
return
}
ownershipMetadataMap := &sync.Map{}
diff --git a/tools/metadata/testdata/generatedEmptyOutputFile.txt b/tools/metadata/testdata/generatedEmptyOutputFile.txt
index 8b13789179..e69de29bb2 100644
--- a/tools/metadata/testdata/generatedEmptyOutputFile.txt
+++ b/tools/metadata/testdata/generatedEmptyOutputFile.txt
@@ -1 +0,0 @@
-
diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py
index b65764b589..29042a5cba 100755
--- a/tools/releasetools/ota_from_target_files.py
+++ b/tools/releasetools/ota_from_target_files.py
@@ -259,6 +259,9 @@ A/B OTA specific options
--vabc_cow_version
Specify the VABC cow version to be used
+
+ --compression_factor
+ Specify the maximum block size to be compressed at once during OTA. supported options: 4k, 8k, 16k, 32k, 64k, 128k
"""
from __future__ import print_function
@@ -331,6 +334,7 @@ OPTIONS.vabc_compression_param = None
OPTIONS.security_patch_level = None
OPTIONS.max_threads = None
OPTIONS.vabc_cow_version = None
+OPTIONS.compression_factor = None
POSTINSTALL_CONFIG = 'META/postinstall_config.txt'
@@ -393,17 +397,6 @@ def ModifyVABCCompressionParam(content, algo):
"""
return ModifyKeyvalueList(content, "virtual_ab_compression_method", algo)
-def SetVABCCowVersion(content, cow_version):
- """ Update virtual_ab_cow_version in dynamic_partitions_info.txt
- Args:
- content: The string content of dynamic_partitions_info.txt
- algo: The cow version be used for VABC. See
- https://cs.android.com/android/platform/superproject/main/+/main:system/core/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h;l=36
- Returns:
- Updated content of dynamic_partitions_info.txt , updated cow version
- """
- return ModifyKeyvalueList(content, "virtual_ab_cow_version", cow_version)
-
def UpdatesInfoForSpecialUpdates(content, partitions_filter,
delete_keys=None):
@@ -1020,6 +1013,8 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None):
target_file, vabc_compression_param)
if OPTIONS.vabc_cow_version:
target_file = ModifyTargetFilesDynamicPartitionInfo(target_file, "virtual_ab_cow_version", OPTIONS.vabc_cow_version)
+ if OPTIONS.compression_factor:
+ target_file = ModifyTargetFilesDynamicPartitionInfo(target_file, "virtual_ab_compression_factor", OPTIONS.compression_factor)
if OPTIONS.skip_postinstall:
target_file = GetTargetFilesZipWithoutPostinstallConfig(target_file)
# Target_file may have been modified, reparse ab_partitions
@@ -1280,6 +1275,13 @@ def main(argv):
else:
raise ValueError("Cannot parse value %r for option %r - only "
"integers are allowed." % (a, o))
+ elif o in ("--compression_factor"):
+ values = ["4k", "8k", "16k", "32k", "64k", "128k"]
+ if a[:-1].isdigit() and a in values and a.endswith("k"):
+ OPTIONS.compression_factor = str(int(a[:-1]) * 1024)
+ else:
+ raise ValueError("Please specify value from following options: 4k, 8k, 16k, 32k, 64k, 128k")
+
elif o == "--vabc_cow_version":
if a.isdigit():
OPTIONS.vabc_cow_version = a
@@ -1335,6 +1337,7 @@ def main(argv):
"security_patch_level=",
"max_threads=",
"vabc_cow_version=",
+ "compression_factor=",
], extra_option_handler=[option_handler, payload_signer.signer_options])
common.InitLogging()
generated by cgit v1.2.3-59-g8ed1b (git 2.39.0) at 2025-11-03 01:22:39 +0000