diff options
| -rwxr-xr-x | imgdiag/ati_download_artifacts.py | 195 | ||||
| -rwxr-xr-x | imgdiag/compare_imgdiag_runs.py | 126 | ||||
| -rw-r--r-- | imgdiag/dirty_image_objects.md | 138 |
3 files changed, 450 insertions, 9 deletions
diff --git a/imgdiag/ati_download_artifacts.py b/imgdiag/ati_download_artifacts.py new file mode 100755 index 0000000000..8c36d2701a --- /dev/null +++ b/imgdiag/ati_download_artifacts.py @@ -0,0 +1,195 @@ +#! /usr/bin/env python3 + +""" +Download test artifacts from ATI (Android Test Investigate). + +This script downloads imgdiag artifacts for specified ATI test runs. + +Usage: + # Download artifacts from specific runs: + ./ati_download_artifacts.py --invocation-id I79100010355578895 --invocation-id I84900010357346481 + + # Download latest 10 imgdiag runs: + ./ati_download_artifacts.py --imgdiag-atp 10 + + # Check all command line flags: + ./ati_download_artifacts.py --help +""" + +import tempfile +import argparse +import subprocess +import time +import os +import concurrent.futures +import json + +try: + from tqdm import tqdm +except: + def tqdm(x, *args, **kwargs): + return x + +LIST_ARTIFACTS_QUERY = """ +SELECT + CONCAT('https://android-build.corp.google.com/builds/', invocation_id, '/', name) AS url +FROM android_build.testartifacts.latest +WHERE invocation_id = '{invocation_id}'; +""" + +LIST_IMGDIAG_RUNS_QUERY = """ +SELECT t.invocation_id, + t.test.name, + t.timing.creation_timestamp +FROM android_build.invocations.latest AS t +WHERE t.test.name like '%imgdiag_top_100%' +ORDER BY t.timing.creation_timestamp DESC +LIMIT {}; +""" + +REQUIRED_IMGDIAG_FILES = [ + "combined_imgdiag_data", + "all-dirty-objects", + "dirty-image-objects-art", + "dirty-image-objects-framework", + "dirty-page-counts", +] + +def filter_artifacts(artifacts): + return [a for a in artifacts if any(x in a for x in REQUIRED_IMGDIAG_FILES)] + +def list_last_imgdiag_runs(run_count): + query_file = tempfile.NamedTemporaryFile() + out_file = tempfile.NamedTemporaryFile() + with open(query_file.name, 'w') as f: + f.write(LIST_IMGDIAG_RUNS_QUERY.format(run_count)) + cmd = f'f1-sql --input_file={query_file.name} --output_file={out_file.name} --csv_output --print_queries=false' + res = subprocess.run(cmd, shell=True, check=True, capture_output=True) + with open(out_file.name) as f: + content = f.read() + content = content.split()[1:] + for i in range(len(content)): + content[i] = content[i].replace('"', '').split(',') + return content + +def list_artifacts(invocation_id): + if not invocation_id: + raise ValueError(f'Invalid invocation: {invocation_id}') + + query_file = tempfile.NamedTemporaryFile() + out_file = tempfile.NamedTemporaryFile() + with open(query_file.name, 'w') as f: + f.write(LIST_ARTIFACTS_QUERY.format(invocation_id=invocation_id)) + cmd = f'f1-sql --input_file={query_file.name} --output_file={out_file.name} --csv_output --print_queries=false --quiet' + execute_command(cmd) + with open(out_file.name) as f: + content = f.read() + content = content.split() + content = content[1:] + for i in range(len(content)): + content[i] = content[i].replace('"', '') + return content + +def execute_command(cmd): + for i in range(5): + try: + subprocess.run(cmd, shell=True, check=True) + return + except Exception as e: + print(f'Failed to run: {cmd}\nException: {e}') + time.sleep(2 ** i) + + raise RuntimeError(f'Failed to run: {cmd}') + +def download_artifacts(res_dir, artifacts): + os.makedirs(res_dir, exist_ok=True) + + commands = [] + for url in artifacts: + filename = url.split('/')[-1] + out_path = os.path.join(res_dir, filename) + cmd = f'sso_client {url} --connect_timeout=120 --dns_timeout=120 --request_timeout=600 --location > {out_path}' + commands.append(cmd) + + if not commands: + return + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + res = list(tqdm(executor.map(execute_command, commands), total=len(commands), leave=False)) + + +def download_invocations(args, invocations): + for invoc_id in tqdm(invocations): + artifacts = list_artifacts(invoc_id) + if not args.download_all: + artifacts = filter_artifacts(artifacts) + + res_dir = os.path.join(args.out_dir, invoc_id) + download_artifacts(res_dir, artifacts) + + +def main(): + parser = argparse.ArgumentParser( + description='Download artifacts from ATI', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--out-dir', + default='./', + help='Output dir for downloads', + ) + parser.add_argument( + '--invocation-id', + action='append', + default=None, + help='Download artifacts from the specified invocations', + ) + parser.add_argument( + '--imgdiag-atp', + metavar='N', + dest='atp_run_count', + type=int, + default=None, + help='Download latest N imgdiag runs', + ) + parser.add_argument( + '--download-all', + action=argparse.BooleanOptionalAction, + default=False, + help='Whether to download all artifacts or combined imgdiag data only', + ) + parser.add_argument( + '--overwrite', + action=argparse.BooleanOptionalAction, + default=False, + help='Download artifacts again even if the invocation_id dir already exists', + ) + args = parser.parse_args() + if not args.invocation_id and not args.atp_run_count: + print('Must specify at least one of: --invocation-id or --imgdiag-atp') + return + + invocations = set() + if args.invocation_id: + invocations.update(args.invocation_id) + if args.atp_run_count: + recent_runs = list_last_imgdiag_runs(args.atp_run_count) + invocations.update({invoc_id for invoc_id, name, timestamp in recent_runs}) + + if not args.overwrite: + existing_downloads = set() + for invoc_id in invocations: + res_dir = os.path.join(args.out_dir, invoc_id) + if os.path.isdir(res_dir): + existing_downloads.add(invoc_id) + + if existing_downloads: + print(f'Skipping existing downloads: {existing_downloads}') + invocations = invocations - existing_downloads + + print(f'Downloading: {invocations}') + download_invocations(args, invocations) + + +if __name__ == '__main__': + main() diff --git a/imgdiag/compare_imgdiag_runs.py b/imgdiag/compare_imgdiag_runs.py new file mode 100755 index 0000000000..780ec2dd22 --- /dev/null +++ b/imgdiag/compare_imgdiag_runs.py @@ -0,0 +1,126 @@ +#! /usr/bin/env python3 + +""" +Compare private dirty page counts between imgdiag test runs. + +This script compares private dirty page counts in the boot-image object section +between specified imgdiag runs. + +Usage: + # Compare multiple imgdiag runs: + ./compare_imgdiag_runs.py ./I1234 ./I1235 ./I1236 + + # Check all command line flags: + ./compare_imgdiag_runs.py --help +""" + +import argparse +import glob +import gzip +import json +import os +import pprint +import statistics + +""" +These thresholds are used to verify that process sets from different runs +are similar enough for meaningful comparison. Constants are based on +the imgdiag run data of 2025-01. Update if necessary. +""" + +# There are about 100 apps in imgdiag_top_100 ATP config + more than 100 system processes. +MIN_COMMON_PROC_COUNT = 200 +# Allow for a relatively small (<20%) mismatch in process sets between runs. +MAX_MISMATCH_PROC_COUNT = 40 + +def main(): + pp = pprint.PrettyPrinter(indent=2) + parser = argparse.ArgumentParser( + description='Compare private dirty page counts between imgdiag ATP runs', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + 'invocation_dirs', + nargs='+', + help='Directories with imgdiag output files', + ) + + parser.add_argument( + '--verbose', + action=argparse.BooleanOptionalAction, + default=False, + help='Print out all mismatched processes and more info about dirty page diffs between individual processes', + ) + args = parser.parse_args() + + runs = [] + for invoc_dir in args.invocation_dirs: + res = glob.glob(os.path.join(invoc_dir, '*dirty-page-counts*')) + if len(res) != 1: + raise ValueError(f"Expected to find exactly one *dirty-page-counts* file in {invoc_dir}, but found: {res}") + + try: + if res[0].endswith('.gz'): + with gzip.open(res[0], 'rb') as f: + contents = json.load(f) + else: + with open(res[0], 'r') as f: + contents = json.load(f) + except: + print('Failed to read: ', res[0]) + raise + runs.append(contents) + + basename = lambda p: os.path.basename(os.path.normpath(p)) + invoc_ids = [basename(path) for path in args.invocation_dirs] + print('Comparing: ', invoc_ids) + + items = list() + for r in runs: + items.append({k[:k.rfind('_')]: v for k, v in r.items()}) + + proc_names = list(set(i.keys()) for i in items) + common_proc_names = set.intersection(*proc_names) + mismatch_proc_names = set.union(*proc_names) - common_proc_names + print('Common proc count (used in the comparison): ', len(common_proc_names)) + if len(common_proc_names) < MIN_COMMON_PROC_COUNT: + print('WARNING: common processes count is too low.') + print(f'Should be at least {MIN_COMMON_PROC_COUNT}.') + + print('Mismatching proc count (not present in all runs): ', len(mismatch_proc_names)) + if len(mismatch_proc_names) > MAX_MISMATCH_PROC_COUNT: + print('WARNING: too many mismatching process names.') + print(f'Should be lower than {MAX_MISMATCH_PROC_COUNT}.') + + if args.verbose: + print("Mismatching process names:") + pp.pprint(mismatch_proc_names) + + dirty_page_sums = list() + for r in items: + dirty_page_sums.append(sum(r[k] for k in common_proc_names)) + + print(f'Total dirty pages:\n{dirty_page_sums}\n') + + mean_dirty_pages = [s / len(common_proc_names) for s in dirty_page_sums] + print(f'Mean dirty pages:\n{mean_dirty_pages}\n') + + median_dirty_pages = [statistics.median(r[name] for name in common_proc_names) for r in items] + print(f'Median dirty pages:\n{median_dirty_pages}\n') + + if args.verbose: + print(f'Largest dirty page diffs:') + for i in range(len(invoc_ids)): + for j in range(len(invoc_ids)): + if j <= i: + continue + + page_count_diffs = [(proc_name, items[i][proc_name], items[j][proc_name], items[j][proc_name] - items[i][proc_name]) for proc_name in common_proc_names] + page_count_diffs = sorted(page_count_diffs, key=lambda x: abs(x[3]), reverse=True) + print(f'Between {invoc_ids[i]} and {invoc_ids[j]}: ') + pp.pprint(page_count_diffs[:10]) + + +if __name__ == '__main__': + main() diff --git a/imgdiag/dirty_image_objects.md b/imgdiag/dirty_image_objects.md index 5d28b04ee6..6726c6fce6 100644 --- a/imgdiag/dirty_image_objects.md +++ b/imgdiag/dirty_image_objects.md @@ -1,6 +1,118 @@ -# How to update dirty-image-objects +# How to update dirty-image-objects (from ATP top 100 run) -1. Install ART APEX with imgdiag and reboot, e.g.: +## 1. Collect dirty objects from most used apps using ATP top100 test run (use specific branch/Android release) + +**IMPORTANT**: Make sure that pinned build for `GIT_TRUNK_RELEASE` branch is up-to-date. Update in `BUILD_IDS` here: +<https://source.corp.google.com/piper///depot/google3/configs/wireless/android/testing/atp/prod/mainline-engprod/common.gcl;l=28-44> + + +**NOTE**: New branches can be added in `master-art_imgdiag_tests` definition here: +<https://source.corp.google.com/piper///depot/google3/configs/wireless/android/testing/atp/prod/mainline-engprod/config.gcl;l=1384-1395> + + +Go to <http://go/abtd> and create a test run with the following parameters: + +* Branch name: `git_main` + +* Test type: `ATP` + +* Test name: select one of the supported imgdiag ATP configurations, see the list here: +<https://atp.googleplex.com/test_configs?testName=%25imgdiag_top_100%25> + + +This will generate a profile based on the following: + +* ART module – latest version from `git_main`. + +* Top 100 apps – up-to-date list of app versions. + +* Platform – pinned version from <https://source.corp.google.com/piper///depot/google3/configs/wireless/android/testing/atp/prod/mainline-engprod/common.gcl;l=28-44> + + +## 2. Create two CLs with new dirty-image-objects files (one for ART, one for frameworks) + + +Download new dirty-image-objects files from the run, either manually from the +website or using the script. Example: +``` +ati_download_artifacts.py --invocation-id I70700010348949160 +``` + + +**NOTE**: Use `ati_download_artifacts.py -h` to see all command line flags. + + +There should be two files named like this: +``` +subprocess-dirty-image-objects-art_9607835265532903390.txt.gz +subprocess-dirty-image-objects-framework_10599183892981195850.txt.gz +``` + +dirty-image-objects-art goes into platform/art repo: <https://cs.android.com/android/platform/superproject/main/+/main:art/build/apex/dirty-image-objects> + + +dirty-image-objects-framework goes into platform/frameworks/base repo: <https://cs.android.com/android/platform/superproject/main/+/main:frameworks/base/config/dirty-image-objects> + + +Upload the two CLs and put them in the same topic. + +## 3. Start an ABTD build from the same branch with dirty-image-objects CLs on top + +Create a build for the same branch and target that was used in step 1, apply CLs with new dirty-image-objects. +For `v2/mainline-engprod/apct/mainline/art/GIT_TRUNK_RELEASE/panther/imgdiag_top_100`: + +* Branch: `git_trunk-release` + +* Target: `panther-userdebug` + +## 4. Run ATP test again, this time with custom build-id from step 3 and with CLs from step 2 + +Rerun the test from step 1 with the following additions: +* Add the CLs with new dirty-image-objects (this will force ABTD to rebuild ART module with the change) +* Select "Configure this test in **Advanced mode**". Scroll down to "**Extra Arguments**" and add **build-id**, set it to the build id from step 3. + +## 5. Download ATP results from step 1 and step 4 and compare them + +Use the script to download both ATP run results, example: +``` +ati_download_artifacts.py --invocation-id I79100010355578895 --invocation-id I84900010357346481 +``` + +Compare the results: +``` +~/compare_imgdiag_runs.py I79100010355578895 I84900010357346481 + +# Example comparison output: + +Comparing: ['I79100010355578895', 'I84900010357346481'] +Common proc count (used in the comparison): 233 +Mismatching proc count (not present in all runs): 10 +Total dirty pages: +[17066, 14799] + +Mean dirty pages: +[73.24463519313305, 63.51502145922747] + +Median dirty pages: +[69, 60] +``` + +Note the number of common processes, it should be at least 200 (approx 100 from top100 apps + another 100+ system processes). If it is lower than that, it might indicate missing processes in one or both runs, which would invalidate the comparison. +The lower the number of mismatching processes the better. Typically less than 40 is fine. + + +In there is a significant difference between the mean and median dirty page counts, it may be useful to check page count diffs for specific processess. Use `--verbose` flag with comparison script to show such info. + +The key number to look at for comparison is "mean dirty pages". In this example new profile saves about 10 pages per process (a mean reduction of 40 KiB memory per process). + +## 6. Submit new dirty-image-objects CLs if the result is better/good enough + +Typical measurement noise for mean dirty pages is less than 2. A new profile with dirty page reduction greater than this threshold is considered an improvement. + + +# How to update dirty-image-objects (manually) + +## 1. Install imgdiag ART APEX on a device with AOSP build, e.g.: ``` . ./build/envsetup.sh @@ -10,7 +122,7 @@ adb install out/dist/test_imgdiag_com.android.art.apex adb reboot ``` -2. Collect imgdiag output. +## 2. Collect imgdiag output. ``` # To see all options check: art/imgdiag/run_imgdiag.py -h @@ -18,7 +130,7 @@ adb reboot art/imgdiag/run_imgdiag.py ``` -3. Create new dirty-image-objects. +## 3. Create new dirty-image-objects files. ``` # To see all options check: art/imgdiag/create_dirty_image_objects.py -h @@ -36,8 +148,14 @@ art/imgdiag/create_dirty_image_objects.py \ ./imgdiag_com.google.android.gms.unstable.txt ``` -The resulting file will contain a list of dirty objects with optional -(enabled by default) sort keys in the following format: +This will generate 3 files: +* dirty-image-objects.txt -- contains all dirty objects + info about the jar they are defined in. +* art\_dirty-image-objects.txt -- dirty objects defined in the ART module, these objects are defined in art/build/apex/dirty-image-objects +* framework\_dirty\_image\_objects.txt -- dirty objects defined in frameworks, these objects are defined in frameworks/base/config/dirty-image-objects + + +The resulting art/framework files will contain lists of dirty objects with +optional (enabled by default) sort keys in the following format: ``` <class_descriptor>[.<reference_field_name>:<reference_field_type>]* [<sort_key>] ``` @@ -54,15 +172,17 @@ All dirty objects will be placed in the dirty bin of the boot image and sorted by the sort\_key values. I.e., dirty entries with sort\_key==N will have lower address than entries with sort\_key==N+1. -4. Push new dirty-image-objects to the device. +## 4. Push new frameworks dirty-image-objects to the device. ``` -adb push dirty-image-objects.txt /etc/dirty-image-objects +adb push framework_dirty-image-objects.txt /etc/dirty-image-objects ``` -5. Reinstall ART APEX to update the boot image. +## 5. Install ART module with new dirty-image-objects ``` +cp ./art_dirty-image-objects.txt $ANDROID_BUILD_TOP/art/build/apex/dirty-image-objects +m apps_only dist adb install out/dist/com.android.art.apex adb reboot ``` |