watchlist: Optimize hash code computation for APK.

ARC team require no divergence with main branch. That is why
this CL is upstreamed and disabled by default for Android and
activated in ARC using config flag.

This service has following features:
   * Aggregated report once per day.
   * Report network operations.
Each above has associated uid that is computed per apks for user id. For
agregated report this means involving most of apks. As as result each
apk is read. Particular this means reading most apks from the system
image during the initial boot and once per day next. In ARC this means
access to squashFS method which is heavy operation. This implements
persistent cache for apk hashes.

Notes: This cherry pick also includes ag/14268703 and ag/19700959.

Test: Locally, no problem found.
      tast.AuthPerf server reports 1+ second improvement in case caches
      are pregenerated However effect of this last longer. Usually
      agregated report takes more time than test run.
      Traced access to squashFS for decompression operations during the
      full initial but. After includes pre-generated cache.
         Before: Compressed 329MB -> Decompressed 804MB in 35 sec.
         After:  Compressed 266MB -> Decompressed 676MB in 22 sec.
Bug: 148229706

Change-Id: I30fbfcb43ff617e57773099e59d50adc1dbebe2c
(cherry picked from commit 2f2dac8c0c7801f4384c3e36a1e8b8bc24471fc7)
(cherry picked from commit ffcad8d2b183708519dddca23a1e21142f4aad20)
(cherry picked from commit 9004131f17d8cbdcea0ea1f8711af1067d4fb804)

5 files changed, 417 insertions, 1 deletions

diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml
index 238f242537cb..1f3aca9dd834 100644
--- a/core/res/res/values/config.xml
+++ b/core/res/res/values/config.xml
@@ -6943,4 +6943,7 @@
 
     <!-- Name of the starting activity for DisplayCompat host. specific to automotive.-->
     <string name="config_defaultDisplayCompatHostActivity" translatable="false"></string>
+
+    <!-- Whether to use file hashes cache in watchlist-->
+    <bool name="config_watchlistUseFileHashesCache">false</bool>
 </resources>
diff --git a/core/res/res/values/symbols.xml b/core/res/res/values/symbols.xml
index 699c8ac3a431..22a0e22efff2 100644
--- a/core/res/res/values/symbols.xml
+++ b/core/res/res/values/symbols.xml
@@ -5336,4 +5336,6 @@
   <java-symbol type="string" name="satellite_notification_open_message" />
   <java-symbol type="string" name="satellite_notification_how_it_works" />
   <java-symbol type="drawable" name="ic_satellite_alt_24px" />
+
+  <java-symbol type="bool" name="config_watchlistUseFileHashesCache" />
 </resources>
diff --git a/services/core/java/com/android/server/net/watchlist/FileHashCache.java b/services/core/java/com/android/server/net/watchlist/FileHashCache.java
new file mode 100644
index 000000000000..f829bc6189f9
--- /dev/null
+++ b/services/core/java/com/android/server/net/watchlist/FileHashCache.java
@@ -0,0 +1,243 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.net.watchlist;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.os.Handler;
+import android.os.SystemClock;
+import android.system.ErrnoException;
+import android.system.Os;
+import android.util.Slog;
+
+import com.android.internal.annotations.VisibleForTesting;
+import com.android.internal.util.HexDump;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.StringTokenizer;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @hide
+ * Utility class that keeps file hashes in cache. This cache is persistent across reboots.
+ * If requested hash does not exist in cache, it is calculated from the target file. Cache gets
+ * persisted once it is changed in deferred mode to prevent multiple savings per many small updates.
+ * Deleted files are detected and removed from the cache during the initial load. If file change is
+ * detected, it is hash is calculated during the next request.
+ * The synchronization is done using Handler. All requests for hashes must be done in context of
+ * handler thread.
+ */
+public class FileHashCache {
+    private static final String TAG = FileHashCache.class.getSimpleName();
+    private static final boolean DEBUG = false;
+    // Turns on the check that validates hash in cache matches one, calculated directly on the
+    // target file. Not to be used in production.
+    private static final boolean VERIFY = false;
+
+    // Used for logging wtf only once during load, see logWtfOnce()
+    private static boolean sLoggedWtf = false;
+
+    @VisibleForTesting
+    static String sPersistFileName = "/data/system/file_hash_cache";
+
+    static long sSaveDeferredDelayMillis = TimeUnit.SECONDS.toMillis(5);
+
+    private static class Entry {
+        public final long mLastModified;
+        public final byte[] mSha256Hash;
+
+        Entry(long lastModified, @NonNull byte[] sha256Hash) {
+            mLastModified = lastModified;
+            mSha256Hash = sha256Hash;
+        }
+    }
+
+    private Handler mHandler;
+    private final Map<File, Entry> mEntries = new HashMap<>();
+
+    private final Runnable mLoadTask = () -> {
+        load();
+    };
+    private final Runnable mSaveTask = () -> {
+        save();
+    };
+
+    /**
+     * @hide
+     */
+    public FileHashCache(@NonNull Handler handler) {
+        mHandler = handler;
+        mHandler.post(mLoadTask);
+    }
+
+    /**
+     * Requests sha256 for the provided file from the cache. If cache entry does not exist or
+     * file was modified, then null is returned.
+     * @hide
+    **/
+    @VisibleForTesting
+    @Nullable
+    byte[] getSha256HashFromCache(@NonNull File file) {
+        if (!mHandler.getLooper().isCurrentThread()) {
+            Slog.wtf(TAG, "Request from invalid thread", new Exception());
+            return null;
+        }
+
+        final Entry entry = mEntries.get(file);
+        if (entry == null) {
+            return null;
+        }
+
+        try {
+            if (entry.mLastModified == Os.stat(file.getAbsolutePath()).st_ctime) {
+                if (VERIFY) {
+                    try {
+                        if (!Arrays.equals(entry.mSha256Hash, DigestUtils.getSha256Hash(file))) {
+                            Slog.wtf(TAG, "Failed to verify entry for " + file);
+                        }
+                    } catch (NoSuchAlgorithmException | IOException e) { }
+                }
+
+                return entry.mSha256Hash;
+            }
+        } catch (ErrnoException e) { }
+
+        if (DEBUG) Slog.v(TAG, "Found stale cached entry for " + file);
+        mEntries.remove(file);
+        return null;
+    }
+
+    /**
+     * Requests sha256 for the provided file. If cache entry does not exist or file was modified,
+     * hash is calculated from the requested file. Otherwise hash from cache is returned.
+     * @hide
+    **/
+    @NonNull
+    public byte[] getSha256Hash(@NonNull File file) throws NoSuchAlgorithmException, IOException {
+        byte[] sha256Hash = getSha256HashFromCache(file);
+        if (sha256Hash != null) {
+            return sha256Hash;
+        }
+
+        try {
+            sha256Hash = DigestUtils.getSha256Hash(file);
+            mEntries.put(file, new Entry(Os.stat(file.getAbsolutePath()).st_ctime, sha256Hash));
+            if (DEBUG) Slog.v(TAG, "New cache entry is created for " + file);
+            scheduleSave();
+            return sha256Hash;
+        } catch (ErrnoException e) {
+            throw new IOException(e);
+        }
+    }
+
+    private static void closeQuietly(@Nullable Closeable closeable) {
+        try {
+            if (closeable != null) {
+                closeable.close();
+            }
+        } catch (IOException e) { }
+    }
+
+    /**
+     * Log an error as wtf only the first instance, then log as warning.
+     */
+    private static void logWtfOnce(@NonNull final String s, final Exception e) {
+        if (!sLoggedWtf) {
+            Slog.wtf(TAG, s, e);
+            sLoggedWtf = true;
+        } else {
+            Slog.w(TAG, s, e);
+        }
+    }
+
+    private void load() {
+        mEntries.clear();
+
+        final long startTime = SystemClock.currentTimeMicro();
+        final File file = new File(sPersistFileName);
+        if (!file.exists()) {
+            if (DEBUG) Slog.v(TAG, "Storage file does not exist. Starting from scratch.");
+            return;
+        }
+
+        BufferedReader reader = null;
+        try {
+            reader = new BufferedReader(new FileReader(file));
+            // forEach rethrows IOException as UncheckedIOException
+            reader.lines().forEach((fileEntry)-> {
+                try {
+                    final StringTokenizer tokenizer = new StringTokenizer(fileEntry, ",");
+                    final File testFile = new File(tokenizer.nextToken());
+                    final long lastModified = Long.parseLong(tokenizer.nextToken());
+                    final byte[] sha256 = HexDump.hexStringToByteArray(tokenizer.nextToken());
+                    mEntries.put(testFile, new Entry(lastModified, sha256));
+                    if (DEBUG) Slog.v(TAG, "Loaded entry for " + testFile);
+                } catch (RuntimeException e) {
+                    // hexStringToByteArray can throw raw RuntimeException on invalid input.  Avoid
+                    // potentially reporting one error per line if the data is corrupt.
+                    logWtfOnce("Invalid entry for " + fileEntry, e);
+                    return;
+                }
+            });
+            if (DEBUG) {
+                Slog.i(TAG, "Loaded " + mEntries.size() + " entries in "
+                        + (SystemClock.currentTimeMicro() - startTime) + " mcs.");
+            }
+        } catch (IOException | UncheckedIOException e) {
+            Slog.e(TAG, "Failed to read storage file", e);
+        } finally {
+            closeQuietly(reader);
+        }
+    }
+
+    private void scheduleSave() {
+        mHandler.removeCallbacks(mSaveTask);
+        mHandler.postDelayed(mSaveTask, sSaveDeferredDelayMillis);
+    }
+
+    private void save() {
+        BufferedWriter writer = null;
+        final long startTime = SystemClock.currentTimeMicro();
+        try {
+            writer = new BufferedWriter(new FileWriter(sPersistFileName));
+            for (Map.Entry<File, Entry> entry : mEntries.entrySet()) {
+                writer.write(entry.getKey() + ","
+                             + entry.getValue().mLastModified + ","
+                             + HexDump.toHexString(entry.getValue().mSha256Hash) + "\n");
+            }
+            if (DEBUG) {
+                Slog.i(TAG, "Saved " + mEntries.size() + " entries in "
+                        + (SystemClock.currentTimeMicro() - startTime) + " mcs.");
+            }
+        } catch (IOException e) {
+            Slog.e(TAG, "Failed to save.", e);
+        } finally {
+            closeQuietly(writer);
+        }
+    }
+}
diff --git a/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java b/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java
index 8ce7b57c55e0..c863cbf327b8 100644
--- a/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java
+++ b/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java
@@ -83,6 +83,8 @@ class WatchlistLoggingHandler extends Handler {
     private final ConcurrentHashMap<Integer, byte[]> mCachedUidDigestMap =
             new ConcurrentHashMap<>();
 
+    private final FileHashCache mApkHashCache;
+
     private interface WatchlistEventKeys {
         String HOST = "host";
         String IP_ADDRESSES = "ipAddresses";
@@ -100,6 +102,13 @@ class WatchlistLoggingHandler extends Handler {
         mSettings = WatchlistSettings.getInstance();
         mDropBoxManager = mContext.getSystemService(DropBoxManager.class);
         mPrimaryUserId = getPrimaryUserId();
+        if (context.getResources().getBoolean(
+                com.android.internal.R.bool.config_watchlistUseFileHashesCache)) {
+            mApkHashCache = new FileHashCache(this);
+            Slog.i(TAG, "Using file hashes cache.");
+        } else {
+            mApkHashCache = null;
+        }
     }
 
     @Override
@@ -345,7 +354,9 @@ class WatchlistLoggingHandler extends Handler {
                             Slog.i(TAG, "Skipping incremental path: " + packageName);
                             continue;
                         }
-                        return DigestUtils.getSha256Hash(new File(apkPath));
+                        return mApkHashCache != null
+                                ? mApkHashCache.getSha256Hash(new File(apkPath))
+                                : DigestUtils.getSha256Hash(new File(apkPath));
                     } catch (NameNotFoundException | NoSuchAlgorithmException | IOException e) {
                         Slog.e(TAG, "Cannot get digest from uid: " + key
                                 + ",pkg: " + packageName, e);
diff --git a/services/tests/servicestests/src/com/android/server/net/watchlist/FileHashCacheTests.java b/services/tests/servicestests/src/com/android/server/net/watchlist/FileHashCacheTests.java
new file mode 100644
index 000000000000..5df7a5e6d788
--- /dev/null
+++ b/services/tests/servicestests/src/com/android/server/net/watchlist/FileHashCacheTests.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.net.watchlist;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import android.annotation.NonNull;
+import android.os.FileUtils;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.Message;
+import android.os.SystemClock;
+import android.system.Os;
+
+import androidx.test.InstrumentationRegistry;
+import androidx.test.filters.SmallTest;
+import androidx.test.runner.AndroidJUnit4;
+
+import com.android.internal.util.HexDump;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * atest frameworks-services -c com.android.server.net.watchlist.FileHashCacheTests
+ */
+@RunWith(AndroidJUnit4.class)
+@SmallTest
+public class FileHashCacheTests {
+
+    private static final String APK_A = "A.apk";
+    private static final String APK_B = "B.apk";
+    private static final String APK_A_CONTENT = "AAA";
+    private static final String APK_A_ALT_CONTENT = "AAA_ALT";
+    private static final String APK_B_CONTENT = "BBB";
+
+    private static final String PERSIST_FILE_NAME_FOR_TEST = "file_hash_cache";
+
+    // Sha256 of "AAA"
+    private static final String APK_A_CONTENT_HASH =
+            "CB1AD2119D8FAFB69566510EE712661F9F14B83385006EF92AEC47F523A38358";
+    // Sha256 of "AAA_ALT"
+    private static final String APK_A_ALT_CONTENT_HASH =
+            "2AB726E3C5B316F4C7507BFCCC3861F0473523D572E0C62BA21601C20693AEF0";
+    // Sha256 of "BBB"
+    private static final String APK_B_CONTENT_HASH =
+            "DCDB704109A454784B81229D2B05F368692E758BFA33CB61D04C1B93791B0273";
+
+    @Before
+    public void setUp() throws Exception {
+        final File persistFile = getFile(PERSIST_FILE_NAME_FOR_TEST);
+        persistFile.delete();
+        FileHashCache.sPersistFileName = persistFile.getAbsolutePath();
+        getFile(APK_A).delete();
+        getFile(APK_B).delete();
+        FileHashCache.sSaveDeferredDelayMillis = 0;
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    @Test
+    public void testFileHashCache_generic() throws Exception {
+        final File apkA = getFile(APK_A);
+        final File apkB = getFile(APK_B);
+
+        Looper.prepare();
+        FileHashCache fileHashCache = new FileHashCache(new InlineHandler());
+
+        assertFalse(getFile(PERSIST_FILE_NAME_FOR_TEST).exists());
+
+        // No hash for non-existing files.
+        assertNull("Found existing entry in the cache",
+                fileHashCache.getSha256HashFromCache(apkA));
+        assertNull("Found existing entry in the cache",
+                fileHashCache.getSha256HashFromCache(apkB));
+        try {
+            fileHashCache.getSha256Hash(apkA);
+            fail("Not reached");
+        } catch (IOException e) { }
+        try {
+            fileHashCache.getSha256Hash(apkB);
+            fail("Not reached");
+        } catch (IOException e) { }
+
+        assertFalse(getFile(PERSIST_FILE_NAME_FOR_TEST).exists());
+        FileUtils.stringToFile(apkA, APK_A_CONTENT);
+        FileUtils.stringToFile(apkB, APK_B_CONTENT);
+
+        assertEquals(APK_A_CONTENT_HASH, HexDump.toHexString(fileHashCache.getSha256Hash(apkA)));
+        assertTrue(getFile(PERSIST_FILE_NAME_FOR_TEST).exists());
+        assertEquals(APK_B_CONTENT_HASH, HexDump.toHexString(fileHashCache.getSha256Hash(apkB)));
+        assertEquals(APK_A_CONTENT_HASH,
+                HexDump.toHexString(fileHashCache.getSha256HashFromCache(apkA)));
+        assertEquals(APK_B_CONTENT_HASH,
+                HexDump.toHexString(fileHashCache.getSha256HashFromCache(apkB)));
+
+        // Recreate handler. It should read persistent state.
+        fileHashCache = new FileHashCache(new InlineHandler());
+        assertEquals(APK_A_CONTENT_HASH,
+                HexDump.toHexString(fileHashCache.getSha256HashFromCache(apkA)));
+        assertEquals(APK_B_CONTENT_HASH,
+                HexDump.toHexString(fileHashCache.getSha256HashFromCache(apkB)));
+
+        // Modify one APK. Cache entry should be invalidated. Make sure that FS timestamp resolution
+        // allows us to detect update.
+        final long before = Os.stat(apkA.getAbsolutePath()).st_ctime;
+        do {
+            FileUtils.stringToFile(apkA, APK_A_ALT_CONTENT);
+        } while (android.system.Os.stat(apkA.getAbsolutePath()).st_ctime == before);
+
+        assertNull("Found stale entry in the cache", fileHashCache.getSha256HashFromCache(apkA));
+        assertEquals(APK_A_ALT_CONTENT_HASH,
+                HexDump.toHexString(fileHashCache.getSha256Hash(apkA)));
+    }
+
+    // Helper handler that executes tasks inline in context of current thread if time is good for
+    // this.
+    private static class InlineHandler extends Handler {
+        @Override
+        public boolean sendMessageAtTime(Message msg, long uptimeMillis) {
+            if (SystemClock.uptimeMillis() >= uptimeMillis && getLooper().isCurrentThread()) {
+                dispatchMessage(msg);
+                return true;
+            }
+            return super.sendMessageAtTime(msg, uptimeMillis);
+        }
+    }
+
+    private File getFile(@NonNull String name) {
+        return new File(InstrumentationRegistry.getContext().getFilesDir(), name);
+    }
+}