Pass bind mount storage data decision from java to zygote

Before we used store dirty data in system property to record
if FUSE for a user is mounted, and zygote uses it to determine
if storage mount is needed. It introduces performance issues and
not reliable.

This CL does the following changes:

- System server determines if storage dirs mount are needed, and store
the record inside system server.

- It passes the verdict to zygote so zygote just need to follow the
input.

- When emulated storage is mounted / unmounted, it will record if
FUSE for that user is ready to use, and will be used for determining
if storage dirs mount are needed when a new process starts.

- After emulated storage is mounted, it will create an async thread
to remount all storage dirs for existing app processes. As we have a
record of pids that storage dirs are not mounted yet, we can use it
directly without scanning the whole /proc in vold.

Bug: 149548518
Test: After flag is enabled, AdoptableHostTest still pass.
Change-Id: Ic99d027d42b2b9a1c7fd03070b36c44882c6e7c5

12 files changed, 218 insertions, 108 deletions

diff --git a/core/java/android/app/ActivityManagerInternal.java b/core/java/android/app/ActivityManagerInternal.java
index ec110435d95c..489a0de0518e 100644
--- a/core/java/android/app/ActivityManagerInternal.java
+++ b/core/java/android/app/ActivityManagerInternal.java
@@ -34,6 +34,7 @@ import android.os.TransactionTooLargeException;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Activity manager local system service interface.
@@ -124,6 +125,12 @@ public abstract class ActivityManagerInternal {
     public abstract int getUidProcessState(int uid);
 
     /**
+     * Get a map of pid and package name that process of that pid Android/data and Android/obb
+     * directory is not mounted to lowerfs.
+     */
+    public abstract Map<Integer, String> getProcessesWithPendingBindMounts(int userId);
+
+    /**
      * @return {@code true} if system is ready, {@code false} otherwise.
      */
     public abstract boolean isSystemReady();
diff --git a/core/java/android/os/Process.java b/core/java/android/os/Process.java
index d7af1b9faa97..a557bd994c13 100644
--- a/core/java/android/os/Process.java
+++ b/core/java/android/os/Process.java
@@ -607,6 +607,7 @@ public class Process {
      *                             started.
      * @param pkgDataInfoMap Map from related package names to private data directory
      *                       volume UUID and inode number.
+     * @param bindMountAppStorageDirs whether zygote needs to mount Android/obb and Android/data.
      * @param zygoteArgs Additional arguments to supply to the zygote process.
      * @return An object that describes the result of the attempt to start the process.
      * @throws RuntimeException on fatal start failure
@@ -630,12 +631,13 @@ public class Process {
                                            @Nullable long[] disabledCompatChanges,
                                            @Nullable Map<String, Pair<String, Long>>
                                                    pkgDataInfoMap,
+                                           boolean bindMountAppStorageDirs,
                                            @Nullable String[] zygoteArgs) {
         return ZYGOTE_PROCESS.start(processClass, niceName, uid, gid, gids,
                     runtimeFlags, mountExternal, targetSdkVersion, seInfo,
                     abi, instructionSet, appDataDir, invokeWith, packageName,
                     zygotePolicyFlags, isTopApp, disabledCompatChanges,
-                    pkgDataInfoMap, zygoteArgs);
+                    pkgDataInfoMap, bindMountAppStorageDirs, zygoteArgs);
     }
 
     /** @hide */
@@ -659,7 +661,7 @@ public class Process {
                     runtimeFlags, mountExternal, targetSdkVersion, seInfo,
                     abi, instructionSet, appDataDir, invokeWith, packageName,
                     /*zygotePolicyFlags=*/ ZYGOTE_POLICY_FLAG_EMPTY, /*isTopApp=*/ false,
-                disabledCompatChanges, /* pkgDataInfoMap */ null, zygoteArgs);
+                disabledCompatChanges, /* pkgDataInfoMap */ null, false, zygoteArgs);
     }
 
     /**
diff --git a/core/java/android/os/ZygoteProcess.java b/core/java/android/os/ZygoteProcess.java
index 34cec061edcd..5f3f14facd75 100644
--- a/core/java/android/os/ZygoteProcess.java
+++ b/core/java/android/os/ZygoteProcess.java
@@ -333,6 +333,7 @@ public class ZygoteProcess {
      *                             started.
      * @param pkgDataInfoMap Map from related package names to private data directory
      *                       volume UUID and inode number.
+     * @param bindMountAppStorageDirs whether zygote needs to mount Android/obb and Android/data.
      *
      * @param zygoteArgs Additional arguments to supply to the Zygote process.
      * @return An object that describes the result of the attempt to start the process.
@@ -354,6 +355,7 @@ public class ZygoteProcess {
                                                   @Nullable long[] disabledCompatChanges,
                                                   @Nullable Map<String, Pair<String, Long>>
                                                           pkgDataInfoMap,
+                                                  boolean bindMountAppStorageDirs,
                                                   @Nullable String[] zygoteArgs) {
         // TODO (chriswailes): Is there a better place to check this value?
         if (fetchUsapPoolEnabledPropWithMinInterval()) {
@@ -365,7 +367,7 @@ public class ZygoteProcess {
                     runtimeFlags, mountExternal, targetSdkVersion, seInfo,
                     abi, instructionSet, appDataDir, invokeWith, /*startChildZygote=*/ false,
                     packageName, zygotePolicyFlags, isTopApp, disabledCompatChanges,
-                    pkgDataInfoMap, zygoteArgs);
+                    pkgDataInfoMap, bindMountAppStorageDirs, zygoteArgs);
         } catch (ZygoteStartFailedEx ex) {
             Log.e(LOG_TAG,
                     "Starting VM process through Zygote failed");
@@ -606,6 +608,7 @@ public class ZygoteProcess {
      * @param disabledCompatChanges a list of disabled compat changes for the process being started.
      * @param pkgDataInfoMap Map from related package names to private data directory volume UUID
      *                       and inode number.
+     * @param bindMountAppStorageDirs whether zygote needs to mount Android/obb and Android/data.
      * @param extraArgs Additional arguments to supply to the zygote process.
      * @return An object that describes the result of the attempt to start the process.
      * @throws ZygoteStartFailedEx if process start failed for any reason
@@ -628,6 +631,7 @@ public class ZygoteProcess {
                                                       @Nullable long[] disabledCompatChanges,
                                                       @Nullable Map<String, Pair<String, Long>>
                                                               pkgDataInfoMap,
+                                                      boolean bindMountAppStorageDirs,
                                                       @Nullable String[] extraArgs)
                                                       throws ZygoteStartFailedEx {
         ArrayList<String> argsForZygote = new ArrayList<>();
@@ -725,6 +729,10 @@ public class ZygoteProcess {
             argsForZygote.add(sb.toString());
         }
 
+        if (bindMountAppStorageDirs) {
+            argsForZygote.add(Zygote.BIND_MOUNT_APP_STORAGE_DIRS);
+        }
+
         if (disabledCompatChanges != null && disabledCompatChanges.length > 0) {
             StringBuilder sb = new StringBuilder();
             sb.append("--disabled-compat-changes=");
@@ -1282,7 +1290,9 @@ public class ZygoteProcess {
                     abi, instructionSet, null /* appDataDir */, null /* invokeWith */,
                     true /* startChildZygote */, null /* packageName */,
                     ZYGOTE_POLICY_FLAG_SYSTEM_PROCESS /* zygotePolicyFlags */, false /* isTopApp */,
-                    null /* disabledCompatChanges */, null /* pkgDataInfoMap */, extraArgs);
+                    null /* disabledCompatChanges */, null /* pkgDataInfoMap */,
+                    /* bindMountAppStorageDirs */ false, extraArgs);
+
         } catch (ZygoteStartFailedEx ex) {
             throw new RuntimeException("Starting child-zygote through Zygote failed", ex);
         }
diff --git a/core/java/android/os/storage/StorageManagerInternal.java b/core/java/android/os/storage/StorageManagerInternal.java
index a2def7fff2d2..f43a2523b5b3 100644
--- a/core/java/android/os/storage/StorageManagerInternal.java
+++ b/core/java/android/os/storage/StorageManagerInternal.java
@@ -100,10 +100,11 @@ public abstract class StorageManagerInternal {
     }
 
     /**
-     * Check if fuse is running in target user, if it's running then setup its obb directories.
-     * TODO: System server should store a list of active pids that obb is not mounted and use it.
+     * Create storage directories if it does not exist.
+     * Return true if the directories were setup correctly, otherwise false.
      */
-    public abstract void prepareObbDirs(int userId, Set<String> packageList, String processName);
+    public abstract boolean prepareStorageDirs(int userId, Set<String> packageList,
+            String processName);
 
     /**
      * Add a listener to listen to reset event in StorageManagerService.
diff --git a/core/java/com/android/internal/os/Zygote.java b/core/java/com/android/internal/os/Zygote.java
index 94924a57e3f2..63b146786391 100644
--- a/core/java/com/android/internal/os/Zygote.java
+++ b/core/java/com/android/internal/os/Zygote.java
@@ -178,6 +178,9 @@ public final class Zygote {
     /** List of packages with the same uid, and its app data info: volume uuid and inode. */
     public static final String PKG_DATA_INFO_MAP = "--pkg-data-info-map";
 
+    /** Bind mount app storage dirs to lower fs not via fuse */
+    public static final String BIND_MOUNT_APP_STORAGE_DIRS = "--bind-mount-storage-dirs";
+
     /**
      * An extraArg passed when a zygote process is forking a child-zygote, specifying a name
      * in the abstract socket namespace. This socket name is what the new child zygote
@@ -283,6 +286,7 @@ public final class Zygote {
      * @param isTopApp true if the process is for top (high priority) application.
      * @param pkgDataInfoList A list that stores related packages and its app data
      * info: volume uuid and inode.
+     * @param bindMountAppStorageDirs  True if the zygote needs to mount storage dirs.
      *
      * @return 0 if this is the child, pid of the child
      * if this is the parent, or -1 on error.
@@ -290,13 +294,13 @@ public final class Zygote {
     static int forkAndSpecialize(int uid, int gid, int[] gids, int runtimeFlags,
             int[][] rlimits, int mountExternal, String seInfo, String niceName, int[] fdsToClose,
             int[] fdsToIgnore, boolean startChildZygote, String instructionSet, String appDataDir,
-            boolean isTopApp, String[] pkgDataInfoList) {
+            boolean isTopApp, String[] pkgDataInfoList, boolean bindMountAppStorageDirs) {
         ZygoteHooks.preFork();
 
         int pid = nativeForkAndSpecialize(
                 uid, gid, gids, runtimeFlags, rlimits, mountExternal, seInfo, niceName, fdsToClose,
                 fdsToIgnore, startChildZygote, instructionSet, appDataDir, isTopApp,
-                pkgDataInfoList);
+                pkgDataInfoList, bindMountAppStorageDirs);
         if (pid == 0) {
             // Note that this event ends at the end of handleChildProc,
             Trace.traceBegin(Trace.TRACE_TAG_ACTIVITY_MANAGER, "PostFork");
@@ -312,7 +316,8 @@ public final class Zygote {
     private static native int nativeForkAndSpecialize(int uid, int gid, int[] gids,
             int runtimeFlags, int[][] rlimits, int mountExternal, String seInfo, String niceName,
             int[] fdsToClose, int[] fdsToIgnore, boolean startChildZygote, String instructionSet,
-            String appDataDir, boolean isTopApp, String[] pkgDataInfoList);
+            String appDataDir, boolean isTopApp, String[] pkgDataInfoList,
+            boolean bindMountAppStorageDirs);
 
     /**
      * Specialize an unspecialized app process.  The current VM must have been started
@@ -339,14 +344,15 @@ public final class Zygote {
      * volume uuid and CE dir inode. For example, pkgDataInfoList = [app_a_pkg_name,
      * app_a_data_volume_uuid, app_a_ce_inode, app_b_pkg_name, app_b_data_volume_uuid,
      * app_b_ce_inode, ...];
+     * @param bindMountAppStorageDirs  True if the zygote needs to mount storage dirs.
      */
     private static void specializeAppProcess(int uid, int gid, int[] gids, int runtimeFlags,
             int[][] rlimits, int mountExternal, String seInfo, String niceName,
             boolean startChildZygote, String instructionSet, String appDataDir, boolean isTopApp,
-            String[] pkgDataInfoList) {
+            String[] pkgDataInfoList, boolean bindMountAppStorageDirs) {
         nativeSpecializeAppProcess(uid, gid, gids, runtimeFlags, rlimits, mountExternal, seInfo,
                 niceName, startChildZygote, instructionSet, appDataDir, isTopApp,
-                pkgDataInfoList);
+                pkgDataInfoList, bindMountAppStorageDirs);
 
         // Note that this event ends at the end of handleChildProc.
         Trace.traceBegin(Trace.TRACE_TAG_ACTIVITY_MANAGER, "PostFork");
@@ -366,7 +372,7 @@ public final class Zygote {
     private static native void nativeSpecializeAppProcess(int uid, int gid, int[] gids,
             int runtimeFlags, int[][] rlimits, int mountExternal, String seInfo, String niceName,
             boolean startChildZygote, String instructionSet, String appDataDir, boolean isTopApp,
-            String[] pkgDataInfoList);
+            String[] pkgDataInfoList, boolean bindMountAppStorageDirs);
 
     /**
      * Called to do any initialization before starting an application.
@@ -691,7 +697,7 @@ public final class Zygote {
                                  args.mRuntimeFlags, rlimits, args.mMountExternal,
                                  args.mSeInfo, args.mNiceName, args.mStartChildZygote,
                                  args.mInstructionSet, args.mAppDataDir, args.mIsTopApp,
-                                 args.mPkgDataInfoList);
+                                 args.mPkgDataInfoList, args.mBindMountAppStorageDirs);
 
             Trace.traceEnd(Trace.TRACE_TAG_ACTIVITY_MANAGER);
 
diff --git a/core/java/com/android/internal/os/ZygoteArguments.java b/core/java/com/android/internal/os/ZygoteArguments.java
index 37f570bba238..1a63765fcaa6 100644
--- a/core/java/com/android/internal/os/ZygoteArguments.java
+++ b/core/java/com/android/internal/os/ZygoteArguments.java
@@ -227,6 +227,11 @@ class ZygoteArguments {
     String[] mPkgDataInfoList;
 
     /**
+     * @see Zygote#BIND_MOUNT_APP_STORAGE_DIRS
+     */
+    boolean mBindMountAppStorageDirs;
+
+    /**
      * Constructs instance and parses args
      *
      * @param args zygote command-line args
@@ -447,6 +452,8 @@ class ZygoteArguments {
                 }
             } else if (arg.startsWith(Zygote.PKG_DATA_INFO_MAP)) {
                 mPkgDataInfoList = getAssignmentList(arg);
+            } else if (arg.equals(Zygote.BIND_MOUNT_APP_STORAGE_DIRS)) {
+                mBindMountAppStorageDirs = true;
             } else {
                 break;
             }
diff --git a/core/java/com/android/internal/os/ZygoteConnection.java b/core/java/com/android/internal/os/ZygoteConnection.java
index 4949811bbe17..bc8dfd4aa402 100644
--- a/core/java/com/android/internal/os/ZygoteConnection.java
+++ b/core/java/com/android/internal/os/ZygoteConnection.java
@@ -258,7 +258,7 @@ class ZygoteConnection {
                 parsedArgs.mRuntimeFlags, rlimits, parsedArgs.mMountExternal, parsedArgs.mSeInfo,
                 parsedArgs.mNiceName, fdsToClose, fdsToIgnore, parsedArgs.mStartChildZygote,
                 parsedArgs.mInstructionSet, parsedArgs.mAppDataDir, parsedArgs.mIsTopApp,
-                parsedArgs.mPkgDataInfoList);
+                parsedArgs.mPkgDataInfoList, parsedArgs.mBindMountAppStorageDirs);
 
         try {
             if (pid == 0) {
diff --git a/core/jni/com_android_internal_os_Zygote.cpp b/core/jni/com_android_internal_os_Zygote.cpp
index 2128f99ff609..027d598e8b61 100644
--- a/core/jni/com_android_internal_os_Zygote.cpp
+++ b/core/jni/com_android_internal_os_Zygote.cpp
@@ -1568,23 +1568,6 @@ static void BindMountStorageDirs(JNIEnv* env, jobjectArray pkg_data_info_list,
   auto extract_fn = std::bind(ExtractJString, env, process_name, managed_nice_name, _1);
   const userid_t user_id = multiuser_get_user_id(uid);
 
-  // If FUSE is not ready for this user, skip it
-  // TODO(148772775): Pass primary volume name from zygote argument to here
-  std::string tmp = GetProperty("vold.fuse_running_users", "");
-  std::istringstream fuse_running_users(tmp);
-  bool user_found = false;
-  std::string s;
-  std::string user_id_str = std::to_string(user_id);
-  while (!user_found && std::getline(fuse_running_users, s, ',')) {
-    if (user_id_str == s) {
-      user_found = true;
-    }
-  }
-  if (!user_found) {
-    ALOGI("User %d is not running fuse yet, fuse_running_users=%s", user_id, tmp.c_str());
-    return;
-  }
-
   // Fuse is ready, so we can start using fuse path.
   int size = (pkg_data_info_list != nullptr) ? env->GetArrayLength(pkg_data_info_list) : 0;
 
@@ -1612,7 +1595,7 @@ static void SpecializeCommon(JNIEnv* env, uid_t uid, gid_t gid, jintArray gids,
                              jstring managed_nice_name, bool is_system_server,
                              bool is_child_zygote, jstring managed_instruction_set,
                              jstring managed_app_data_dir, bool is_top_app,
-                             jobjectArray pkg_data_info_list) {
+                             jobjectArray pkg_data_info_list, bool mount_storage_dirs) {
   const char* process_name = is_system_server ? "system_server" : "zygote";
   auto fail_fn = std::bind(ZygoteFailure, env, process_name, managed_nice_name, _1);
   auto extract_fn = std::bind(ExtractJString, env, process_name, managed_nice_name, _1);
@@ -1651,10 +1634,7 @@ static void SpecializeCommon(JNIEnv* env, uid_t uid, gid_t gid, jintArray gids,
     isolateAppData(env, pkg_data_info_list, uid, process_name, managed_nice_name, fail_fn);
     isolateJitProfile(env, pkg_data_info_list, uid, process_name, managed_nice_name, fail_fn);
   }
-
-  if ((mount_external != MOUNT_EXTERNAL_INSTALLER) &&
-        GetBoolProperty(kPropFuse, false) &&
-        GetBoolProperty(ANDROID_VOLD_APP_DATA_ISOLATION_ENABLED_PROPERTY, false)) {
+  if ((mount_external != MOUNT_EXTERNAL_INSTALLER) && mount_storage_dirs) {
     BindMountStorageDirs(env, pkg_data_info_list, uid, process_name, managed_nice_name, fail_fn);
   }
 
@@ -2021,7 +2001,7 @@ static jint com_android_internal_os_Zygote_nativeForkAndSpecialize(
         jint mount_external, jstring se_info, jstring nice_name,
         jintArray managed_fds_to_close, jintArray managed_fds_to_ignore, jboolean is_child_zygote,
         jstring instruction_set, jstring app_data_dir, jboolean is_top_app,
-        jobjectArray pkg_data_info_list) {
+        jobjectArray pkg_data_info_list, jboolean mount_storage_dirs) {
     jlong capabilities = CalculateCapabilities(env, uid, gid, gids, is_child_zygote);
 
     if (UNLIKELY(managed_fds_to_close == nullptr)) {
@@ -2058,7 +2038,8 @@ static jint com_android_internal_os_Zygote_nativeForkAndSpecialize(
                        capabilities, capabilities,
                        mount_external, se_info, nice_name, false,
                        is_child_zygote == JNI_TRUE, instruction_set, app_data_dir,
-                       is_top_app == JNI_TRUE, pkg_data_info_list);
+                       is_top_app == JNI_TRUE, pkg_data_info_list,
+                       mount_storage_dirs == JNI_TRUE);
     }
     return pid;
 }
@@ -2093,7 +2074,7 @@ static jint com_android_internal_os_Zygote_nativeForkSystemServer(
                        permitted_capabilities, effective_capabilities,
                        MOUNT_EXTERNAL_DEFAULT, nullptr, nullptr, true,
                        false, nullptr, nullptr, /* is_top_app= */ false,
-                       /* pkg_data_info_list */ nullptr);
+                       /* pkg_data_info_list */ nullptr, false);
   } else if (pid > 0) {
       // The zygote process checks whether the child process has died or not.
       ALOGI("System server process %d has been created", pid);
@@ -2223,14 +2204,15 @@ static void com_android_internal_os_Zygote_nativeSpecializeAppProcess(
     jint runtime_flags, jobjectArray rlimits,
     jint mount_external, jstring se_info, jstring nice_name,
     jboolean is_child_zygote, jstring instruction_set, jstring app_data_dir, jboolean is_top_app,
-    jobjectArray pkg_data_info_list) {
+    jobjectArray pkg_data_info_list, jboolean mount_storage_dirs) {
   jlong capabilities = CalculateCapabilities(env, uid, gid, gids, is_child_zygote);
 
   SpecializeCommon(env, uid, gid, gids, runtime_flags, rlimits,
                    capabilities, capabilities,
                    mount_external, se_info, nice_name, false,
                    is_child_zygote == JNI_TRUE, instruction_set, app_data_dir,
-                   is_top_app == JNI_TRUE, pkg_data_info_list);
+                   is_top_app == JNI_TRUE, pkg_data_info_list,
+                   mount_storage_dirs == JNI_TRUE);
 }
 
 /**
@@ -2424,7 +2406,7 @@ static jint com_android_internal_os_Zygote_nativeParseSigChld(JNIEnv* env, jclas
 static const JNINativeMethod gMethods[] = {
         {"nativeForkAndSpecialize",
          "(II[II[[IILjava/lang/String;Ljava/lang/String;[I[IZLjava/lang/String;Ljava/lang/"
-         "String;Z[Ljava/lang/String;)I",
+         "String;Z[Ljava/lang/String;Z)I",
          (void*)com_android_internal_os_Zygote_nativeForkAndSpecialize},
         {"nativeForkSystemServer", "(II[II[[IJJ)I",
          (void*)com_android_internal_os_Zygote_nativeForkSystemServer},
@@ -2437,7 +2419,7 @@ static const JNINativeMethod gMethods[] = {
         {"nativeForkUsap", "(II[IZ)I", (void*)com_android_internal_os_Zygote_nativeForkUsap},
         {"nativeSpecializeAppProcess",
          "(II[II[[IILjava/lang/String;Ljava/lang/String;ZLjava/lang/String;Ljava/lang/"
-         "String;Z[Ljava/lang/String;)V",
+         "String;Z[Ljava/lang/String;Z)V",
          (void*)com_android_internal_os_Zygote_nativeSpecializeAppProcess},
         {"nativeInitNativeState", "(Z)V",
          (void*)com_android_internal_os_Zygote_nativeInitNativeState},
diff --git a/services/core/java/com/android/server/StorageManagerService.java b/services/core/java/com/android/server/StorageManagerService.java
index b7d050a25484..2c220d3f8861 100644
--- a/services/core/java/com/android/server/StorageManagerService.java
+++ b/services/core/java/com/android/server/StorageManagerService.java
@@ -154,7 +154,6 @@ import com.android.internal.util.HexDump;
 import com.android.internal.util.IndentingPrintWriter;
 import com.android.internal.util.Preconditions;
 import com.android.internal.widget.LockPatternUtils;
-import com.android.server.SystemService.TargetUser;
 import com.android.server.pm.Installer;
 import com.android.server.storage.AppFuseBridge;
 import com.android.server.storage.StorageSessionController;
@@ -233,6 +232,8 @@ class StorageManagerService extends IStorageManager.Stub
     private static final String FUSE_ENABLED = "fuse_enabled";
     private static final boolean DEFAULT_FUSE_ENABLED = true;
 
+    private final Set<Integer> mFuseMountedUser = new ArraySet<>();
+
     public static class Lifecycle extends SystemService {
         private StorageManagerService mStorageManagerService;
 
@@ -1497,6 +1498,9 @@ class StorageManagerService extends IStorageManager.Stub
 
     @GuardedBy("mLock")
     private void onVolumeStateChangedLocked(VolumeInfo vol, int oldState, int newState) {
+        if (vol.type == VolumeInfo.TYPE_EMULATED && newState != VolumeInfo.STATE_MOUNTED) {
+            mFuseMountedUser.remove(vol.getMountUserId());
+        }
         // Remember that we saw this volume so we're ready to accept user
         // metadata, or so we can annoy them when a private volume is ejected
         if (!TextUtils.isEmpty(vol.fsUuid)) {
@@ -2075,39 +2079,86 @@ class StorageManagerService extends IStorageManager.Stub
         mount(vol);
     }
 
+    private void remountAppStorageDirs(Map<Integer, String> pidPkgMap, int userId) {
+        for (Entry<Integer, String> entry : pidPkgMap.entrySet()) {
+            final int pid = entry.getKey();
+            final String packageName = entry.getValue();
+            Slog.i(TAG, "Remounting storage for pid: " + pid);
+            final String[] sharedPackages =
+                    mPmInternal.getSharedUserPackagesForPackage(packageName, userId);
+            final int uid = mPmInternal.getPackageUidInternal(packageName, 0, userId);
+            final String[] packages =
+                    sharedPackages.length != 0 ? sharedPackages : new String[]{packageName};
+            try {
+                mVold.remountAppStorageDirs(uid, pid, packages);
+            } catch (RemoteException e) {
+                throw e.rethrowAsRuntimeException();
+            }
+        }
+    }
+
     private void mount(VolumeInfo vol) {
         try {
             // TODO(b/135341433): Remove paranoid logging when FUSE is stable
             Slog.i(TAG, "Mounting volume " + vol);
             mVold.mount(vol.id, vol.mountFlags, vol.mountUserId, new IVoldMountCallback.Stub() {
-                    @Override
-                    public boolean onVolumeChecking(FileDescriptor fd, String path,
-                            String internalPath) {
-                        vol.path = path;
-                        vol.internalPath = internalPath;
-                        ParcelFileDescriptor pfd = new ParcelFileDescriptor(fd);
+                @Override
+                public boolean onVolumeChecking(FileDescriptor fd, String path,
+                        String internalPath) {
+                    vol.path = path;
+                    vol.internalPath = internalPath;
+                    ParcelFileDescriptor pfd = new ParcelFileDescriptor(fd);
+                    try {
+                        mStorageSessionController.onVolumeMount(pfd, vol);
+                        return true;
+                    } catch (ExternalStorageServiceException e) {
+                        Slog.e(TAG, "Failed to mount volume " + vol, e);
+
+                        int nextResetSeconds = REMOTE_TIMEOUT_SECONDS * 2;
+                        Slog.i(TAG, "Scheduling reset in " + nextResetSeconds + "s");
+                        mHandler.removeMessages(H_RESET);
+                        mHandler.sendMessageDelayed(mHandler.obtainMessage(H_RESET),
+                                TimeUnit.SECONDS.toMillis(nextResetSeconds));
+                        return false;
+                    } finally {
                         try {
-                            mStorageSessionController.onVolumeMount(pfd, vol);
-                            return true;
-                        } catch (ExternalStorageServiceException e) {
-                            Slog.e(TAG, "Failed to mount volume " + vol, e);
-
-                            int nextResetSeconds = REMOTE_TIMEOUT_SECONDS * 2;
-                            Slog.i(TAG, "Scheduling reset in " + nextResetSeconds + "s");
-                            mHandler.removeMessages(H_RESET);
-                            mHandler.sendMessageDelayed(mHandler.obtainMessage(H_RESET),
-                                    TimeUnit.SECONDS.toMillis(nextResetSeconds));
-                            return false;
-                        } finally {
-                            try {
-                                pfd.close();
-                            } catch (Exception e) {
-                                Slog.e(TAG, "Failed to close FUSE device fd", e);
-                            }
+                            pfd.close();
+                        } catch (Exception e) {
+                            Slog.e(TAG, "Failed to close FUSE device fd", e);
                         }
                     }
-                });
+                }
+            });
             Slog.i(TAG, "Mounted volume " + vol);
+            if (vol.type == VolumeInfo.TYPE_EMULATED) {
+                final int userId = vol.getMountUserId();
+                mFuseMountedUser.add(userId);
+                // Async remount app storage so it won't block the main thread.
+                new Thread(() -> {
+                    Map<Integer, String> pidPkgMap = null;
+                    // getProcessesWithPendingBindMounts() could fail when a new app process is
+                    // starting and it's not planning to mount storage dirs in zygote, but it's
+                    // rare, so we retry 5 times and hope we can get the result successfully.
+                    for (int i = 0; i < 5; i++) {
+                        try {
+                            pidPkgMap = LocalServices.getService(ActivityManagerInternal.class)
+                                    .getProcessesWithPendingBindMounts(vol.getMountUserId());
+                            break;
+                        } catch (IllegalStateException e) {
+                            Slog.i(TAG, "Some processes are starting, retry");
+                            // Wait 100ms and retry so hope the pending process is started.
+                            SystemClock.sleep(100);
+                        }
+                    }
+                    if (pidPkgMap != null) {
+                        remountAppStorageDirs(pidPkgMap, userId);
+                    } else {
+                        Slog.wtf(TAG, "Not able to getStorageNotOptimizedProcesses() after"
+                                + " 5 retries");
+                    }
+
+                }).start();
+            }
         } catch (Exception e) {
             Slog.wtf(TAG, e);
         }
@@ -4309,7 +4360,8 @@ class StorageManagerService extends IStorageManager.Stub
             pw.println();
             pw.println("mObbPathToStateMap:");
             pw.increaseIndent();
-            final Iterator<Entry<String, ObbState>> maps = mObbPathToStateMap.entrySet().iterator();
+            final Iterator<Entry<String, ObbState>> maps =
+                    mObbPathToStateMap.entrySet().iterator();
             while (maps.hasNext()) {
                 final Entry<String, ObbState> e = maps.next();
                 pw.print(e.getKey());
@@ -4350,45 +4402,41 @@ class StorageManagerService extends IStorageManager.Stub
         }
 
         /**
-         * Check if fuse is running in target user, if it's running then setup its obb directories.
-         * TODO: System server should store a list of active pids that obb is not mounted and use it.
+         * Check if fuse is running in target user, if it's running then setup its storage dirs.
+         * Return true if storage dirs are mounted.
          */
         @Override
-        public void prepareObbDirs(int userId, Set<String> packageList, String processName) {
-            String fuseRunningUsersList = SystemProperties.get("vold.fuse_running_users", "");
-            String[] fuseRunningUsers = fuseRunningUsersList.split(",");
-            boolean fuseReady = false;
-            String targetUserId = String.valueOf(userId);
-            for (String user : fuseRunningUsers) {
-                if (targetUserId.equals(user)) {
-                    fuseReady = true;
-                }
+        public boolean prepareStorageDirs(int userId, Set<String> packageList,
+                String processName) {
+            if (!mFuseMountedUser.contains(userId)) {
+                Slog.w(TAG, "User " + userId + " is not unlocked yet so skip mounting obb");
+                return false;
             }
-            if (fuseReady) {
-                try {
-                    final IVold vold = IVold.Stub.asInterface(
-                            ServiceManager.getServiceOrThrow("vold"));
-                    for (String pkg : packageList) {
-                        final String packageObbDir =
-                                String.format("/storage/emulated/%d/Android/obb/%s/", userId, pkg);
-                        final String packageDataDir =
-                                String.format("/storage/emulated/%d/Android/data/%s/",
-                                        userId, pkg);
-
-                        // Create package obb and data dir if it doesn't exist.
-                        File file = new File(packageObbDir);
-                        if (!file.exists()) {
-                            vold.setupAppDir(packageObbDir, mPmInternal.getPackage(pkg).getUid());
-                        }
-                        file = new File(packageDataDir);
-                        if (!file.exists()) {
-                            vold.setupAppDir(packageDataDir, mPmInternal.getPackage(pkg).getUid());
-                        }
+            try {
+                final IVold vold = IVold.Stub.asInterface(
+                        ServiceManager.getServiceOrThrow("vold"));
+                for (String pkg : packageList) {
+                    final String packageObbDir =
+                            String.format("/storage/emulated/%d/Android/obb/%s/", userId, pkg);
+                    final String packageDataDir =
+                            String.format("/storage/emulated/%d/Android/data/%s/",
+                                    userId, pkg);
+
+                    // Create package obb and data dir if it doesn't exist.
+                    File file = new File(packageObbDir);
+                    if (!file.exists()) {
+                        vold.setupAppDir(packageObbDir, mPmInternal.getPackage(pkg).getUid());
+                    }
+                    file = new File(packageDataDir);
+                    if (!file.exists()) {
+                        vold.setupAppDir(packageDataDir, mPmInternal.getPackage(pkg).getUid());
                     }
-                } catch (ServiceManager.ServiceNotFoundException | RemoteException e) {
-                    Slog.e(TAG, "Unable to create obb and data directories for " + processName, e);
                 }
+            } catch (ServiceManager.ServiceNotFoundException | RemoteException e) {
+                Slog.e(TAG, "Unable to create obb and data directories for " + processName,e);
+                return false;
             }
+            return true;
         }
 
         @Override
diff --git a/services/core/java/com/android/server/am/ActivityManagerService.java b/services/core/java/com/android/server/am/ActivityManagerService.java
index f64272bf08d1..ac25c4666dbf 100644
--- a/services/core/java/com/android/server/am/ActivityManagerService.java
+++ b/services/core/java/com/android/server/am/ActivityManagerService.java
@@ -18714,6 +18714,11 @@ public class ActivityManagerService extends IActivityManager.Stub
         }
 
         @Override
+        public Map<Integer, String> getProcessesWithPendingBindMounts(int userId) {
+            return mProcessList.getProcessesWithPendingBindMounts(userId);
+        }
+
+        @Override
         public boolean isSystemReady() {
             // no need to synchronize(this) just to read & return the value
             return mSystemReady;
diff --git a/services/core/java/com/android/server/am/ProcessList.java b/services/core/java/com/android/server/am/ProcessList.java
index 6b165139aefd..b7c02184db79 100644
--- a/services/core/java/com/android/server/am/ProcessList.java
+++ b/services/core/java/com/android/server/am/ProcessList.java
@@ -97,6 +97,7 @@ import android.provider.DeviceConfig;
 import android.system.Os;
 import android.text.TextUtils;
 import android.util.ArrayMap;
+import android.util.ArraySet;
 import android.util.EventLog;
 import android.util.LongSparseArray;
 import android.util.Pair;
@@ -134,8 +135,11 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Activity manager code dealing with processes.
@@ -152,7 +156,8 @@ public final class ProcessList {
             "persist.sys.vold_app_data_isolation_enabled";
 
     // A device config to control the minimum target SDK to enable app data isolation
-    static final String ANDROID_APP_DATA_ISOLATION_MIN_SDK = "android_app_data_isolation_min_sdk";
+    static final String ANDROID_APP_DATA_ISOLATION_MIN_SDK =
+            "android_app_data_isolation_min_sdk";
 
     // The minimum time we allow between crashes, for us to consider this
     // application to be bad and stop and its services and reject broadcasts.
@@ -789,6 +794,31 @@ public final class ProcessList {
         }
     }
 
+    /**
+     * Get a map of pid and package name that process of that pid Android/data and Android/obb
+     * directory is not mounted to lowerfs to speed up access.
+     */
+    Map<Integer, String> getProcessesWithPendingBindMounts(int userId) {
+        final Map<Integer, String> pidPackageMap = new HashMap<>();
+        synchronized (mService) {
+            for (int i = mLruProcesses.size() - 1; i >= 0; i--) {
+                final ProcessRecord record = mLruProcesses.get(i);
+                if (record.userId != userId || !record.bindMountPending) {
+                    continue;
+                }
+                final int pid = record.pid;
+                // It can happen when app process is starting, but zygote work is not done yet so
+                // system does not this pid record yet.
+                if (pid == 0) {
+                    throw new IllegalStateException("Pending process is not started yet,"
+                            + "retry later");
+                }
+                pidPackageMap.put(pid, record.info.packageName);
+            }
+            return pidPackageMap;
+        }
+    }
+
     private void updateOomLevels(int displayWidth, int displayHeight, boolean write) {
         // Scale buckets from avail memory: at 300MB we use the lowest values to
         // 700MB or more for the top values.
@@ -1648,6 +1678,7 @@ public final class ProcessList {
         if (app.pid > 0 && app.pid != ActivityManagerService.MY_PID) {
             checkSlow(startTime, "startProcess: removing from pids map");
             mService.removePidLocked(app);
+            app.bindMountPending = false;
             mService.mHandler.removeMessages(PROC_START_TIMEOUT_MSG, app);
             checkSlow(startTime, "startProcess: done removing from pids map");
             app.setPid(0);
@@ -2128,8 +2159,10 @@ public final class ProcessList {
             }
 
             final Map<String, Pair<String, Long>> pkgDataInfoMap;
+            boolean bindMountAppStorageDirs = false;
 
             if (shouldIsolateAppData(app)) {
+                bindMountAppStorageDirs = mVoldAppDataIsolationEnabled;
                 // Get all packages belongs to the same shared uid. sharedPackages is empty array
                 // if it doesn't have shared uid.
                 final PackageManagerInternal pmInt = mService.getPackageManagerInternalLocked();
@@ -2138,11 +2171,17 @@ public final class ProcessList {
                 pkgDataInfoMap = getPackageAppDataInfoMap(pmInt, sharedPackages.length == 0
                         ? new String[]{app.info.packageName} : sharedPackages, uid);
 
+                int userId = UserHandle.getUserId(uid);
                 if (mVoldAppDataIsolationEnabled) {
                     StorageManagerInternal storageManagerInternal = LocalServices.getService(
-                                StorageManagerInternal.class);
-                    storageManagerInternal.prepareObbDirs(UserHandle.getUserId(uid),
-                            pkgDataInfoMap.keySet(), app.processName);
+                            StorageManagerInternal.class);
+                    if (!storageManagerInternal.prepareStorageDirs(userId, pkgDataInfoMap.keySet(),
+                            app.processName)) {
+                        // Cannot prepare Android/app and Android/obb directory,
+                        // so we won't mount it in zygote.
+                        app.bindMountPending = true;
+                        bindMountAppStorageDirs = false;
+                    }
                 }
             } else {
                 pkgDataInfoMap = null;
@@ -2163,7 +2202,7 @@ public final class ProcessList {
                         app.info.targetSdkVersion, seInfo, requiredAbi, instructionSet,
                         app.info.dataDir, null, app.info.packageName,
                         /*zygotePolicyFlags=*/ ZYGOTE_POLICY_FLAG_EMPTY, isTopApp,
-                        app.mDisabledCompatChanges, pkgDataInfoMap,
+                        app.mDisabledCompatChanges, pkgDataInfoMap, bindMountAppStorageDirs,
                         new String[]{PROC_START_SEQ_IDENT + app.startSeq});
             } else {
                 startResult = Process.start(entryPoint,
@@ -2171,6 +2210,7 @@ public final class ProcessList {
                         app.info.targetSdkVersion, seInfo, requiredAbi, instructionSet,
                         app.info.dataDir, invokeWith, app.info.packageName, zygotePolicyFlags,
                         isTopApp, app.mDisabledCompatChanges, pkgDataInfoMap,
+                        bindMountAppStorageDirs,
                         new String[]{PROC_START_SEQ_IDENT + app.startSeq});
             }
             checkSlow(startTime, "startProcess: returned from zygote!");
@@ -2629,6 +2669,7 @@ public final class ProcessList {
             int pid = app.pid;
             if (pid > 0) {
                 mService.removePidLocked(app);
+                app.bindMountPending = false;
                 mService.mHandler.removeMessages(PROC_START_TIMEOUT_MSG, app);
                 mService.mBatteryStatsService.noteProcessFinish(app.processName, app.info.uid);
                 if (app.isolated) {
diff --git a/services/core/java/com/android/server/am/ProcessRecord.java b/services/core/java/com/android/server/am/ProcessRecord.java
index c0298110580e..ec5999d474fc 100644
--- a/services/core/java/com/android/server/am/ProcessRecord.java
+++ b/services/core/java/com/android/server/am/ProcessRecord.java
@@ -251,6 +251,7 @@ class ProcessRecord implements WindowProcessListener {
     int adjSourceProcState;     // Debugging: proc state of adjSource's process.
     Object adjTarget;           // Debugging: target component impacting oom_adj.
     Runnable crashHandler;      // Optional local handler to be invoked in the process crash.
+    boolean bindMountPending;   // True if Android/obb and Android/data need to be bind mount .
 
     // Cache of last retrieve memory info and uptime, to throttle how frequently
     // apps can requyest it.