diff options
14 files changed, 424 insertions, 293 deletions
diff --git a/core/java/android/os/incremental/IIncrementalService.aidl b/core/java/android/os/incremental/IIncrementalService.aidl index 52475e9cd89d..ca92ad5deae6 100644 --- a/core/java/android/os/incremental/IIncrementalService.aidl +++ b/core/java/android/os/incremental/IIncrementalService.aidl @@ -144,4 +144,14 @@ interface IIncrementalService { * Stop listening for the loading progress change for a storage. */ boolean unregisterLoadingProgressListener(int storageId); + + /** + * Register storage health status listener. + */ + boolean registerStorageHealthListener(int storageId, in StorageHealthCheckParams params, in IStorageHealthListener listener); + + /** + * Register storage health status listener. + */ + void unregisterStorageHealthListener(int storageId); } diff --git a/core/java/android/os/incremental/IStorageHealthListener.aidl b/core/java/android/os/incremental/IStorageHealthListener.aidl index 9f93ede5c9fc..c71e73f9ec8e 100644 --- a/core/java/android/os/incremental/IStorageHealthListener.aidl +++ b/core/java/android/os/incremental/IStorageHealthListener.aidl @@ -26,9 +26,15 @@ oneway interface IStorageHealthListener { /** There are reads pending for params.blockedTimeoutMs, waiting till * params.unhealthyTimeoutMs to confirm unhealthy state. */ const int HEALTH_STATUS_BLOCKED = 2; - /** There are reads pending for params.unhealthyTimeoutMs>, - * marking storage as unhealthy. */ + /** There are reads pending for params.unhealthyTimeoutMs, + * marking storage as unhealthy due to unknown issues. */ const int HEALTH_STATUS_UNHEALTHY = 3; + /** There are reads pending for params.unhealthyTimeoutMs, + * due to data transportation issues. */ + const int HEALTH_STATUS_UNHEALTHY_TRANSPORT = 4; + /** There are reads pending for params.unhealthyTimeoutMs, + * due to limited storage space. */ + const int HEALTH_STATUS_UNHEALTHY_STORAGE = 5; /** Health status callback. */ void onHealthStatus(in int storageId, in int status); diff --git a/core/java/android/os/incremental/IncrementalManager.java b/core/java/android/os/incremental/IncrementalManager.java index 768ef975bd99..fb47ef04b231 100644 --- a/core/java/android/os/incremental/IncrementalManager.java +++ b/core/java/android/os/incremental/IncrementalManager.java @@ -283,6 +283,7 @@ public final class IncrementalManager { return; } mLoadingProgressCallbacks.cleanUpCallbacks(storage); + unregisterHealthListener(codePath); mService.deleteStorage(storage.getId()); } catch (RemoteException e) { throw e.rethrowFromSystemServer(); @@ -297,7 +298,7 @@ public final class IncrementalManager { * @param callback To report loading progress to. * @return True if the package name and associated storage id are valid. False otherwise. */ - public boolean registerCallback(@NonNull String codePath, + public boolean registerLoadingProgressCallback(@NonNull String codePath, @NonNull IPackageLoadingProgressCallback callback) { final IncrementalStorage storage = openStorage(codePath); if (storage == null) { @@ -314,7 +315,7 @@ public final class IncrementalManager { * @param codePath Path of the installed package * @return True if the package name and associated storage id are valid. False otherwise. */ - public boolean unregisterCallback(@NonNull String codePath, + public boolean unregisterLoadingProgressCallback(@NonNull String codePath, @NonNull IPackageLoadingProgressCallback callback) { final IncrementalStorage storage = openStorage(codePath); if (storage == null) { @@ -414,6 +415,38 @@ public final class IncrementalManager { } } + /** + * Specify the health check params and listener for listening to Incremental Storage health + * status changes. Notice that this will overwrite the previously registered listener. + * @param codePath Path of the installed package. This path is on an Incremental Storage. + * @param healthCheckParams The params for health state change timeouts. + * @param listener To report health status change. + * @return True if listener was successfully registered. + */ + public boolean registerHealthListener(@NonNull String codePath, + @NonNull StorageHealthCheckParams healthCheckParams, + @NonNull IStorageHealthListener.Stub listener) { + final IncrementalStorage storage = openStorage(codePath); + if (storage == null) { + // storage does not exist, package not installed + return false; + } + return storage.registerStorageHealthListener(healthCheckParams, listener); + } + + /** + * Stop listening to health status changes on an Incremental Storage. + * @param codePath Path of the installed package. This path is on an Incremental Storage. + */ + public void unregisterHealthListener(@NonNull String codePath) { + final IncrementalStorage storage = openStorage(codePath); + if (storage == null) { + // storage does not exist, package not installed + return; + } + storage.unregisterStorageHealthListener(); + } + /* Native methods */ private static native boolean nativeIsEnabled(); private static native boolean nativeIsIncrementalPath(@NonNull String path); diff --git a/core/java/android/os/incremental/IncrementalStorage.java b/core/java/android/os/incremental/IncrementalStorage.java index a1c3cc697e02..b913faf9cc83 100644 --- a/core/java/android/os/incremental/IncrementalStorage.java +++ b/core/java/android/os/incremental/IncrementalStorage.java @@ -545,4 +545,31 @@ public final class IncrementalStorage { return false; } } + + /** + * Register to listen to the status changes of the storage health. + * @param healthCheckParams Params to specify status change timeouts. + * @param listener To report health status change from Incremental Service to the caller. + */ + public boolean registerStorageHealthListener(StorageHealthCheckParams healthCheckParams, + IStorageHealthListener listener) { + try { + return mService.registerStorageHealthListener(mId, healthCheckParams, listener); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + return false; + } + } + + /** + * Stops listening to the status changes of the storage health. + */ + public void unregisterStorageHealthListener() { + try { + mService.unregisterStorageHealthListener(mId); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + return; + } + } } diff --git a/services/core/java/com/android/server/pm/IncrementalStates.java b/services/core/java/com/android/server/pm/IncrementalStates.java index ababb8365c4f..43f4a3477a98 100644 --- a/services/core/java/com/android/server/pm/IncrementalStates.java +++ b/services/core/java/com/android/server/pm/IncrementalStates.java @@ -16,18 +16,21 @@ package com.android.server.pm; -import android.content.pm.IDataLoaderStatusListener; +import static android.os.incremental.IStorageHealthListener.HEALTH_STATUS_OK; +import static android.os.incremental.IStorageHealthListener.HEALTH_STATUS_UNHEALTHY; +import static android.os.incremental.IStorageHealthListener.HEALTH_STATUS_UNHEALTHY_STORAGE; +import static android.os.incremental.IStorageHealthListener.HEALTH_STATUS_UNHEALTHY_TRANSPORT; + import android.content.pm.IncrementalStatesInfo; import android.content.pm.PackageManager; import android.os.Handler; -import android.os.incremental.IStorageHealthListener; import android.util.Slog; import com.android.internal.annotations.GuardedBy; import com.android.internal.os.BackgroundThread; import com.android.internal.util.function.pooled.PooledLambda; -import java.util.function.BiConsumer; +import java.util.function.Consumer; /** * Manages state transitions of a package installed on Incremental File System. Currently manages: @@ -36,8 +39,7 @@ import java.util.function.BiConsumer; * * The following events might change the states of a package: * 1. Installation commit - * 2. Incremental storage health - * 3. Data loader stream health + * 2. Incremental storage health changes * 4. Loading progress changes * * @hide @@ -48,16 +50,14 @@ public final class IncrementalStates { private final Handler mHandler = BackgroundThread.getHandler(); private final Object mLock = new Object(); @GuardedBy("mLock") - private int mStreamStatus = IDataLoaderStatusListener.STREAM_HEALTHY; - @GuardedBy("mLock") - private int mStorageHealthStatus = IStorageHealthListener.HEALTH_STATUS_OK; + private int mStorageHealthStatus = HEALTH_STATUS_OK; @GuardedBy("mLock") private final LoadingState mLoadingState; @GuardedBy("mLock") private StartableState mStartableState; @GuardedBy("mLock") private Callback mCallback = null; - private final BiConsumer<Integer, Integer> mStatusConsumer; + private final Consumer<Integer> mStatusConsumer; public IncrementalStates() { // By default the package is not startable and not fully loaded (i.e., is loading) @@ -148,12 +148,9 @@ public final class IncrementalStates { } } - private class StatusConsumer implements BiConsumer<Integer, Integer> { + private class StatusConsumer implements Consumer<Integer> { @Override - public void accept(Integer streamStatus, Integer storageStatus) { - if (streamStatus == null && storageStatus == null) { - return; - } + public void accept(Integer storageStatus) { final boolean oldState, newState; synchronized (mLock) { if (!mLoadingState.isLoading()) { @@ -161,12 +158,7 @@ public final class IncrementalStates { return; } oldState = mStartableState.isStartable(); - if (streamStatus != null) { - mStreamStatus = (Integer) streamStatus; - } - if (storageStatus != null) { - mStorageHealthStatus = (Integer) storageStatus; - } + mStorageHealthStatus = storageStatus; updateStartableStateLocked(); newState = mStartableState.isStartable(); } @@ -188,21 +180,7 @@ public final class IncrementalStates { Slog.i(TAG, "received storage health status changed event : storageHealthStatus=" + storageHealthStatus); } - mStatusConsumer.accept(null, storageHealthStatus); - } - - /** - * By calling this method, the caller indicates that the stream status of the package has - * been - * changed. This could indicate a streaming error. The state will change according to the - * status - * code defined in {@code IDataLoaderStatusListener}. - */ - public void onStreamStatusChanged(int streamState) { - if (DEBUG) { - Slog.i(TAG, "received stream status changed event : streamState=" + streamState); - } - mStatusConsumer.accept(streamState, null); + mStatusConsumer.accept(storageHealthStatus); } /** @@ -284,35 +262,16 @@ public final class IncrementalStates { final boolean currentState = mStartableState.isStartable(); boolean nextState = currentState; if (!currentState) { - if (mStorageHealthStatus == IStorageHealthListener.HEALTH_STATUS_OK - && mStreamStatus == IDataLoaderStatusListener.STREAM_HEALTHY) { - // change from unstartable -> startable when both stream and storage are healthy + if (mStorageHealthStatus == HEALTH_STATUS_OK) { + // change from unstartable -> startable nextState = true; } } else { - if (mStorageHealthStatus == IStorageHealthListener.HEALTH_STATUS_UNHEALTHY) { - // unrecoverable if storage is unhealthy + if (mStorageHealthStatus == HEALTH_STATUS_UNHEALTHY + || mStorageHealthStatus == HEALTH_STATUS_UNHEALTHY_STORAGE + || mStorageHealthStatus == HEALTH_STATUS_UNHEALTHY_TRANSPORT) { + // change from startable -> unstartable nextState = false; - } else { - switch (mStreamStatus) { - case IDataLoaderStatusListener.STREAM_INTEGRITY_ERROR: - // unrecoverable, fall through - case IDataLoaderStatusListener.STREAM_SOURCE_ERROR: { - // unrecoverable - nextState = false; - break; - } - case IDataLoaderStatusListener.STREAM_STORAGE_ERROR: { - if (mStorageHealthStatus != IStorageHealthListener.HEALTH_STATUS_OK) { - // unrecoverable if there is a pending read AND storage is limited - nextState = false; - } - break; - } - default: - // anything else, remain startable - break; - } } } if (nextState == currentState) { @@ -370,17 +329,11 @@ public final class IncrementalStates { return PackageManager.UNSTARTABLE_REASON_UNKNOWN; } // Translate stream status to reason for unstartable state - switch (mStreamStatus) { - case IDataLoaderStatusListener.STREAM_TRANSPORT_ERROR: - // fall through - case IDataLoaderStatusListener.STREAM_INTEGRITY_ERROR: - // fall through - case IDataLoaderStatusListener.STREAM_SOURCE_ERROR: { - return PackageManager.UNSTARTABLE_REASON_CONNECTION_ERROR; - } - case IDataLoaderStatusListener.STREAM_STORAGE_ERROR: { + switch (mStorageHealthStatus) { + case HEALTH_STATUS_UNHEALTHY_STORAGE: return PackageManager.UNSTARTABLE_REASON_INSUFFICIENT_STORAGE; - } + case HEALTH_STATUS_UNHEALTHY_TRANSPORT: + return PackageManager.UNSTARTABLE_REASON_CONNECTION_ERROR; default: return PackageManager.UNSTARTABLE_REASON_UNKNOWN; } @@ -464,7 +417,6 @@ public final class IncrementalStates { } IncrementalStates l = (IncrementalStates) o; return l.mStorageHealthStatus == mStorageHealthStatus - && l.mStreamStatus == mStreamStatus && l.mStartableState.equals(mStartableState) && l.mLoadingState.equals(mLoadingState); } @@ -474,7 +426,6 @@ public final class IncrementalStates { int hashCode = mStartableState.hashCode(); hashCode = 31 * hashCode + mLoadingState.hashCode(); hashCode = 31 * hashCode + mStorageHealthStatus; - hashCode = 31 * hashCode + mStreamStatus; return hashCode; } } diff --git a/services/core/java/com/android/server/pm/PackageInstallerSession.java b/services/core/java/com/android/server/pm/PackageInstallerSession.java index 649cafb1cb06..5d2928e1a854 100644 --- a/services/core/java/com/android/server/pm/PackageInstallerSession.java +++ b/services/core/java/com/android/server/pm/PackageInstallerSession.java @@ -143,7 +143,6 @@ import com.android.internal.util.ArrayUtils; import com.android.internal.util.FrameworkStatsLog; import com.android.internal.util.IndentingPrintWriter; import com.android.internal.util.Preconditions; -import com.android.internal.util.function.pooled.PooledLambda; import com.android.server.LocalServices; import com.android.server.pm.Installer.InstallerException; import com.android.server.pm.dex.DexManager; @@ -1702,28 +1701,6 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub { dispatchSessionFinished(error, detailedMessage, null); } - private void onStorageHealthStatusChanged(int status) { - final String packageName = getPackageName(); - if (TextUtils.isEmpty(packageName)) { - // The package has not been installed. - return; - } - mHandler.post(PooledLambda.obtainRunnable( - PackageManagerService::onStorageHealthStatusChanged, - mPm, packageName, status, userId).recycleOnUse()); - } - - private void onStreamHealthStatusChanged(int status) { - final String packageName = getPackageName(); - if (TextUtils.isEmpty(packageName)) { - // The package has not been installed. - return; - } - mHandler.post(PooledLambda.obtainRunnable( - PackageManagerService::onStreamStatusChanged, - mPm, packageName, status, userId).recycleOnUse()); - } - /** * If session should be sealed, then it's sealed to prevent further modification. * If the session can't be sealed then it's destroyed. @@ -3315,19 +3292,11 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub { return; } - final boolean isDestroyedOrDataLoaderFinished; synchronized (mLock) { - isDestroyedOrDataLoaderFinished = mDestroyed || mDataLoaderFinished; - } - if (isDestroyedOrDataLoaderFinished) { - switch (status) { - case IDataLoaderStatusListener.DATA_LOADER_UNRECOVERABLE: - // treat as unhealthy storage - onStorageHealthStatusChanged( - IStorageHealthListener.HEALTH_STATUS_UNHEALTHY); - return; + if (mDestroyed || mDataLoaderFinished) { + // No need to worry about post installation + return; } - return; } try { @@ -3423,13 +3392,10 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub { } @Override public void reportStreamHealth(int dataLoaderId, int streamStatus) { - synchronized (mLock) { - if (!mDestroyed && !mDataLoaderFinished) { - // ignore streaming status if package isn't installed - return; - } - } - onStreamHealthStatusChanged(streamStatus); + // Currently the stream status is not used during package installation. It is + // technically possible for the data loader to report stream status via this + // callback, but if something is wrong with the streaming, it is more likely that + // prepareDataLoaderLocked will return false and the installation will be aborted. } }; @@ -3438,20 +3404,16 @@ public class PackageInstallerSession extends IPackageInstallerSession.Stub { healthCheckParams.blockedTimeoutMs = INCREMENTAL_STORAGE_BLOCKED_TIMEOUT_MS; healthCheckParams.unhealthyTimeoutMs = INCREMENTAL_STORAGE_UNHEALTHY_TIMEOUT_MS; healthCheckParams.unhealthyMonitoringMs = INCREMENTAL_STORAGE_UNHEALTHY_MONITORING_MS; - final boolean systemDataLoader = params.getComponentName().getPackageName() == SYSTEM_DATA_LOADER_PACKAGE; final IStorageHealthListener healthListener = new IStorageHealthListener.Stub() { @Override public void onHealthStatus(int storageId, int status) { - final boolean isDestroyedOrDataLoaderFinished; synchronized (mLock) { - isDestroyedOrDataLoaderFinished = mDestroyed || mDataLoaderFinished; - } - if (isDestroyedOrDataLoaderFinished) { - // App's installed. - onStorageHealthStatusChanged(status); - return; + if (mDestroyed || mDataLoaderFinished) { + // No need to worry about post installation + return; + } } switch (status) { diff --git a/services/core/java/com/android/server/pm/PackageManagerService.java b/services/core/java/com/android/server/pm/PackageManagerService.java index ff84e2ebfa51..2ea800897c01 100644 --- a/services/core/java/com/android/server/pm/PackageManagerService.java +++ b/services/core/java/com/android/server/pm/PackageManagerService.java @@ -271,8 +271,10 @@ import android.os.Trace; import android.os.UserHandle; import android.os.UserManager; import android.os.UserManagerInternal; +import android.os.incremental.IStorageHealthListener; import android.os.incremental.IncrementalManager; import android.os.incremental.IncrementalStorage; +import android.os.incremental.StorageHealthCheckParams; import android.os.storage.DiskInfo; import android.os.storage.IStorageManager; import android.os.storage.StorageEventListener; @@ -732,6 +734,14 @@ public class PackageManagerService extends IPackageManager.Stub private static final String RANDOM_DIR_PREFIX = "~~"; + /** + * Timeout configurations for incremental storage health monitor. + * See {@link IStorageHealthListener} + */ + private static final int INCREMENTAL_STORAGE_BLOCKED_TIMEOUT_MS = 2000; + private static final int INCREMENTAL_STORAGE_UNHEALTHY_TIMEOUT_MS = 7000; + private static final int INCREMENTAL_STORAGE_UNHEALTHY_MONITORING_MS = 60000; + final ServiceThread mHandlerThread; final Handler mHandler; @@ -9696,6 +9706,18 @@ public class PackageManagerService extends IPackageManager.Stub mSettings.disableSystemPackageLPw(parsedPackage.getPackageName(), true); } } + if (mIncrementalManager != null && isIncrementalPath(parsedPackage.getPath())) { + if (pkgSetting != null && pkgSetting.isPackageLoading()) { + final StorageHealthCheckParams healthCheckParams = new StorageHealthCheckParams(); + healthCheckParams.blockedTimeoutMs = INCREMENTAL_STORAGE_BLOCKED_TIMEOUT_MS; + healthCheckParams.unhealthyTimeoutMs = INCREMENTAL_STORAGE_UNHEALTHY_TIMEOUT_MS; + healthCheckParams.unhealthyMonitoringMs = + INCREMENTAL_STORAGE_UNHEALTHY_MONITORING_MS; + mIncrementalManager.registerHealthListener(parsedPackage.getPath(), + healthCheckParams, + new IncrementalHealthListener(parsedPackage.getPackageName())); + } + } return scanResult.pkgSetting.pkg; } @@ -16356,12 +16378,25 @@ public class PackageManagerService extends IPackageManager.Stub // TODO(b/169721400): generalize Incremental States and create a Callback object // that can be used for all the packages. - final IncrementalStatesCallback incrementalStatesCallback = - new IncrementalStatesCallback(ps, userId); final String codePath = ps.getPathString(); if (IncrementalManager.isIncrementalPath(codePath) && mIncrementalManager != null) { - mIncrementalManager.registerCallback(codePath, incrementalStatesCallback); + final IncrementalStatesCallback incrementalStatesCallback = + new IncrementalStatesCallback(ps.name, + UserHandle.getUid(userId, ps.appId), + getInstalledUsers(ps, userId)); ps.setIncrementalStatesCallback(incrementalStatesCallback); + mIncrementalManager.registerLoadingProgressCallback(codePath, + new IncrementalProgressListener(ps.name)); + final IncrementalHealthListener incrementalHealthListener = + new IncrementalHealthListener(ps.name); + final StorageHealthCheckParams healthCheckParams = + new StorageHealthCheckParams(); + healthCheckParams.blockedTimeoutMs = INCREMENTAL_STORAGE_BLOCKED_TIMEOUT_MS; + healthCheckParams.unhealthyTimeoutMs = INCREMENTAL_STORAGE_UNHEALTHY_TIMEOUT_MS; + healthCheckParams.unhealthyMonitoringMs = + INCREMENTAL_STORAGE_UNHEALTHY_MONITORING_MS; + mIncrementalManager.registerHealthListener(codePath, + new StorageHealthCheckParams(), incrementalHealthListener); } // Ensure that the uninstall reason is UNKNOWN for users with the package installed. @@ -17264,45 +17299,39 @@ public class PackageManagerService extends IPackageManager.Stub NativeLibraryHelper.waitForNativeBinariesExtraction(incrementalStorages); } - private class IncrementalStatesCallback extends IPackageLoadingProgressCallback.Stub - implements IncrementalStates.Callback { - @GuardedBy("mPackageSetting") - private final PackageSetting mPackageSetting; - private final String mPackageName; - private final String mPathString; - private final int mUid; - private final int[] mInstalledUserIds; - - IncrementalStatesCallback(PackageSetting packageSetting, int userId) { - mPackageSetting = packageSetting; - mPackageName = packageSetting.name; - mUid = UserHandle.getUid(userId, packageSetting.appId); - mPathString = packageSetting.getPathString(); - final int[] allUserIds = resolveUserIds(userId); - final ArrayList<Integer> installedUserIds = new ArrayList<>(); - for (int i = 0; i < allUserIds.length; i++) { - if (packageSetting.getInstalled(allUserIds[i])) { - installedUserIds.add(allUserIds[i]); - } - } - final int numInstalledUserId = installedUserIds.size(); - mInstalledUserIds = new int[numInstalledUserId]; - for (int i = 0; i < numInstalledUserId; i++) { - mInstalledUserIds[i] = installedUserIds.get(i); + private int[] getInstalledUsers(PackageSetting ps, int userId) { + final int[] allUserIds = resolveUserIds(userId); + final ArrayList<Integer> installedUserIdsList = new ArrayList<>(); + for (int i = 0; i < allUserIds.length; i++) { + if (ps.getInstalled(allUserIds[i])) { + installedUserIdsList.add(allUserIds[i]); } } + final int numInstalledUserId = installedUserIdsList.size(); + final int[] installedUserIds = new int[numInstalledUserId]; + for (int i = 0; i < numInstalledUserId; i++) { + installedUserIds[i] = installedUserIdsList.get(i); + } + return installedUserIds; + } - @Override - public void onPackageLoadingProgressChanged(float progress) { - synchronized (mPackageSetting) { - mPackageSetting.setLoadingProgress(progress); - } + /** + * Package states callback, used to listen for package state changes and send broadcasts + */ + private final class IncrementalStatesCallback implements IncrementalStates.Callback { + private final String mPackageName; + private final int mUid; + private final int[] mInstalledUserIds; + IncrementalStatesCallback(String packageName, int uid, int[] installedUserIds) { + mPackageName = packageName; + mUid = uid; + mInstalledUserIds = installedUserIds; } @Override public void onPackageFullyLoaded() { - mIncrementalManager.unregisterCallback(mPathString, this); final SparseArray<int[]> newBroadcastAllowList; + final String codePath; synchronized (mLock) { final PackageSetting ps = mSettings.mPackages.get(mPackageName); if (ps == null) { @@ -17310,6 +17339,7 @@ public class PackageManagerService extends IPackageManager.Stub } newBroadcastAllowList = mAppsFilter.getVisibilityAllowList( ps, mInstalledUserIds, mSettings.mPackages); + codePath = ps.getPathString(); } Bundle extras = new Bundle(); extras.putInt(Intent.EXTRA_UID, mUid); @@ -17318,6 +17348,8 @@ public class PackageManagerService extends IPackageManager.Stub extras, 0 /*flags*/, null /*targetPackage*/, null /*finishedReceiver*/, mInstalledUserIds, null /* instantUserIds */, newBroadcastAllowList); + // Unregister health listener as it will always be healthy from now + mIncrementalManager.unregisterHealthListener(codePath); } @Override @@ -17365,37 +17397,48 @@ public class PackageManagerService extends IPackageManager.Stub } /** - * This is an internal method that is used to indicate changes on the health status of the - * Incremental Storage used by an installed package with an associated user id. This might - * result in a change in the loading state of the package. + * Loading progress callback, used to listen for progress changes and update package setting */ - public void onStorageHealthStatusChanged(String packageName, int status, int userId) { - final int callingUid = Binder.getCallingUid(); - mPermissionManager.enforceCrossUserPermission( - callingUid, userId, true, false, - "onStorageHealthStatusChanged"); - final PackageSetting ps = getPackageSettingForUser(packageName, callingUid, userId); - if (ps == null) { - return; + private class IncrementalProgressListener extends IPackageLoadingProgressCallback.Stub { + private final String mPackageName; + IncrementalProgressListener(String packageName) { + mPackageName = packageName; + } + + @Override + public void onPackageLoadingProgressChanged(float progress) { + final PackageSetting ps; + synchronized (mLock) { + ps = mSettings.mPackages.get(mPackageName); + } + if (ps == null) { + return; + } + ps.setLoadingProgress(progress); } - ps.setStorageHealthStatus(status); } /** - * This is an internal method that is used to indicate changes on the stream status of the - * data loader used by an installed package with an associated user id. This might - * result in a change in the loading state of the package. + * Incremental storage health status callback, used to listen for monitoring changes and update + * package setting. */ - public void onStreamStatusChanged(String packageName, int status, int userId) { - final int callingUid = Binder.getCallingUid(); - mPermissionManager.enforceCrossUserPermission( - callingUid, userId, true, false, - "onStreamStatusChanged"); - final PackageSetting ps = getPackageSettingForUser(packageName, callingUid, userId); - if (ps == null) { - return; + private class IncrementalHealthListener extends IStorageHealthListener.Stub { + private final String mPackageName; + IncrementalHealthListener(String packageName) { + mPackageName = packageName; + } + + @Override + public void onHealthStatus(int storageId, int status) throws RemoteException { + final PackageSetting ps; + synchronized (mLock) { + ps = mSettings.mPackages.get(mPackageName); + } + if (ps == null) { + return; + } + ps.setStorageHealthStatus(status); } - ps.setStreamStatus(status); } @Nullable PackageSetting getPackageSettingForUser(String packageName, int callingUid, @@ -25637,7 +25680,7 @@ public class PackageManagerService extends IPackageManager.Stub "Failed registering loading progress callback. Incremental is not enabled"); return false; } - return mIncrementalManager.registerCallback(ps.getPathString(), + return mIncrementalManager.registerLoadingProgressCallback(ps.getPathString(), (IPackageLoadingProgressCallback) callback.getBinder()); } @@ -25656,7 +25699,7 @@ public class PackageManagerService extends IPackageManager.Stub if (mIncrementalManager == null) { return false; } - return mIncrementalManager.unregisterCallback(ps.getPathString(), + return mIncrementalManager.unregisterLoadingProgressCallback(ps.getPathString(), (IPackageLoadingProgressCallback) callback.getBinder()); } } diff --git a/services/core/java/com/android/server/pm/PackageSettingBase.java b/services/core/java/com/android/server/pm/PackageSettingBase.java index d52ad46d4b7e..be7c7c6ff1d6 100644 --- a/services/core/java/com/android/server/pm/PackageSettingBase.java +++ b/services/core/java/com/android/server/pm/PackageSettingBase.java @@ -793,13 +793,6 @@ public abstract class PackageSettingBase extends SettingBase { incrementalStates.onStorageHealthStatusChanged(status); } - /** - * @see IncrementalStates#onStreamStatusChanged(int) - */ - public void setStreamStatus(int status) { - incrementalStates.onStreamStatusChanged(status); - } - protected PackageSettingBase updateFrom(PackageSettingBase other) { super.copyFrom(other); setPath(other.getPath()); diff --git a/services/incremental/BinderIncrementalService.cpp b/services/incremental/BinderIncrementalService.cpp index 2f8825b064ce..a31aac96eb48 100644 --- a/services/incremental/BinderIncrementalService.cpp +++ b/services/incremental/BinderIncrementalService.cpp @@ -323,6 +323,22 @@ binder::Status BinderIncrementalService::unregisterLoadingProgressListener(int32 return ok(); } +binder::Status BinderIncrementalService::registerStorageHealthListener( + int32_t storageId, + const ::android::os::incremental::StorageHealthCheckParams& healthCheckParams, + const ::android::sp<IStorageHealthListener>& healthListener, bool* _aidl_return) { + *_aidl_return = mImpl.registerStorageHealthListener(storageId, + const_cast<StorageHealthCheckParams&&>( + healthCheckParams), + healthListener); + return ok(); +} + +binder::Status BinderIncrementalService::unregisterStorageHealthListener(int32_t storageId) { + mImpl.unregisterStorageHealthListener(storageId); + return ok(); +} + } // namespace android::os::incremental jlong Incremental_IncrementalService_Start(JNIEnv* env) { diff --git a/services/incremental/BinderIncrementalService.h b/services/incremental/BinderIncrementalService.h index 0a89166f4868..8afa0f7bb117 100644 --- a/services/incremental/BinderIncrementalService.h +++ b/services/incremental/BinderIncrementalService.h @@ -89,6 +89,11 @@ public: progressListener, bool* _aidl_return) final; binder::Status unregisterLoadingProgressListener(int32_t storageId, bool* _aidl_return) final; + binder::Status registerStorageHealthListener( + int32_t storageId, + const ::android::os::incremental::StorageHealthCheckParams& healthCheckParams, + const ::android::sp<IStorageHealthListener>& healthListener, bool* _aidl_return) final; + binder::Status unregisterStorageHealthListener(int32_t storageId) final; private: android::incremental::IncrementalService mImpl; diff --git a/services/incremental/IncrementalService.cpp b/services/incremental/IncrementalService.cpp index 5f145f33f628..599ac9344e73 100644 --- a/services/incremental/IncrementalService.cpp +++ b/services/incremental/IncrementalService.cpp @@ -1801,6 +1801,31 @@ bool IncrementalService::unregisterLoadingProgressListener(StorageId storage) { return removeTimedJobs(*mProgressUpdateJobQueue, storage); } +bool IncrementalService::registerStorageHealthListener( + StorageId storage, StorageHealthCheckParams&& healthCheckParams, + const StorageHealthListener& healthListener) { + DataLoaderStubPtr dataLoaderStub; + { + std::unique_lock l(mLock); + const auto& ifs = getIfsLocked(storage); + if (!ifs) { + return false; + } + dataLoaderStub = ifs->dataLoaderStub; + if (!dataLoaderStub) { + return false; + } + } + dataLoaderStub->setHealthListener(std::move(healthCheckParams), &healthListener); + return true; +} + +void IncrementalService::unregisterStorageHealthListener(StorageId storage) { + StorageHealthCheckParams invalidCheckParams; + invalidCheckParams.blockedTimeoutMs = -1; + registerStorageHealthListener(storage, std::move(invalidCheckParams), {}); +} + bool IncrementalService::perfLoggingEnabled() { static const bool enabled = base::GetBoolProperty("incremental.perflogging", false); return enabled; @@ -2137,6 +2162,19 @@ binder::Status IncrementalService::DataLoaderStub::onStatusChanged(MountId mount binder::Status IncrementalService::DataLoaderStub::reportStreamHealth(MountId mountId, int newStatus) { + if (!isValid()) { + return binder::Status:: + fromServiceSpecificError(-EINVAL, + "reportStreamHealth came to invalid DataLoaderStub"); + } + if (id() != mountId) { + LOG(ERROR) << "Mount ID mismatch: expected " << id() << ", but got: " << mountId; + return binder::Status::fromServiceSpecificError(-EPERM, "Mount ID mismatch."); + } + { + std::lock_guard lock(mMutex); + mStreamStatus = newStatus; + } return binder::Status::ok(); } @@ -2153,6 +2191,33 @@ void IncrementalService::DataLoaderStub::onHealthStatus(StorageHealthListener he } } +static int adjustHealthStatus(int healthStatus, int streamStatus) { + if (healthStatus == IStorageHealthListener::HEALTH_STATUS_OK) { + // everything is good; no need to change status + return healthStatus; + } + int newHeathStatus = healthStatus; + switch (streamStatus) { + case IDataLoaderStatusListener::STREAM_STORAGE_ERROR: + // storage is limited and storage not healthy + newHeathStatus = IStorageHealthListener::HEALTH_STATUS_UNHEALTHY_STORAGE; + break; + case IDataLoaderStatusListener::STREAM_INTEGRITY_ERROR: + // fall through + case IDataLoaderStatusListener::STREAM_SOURCE_ERROR: + // fall through + case IDataLoaderStatusListener::STREAM_TRANSPORT_ERROR: + if (healthStatus == IStorageHealthListener::HEALTH_STATUS_UNHEALTHY) { + newHeathStatus = IStorageHealthListener::HEALTH_STATUS_UNHEALTHY_TRANSPORT; + } + // pending/blocked status due to transportation issues is not regarded as unhealthy + break; + default: + break; + } + return newHeathStatus; +} + void IncrementalService::DataLoaderStub::updateHealthStatus(bool baseline) { LOG(DEBUG) << id() << ": updateHealthStatus" << (baseline ? " (baseline)" : ""); @@ -2232,6 +2297,8 @@ void IncrementalService::DataLoaderStub::updateHealthStatus(bool baseline) { checkBackAfter = unhealthyMonitoring; healthStatusToReport = IStorageHealthListener::HEALTH_STATUS_UNHEALTHY; } + // Adjust health status based on stream status + healthStatusToReport = adjustHealthStatus(healthStatusToReport, mStreamStatus); LOG(DEBUG) << id() << ": updateHealthStatus in " << double(checkBackAfter.count()) / 1000.0 << "secs"; mService.addTimedJob(*mService.mTimedQueue, id(), checkBackAfter, @@ -2321,6 +2388,18 @@ void IncrementalService::DataLoaderStub::unregisterFromPendingReads() { mService.mLooper->wake(); } +void IncrementalService::DataLoaderStub::setHealthListener( + StorageHealthCheckParams&& healthCheckParams, const StorageHealthListener* healthListener) { + std::lock_guard lock(mMutex); + mHealthCheckParams = std::move(healthCheckParams); + if (healthListener == nullptr) { + // reset listener and params + mHealthListener = {}; + } else { + mHealthListener = *healthListener; + } +} + void IncrementalService::DataLoaderStub::onDump(int fd) { dprintf(fd, " dataLoader: {\n"); dprintf(fd, " currentStatus: %d\n", mCurrentStatus); diff --git a/services/incremental/IncrementalService.h b/services/incremental/IncrementalService.h index 504c02a57b86..4c4b8bd1ba50 100644 --- a/services/incremental/IncrementalService.h +++ b/services/incremental/IncrementalService.h @@ -140,7 +140,10 @@ public: bool registerLoadingProgressListener(StorageId storage, const StorageLoadingProgressListener& progressListener); bool unregisterLoadingProgressListener(StorageId storage); - + bool registerStorageHealthListener(StorageId storage, + StorageHealthCheckParams&& healthCheckParams, + const StorageHealthListener& healthListener); + void unregisterStorageHealthListener(StorageId storage); RawMetadata getMetadata(StorageId storage, std::string_view path) const; RawMetadata getMetadata(StorageId storage, FileId node) const; @@ -197,6 +200,8 @@ private: MountId id() const { return mId.load(std::memory_order_relaxed); } const content::pm::DataLoaderParamsParcel& params() const { return mParams; } + void setHealthListener(StorageHealthCheckParams&& healthCheckParams, + const StorageHealthListener* healthListener); private: binder::Status onStatusChanged(MountId mount, int newStatus) final; @@ -251,6 +256,7 @@ private: BootClockTsUs kernelTsUs; } mHealthBase = {TimePoint::max(), kMaxBootClockTsUs}; StorageHealthCheckParams mHealthCheckParams; + int mStreamStatus = content::pm::IDataLoaderStatusListener::STREAM_HEALTHY; }; using DataLoaderStubPtr = sp<DataLoaderStub>; diff --git a/services/incremental/test/IncrementalServiceTest.cpp b/services/incremental/test/IncrementalServiceTest.cpp index aec9fa1c3277..867312e0eb2f 100644 --- a/services/incremental/test/IncrementalServiceTest.cpp +++ b/services/incremental/test/IncrementalServiceTest.cpp @@ -177,6 +177,18 @@ public: } return binder::Status::ok(); } + binder::Status storageError(int32_t id) { + if (mListener) { + mListener->reportStreamHealth(id, IDataLoaderStatusListener::STREAM_STORAGE_ERROR); + } + return binder::Status::ok(); + } + binder::Status transportError(int32_t id) { + if (mListener) { + mListener->reportStreamHealth(id, IDataLoaderStatusListener::STREAM_INTEGRITY_ERROR); + } + return binder::Status::ok(); + } int32_t setStorageParams(bool enableReadLogs) { int32_t result = -1; EXPECT_NE(mServiceConnector.get(), nullptr); @@ -1221,4 +1233,83 @@ TEST_F(IncrementalServiceTest, testRegisterLoadingProgressListenerFailsToGetProg EXPECT_CALL(*listenerMock, onStorageLoadingProgressChanged(_, _)).Times(0); mIncrementalService->registerLoadingProgressListener(storageId, listener); } + +TEST_F(IncrementalServiceTest, testRegisterStorageHealthListenerSuccess) { + mIncFs->openMountSuccess(); + sp<NiceMock<MockStorageHealthListener>> listener{new NiceMock<MockStorageHealthListener>}; + sp<NiceMock<MockStorageHealthListener>> newListener{new NiceMock<MockStorageHealthListener>}; + NiceMock<MockStorageHealthListener>* newListenerMock = newListener.get(); + + TemporaryDir tempDir; + int storageId = mIncrementalService->createStorage(tempDir.path, std::move(mDataLoaderParcel), + IncrementalService::CreateOptions::CreateNew, + {}, StorageHealthCheckParams{}, listener); + ASSERT_GE(storageId, 0); + StorageHealthCheckParams newParams; + newParams.blockedTimeoutMs = 10000; + newParams.unhealthyTimeoutMs = 20000; + newParams.unhealthyMonitoringMs = 30000; + ASSERT_TRUE(mIncrementalService->registerStorageHealthListener(storageId, std::move(newParams), + newListener)); + + using MS = std::chrono::milliseconds; + using MCS = std::chrono::microseconds; + + const auto blockedTimeout = MS(newParams.blockedTimeoutMs); + const auto unhealthyTimeout = MS(newParams.unhealthyTimeoutMs); + + const uint64_t kFirstTimestampUs = 1000000000ll; + const uint64_t kBlockedTimestampUs = + kFirstTimestampUs - std::chrono::duration_cast<MCS>(blockedTimeout).count(); + const uint64_t kUnhealthyTimestampUs = + kFirstTimestampUs - std::chrono::duration_cast<MCS>(unhealthyTimeout).count(); + + // test that old listener was not called + EXPECT_CALL(*listener.get(), + onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_READS_PENDING)) + .Times(0); + EXPECT_CALL(*newListenerMock, + onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_READS_PENDING)) + .Times(1); + EXPECT_CALL(*newListenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_BLOCKED)) + .Times(1); + EXPECT_CALL(*newListenerMock, + onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_UNHEALTHY_STORAGE)) + .Times(1); + EXPECT_CALL(*newListenerMock, + onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_UNHEALTHY_TRANSPORT)) + .Times(1); + mIncFs->waitForPendingReadsSuccess(kFirstTimestampUs); + mLooper->mCallback(-1, -1, mLooper->mCallbackData); + + ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_READS_PENDING, newListener->mStatus); + ASSERT_EQ(storageId, newListener->mStorageId); + + auto timedCallback = mTimedQueue->mWhat; + mTimedQueue->clearJob(storageId); + + // test when health status is blocked with transport error + mDataLoader->transportError(storageId); + mIncFs->waitForPendingReadsSuccess(kBlockedTimestampUs); + timedCallback(); + ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_BLOCKED, newListener->mStatus); + timedCallback = mTimedQueue->mWhat; + mTimedQueue->clearJob(storageId); + + // test when health status is blocked with storage error + mDataLoader->storageError(storageId); + mIncFs->waitForPendingReadsSuccess(kBlockedTimestampUs); + timedCallback(); + ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY_STORAGE, newListener->mStatus); + timedCallback = mTimedQueue->mWhat; + mTimedQueue->clearJob(storageId); + + // test when health status is unhealthy with transport error + mDataLoader->transportError(storageId); + mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs); + timedCallback(); + ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY_TRANSPORT, newListener->mStatus); + mTimedQueue->clearJob(storageId); +} + } // namespace android::os::incremental diff --git a/services/tests/servicestests/src/com/android/server/pm/IncrementalStatesTest.java b/services/tests/servicestests/src/com/android/server/pm/IncrementalStatesTest.java index c4c2f68e8219..86758f18a407 100644 --- a/services/tests/servicestests/src/com/android/server/pm/IncrementalStatesTest.java +++ b/services/tests/servicestests/src/com/android/server/pm/IncrementalStatesTest.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import android.content.pm.IDataLoaderStatusListener; import android.content.pm.PackageManager; import android.os.ConditionVariable; import android.os.incremental.IStorageHealthListener; @@ -113,71 +112,12 @@ public class IncrementalStatesTest { } /** - * Test that the package is still startable when Incremental Storage is at blocked status. + * Test that the package becomes unstartable when health status indicate storage issues. */ @Test public void testStartableTransition_IncrementalStorageBlocked() { mIncrementalStates.onStorageHealthStatusChanged( - IStorageHealthListener.HEALTH_STATUS_BLOCKED); - // Test that package is still startable - assertFalse(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertTrue(mIncrementalStates.isStartable()); - } - - /** - * Test that the package is still startable when Data Loader has unknown transportation issues. - */ - @Test - public void testStartableTransition_DataLoaderTransportError() { - mIncrementalStates.onStreamStatusChanged( - IDataLoaderStatusListener.STREAM_TRANSPORT_ERROR); - // Test that package is still startable - assertFalse(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertTrue(mIncrementalStates.isStartable()); - } - - /** - * Test that the package becomes unstartable when Data Loader has data integrity issues. - */ - @Test - public void testStartableTransition_DataLoaderIntegrityError() { - mIncrementalStates.onStreamStatusChanged( - IDataLoaderStatusListener.STREAM_INTEGRITY_ERROR); - // Test that package is now unstartable - assertTrue(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertFalse(mIncrementalStates.isStartable()); - assertEquals(PackageManager.UNSTARTABLE_REASON_CONNECTION_ERROR, - mUnstartableReason.get()); - } - - /** - * Test that the package becomes unstartable when Data Loader has data source issues. - */ - @Test - public void testStartableTransition_DataLoaderSourceError() { - mIncrementalStates.onStreamStatusChanged( - IDataLoaderStatusListener.STREAM_SOURCE_ERROR); - // Test that package is now unstartable - assertTrue(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertFalse(mIncrementalStates.isStartable()); - assertEquals(PackageManager.UNSTARTABLE_REASON_CONNECTION_ERROR, - mUnstartableReason.get()); - } - - /** - * Test that the package becomes unstartable when Data Loader hits limited storage while - * Incremental storage has a pending reads. - */ - @Test - public void testStartableTransition_DataLoaderStorageErrorWhenIncrementalStoragePending() - throws InterruptedException { - mIncrementalStates.onStreamStatusChanged( - IDataLoaderStatusListener.STREAM_STORAGE_ERROR); - // Test that package is still startable - assertFalse(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertTrue(mIncrementalStates.isStartable()); - mIncrementalStates.onStorageHealthStatusChanged( - IStorageHealthListener.HEALTH_STATUS_READS_PENDING); + IStorageHealthListener.HEALTH_STATUS_UNHEALTHY_STORAGE); // Test that package is now unstartable assertTrue(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); assertFalse(mIncrementalStates.isStartable()); @@ -186,23 +126,16 @@ public class IncrementalStatesTest { } /** - * Test that the package becomes unstartable when Data Loader hits limited storage while - * Incremental storage is at blocked status. + * Test that the package becomes unstartable when health status indicates transport issues. */ @Test - public void testStartableTransition_DataLoaderStorageErrorWhenIncrementalStorageBlocked() - throws InterruptedException { - mIncrementalStates.onStreamStatusChanged( - IDataLoaderStatusListener.STREAM_STORAGE_ERROR); - // Test that package is still startable - assertFalse(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertTrue(mIncrementalStates.isStartable()); + public void testStartableTransition_DataLoaderIntegrityError() { mIncrementalStates.onStorageHealthStatusChanged( - IStorageHealthListener.HEALTH_STATUS_BLOCKED); + IStorageHealthListener.HEALTH_STATUS_UNHEALTHY_TRANSPORT); // Test that package is now unstartable assertTrue(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); assertFalse(mIncrementalStates.isStartable()); - assertEquals(PackageManager.UNSTARTABLE_REASON_INSUFFICIENT_STORAGE, + assertEquals(PackageManager.UNSTARTABLE_REASON_CONNECTION_ERROR, mUnstartableReason.get()); } @@ -227,42 +160,18 @@ public class IncrementalStatesTest { } /** - * Test that the package becomes unstartable when Data Loader has data integrity issue, and it - * becomes startable again when Data Loader is healthy again. + * Test that the package becomes unstartable when health status indicates transportation issue, + * and it becomes startable again when health status is ok again. */ @Test public void testStartableTransition_DataLoaderUnhealthyBackToHealthy() throws InterruptedException { - mIncrementalStates.onStreamStatusChanged(IDataLoaderStatusListener.STREAM_INTEGRITY_ERROR); - // Test that package is unstartable - assertTrue(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertFalse(mIncrementalStates.isStartable()); - - mIncrementalStates.onStreamStatusChanged(IDataLoaderStatusListener.STREAM_HEALTHY); - // Test that package is now startable - assertTrue(mStartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertTrue(mIncrementalStates.isStartable()); - } - - /** - * Test that the package becomes unstartable when both Incremental Storage and Data Loader - * are unhealthy, and it becomes startable again when both Incremental Storage and Data Loader - * are healthy again. - */ - @Test - public void testStartableTransition_DataLoaderAndIncrementalStorageUnhealthyBackToHealthy() - throws InterruptedException { mIncrementalStates.onStorageHealthStatusChanged( - IStorageHealthListener.HEALTH_STATUS_UNHEALTHY); - mIncrementalStates.onStreamStatusChanged(IDataLoaderStatusListener.STREAM_INTEGRITY_ERROR); + IStorageHealthListener.HEALTH_STATUS_UNHEALTHY_TRANSPORT); // Test that package is unstartable assertTrue(mUnstartableCalled.block(WAIT_TIMEOUT_MILLIS)); assertFalse(mIncrementalStates.isStartable()); - mIncrementalStates.onStreamStatusChanged(IDataLoaderStatusListener.STREAM_HEALTHY); - // Test that package is still unstartable - assertFalse(mStartableCalled.block(WAIT_TIMEOUT_MILLIS)); - assertFalse(mIncrementalStates.isStartable()); mIncrementalStates.onStorageHealthStatusChanged(IStorageHealthListener.HEALTH_STATUS_OK); // Test that package is now startable assertTrue(mStartableCalled.block(WAIT_TIMEOUT_MILLIS)); |