DataLoader lifecycle.

- restarting DataLoaders for not fully downloaded mounts,
- exponential backoff retry on DataLoader restart.

Bug: 173223115
Bug: 160634487
Test: atest PackageManagerShellCommandTest PackageManagerShellCommandIncrementalTest IncrementalServiceTest PackageManagerServiceTest ChecksumsTest
Change-Id: I5a93fd515ef0fec09452ce7bb4cd79fbd283d45f

5 files changed, 190 insertions, 25 deletions

diff --git a/services/incremental/IncrementalService.cpp b/services/incremental/IncrementalService.cpp
index 56cb3d11d552..886c1e5b98ea 100644
--- a/services/incremental/IncrementalService.cpp
+++ b/services/incremental/IncrementalService.cpp
@@ -69,6 +69,14 @@ struct Constants {
     static constexpr auto progressUpdateInterval = 1000ms;
     static constexpr auto perUidTimeoutOffset = progressUpdateInterval * 2;
     static constexpr auto minPerUidTimeout = progressUpdateInterval * 3;
+
+    // If DL was up and not crashing for 10mins, we consider it healthy and reset all delays.
+    static constexpr auto healthyDataLoaderUptime = 10min;
+    // 10s, 100s (~2min), 1000s (~15min), 10000s (~3hrs)
+    static constexpr auto minBindDelay = 10s;
+    static constexpr auto maxBindDelay = 10000s;
+    static constexpr auto bindDelayMultiplier = 10;
+    static constexpr auto bindDelayJitterDivider = 10;
 };
 
 static const Constants& constants() {
@@ -386,6 +394,28 @@ void IncrementalService::onDump(int fd) {
     dprintf(fd, "}\n");
 }
 
+bool IncrementalService::needStartDataLoaderLocked(IncFsMount& ifs) {
+    if (ifs.dataLoaderStub->params().packageName == Constants::systemPackage) {
+        return true;
+    }
+
+    // Check all permanent binds.
+    for (auto&& [_, bindPoint] : ifs.bindPoints) {
+        if (bindPoint.kind != BindKind::Permanent) {
+            continue;
+        }
+        const auto progress = getLoadingProgressFromPath(ifs, bindPoint.sourceDir,
+                                                         /*stopOnFirstIncomplete=*/true);
+        if (!progress.isError() && !progress.fullyLoaded()) {
+            LOG(INFO) << "Non system mount: [" << bindPoint.sourceDir
+                      << "], partial progress: " << progress.getProgress() * 100 << "%";
+            return true;
+        }
+    }
+
+    return false;
+}
+
 void IncrementalService::onSystemReady() {
     if (mSystemReady.exchange(true)) {
         return;
@@ -396,8 +426,11 @@ void IncrementalService::onSystemReady() {
         std::lock_guard l(mLock);
         mounts.reserve(mMounts.size());
         for (auto&& [id, ifs] : mMounts) {
-            if (ifs->mountId == id &&
-                ifs->dataLoaderStub->params().packageName == Constants::systemPackage) {
+            if (ifs->mountId != id) {
+                continue;
+            }
+
+            if (needStartDataLoaderLocked(*ifs)) {
                 mounts.push_back(ifs);
             }
         }
@@ -1539,6 +1572,11 @@ static long elapsedMcs(Duration start, Duration end) {
     return std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
 }
 
+template <class Duration>
+static constexpr auto castToMs(Duration d) {
+    return std::chrono::duration_cast<std::chrono::milliseconds>(d);
+}
+
 // Extract lib files from zip, create new files in incfs and write data to them
 // Lib files should be placed next to the APK file in the following matter:
 // Example:
@@ -2134,9 +2172,43 @@ void IncrementalService::DataLoaderStub::setTargetStatusLocked(int status) {
                << status << " (current " << mCurrentStatus << ")";
 }
 
+Milliseconds IncrementalService::DataLoaderStub::updateBindDelay() {
+    std::unique_lock lock(mMutex);
+    const auto previousBindTs = mPreviousBindTs;
+    const auto now = Clock::now();
+    mPreviousBindTs = now;
+
+    const auto nonCrashingInterval = std::max(castToMs(now - previousBindTs), 100ms);
+    if (previousBindTs.time_since_epoch() == Clock::duration::zero() ||
+        nonCrashingInterval > Constants::healthyDataLoaderUptime) {
+        mPreviousBindDelay = 0ms;
+        return mPreviousBindDelay;
+    }
+
+    constexpr auto minBindDelayMs = castToMs(Constants::minBindDelay);
+    constexpr auto maxBindDelayMs = castToMs(Constants::maxBindDelay);
+
+    const auto bindDelayMs =
+            std::min(std::max(mPreviousBindDelay * Constants::bindDelayMultiplier, minBindDelayMs),
+                     maxBindDelayMs)
+                    .count();
+    const auto bindDelayJitterRangeMs = bindDelayMs / Constants::bindDelayJitterDivider;
+    const auto bindDelayJitterMs = rand() % (bindDelayJitterRangeMs * 2) - bindDelayJitterRangeMs;
+    mPreviousBindDelay = std::chrono::milliseconds(bindDelayMs + bindDelayJitterMs);
+
+    return mPreviousBindDelay;
+}
+
 bool IncrementalService::DataLoaderStub::bind() {
+    const auto bindDelay = updateBindDelay();
+    if (bindDelay > 1s) {
+        LOG(INFO) << "Delaying bind to " << mParams.packageName << " by "
+                  << bindDelay.count() / 1000 << "s";
+    }
+
     bool result = false;
-    auto status = mService.mDataLoaderManager->bindToDataLoader(id(), mParams, this, &result);
+    auto status = mService.mDataLoaderManager->bindToDataLoader(id(), mParams, bindDelay.count(),
+                                                                this, &result);
     if (!status.isOk() || !result) {
         LOG(ERROR) << "Failed to bind a data loader for mount " << id();
         return false;
@@ -2249,7 +2321,8 @@ binder::Status IncrementalService::DataLoaderStub::onStatusChanged(MountId mount
 
         listener = mStatusListener;
 
-        if (mCurrentStatus == IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE) {
+        if (mCurrentStatus == IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE ||
+            mCurrentStatus == IDataLoaderStatusListener::DATA_LOADER_UNRECOVERABLE) {
             // For unavailable, unbind from DataLoader to ensure proper re-commit.
             setTargetStatusLocked(IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
         }
@@ -2544,6 +2617,9 @@ void IncrementalService::DataLoaderStub::onDump(int fd) {
         dprintf(fd, "          blockIndex: %d\n", pendingRead.block);
         dprintf(fd, "          bootClockTsUs: %lld\n", (long long)pendingRead.bootClockTsUs);
     }
+    dprintf(fd, "        bind: %llds ago (delay: %llds)\n",
+            (long long)(elapsedMcs(mPreviousBindTs, Clock::now()) / 1000000),
+            (long long)(mPreviousBindDelay.count() / 1000));
     dprintf(fd, "      }\n");
     const auto& params = mParams;
     dprintf(fd, "      dataLoaderParams: {\n");
diff --git a/services/incremental/IncrementalService.h b/services/incremental/IncrementalService.h
index 5d53bac777b5..459ed3293510 100644
--- a/services/incremental/IncrementalService.h
+++ b/services/incremental/IncrementalService.h
@@ -245,7 +245,6 @@ private:
         void setTargetStatusLocked(int status);
 
         bool fsmStep();
-        bool fsmStep(int currentStatus, int targetStatus);
 
         void onHealthStatus(StorageHealthListener healthListener, int healthStatus);
         void updateHealthStatus(bool baseline = false);
@@ -259,6 +258,8 @@ private:
 
         BootClockTsUs getOldestPendingReadTs();
 
+        Milliseconds updateBindDelay();
+
         void registerForPendingReads();
         void unregisterFromPendingReads();
 
@@ -276,6 +277,9 @@ private:
         int mTargetStatus = content::pm::IDataLoaderStatusListener::DATA_LOADER_DESTROYED;
         TimePoint mTargetStatusTs = {};
 
+        TimePoint mPreviousBindTs = {};
+        Milliseconds mPreviousBindDelay = {};
+
         std::string mHealthPath;
         incfs::UniqueControl mHealthControl;
         struct {
@@ -370,6 +374,8 @@ private:
     void addBindMountRecordLocked(IncFsMount& ifs, StorageId storage, std::string&& metadataName,
                                   std::string&& source, std::string&& target, BindKind kind);
 
+    bool needStartDataLoaderLocked(IncFsMount& ifs);
+
     DataLoaderStubPtr prepareDataLoader(IncFsMount& ifs,
                                         content::pm::DataLoaderParamsParcel&& params,
                                         const DataLoaderStatusListener* statusListener = nullptr,
diff --git a/services/incremental/ServiceWrappers.cpp b/services/incremental/ServiceWrappers.cpp
index 25d3f77fa58b..659d6503e11c 100644
--- a/services/incremental/ServiceWrappers.cpp
+++ b/services/incremental/ServiceWrappers.cpp
@@ -70,9 +70,10 @@ public:
     ~RealDataLoaderManager() = default;
     binder::Status bindToDataLoader(MountId mountId,
                                     const content::pm::DataLoaderParamsParcel& params,
+                                    int bindDelayMs,
                                     const sp<content::pm::IDataLoaderStatusListener>& listener,
                                     bool* _aidl_return) const final {
-        return mInterface->bindToDataLoader(mountId, params, listener, _aidl_return);
+        return mInterface->bindToDataLoader(mountId, params, bindDelayMs, listener, _aidl_return);
     }
     binder::Status getDataLoader(MountId mountId,
                                  sp<content::pm::IDataLoader>* _aidl_return) const final {
diff --git a/services/incremental/ServiceWrappers.h b/services/incremental/ServiceWrappers.h
index 71fd3acc6489..d60035a9274d 100644
--- a/services/incremental/ServiceWrappers.h
+++ b/services/incremental/ServiceWrappers.h
@@ -63,7 +63,7 @@ class DataLoaderManagerWrapper {
 public:
     virtual ~DataLoaderManagerWrapper() = default;
     virtual binder::Status bindToDataLoader(
-            MountId mountId, const content::pm::DataLoaderParamsParcel& params,
+            MountId mountId, const content::pm::DataLoaderParamsParcel& params, int bindDelayMs,
             const sp<content::pm::IDataLoaderStatusListener>& listener, bool* result) const = 0;
     virtual binder::Status getDataLoader(MountId mountId,
                                          sp<content::pm::IDataLoader>* result) const = 0;
diff --git a/services/incremental/test/IncrementalServiceTest.cpp b/services/incremental/test/IncrementalServiceTest.cpp
index 8713f9d3d821..ab491efe8583 100644
--- a/services/incremental/test/IncrementalServiceTest.cpp
+++ b/services/incremental/test/IncrementalServiceTest.cpp
@@ -208,8 +208,9 @@ public:
         EXPECT_TRUE(mDataLoaderHolder != nullptr);
     }
 
-    MOCK_CONST_METHOD4(bindToDataLoader,
+    MOCK_CONST_METHOD5(bindToDataLoader,
                        binder::Status(int32_t mountId, const DataLoaderParamsParcel& params,
+                                      int bindDelayMs,
                                       const sp<IDataLoaderStatusListener>& listener,
                                       bool* _aidl_return));
     MOCK_CONST_METHOD2(getDataLoader,
@@ -217,11 +218,11 @@ public:
     MOCK_CONST_METHOD1(unbindFromDataLoader, binder::Status(int32_t mountId));
 
     void bindToDataLoaderSuccess() {
-        ON_CALL(*this, bindToDataLoader(_, _, _, _))
+        ON_CALL(*this, bindToDataLoader(_, _, _, _, _))
                 .WillByDefault(Invoke(this, &MockDataLoaderManager::bindToDataLoaderOk));
     }
     void bindToDataLoaderFails() {
-        ON_CALL(*this, bindToDataLoader(_, _, _, _))
+        ON_CALL(*this, bindToDataLoader(_, _, _, _, _))
                 .WillByDefault(Return(
                         (binder::Status::fromExceptionCode(1, String8("failed to prepare")))));
     }
@@ -234,6 +235,7 @@ public:
                 .WillByDefault(Invoke(this, &MockDataLoaderManager::unbindFromDataLoaderOk));
     }
     binder::Status bindToDataLoaderOk(int32_t mountId, const DataLoaderParamsParcel& params,
+                                      int bindDelayMs,
                                       const sp<IDataLoaderStatusListener>& listener,
                                       bool* _aidl_return) {
         mId = mountId;
@@ -245,6 +247,40 @@ public:
         }
         return binder::Status::ok();
     }
+    binder::Status bindToDataLoaderOkWith10sDelay(int32_t mountId,
+                                                  const DataLoaderParamsParcel& params,
+                                                  int bindDelayMs,
+                                                  const sp<IDataLoaderStatusListener>& listener,
+                                                  bool* _aidl_return) {
+        CHECK(1000 * 9 <= bindDelayMs && bindDelayMs <= 1000 * 11) << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+    binder::Status bindToDataLoaderOkWith100sDelay(int32_t mountId,
+                                                   const DataLoaderParamsParcel& params,
+                                                   int bindDelayMs,
+                                                   const sp<IDataLoaderStatusListener>& listener,
+                                                   bool* _aidl_return) {
+        CHECK(1000 * 9 * 9 < bindDelayMs && bindDelayMs < 1000 * 11 * 11) << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+    binder::Status bindToDataLoaderOkWith1000sDelay(int32_t mountId,
+                                                    const DataLoaderParamsParcel& params,
+                                                    int bindDelayMs,
+                                                    const sp<IDataLoaderStatusListener>& listener,
+                                                    bool* _aidl_return) {
+        CHECK(1000 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 1000 * 11 * 11 * 11) << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+    binder::Status bindToDataLoaderOkWith10000sDelay(int32_t mountId,
+                                                     const DataLoaderParamsParcel& params,
+                                                     int bindDelayMs,
+                                                     const sp<IDataLoaderStatusListener>& listener,
+                                                     bool* _aidl_return) {
+        CHECK(1000 * 9 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 1000 * 11 * 11 * 11 * 11)
+                << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+
     binder::Status getDataLoaderOk(int32_t mountId, sp<IDataLoader>* _aidl_return) {
         *_aidl_return = mDataLoader;
         return binder::Status::ok();
@@ -261,6 +297,9 @@ public:
     void setDataLoaderStatusUnavailable() {
         mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE);
     }
+    void setDataLoaderStatusUnrecoverable() {
+        mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_UNRECOVERABLE);
+    }
     binder::Status unbindFromDataLoaderOk(int32_t id) {
         if (mDataLoader) {
             if (auto status = mDataLoader->destroy(id); !status.isOk()) {
@@ -676,7 +715,7 @@ protected:
 
 TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsFails) {
     mVold->mountIncFsFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     TemporaryDir tempDir;
     int storageId =
             mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
@@ -686,7 +725,7 @@ TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsFails) {
 
 TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsInvalidControlParcel) {
     mVold->mountIncFsInvalidControlParcel();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     TemporaryDir tempDir;
     int storageId =
@@ -698,7 +737,7 @@ TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsInvalidControlParcel)
 TEST_F(IncrementalServiceTest, testCreateStorageMakeFileFails) {
     mVold->mountIncFsSuccess();
     mIncFs->makeFileFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     EXPECT_CALL(*mVold, unmountIncFs(_));
     TemporaryDir tempDir;
@@ -712,7 +751,7 @@ TEST_F(IncrementalServiceTest, testCreateStorageBindMountFails) {
     mVold->mountIncFsSuccess();
     mIncFs->makeFileSuccess();
     mVold->bindMountFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     EXPECT_CALL(*mVold, unmountIncFs(_));
     TemporaryDir tempDir;
@@ -727,7 +766,7 @@ TEST_F(IncrementalServiceTest, testCreateStoragePrepareDataLoaderFails) {
     mIncFs->makeFileSuccess();
     mVold->bindMountSuccess();
     mDataLoaderManager->bindToDataLoaderFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoader, start(_)).Times(0);
@@ -755,11 +794,11 @@ TEST_F(IncrementalServiceTest, testDeleteStorageSuccess) {
     mIncrementalService->deleteStorage(storageId);
 }
 
-TEST_F(IncrementalServiceTest, testDataLoaderDestroyed) {
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(2);
+TEST_F(IncrementalServiceTest, testDataLoaderDestroyedAndDelayed) {
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(6);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
-    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(2);
-    EXPECT_CALL(*mDataLoader, start(_)).Times(2);
+    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(6);
+    EXPECT_CALL(*mDataLoader, start(_)).Times(6);
     EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
     EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
     TemporaryDir tempDir;
@@ -768,13 +807,38 @@ TEST_F(IncrementalServiceTest, testDataLoaderDestroyed) {
                                                IncrementalService::CreateOptions::CreateNew);
     ASSERT_GE(storageId, 0);
     mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
+
     // Simulated crash/other connection breakage.
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith10sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith100sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith1000sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
     mDataLoaderManager->setDataLoaderStatusDestroyed();
 }
 
 TEST_F(IncrementalServiceTest, testStartDataLoaderCreate) {
     mDataLoader->initializeCreateOkNoStatus();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);
@@ -793,7 +857,7 @@ TEST_F(IncrementalServiceTest, testStartDataLoaderCreate) {
 
 TEST_F(IncrementalServiceTest, testStartDataLoaderPendingStart) {
     mDataLoader->initializeCreateOkNoStatus();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);
@@ -811,7 +875,7 @@ TEST_F(IncrementalServiceTest, testStartDataLoaderPendingStart) {
 
 TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnavailable) {
     mDataLoader->initializeCreateOkNoStatus();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(0);
@@ -827,12 +891,30 @@ TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnavailable) {
     mDataLoaderManager->setDataLoaderStatusUnavailable();
 }
 
+TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnrecoverable) {
+    mDataLoader->initializeCreateOkNoStatus();
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
+    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoader, start(_)).Times(0);
+    EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
+    EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
+    TemporaryDir tempDir;
+    int storageId =
+            mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
+                                               IncrementalService::CreateOptions::CreateNew);
+    ASSERT_GE(storageId, 0);
+    ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
+                                                  {}, {}));
+    mDataLoaderManager->setDataLoaderStatusUnrecoverable();
+}
+
 TEST_F(IncrementalServiceTest, testStartDataLoaderRecreateOnPendingReads) {
     mIncFs->waitForPendingReadsSuccess();
     mIncFs->openMountSuccess();
     mDataLoader->initializeCreateOkNoStatus();
 
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(2);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(2);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(2);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(2);
     EXPECT_CALL(*mDataLoader, start(_)).Times(0);
@@ -856,7 +938,7 @@ TEST_F(IncrementalServiceTest, testStartDataLoaderRecreateOnPendingReads) {
 TEST_F(IncrementalServiceTest, testStartDataLoaderUnhealthyStorage) {
     mIncFs->openMountSuccess();
 
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);
@@ -1406,7 +1488,7 @@ static ErrorCode checkPerUidTimeoutsEmpty(
 }
 
 TEST_F(IncrementalServiceTest, testPerUidTimeoutsTooShort) {
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);