EGL Multifile Blobcache: Handle lost cache

During execution, if the app's cache is cleared from outside
our control, two issues occur:

* New entries won't persist to disk because the multifile directory
  is missing.
* applyLRU will abort eviction, allowing entries beyond the limits.

To address this, this CL:
* Adds missing directory detection to our deferred write thread
  which then attempts to recreate it and continue.
* Updates eviction to issue a warning rather than abort so it can
  update tracking and continue.

For missing entries, the app will get hits from hotcache, but
anything beyond that will become a miss.

Additional tests:
* RecoverFromLostCache
* EvictAfterLostCache

Based on work by: Igor Nazarov <i.nazarov@samsung.com>

Test: libEGL_test, EGL_test, ANGLE trace tests, apps
Bug: b/351867582, b/380483358
Flag: com.android.graphics.egl.flags.multifile_blobcache_advanced_usage
Change-Id: I13c8cdf58c957163eed4498c0d4be180574bf03e

2 files changed, 161 insertions, 6 deletions

diff --git a/opengl/libs/EGL/MultifileBlobCache.cpp b/opengl/libs/EGL/MultifileBlobCache.cpp
index 53a08bb59d..1f6d4d04e7 100644
--- a/opengl/libs/EGL/MultifileBlobCache.cpp
+++ b/opengl/libs/EGL/MultifileBlobCache.cpp
@@ -137,7 +137,7 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s
     // Check that our cacheVersion and buildId match
     struct stat st;
     if (stat(mMultifileDirName.c_str(), &st) == 0) {
-        if (checkStatus(mMultifileDirName.c_str())) {
+        if (checkStatus(mMultifileDirName)) {
             statusGood = true;
         } else {
             ALOGV("INIT: Cache status has changed, clearing the cache");
@@ -851,8 +851,8 @@ bool MultifileBlobCache::applyLRU(size_t cacheSizeLimit, size_t cacheEntryLimit)
         // Remove it from the system
         std::string entryPath = mMultifileDirName + "/" + std::to_string(entryHash);
         if (remove(entryPath.c_str()) != 0) {
-            ALOGE("LRU: Error removing %s: %s", entryPath.c_str(), std::strerror(errno));
-            return false;
+            // Continue evicting invalid item (app's cache might be cleared)
+            ALOGW("LRU: Error removing %s: %s", entryPath.c_str(), std::strerror(errno));
         }
 
         // Increment the iterator before clearing the entry
@@ -945,9 +945,36 @@ void MultifileBlobCache::processTask(DeferredTask& task) {
             // Create the file or reset it if already present, read+write for user only
             int fd = open(fullPath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
             if (fd == -1) {
-                ALOGE("Cache error in SET - failed to open fullPath: %s, error: %s",
-                      fullPath.c_str(), std::strerror(errno));
-                return;
+                if (flags::multifile_blobcache_advanced_usage()) {
+                    struct stat st;
+                    if (stat(mMultifileDirName.c_str(), &st) == -1) {
+                        ALOGW("Cache directory missing (app's cache cleared?). Recreating...");
+
+                        // Restore the multifile directory
+                        if (mkdir(mMultifileDirName.c_str(), 0755) != 0 && (errno != EEXIST)) {
+                            ALOGE("Cache error in SET - Unable to create directory (%s), errno "
+                                  "(%i)",
+                                  mMultifileDirName.c_str(), errno);
+                            return;
+                        }
+
+                        // Create new status file
+                        if (!createStatus(mMultifileDirName.c_str())) {
+                            ALOGE("Cache error in SET - Failed to create status file!");
+                            return;
+                        }
+
+                        // Try to open the file again
+                        fd = open(fullPath.c_str(), O_WRONLY | O_CREAT | O_TRUNC,
+                                  S_IRUSR | S_IWUSR);
+                    }
+                }
+
+                if (fd == -1) {
+                    ALOGE("Cache error in SET - failed to open fullPath: %s, error: %s",
+                          fullPath.c_str(), std::strerror(errno));
+                    return;
+                }
             }
 
             ALOGV("DEFERRED: Opened fd %i from %s", fd, fullPath.c_str());
diff --git a/opengl/libs/EGL/MultifileBlobCache_test.cpp b/opengl/libs/EGL/MultifileBlobCache_test.cpp
index 74352d477d..fb765a73bc 100644
--- a/opengl/libs/EGL/MultifileBlobCache_test.cpp
+++ b/opengl/libs/EGL/MultifileBlobCache_test.cpp
@@ -59,6 +59,7 @@ protected:
     std::vector<std::string> getCacheEntries();
 
     void clearProperties();
+    bool clearCache();
 
     std::unique_ptr<TemporaryFile> mTempFile;
     std::unique_ptr<MultifileBlobCache> mMBC;
@@ -727,4 +728,131 @@ TEST_F(MultifileBlobCacheTest, GetUpdatesAccessTime) {
     }
 }
 
+bool MultifileBlobCacheTest::clearCache() {
+    std::string cachePath = &mTempFile->path[0];
+    std::string multifileDirName = cachePath + ".multifile";
+
+    DIR* dir = opendir(multifileDirName.c_str());
+    if (dir == nullptr) {
+        printf("Error opening directory: %s\n", multifileDirName.c_str());
+        return false;
+    }
+
+    struct dirent* entry;
+    while ((entry = readdir(dir)) != nullptr) {
+        // Skip "." and ".." entries
+        if (std::string(entry->d_name) == "." || std::string(entry->d_name) == "..") {
+            continue;
+        }
+
+        std::string entryPath = multifileDirName + "/" + entry->d_name;
+
+        // Delete the entry (we assert it's a file, nothing nested here)
+        if (unlink(entryPath.c_str()) != 0) {
+            printf("Error deleting file: %s\n", entryPath.c_str());
+            closedir(dir);
+            return false;
+        }
+    }
+
+    closedir(dir);
+
+    // Delete the empty directory itself
+    if (rmdir(multifileDirName.c_str()) != 0) {
+        printf("Error deleting directory %s, error %s\n", multifileDirName.c_str(),
+               std::strerror(errno));
+        return false;
+    }
+
+    return true;
+}
+
+// Recover from lost cache in the case of app clearing it
+TEST_F(MultifileBlobCacheTest, RecoverFromLostCache) {
+    if (!flags::multifile_blobcache_advanced_usage()) {
+        GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
+    }
+
+    int entry = 0;
+    int result = 0;
+
+    uint32_t kEntryCount = 10;
+
+    // Add some entries
+    for (entry = 0; entry < kEntryCount; entry++) {
+        mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
+        ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
+        ASSERT_EQ(entry, result);
+    }
+
+    // For testing, wait until the entries have completed writing
+    mMBC->finish();
+
+    // Manually delete the cache!
+    ASSERT_TRUE(clearCache());
+
+    // Cache should not contain any entries
+    for (entry = 0; entry < kEntryCount; entry++) {
+        ASSERT_EQ(size_t(0), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
+    }
+
+    // Ensure we can still add new ones
+    for (entry = kEntryCount; entry < kEntryCount * 2; entry++) {
+        mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
+        ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
+        ASSERT_EQ(entry, result);
+    }
+
+    // Close the cache so everything writes out
+    mMBC->finish();
+    mMBC.reset();
+
+    // Open the cache again
+    mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
+                                      &mTempFile->path[0]));
+
+    // Before fixes, writing the second entries to disk should have failed due to missing
+    // cache dir.  But now they should have survived our shutdown above.
+    for (entry = kEntryCount; entry < kEntryCount * 2; entry++) {
+        ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
+        ASSERT_EQ(entry, result);
+    }
+}
+
+// Ensure cache eviction succeeds if the cache is deleted
+TEST_F(MultifileBlobCacheTest, EvictAfterLostCache) {
+    if (!flags::multifile_blobcache_advanced_usage()) {
+        GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
+    }
+
+    int entry = 0;
+    int result = 0;
+
+    uint32_t kEntryCount = 10;
+
+    // Add some entries
+    for (entry = 0; entry < kEntryCount; entry++) {
+        mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
+        ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
+        ASSERT_EQ(entry, result);
+    }
+
+    // For testing, wait until the entries have completed writing
+    mMBC->finish();
+
+    // Manually delete the cache!
+    ASSERT_TRUE(clearCache());
+
+    // Now start adding entries to trigger eviction, cache should survive
+    for (entry = kEntryCount; entry < 2 * kMaxTotalEntries; entry++) {
+        mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
+        ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
+        ASSERT_EQ(entry, result);
+    }
+
+    // We should have triggered multiple evictions above and remain at or below the
+    // max amount of entries
+    ASSERT_LE(getCacheEntries().size(), kMaxTotalEntries);
+}
+
 } // namespace android