Reland "EGL BlobCache: Support multifile cache and flexible size limit"

This reverts commit 82a9353d96954edd78d5e2f342a669dd8ddce5d1 and
incorporates a change to only use the fixed upper cache limit instead
of reading one from GraphicsEnvironment.

Test: Multiple apps and ANGLE traces
Test: atest CtsSdkSandboxInprocessTests
Test: /data/nativetest64/EGL_test/EGL_test
Bug: b/246966894
Change-Id: Iae44e06377de48fe2101bf547b02d3aaf37443d9

8 files changed, 608 insertions, 37 deletions

diff --git a/opengl/libs/Android.bp b/opengl/libs/Android.bp
index 62cf2555ca..750338bd84 100644
--- a/opengl/libs/Android.bp
+++ b/opengl/libs/Android.bp
@@ -160,6 +160,7 @@ cc_library_shared {
     srcs: [
         "EGL/egl_tls.cpp",
         "EGL/egl_cache.cpp",
+        "EGL/egl_cache_multifile.cpp",
         "EGL/egl_display.cpp",
         "EGL/egl_object.cpp",
         "EGL/egl_layers.cpp",
diff --git a/opengl/libs/EGL/FileBlobCache.cpp b/opengl/libs/EGL/FileBlobCache.cpp
index 751f3be81a..3f7ae7e5f4 100644
--- a/opengl/libs/EGL/FileBlobCache.cpp
+++ b/opengl/libs/EGL/FileBlobCache.cpp
@@ -185,4 +185,10 @@ void FileBlobCache::writeToFile() {
     }
 }
 
+size_t FileBlobCache::getSize() {
+    if (mFilename.length() > 0) {
+        return getFlattenedSize() + cacheFileHeaderSize;
+    }
+    return 0;
+}
 }
diff --git a/opengl/libs/EGL/FileBlobCache.h b/opengl/libs/EGL/FileBlobCache.h
index 393703f234..8220723b2c 100644
--- a/opengl/libs/EGL/FileBlobCache.h
+++ b/opengl/libs/EGL/FileBlobCache.h
@@ -33,6 +33,9 @@ public:
     // disk.
     void writeToFile();
 
+    // Return the total size of the cache
+    size_t getSize();
+
 private:
     // mFilename is the name of the file for storing cache contents.
     std::string mFilename;
diff --git a/opengl/libs/EGL/egl_cache.cpp b/opengl/libs/EGL/egl_cache.cpp
index 8348d6cb0a..6225c26c9b 100644
--- a/opengl/libs/EGL/egl_cache.cpp
+++ b/opengl/libs/EGL/egl_cache.cpp
@@ -16,6 +16,8 @@
 
 #include "egl_cache.h"
 
+#include <android-base/properties.h>
+#include <inttypes.h>
 #include <log/log.h>
 #include <private/EGL/cache.h>
 #include <unistd.h>
@@ -23,16 +25,23 @@
 #include <thread>
 
 #include "../egl_impl.h"
+#include "egl_cache_multifile.h"
 #include "egl_display.h"
 
-// Cache size limits.
+// Monolithic cache size limits.
 static const size_t maxKeySize = 12 * 1024;
 static const size_t maxValueSize = 64 * 1024;
 static const size_t maxTotalSize = 32 * 1024 * 1024;
 
-// The time in seconds to wait before saving newly inserted cache entries.
+// The time in seconds to wait before saving newly inserted monolithic cache entries.
 static const unsigned int deferredSaveDelay = 4;
 
+// Multifile cache size limit
+constexpr size_t kMultifileCacheByteLimit = 64 * 1024 * 1024;
+
+// Delay before cleaning up multifile cache entries
+static const unsigned int deferredMultifileCleanupDelaySeconds = 1;
+
 namespace android {
 
 #define BC_EXT_STR "EGL_ANDROID_blob_cache"
@@ -58,7 +67,8 @@ static EGLsizeiANDROID getBlob(const void* key, EGLsizeiANDROID keySize, void* v
 //
 // egl_cache_t definition
 //
-egl_cache_t::egl_cache_t() : mInitialized(false) {}
+egl_cache_t::egl_cache_t()
+      : mInitialized(false), mMultifileMode(true), mCacheByteLimit(maxTotalSize) {}
 
 egl_cache_t::~egl_cache_t() {}
 
@@ -101,6 +111,18 @@ void egl_cache_t::initialize(egl_display_t* display) {
         }
     }
 
+    mMultifileMode = true;
+
+    // Allow forcing monolithic cache for debug purposes
+    if (base::GetProperty("debug.egl.blobcache.multifilemode", "") == "false") {
+        ALOGD("Forcing monolithic cache due to debug.egl.blobcache.multifilemode == \"false\"");
+        mMultifileMode = false;
+    }
+
+    if (mMultifileMode) {
+        mCacheByteLimit = kMultifileCacheByteLimit;
+    }
+
     mInitialized = true;
 }
 
@@ -110,6 +132,11 @@ void egl_cache_t::terminate() {
         mBlobCache->writeToFile();
     }
     mBlobCache = nullptr;
+    if (mMultifileMode) {
+        checkMultifileCacheSize(mCacheByteLimit);
+    }
+    mMultifileMode = false;
+    mInitialized = false;
 }
 
 void egl_cache_t::setBlob(const void* key, EGLsizeiANDROID keySize, const void* value,
@@ -122,20 +149,37 @@ void egl_cache_t::setBlob(const void* key, EGLsizeiANDROID keySize, const void*
     }
 
     if (mInitialized) {
-        BlobCache* bc = getBlobCacheLocked();
-        bc->set(key, keySize, value, valueSize);
-
-        if (!mSavePending) {
-            mSavePending = true;
-            std::thread deferredSaveThread([this]() {
-                sleep(deferredSaveDelay);
-                std::lock_guard<std::mutex> lock(mMutex);
-                if (mInitialized && mBlobCache) {
-                    mBlobCache->writeToFile();
-                }
-                mSavePending = false;
-            });
-            deferredSaveThread.detach();
+        if (mMultifileMode) {
+            setBlobMultifile(key, keySize, value, valueSize, mFilename);
+
+            if (!mMultifileCleanupPending) {
+                mMultifileCleanupPending = true;
+                // Kick off a thread to cull cache files below limit
+                std::thread deferredMultifileCleanupThread([this]() {
+                    sleep(deferredMultifileCleanupDelaySeconds);
+                    std::lock_guard<std::mutex> lock(mMutex);
+                    // Check the size of cache and remove entries to stay under limit
+                    checkMultifileCacheSize(mCacheByteLimit);
+                    mMultifileCleanupPending = false;
+                });
+                deferredMultifileCleanupThread.detach();
+            }
+        } else {
+            BlobCache* bc = getBlobCacheLocked();
+            bc->set(key, keySize, value, valueSize);
+
+            if (!mSavePending) {
+                mSavePending = true;
+                std::thread deferredSaveThread([this]() {
+                    sleep(deferredSaveDelay);
+                    std::lock_guard<std::mutex> lock(mMutex);
+                    if (mInitialized && mBlobCache) {
+                        mBlobCache->writeToFile();
+                    }
+                    mSavePending = false;
+                });
+                deferredSaveThread.detach();
+            }
         }
     }
 }
@@ -145,13 +189,17 @@ EGLsizeiANDROID egl_cache_t::getBlob(const void* key, EGLsizeiANDROID keySize, v
     std::lock_guard<std::mutex> lock(mMutex);
 
     if (keySize < 0 || valueSize < 0) {
-        ALOGW("EGL_ANDROID_blob_cache set: negative sizes are not allowed");
+        ALOGW("EGL_ANDROID_blob_cache get: negative sizes are not allowed");
         return 0;
     }
 
     if (mInitialized) {
-        BlobCache* bc = getBlobCacheLocked();
-        return bc->get(key, keySize, value, valueSize);
+        if (mMultifileMode) {
+            return getBlobMultifile(key, keySize, value, valueSize, mFilename);
+        } else {
+            BlobCache* bc = getBlobCacheLocked();
+            return bc->get(key, keySize, value, valueSize);
+        }
     }
     return 0;
 }
@@ -161,9 +209,34 @@ void egl_cache_t::setCacheFilename(const char* filename) {
     mFilename = filename;
 }
 
+void egl_cache_t::setCacheLimit(int64_t cacheByteLimit) {
+    std::lock_guard<std::mutex> lock(mMutex);
+
+    if (!mMultifileMode) {
+        // If we're not in multifile mode, ensure the cache limit is only being lowered,
+        // not increasing above the hard coded platform limit
+        if (cacheByteLimit > maxTotalSize) {
+            return;
+        }
+    }
+
+    mCacheByteLimit = cacheByteLimit;
+}
+
+size_t egl_cache_t::getCacheSize() {
+    std::lock_guard<std::mutex> lock(mMutex);
+    if (mMultifileMode) {
+        return getMultifileCacheSize();
+    }
+    if (mBlobCache) {
+        return mBlobCache->getSize();
+    }
+    return 0;
+}
+
 BlobCache* egl_cache_t::getBlobCacheLocked() {
     if (mBlobCache == nullptr) {
-        mBlobCache.reset(new FileBlobCache(maxKeySize, maxValueSize, maxTotalSize, mFilename));
+        mBlobCache.reset(new FileBlobCache(maxKeySize, maxValueSize, mCacheByteLimit, mFilename));
     }
     return mBlobCache.get();
 }
diff --git a/opengl/libs/EGL/egl_cache.h b/opengl/libs/EGL/egl_cache.h
index d10a6154b7..2dcd803324 100644
--- a/opengl/libs/EGL/egl_cache.h
+++ b/opengl/libs/EGL/egl_cache.h
@@ -64,6 +64,12 @@ public:
     // cache contents from one program invocation to another.
     void setCacheFilename(const char* filename);
 
+    // Allow the fixed cache limit to be overridden
+    void setCacheLimit(int64_t cacheByteLimit);
+
+    // Return the byte total for cache file(s)
+    size_t getCacheSize();
+
 private:
     // Creation and (the lack of) destruction is handled internally.
     egl_cache_t();
@@ -112,6 +118,16 @@ private:
 
     // sCache is the singleton egl_cache_t object.
     static egl_cache_t sCache;
+
+    // Whether to use multiple files to store cache entries
+    bool mMultifileMode;
+
+    // Cache limit
+    int64_t mCacheByteLimit;
+
+    // Whether we've kicked off a side thread that will check the multifile
+    // cache size and remove entries if needed.
+    bool mMultifileCleanupPending;
 };
 
 }; // namespace android
diff --git a/opengl/libs/EGL/egl_cache_multifile.cpp b/opengl/libs/EGL/egl_cache_multifile.cpp
new file mode 100644
index 0000000000..48e557f190
--- /dev/null
+++ b/opengl/libs/EGL/egl_cache_multifile.cpp
@@ -0,0 +1,343 @@
+/*
+ ** Copyright 2022, The Android Open Source Project
+ **
+ ** Licensed under the Apache License, Version 2.0 (the "License");
+ ** you may not use this file except in compliance with the License.
+ ** You may obtain a copy of the License at
+ **
+ **     http://www.apache.org/licenses/LICENSE-2.0
+ **
+ ** Unless required by applicable law or agreed to in writing, software
+ ** distributed under the License is distributed on an "AS IS" BASIS,
+ ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ** See the License for the specific language governing permissions and
+ ** limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+
+#include "egl_cache_multifile.h"
+
+#include <android-base/properties.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <log/log.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <utime.h>
+
+#include <algorithm>
+#include <chrono>
+#include <fstream>
+#include <limits>
+#include <locale>
+#include <map>
+#include <sstream>
+#include <unordered_map>
+
+#include <utils/JenkinsHash.h>
+
+static std::string multifileDirName = "";
+
+using namespace std::literals;
+
+namespace {
+
+// Create a directory for tracking multiple files
+void setupMultifile(const std::string& baseDir) {
+    // If we've already set up the multifile dir in this base directory, we're done
+    if (!multifileDirName.empty() && multifileDirName.find(baseDir) != std::string::npos) {
+        return;
+    }
+
+    // Otherwise, create it
+    multifileDirName = baseDir + ".multifile";
+    if (mkdir(multifileDirName.c_str(), 0755) != 0 && (errno != EEXIST)) {
+        ALOGW("Unable to create directory (%s), errno (%i)", multifileDirName.c_str(), errno);
+    }
+}
+
+// Create a filename that is based on the hash of the key
+std::string getCacheEntryFilename(const void* key, EGLsizeiANDROID keySize,
+                                  const std::string& baseDir) {
+    // Hash the key into a string
+    std::stringstream keyName;
+    keyName << android::JenkinsHashMixBytes(0, static_cast<const uint8_t*>(key), keySize);
+
+    // Build a filename using dir and hash
+    return baseDir + "/" + keyName.str();
+}
+
+// Determine file age based on stat modification time
+// Newer files have a higher age (time since epoch)
+time_t getFileAge(const std::string& filePath) {
+    struct stat st;
+    if (stat(filePath.c_str(), &st) == 0) {
+        ALOGD("getFileAge returning %" PRId64 " for file age", static_cast<uint64_t>(st.st_mtime));
+        return st.st_mtime;
+    } else {
+        ALOGW("Failed to stat %s", filePath.c_str());
+        return 0;
+    }
+}
+
+size_t getFileSize(const std::string& filePath) {
+    struct stat st;
+    if (stat(filePath.c_str(), &st) != 0) {
+        ALOGE("Unable to stat %s", filePath.c_str());
+        return 0;
+    }
+    return st.st_size;
+}
+
+// Walk through directory entries and track age and size
+// Then iterate through the entries, oldest first, and remove them until under the limit.
+// This will need to be updated if we move to a multilevel cache dir.
+bool applyLRU(size_t cacheLimit) {
+    // Build a multimap of files indexed by age.
+    // They will be automatically sorted smallest (oldest) to largest (newest)
+    std::multimap<time_t, std::string> agesToFiles;
+
+    // Map files to sizes
+    std::unordered_map<std::string, size_t> filesToSizes;
+
+    size_t totalCacheSize = 0;
+
+    DIR* dir;
+    struct dirent* entry;
+    if ((dir = opendir(multifileDirName.c_str())) != nullptr) {
+        while ((entry = readdir(dir)) != nullptr) {
+            if (entry->d_name == "."s || entry->d_name == ".."s) {
+                continue;
+            }
+
+            // Look up each file age
+            std::string fullPath = multifileDirName + "/" + entry->d_name;
+            time_t fileAge = getFileAge(fullPath);
+
+            // Track the files, sorted by age
+            agesToFiles.insert(std::make_pair(fileAge, fullPath));
+
+            // Also track the size so we know how much room we have freed
+            size_t fileSize = getFileSize(fullPath);
+            filesToSizes[fullPath] = fileSize;
+            totalCacheSize += fileSize;
+        }
+        closedir(dir);
+    } else {
+        ALOGE("Unable to open filename: %s", multifileDirName.c_str());
+        return false;
+    }
+
+    if (totalCacheSize <= cacheLimit) {
+        // If LRU was called on a sufficiently small cache, no need to remove anything
+        return true;
+    }
+
+    // Walk through the map of files until we're under the cache size
+    for (const auto& cacheEntryIter : agesToFiles) {
+        time_t entryAge = cacheEntryIter.first;
+        const std::string entryPath = cacheEntryIter.second;
+
+        ALOGD("Removing %s with age %ld", entryPath.c_str(), entryAge);
+        if (std::remove(entryPath.c_str()) != 0) {
+            ALOGE("Error removing %s: %s", entryPath.c_str(), std::strerror(errno));
+            return false;
+        }
+
+        totalCacheSize -= filesToSizes[entryPath];
+        if (totalCacheSize <= cacheLimit) {
+            // Success
+            ALOGV("Reduced cache to %zu", totalCacheSize);
+            return true;
+        } else {
+            ALOGD("Cache size is still too large (%zu), removing more files", totalCacheSize);
+        }
+    }
+
+    // Should never reach this return
+    return false;
+}
+
+} // namespace
+
+namespace android {
+
+void setBlobMultifile(const void* key, EGLsizeiANDROID keySize, const void* value,
+                      EGLsizeiANDROID valueSize, const std::string& baseDir) {
+    if (baseDir.empty()) {
+        return;
+    }
+
+    setupMultifile(baseDir);
+    std::string filename = getCacheEntryFilename(key, keySize, multifileDirName);
+
+    ALOGD("Attempting to open filename for set: %s", filename.c_str());
+    std::ofstream outfile(filename, std::ofstream::binary);
+    if (outfile.fail()) {
+        ALOGW("Unable to open filename: %s", filename.c_str());
+        return;
+    }
+
+    // First write the key
+    outfile.write(static_cast<const char*>(key), keySize);
+    if (outfile.bad()) {
+        ALOGW("Unable to write key to filename: %s", filename.c_str());
+        outfile.close();
+        return;
+    }
+    ALOGD("Wrote %i bytes to out file for key", static_cast<int>(outfile.tellp()));
+
+    // Then write the value
+    outfile.write(static_cast<const char*>(value), valueSize);
+    if (outfile.bad()) {
+        ALOGW("Unable to write value to filename: %s", filename.c_str());
+        outfile.close();
+        return;
+    }
+    ALOGD("Wrote %i bytes to out file for full entry", static_cast<int>(outfile.tellp()));
+
+    outfile.close();
+}
+
+EGLsizeiANDROID getBlobMultifile(const void* key, EGLsizeiANDROID keySize, void* value,
+                                 EGLsizeiANDROID valueSize, const std::string& baseDir) {
+    if (baseDir.empty()) {
+        return 0;
+    }
+
+    setupMultifile(baseDir);
+    std::string filename = getCacheEntryFilename(key, keySize, multifileDirName);
+
+    // Open the hashed filename path
+    ALOGD("Attempting to open filename for get: %s", filename.c_str());
+    int fd = open(filename.c_str(), O_RDONLY);
+
+    // File doesn't exist, this is a MISS, return zero bytes read
+    if (fd == -1) {
+        ALOGD("Cache MISS - failed to open filename: %s, error: %s", filename.c_str(),
+              std::strerror(errno));
+        return 0;
+    }
+
+    ALOGD("Cache HIT - opened filename: %s", filename.c_str());
+
+    // Get the size of the file
+    size_t entrySize = getFileSize(filename);
+    if (keySize > entrySize) {
+        ALOGW("keySize (%lu) is larger than entrySize (%zu). This is a hash collision or modified "
+              "file",
+              keySize, entrySize);
+        close(fd);
+        return 0;
+    }
+
+    // Memory map the file
+    uint8_t* cacheEntry =
+            reinterpret_cast<uint8_t*>(mmap(nullptr, entrySize, PROT_READ, MAP_PRIVATE, fd, 0));
+    if (cacheEntry == MAP_FAILED) {
+        ALOGE("Failed to mmap cacheEntry, error: %s", std::strerror(errno));
+        close(fd);
+        return 0;
+    }
+
+    // Compare the incoming key with our stored version (the beginning of the entry)
+    int compare = memcmp(cacheEntry, key, keySize);
+    if (compare != 0) {
+        ALOGW("Cached key and new key do not match! This is a hash collision or modified file");
+        munmap(cacheEntry, entrySize);
+        close(fd);
+        return 0;
+    }
+
+    // Keys matched, so remaining cache is value size
+    size_t cachedValueSize = entrySize - keySize;
+
+    // Return actual value size if valueSize is not large enough
+    if (cachedValueSize > valueSize) {
+        ALOGD("Skipping file read, not enough room provided (valueSize): %lu, "
+              "returning required space as %zu",
+              valueSize, cachedValueSize);
+        munmap(cacheEntry, entrySize);
+        close(fd);
+        return cachedValueSize;
+    }
+
+    // Remaining entry following the key is the value
+    uint8_t* cachedValue = cacheEntry + keySize;
+    memcpy(value, cachedValue, cachedValueSize);
+    munmap(cacheEntry, entrySize);
+    close(fd);
+
+    ALOGD("Read %zu bytes from %s", cachedValueSize, filename.c_str());
+    return cachedValueSize;
+}
+
+// Walk through the files in our flat directory, checking the size of each one.
+// Return the total size of normal files in the directory.
+// This will need to be updated if we move to a multilevel cache dir.
+size_t getMultifileCacheSize() {
+    if (multifileDirName.empty()) {
+        return 0;
+    }
+
+    DIR* dir;
+    struct dirent* entry;
+    size_t size = 0;
+
+    ALOGD("Using %s as the multifile cache dir ", multifileDirName.c_str());
+
+    if ((dir = opendir(multifileDirName.c_str())) != nullptr) {
+        while ((entry = readdir(dir)) != nullptr) {
+            if (entry->d_name == "."s || entry->d_name == ".."s) {
+                continue;
+            }
+
+            // Add up the size of all files in the dir
+            std::string fullPath = multifileDirName + "/" + entry->d_name;
+            size += getFileSize(fullPath);
+        }
+        closedir(dir);
+    } else {
+        ALOGW("Unable to open filename: %s", multifileDirName.c_str());
+        return 0;
+    }
+
+    return size;
+}
+
+// When removing files, what fraction of the overall limit should be reached when removing files
+// A divisor of two will decrease the cache to 50%, four to 25% and so on
+constexpr uint32_t kCacheLimitDivisor = 2;
+
+// Calculate the cache size and remove old entries until under the limit
+void checkMultifileCacheSize(size_t cacheByteLimit) {
+    // Start with the value provided by egl_cache
+    size_t limit = cacheByteLimit;
+
+    // Check for a debug value
+    int debugCacheSize = base::GetIntProperty("debug.egl.blobcache.bytelimit", -1);
+    if (debugCacheSize >= 0) {
+        ALOGV("Overriding cache limit %zu with %i from debug.egl.blobcache.bytelimit", limit,
+              debugCacheSize);
+        limit = debugCacheSize;
+    }
+
+    // Tally up the initial amount of cache in use
+    size_t size = getMultifileCacheSize();
+    ALOGD("Multifile cache dir size: %zu", size);
+
+    // If size is larger than the threshold, remove files using LRU
+    if (size > limit) {
+        ALOGV("Multifile cache size is larger than %zu, removing old entries", cacheByteLimit);
+        if (!applyLRU(limit / kCacheLimitDivisor)) {
+            ALOGE("Error when clearing multifile shader cache");
+            return;
+        }
+    }
+    ALOGD("Multifile cache size after reduction: %zu", getMultifileCacheSize());
+}
+
+}; // namespace android
\ No newline at end of file
diff --git a/opengl/libs/EGL/egl_cache_multifile.h b/opengl/libs/EGL/egl_cache_multifile.h
new file mode 100644
index 0000000000..ee5fe8108d
--- /dev/null
+++ b/opengl/libs/EGL/egl_cache_multifile.h
@@ -0,0 +1,36 @@
+/*
+ ** Copyright 2022, The Android Open Source Project
+ **
+ ** Licensed under the Apache License, Version 2.0 (the "License");
+ ** you may not use this file except in compliance with the License.
+ ** You may obtain a copy of the License at
+ **
+ **     http://www.apache.org/licenses/LICENSE-2.0
+ **
+ ** Unless required by applicable law or agreed to in writing, software
+ ** distributed under the License is distributed on an "AS IS" BASIS,
+ ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ** See the License for the specific language governing permissions and
+ ** limitations under the License.
+ */
+
+#ifndef ANDROID_EGL_CACHE_MULTIFILE_H
+#define ANDROID_EGL_CACHE_MULTIFILE_H
+
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+
+#include <string>
+
+namespace android {
+
+void setBlobMultifile(const void* key, EGLsizeiANDROID keySize, const void* value,
+                      EGLsizeiANDROID valueSize, const std::string& baseDir);
+EGLsizeiANDROID getBlobMultifile(const void* key, EGLsizeiANDROID keySize, void* value,
+                                 EGLsizeiANDROID valueSize, const std::string& baseDir);
+size_t getMultifileCacheSize();
+void checkMultifileCacheSize(size_t cacheByteLimit);
+
+}; // namespace android
+
+#endif // ANDROID_EGL_CACHE_MULTIFILE_H
diff --git a/opengl/tests/EGLTest/egl_cache_test.cpp b/opengl/tests/EGLTest/egl_cache_test.cpp
index c974f63c13..c4fa65f044 100644
--- a/opengl/tests/EGLTest/egl_cache_test.cpp
+++ b/opengl/tests/EGLTest/egl_cache_test.cpp
@@ -24,24 +24,33 @@
 #include <android-base/test_utils.h>
 
 #include "egl_cache.h"
+#include "egl_cache_multifile.h"
 #include "egl_display.h"
 
 #include <memory>
 
+using namespace std::literals;
+
 namespace android {
 
 class EGLCacheTest : public ::testing::Test {
 protected:
     virtual void SetUp() {
         mCache = egl_cache_t::get();
+        mTempFile.reset(new TemporaryFile());
+        mCache->setCacheFilename(&mTempFile->path[0]);
     }
 
     virtual void TearDown() {
-        mCache->setCacheFilename("");
         mCache->terminate();
+        mCache->setCacheFilename("");
+        mTempFile.reset(nullptr);
     }
 
+    std::string getCachefileName();
+
     egl_cache_t* mCache;
+    std::unique_ptr<TemporaryFile> mTempFile;
 };
 
 TEST_F(EGLCacheTest, UninitializedCacheAlwaysMisses) {
@@ -77,35 +86,119 @@ TEST_F(EGLCacheTest, TerminatedCacheAlwaysMisses) {
     ASSERT_EQ(0xee, buf[3]);
 }
 
-class EGLCacheSerializationTest : public EGLCacheTest {
-
-protected:
+TEST_F(EGLCacheTest, ReinitializedCacheContainsValues) {
+    uint8_t buf[4] = { 0xee, 0xee, 0xee, 0xee };
+    mCache->initialize(egl_display_t::get(EGL_DEFAULT_DISPLAY));
+    mCache->setBlob("abcd", 4, "efgh", 4);
+    mCache->terminate();
+    mCache->initialize(egl_display_t::get(EGL_DEFAULT_DISPLAY));
+    ASSERT_EQ(4, mCache->getBlob("abcd", 4, buf, 4));
+    ASSERT_EQ('e', buf[0]);
+    ASSERT_EQ('f', buf[1]);
+    ASSERT_EQ('g', buf[2]);
+    ASSERT_EQ('h', buf[3]);
+}
 
-    virtual void SetUp() {
-        EGLCacheTest::SetUp();
-        mTempFile.reset(new TemporaryFile());
+std::string EGLCacheTest::getCachefileName() {
+    // Return the monolithic filename unless we find the multifile dir
+    std::string cachefileName = &mTempFile->path[0];
+    std::string multifileDirName = cachefileName + ".multifile";
+
+    struct stat info;
+    if (stat(multifileDirName.c_str(), &info) == 0) {
+
+        // Ensure we only have one file to manage
+        int realFileCount = 0;
+
+        // We have a multifile dir. Return the only real file in it.
+        DIR* dir;
+        struct dirent* entry;
+        if ((dir = opendir(multifileDirName.c_str())) != nullptr) {
+            while ((entry = readdir(dir)) != nullptr) {
+                if (entry->d_name == "."s || entry->d_name == ".."s) {
+                    continue;
+                }
+                cachefileName = multifileDirName + "/" + entry->d_name;
+                realFileCount++;
+            }
+        }
+
+        if (realFileCount != 1) {
+            // If there was more than one real file in the directory, this
+            // violates test assumptions
+            cachefileName = "";
+        }
     }
 
-    virtual void TearDown() {
-        mTempFile.reset(nullptr);
-        EGLCacheTest::TearDown();
-    }
-
-    std::unique_ptr<TemporaryFile> mTempFile;
-};
+    return cachefileName;
+}
 
-TEST_F(EGLCacheSerializationTest, ReinitializedCacheContainsValues) {
+TEST_F(EGLCacheTest, ModifiedCacheMisses) {
     uint8_t buf[4] = { 0xee, 0xee, 0xee, 0xee };
-    mCache->setCacheFilename(&mTempFile->path[0]);
     mCache->initialize(egl_display_t::get(EGL_DEFAULT_DISPLAY));
+
     mCache->setBlob("abcd", 4, "efgh", 4);
+    ASSERT_EQ(4, mCache->getBlob("abcd", 4, buf, 4));
+    ASSERT_EQ('e', buf[0]);
+    ASSERT_EQ('f', buf[1]);
+    ASSERT_EQ('g', buf[2]);
+    ASSERT_EQ('h', buf[3]);
+
+    // Depending on the cache mode, the file will be in different locations
+    std::string cachefileName = getCachefileName();
+    ASSERT_TRUE(cachefileName.length() > 0);
+
+    // Ensure the cache file is written to disk
     mCache->terminate();
+
+    // Stomp on the beginning of the cache file, breaking the key match
+    const long stomp = 0xbadf00d;
+    FILE *file = fopen(cachefileName.c_str(), "w");
+    fprintf(file, "%ld", stomp);
+    fflush(file);
+    fclose(file);
+
+    // Ensure no cache hit
     mCache->initialize(egl_display_t::get(EGL_DEFAULT_DISPLAY));
+    uint8_t buf2[4] = { 0xee, 0xee, 0xee, 0xee };
+    ASSERT_EQ(0, mCache->getBlob("abcd", 4, buf2, 4));
+    ASSERT_EQ(0xee, buf2[0]);
+    ASSERT_EQ(0xee, buf2[1]);
+    ASSERT_EQ(0xee, buf2[2]);
+    ASSERT_EQ(0xee, buf2[3]);
+}
+
+TEST_F(EGLCacheTest, TerminatedCacheBelowCacheLimit) {
+    uint8_t buf[4] = { 0xee, 0xee, 0xee, 0xee };
+    mCache->initialize(egl_display_t::get(EGL_DEFAULT_DISPLAY));
+
+    mCache->setBlob("abcd", 4, "efgh", 4);
     ASSERT_EQ(4, mCache->getBlob("abcd", 4, buf, 4));
     ASSERT_EQ('e', buf[0]);
     ASSERT_EQ('f', buf[1]);
     ASSERT_EQ('g', buf[2]);
     ASSERT_EQ('h', buf[3]);
+
+    mCache->setBlob("ijkl", 4, "mnop", 4);
+    ASSERT_EQ(4, mCache->getBlob("ijkl", 4, buf, 4));
+    ASSERT_EQ('m', buf[0]);
+    ASSERT_EQ('n', buf[1]);
+    ASSERT_EQ('o', buf[2]);
+    ASSERT_EQ('p', buf[3]);
+
+    mCache->setBlob("qrst", 4, "uvwx", 4);
+    ASSERT_EQ(4, mCache->getBlob("qrst", 4, buf, 4));
+    ASSERT_EQ('u', buf[0]);
+    ASSERT_EQ('v', buf[1]);
+    ASSERT_EQ('w', buf[2]);
+    ASSERT_EQ('x', buf[3]);
+
+    // Cache should contain both the key and the value
+    // So 8 bytes per entry, at least 24 bytes
+    ASSERT_GE(mCache->getCacheSize(), 24);
+    mCache->setCacheLimit(4);
+    mCache->terminate();
+    ASSERT_LE(mCache->getCacheSize(), 4);
 }
 
 }