diff options
Diffstat (limited to 'opengl')
| -rw-r--r-- | opengl/Android.bp | 4 | ||||
| -rw-r--r-- | opengl/libs/EGL/MultifileBlobCache.cpp | 224 | ||||
| -rw-r--r-- | opengl/libs/EGL/MultifileBlobCache.h | 31 | ||||
| -rw-r--r-- | opengl/libs/EGL/MultifileBlobCache_test.cpp | 253 | ||||
| -rw-r--r-- | opengl/libs/EGL/egl_cache.cpp | 3 | ||||
| -rw-r--r-- | opengl/libs/EGL/fuzzer/MultifileBlobCache_fuzzer.cpp | 11 | ||||
| -rw-r--r-- | opengl/tests/EGLTest/egl_cache_test.cpp | 11 |
7 files changed, 500 insertions, 37 deletions
diff --git a/opengl/Android.bp b/opengl/Android.bp index b15694bf50..4454f36b67 100644 --- a/opengl/Android.bp +++ b/opengl/Android.bp @@ -72,6 +72,10 @@ cc_library_headers { llndk: { llndk_headers: true, }, + apex_available: [ + "//apex_available:platform", + "com.android.virt", + ], } subdirs = [ diff --git a/opengl/libs/EGL/MultifileBlobCache.cpp b/opengl/libs/EGL/MultifileBlobCache.cpp index ed3c616b92..9905210014 100644 --- a/opengl/libs/EGL/MultifileBlobCache.cpp +++ b/opengl/libs/EGL/MultifileBlobCache.cpp @@ -18,6 +18,7 @@ #include "MultifileBlobCache.h" +#include <android-base/properties.h> #include <dirent.h> #include <fcntl.h> #include <inttypes.h> @@ -62,12 +63,15 @@ void freeHotCacheEntry(android::MultifileHotCache& entry) { namespace android { MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, size_t maxTotalSize, - const std::string& baseDir) + size_t maxTotalEntries, const std::string& baseDir) : mInitialized(false), + mCacheVersion(0), mMaxKeySize(maxKeySize), mMaxValueSize(maxValueSize), mMaxTotalSize(maxTotalSize), + mMaxTotalEntries(maxTotalEntries), mTotalCacheSize(0), + mTotalCacheEntries(0), mHotCacheLimit(0), mHotCacheSize(0), mWorkerThreadIdle(true) { @@ -76,6 +80,26 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s return; } + // Set the cache version, override if debug value set + mCacheVersion = kMultifileBlobCacheVersion; + int debugCacheVersion = base::GetIntProperty("debug.egl.blobcache.cache_version", -1); + if (debugCacheVersion >= 0) { + ALOGV("INIT: Using %u as cacheVersion instead of %u", debugCacheVersion, mCacheVersion); + mCacheVersion = debugCacheVersion; + } + + // Set the platform build ID, override if debug value set + mBuildId = base::GetProperty("ro.build.id", ""); + std::string debugBuildId = base::GetProperty("debug.egl.blobcache.build_id", ""); + if (!debugBuildId.empty()) { + ALOGV("INIT: Using %s as buildId instead of %s", debugBuildId.c_str(), mBuildId.c_str()); + if (debugBuildId.length() > PROP_VALUE_MAX) { + ALOGV("INIT: debugBuildId is too long (%zu), reduce it to %u", debugBuildId.length(), + PROP_VALUE_MAX); + } + mBuildId = debugBuildId; + } + // Establish the name of our multifile directory mMultifileDirName = baseDir + ".multifile"; @@ -93,14 +117,30 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s mTaskThread = std::thread(&MultifileBlobCache::processTasks, this); // See if the dir exists, and initialize using its contents + bool statusGood = false; + + // Check that our cacheVersion and buildId match struct stat st; if (stat(mMultifileDirName.c_str(), &st) == 0) { + if (checkStatus(mMultifileDirName.c_str())) { + statusGood = true; + } else { + ALOGV("INIT: Cache status has changed, clearing the cache"); + if (!clearCache()) { + ALOGE("INIT: Unable to clear cache"); + return; + } + } + } + + if (statusGood) { // Read all the files and gather details, then preload their contents DIR* dir; struct dirent* entry; if ((dir = opendir(mMultifileDirName.c_str())) != nullptr) { while ((entry = readdir(dir)) != nullptr) { - if (entry->d_name == "."s || entry->d_name == ".."s) { + if (entry->d_name == "."s || entry->d_name == ".."s || + strcmp(entry->d_name, kMultifileBlobCacheStatusFile) == 0) { continue; } @@ -123,7 +163,8 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s if (st.st_size <= 0 || st.st_atime <= 0) { ALOGE("INIT: Entry %u has invalid stats! Removing.", entryHash); if (remove(fullPath.c_str()) != 0) { - ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno)); + ALOGE("INIT: Error removing %s: %s", fullPath.c_str(), + std::strerror(errno)); } continue; } @@ -140,7 +181,7 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s MultifileHeader header; size_t result = read(fd, static_cast<void*>(&header), sizeof(MultifileHeader)); if (result != sizeof(MultifileHeader)) { - ALOGE("Error reading MultifileHeader from cache entry (%s): %s", + ALOGE("INIT: Error reading MultifileHeader from cache entry (%s): %s", fullPath.c_str(), std::strerror(errno)); close(fd); return; @@ -150,7 +191,8 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s if (header.magic != kMultifileMagic) { ALOGE("INIT: Entry %u has bad magic (%u)! Removing.", entryHash, header.magic); if (remove(fullPath.c_str()) != 0) { - ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno)); + ALOGE("INIT: Error removing %s: %s", fullPath.c_str(), + std::strerror(errno)); } close(fd); continue; @@ -175,7 +217,7 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s if (header.crc != crc32c(mappedEntry + sizeof(MultifileHeader), fileSize - sizeof(MultifileHeader))) { - ALOGE("INIT: Entry %u failed CRC check! Removing.", entryHash); + ALOGV("INIT: Entry %u failed CRC check! Removing.", entryHash); if (remove(fullPath.c_str()) != 0) { ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno)); } @@ -184,11 +226,12 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s // If the cache entry is damaged or no good, remove it if (header.keySize <= 0 || header.valueSize <= 0) { - ALOGE("INIT: Entry %u has a bad header keySize (%lu) or valueSize (%lu), " + ALOGV("INIT: Entry %u has a bad header keySize (%lu) or valueSize (%lu), " "removing.", entryHash, header.keySize, header.valueSize); if (remove(fullPath.c_str()) != 0) { - ALOGE("Error removing %s: %s", fullPath.c_str(), std::strerror(errno)); + ALOGE("INIT: Error removing %s: %s", fullPath.c_str(), + std::strerror(errno)); } continue; } @@ -226,9 +269,17 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s // If the multifile directory does not exist, create it and start from scratch if (mkdir(mMultifileDirName.c_str(), 0755) != 0 && (errno != EEXIST)) { ALOGE("Unable to create directory (%s), errno (%i)", mMultifileDirName.c_str(), errno); + return; + } + + // Create new status file + if (!createStatus(mMultifileDirName.c_str())) { + ALOGE("INIT: Failed to create status file!"); + return; } } + ALOGV("INIT: Multifile BlobCache initialization succeeded"); mInitialized = true; } @@ -270,7 +321,7 @@ void MultifileBlobCache::set(const void* key, EGLsizeiANDROID keySize, const voi size_t fileSize = sizeof(MultifileHeader) + keySize + valueSize; // If we're going to be over the cache limit, kick off a trim to clear space - if (getTotalSize() + fileSize > mMaxTotalSize) { + if (getTotalSize() + fileSize > mMaxTotalSize || getTotalEntries() + 1 > mMaxTotalEntries) { ALOGV("SET: Cache is full, calling trimCache to clear space"); trimCache(); } @@ -469,6 +520,112 @@ void MultifileBlobCache::finish() { } } +bool MultifileBlobCache::createStatus(const std::string& baseDir) { + // Populate the status struct + struct MultifileStatus status; + memset(&status, 0, sizeof(status)); + status.magic = kMultifileMagic; + status.cacheVersion = mCacheVersion; + + // Copy the buildId string in, up to our allocated space + strncpy(status.buildId, mBuildId.c_str(), + mBuildId.length() > PROP_VALUE_MAX ? PROP_VALUE_MAX : mBuildId.length()); + + // Finally update the crc, using cacheVersion and everything the follows + status.crc = + crc32c(reinterpret_cast<uint8_t*>(&status) + offsetof(MultifileStatus, cacheVersion), + sizeof(status) - offsetof(MultifileStatus, cacheVersion)); + + // Create the status file + std::string cacheStatus = baseDir + "/" + kMultifileBlobCacheStatusFile; + int fd = open(cacheStatus.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (fd == -1) { + ALOGE("STATUS(CREATE): Unable to create status file: %s, error: %s", cacheStatus.c_str(), + std::strerror(errno)); + return false; + } + + // Write the buffer contents to disk + ssize_t result = write(fd, &status, sizeof(status)); + close(fd); + if (result != sizeof(status)) { + ALOGE("STATUS(CREATE): Error writing cache status file: %s, error %s", cacheStatus.c_str(), + std::strerror(errno)); + return false; + } + + ALOGV("STATUS(CREATE): Created status file: %s", cacheStatus.c_str()); + return true; +} + +bool MultifileBlobCache::checkStatus(const std::string& baseDir) { + std::string cacheStatus = baseDir + "/" + kMultifileBlobCacheStatusFile; + + // Does status exist + struct stat st; + if (stat(cacheStatus.c_str(), &st) != 0) { + ALOGV("STATUS(CHECK): Status file (%s) missing", cacheStatus.c_str()); + return false; + } + + // If the status entry is damaged or no good, remove it + if (st.st_size <= 0 || st.st_atime <= 0) { + ALOGE("STATUS(CHECK): Cache status has invalid stats!"); + return false; + } + + // Open the file so we can read its header + int fd = open(cacheStatus.c_str(), O_RDONLY); + if (fd == -1) { + ALOGE("STATUS(CHECK): Cache error - failed to open cacheStatus: %s, error: %s", + cacheStatus.c_str(), std::strerror(errno)); + return false; + } + + // Read in the status header + MultifileStatus status; + size_t result = read(fd, static_cast<void*>(&status), sizeof(MultifileStatus)); + close(fd); + if (result != sizeof(MultifileStatus)) { + ALOGE("STATUS(CHECK): Error reading cache status (%s): %s", cacheStatus.c_str(), + std::strerror(errno)); + return false; + } + + // Verify header magic + if (status.magic != kMultifileMagic) { + ALOGE("STATUS(CHECK): Cache status has bad magic (%u)!", status.magic); + return false; + } + + // Ensure we have a good CRC + if (status.crc != + crc32c(reinterpret_cast<uint8_t*>(&status) + offsetof(MultifileStatus, cacheVersion), + sizeof(status) - offsetof(MultifileStatus, cacheVersion))) { + ALOGE("STATUS(CHECK): Cache status failed CRC check!"); + return false; + } + + // Check cacheVersion + if (status.cacheVersion != mCacheVersion) { + ALOGV("STATUS(CHECK): Cache version has changed! old(%u) new(%u)", status.cacheVersion, + mCacheVersion); + return false; + } + + // Check buildId + if (strcmp(status.buildId, mBuildId.c_str()) != 0) { + ALOGV("STATUS(CHECK): BuildId has changed! old(%s) new(%s)", status.buildId, + mBuildId.c_str()); + return false; + } + + // All checks passed! + ALOGV("STATUS(CHECK): Status file is good! cacheVersion(%u), buildId(%s) file(%s)", + status.cacheVersion, status.buildId, cacheStatus.c_str()); + return true; +} + void MultifileBlobCache::trackEntry(uint32_t entryHash, EGLsizeiANDROID valueSize, size_t fileSize, time_t accessTime) { mEntries.insert(entryHash); @@ -485,10 +642,12 @@ MultifileEntryStats MultifileBlobCache::getEntryStats(uint32_t entryHash) { void MultifileBlobCache::increaseTotalCacheSize(size_t fileSize) { mTotalCacheSize += fileSize; + mTotalCacheEntries++; } void MultifileBlobCache::decreaseTotalCacheSize(size_t fileSize) { mTotalCacheSize -= fileSize; + mTotalCacheEntries--; } bool MultifileBlobCache::addToHotCache(uint32_t newEntryHash, int newFd, uint8_t* newEntryBuffer, @@ -557,7 +716,7 @@ bool MultifileBlobCache::removeFromHotCache(uint32_t entryHash) { return false; } -bool MultifileBlobCache::applyLRU(size_t cacheLimit) { +bool MultifileBlobCache::applyLRU(size_t cacheSizeLimit, size_t cacheEntryLimit) { // Walk through our map of sorted last access times and remove files until under the limit for (auto cacheEntryIter = mEntryStats.begin(); cacheEntryIter != mEntryStats.end();) { uint32_t entryHash = cacheEntryIter->first; @@ -590,9 +749,10 @@ bool MultifileBlobCache::applyLRU(size_t cacheLimit) { // See if it has been reduced enough size_t totalCacheSize = getTotalSize(); - if (totalCacheSize <= cacheLimit) { + size_t totalCacheEntries = getTotalEntries(); + if (totalCacheSize <= cacheSizeLimit && totalCacheEntries <= cacheEntryLimit) { // Success - ALOGV("LRU: Reduced cache to %zu", totalCacheSize); + ALOGV("LRU: Reduced cache to size %zu entries %zu", totalCacheSize, totalCacheEntries); return true; } } @@ -601,8 +761,43 @@ bool MultifileBlobCache::applyLRU(size_t cacheLimit) { return false; } +// Clear the cache by removing all entries and deleting the directory +bool MultifileBlobCache::clearCache() { + DIR* dir; + struct dirent* entry; + dir = opendir(mMultifileDirName.c_str()); + if (dir == nullptr) { + ALOGE("CLEAR: Unable to open multifile dir: %s", mMultifileDirName.c_str()); + return false; + } + + // Delete all entries and the status file + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_name == "."s || entry->d_name == ".."s) { + continue; + } + + std::string entryName = entry->d_name; + std::string fullPath = mMultifileDirName + "/" + entryName; + if (remove(fullPath.c_str()) != 0) { + ALOGE("CLEAR: Error removing %s: %s", fullPath.c_str(), std::strerror(errno)); + return false; + } + } + + // Delete the directory + if (remove(mMultifileDirName.c_str()) != 0) { + ALOGE("CLEAR: Error removing %s: %s", mMultifileDirName.c_str(), std::strerror(errno)); + return false; + } + + ALOGV("CLEAR: Cleared the multifile blobcache"); + return true; +} + // When removing files, what fraction of the overall limit should be reached when removing files // A divisor of two will decrease the cache to 50%, four to 25% and so on +// We use the same limit to manage size and entry count constexpr uint32_t kCacheLimitDivisor = 2; // Calculate the cache size and remove old entries until under the limit @@ -611,8 +806,9 @@ void MultifileBlobCache::trimCache() { ALOGV("TRIM: Waiting for work to complete."); waitForWorkComplete(); - ALOGV("TRIM: Reducing multifile cache size to %zu", mMaxTotalSize / kCacheLimitDivisor); - if (!applyLRU(mMaxTotalSize / kCacheLimitDivisor)) { + ALOGV("TRIM: Reducing multifile cache size to %zu, entries %zu", + mMaxTotalSize / kCacheLimitDivisor, mMaxTotalEntries / kCacheLimitDivisor); + if (!applyLRU(mMaxTotalSize / kCacheLimitDivisor, mMaxTotalEntries / kCacheLimitDivisor)) { ALOGE("Error when clearing multifile shader cache"); return; } diff --git a/opengl/libs/EGL/MultifileBlobCache.h b/opengl/libs/EGL/MultifileBlobCache.h index 5e527dcf35..18566c2892 100644 --- a/opengl/libs/EGL/MultifileBlobCache.h +++ b/opengl/libs/EGL/MultifileBlobCache.h @@ -21,6 +21,7 @@ #include <EGL/eglext.h> #include <android-base/thread_annotations.h> +#include <cutils/properties.h> #include <future> #include <map> #include <queue> @@ -33,6 +34,9 @@ namespace android { +constexpr uint32_t kMultifileBlobCacheVersion = 1; +constexpr char kMultifileBlobCacheStatusFile[] = "cache.status"; + struct MultifileHeader { uint32_t magic; uint32_t crc; @@ -46,6 +50,13 @@ struct MultifileEntryStats { time_t accessTime; }; +struct MultifileStatus { + uint32_t magic; + uint32_t crc; + uint32_t cacheVersion; + char buildId[PROP_VALUE_MAX]; +}; + struct MultifileHotCache { int entryFd; uint8_t* entryBuffer; @@ -92,7 +103,7 @@ private: class MultifileBlobCache { public: MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, size_t maxTotalSize, - const std::string& baseDir); + size_t maxTotalEntries, const std::string& baseDir); ~MultifileBlobCache(); void set(const void* key, EGLsizeiANDROID keySize, const void* value, @@ -103,6 +114,13 @@ public: void finish(); size_t getTotalSize() const { return mTotalCacheSize; } + size_t getTotalEntries() const { return mTotalCacheEntries; } + + const std::string& getCurrentBuildId() const { return mBuildId; } + void setCurrentBuildId(const std::string& buildId) { mBuildId = buildId; } + + uint32_t getCurrentCacheVersion() const { return mCacheVersion; } + void setCurrentCacheVersion(uint32_t cacheVersion) { mCacheVersion = cacheVersion; } private: void trackEntry(uint32_t entryHash, EGLsizeiANDROID valueSize, size_t fileSize, @@ -111,6 +129,9 @@ private: bool removeEntry(uint32_t entryHash); MultifileEntryStats getEntryStats(uint32_t entryHash); + bool createStatus(const std::string& baseDir); + bool checkStatus(const std::string& baseDir); + size_t getFileSize(uint32_t entryHash); size_t getValueSize(uint32_t entryHash); @@ -120,12 +141,16 @@ private: bool addToHotCache(uint32_t entryHash, int fd, uint8_t* entryBufer, size_t entrySize); bool removeFromHotCache(uint32_t entryHash); + bool clearCache(); void trimCache(); - bool applyLRU(size_t cacheLimit); + bool applyLRU(size_t cacheSizeLimit, size_t cacheEntryLimit); bool mInitialized; std::string mMultifileDirName; + std::string mBuildId; + uint32_t mCacheVersion; + std::unordered_set<uint32_t> mEntries; std::unordered_map<uint32_t, MultifileEntryStats> mEntryStats; std::unordered_map<uint32_t, MultifileHotCache> mHotCache; @@ -133,7 +158,9 @@ private: size_t mMaxKeySize; size_t mMaxValueSize; size_t mMaxTotalSize; + size_t mMaxTotalEntries; size_t mTotalCacheSize; + size_t mTotalCacheEntries; size_t mHotCacheLimit; size_t mHotCacheEntryLimit; size_t mHotCacheSize; diff --git a/opengl/libs/EGL/MultifileBlobCache_test.cpp b/opengl/libs/EGL/MultifileBlobCache_test.cpp index 1639be6480..90a0f1ee06 100644 --- a/opengl/libs/EGL/MultifileBlobCache_test.cpp +++ b/opengl/libs/EGL/MultifileBlobCache_test.cpp @@ -16,13 +16,17 @@ #include "MultifileBlobCache.h" +#include <android-base/properties.h> #include <android-base/test_utils.h> #include <fcntl.h> #include <gtest/gtest.h> #include <stdio.h> +#include <fstream> #include <memory> +using namespace std::literals; + namespace android { template <typename T> @@ -31,23 +35,40 @@ using sp = std::shared_ptr<T>; constexpr size_t kMaxKeySize = 2 * 1024; constexpr size_t kMaxValueSize = 6 * 1024; constexpr size_t kMaxTotalSize = 32 * 1024; +constexpr size_t kMaxTotalEntries = 64; class MultifileBlobCacheTest : public ::testing::Test { protected: virtual void SetUp() { + clearProperties(); mTempFile.reset(new TemporaryFile()); mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, - &mTempFile->path[0])); + kMaxTotalEntries, &mTempFile->path[0])); } - virtual void TearDown() { mMBC.reset(); } + virtual void TearDown() { + clearProperties(); + mMBC.reset(); + } int getFileDescriptorCount(); + std::vector<std::string> getCacheEntries(); + + void clearProperties(); std::unique_ptr<TemporaryFile> mTempFile; std::unique_ptr<MultifileBlobCache> mMBC; }; +void MultifileBlobCacheTest::clearProperties() { + // Clear any debug properties used in the tests + base::SetProperty("debug.egl.blobcache.cache_version", ""); + base::WaitForProperty("debug.egl.blobcache.cache_version", ""); + + base::SetProperty("debug.egl.blobcache.build_id", ""); + base::WaitForProperty("debug.egl.blobcache.build_id", ""); +} + TEST_F(MultifileBlobCacheTest, CacheSingleValueSucceeds) { unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee}; mMBC->set("abcd", 4, "efgh", 4); @@ -211,6 +232,23 @@ TEST_F(MultifileBlobCacheTest, CacheMaxKeyAndValueSizeSucceeds) { } } +TEST_F(MultifileBlobCacheTest, CacheMaxEntrySucceeds) { + // Fill the cache with max entries + int i = 0; + for (i = 0; i < kMaxTotalEntries; i++) { + mMBC->set(std::to_string(i).c_str(), sizeof(i), std::to_string(i).c_str(), sizeof(i)); + } + + // Ensure it is full + ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries); + + // Add another entry + mMBC->set(std::to_string(i).c_str(), sizeof(i), std::to_string(i).c_str(), sizeof(i)); + + // Ensure total entries is cut in half + 1 + ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries / 2 + 1); +} + TEST_F(MultifileBlobCacheTest, CacheMinKeyAndValueSizeSucceeds) { unsigned char buf[1] = {0xee}; mMBC->set("x", 1, "y", 1); @@ -234,8 +272,7 @@ int MultifileBlobCacheTest::getFileDescriptorCount() { TEST_F(MultifileBlobCacheTest, EnsureFileDescriptorsClosed) { // Populate the cache with a bunch of entries - size_t kLargeNumberOfEntries = 1024; - for (int i = 0; i < kLargeNumberOfEntries; i++) { + for (int i = 0; i < kMaxTotalEntries; i++) { // printf("Caching: %i", i); // Use the index as the key and value @@ -247,27 +284,223 @@ TEST_F(MultifileBlobCacheTest, EnsureFileDescriptorsClosed) { } // Ensure we don't have a bunch of open fds - ASSERT_LT(getFileDescriptorCount(), kLargeNumberOfEntries / 2); + ASSERT_LT(getFileDescriptorCount(), kMaxTotalEntries / 2); // Close the cache so everything writes out mMBC->finish(); mMBC.reset(); // Now open it again and ensure we still don't have a bunch of open fds - mMBC.reset( - new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, &mTempFile->path[0])); + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); // Check after initialization - ASSERT_LT(getFileDescriptorCount(), kLargeNumberOfEntries / 2); + ASSERT_LT(getFileDescriptorCount(), kMaxTotalEntries / 2); - for (int i = 0; i < kLargeNumberOfEntries; i++) { + for (int i = 0; i < kMaxTotalEntries; i++) { int result = 0; ASSERT_EQ(sizeof(i), mMBC->get(&i, sizeof(i), &result, sizeof(result))); ASSERT_EQ(i, result); } // And again after we've actually used it - ASSERT_LT(getFileDescriptorCount(), kLargeNumberOfEntries / 2); + ASSERT_LT(getFileDescriptorCount(), kMaxTotalEntries / 2); +} + +std::vector<std::string> MultifileBlobCacheTest::getCacheEntries() { + std::string cachePath = &mTempFile->path[0]; + std::string multifileDirName = cachePath + ".multifile"; + std::vector<std::string> cacheEntries; + + struct stat info; + if (stat(multifileDirName.c_str(), &info) == 0) { + // We have a multifile dir. Skip the status file and return the only entry. + DIR* dir; + struct dirent* entry; + if ((dir = opendir(multifileDirName.c_str())) != nullptr) { + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_name == "."s || entry->d_name == ".."s) { + continue; + } + if (strcmp(entry->d_name, kMultifileBlobCacheStatusFile) == 0) { + continue; + } + cacheEntries.push_back(multifileDirName + "/" + entry->d_name); + } + } else { + printf("Unable to open %s, error: %s\n", multifileDirName.c_str(), + std::strerror(errno)); + } + } else { + printf("Unable to stat %s, error: %s\n", multifileDirName.c_str(), std::strerror(errno)); + } + + return cacheEntries; +} + +TEST_F(MultifileBlobCacheTest, CacheContainsStatus) { + struct stat info; + std::stringstream statusFile; + statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile; + + // After INIT, cache should have a status + ASSERT_TRUE(stat(statusFile.str().c_str(), &info) == 0); + + // Set one entry + mMBC->set("abcd", 4, "efgh", 4); + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Ensure status lives after closing the cache + ASSERT_TRUE(stat(statusFile.str().c_str(), &info) == 0); + + // Open the cache again + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Ensure we still have a status + ASSERT_TRUE(stat(statusFile.str().c_str(), &info) == 0); +} + +// Verify missing cache status file causes cache the be cleared +TEST_F(MultifileBlobCacheTest, MissingCacheStatusClears) { + // Set one entry + mMBC->set("abcd", 4, "efgh", 4); + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Ensure there is one cache entry + ASSERT_EQ(getCacheEntries().size(), 1); + + // Delete the status file + std::stringstream statusFile; + statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile; + remove(statusFile.str().c_str()); + + // Open the cache again and ensure no cache hits + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Ensure we have no entries + ASSERT_EQ(getCacheEntries().size(), 0); +} + +// Verify modified cache status file BEGIN causes cache to be cleared +TEST_F(MultifileBlobCacheTest, ModifiedCacheStatusBeginClears) { + // Set one entry + mMBC->set("abcd", 4, "efgh", 4); + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Ensure there is one cache entry + ASSERT_EQ(getCacheEntries().size(), 1); + + // Modify the status file + std::stringstream statusFile; + statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile; + + // Stomp on the beginning of the cache file + const char* stomp = "BADF00D"; + std::fstream fs(statusFile.str()); + fs.seekp(0, std::ios_base::beg); + fs.write(stomp, strlen(stomp)); + fs.flush(); + fs.close(); + + // Open the cache again and ensure no cache hits + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Ensure we have no entries + ASSERT_EQ(getCacheEntries().size(), 0); +} + +// Verify modified cache status file END causes cache to be cleared +TEST_F(MultifileBlobCacheTest, ModifiedCacheStatusEndClears) { + // Set one entry + mMBC->set("abcd", 4, "efgh", 4); + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Ensure there is one cache entry + ASSERT_EQ(getCacheEntries().size(), 1); + + // Modify the status file + std::stringstream statusFile; + statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile; + + // Stomp on the END of the cache status file, modifying its contents + const char* stomp = "BADF00D"; + std::fstream fs(statusFile.str()); + fs.seekp(-strlen(stomp), std::ios_base::end); + fs.write(stomp, strlen(stomp)); + fs.flush(); + fs.close(); + + // Open the cache again and ensure no cache hits + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Ensure we have no entries + ASSERT_EQ(getCacheEntries().size(), 0); +} + +// Verify mismatched cacheVersion causes cache to be cleared +TEST_F(MultifileBlobCacheTest, MismatchedCacheVersionClears) { + // Set one entry + mMBC->set("abcd", 4, "efgh", 4); + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Ensure there is one cache entry + ASSERT_EQ(getCacheEntries().size(), 1); + + // Set a debug cacheVersion + std::string newCacheVersion = std::to_string(kMultifileBlobCacheVersion + 1); + ASSERT_TRUE(base::SetProperty("debug.egl.blobcache.cache_version", newCacheVersion.c_str())); + ASSERT_TRUE( + base::WaitForProperty("debug.egl.blobcache.cache_version", newCacheVersion.c_str())); + + // Open the cache again and ensure no cache hits + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Ensure we have no entries + ASSERT_EQ(getCacheEntries().size(), 0); +} + +// Verify mismatched buildId causes cache to be cleared +TEST_F(MultifileBlobCacheTest, MismatchedBuildIdClears) { + // Set one entry + mMBC->set("abcd", 4, "efgh", 4); + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Ensure there is one cache entry + ASSERT_EQ(getCacheEntries().size(), 1); + + // Set a debug buildId + base::SetProperty("debug.egl.blobcache.build_id", "foo"); + base::WaitForProperty("debug.egl.blobcache.build_id", "foo"); + + // Open the cache again and ensure no cache hits + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Ensure we have no entries + ASSERT_EQ(getCacheEntries().size(), 0); } } // namespace android diff --git a/opengl/libs/EGL/egl_cache.cpp b/opengl/libs/EGL/egl_cache.cpp index 1b68344cb1..98ff1d12cc 100644 --- a/opengl/libs/EGL/egl_cache.cpp +++ b/opengl/libs/EGL/egl_cache.cpp @@ -41,6 +41,7 @@ static const unsigned int kDeferredMonolithicSaveDelay = 4; constexpr uint32_t kMaxMultifileKeySize = 1 * 1024 * 1024; constexpr uint32_t kMaxMultifileValueSize = 8 * 1024 * 1024; constexpr uint32_t kMaxMultifileTotalSize = 32 * 1024 * 1024; +constexpr uint32_t kMaxMultifileTotalEntries = 4 * 1024; namespace android { @@ -277,7 +278,7 @@ MultifileBlobCache* egl_cache_t::getMultifileBlobCacheLocked() { if (mMultifileBlobCache == nullptr) { mMultifileBlobCache.reset(new MultifileBlobCache(kMaxMultifileKeySize, kMaxMultifileValueSize, mCacheByteLimit, - mFilename)); + kMaxMultifileTotalEntries, mFilename)); } return mMultifileBlobCache.get(); } diff --git a/opengl/libs/EGL/fuzzer/MultifileBlobCache_fuzzer.cpp b/opengl/libs/EGL/fuzzer/MultifileBlobCache_fuzzer.cpp index 633cc9c51b..2d9ee3a49e 100644 --- a/opengl/libs/EGL/fuzzer/MultifileBlobCache_fuzzer.cpp +++ b/opengl/libs/EGL/fuzzer/MultifileBlobCache_fuzzer.cpp @@ -28,6 +28,7 @@ namespace android { constexpr size_t kMaxKeySize = 2 * 1024; constexpr size_t kMaxValueSize = 6 * 1024; constexpr size_t kMaxTotalSize = 32 * 1024; +constexpr size_t kMaxTotalEntries = 64; extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // To fuzz this, we're going to create a key/value pair from data @@ -79,8 +80,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { std::unique_ptr<MultifileBlobCache> mbc; tempFile.reset(new TemporaryFile()); - mbc.reset( - new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, &tempFile->path[0])); + mbc.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &tempFile->path[0])); // With remaining data, select different paths below int loopCount = 1; uint8_t bumpCount = 0; @@ -131,8 +132,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // Place the maxKey/maxValue twice // The first will fit, the second will trigger hot cache trimming tempFile.reset(new TemporaryFile()); - mbc.reset( - new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, &tempFile->path[0])); + mbc.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &tempFile->path[0])); uint8_t* buffer = new uint8_t[kMaxValueSize]; mbc->set(maxKey1.data(), kMaxKeySize, maxValue1.data(), kMaxValueSize); mbc->set(maxKey2.data(), kMaxKeySize, maxValue2.data(), kMaxValueSize); @@ -145,7 +146,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // overflow tempFile.reset(new TemporaryFile()); mbc.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, 2 * (kMaxKeySize + kMaxValueSize), - &tempFile->path[0])); + kMaxTotalEntries, &tempFile->path[0])); mbc->set(maxKey1.data(), kMaxKeySize, maxValue1.data(), kMaxValueSize); mbc->set(maxKey2.data(), kMaxKeySize, maxValue2.data(), kMaxValueSize); mbc->get(maxKey1.data(), kMaxKeySize, buffer, kMaxValueSize); diff --git a/opengl/tests/EGLTest/egl_cache_test.cpp b/opengl/tests/EGLTest/egl_cache_test.cpp index f81c68f66e..ce5818229a 100644 --- a/opengl/tests/EGLTest/egl_cache_test.cpp +++ b/opengl/tests/EGLTest/egl_cache_test.cpp @@ -114,25 +114,26 @@ std::string EGLCacheTest::getCachefileName() { struct stat info; if (stat(multifileDirName.c_str(), &info) == 0) { // Ensure we only have one file to manage - int realFileCount = 0; + int entryFileCount = 0; - // We have a multifile dir. Return the only real file in it. + // We have a multifile dir. Return the only entry file in it. DIR* dir; struct dirent* entry; if ((dir = opendir(multifileDirName.c_str())) != nullptr) { while ((entry = readdir(dir)) != nullptr) { - if (entry->d_name == "."s || entry->d_name == ".."s) { + if (entry->d_name == "."s || entry->d_name == ".."s || + strcmp(entry->d_name, kMultifileBlobCacheStatusFile) == 0) { continue; } cachefileName = multifileDirName + "/" + entry->d_name; - realFileCount++; + entryFileCount++; } } else { printf("Unable to open %s, error: %s\n", multifileDirName.c_str(), std::strerror(errno)); } - if (realFileCount != 1) { + if (entryFileCount != 1) { // If there was more than one real file in the directory, this // violates test assumptions cachefileName = ""; |