diff options
Diffstat (limited to 'opengl/libs')
| -rw-r--r-- | opengl/libs/Android.bp | 15 | ||||
| -rw-r--r-- | opengl/libs/EGL/MultifileBlobCache.cpp | 233 | ||||
| -rw-r--r-- | opengl/libs/EGL/MultifileBlobCache.h | 37 | ||||
| -rw-r--r-- | opengl/libs/EGL/MultifileBlobCache_test.cpp | 413 | ||||
| -rw-r--r-- | opengl/libs/EGL/egl_flags.aconfig | 13 | ||||
| -rw-r--r-- | opengl/libs/EGL/fuzzer/Android.bp | 1 |
6 files changed, 672 insertions, 40 deletions
diff --git a/opengl/libs/Android.bp b/opengl/libs/Android.bp index b19a862b6c..91250b9945 100644 --- a/opengl/libs/Android.bp +++ b/opengl/libs/Android.bp @@ -63,6 +63,18 @@ ndk_library { unversioned_until: "current", } +aconfig_declarations { + name: "egl_flags", + package: "com.android.graphics.egl.flags", + container: "system", + srcs: ["EGL/egl_flags.aconfig"], +} + +cc_aconfig_library { + name: "libegl_flags", + aconfig_declarations: "egl_flags", +} + cc_defaults { name: "gl_libs_defaults", cflags: [ @@ -136,6 +148,7 @@ cc_library_static { ], export_include_dirs: ["EGL"], shared_libs: [ + "libegl_flags", "libz", ], } @@ -166,6 +179,7 @@ cc_library_shared { "android.hardware.configstore@1.0", "android.hardware.configstore-utils", "libbase", + "libegl_flags", "libhidlbase", "libnativebridge_lazy", "libnativeloader_lazy", @@ -202,6 +216,7 @@ cc_test { "EGL/MultifileBlobCache_test.cpp", ], shared_libs: [ + "libegl_flags", "libutils", "libz", ], diff --git a/opengl/libs/EGL/MultifileBlobCache.cpp b/opengl/libs/EGL/MultifileBlobCache.cpp index f7e33b383f..04c525e93d 100644 --- a/opengl/libs/EGL/MultifileBlobCache.cpp +++ b/opengl/libs/EGL/MultifileBlobCache.cpp @@ -38,11 +38,20 @@ #include <utils/JenkinsHash.h> +#include <com_android_graphics_egl_flags.h> + +using namespace com::android::graphics::egl; + using namespace std::literals; constexpr uint32_t kMultifileMagic = 'MFB$'; constexpr uint32_t kCrcPlaceholder = 0; +// When removing files, what fraction of the overall limit should be reached when removing files +// A divisor of two will decrease the cache to 50%, four to 25% and so on +// We use the same limit to manage size and entry count +constexpr uint32_t kCacheLimitDivisor = 2; + namespace { // Helper function to close entries or free them @@ -72,6 +81,7 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s mMaxTotalEntries(maxTotalEntries), mTotalCacheSize(0), mTotalCacheEntries(0), + mTotalCacheSizeDivisor(kCacheLimitDivisor), mHotCacheLimit(0), mHotCacheSize(0), mWorkerThreadIdle(true) { @@ -80,8 +90,13 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s return; } - // Set the cache version, override if debug value set + // Set the cache version mCacheVersion = kMultifileBlobCacheVersion; + // Bump the version if we're using flagged features + if (flags::multifile_blobcache_advanced_usage()) { + mCacheVersion++; + } + // Override if debug value set int debugCacheVersion = base::GetIntProperty("debug.egl.blobcache.cache_version", -1); if (debugCacheVersion >= 0) { ALOGV("INIT: Using %u as cacheVersion instead of %u", debugCacheVersion, mCacheVersion); @@ -122,7 +137,7 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s // Check that our cacheVersion and buildId match struct stat st; if (stat(mMultifileDirName.c_str(), &st) == 0) { - if (checkStatus(mMultifileDirName.c_str())) { + if (checkStatus(mMultifileDirName)) { statusGood = true; } else { ALOGV("INIT: Cache status has changed, clearing the cache"); @@ -237,11 +252,9 @@ MultifileBlobCache::MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, s ALOGV("INIT: Entry %u is good, tracking it now.", entryHash); - // Track details for rapid lookup later - trackEntry(entryHash, header.valueSize, fileSize, st.st_atime); - - // Track the total size - increaseTotalCacheSize(fileSize); + // Track details for rapid lookup later and update total size + // Note access time is a full timespec instead of just seconds + trackEntry(entryHash, header.valueSize, fileSize, st.st_atim); // Preload the entry for fast retrieval if ((mHotCacheSize + fileSize) < mHotCacheLimit) { @@ -317,6 +330,28 @@ void MultifileBlobCache::set(const void* key, EGLsizeiANDROID keySize, const voi // Generate a hash of the key and use it to track this entry uint32_t entryHash = android::JenkinsHashMixBytes(0, static_cast<const uint8_t*>(key), keySize); + std::string fullPath = mMultifileDirName + "/" + std::to_string(entryHash); + + // See if we already have this file + if (flags::multifile_blobcache_advanced_usage() && contains(entryHash)) { + // Remove previous entry from hot cache + removeFromHotCache(entryHash); + + // Remove previous entry and update the overall cache size + removeEntry(entryHash); + + // If valueSize is zero, this is an indication that the user wants to remove the entry from + // cache It has already been removed from tracking, now remove it from disk It is safe to do + // this immediately because we drained the write queue in removeFromHotCache + if (valueSize == 0) { + ALOGV("SET: Zero size detected for existing entry, removing %u from cache", entryHash); + if (remove(fullPath.c_str()) != 0) { + ALOGW("SET: Error removing %s: %s", fullPath.c_str(), std::strerror(errno)); + } + return; + } + } + size_t fileSize = sizeof(MultifileHeader) + keySize + valueSize; // If we're going to be over the cache limit, kick off a trim to clear space @@ -339,13 +374,12 @@ void MultifileBlobCache::set(const void* key, EGLsizeiANDROID keySize, const voi memcpy(static_cast<void*>(buffer + sizeof(MultifileHeader) + keySize), static_cast<const void*>(value), valueSize); - std::string fullPath = mMultifileDirName + "/" + std::to_string(entryHash); - - // Track the size and access time for quick recall - trackEntry(entryHash, valueSize, fileSize, time(0)); - - // Update the overall cache size - increaseTotalCacheSize(fileSize); + // Track the size and access time for quick recall and update the overall cache size + struct timespec time = {0, 0}; + if (flags::multifile_blobcache_advanced_usage()) { + clock_gettime(CLOCK_REALTIME, &time); + } + trackEntry(entryHash, valueSize, fileSize, time); // Keep the entry in hot cache for quick retrieval ALOGV("SET: Adding %u to hot cache.", entryHash); @@ -427,6 +461,14 @@ EGLsizeiANDROID MultifileBlobCache::get(const void* key, EGLsizeiANDROID keySize if (mHotCache.find(entryHash) != mHotCache.end()) { ALOGV("GET: HotCache HIT for entry %u", entryHash); cacheEntry = mHotCache[entryHash].entryBuffer; + + if (flags::multifile_blobcache_advanced_usage()) { + // Update last access time on disk + struct timespec times[2]; + times[0].tv_nsec = UTIME_NOW; + times[1].tv_nsec = UTIME_OMIT; + utimensat(0, fullPath.c_str(), times, 0); + } } else { ALOGV("GET: HotCache MISS for entry: %u", entryHash); @@ -455,6 +497,14 @@ EGLsizeiANDROID MultifileBlobCache::get(const void* key, EGLsizeiANDROID keySize cacheEntry = reinterpret_cast<uint8_t*>(mmap(nullptr, fileSize, PROT_READ, MAP_PRIVATE, fd, 0)); + if (flags::multifile_blobcache_advanced_usage()) { + // Update last access time and omit last modify time + struct timespec times[2]; + times[0].tv_nsec = UTIME_NOW; + times[1].tv_nsec = UTIME_OMIT; + futimens(fd, times); + } + // We can close the file now and the mmap will remain close(fd); @@ -491,6 +541,13 @@ EGLsizeiANDROID MultifileBlobCache::get(const void* key, EGLsizeiANDROID keySize return 0; } + if (flags::multifile_blobcache_advanced_usage()) { + // Update the entry time for this hash, so it reflects LRU + struct timespec time; + clock_gettime(CLOCK_REALTIME, &time); + updateEntryTime(entryHash, time); + } + // Remaining entry following the key is the value uint8_t* cachedValue = cacheEntry + (keySize + sizeof(MultifileHeader)); memcpy(value, cachedValue, cachedValueSize); @@ -626,9 +683,47 @@ bool MultifileBlobCache::checkStatus(const std::string& baseDir) { } void MultifileBlobCache::trackEntry(uint32_t entryHash, EGLsizeiANDROID valueSize, size_t fileSize, - time_t accessTime) { + const timespec& accessTime) { +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + // When we add this entry to the map, it is sorted by accessTime + MultifileEntryStatsMapIter entryStatsIter = + mEntryStats.emplace(std::piecewise_construct, std::forward_as_tuple(accessTime), + std::forward_as_tuple(entryHash, valueSize, fileSize)); + + // Track all entries with quick access to its stats + mEntries.emplace(entryHash, entryStatsIter); +#else + (void)accessTime; mEntries.insert(entryHash); - mEntryStats[entryHash] = {valueSize, fileSize, accessTime}; + mEntryStats[entryHash] = {entryHash, valueSize, fileSize}; +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + + increaseTotalCacheSize(fileSize); +} + +bool MultifileBlobCache::removeEntry(uint32_t entryHash) { + auto entryIter = mEntries.find(entryHash); + if (entryIter == mEntries.end()) { + return false; + } + +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + MultifileEntryStatsMapIter entryStatsIter = entryIter->second; + MultifileEntryStats entryStats = entryStatsIter->second; + decreaseTotalCacheSize(entryStats.fileSize); +#else + auto entryStatsIter = mEntryStats.find(entryHash); + if (entryStatsIter == mEntryStats.end()) { + ALOGE("Failed to remove entryHash (%u) from mEntryStats", entryHash); + return false; + } + decreaseTotalCacheSize(entryStatsIter->second.fileSize); +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + + mEntryStats.erase(entryStatsIter); + mEntries.erase(entryIter); + + return true; } bool MultifileBlobCache::contains(uint32_t hashEntry) const { @@ -636,7 +731,40 @@ bool MultifileBlobCache::contains(uint32_t hashEntry) const { } MultifileEntryStats MultifileBlobCache::getEntryStats(uint32_t entryHash) { +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + auto entryIter = mEntries.find(entryHash); + if (entryIter == mEntries.end()) { + return {}; + } + + MultifileEntryStatsMapIter entryStatsIter = entryIter->second; + MultifileEntryStats entryStats = entryStatsIter->second; + return entryStats; +#else return mEntryStats[entryHash]; +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) +} + +void MultifileBlobCache::updateEntryTime(uint32_t entryHash, const timespec& newTime) { +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + // This function updates the ordering of the map by removing the old iterators + // and re-adding them. If should be perforant as it does not perform a full re-sort. + // First, pull out the old entryStats + auto entryIter = mEntries.find(entryHash); + MultifileEntryStatsMapIter entryStatsIter = entryIter->second; + MultifileEntryStats entryStats = std::move(entryStatsIter->second); + + // Remove the old iterators + mEntryStats.erase(entryStatsIter); + mEntries.erase(entryIter); + + // Insert the new with updated time + entryStatsIter = mEntryStats.emplace(std::make_pair(newTime, std::move(entryStats))); + mEntries.emplace(entryHash, entryStatsIter); +#else + (void)entryHash; + (void)newTime; +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) } void MultifileBlobCache::increaseTotalCacheSize(size_t fileSize) { @@ -718,31 +846,32 @@ bool MultifileBlobCache::removeFromHotCache(uint32_t entryHash) { bool MultifileBlobCache::applyLRU(size_t cacheSizeLimit, size_t cacheEntryLimit) { // Walk through our map of sorted last access times and remove files until under the limit for (auto cacheEntryIter = mEntryStats.begin(); cacheEntryIter != mEntryStats.end();) { +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + const MultifileEntryStats& entryStats = cacheEntryIter->second; + uint32_t entryHash = entryStats.entryHash; +#else uint32_t entryHash = cacheEntryIter->first; + const MultifileEntryStats& entryStats = cacheEntryIter->second; +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) ALOGV("LRU: Removing entryHash %u", entryHash); - // Track the overall size - MultifileEntryStats entryStats = getEntryStats(entryHash); - decreaseTotalCacheSize(entryStats.fileSize); - // Remove it from hot cache if present removeFromHotCache(entryHash); // Remove it from the system std::string entryPath = mMultifileDirName + "/" + std::to_string(entryHash); if (remove(entryPath.c_str()) != 0) { - ALOGE("LRU: Error removing %s: %s", entryPath.c_str(), std::strerror(errno)); - return false; + // Continue evicting invalid item (app's cache might be cleared) + ALOGW("LRU: Error removing %s: %s", entryPath.c_str(), std::strerror(errno)); } // Increment the iterator before clearing the entry cacheEntryIter++; - // Delete the entry from our tracking - size_t count = mEntryStats.erase(entryHash); - if (count != 1) { - ALOGE("LRU: Failed to remove entryHash (%u) from mEntryStats", entryHash); + // Delete the entry from our tracking and update the overall cache size + if (!removeEntry(entryHash)) { + ALOGE("LRU: Failed to remove entryHash %u", entryHash); return false; } @@ -794,11 +923,6 @@ bool MultifileBlobCache::clearCache() { return true; } -// When removing files, what fraction of the overall limit should be reached when removing files -// A divisor of two will decrease the cache to 50%, four to 25% and so on -// We use the same limit to manage size and entry count -constexpr uint32_t kCacheLimitDivisor = 2; - // Calculate the cache size and remove old entries until under the limit void MultifileBlobCache::trimCache() { // Wait for all deferred writes to complete @@ -806,8 +930,10 @@ void MultifileBlobCache::trimCache() { waitForWorkComplete(); ALOGV("TRIM: Reducing multifile cache size to %zu, entries %zu", - mMaxTotalSize / kCacheLimitDivisor, mMaxTotalEntries / kCacheLimitDivisor); - if (!applyLRU(mMaxTotalSize / kCacheLimitDivisor, mMaxTotalEntries / kCacheLimitDivisor)) { + mMaxTotalSize / mTotalCacheSizeDivisor, mMaxTotalEntries / mTotalCacheSizeDivisor); + + if (!applyLRU(mMaxTotalSize / mTotalCacheSizeDivisor, + mMaxTotalEntries / mTotalCacheSizeDivisor)) { ALOGE("Error when clearing multifile shader cache"); return; } @@ -830,9 +956,36 @@ void MultifileBlobCache::processTask(DeferredTask& task) { // Create the file or reset it if already present, read+write for user only int fd = open(fullPath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); if (fd == -1) { - ALOGE("Cache error in SET - failed to open fullPath: %s, error: %s", - fullPath.c_str(), std::strerror(errno)); - return; + if (flags::multifile_blobcache_advanced_usage()) { + struct stat st; + if (stat(mMultifileDirName.c_str(), &st) == -1) { + ALOGW("Cache directory missing (app's cache cleared?). Recreating..."); + + // Restore the multifile directory + if (mkdir(mMultifileDirName.c_str(), 0755) != 0 && (errno != EEXIST)) { + ALOGE("Cache error in SET - Unable to create directory (%s), errno " + "(%i)", + mMultifileDirName.c_str(), errno); + return; + } + + // Create new status file + if (!createStatus(mMultifileDirName.c_str())) { + ALOGE("Cache error in SET - Failed to create status file!"); + return; + } + + // Try to open the file again + fd = open(fullPath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); + } + } + + if (fd == -1) { + ALOGE("Cache error in SET - failed to open fullPath: %s, error: %s", + fullPath.c_str(), std::strerror(errno)); + return; + } } ALOGV("DEFERRED: Opened fd %i from %s", fd, fullPath.c_str()); @@ -849,6 +1002,14 @@ void MultifileBlobCache::processTask(DeferredTask& task) { return; } + if (flags::multifile_blobcache_advanced_usage()) { + // Update last access time and last modify time + struct timespec times[2]; + times[0].tv_nsec = UTIME_NOW; + times[1].tv_nsec = UTIME_NOW; + futimens(fd, times); + } + ALOGV("DEFERRED: Completed write for: %s", fullPath.c_str()); close(fd); diff --git a/opengl/libs/EGL/MultifileBlobCache.h b/opengl/libs/EGL/MultifileBlobCache.h index 65aa2db344..3bd393f068 100644 --- a/opengl/libs/EGL/MultifileBlobCache.h +++ b/opengl/libs/EGL/MultifileBlobCache.h @@ -32,6 +32,10 @@ #include "FileBlobCache.h" +#include <com_android_graphics_egl_flags.h> + +using namespace com::android::graphics::egl; + namespace android { constexpr uint32_t kMultifileBlobCacheVersion = 2; @@ -45,9 +49,9 @@ struct MultifileHeader { }; struct MultifileEntryStats { + uint32_t entryHash; EGLsizeiANDROID valueSize; size_t fileSize; - time_t accessTime; }; struct MultifileStatus { @@ -100,6 +104,26 @@ private: size_t mBufferSize; }; +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) +struct MultifileTimeLess { + bool operator()(const struct timespec& t1, const struct timespec& t2) const { + if (t1.tv_sec == t2.tv_sec) { + // If seconds are equal, check nanoseconds + return t1.tv_nsec < t2.tv_nsec; + } else { + // Otherwise, compare seconds + return t1.tv_sec < t2.tv_sec; + } + } +}; + +// The third parameter here causes all entries to be sorted by access time, +// so oldest will be accessed first in applyLRU +using MultifileEntryStatsMap = + std::multimap<struct timespec, MultifileEntryStats, MultifileTimeLess>; +using MultifileEntryStatsMapIter = MultifileEntryStatsMap::iterator; +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + class MultifileBlobCache { public: MultifileBlobCache(size_t maxKeySize, size_t maxValueSize, size_t maxTotalSize, @@ -115,6 +139,7 @@ public: size_t getTotalSize() const { return mTotalCacheSize; } size_t getTotalEntries() const { return mTotalCacheEntries; } + size_t getTotalCacheSizeDivisor() const { return mTotalCacheSizeDivisor; } const std::string& getCurrentBuildId() const { return mBuildId; } void setCurrentBuildId(const std::string& buildId) { mBuildId = buildId; } @@ -124,10 +149,11 @@ public: private: void trackEntry(uint32_t entryHash, EGLsizeiANDROID valueSize, size_t fileSize, - time_t accessTime); + const timespec& accessTime); bool contains(uint32_t entryHash) const; bool removeEntry(uint32_t entryHash); MultifileEntryStats getEntryStats(uint32_t entryHash); + void updateEntryTime(uint32_t entryHash, const timespec& newTime); bool createStatus(const std::string& baseDir); bool checkStatus(const std::string& baseDir); @@ -151,8 +177,14 @@ private: std::string mBuildId; uint32_t mCacheVersion; +#if COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + std::unordered_map<uint32_t, MultifileEntryStatsMapIter> mEntries; + MultifileEntryStatsMap mEntryStats; +#else std::unordered_set<uint32_t> mEntries; std::unordered_map<uint32_t, MultifileEntryStats> mEntryStats; +#endif // COM_ANDROID_GRAPHICS_EGL_FLAGS(MULTIFILE_BLOBCACHE_ADVANCED_USAGE) + std::unordered_map<uint32_t, MultifileHotCache> mHotCache; size_t mMaxKeySize; @@ -161,6 +193,7 @@ private: size_t mMaxTotalEntries; size_t mTotalCacheSize; size_t mTotalCacheEntries; + size_t mTotalCacheSizeDivisor; size_t mHotCacheLimit; size_t mHotCacheEntryLimit; size_t mHotCacheSize; diff --git a/opengl/libs/EGL/MultifileBlobCache_test.cpp b/opengl/libs/EGL/MultifileBlobCache_test.cpp index 90a0f1ee06..85fb29ec40 100644 --- a/opengl/libs/EGL/MultifileBlobCache_test.cpp +++ b/opengl/libs/EGL/MultifileBlobCache_test.cpp @@ -21,10 +21,15 @@ #include <fcntl.h> #include <gtest/gtest.h> #include <stdio.h> +#include <utils/JenkinsHash.h> #include <fstream> #include <memory> +#include <com_android_graphics_egl_flags.h> + +using namespace com::android::graphics::egl; + using namespace std::literals; namespace android { @@ -55,6 +60,7 @@ protected: std::vector<std::string> getCacheEntries(); void clearProperties(); + bool clearCache(); std::unique_ptr<TemporaryFile> mTempFile; std::unique_ptr<MultifileBlobCache> mMBC; @@ -314,7 +320,7 @@ std::vector<std::string> MultifileBlobCacheTest::getCacheEntries() { struct stat info; if (stat(multifileDirName.c_str(), &info) == 0) { - // We have a multifile dir. Skip the status file and return the only entry. + // We have a multifile dir. Skip the status file and return the entries. DIR* dir; struct dirent* entry; if ((dir = opendir(multifileDirName.c_str())) != nullptr) { @@ -325,6 +331,7 @@ std::vector<std::string> MultifileBlobCacheTest::getCacheEntries() { if (strcmp(entry->d_name, kMultifileBlobCacheStatusFile) == 0) { continue; } + // printf("Found entry: %s\n", entry->d_name); cacheEntries.push_back(multifileDirName + "/" + entry->d_name); } } else { @@ -458,6 +465,8 @@ TEST_F(MultifileBlobCacheTest, MismatchedCacheVersionClears) { // Set one entry mMBC->set("abcd", 4, "efgh", 4); + uint32_t initialCacheVersion = mMBC->getCurrentCacheVersion(); + // Close the cache so everything writes out mMBC->finish(); mMBC.reset(); @@ -466,7 +475,7 @@ TEST_F(MultifileBlobCacheTest, MismatchedCacheVersionClears) { ASSERT_EQ(getCacheEntries().size(), 1); // Set a debug cacheVersion - std::string newCacheVersion = std::to_string(kMultifileBlobCacheVersion + 1); + std::string newCacheVersion = std::to_string(initialCacheVersion + 1); ASSERT_TRUE(base::SetProperty("debug.egl.blobcache.cache_version", newCacheVersion.c_str())); ASSERT_TRUE( base::WaitForProperty("debug.egl.blobcache.cache_version", newCacheVersion.c_str())); @@ -503,4 +512,404 @@ TEST_F(MultifileBlobCacheTest, MismatchedBuildIdClears) { ASSERT_EQ(getCacheEntries().size(), 0); } +// Ensure cache is correct when a key is reused +TEST_F(MultifileBlobCacheTest, SameKeyDifferentValues) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee}; + + size_t startingSize = mMBC->getTotalSize(); + + // New cache should be empty + ASSERT_EQ(startingSize, 0); + + // Set an initial value + mMBC->set("ab", 2, "cdef", 4); + + // Grab the new size + size_t firstSize = mMBC->getTotalSize(); + + // Ensure the size went up + // Note: Checking for an exact size is challenging, as the + // file size can differ between platforms. + ASSERT_GT(firstSize, startingSize); + + // Verify the cache is correct + ASSERT_EQ(size_t(4), mMBC->get("ab", 2, buf, 4)); + ASSERT_EQ('c', buf[0]); + ASSERT_EQ('d', buf[1]); + ASSERT_EQ('e', buf[2]); + ASSERT_EQ('f', buf[3]); + + // Now reuse the key with a smaller value + mMBC->set("ab", 2, "gh", 2); + + // Grab the new size + size_t secondSize = mMBC->getTotalSize(); + + // Ensure it decreased in size + ASSERT_LT(secondSize, firstSize); + + // Verify the cache is correct + ASSERT_EQ(size_t(2), mMBC->get("ab", 2, buf, 2)); + ASSERT_EQ('g', buf[0]); + ASSERT_EQ('h', buf[1]); + + // Now put back the original value + mMBC->set("ab", 2, "cdef", 4); + + // And we should get back a stable size + size_t finalSize = mMBC->getTotalSize(); + ASSERT_EQ(firstSize, finalSize); +} + +// Ensure cache is correct when a key is reused with large value size +TEST_F(MultifileBlobCacheTest, SameKeyLargeValues) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + // Create the cache with larger limits to stress test reuse + constexpr uint32_t kLocalMaxKeySize = 1 * 1024 * 1024; + constexpr uint32_t kLocalMaxValueSize = 4 * 1024 * 1024; + constexpr uint32_t kLocalMaxTotalSize = 32 * 1024 * 1024; + mMBC.reset(new MultifileBlobCache(kLocalMaxKeySize, kLocalMaxValueSize, kLocalMaxTotalSize, + kMaxTotalEntries, &mTempFile->path[0])); + + constexpr uint32_t kLargeValueCount = 8; + constexpr uint32_t kLargeValueSize = 64 * 1024; + + // Create a several really large values + unsigned char largeValue[kLargeValueCount][kLargeValueSize]; + for (int i = 0; i < kLargeValueCount; i++) { + for (int j = 0; j < kLargeValueSize; j++) { + // Fill the value with the index for uniqueness + largeValue[i][j] = i; + } + } + + size_t startingSize = mMBC->getTotalSize(); + + // New cache should be empty + ASSERT_EQ(startingSize, 0); + + // Cycle through the values and set them all in sequence + for (int i = 0; i < kLargeValueCount; i++) { + mMBC->set("abcd", 4, largeValue[i], kLargeValueSize); + } + + // Ensure we get the last one back + unsigned char outBuf[kLargeValueSize]; + mMBC->get("abcd", 4, outBuf, kLargeValueSize); + + for (int i = 0; i < kLargeValueSize; i++) { + // Buffer should contain highest index value + ASSERT_EQ(kLargeValueCount - 1, outBuf[i]); + } +} + +// Ensure cache eviction is LRU +TEST_F(MultifileBlobCacheTest, CacheEvictionIsLRU) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + // Fill the cache with exactly how much it can hold + int entry = 0; + for (entry = 0; entry < kMaxTotalEntries; entry++) { + // Use the index as the key and value + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + + int result = 0; + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // Ensure the cache is full + ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries); + + // Add one more entry to trigger eviction + size_t overflowEntry = kMaxTotalEntries; + mMBC->set(&overflowEntry, sizeof(overflowEntry), &overflowEntry, sizeof(overflowEntry)); + + // Verify it contains the right amount, which will be one more than reduced size + // because we evict the cache before adding a new entry + size_t evictionLimit = kMaxTotalEntries / mMBC->getTotalCacheSizeDivisor(); + ASSERT_EQ(mMBC->getTotalEntries(), evictionLimit + 1); + + // Ensure cache is as expected, with old entries removed, newer entries remaining + for (entry = 0; entry < kMaxTotalEntries; entry++) { + int result = 0; + mMBC->get(&entry, sizeof(entry), &result, sizeof(result)); + + if (entry < evictionLimit) { + // We should get no hits on evicted entries, i.e. the first added + ASSERT_EQ(result, 0); + } else { + // Above the limit should still be present + ASSERT_EQ(result, entry); + } + } +} + +// Ensure calling GET on an entry updates its access time, even if already in hotcache +TEST_F(MultifileBlobCacheTest, GetUpdatesAccessTime) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + // Fill the cache with exactly how much it can hold + int entry = 0; + int result = 0; + for (entry = 0; entry < kMaxTotalEntries; entry++) { + // Use the index as the key and value + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // Ensure the cache is full + ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries); + + // GET the first few entries to update their access time + std::vector<int> accessedEntries = {1, 2, 3}; + for (int i = 0; i < accessedEntries.size(); i++) { + entry = accessedEntries[i]; + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + } + + // Add one more entry to trigger eviction + size_t overflowEntry = kMaxTotalEntries; + mMBC->set(&overflowEntry, sizeof(overflowEntry), &overflowEntry, sizeof(overflowEntry)); + + size_t evictionLimit = kMaxTotalEntries / mMBC->getTotalCacheSizeDivisor(); + + // Ensure cache is as expected, with old entries removed, newer entries remaining + for (entry = 0; entry < kMaxTotalEntries; entry++) { + int result = 0; + mMBC->get(&entry, sizeof(entry), &result, sizeof(result)); + + if (std::find(accessedEntries.begin(), accessedEntries.end(), entry) != + accessedEntries.end()) { + // If this is one of the handful we accessed after filling the cache, + // they should still be in the cache because LRU + ASSERT_EQ(result, entry); + } else if (entry >= (evictionLimit + accessedEntries.size())) { + // If they were above the eviction limit (plus three for our updated entries), + // they should still be present + ASSERT_EQ(result, entry); + } else { + // Otherwise, they shold be evicted and no longer present + ASSERT_EQ(result, 0); + } + } + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Open the cache again + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Check the cache again, ensuring the updated access time made it to disk + for (entry = 0; entry < kMaxTotalEntries; entry++) { + int result = 0; + mMBC->get(&entry, sizeof(entry), &result, sizeof(result)); + if (std::find(accessedEntries.begin(), accessedEntries.end(), entry) != + accessedEntries.end()) { + ASSERT_EQ(result, entry); + } else if (entry >= (evictionLimit + accessedEntries.size())) { + ASSERT_EQ(result, entry); + } else { + ASSERT_EQ(result, 0); + } + } +} + +bool MultifileBlobCacheTest::clearCache() { + std::string cachePath = &mTempFile->path[0]; + std::string multifileDirName = cachePath + ".multifile"; + + DIR* dir = opendir(multifileDirName.c_str()); + if (dir == nullptr) { + printf("Error opening directory: %s\n", multifileDirName.c_str()); + return false; + } + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + // Skip "." and ".." entries + if (std::string(entry->d_name) == "." || std::string(entry->d_name) == "..") { + continue; + } + + std::string entryPath = multifileDirName + "/" + entry->d_name; + + // Delete the entry (we assert it's a file, nothing nested here) + if (unlink(entryPath.c_str()) != 0) { + printf("Error deleting file: %s\n", entryPath.c_str()); + closedir(dir); + return false; + } + } + + closedir(dir); + + // Delete the empty directory itself + if (rmdir(multifileDirName.c_str()) != 0) { + printf("Error deleting directory %s, error %s\n", multifileDirName.c_str(), + std::strerror(errno)); + return false; + } + + return true; +} + +// Recover from lost cache in the case of app clearing it +TEST_F(MultifileBlobCacheTest, RecoverFromLostCache) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + int entry = 0; + int result = 0; + + uint32_t kEntryCount = 10; + + // Add some entries + for (entry = 0; entry < kEntryCount; entry++) { + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // For testing, wait until the entries have completed writing + mMBC->finish(); + + // Manually delete the cache! + ASSERT_TRUE(clearCache()); + + // Cache should not contain any entries + for (entry = 0; entry < kEntryCount; entry++) { + ASSERT_EQ(size_t(0), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + } + + // Ensure we can still add new ones + for (entry = kEntryCount; entry < kEntryCount * 2; entry++) { + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // Close the cache so everything writes out + mMBC->finish(); + mMBC.reset(); + + // Open the cache again + mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries, + &mTempFile->path[0])); + + // Before fixes, writing the second entries to disk should have failed due to missing + // cache dir. But now they should have survived our shutdown above. + for (entry = kEntryCount; entry < kEntryCount * 2; entry++) { + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } +} + +// Ensure cache eviction succeeds if the cache is deleted +TEST_F(MultifileBlobCacheTest, EvictAfterLostCache) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + int entry = 0; + int result = 0; + + uint32_t kEntryCount = 10; + + // Add some entries + for (entry = 0; entry < kEntryCount; entry++) { + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // For testing, wait until the entries have completed writing + mMBC->finish(); + + // Manually delete the cache! + ASSERT_TRUE(clearCache()); + + // Now start adding entries to trigger eviction, cache should survive + for (entry = kEntryCount; entry < 2 * kMaxTotalEntries; entry++) { + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // We should have triggered multiple evictions above and remain at or below the + // max amount of entries + ASSERT_LE(getCacheEntries().size(), kMaxTotalEntries); +} + +// Remove from cache when size is zero +TEST_F(MultifileBlobCacheTest, ZeroSizeRemovesEntry) { + if (!flags::multifile_blobcache_advanced_usage()) { + GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag"; + } + + // Put some entries in + int entry = 0; + int result = 0; + + uint32_t kEntryCount = 20; + + // Add some entries + for (entry = 0; entry < kEntryCount; entry++) { + mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry)); + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(entry, result); + } + + // Send some of them again with size zero + std::vector<int> removedEntries = {5, 10, 18}; + for (int i = 0; i < removedEntries.size(); i++) { + entry = removedEntries[i]; + mMBC->set(&entry, sizeof(entry), nullptr, 0); + } + + // Ensure they do not get a hit + for (int i = 0; i < removedEntries.size(); i++) { + entry = removedEntries[i]; + ASSERT_EQ(size_t(0), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + } + + // And have been removed from disk + std::vector<std::string> diskEntries = getCacheEntries(); + ASSERT_EQ(diskEntries.size(), kEntryCount - removedEntries.size()); + for (int i = 0; i < removedEntries.size(); i++) { + entry = removedEntries[i]; + // Generate a hash for our removed entries and ensure they are not contained + // Note our entry and key and the same here, so we're hashing the key just like + // the multifile blobcache does. + uint32_t entryHash = + android::JenkinsHashMixBytes(0, reinterpret_cast<uint8_t*>(&entry), sizeof(entry)); + ASSERT_EQ(std::find(diskEntries.begin(), diskEntries.end(), std::to_string(entryHash)), + diskEntries.end()); + } + + // Ensure the others are still present + for (entry = 0; entry < kEntryCount; entry++) { + if (std::find(removedEntries.begin(), removedEntries.end(), entry) == + removedEntries.end()) { + ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result))); + ASSERT_EQ(result, entry); + } + } +} + } // namespace android diff --git a/opengl/libs/EGL/egl_flags.aconfig b/opengl/libs/EGL/egl_flags.aconfig new file mode 100644 index 0000000000..115797071c --- /dev/null +++ b/opengl/libs/EGL/egl_flags.aconfig @@ -0,0 +1,13 @@ +package: "com.android.graphics.egl.flags" +container: "system" + +flag { + name: "multifile_blobcache_advanced_usage" + namespace: "gpu" + description: "This flag controls new behaviors to address bugs found via advanced usage" + bug: "380483358" + is_fixed_read_only: true + metadata { + purpose: PURPOSE_BUGFIX + } +} diff --git a/opengl/libs/EGL/fuzzer/Android.bp b/opengl/libs/EGL/fuzzer/Android.bp index 4947e5ff6c..fe5f2a6de6 100644 --- a/opengl/libs/EGL/fuzzer/Android.bp +++ b/opengl/libs/EGL/fuzzer/Android.bp @@ -37,6 +37,7 @@ cc_fuzz { ], shared_libs: [ + "libegl_flags", "libz", ], |