[RenderEngine] Do not cache protected EGLImage between runs.

Normally on a device the amount of protected memory is limited, however,
currently RenderEngine caches all generated EGLImage until layer is
destroyed, and hence eventually we run out of memory when trying to
create a protected EGLImage. This patch makes sure we unbind the buffer
and destroy the EGLImage once it's finished.

Bug: b/171098727
Test: DRM contents with big buffers playback work
Change-Id: Id24000d5e89220c849bf3cdc4917a6fa6b59e471

2 files changed, 11 insertions, 3 deletions

diff --git a/libs/renderengine/gl/GLESRenderEngine.cpp b/libs/renderengine/gl/GLESRenderEngine.cpp
index 6adcbeaf76..13577f7d83 100644
--- a/libs/renderengine/gl/GLESRenderEngine.cpp
+++ b/libs/renderengine/gl/GLESRenderEngine.cpp
@@ -735,9 +735,9 @@ status_t GLESRenderEngine::cacheExternalTextureBufferInternal(const sp<GraphicBu
     bool created = newImage->setNativeWindowBuffer(buffer->getNativeBuffer(),
                                                    buffer->getUsage() & GRALLOC_USAGE_PROTECTED);
     if (!created) {
-        ALOGE("Failed to create image. size=%ux%u st=%u usage=%#" PRIx64 " fmt=%d",
-              buffer->getWidth(), buffer->getHeight(), buffer->getStride(), buffer->getUsage(),
-              buffer->getPixelFormat());
+        ALOGE("Failed to create image. id=%" PRIx64 " size=%ux%u st=%u usage=%#" PRIx64 " fmt=%d",
+              buffer->getId(), buffer->getWidth(), buffer->getHeight(), buffer->getStride(),
+              buffer->getUsage(), buffer->getPixelFormat());
         return NO_INIT;
     }
 
@@ -1218,6 +1218,11 @@ status_t GLESRenderEngine::drawLayers(const DisplaySettings& display,
             texCoords[2] = vec2(1.0, 1.0);
             texCoords[3] = vec2(1.0, 0.0);
             setupLayerTexturing(texture);
+
+            // Do not cache protected EGLImage, protected memory is limited.
+            if (gBuf->getUsage() & GRALLOC_USAGE_PROTECTED) {
+                unbindExternalTextureBuffer(gBuf->getId());
+            }
         }
 
         const half3 solidColor = layer->source.solidColor;
diff --git a/services/surfaceflinger/BufferLayerConsumer.cpp b/services/surfaceflinger/BufferLayerConsumer.cpp
index 7c73df2de5..69d2d11a4d 100644
--- a/services/surfaceflinger/BufferLayerConsumer.cpp
+++ b/services/surfaceflinger/BufferLayerConsumer.cpp
@@ -503,6 +503,9 @@ void BufferLayerConsumer::dumpLocked(String8& result, const char* prefix) const
 BufferLayerConsumer::Image::Image(const sp<GraphicBuffer>& graphicBuffer,
                                   renderengine::RenderEngine& engine)
       : mGraphicBuffer(graphicBuffer), mRE(engine) {
+    if (graphicBuffer != nullptr && (graphicBuffer->getUsage() & GRALLOC_USAGE_PROTECTED)) {
+        return;
+    }
     mRE.cacheExternalTextureBuffer(mGraphicBuffer);
 }