Share Vulkan resources between RenderThread and HardwareBitmapUploader

The VkInstance and VkDevice are constructs that can be shared between
threads and avoid the extra driver allocations associated with them.

This CL also fixes a bug where Vulkan uploads were occuring on the
thread that invoked the upload and not occuring on the upload thread.

Bug: b/162628999
Test: atest CtsUiRenderingTestCases
Change-Id: I0289938333e2c3dafcc2ce18e72dca3acf4d5b79

5 files changed, 100 insertions, 75 deletions

diff --git a/libs/hwui/HardwareBitmapUploader.cpp b/libs/hwui/HardwareBitmapUploader.cpp
index 87244427a719..ab9b8b55a4cb 100644
--- a/libs/hwui/HardwareBitmapUploader.cpp
+++ b/libs/hwui/HardwareBitmapUploader.cpp
@@ -56,12 +56,6 @@ class AHBUploader : public RefBase {
 public:
     virtual ~AHBUploader() {}
 
-    // Called to start creation of the Vulkan and EGL contexts on another thread before we actually
-    // need to do an upload.
-    void initialize() {
-        onInitialize();
-    }
-
     void destroy() {
         std::lock_guard _lock{mLock};
         LOG_ALWAYS_FATAL_IF(mPendingUploads, "terminate called while uploads in progress");
@@ -91,7 +85,6 @@ protected:
     sp<ThreadBase> mUploadThread = nullptr;
 
 private:
-    virtual void onInitialize() = 0;
     virtual void onIdle() = 0;
     virtual void onDestroy() = 0;
 
@@ -141,7 +134,6 @@ private:
 
 class EGLUploader : public AHBUploader {
 private:
-    void onInitialize() override {}
     void onDestroy() override {
         mEglManager.destroy();
     }
@@ -231,62 +223,67 @@ private:
 
 class VkUploader : public AHBUploader {
 private:
-    void onInitialize() override {
-        std::lock_guard _lock{mLock};
-        if (!mUploadThread) {
-            mUploadThread = new ThreadBase{};
-        }
-        if (!mUploadThread->isRunning()) {
-            mUploadThread->start("GrallocUploadThread");
-        }
-
-        mUploadThread->queue().post([this]() {
-            std::lock_guard _lock{mVkLock};
-            if (!mVulkanManager.hasVkContext()) {
-                mVulkanManager.initialize();
-            }
-        });
-    }
     void onDestroy() override {
+        std::lock_guard _lock{mVkLock};
         mGrContext.reset();
-        mVulkanManager.destroy();
+        mVulkanManagerStrong.clear();
     }
     void onIdle() override {
-        mGrContext.reset();
+        onDestroy();
     }
 
-    void onBeginUpload() override {
-        {
-            std::lock_guard _lock{mVkLock};
-            if (!mVulkanManager.hasVkContext()) {
-                LOG_ALWAYS_FATAL_IF(mGrContext,
-                    "GrContext exists with no VulkanManager for vulkan uploads");
-                mUploadThread->queue().runSync([this]() {
-                    mVulkanManager.initialize();
-                });
-            }
-        }
-        if (!mGrContext) {
-            GrContextOptions options;
-            mGrContext = mVulkanManager.createContext(options);
-            LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
-            this->postIdleTimeoutCheck();
-        }
-    }
+    void onBeginUpload() override {}
 
     bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
                                 AHardwareBuffer* ahb) override {
-        ATRACE_CALL();
+        bool uploadSucceeded = false;
+        mUploadThread->queue().runSync([this, &uploadSucceeded, bitmap, ahb]() {
+          ATRACE_CALL();
+          std::lock_guard _lock{mVkLock};
+
+          renderthread::VulkanManager* vkManager = getVulkanManager();
+          if (!vkManager->hasVkContext()) {
+              LOG_ALWAYS_FATAL_IF(mGrContext,
+                                  "GrContext exists with no VulkanManager for vulkan uploads");
+              vkManager->initialize();
+          }
+
+          if (!mGrContext) {
+              GrContextOptions options;
+              mGrContext = vkManager->createContext(options,
+                      renderthread::VulkanManager::ContextType::kUploadThread);
+              LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
+              this->postIdleTimeoutCheck();
+          }
+
+          sk_sp<SkImage> image =
+              SkImage::MakeFromAHardwareBufferWithData(mGrContext.get(), bitmap.pixmap(), ahb);
+          mGrContext->submit(true);
+
+          uploadSucceeded = (image.get() != nullptr);
+        });
+        return uploadSucceeded;
+    }
 
-        std::lock_guard _lock{mLock};
+    /* must be called on the upload thread after the vkLock has been acquired  */
+    renderthread::VulkanManager* getVulkanManager() {
+        if (!mVulkanManagerStrong) {
+            mVulkanManagerStrong = mVulkanManagerWeak.promote();
+
+            // create a new manager if we couldn't promote the weak ref
+            if (!mVulkanManagerStrong) {
+                mVulkanManagerStrong = renderthread::VulkanManager::getInstance();
+                mGrContext.reset();
+                mVulkanManagerWeak = mVulkanManagerStrong;
+            }
+        }
 
-        sk_sp<SkImage> image =
-                SkImage::MakeFromAHardwareBufferWithData(mGrContext.get(), bitmap.pixmap(), ahb);
-        return (image.get() != nullptr);
+        return mVulkanManagerStrong.get();
     }
 
     sk_sp<GrDirectContext> mGrContext;
-    renderthread::VulkanManager mVulkanManager;
+    sp<renderthread::VulkanManager> mVulkanManagerStrong;
+    wp<renderthread::VulkanManager> mVulkanManagerWeak;
     std::mutex mVkLock;
 };
 
@@ -428,7 +425,6 @@ void HardwareBitmapUploader::initialize() {
     bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
             uirenderer::RenderPipelineType::SkiaGL;
     createUploader(usingGL);
-    sUploader->initialize();
 }
 
 void HardwareBitmapUploader::terminate() {
diff --git a/libs/hwui/renderthread/RenderThread.cpp b/libs/hwui/renderthread/RenderThread.cpp
index 565fb61c8994..4dcbc4458e97 100644
--- a/libs/hwui/renderthread/RenderThread.cpp
+++ b/libs/hwui/renderthread/RenderThread.cpp
@@ -131,8 +131,7 @@ RenderThread::RenderThread()
         , mFrameCallbackTaskPending(false)
         , mRenderState(nullptr)
         , mEglManager(nullptr)
-        , mFunctorManager(WebViewFunctorManager::instance())
-        , mVkManager(nullptr) {
+        , mFunctorManager(WebViewFunctorManager::instance()) {
     Properties::load();
     start("RenderThread");
 }
@@ -166,7 +165,7 @@ void RenderThread::initThreadLocals() {
     initializeChoreographer();
     mEglManager = new EglManager();
     mRenderState = new RenderState(*this);
-    mVkManager = new VulkanManager();
+    mVkManager = VulkanManager::getInstance();
     mCacheManager = new CacheManager();
 }
 
@@ -196,7 +195,8 @@ void RenderThread::requireGlContext() {
 }
 
 void RenderThread::requireVkContext() {
-    if (mVkManager->hasVkContext()) {
+    // the getter creates the context in the event it had been destroyed by destroyRenderingContext
+    if (vulkanManager().hasVkContext()) {
         return;
     }
     mVkManager->initialize();
@@ -222,11 +222,16 @@ void RenderThread::destroyRenderingContext() {
             mEglManager->destroy();
         }
     } else {
-        if (vulkanManager().hasVkContext()) {
-            setGrContext(nullptr);
-            vulkanManager().destroy();
-        }
+        setGrContext(nullptr);
+        mVkManager.clear();
+    }
+}
+
+VulkanManager& RenderThread::vulkanManager() {
+    if (!mVkManager.get()) {
+        mVkManager = VulkanManager::getInstance();
     }
+    return *mVkManager.get();
 }
 
 void RenderThread::dumpGraphicsMemory(int fd) {
diff --git a/libs/hwui/renderthread/RenderThread.h b/libs/hwui/renderthread/RenderThread.h
index b8ce55650516..d7dc00b8a5c1 100644
--- a/libs/hwui/renderthread/RenderThread.h
+++ b/libs/hwui/renderthread/RenderThread.h
@@ -110,7 +110,7 @@ public:
     void setGrContext(sk_sp<GrDirectContext> cxt);
 
     CacheManager& cacheManager() { return *mCacheManager; }
-    VulkanManager& vulkanManager() { return *mVkManager; }
+    VulkanManager& vulkanManager();
 
     sk_sp<Bitmap> allocateHardwareBitmap(SkBitmap& skBitmap);
     void dumpGraphicsMemory(int fd);
@@ -188,7 +188,7 @@ private:
 
     sk_sp<GrDirectContext> mGrContext;
     CacheManager* mCacheManager;
-    VulkanManager* mVkManager;
+    sp<VulkanManager> mVkManager;
 };
 
 } /* namespace renderthread */
diff --git a/libs/hwui/renderthread/VulkanManager.cpp b/libs/hwui/renderthread/VulkanManager.cpp
index 249936eb485e..0c5cf682e566 100644
--- a/libs/hwui/renderthread/VulkanManager.cpp
+++ b/libs/hwui/renderthread/VulkanManager.cpp
@@ -57,12 +57,22 @@ static void free_features_extensions_structs(const VkPhysicalDeviceFeatures2& fe
 #define GET_INST_PROC(F) m##F = (PFN_vk##F)vkGetInstanceProcAddr(mInstance, "vk" #F)
 #define GET_DEV_PROC(F) m##F = (PFN_vk##F)vkGetDeviceProcAddr(mDevice, "vk" #F)
 
-void VulkanManager::destroy() {
-    if (VK_NULL_HANDLE != mCommandPool) {
-        mDestroyCommandPool(mDevice, mCommandPool, nullptr);
-        mCommandPool = VK_NULL_HANDLE;
+sp<VulkanManager> VulkanManager::getInstance() {
+    // cache a weakptr to the context to enable a second thread to share the same vulkan state
+    static wp<VulkanManager> sWeakInstance = nullptr;
+    static std::mutex sLock;
+
+    std::lock_guard _lock{sLock};
+    sp<VulkanManager> vulkanManager = sWeakInstance.promote();
+    if (!vulkanManager.get()) {
+        vulkanManager = new VulkanManager();
+        sWeakInstance = vulkanManager;
     }
 
+    return vulkanManager;
+}
+
+VulkanManager::~VulkanManager() {
     if (mDevice != VK_NULL_HANDLE) {
         mDeviceWaitIdle(mDevice);
         mDestroyDevice(mDevice, nullptr);
@@ -73,6 +83,7 @@ void VulkanManager::destroy() {
     }
 
     mGraphicsQueue = VK_NULL_HANDLE;
+    mAHBUploadQueue = VK_NULL_HANDLE;
     mPresentQueue = VK_NULL_HANDLE;
     mDevice = VK_NULL_HANDLE;
     mPhysicalDevice = VK_NULL_HANDLE;
@@ -175,6 +186,7 @@ void VulkanManager::setupDevice(GrVkExtensions& grExtensions, VkPhysicalDeviceFe
     for (uint32_t i = 0; i < queueCount; i++) {
         if (queueProps[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) {
             mGraphicsQueueIndex = i;
+            LOG_ALWAYS_FATAL_IF(queueProps[i].queueCount < 2);
             break;
         }
     }
@@ -283,7 +295,7 @@ void VulkanManager::setupDevice(GrVkExtensions& grExtensions, VkPhysicalDeviceFe
                     queueNextPtr,                                // pNext
                     0,                                           // VkDeviceQueueCreateFlags
                     mGraphicsQueueIndex,                         // queueFamilyIndex
-                    1,                                           // queueCount
+                    2,                                           // queueCount
                     queuePriorities,                             // pQueuePriorities
             },
             {
@@ -347,6 +359,7 @@ void VulkanManager::initialize() {
     this->setupDevice(mExtensions, mPhysicalDeviceFeatures2);
 
     mGetDeviceQueue(mDevice, mGraphicsQueueIndex, 0, &mGraphicsQueue);
+    mGetDeviceQueue(mDevice, mGraphicsQueueIndex, 1, &mAHBUploadQueue);
 
     // create the command pool for the command buffers
     if (VK_NULL_HANDLE == mCommandPool) {
@@ -369,7 +382,8 @@ void VulkanManager::initialize() {
     }
 }
 
-sk_sp<GrDirectContext> VulkanManager::createContext(const GrContextOptions& options) {
+sk_sp<GrDirectContext> VulkanManager::createContext(const GrContextOptions& options,
+                                                    ContextType contextType) {
     auto getProc = [](const char* proc_name, VkInstance instance, VkDevice device) {
         if (device != VK_NULL_HANDLE) {
             return vkGetDeviceProcAddr(device, proc_name);
@@ -381,7 +395,8 @@ sk_sp<GrDirectContext> VulkanManager::createContext(const GrContextOptions& opti
     backendContext.fInstance = mInstance;
     backendContext.fPhysicalDevice = mPhysicalDevice;
     backendContext.fDevice = mDevice;
-    backendContext.fQueue = mGraphicsQueue;
+    backendContext.fQueue = (contextType == ContextType::kRenderThread) ? mGraphicsQueue
+                                                                        : mAHBUploadQueue;
     backendContext.fGraphicsQueueIndex = mGraphicsQueueIndex;
     backendContext.fMaxAPIVersion = mAPIVersion;
     backendContext.fVkExtensions = &mExtensions;
diff --git a/libs/hwui/renderthread/VulkanManager.h b/libs/hwui/renderthread/VulkanManager.h
index 3f2df8d75d89..13335f32ef06 100644
--- a/libs/hwui/renderthread/VulkanManager.h
+++ b/libs/hwui/renderthread/VulkanManager.h
@@ -43,10 +43,9 @@ class RenderThread;
 // This class contains the shared global Vulkan objects, such as VkInstance, VkDevice and VkQueue,
 // which are re-used by CanvasContext. This class is created once and should be used by all vulkan
 // windowing contexts. The VulkanManager must be initialized before use.
-class VulkanManager {
+class VulkanManager final : public RefBase {
 public:
-    explicit VulkanManager() {}
-    ~VulkanManager() { destroy(); }
+    static sp<VulkanManager> getInstance();
 
     // Sets up the vulkan context that is shared amonst all clients of the VulkanManager. This must
     // be call once before use of the VulkanManager. Multiple calls after the first will simiply
@@ -68,9 +67,6 @@ public:
     Frame dequeueNextBuffer(VulkanSurface* surface);
     void swapBuffers(VulkanSurface* surface, const SkRect& dirtyRect);
 
-    // Cleans up all the global state in the VulkanManger.
-    void destroy();
-
     // Inserts a wait on fence command into the Vulkan command buffer.
     status_t fenceWait(int fence, GrDirectContext* grContext);
 
@@ -83,12 +79,24 @@ public:
     // the internal state of VulkanManager: VulkanManager must be alive to use the returned value.
     VkFunctorInitParams getVkFunctorInitParams() const;
 
-    sk_sp<GrDirectContext> createContext(const GrContextOptions& options);
+
+    enum class ContextType {
+        kRenderThread,
+        kUploadThread
+    };
+
+    // returns a Skia graphic context used to draw content on the specified thread
+    sk_sp<GrDirectContext> createContext(const GrContextOptions& options,
+                                         ContextType contextType = ContextType::kRenderThread);
 
     uint32_t getDriverVersion() const { return mDriverVersion; }
 
 private:
     friend class VulkanSurface;
+
+    explicit VulkanManager() {}
+    ~VulkanManager();
+
     // Sets up the VkInstance and VkDevice objects. Also fills out the passed in
     // VkPhysicalDeviceFeatures struct.
     void setupDevice(GrVkExtensions&, VkPhysicalDeviceFeatures2&);
@@ -154,6 +162,7 @@ private:
 
     uint32_t mGraphicsQueueIndex;
     VkQueue mGraphicsQueue = VK_NULL_HANDLE;
+    VkQueue mAHBUploadQueue = VK_NULL_HANDLE;
     uint32_t mPresentQueueIndex;
     VkQueue mPresentQueue = VK_NULL_HANDLE;
     VkCommandPool mCommandPool = VK_NULL_HANDLE;