Resample gainmaps during region decoding.

Implicitly, region decoding is a cropping operation, which previously
was introducing quality issues when decoding a gainmapped image. For
instance:

(a) Consider a 48x48 image with a 12x12 gainmap.
(b) Consider decoding a 10x10 region of the image. The decoder must choose
  to either decode a 2x2 or 3x3 region of the gainmap, which does not
  match the 4x scale ratio of the source, so either we decode too much
  or too little of the image
(c) When displaying the image, we bilinearly scale both the image and
the gainmap to the destination, then apply the gainmap. But, because of
(b), the gainmap is misaligned with the base image, introducing haloing
artifacts.

To fix this, we change (b) by always decoding a slightly larger
region (in the examplar case, either a decode a 3x3 or 4x4 region), but
retain information about the "logical" region we intended. Then, we
resample from the "logical" region up to the decoded bounds.

In the above example, this means that we decode a 3x3 or 4x4 bitmap,
that holds the logical 2.5x2.5 region of the gainmap, ensuring that (0, 0)
in the resulting bitmap maps to (0, 0) in the bitmap for the decoded region
from the base image, so that the gainmap content lines up with the base
image after the gainmap is upsampled during rendering.

We do the actual resampling inside of the recycling allocator, since we
sometimes perform a bitmap copy there anyways, so we can resample during
the copy.

Hide this behind a flag, since I broke decoding in about 5 different
ways before settling on this.

Bug: 352847821
Flag: com.android.graphics.hwui.flags.resample_gainmap_regions
Test: Decoding works in Photos, Files, and modified SilkFX
Change-Id: Ic21d44011858619273b11c20ee746614a1749a73

6 files changed, 145 insertions, 45 deletions

diff --git a/libs/hwui/Properties.cpp b/libs/hwui/Properties.cpp
index d184f64b1c2c..1217b47664dd 100644
--- a/libs/hwui/Properties.cpp
+++ b/libs/hwui/Properties.cpp
@@ -42,6 +42,9 @@ constexpr bool hdr_10bit_plus() {
 constexpr bool initialize_gl_always() {
     return false;
 }
+constexpr bool resample_gainmap_regions() {
+    return false;
+}
 }  // namespace hwui_flags
 #endif
 
@@ -100,6 +103,7 @@ float Properties::maxHdrHeadroomOn8bit = 5.f;  // TODO: Refine this number
 
 bool Properties::clipSurfaceViews = false;
 bool Properties::hdr10bitPlus = false;
+bool Properties::resampleGainmapRegions = false;
 
 int Properties::timeoutMultiplier = 1;
 
@@ -175,6 +179,8 @@ bool Properties::load() {
     clipSurfaceViews =
             base::GetBoolProperty("debug.hwui.clip_surfaceviews", hwui_flags::clip_surfaceviews());
     hdr10bitPlus = hwui_flags::hdr_10bit_plus();
+    resampleGainmapRegions = base::GetBoolProperty("debug.hwui.resample_gainmap_regions",
+                                                   hwui_flags::resample_gainmap_regions());
 
     timeoutMultiplier = android::base::GetIntProperty("ro.hw_timeout_multiplier", 1);
 
diff --git a/libs/hwui/Properties.h b/libs/hwui/Properties.h
index e2646422030e..73e80ce4afd0 100644
--- a/libs/hwui/Properties.h
+++ b/libs/hwui/Properties.h
@@ -342,6 +342,7 @@ public:
 
     static bool clipSurfaceViews;
     static bool hdr10bitPlus;
+    static bool resampleGainmapRegions;
 
     static int timeoutMultiplier;
 
diff --git a/libs/hwui/aconfig/hwui_flags.aconfig b/libs/hwui/aconfig/hwui_flags.aconfig
index cd3ae5342f4e..13c0b00daa21 100644
--- a/libs/hwui/aconfig/hwui_flags.aconfig
+++ b/libs/hwui/aconfig/hwui_flags.aconfig
@@ -97,3 +97,13 @@ flag {
   description: "Initialize GL even when HWUI is set to use Vulkan. This improves app startup time for apps using GL."
   bug: "335172671"
 }
+
+flag {
+  name: "resample_gainmap_regions"
+  namespace: "core_graphics"
+  description: "Resample gainmaps when decoding regions, to improve visual quality"
+  bug: "352847821"
+  metadata {
+    purpose: PURPOSE_BUGFIX
+  }
+}
diff --git a/libs/hwui/jni/BitmapRegionDecoder.cpp b/libs/hwui/jni/BitmapRegionDecoder.cpp
index ea5c14486ea4..6a65b8273194 100644
--- a/libs/hwui/jni/BitmapRegionDecoder.cpp
+++ b/libs/hwui/jni/BitmapRegionDecoder.cpp
@@ -87,8 +87,17 @@ public:
                                            requireUnpremul, prefColorSpace);
     }
 
-    bool decodeGainmapRegion(sp<uirenderer::Gainmap>* outGainmap, int outWidth, int outHeight,
-                             const SkIRect& desiredSubset, int sampleSize, bool requireUnpremul) {
+    // Decodes the gainmap region. If decoding succeeded, returns true and
+    // populate outGainmap with the decoded gainmap. Otherwise, returns false.
+    //
+    // Note that the desiredSubset is the logical region within the source
+    // gainmap that we want to decode. This is used for scaling into the final
+    // bitmap, since we do not want to include portions of the gainmap outside
+    // of this region. desiredSubset is also _not_ guaranteed to be
+    // pixel-aligned, so it's not possible to simply resize the resulting
+    // bitmap to accomplish this.
+    bool decodeGainmapRegion(sp<uirenderer::Gainmap>* outGainmap, SkISize bitmapDimensions,
+                             const SkRect& desiredSubset, int sampleSize, bool requireUnpremul) {
         SkColorType decodeColorType = mGainmapBRD->computeOutputColorType(kN32_SkColorType);
         sk_sp<SkColorSpace> decodeColorSpace =
                 mGainmapBRD->computeOutputColorSpace(decodeColorType, nullptr);
@@ -107,13 +116,30 @@ public:
         // allocation type. RecyclingClippingPixelAllocator will populate this with the
         // actual alpha type in either allocPixelRef() or copyIfNecessary()
         sk_sp<Bitmap> nativeBitmap = Bitmap::allocateHeapBitmap(SkImageInfo::Make(
-                outWidth, outHeight, decodeColorType, kPremul_SkAlphaType, decodeColorSpace));
+                bitmapDimensions, decodeColorType, kPremul_SkAlphaType, decodeColorSpace));
         if (!nativeBitmap) {
             ALOGE("OOM allocating Bitmap for Gainmap");
             return false;
         }
-        RecyclingClippingPixelAllocator allocator(nativeBitmap.get(), false);
-        if (!mGainmapBRD->decodeRegion(&bm, &allocator, desiredSubset, sampleSize, decodeColorType,
+
+        // Round out the subset so that we decode a slightly larger region, in
+        // case the subset has fractional components.
+        SkIRect roundedSubset = desiredSubset.roundOut();
+
+        // Map the desired subset to the space of the decoded gainmap. The
+        // subset is repositioned relative to the resulting bitmap, and then
+        // scaled to respect the sampleSize.
+        // This assumes that the subset will not be modified by the decoder, which is true
+        // for existing gainmap formats.
+        SkRect logicalSubset = desiredSubset.makeOffset(-std::floorf(desiredSubset.left()),
+                                                        -std::floorf(desiredSubset.top()));
+        logicalSubset.fLeft /= sampleSize;
+        logicalSubset.fTop /= sampleSize;
+        logicalSubset.fRight /= sampleSize;
+        logicalSubset.fBottom /= sampleSize;
+
+        RecyclingClippingPixelAllocator allocator(nativeBitmap.get(), false, logicalSubset);
+        if (!mGainmapBRD->decodeRegion(&bm, &allocator, roundedSubset, sampleSize, decodeColorType,
                                        requireUnpremul, decodeColorSpace)) {
             ALOGE("Error decoding Gainmap region");
             return false;
@@ -130,16 +156,31 @@ public:
         return true;
     }
 
-    SkIRect calculateGainmapRegion(const SkIRect& mainImageRegion, int* inOutWidth,
-                                   int* inOutHeight) {
+    struct Projection {
+        SkRect srcRect;
+        SkISize destSize;
+    };
+    Projection calculateGainmapRegion(const SkIRect& mainImageRegion, SkISize dimensions) {
         const float scaleX = ((float)mGainmapBRD->width()) / mMainImageBRD->width();
         const float scaleY = ((float)mGainmapBRD->height()) / mMainImageBRD->height();
-        *inOutWidth *= scaleX;
-        *inOutHeight *= scaleY;
-        // TODO: Account for rounding error?
-        return SkIRect::MakeLTRB(mainImageRegion.left() * scaleX, mainImageRegion.top() * scaleY,
-                                 mainImageRegion.right() * scaleX,
-                                 mainImageRegion.bottom() * scaleY);
+
+        if (uirenderer::Properties::resampleGainmapRegions) {
+            const auto srcRect = SkRect::MakeLTRB(
+                    mainImageRegion.left() * scaleX, mainImageRegion.top() * scaleY,
+                    mainImageRegion.right() * scaleX, mainImageRegion.bottom() * scaleY);
+            // Request a slightly larger destination size so that the gainmap
+            // subset we want fits entirely in this size.
+            const auto destSize = SkISize::Make(std::ceil(dimensions.width() * scaleX),
+                                                std::ceil(dimensions.height() * scaleY));
+            return Projection{.srcRect = srcRect, .destSize = destSize};
+        } else {
+            const auto srcRect = SkRect::Make(SkIRect::MakeLTRB(
+                    mainImageRegion.left() * scaleX, mainImageRegion.top() * scaleY,
+                    mainImageRegion.right() * scaleX, mainImageRegion.bottom() * scaleY));
+            const auto destSize =
+                    SkISize::Make(dimensions.width() * scaleX, dimensions.height() * scaleY);
+            return Projection{.srcRect = srcRect, .destSize = destSize};
+        }
     }
 
     bool hasGainmap() { return mGainmapBRD != nullptr; }
@@ -327,16 +368,16 @@ static jobject nativeDecodeRegion(JNIEnv* env, jobject, jlong brdHandle, jint in
     sp<uirenderer::Gainmap> gainmap;
     bool hasGainmap = brd->hasGainmap();
     if (hasGainmap) {
-        int gainmapWidth = bitmap.width();
-        int gainmapHeight = bitmap.height();
+        SkISize gainmapDims = SkISize::Make(bitmap.width(), bitmap.height());
         if (javaBitmap) {
             // If we are recycling we must match the inBitmap's relative dimensions
-            gainmapWidth = recycledBitmap->width();
-            gainmapHeight = recycledBitmap->height();
+            gainmapDims.fWidth = recycledBitmap->width();
+            gainmapDims.fHeight = recycledBitmap->height();
         }
-        SkIRect gainmapSubset = brd->calculateGainmapRegion(subset, &gainmapWidth, &gainmapHeight);
-        if (!brd->decodeGainmapRegion(&gainmap, gainmapWidth, gainmapHeight, gainmapSubset,
-                                      sampleSize, requireUnpremul)) {
+        BitmapRegionDecoderWrapper::Projection gainmapProjection =
+                brd->calculateGainmapRegion(subset, gainmapDims);
+        if (!brd->decodeGainmapRegion(&gainmap, gainmapProjection.destSize,
+                                      gainmapProjection.srcRect, sampleSize, requireUnpremul)) {
             // If there is an error decoding Gainmap - we don't fail. We just don't provide Gainmap
             hasGainmap = false;
         }
diff --git a/libs/hwui/jni/Graphics.cpp b/libs/hwui/jni/Graphics.cpp
index a88139d6b5d6..258bf91f2124 100644
--- a/libs/hwui/jni/Graphics.cpp
+++ b/libs/hwui/jni/Graphics.cpp
@@ -1,12 +1,14 @@
 #include <assert.h>
+#include <cutils/ashmem.h>
+#include <hwui/Canvas.h>
+#include <log/log.h>
+#include <nativehelper/JNIHelp.h>
 #include <unistd.h>
 
-#include "jni.h"
-#include <nativehelper/JNIHelp.h>
 #include "GraphicsJNI.h"
-
 #include "SkBitmap.h"
 #include "SkCanvas.h"
+#include "SkColor.h"
 #include "SkColorSpace.h"
 #include "SkFontMetrics.h"
 #include "SkImageInfo.h"
@@ -14,10 +16,9 @@
 #include "SkPoint.h"
 #include "SkRect.h"
 #include "SkRegion.h"
+#include "SkSamplingOptions.h"
 #include "SkTypes.h"
-#include <cutils/ashmem.h>
-#include <hwui/Canvas.h>
-#include <log/log.h>
+#include "jni.h"
 
 using namespace android;
 
@@ -630,13 +631,15 @@ bool HeapAllocator::allocPixelRef(SkBitmap* bitmap) {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-RecyclingClippingPixelAllocator::RecyclingClippingPixelAllocator(android::Bitmap* recycledBitmap,
-                                                                 bool mustMatchColorType)
+RecyclingClippingPixelAllocator::RecyclingClippingPixelAllocator(
+        android::Bitmap* recycledBitmap, bool mustMatchColorType,
+        std::optional<SkRect> desiredSubset)
         : mRecycledBitmap(recycledBitmap)
         , mRecycledBytes(recycledBitmap ? recycledBitmap->getAllocationByteCount() : 0)
         , mSkiaBitmap(nullptr)
         , mNeedsCopy(false)
-        , mMustMatchColorType(mustMatchColorType) {}
+        , mMustMatchColorType(mustMatchColorType)
+        , mDesiredSubset(getSourceBoundsForUpsample(desiredSubset)) {}
 
 RecyclingClippingPixelAllocator::~RecyclingClippingPixelAllocator() {}
 
@@ -668,7 +671,8 @@ bool RecyclingClippingPixelAllocator::allocPixelRef(SkBitmap* bitmap) {
     const SkImageInfo maxInfo = bitmap->info().makeWH(maxWidth, maxHeight);
     const size_t rowBytes = maxInfo.minRowBytes();
     const size_t bytesNeeded = maxInfo.computeByteSize(rowBytes);
-    if (bytesNeeded <= mRecycledBytes) {
+
+    if (!mDesiredSubset && bytesNeeded <= mRecycledBytes) {
         // Here we take advantage of reconfigure() to reset the rowBytes
         // of mRecycledBitmap.  It is very important that we pass in
         // mRecycledBitmap->info() for the SkImageInfo.  According to the
@@ -712,20 +716,31 @@ void RecyclingClippingPixelAllocator::copyIfNecessary() {
     if (mNeedsCopy) {
         mRecycledBitmap->ref();
         android::Bitmap* recycledPixels = mRecycledBitmap;
-        void* dst = recycledPixels->pixels();
-        const size_t dstRowBytes = mRecycledBitmap->rowBytes();
-        const size_t bytesToCopy = std::min(mRecycledBitmap->info().minRowBytes(),
-                mSkiaBitmap->info().minRowBytes());
-        const int rowsToCopy = std::min(mRecycledBitmap->info().height(),
-                mSkiaBitmap->info().height());
-        for (int y = 0; y < rowsToCopy; y++) {
-            memcpy(dst, mSkiaBitmap->getAddr(0, y), bytesToCopy);
-            // Cast to bytes in order to apply the dstRowBytes offset correctly.
-            dst = reinterpret_cast<void*>(
-                    reinterpret_cast<uint8_t*>(dst) + dstRowBytes);
+        if (mDesiredSubset) {
+            recycledPixels->setAlphaType(mSkiaBitmap->alphaType());
+            recycledPixels->setColorSpace(mSkiaBitmap->refColorSpace());
+
+            auto canvas = SkCanvas(recycledPixels->getSkBitmap());
+            SkRect destination = SkRect::Make(recycledPixels->info().bounds());
+            destination.intersect(SkRect::Make(mSkiaBitmap->info().bounds()));
+            canvas.drawImageRect(mSkiaBitmap->asImage(), *mDesiredSubset, destination,
+                                 SkSamplingOptions(SkFilterMode::kLinear), nullptr,
+                                 SkCanvas::kFast_SrcRectConstraint);
+        } else {
+            void* dst = recycledPixels->pixels();
+            const size_t dstRowBytes = mRecycledBitmap->rowBytes();
+            const size_t bytesToCopy = std::min(mRecycledBitmap->info().minRowBytes(),
+                                                mSkiaBitmap->info().minRowBytes());
+            const int rowsToCopy =
+                    std::min(mRecycledBitmap->info().height(), mSkiaBitmap->info().height());
+            for (int y = 0; y < rowsToCopy; y++) {
+                memcpy(dst, mSkiaBitmap->getAddr(0, y), bytesToCopy);
+                // Cast to bytes in order to apply the dstRowBytes offset correctly.
+                dst = reinterpret_cast<void*>(reinterpret_cast<uint8_t*>(dst) + dstRowBytes);
+            }
+            recycledPixels->setAlphaType(mSkiaBitmap->alphaType());
+            recycledPixels->setColorSpace(mSkiaBitmap->refColorSpace());
         }
-        recycledPixels->setAlphaType(mSkiaBitmap->alphaType());
-        recycledPixels->setColorSpace(mSkiaBitmap->refColorSpace());
         recycledPixels->notifyPixelsChanged();
         recycledPixels->unref();
     }
@@ -733,6 +748,20 @@ void RecyclingClippingPixelAllocator::copyIfNecessary() {
     mSkiaBitmap = nullptr;
 }
 
+std::optional<SkRect> RecyclingClippingPixelAllocator::getSourceBoundsForUpsample(
+        std::optional<SkRect> subset) {
+    if (!uirenderer::Properties::resampleGainmapRegions || !subset || subset->isEmpty()) {
+        return std::nullopt;
+    }
+
+    if (subset->left() == floor(subset->left()) && subset->top() == floor(subset->top()) &&
+        subset->right() == floor(subset->right()) && subset->bottom() == floor(subset->bottom())) {
+        return std::nullopt;
+    }
+
+    return subset;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 AshmemPixelAllocator::AshmemPixelAllocator(JNIEnv *env) {
diff --git a/libs/hwui/jni/GraphicsJNI.h b/libs/hwui/jni/GraphicsJNI.h
index b0a1074d6693..4b08f8dc7a93 100644
--- a/libs/hwui/jni/GraphicsJNI.h
+++ b/libs/hwui/jni/GraphicsJNI.h
@@ -216,8 +216,8 @@ private:
  */
 class RecyclingClippingPixelAllocator : public android::skia::BRDAllocator {
 public:
-    RecyclingClippingPixelAllocator(android::Bitmap* recycledBitmap,
-                                    bool mustMatchColorType = true);
+    RecyclingClippingPixelAllocator(android::Bitmap* recycledBitmap, bool mustMatchColorType = true,
+                                    std::optional<SkRect> desiredSubset = std::nullopt);
 
     ~RecyclingClippingPixelAllocator();
 
@@ -241,11 +241,24 @@ public:
     SkCodec::ZeroInitialized zeroInit() const override { return SkCodec::kNo_ZeroInitialized; }
 
 private:
+    /**
+     *  Optionally returns a subset rectangle that we need to upsample from.
+     *  E.g., a gainmap subset may be decoded in a slightly larger rectangle
+     *  than is needed (in order to correctly preserve gainmap alignment when
+     *  rendering at display time), so we need to re-sample the "intended"
+     *  gainmap back up to the bitmap dimensions.
+     *
+     *  If we don't need to upsample from a subregion, then returns an empty
+     *  optional
+     */
+    static std::optional<SkRect> getSourceBoundsForUpsample(std::optional<SkRect> subset);
+
     android::Bitmap* mRecycledBitmap;
     const size_t     mRecycledBytes;
     SkBitmap*        mSkiaBitmap;
     bool             mNeedsCopy;
     const bool mMustMatchColorType;
+    const std::optional<SkRect> mDesiredSubset;
 };
 
 class AshmemPixelAllocator : public SkBitmap::Allocator {