surfaceflinger: respect install orientation in DisplayRenderArea

DisplayRenderArea getTransform takes install orientation into
consideration.  Fix getSourceCrop and getRotationFlags to respect
install orientation as well.

This results in some changes

 - source crop is no longer rotated by install orientation for
   LayerRenderArea
 - rotation flags is no longer rotated by install orientation for
   LayerRenderArea
 - source crop is no longer set to getBounds by default for
   LayerRenderArea (it is already done in captureLayers)
 - source crop is always rotated by install orientation for
   DisplayRenderArea

AFAICT, all the changes are actually fixes.  Quickly verified by
forcing primaryDisplayOrientation to DisplayState::eOrientation90.

Bug: 113041375
Test: take screenshot, rotate screen, screencap
Change-Id: If19df222ae52f6b276f9b0572e7b9bec872e3ae4
Merged-In: If19df222ae52f6b276f9b0572e7b9bec872e3ae4

2 files changed, 66 insertions, 57 deletions

diff --git a/services/surfaceflinger/DisplayDevice.h b/services/surfaceflinger/DisplayDevice.h
index 21af37f043..0e8867f609 100644
--- a/services/surfaceflinger/DisplayDevice.h
+++ b/services/surfaceflinger/DisplayDevice.h
@@ -25,6 +25,7 @@
 #include <math/mat4.h>
 
 #include <binder/IBinder.h>
+#include <gui/LayerState.h>
 #include <hardware/hwcomposer_defs.h>
 #include <ui/GraphicTypes.h>
 #include <ui/HdrCapabilities.h>
@@ -344,8 +345,10 @@ public:
                               rotation) {}
     DisplayRenderArea(const sp<const DisplayDevice> device, Rect sourceCrop, uint32_t reqWidth,
                       uint32_t reqHeight, Transform::orientation_flags rotation)
-          : RenderArea(reqWidth, reqHeight, CaptureFill::OPAQUE, rotation), mDevice(device),
-                              mSourceCrop(sourceCrop) {}
+          : RenderArea(reqWidth, reqHeight, CaptureFill::OPAQUE,
+                       getDisplayRotation(rotation, device->getInstallOrientation())),
+            mDevice(device),
+            mSourceCrop(sourceCrop) {}
 
     const Transform& getTransform() const override { return mDevice->getTransform(); }
     Rect getBounds() const override { return mDevice->getBounds(); }
@@ -353,9 +356,67 @@ public:
     int getWidth() const override { return mDevice->getWidth(); }
     bool isSecure() const override { return mDevice->isSecure(); }
     bool needsFiltering() const override { return mDevice->needsFiltering(); }
-    Rect getSourceCrop() const override { return mSourceCrop; }
+
+    Rect getSourceCrop() const override {
+        const int orientation = mDevice->getInstallOrientation();
+        if (orientation == DisplayState::eOrientationDefault) {
+            return mSourceCrop;
+        }
+
+        uint32_t flags = 0x00;
+        switch (orientation) {
+            case DisplayState::eOrientation90:
+                flags = Transform::ROT_90;
+                break;
+            case DisplayState::eOrientation180:
+                flags = Transform::ROT_180;
+                break;
+            case DisplayState::eOrientation270:
+                flags = Transform::ROT_270;
+                break;
+        }
+        Transform tr;
+        tr.set(flags, getWidth(), getHeight());
+        return tr.transform(mSourceCrop);
+    }
 
 private:
+    static Transform::orientation_flags getDisplayRotation(
+            Transform::orientation_flags rotation, int orientation) {
+        if (orientation == DisplayState::eOrientationDefault) {
+            return rotation;
+        }
+
+        // convert hw orientation into flag presentation
+        // here inverse transform needed
+        uint8_t hw_rot_90 = 0x00;
+        uint8_t hw_flip_hv = 0x00;
+        switch (orientation) {
+            case DisplayState::eOrientation90:
+                hw_rot_90 = Transform::ROT_90;
+                hw_flip_hv = Transform::ROT_180;
+                break;
+            case DisplayState::eOrientation180:
+                hw_flip_hv = Transform::ROT_180;
+                break;
+            case DisplayState::eOrientation270:
+                hw_rot_90 = Transform::ROT_90;
+                break;
+        }
+
+        // transform flags operation
+        // 1) flip H V if both have ROT_90 flag
+        // 2) XOR these flags
+        uint8_t rotation_rot_90 = rotation & Transform::ROT_90;
+        uint8_t rotation_flip_hv = rotation & Transform::ROT_180;
+        if (rotation_rot_90 & hw_rot_90) {
+            rotation_flip_hv = (~rotation_flip_hv) & Transform::ROT_180;
+        }
+
+        return static_cast<Transform::orientation_flags>(
+                (rotation_rot_90 ^ hw_rot_90) | (rotation_flip_hv ^ hw_flip_hv));
+    }
+
     const sp<const DisplayDevice> mDevice;
     const Rect mSourceCrop;
 };
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index a503bba23b..cadbf8afbd 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -5095,7 +5095,8 @@ void SurfaceFlinger::renderScreenImplLocked(const RenderArea& renderArea,
 
     const auto reqWidth = renderArea.getReqWidth();
     const auto reqHeight = renderArea.getReqHeight();
-    Rect sourceCrop = renderArea.getSourceCrop();
+    const auto sourceCrop = renderArea.getSourceCrop();
+    const auto rotation = renderArea.getRotationFlags();
 
     bool filtering = false;
     if (primaryDisplayOrientation & DisplayState::eOrientationSwapMask) {
@@ -5106,28 +5107,6 @@ void SurfaceFlinger::renderScreenImplLocked(const RenderArea& renderArea,
                 static_cast<int32_t>(reqHeight) != raHeight;
     }
 
-    // if a default or invalid sourceCrop is passed in, set reasonable values
-    if (sourceCrop.width() == 0 || sourceCrop.height() == 0 || !sourceCrop.isValid()) {
-        sourceCrop.setLeftTop(Point(0, 0));
-        sourceCrop.setRightBottom(Point(raWidth, raHeight));
-    } else if (primaryDisplayOrientation != DisplayState::eOrientationDefault) {
-        Transform tr;
-        uint32_t flags = 0x00;
-        switch (primaryDisplayOrientation) {
-            case DisplayState::eOrientation90:
-                flags = Transform::ROT_90;
-                break;
-            case DisplayState::eOrientation180:
-                flags = Transform::ROT_180;
-                break;
-            case DisplayState::eOrientation270:
-                flags = Transform::ROT_270;
-                break;
-        }
-        tr.set(flags, raWidth, raHeight);
-        sourceCrop = tr.transform(sourceCrop);
-    }
-
     // assume ColorMode::SRGB / RenderIntent::COLORIMETRIC
     engine.setOutputDataSpace(Dataspace::SRGB);
     engine.setDisplayMaxLuminance(DisplayDevice::sDefaultMaxLumiance);
@@ -5135,37 +5114,6 @@ void SurfaceFlinger::renderScreenImplLocked(const RenderArea& renderArea,
     // make sure to clear all GL error flags
     engine.checkErrors();
 
-    Transform::orientation_flags rotation = renderArea.getRotationFlags();
-    if (primaryDisplayOrientation != DisplayState::eOrientationDefault) {
-        // convert hw orientation into flag presentation
-        // here inverse transform needed
-        uint8_t hw_rot_90  = 0x00;
-        uint8_t hw_flip_hv = 0x00;
-        switch (primaryDisplayOrientation) {
-            case DisplayState::eOrientation90:
-                hw_rot_90 = Transform::ROT_90;
-                hw_flip_hv = Transform::ROT_180;
-                break;
-            case DisplayState::eOrientation180:
-                hw_flip_hv = Transform::ROT_180;
-                break;
-            case DisplayState::eOrientation270:
-                hw_rot_90  = Transform::ROT_90;
-                break;
-        }
-
-        // transform flags operation
-        // 1) flip H V if both have ROT_90 flag
-        // 2) XOR these flags
-        uint8_t rotation_rot_90  = rotation & Transform::ROT_90;
-        uint8_t rotation_flip_hv = rotation & Transform::ROT_180;
-        if (rotation_rot_90 & hw_rot_90) {
-            rotation_flip_hv = (~rotation_flip_hv) & Transform::ROT_180;
-        }
-        rotation = static_cast<Transform::orientation_flags>
-                   ((rotation_rot_90 ^ hw_rot_90) | (rotation_flip_hv ^ hw_flip_hv));
-    }
-
     // set-up our viewport
     engine.setViewportAndProjection(reqWidth, reqHeight, sourceCrop, raHeight, yswap,
                                     rotation);