Add clamp to Layer and update Transform inverse.

Ensure that the adjusted transparent region is within
the display bounds by clamping after the transformation
is applied and clean up transform's inverse function.

Bug: 18452546

Change-Id: Ia473e483ee8374177bcb84d8192eb1f0e86f022a

2 files changed, 29 insertions, 17 deletions

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index c3e1a768e3..5e3dfabb39 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -332,8 +332,12 @@ FloatRect Layer::computeCrop(const sp<const DisplayDevice>& hw) const {
     activeCrop.intersect(hw->getViewport(), &activeCrop);
     activeCrop = s.transform.inverse().transform(activeCrop);
 
-    // paranoia: make sure the window-crop is constrained in the
-    // window's bounds
+    // This needs to be here as transform.transform(Rect) computes the
+    // transformed rect and then takes the bounding box of the result before
+    // returning. This means
+    // transform.inverse().transform(transform.transform(Rect)) != Rect
+    // in which case we need to make sure the final rect is clipped to the
+    // display bounds.
     activeCrop.intersect(Rect(s.active.w, s.active.h), &activeCrop);
 
     // subtract the transparent region and snap to the bounds
@@ -432,6 +436,13 @@ void Layer::setGeometry(
         activeCrop = s.transform.transform(activeCrop);
         activeCrop.intersect(hw->getViewport(), &activeCrop);
         activeCrop = s.transform.inverse().transform(activeCrop);
+        // This needs to be here as transform.transform(Rect) computes the
+        // transformed rect and then takes the bounding box of the result before
+        // returning. This means
+        // transform.inverse().transform(transform.transform(Rect)) != Rect
+        // in which case we need to make sure the final rect is clipped to the
+        // display bounds.
+        activeCrop.intersect(Rect(s.active.w, s.active.h), &activeCrop);
         // mark regions outside the crop as transparent
         activeTransparentRegion.orSelf(Rect(0, 0, s.active.w, activeCrop.top));
         activeTransparentRegion.orSelf(Rect(0, activeCrop.bottom,
diff --git a/services/surfaceflinger/Transform.cpp b/services/surfaceflinger/Transform.cpp
index 3456abfa6f..35e7e7dac1 100644
--- a/services/surfaceflinger/Transform.cpp
+++ b/services/surfaceflinger/Transform.cpp
@@ -301,16 +301,16 @@ Transform Transform::inverse() const {
     // (T*M)^-1 = M^-1 * T^-1
     Transform result;
     if (mType <= TRANSLATE) {
-        // 1 0 x
-        // 0 1 y
-        // 0 0 1
+        // 1 0 0
+        // 0 1 0
+        // x y 1
         result = *this;
         result.mMatrix[2][0] = -result.mMatrix[2][0];
         result.mMatrix[2][1] = -result.mMatrix[2][1];
     } else {
-        // a c x
-        // b d y
-        // 0 0 1
+        // a c 0
+        // b d 0
+        // x y 1
         const mat33& M(mMatrix);
         const float a = M[0][0];
         const float b = M[1][0];
@@ -319,16 +319,17 @@ Transform Transform::inverse() const {
         const float x = M[2][0];
         const float y = M[2][1];
 
-        Transform R, T;
         const float idet = 1.0 / (a*d - b*c);
-        R.mMatrix[0][0] =  d*idet;    R.mMatrix[0][1] = -c*idet;
-        R.mMatrix[1][0] = -b*idet;    R.mMatrix[1][1] =  a*idet;
-        R.mType = mType &= ~TRANSLATE;
-
-        T.mMatrix[2][0] = -x;
-        T.mMatrix[2][1] = -y;
-        T.mType = TRANSLATE;
-        result =  R * T;
+        result.mMatrix[0][0] =  d*idet;
+        result.mMatrix[0][1] = -c*idet;
+        result.mMatrix[1][0] = -b*idet;
+        result.mMatrix[1][1] =  a*idet;
+        result.mType = mType;
+
+        vec2 T(-x, -y);
+        T = result.transform(T);
+        result.mMatrix[2][0] = T[0];
+        result.mMatrix[2][1] = T[1];
     }
     return result;
 }