Hide layers that have irreversible transforms

Surface API doesn't prohibit putting interesting values in transforms.
It's especially possible to scale something to 0 at the begining or the
end of an animation, but we should guard SF against ill-intentioned
apps.

Bug: 210837722
Test: KeyguardTests#testDismissKeyguardActivity doesn't crash SF on CF
PC and the warning log is in logcat.

Change-Id: Idaa1cdb0676102a580c4478c860327796c8213f3

4 files changed, 22 insertions, 1 deletions

diff --git a/libs/ui/Transform.cpp b/libs/ui/Transform.cpp
index b34d90699f..42dd85e959 100644
--- a/libs/ui/Transform.cpp
+++ b/libs/ui/Transform.cpp
@@ -134,6 +134,10 @@ float Transform::dsdy() const {
     return mMatrix[1][1];
 }
 
+float Transform::det() const {
+    return mMatrix[0][0] * mMatrix[1][1] - mMatrix[0][1] * mMatrix[1][0];
+}
+
 float Transform::getScaleX() const {
     return sqrt((dsdx() * dsdx()) + (dtdx() * dtdx()));
 }
@@ -390,7 +394,7 @@ Transform Transform::inverse() const {
         const float x = M[2][0];
         const float y = M[2][1];
 
-        const float idet = 1.0f / (a*d - b*c);
+        const float idet = 1.0f / det();
         result.mMatrix[0][0] =  d*idet;
         result.mMatrix[0][1] = -c*idet;
         result.mMatrix[1][0] = -b*idet;
diff --git a/libs/ui/include/ui/Transform.h b/libs/ui/include/ui/Transform.h
index 33fbe05035..f1178cab35 100644
--- a/libs/ui/include/ui/Transform.h
+++ b/libs/ui/include/ui/Transform.h
@@ -77,6 +77,7 @@ public:
     float dtdx() const;
     float dtdy() const;
     float dsdy() const;
+    float det() const;
 
     float getScaleX() const;
     float getScaleY() const;
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 533acfdefe..4de8dc2ea9 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -338,6 +338,12 @@ void Layer::computeBounds(FloatRect parentBounds, ui::Transform parentTransform,
     // Calculate effective layer transform
     mEffectiveTransform = parentTransform * getActiveTransform(s);
 
+    if (CC_UNLIKELY(!isTransformValid())) {
+        ALOGW("Stop computing bounds for %s because it has invalid transformation.",
+              getDebugName());
+        return;
+    }
+
     // Transform parent bounds to layer space
     parentBounds = getActiveTransform(s).inverse().transform(parentBounds);
 
@@ -1326,6 +1332,10 @@ bool Layer::isHiddenByPolicy() const {
             }
         }
     }
+    if (CC_UNLIKELY(!isTransformValid())) {
+        ALOGW("Hide layer %s because it has invalid transformation.", getDebugName());
+        return true;
+    }
     return s.flags & layer_state_t::eLayerHidden;
 }
 
@@ -1858,6 +1868,11 @@ ui::Transform Layer::getTransform() const {
     return mEffectiveTransform;
 }
 
+bool Layer::isTransformValid() const {
+    float transformDet = getTransform().det();
+    return transformDet != 0 && !isinf(transformDet) && !isnan(transformDet);
+}
+
 half Layer::getAlpha() const {
     const auto& p = mDrawingParent.promote();
 
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 21dd5f4954..2b5e337095 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -752,6 +752,7 @@ public:
                                   FrameEventHistoryDelta* outDelta);
 
     ui::Transform getTransform() const;
+    bool isTransformValid() const;
 
     // Returns the Alpha of the Surface, accounting for the Alpha
     // of parent Surfaces in the hierarchy (alpha's will be multiplied