Add vector of TouchVideoFrame to NotifyMotionArgs

NotifyMotionArgs is passed to InputListener. We add a vector of
TouchVideoFrame here to NotifyMotionArgs  in order to provide heatmap
information about touch. This heatmap information can later be used to
do further processing of touch. For example, it could be used to
determine whether a given touch stream represents a user intentionally
pressing harder on the screen.

Introduce a new class, TouchVideoFrame, that will represent a single
scan of the touch heatmap for a given touch event.

Test: integration tested by adding a debug log to the case where the
finger area is large. That means, the video frames are being passed
locally to the InputClassfier HAL. Although HAL is not being submitted
together with this change, the current change will make the subsequent
CLs simpler to review.
Bug: 111480215

Change-Id: I4f16c8b2bd726c6bd4bbd5d2b18d2536a2347bda

4 files changed, 96 insertions, 21 deletions

diff --git a/include/input/TouchVideoFrame.h b/include/input/TouchVideoFrame.h
new file mode 100644
index 0000000000..d68f27431a
--- /dev/null
+++ b/include/input/TouchVideoFrame.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _LIBINPUT_TOUCHVIDEOFRAME_H
+#define _LIBINPUT_TOUCHVIDEOFRAME_H
+
+#include <stdint.h>
+#include <sys/time.h>
+#include <vector>
+
+namespace android {
+
+/**
+ * Represents data from a single scan of the touchscreen device.
+ * Similar in concept to a video frame, but the touch strength is used as
+ * the values instead.
+ */
+class TouchVideoFrame {
+public:
+    TouchVideoFrame(uint32_t width, uint32_t height, std::vector<int16_t> data,
+            const struct timeval& timestamp) :
+            mWidth(width), mHeight(height), mData(std::move(data)), mTimestamp(timestamp) {
+    }
+
+    /**
+     * Width of the frame
+     */
+    uint32_t getWidth() const { return mWidth; }
+    /**
+     * Height of the frame
+     */
+    uint32_t getHeight() const { return mHeight; }
+    /**
+     * The touch strength data.
+     * The array is a 2-D row-major matrix, with dimensions (height, width).
+     * Total size of the array should equal getHeight() * getWidth().
+     * Data is allowed to be negative.
+     */
+    const std::vector<int16_t>& getData() const { return mData; }
+    /**
+     * Time at which the heatmap was taken.
+     */
+    const struct timeval& getTimestamp() const { return mTimestamp; }
+
+private:
+    uint32_t mWidth;
+    uint32_t mHeight;
+    std::vector<int16_t> mData;
+    struct timeval mTimestamp;
+};
+
+} // namespace android
+
+#endif // _LIBINPUT_TOUCHVIDEOFRAME_H
diff --git a/services/inputflinger/InputListener.cpp b/services/inputflinger/InputListener.cpp
index 23cceb4713..b4eb37026b 100644
--- a/services/inputflinger/InputListener.cpp
+++ b/services/inputflinger/InputListener.cpp
@@ -74,14 +74,16 @@ NotifyMotionArgs::NotifyMotionArgs(uint32_t sequenceNum, nsecs_t eventTime, int3
         int32_t buttonState, int32_t edgeFlags, uint32_t deviceTimestamp,
         uint32_t pointerCount,
         const PointerProperties* pointerProperties, const PointerCoords* pointerCoords,
-        float xPrecision, float yPrecision, nsecs_t downTime) :
+        float xPrecision, float yPrecision, nsecs_t downTime,
+        const std::vector<TouchVideoFrame>& videoFrames) :
         NotifyArgs(sequenceNum), eventTime(eventTime), deviceId(deviceId), source(source),
         displayId(displayId), policyFlags(policyFlags),
         action(action), actionButton(actionButton),
         flags(flags), metaState(metaState), buttonState(buttonState),
         edgeFlags(edgeFlags), deviceTimestamp(deviceTimestamp),
         pointerCount(pointerCount),
-        xPrecision(xPrecision), yPrecision(yPrecision), downTime(downTime) {
+        xPrecision(xPrecision), yPrecision(yPrecision), downTime(downTime),
+        videoFrames(videoFrames) {
     for (uint32_t i = 0; i < pointerCount; i++) {
         this->pointerProperties[i].copyFrom(pointerProperties[i]);
         this->pointerCoords[i].copyFrom(pointerCoords[i]);
@@ -95,7 +97,8 @@ NotifyMotionArgs::NotifyMotionArgs(const NotifyMotionArgs& other) :
         metaState(other.metaState), buttonState(other.buttonState),
         edgeFlags(other.edgeFlags),
         deviceTimestamp(other.deviceTimestamp), pointerCount(other.pointerCount),
-        xPrecision(other.xPrecision), yPrecision(other.yPrecision), downTime(other.downTime) {
+        xPrecision(other.xPrecision), yPrecision(other.yPrecision), downTime(other.downTime),
+        videoFrames(other.videoFrames) {
     for (uint32_t i = 0; i < pointerCount; i++) {
         pointerProperties[i].copyFrom(other.pointerProperties[i]);
         pointerCoords[i].copyFrom(other.pointerCoords[i]);
diff --git a/services/inputflinger/InputReader.cpp b/services/inputflinger/InputReader.cpp
index 5d0b894dd0..2d342bce94 100644
--- a/services/inputflinger/InputReader.cpp
+++ b/services/inputflinger/InputReader.cpp
@@ -2832,7 +2832,7 @@ void CursorInputMapper::sync(nsecs_t when) {
                         AMOTION_EVENT_ACTION_BUTTON_RELEASE, actionButton, 0,
                         metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,
                         /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                        mXPrecision, mYPrecision, downTime);
+                        mXPrecision, mYPrecision, downTime, /* videoFrames */ {});
                 getListener()->notifyMotion(&releaseArgs);
             }
         }
@@ -2841,7 +2841,7 @@ void CursorInputMapper::sync(nsecs_t when) {
                 displayId, policyFlags, motionEventAction, 0, 0, metaState, currentButtonState,
                 AMOTION_EVENT_EDGE_FLAG_NONE,
                 /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                mXPrecision, mYPrecision, downTime);
+                mXPrecision, mYPrecision, downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
 
         if (buttonsPressed) {
@@ -2853,7 +2853,7 @@ void CursorInputMapper::sync(nsecs_t when) {
                         mSource, displayId, policyFlags, AMOTION_EVENT_ACTION_BUTTON_PRESS,
                         actionButton, 0, metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,
                         /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                        mXPrecision, mYPrecision, downTime);
+                        mXPrecision, mYPrecision, downTime, /* videoFrames */ {});
                 getListener()->notifyMotion(&pressArgs);
             }
         }
@@ -2867,7 +2867,7 @@ void CursorInputMapper::sync(nsecs_t when) {
                     mSource, displayId, policyFlags, AMOTION_EVENT_ACTION_HOVER_MOVE, 0, 0,
                     metaState, currentButtonState, AMOTION_EVENT_EDGE_FLAG_NONE,
                     /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                    mXPrecision, mYPrecision, downTime);
+                    mXPrecision, mYPrecision, downTime, /* videoFrames */ {});
             getListener()->notifyMotion(&hoverArgs);
         }
 
@@ -2881,7 +2881,7 @@ void CursorInputMapper::sync(nsecs_t when) {
                     AMOTION_EVENT_ACTION_SCROLL, 0, 0, metaState, currentButtonState,
                     AMOTION_EVENT_EDGE_FLAG_NONE,
                     /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                    mXPrecision, mYPrecision, downTime);
+                    mXPrecision, mYPrecision, downTime, /* videoFrames */ {});
             getListener()->notifyMotion(&scrollArgs);
         }
     }
@@ -3013,7 +3013,7 @@ void RotaryEncoderInputMapper::sync(nsecs_t when) {
                 AMOTION_EVENT_ACTION_SCROLL, 0, 0, metaState, 0,
                 AMOTION_EVENT_EDGE_FLAG_NONE,
                 /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                0, 0, 0);
+                0, 0, 0, /* videoFrames */ {});
         getListener()->notifyMotion(&scrollArgs);
     }
 
@@ -5400,7 +5400,7 @@ void TouchInputMapper::dispatchPointerGestures(nsecs_t when, uint32_t policyFlag
                 AMOTION_EVENT_ACTION_HOVER_MOVE, 0, 0,
                 metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,
                 /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-                0, 0, mPointerGesture.downTime);
+                0, 0, mPointerGesture.downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
     }
 
@@ -6319,13 +6319,13 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
         mPointerSimple.down = false;
 
         // Send up.
-        NotifyMotionArgs args(mContext->getNextSequenceNum(), when, getDeviceId(), 
+        NotifyMotionArgs args(mContext->getNextSequenceNum(), when, getDeviceId(),
                 mSource, mViewport.displayId, policyFlags,
                 AMOTION_EVENT_ACTION_UP, 0, 0, metaState, mLastRawState.buttonState, 0,
                 /* deviceTimestamp */ 0,
                 1, &mPointerSimple.lastProperties, &mPointerSimple.lastCoords,
                 mOrientedXPrecision, mOrientedYPrecision,
-                mPointerSimple.downTime);
+                mPointerSimple.downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
     }
 
@@ -6339,7 +6339,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
                 /* deviceTimestamp */ 0,
                 1, &mPointerSimple.lastProperties, &mPointerSimple.lastCoords,
                 mOrientedXPrecision, mOrientedYPrecision,
-                mPointerSimple.downTime);
+                mPointerSimple.downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
     }
 
@@ -6355,7 +6355,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
                     /* deviceTimestamp */ 0,
                     1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,
                     mOrientedXPrecision, mOrientedYPrecision,
-                    mPointerSimple.downTime);
+                    mPointerSimple.downTime, /* videoFrames */ {});
             getListener()->notifyMotion(&args);
         }
 
@@ -6366,7 +6366,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
                 /* deviceTimestamp */ 0,
                 1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,
                 mOrientedXPrecision, mOrientedYPrecision,
-                mPointerSimple.downTime);
+                mPointerSimple.downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
     }
 
@@ -6382,7 +6382,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
                     /* deviceTimestamp */ 0,
                     1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,
                     mOrientedXPrecision, mOrientedYPrecision,
-                    mPointerSimple.downTime);
+                    mPointerSimple.downTime, /* videoFrames */ {});
             getListener()->notifyMotion(&args);
         }
 
@@ -6394,7 +6394,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
                 /* deviceTimestamp */ 0,
                 1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,
                 mOrientedXPrecision, mOrientedYPrecision,
-                mPointerSimple.downTime);
+                mPointerSimple.downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
     }
 
@@ -6416,7 +6416,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags,
                 /* deviceTimestamp */ 0,
                 1, &mPointerSimple.currentProperties, &pointerCoords,
                 mOrientedXPrecision, mOrientedYPrecision,
-                mPointerSimple.downTime);
+                mPointerSimple.downTime, /* videoFrames */ {});
         getListener()->notifyMotion(&args);
     }
 
@@ -6478,7 +6478,7 @@ void TouchInputMapper::dispatchMotion(nsecs_t when, uint32_t policyFlags, uint32
             source, mViewport.displayId, policyFlags,
             action, actionButton, flags, metaState, buttonState, edgeFlags,
             deviceTimestamp, pointerCount, pointerProperties, pointerCoords,
-            xPrecision, yPrecision, downTime);
+            xPrecision, yPrecision, downTime, /* videoFrames */ {});
     getListener()->notifyMotion(&args);
 }
 
@@ -7403,7 +7403,7 @@ void JoystickInputMapper::sync(nsecs_t when, bool force) {
             AINPUT_SOURCE_JOYSTICK, ADISPLAY_ID_NONE, policyFlags,
             AMOTION_EVENT_ACTION_MOVE, 0, 0, metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,
             /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,
-            0, 0, 0);
+            0, 0, 0, /* videoFrames */ {});
     getListener()->notifyMotion(&args);
 }
 
diff --git a/services/inputflinger/include/InputListener.h b/services/inputflinger/include/InputListener.h
index f3a30ab684..2442cc052b 100644
--- a/services/inputflinger/include/InputListener.h
+++ b/services/inputflinger/include/InputListener.h
@@ -17,7 +17,10 @@
 #ifndef _UI_INPUT_LISTENER_H
 #define _UI_INPUT_LISTENER_H
 
+#include <vector>
+
 #include <input/Input.h>
+#include <input/TouchVideoFrame.h>
 #include <utils/RefBase.h>
 #include <utils/Vector.h>
 
@@ -110,6 +113,7 @@ struct NotifyMotionArgs : public NotifyArgs {
     float xPrecision;
     float yPrecision;
     nsecs_t downTime;
+    std::vector<TouchVideoFrame> videoFrames;
 
     inline NotifyMotionArgs() { }
 
@@ -119,7 +123,8 @@ struct NotifyMotionArgs : public NotifyArgs {
             int32_t metaState, int32_t buttonState,
             int32_t edgeFlags, uint32_t deviceTimestamp, uint32_t pointerCount,
             const PointerProperties* pointerProperties, const PointerCoords* pointerCoords,
-            float xPrecision, float yPrecision, nsecs_t downTime);
+            float xPrecision, float yPrecision, nsecs_t downTime,
+            const std::vector<TouchVideoFrame>& videoFrames);
 
     NotifyMotionArgs(const NotifyMotionArgs& other);