diff options
28 files changed, 764 insertions, 4 deletions
diff --git a/core/java/android/view/MotionPredictor.java b/core/java/android/view/MotionPredictor.java new file mode 100644 index 000000000000..3e58a31745de --- /dev/null +++ b/core/java/android/view/MotionPredictor.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view; + +import android.annotation.NonNull; +import android.content.Context; + +import libcore.util.NativeAllocationRegistry; + +import java.util.Arrays; +import java.util.List; + +/** + * Calculates motion predictions. + * + * Add motions here to get predicted events! + * @hide + */ +// Acts as a pass-through to the native MotionPredictor object. +// Do not store any state in this Java layer, or add any business logic here. All of the +// implementation details should go into the native MotionPredictor. +// The context / resource access must be here rather than in native layer due to the lack of the +// corresponding native API surface. +public final class MotionPredictor { + + private static class RegistryHolder { + public static final NativeAllocationRegistry REGISTRY = + NativeAllocationRegistry.createMalloced( + MotionPredictor.class.getClassLoader(), + nativeGetNativeMotionPredictorFinalizer()); + } + + // Pointer to the native object. + private final long mPtr; + private final Context mContext; + + /** + * Create a new MotionPredictor for the provided {@link Context}. + * @param context The context for the predictions + */ + public MotionPredictor(@NonNull Context context) { + mContext = context; + final int offsetNanos = mContext.getResources().getInteger( + com.android.internal.R.integer.config_motionPredictionOffsetNanos); + mPtr = nativeInitialize(offsetNanos); + RegistryHolder.REGISTRY.registerNativeAllocation(this, mPtr); + } + + /** + * Record a movement so that in the future, a prediction for the current gesture can be + * generated. Ensure to add all motions from the gesture of interest to generate the correct + * prediction. + * @param event The received event + */ + public void record(@NonNull MotionEvent event) { + nativeRecord(mPtr, event); + } + + /** + * Get predicted events for all gestures that have been provided to the 'record' function. + * If events from multiple devices were sent to 'record', this will produce a separate + * {@link MotionEvent} for each device id. The returned list may be empty if no predictions for + * any of the added events are available. + * Predictions may not reach the requested timestamp if the confidence in the prediction results + * is low. + * + * @param predictionTimeNanos The time that the prediction should target, in the + * {@link android.os.SystemClock#uptimeMillis} time base, but in nanoseconds. + * + * @return the list of predicted motion events, for each device id. Ensure to check the + * historical data in addition to the latest ({@link MotionEvent#getX getX()}, + * {@link MotionEvent#getY getY()}) coordinates for smoothest prediction curves. Empty list is + * returned if predictions are not supported, or not possible for the current set of gestures. + */ + @NonNull + public List<MotionEvent> predict(long predictionTimeNanos) { + return Arrays.asList(nativePredict(mPtr, predictionTimeNanos)); + } + + /** + * Check whether this device supports motion predictions for the given source type. + * + * @param deviceId The input device id + * @param source The source of input events + * @return True if the current device supports predictions, false otherwise. + */ + public boolean isPredictionAvailable(int deviceId, int source) { + // Device-specific override + if (!mContext.getResources().getBoolean( + com.android.internal.R.bool.config_enableMotionPrediction)) { + return false; + } + return nativeIsPredictionAvailable(mPtr, deviceId, source); + } + + private static native long nativeInitialize(int offsetNanos); + private static native void nativeRecord(long nativePtr, MotionEvent event); + private static native MotionEvent[] nativePredict(long nativePtr, long predictionTimeNanos); + private static native boolean nativeIsPredictionAvailable(long nativePtr, int deviceId, + int source); + private static native long nativeGetNativeMotionPredictorFinalizer(); +} diff --git a/core/jni/Android.bp b/core/jni/Android.bp index a43f0b38ddce..21f1d6d073fc 100644 --- a/core/jni/Android.bp +++ b/core/jni/Android.bp @@ -129,6 +129,7 @@ cc_library_shared { "android_view_KeyCharacterMap.cpp", "android_view_KeyEvent.cpp", "android_view_MotionEvent.cpp", + "android_view_MotionPredictor.cpp", "android_view_PointerIcon.cpp", "android_view_Surface.cpp", "android_view_SurfaceControl.cpp", @@ -283,6 +284,7 @@ cc_library_shared { "libhwui", "libmediandk", "libpermission", + "libPlatformProperties", "libsensor", "libinput", "libcamera_client", diff --git a/core/jni/AndroidRuntime.cpp b/core/jni/AndroidRuntime.cpp index 6ceffde68e87..578cf2472b9a 100644 --- a/core/jni/AndroidRuntime.cpp +++ b/core/jni/AndroidRuntime.cpp @@ -187,6 +187,7 @@ extern int register_android_view_InputQueue(JNIEnv* env); extern int register_android_view_KeyCharacterMap(JNIEnv *env); extern int register_android_view_KeyEvent(JNIEnv* env); extern int register_android_view_MotionEvent(JNIEnv* env); +extern int register_android_view_MotionPredictor(JNIEnv* env); extern int register_android_view_PointerIcon(JNIEnv* env); extern int register_android_view_VelocityTracker(JNIEnv* env); extern int register_android_view_VerifiedKeyEvent(JNIEnv* env); @@ -1643,6 +1644,7 @@ static const RegJNIRec gRegJNI[] = { REG_JNI(register_android_view_InputQueue), REG_JNI(register_android_view_KeyEvent), REG_JNI(register_android_view_MotionEvent), + REG_JNI(register_android_view_MotionPredictor), REG_JNI(register_android_view_PointerIcon), REG_JNI(register_android_view_VelocityTracker), REG_JNI(register_android_view_VerifiedKeyEvent), diff --git a/core/jni/android_view_InputDevice.cpp b/core/jni/android_view_InputDevice.cpp index 7002d9b4c489..7d379e5e69c2 100644 --- a/core/jni/android_view_InputDevice.cpp +++ b/core/jni/android_view_InputDevice.cpp @@ -97,7 +97,6 @@ jobject android_view_InputDevice_create(JNIEnv* env, const InputDeviceInfo& devi return env->NewLocalRef(inputDeviceObj.get()); } - int register_android_view_InputDevice(JNIEnv* env) { gInputDeviceClassInfo.clazz = FindClassOrDie(env, "android/view/InputDevice"); @@ -108,9 +107,8 @@ int register_android_view_InputDevice(JNIEnv* env) "String;ZIILandroid/view/KeyCharacterMap;Ljava/" "lang/String;Ljava/lang/String;ZZZZZZ)V"); - gInputDeviceClassInfo.addMotionRange = GetMethodIDOrDie(env, gInputDeviceClassInfo.clazz, - "addMotionRange", "(IIFFFFF)V"); - + gInputDeviceClassInfo.addMotionRange = + GetMethodIDOrDie(env, gInputDeviceClassInfo.clazz, "addMotionRange", "(IIFFFFF)V"); return 0; } diff --git a/core/jni/android_view_MotionEvent.cpp b/core/jni/android_view_MotionEvent.cpp index 403c5836d9dd..91e545954c6e 100644 --- a/core/jni/android_view_MotionEvent.cpp +++ b/core/jni/android_view_MotionEvent.cpp @@ -102,6 +102,20 @@ jobject android_view_MotionEvent_obtainAsCopy(JNIEnv* env, const MotionEvent& ev return eventObj; } +jobject android_view_MotionEvent_obtainFromNative(JNIEnv* env, std::unique_ptr<MotionEvent> event) { + if (event == nullptr) { + return nullptr; + } + jobject eventObj = + env->CallStaticObjectMethod(gMotionEventClassInfo.clazz, gMotionEventClassInfo.obtain); + if (env->ExceptionCheck() || !eventObj) { + LOGE_EX(env); + LOG_ALWAYS_FATAL("An exception occurred while obtaining a Java motion event."); + } + android_view_MotionEvent_setNativePtr(env, eventObj, event.release()); + return eventObj; +} + status_t android_view_MotionEvent_recycle(JNIEnv* env, jobject eventObj) { env->CallVoidMethod(eventObj, gMotionEventClassInfo.recycle); if (env->ExceptionCheck()) { diff --git a/core/jni/android_view_MotionEvent.h b/core/jni/android_view_MotionEvent.h index 32a280ec1974..e81213608d68 100644 --- a/core/jni/android_view_MotionEvent.h +++ b/core/jni/android_view_MotionEvent.h @@ -28,6 +28,11 @@ class MotionEvent; * Returns NULL on error. */ extern jobject android_view_MotionEvent_obtainAsCopy(JNIEnv* env, const MotionEvent& event); +/* Obtains an instance of a Java MotionEvent object, taking over the ownership of the provided + * native MotionEvent instance. Crashes on error. */ +extern jobject android_view_MotionEvent_obtainFromNative(JNIEnv* env, + std::unique_ptr<MotionEvent> event); + /* Gets the underlying native MotionEvent instance within a DVM MotionEvent object. * Returns NULL if the event is NULL or if it is uninitialized. */ extern MotionEvent* android_view_MotionEvent_getNativePtr(JNIEnv* env, jobject eventObj); diff --git a/core/jni/android_view_MotionPredictor.cpp b/core/jni/android_view_MotionPredictor.cpp new file mode 100644 index 000000000000..2c232fadbbc5 --- /dev/null +++ b/core/jni/android_view_MotionPredictor.cpp @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "MotionPredictor-JNI" + +#include <input/Input.h> +#include <input/MotionPredictor.h> + +#include "android_view_MotionEvent.h" +#include "core_jni_converters.h" +#include "core_jni_helpers.h" + +/** + * This file is a bridge from Java to native for MotionPredictor class. + * It should be pass-through only. Do not store any state or put any business logic into this file. + */ + +namespace android { + +// ---------------------------------------------------------------------------- + +static struct { + jclass clazz; +} gMotionEventClassInfo; + +// ---------------------------------------------------------------------------- + +static void release(void* ptr) { + delete reinterpret_cast<MotionPredictor*>(ptr); +} + +static jlong android_view_MotionPredictor_nativeGetNativeMotionPredictorFinalizer(JNIEnv* env, + jclass clazz) { + return reinterpret_cast<jlong>(release); +} + +static jlong android_view_MotionPredictor_nativeInitialize(JNIEnv* env, jclass clazz, + jint offsetNanos) { + const nsecs_t offset = static_cast<nsecs_t>(offsetNanos); + return reinterpret_cast<jlong>(new MotionPredictor(offset)); +} + +static void android_view_MotionPredictor_nativeRecord(JNIEnv* env, jclass clazz, jlong ptr, + jobject event) { + MotionPredictor* predictor = reinterpret_cast<MotionPredictor*>(ptr); + MotionEvent* motionEvent = android_view_MotionEvent_getNativePtr(env, event); + predictor->record(*motionEvent); +} + +static jobject android_view_MotionPredictor_nativePredict(JNIEnv* env, jclass clazz, jlong ptr, + jlong predictionTimeNanos) { + MotionPredictor* predictor = reinterpret_cast<MotionPredictor*>(ptr); + return toJavaArray(env, predictor->predict(static_cast<nsecs_t>(predictionTimeNanos)), + gMotionEventClassInfo.clazz, &android_view_MotionEvent_obtainFromNative); +} + +static jboolean android_view_MotionPredictor_nativeIsPredictionAvailable(JNIEnv* env, jclass clazz, + jlong ptr, jint deviceId, + jint source) { + MotionPredictor* predictor = reinterpret_cast<MotionPredictor*>(ptr); + return predictor->isPredictionAvailable(static_cast<int32_t>(deviceId), + static_cast<int32_t>(source)); +} + +// ---------------------------------------------------------------------------- + +static const std::array<JNINativeMethod, 5> gMotionPredictorMethods{{ + /* name, signature, funcPtr */ + {"nativeInitialize", "(I)J", (void*)android_view_MotionPredictor_nativeInitialize}, + {"nativeGetNativeMotionPredictorFinalizer", "()J", + (void*)android_view_MotionPredictor_nativeGetNativeMotionPredictorFinalizer}, + {"nativeRecord", "(JLandroid/view/MotionEvent;)V", + (void*)android_view_MotionPredictor_nativeRecord}, + {"nativePredict", "(JJ)[Landroid/view/MotionEvent;", + (void*)android_view_MotionPredictor_nativePredict}, + {"nativeIsPredictionAvailable", "(JII)Z", + (void*)android_view_MotionPredictor_nativeIsPredictionAvailable}, +}}; + +int register_android_view_MotionPredictor(JNIEnv* env) { + jclass motionEventClazz = FindClassOrDie(env, "android/view/MotionEvent"); + gMotionEventClassInfo.clazz = MakeGlobalRefOrDie(env, motionEventClazz); + return RegisterMethodsOrDie(env, "android/view/MotionPredictor", gMotionPredictorMethods.data(), + gMotionPredictorMethods.size()); +} + +} // namespace android diff --git a/core/jni/core_jni_converters.h b/core/jni/core_jni_converters.h new file mode 100644 index 000000000000..cb9bdf76d0a4 --- /dev/null +++ b/core/jni/core_jni_converters.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <nativehelper/scoped_local_ref.h> + +template <class T> +static jobject toJavaArray(JNIEnv* env, std::vector<T>&& list, jclass clazz, + jobject (*convert)(JNIEnv* env, T)) { + jobjectArray arr = env->NewObjectArray(list.size(), clazz, nullptr); + LOG_ALWAYS_FATAL_IF(arr == nullptr); + for (size_t i = 0; i < list.size(); i++) { + T& t = list[i]; + ScopedLocalRef<jobject> javaObj(env, convert(env, std::move(t))); + env->SetObjectArrayElement(arr, i, javaObj.get()); + } + return arr; +}
\ No newline at end of file diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml index a4d6fdd28054..72657a09e2e0 100644 --- a/core/res/res/values/config.xml +++ b/core/res/res/values/config.xml @@ -2344,6 +2344,24 @@ display, this value should be true. --> <bool name="config_perDisplayFocusEnabled">false</bool> + <!-- Whether the system enables motion prediction. Only enable this after confirming that the + model works well on your device. To enable system-based prediction, set this value to true. + --> + <bool name="config_enableMotionPrediction">true</bool> + + <!-- Additional offset to use for motion prediction, in nanoseconds. A positive number indicates + that the prediction will take place further in the future. For example, suppose a + MotionEvent arrives with timestamp t=1, and the current expected presentation time is t=2. + Typically, the prediction will target the presentation time, t=2. If you'd like to make + prediction more aggressive, you could set the offset to a positive number. + Setting the offset to 1 here would mean that the prediction will be done for time t=3. + A negative number may also be provided, to make the prediction less aggressive. In general, + the offset here should represent some built-in hardware delays that may not be accounted + for by the "expected present time". See also: + https://developer.android.com/reference/android/view/ + Choreographer.FrameTimeline#getExpectedPresentationTimeNanos() --> + <integer name="config_motionPredictionOffsetNanos">0</integer> + <!-- Whether a software navigation bar should be shown. NOTE: in the future this may be autodetected from the Configuration. --> <bool name="config_showNavigationBar">false</bool> diff --git a/core/res/res/values/symbols.xml b/core/res/res/values/symbols.xml index cd39e590310b..00bee5c9cb35 100644 --- a/core/res/res/values/symbols.xml +++ b/core/res/res/values/symbols.xml @@ -1684,6 +1684,8 @@ <java-symbol type="bool" name="config_lockUiMode" /> <java-symbol type="bool" name="config_reverseDefaultRotation" /> <java-symbol type="bool" name="config_perDisplayFocusEnabled" /> + <java-symbol type="bool" name="config_enableMotionPrediction" /> + <java-symbol type="integer" name="config_motionPredictionOffsetNanos" /> <java-symbol type="bool" name="config_showNavigationBar" /> <java-symbol type="bool" name="config_supportAutoRotation" /> <java-symbol type="bool" name="config_dockedStackDividerFreeSnapMode" /> diff --git a/tests/Input/Android.bp b/tests/Input/Android.bp index de9bbb6ef9fa..83893ba46885 100644 --- a/tests/Input/Android.bp +++ b/tests/Input/Android.bp @@ -18,6 +18,7 @@ android_test { static_libs: [ "androidx.test.ext.junit", "androidx.test.rules", + "mockito-target-minus-junit4", "services.core.unboosted", "testables", "truth-prebuilt", diff --git a/tests/Input/src/com/android/test/input/MotionPredictorTest.kt b/tests/Input/src/com/android/test/input/MotionPredictorTest.kt new file mode 100644 index 000000000000..46aad9f635ca --- /dev/null +++ b/tests/Input/src/com/android/test/input/MotionPredictorTest.kt @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.test.input + +import android.content.Context +import android.content.res.Resources +import android.os.SystemProperties +import android.view.InputDevice +import android.view.MotionEvent +import android.view.MotionEvent.ACTION_DOWN +import android.view.MotionEvent.ACTION_MOVE +import android.view.MotionEvent.PointerCoords +import android.view.MotionEvent.PointerProperties +import android.view.MotionPredictor + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.filters.SmallTest +import androidx.test.platform.app.InstrumentationRegistry + +import org.mockito.Mockito.mock +import org.mockito.Mockito.`when` + +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith + +import java.time.Duration + +private fun getStylusMotionEvent( + eventTime: Duration, + action: Int, + x: Float, + y: Float, + ): MotionEvent{ + // One-time: send a DOWN event + val pointerCount = 1 + val properties = arrayOfNulls<MotionEvent.PointerProperties>(pointerCount) + val coords = arrayOfNulls<MotionEvent.PointerCoords>(pointerCount) + + for (i in 0 until pointerCount) { + properties[i] = PointerProperties() + properties[i]!!.id = i + properties[i]!!.toolType = MotionEvent.TOOL_TYPE_STYLUS + coords[i] = PointerCoords() + coords[i]!!.x = x + coords[i]!!.y = y + } + + return MotionEvent.obtain(/*downTime=*/0, eventTime.toMillis(), action, properties.size, + properties, coords, /*metaState=*/0, /*buttonState=*/0, + /*xPrecision=*/0f, /*yPrecision=*/0f, /*deviceId=*/0, /*edgeFlags=*/0, + InputDevice.SOURCE_STYLUS, /*flags=*/0) +} + +private fun getPredictionContext(offset: Duration, enablePrediction: Boolean): Context { + val context = mock(Context::class.java) + val resources: Resources = mock(Resources::class.java) + `when`(context.getResources()).thenReturn(resources) + `when`(resources.getInteger( + com.android.internal.R.integer.config_motionPredictionOffsetNanos)).thenReturn( + offset.toNanos().toInt()) + `when`(resources.getBoolean( + com.android.internal.R.bool.config_enableMotionPrediction)).thenReturn(enablePrediction) + return context +} + +@RunWith(AndroidJUnit4::class) +@SmallTest +class MotionPredictorTest { + private val instrumentation = InstrumentationRegistry.getInstrumentation() + val initialPropertyValue = SystemProperties.get("persist.input.enable_motion_prediction") + + @Before + fun setUp() { + instrumentation.uiAutomation.executeShellCommand( + "setprop persist.input.enable_motion_prediction true") + } + + @After + fun tearDown() { + instrumentation.uiAutomation.executeShellCommand( + "setprop persist.input.enable_motion_prediction $initialPropertyValue") + } + + /** + * In a typical usage, app will send the event to the predictor and then call .predict to draw + * a prediction. Here, we send 2 events to the predictor and check the returned event. + * Input: + * t = 0 x = 0 y = 0 + * t = 1 x = 1 y = 2 + * Output (expected): + * t = 3 x = 3 y = 6 + * + * Historical data is ignored for simplicity. + */ + @Test + fun testPredictedCoordinatesAndTime() { + val context = getPredictionContext( + /*offset=*/Duration.ofMillis(1), /*enablePrediction=*/true) + val predictor = MotionPredictor(context) + var eventTime = Duration.ofMillis(0) + val downEvent = getStylusMotionEvent(eventTime, ACTION_DOWN, /*x=*/0f, /*y=*/0f) + // ACTION_DOWN t=0 x=0 y=0 + predictor.record(downEvent) + + eventTime += Duration.ofMillis(1) + val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /*x=*/1f, /*y=*/2f) + // ACTION_MOVE t=1 x=1 y=2 + predictor.record(moveEvent) + + val predicted = predictor.predict(Duration.ofMillis(2).toNanos()) + assertEquals(1, predicted.size) + val event = predicted[0] + assertNotNull(event) + + // Prediction will happen for t=3 (2 + 1, since offset is 1 and present time is 2) + assertEquals(3, event.eventTime) + assertEquals(3f, event.x, /*delta=*/0.001f) + assertEquals(6f, event.y, /*delta=*/0.001f) + } +} diff --git a/tests/MotionPrediction/Android.bp b/tests/MotionPrediction/Android.bp new file mode 100644 index 000000000000..b9d01da263aa --- /dev/null +++ b/tests/MotionPrediction/Android.bp @@ -0,0 +1,31 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "frameworks_base_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["frameworks_base_license"], +} + +android_app { + name: "MotionPrediction", + srcs: ["**/*.kt"], + platform_apis: true, + certificate: "platform", +} diff --git a/tests/MotionPrediction/AndroidManifest.xml b/tests/MotionPrediction/AndroidManifest.xml new file mode 100644 index 000000000000..3f8c2f278623 --- /dev/null +++ b/tests/MotionPrediction/AndroidManifest.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="test.motionprediction"> + + <application android:allowBackup="false" + android:icon="@mipmap/ic_launcher" + android:label="@string/app_name" + android:supportsRtl="true" + android:theme="@style/AppTheme"> + <activity android:name=".MainActivity" + android:exported="true"> + <intent-filter> + <action android:name="android.intent.action.MAIN"/> + <category android:name="android.intent.category.LAUNCHER"/> + </intent-filter> + </activity> + </application> + +</manifest> diff --git a/tests/MotionPrediction/OWNERS b/tests/MotionPrediction/OWNERS new file mode 100644 index 000000000000..c88bfe97cab9 --- /dev/null +++ b/tests/MotionPrediction/OWNERS @@ -0,0 +1 @@ +include platform/frameworks/base:/INPUT_OWNERS diff --git a/tests/MotionPrediction/res/layout/activity_main.xml b/tests/MotionPrediction/res/layout/activity_main.xml new file mode 100644 index 000000000000..65dc325befdb --- /dev/null +++ b/tests/MotionPrediction/res/layout/activity_main.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:tools="http://schemas.android.com/tools" + android:layout_width="match_parent" + android:layout_height="match_parent" + android:orientation="vertical" + android:paddingBottom="@dimen/activity_vertical_margin" + android:paddingLeft="@dimen/activity_horizontal_margin" + android:paddingRight="@dimen/activity_horizontal_margin" + android:paddingTop="@dimen/activity_vertical_margin" + tools:context="test.motionprediction.MainActivity"> + + <test.motionprediction.DrawingView + android:layout_width="match_parent" + android:layout_height="match_parent" + android:id="@+id/output" /> + +</LinearLayout> diff --git a/tests/MotionPrediction/res/mipmap-hdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-hdpi/ic_launcher.png Binary files differnew file mode 100644 index 000000000000..cde69bcccec6 --- /dev/null +++ b/tests/MotionPrediction/res/mipmap-hdpi/ic_launcher.png diff --git a/tests/MotionPrediction/res/mipmap-mdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-mdpi/ic_launcher.png Binary files differnew file mode 100644 index 000000000000..c133a0cbd379 --- /dev/null +++ b/tests/MotionPrediction/res/mipmap-mdpi/ic_launcher.png diff --git a/tests/MotionPrediction/res/mipmap-xhdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-xhdpi/ic_launcher.png Binary files differnew file mode 100644 index 000000000000..bfa42f0e7b91 --- /dev/null +++ b/tests/MotionPrediction/res/mipmap-xhdpi/ic_launcher.png diff --git a/tests/MotionPrediction/res/mipmap-xxhdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-xxhdpi/ic_launcher.png Binary files differnew file mode 100644 index 000000000000..324e72cdd748 --- /dev/null +++ b/tests/MotionPrediction/res/mipmap-xxhdpi/ic_launcher.png diff --git a/tests/MotionPrediction/res/mipmap-xxxhdpi/ic_launcher.png b/tests/MotionPrediction/res/mipmap-xxxhdpi/ic_launcher.png Binary files differnew file mode 100644 index 000000000000..aee44e138434 --- /dev/null +++ b/tests/MotionPrediction/res/mipmap-xxxhdpi/ic_launcher.png diff --git a/tests/MotionPrediction/res/values-w820dp/dimens.xml b/tests/MotionPrediction/res/values-w820dp/dimens.xml new file mode 100644 index 000000000000..95669e6aa6fa --- /dev/null +++ b/tests/MotionPrediction/res/values-w820dp/dimens.xml @@ -0,0 +1,20 @@ +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <!-- Example customization of dimensions originally defined in res/values/dimens.xml + (such as screen margins) for screens with more than 820dp of available width. This + would include 7" and 10" devices in landscape (~960dp and ~1280dp respectively). --> + <dimen name="activity_horizontal_margin">64dp</dimen> +</resources> diff --git a/tests/MotionPrediction/res/values/colors.xml b/tests/MotionPrediction/res/values/colors.xml new file mode 100644 index 000000000000..139eb1d1303b --- /dev/null +++ b/tests/MotionPrediction/res/values/colors.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <color name="colorPrimary">#3F51B5</color> + <color name="colorPrimaryDark">#303F9F</color> + <color name="colorAccent">#FF4081</color> +</resources> diff --git a/tests/MotionPrediction/res/values/dimens.xml b/tests/MotionPrediction/res/values/dimens.xml new file mode 100644 index 000000000000..d26136f18c29 --- /dev/null +++ b/tests/MotionPrediction/res/values/dimens.xml @@ -0,0 +1,19 @@ +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <!-- Default screen margins, per the Android Design guidelines. --> + <dimen name="activity_horizontal_margin">16dp</dimen> + <dimen name="activity_vertical_margin">16dp</dimen> +</resources> diff --git a/tests/MotionPrediction/res/values/strings.xml b/tests/MotionPrediction/res/values/strings.xml new file mode 100644 index 000000000000..16a2bdf34d13 --- /dev/null +++ b/tests/MotionPrediction/res/values/strings.xml @@ -0,0 +1,17 @@ +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <string name="app_name">Motion Prediction</string> +</resources> diff --git a/tests/MotionPrediction/res/values/styles.xml b/tests/MotionPrediction/res/values/styles.xml new file mode 100644 index 000000000000..cfb5e3d83c0d --- /dev/null +++ b/tests/MotionPrediction/res/values/styles.xml @@ -0,0 +1,23 @@ +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<resources> + <!-- Base application theme. --> + <style name="AppTheme" parent="@android:style/Theme.Material.Light.DarkActionBar"> + <!-- Customize your theme here. --> + <item name="android:colorPrimary">@color/colorPrimary</item> + <item name="android:colorPrimaryDark">@color/colorPrimaryDark</item> + <item name="android:colorAccent">@color/colorAccent</item> + </style> +</resources> diff --git a/tests/MotionPrediction/src/test/motionprediction/DrawingView.kt b/tests/MotionPrediction/src/test/motionprediction/DrawingView.kt new file mode 100644 index 000000000000..f529bf77f32a --- /dev/null +++ b/tests/MotionPrediction/src/test/motionprediction/DrawingView.kt @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test.motionprediction + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.util.AttributeSet +import android.view.MotionEvent +import android.view.MotionEvent.ACTION_DOWN +import android.view.MotionPredictor +import android.view.View + +import java.util.Vector + +private fun drawLine(canvas: Canvas, from: MotionEvent, to: MotionEvent, paint: Paint) { + canvas.apply { + val x0 = from.getX() + val y0 = from.getY() + val x1 = to.getX() + val y1 = to.getY() + // TODO: handle historical data + drawLine(x0, y0, x1, y1, paint) + } +} + +/** + * Draw the current stroke and predicted values + */ +class DrawingView(context: Context, attrs: AttributeSet) : View(context, attrs) { + private val TAG = "DrawingView" + + val events: MutableMap<Int, Vector<MotionEvent>> = mutableMapOf<Int, Vector<MotionEvent>>() + + var isPredictionAvailable = false + private val predictor = MotionPredictor(getContext()) + + private var predictionPaint = Paint() + private var realPaint = Paint() + + init { + setBackgroundColor(Color.WHITE) + predictionPaint.color = Color.BLACK + predictionPaint.setStrokeWidth(5f) + realPaint.color = Color.RED + realPaint.setStrokeWidth(5f) + } + + private fun addEvent(event: MotionEvent) { + if (event.getActionMasked() == ACTION_DOWN) { + events.remove(event.deviceId) + } + var vec = events.getOrPut(event.deviceId) { Vector<MotionEvent>() } + vec.add(MotionEvent.obtain(event)) + predictor.record(event) + invalidate() + } + + public override fun onTouchEvent(event: MotionEvent): Boolean { + isPredictionAvailable = predictor.isPredictionAvailable(event.getDeviceId(), + event.getSource()) + addEvent(event) + return true + } + + public override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) + if (!isPredictionAvailable) { + canvas.apply { + drawRect(0f, 0f, 200f, 200f, realPaint) + } + } + + var eventTime = 0L + + // Draw real events + for ((_, vec) in events ) { + for (i in 1 until vec.size) { + drawLine(canvas, vec[i - 1], vec[i], realPaint) + } + eventTime = vec.lastElement().eventTime + } + + // Draw predictions. Convert to nanos and hardcode to +20ms into the future + val predictionList = predictor.predict(eventTime * 1000000 + 20000000) + for (prediction in predictionList) { + val realEvents = events.get(prediction.deviceId)!! + drawLine(canvas, realEvents[realEvents.size - 1], prediction, predictionPaint) + } + } +} diff --git a/tests/MotionPrediction/src/test/motionprediction/MainActivity.kt b/tests/MotionPrediction/src/test/motionprediction/MainActivity.kt new file mode 100644 index 000000000000..cec2c06157a1 --- /dev/null +++ b/tests/MotionPrediction/src/test/motionprediction/MainActivity.kt @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test.motionprediction + +import android.app.Activity +import android.os.Bundle + +class MainActivity : Activity() { + val TAG = "MotionPrediction" + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + } +} |
