diff options
35 files changed, 927 insertions, 278 deletions
diff --git a/core/java/android/app/WallpaperManager.java b/core/java/android/app/WallpaperManager.java index 6592019dc0d9..f6733047d965 100644 --- a/core/java/android/app/WallpaperManager.java +++ b/core/java/android/app/WallpaperManager.java @@ -822,6 +822,10 @@ public class WallpaperManager { */ @TestApi public boolean isLockscreenLiveWallpaperEnabled() { + return isLockscreenLiveWallpaperEnabledHelper(); + } + + private static boolean isLockscreenLiveWallpaperEnabledHelper() { if (sGlobals == null) { sIsLockscreenLiveWallpaperEnabled = SystemProperties.getBoolean( "persist.wm.debug.lockscreen_live_wallpaper", false); @@ -2757,7 +2761,7 @@ public class WallpaperManager { public static InputStream openDefaultWallpaper(Context context, @SetWallpaperFlags int which) { final String whichProp; final int defaultResId; - if (which == FLAG_LOCK && !sIsLockscreenLiveWallpaperEnabled) { + if (which == FLAG_LOCK && !isLockscreenLiveWallpaperEnabledHelper()) { /* Factory-default lock wallpapers are not yet supported whichProp = PROP_LOCK_WALLPAPER; defaultResId = com.android.internal.R.drawable.default_lock_wallpaper; diff --git a/core/java/android/hardware/soundtrigger/ConversionUtil.java b/core/java/android/hardware/soundtrigger/ConversionUtil.java index 888047d95f80..21fe686fa2e3 100644 --- a/core/java/android/hardware/soundtrigger/ConversionUtil.java +++ b/core/java/android/hardware/soundtrigger/ConversionUtil.java @@ -32,10 +32,12 @@ import android.media.soundtrigger.RecognitionConfig; import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionMode; import android.media.soundtrigger.SoundModel; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; import android.os.ParcelFileDescriptor; -import android.system.ErrnoException; import android.os.SharedMemory; +import android.system.ErrnoException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -219,36 +221,40 @@ public class ConversionUtil { return new SoundTrigger.ConfidenceLevel(apiLevel.userId, apiLevel.levelPercent); } - public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent( - int modelHandle, int captureSession, RecognitionEvent aidlEvent) { + public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent(int modelHandle, + int captureSession, RecognitionEventSys aidlEvent) { + RecognitionEvent recognitionEvent = aidlEvent.recognitionEvent; // The API recognition event doesn't allow for a null audio format, even though it doesn't // always make sense. We thus replace it with a default. - AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(aidlEvent.audioConfig, + AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(recognitionEvent.audioConfig, true /*isInput*/); - // TODO(b/265852186) propagate a timestamp from aidl interfaces - return new SoundTrigger.GenericRecognitionEvent(aidlEvent.status, modelHandle, - aidlEvent.captureAvailable, captureSession, aidlEvent.captureDelayMs, - aidlEvent.capturePreambleMs, aidlEvent.triggerInData, audioFormat, aidlEvent.data, - aidlEvent.recognitionStillActive, -1 /* halEventReceivedMillis */); + return new SoundTrigger.GenericRecognitionEvent(recognitionEvent.status, modelHandle, + recognitionEvent.captureAvailable, captureSession, recognitionEvent.captureDelayMs, + recognitionEvent.capturePreambleMs, recognitionEvent.triggerInData, audioFormat, + recognitionEvent.data, + recognitionEvent.recognitionStillActive, aidlEvent.halEventReceivedMillis); } public static SoundTrigger.RecognitionEvent aidl2apiPhraseRecognitionEvent( - int modelHandle, int captureSession, - PhraseRecognitionEvent aidlEvent) { + int modelHandle, int captureSession, PhraseRecognitionEventSys aidlEvent) { + PhraseRecognitionEvent recognitionEvent = aidlEvent.phraseRecognitionEvent; SoundTrigger.KeyphraseRecognitionExtra[] apiExtras = - new SoundTrigger.KeyphraseRecognitionExtra[aidlEvent.phraseExtras.length]; - for (int i = 0; i < aidlEvent.phraseExtras.length; ++i) { - apiExtras[i] = aidl2apiPhraseRecognitionExtra(aidlEvent.phraseExtras[i]); + new SoundTrigger.KeyphraseRecognitionExtra[recognitionEvent.phraseExtras.length]; + for (int i = 0; i < recognitionEvent.phraseExtras.length; ++i) { + apiExtras[i] = aidl2apiPhraseRecognitionExtra(recognitionEvent.phraseExtras[i]); } // The API recognition event doesn't allow for a null audio format, even though it doesn't // always make sense. We thus replace it with a default. - AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(aidlEvent.common.audioConfig, + AudioFormat audioFormat = aidl2apiAudioFormatWithDefault( + recognitionEvent.common.audioConfig, true /*isInput*/); - // TODO(b/265852186) propagate a timestamp from aidl interfaces - return new SoundTrigger.KeyphraseRecognitionEvent(aidlEvent.common.status, modelHandle, - aidlEvent.common.captureAvailable, captureSession, aidlEvent.common.captureDelayMs, - aidlEvent.common.capturePreambleMs, aidlEvent.common.triggerInData, audioFormat, - aidlEvent.common.data, apiExtras, -1 /* halEventReceivedMillis */); + return new SoundTrigger.KeyphraseRecognitionEvent(recognitionEvent.common.status, + modelHandle, + recognitionEvent.common.captureAvailable, captureSession, + recognitionEvent.common.captureDelayMs, + recognitionEvent.common.capturePreambleMs, recognitionEvent.common.triggerInData, + audioFormat, + recognitionEvent.common.data, apiExtras, aidlEvent.halEventReceivedMillis); } // In case of a null input returns a non-null valid output. diff --git a/core/java/android/hardware/soundtrigger/SoundTriggerModule.java b/core/java/android/hardware/soundtrigger/SoundTriggerModule.java index 37c5213e0359..5cdbe233aa3b 100644 --- a/core/java/android/hardware/soundtrigger/SoundTriggerModule.java +++ b/core/java/android/hardware/soundtrigger/SoundTriggerModule.java @@ -22,13 +22,13 @@ import android.compat.annotation.UnsupportedAppUsage; import android.media.permission.ClearCallingIdentityContext; import android.media.permission.Identity; import android.media.permission.SafeCloseable; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.Build; import android.os.Handler; import android.os.IBinder; @@ -398,7 +398,7 @@ public class SoundTriggerModule { } @Override - public synchronized void onRecognition(int handle, RecognitionEvent event, + public synchronized void onRecognition(int handle, RecognitionEventSys event, int captureSession) throws RemoteException { Message m = mHandler.obtainMessage(EVENT_RECOGNITION, @@ -407,7 +407,7 @@ public class SoundTriggerModule { } @Override - public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEvent event, + public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEventSys event, int captureSession) throws RemoteException { Message m = mHandler.obtainMessage(EVENT_RECOGNITION, diff --git a/core/java/android/provider/Settings.java b/core/java/android/provider/Settings.java index 4a46beb670de..329a2fa57c08 100644 --- a/core/java/android/provider/Settings.java +++ b/core/java/android/provider/Settings.java @@ -7175,6 +7175,13 @@ public final class Settings { public static final String CREDENTIAL_SERVICE = "credential_service"; /** + * The currently selected primary credential service flattened ComponentName. + * + * @hide + */ + public static final String CREDENTIAL_SERVICE_PRIMARY = "credential_service_primary"; + + /** * The currently selected autofill service flattened ComponentName. * @hide */ diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml index 7bc3ab80da99..87ba59abdebe 100644 --- a/core/res/res/values/config.xml +++ b/core/res/res/values/config.xml @@ -991,6 +991,25 @@ <!-- Nominal White Z --> <item>1.089058</item> </string-array> + <!-- The CCT closest to the white coordinates (primary) above and in SurfaceControl. --> + <integer name="config_displayWhiteBalanceDisplayNominalWhiteCct">6500</integer> + + <!-- Range minimums corresponding to config_displayWhiteBalanceDisplaySteps. For example, if the + range minimums are [0, 3000] and the steps are [10, 20] then between 0 and 3000, exclusive, + the step between them will be 10 (i.e. 0, 10, 20, etc.) and the step between 3000 and the + maximum value is 20 (i.e. 3000, 3020, 3040, etc.). --> + <integer-array name="config_displayWhiteBalanceDisplayRangeMinimums"> + <item>0</item> + </integer-array> + + <!-- Steps corresponding to config_displayWhiteBalanceDisplayRangeMinimums. For example, if the + range minimums are [0, 3000] and the steps are [10, 20] then between 0 and 3000, exclusive, + the step between them will be 10 (i.e. 0, 10, 20, etc.) and the step between 3000 and the + maximum value is 20 (i.e. 3000, 3020, 3040, etc.). --> + <integer-array name="config_displayWhiteBalanceDisplaySteps"> + <item>1</item> + </integer-array> + <!-- Boolean indicating whether light mode is allowed when DWB is turned on. --> <bool name="config_displayWhiteBalanceLightModeAllowed">true</bool> diff --git a/core/res/res/values/symbols.xml b/core/res/res/values/symbols.xml index e25425d6cb18..218bbc24bbe6 100644 --- a/core/res/res/values/symbols.xml +++ b/core/res/res/values/symbols.xml @@ -3423,6 +3423,9 @@ <java-symbol type="integer" name="config_displayWhiteBalanceColorTemperatureDefault" /> <java-symbol type="array" name="config_displayWhiteBalanceDisplayPrimaries" /> <java-symbol type="array" name="config_displayWhiteBalanceDisplayNominalWhite" /> + <java-symbol type="integer" name="config_displayWhiteBalanceDisplayNominalWhiteCct" /> + <java-symbol type="array" name="config_displayWhiteBalanceDisplayRangeMinimums" /> + <java-symbol type="array" name="config_displayWhiteBalanceDisplaySteps" /> <java-symbol type="bool" name="config_displayWhiteBalanceLightModeAllowed" /> <java-symbol type="integer" name="config_displayWhiteBalanceTransitionTime" /> diff --git a/media/aidl/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl b/media/aidl/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl index 6092ac53ec39..82fc33edf08a 100644 --- a/media/aidl/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl +++ b/media/aidl/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl @@ -15,8 +15,8 @@ */ package android.media.soundtrigger_middleware; -import android.media.soundtrigger.RecognitionEvent; -import android.media.soundtrigger.PhraseRecognitionEvent; +import android.media.soundtrigger_middleware.RecognitionEventSys; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; /** * Main interface for a client to get notifications of events coming from this module. @@ -31,7 +31,7 @@ oneway interface ISoundTriggerCallback { * In case of abortion, the caller may retry after the next onRecognitionAvailabilityChange() * callback. */ - void onRecognition(int modelHandle, in RecognitionEvent event, int captureSession); + void onRecognition(int modelHandle, in RecognitionEventSys event, int captureSession); /** * Invoked whenever a phrase recognition event is triggered (typically, on recognition, but * also in case of external aborting of a recognition or a forced recognition event - see the @@ -39,7 +39,7 @@ oneway interface ISoundTriggerCallback { * In case of abortion, the caller may retry after the next onRecognitionAvailabilityChange() * callback. */ - void onPhraseRecognition(int modelHandle, in PhraseRecognitionEvent event, int captureSession); + void onPhraseRecognition(int modelHandle, in PhraseRecognitionEventSys event, int captureSession); /** * Notifies the client that some start/load operations that have previously failed for resource * reasons (threw a ServiceSpecificException(RESOURCE_CONTENTION) or have been preempted) may diff --git a/media/aidl/android/media/soundtrigger_middleware/PhraseRecognitionEventSys.aidl b/media/aidl/android/media/soundtrigger_middleware/PhraseRecognitionEventSys.aidl new file mode 100644 index 000000000000..6c912ed7e056 --- /dev/null +++ b/media/aidl/android/media/soundtrigger_middleware/PhraseRecognitionEventSys.aidl @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger.PhraseRecognitionEvent; + +/** + * Wrapper to android.media.soundtrigger.RecognitionEvent providing additional fields used by the + * framework. + */ +parcelable PhraseRecognitionEventSys { + + PhraseRecognitionEvent phraseRecognitionEvent; + /** + * Timestamp of when the trigger event from SoundTriggerHal was received by the + * framework. + * + * <p>same units and timebase as {@link SystemClock#elapsedRealtime()}. + * The value will be -1 if the event was not generated from the HAL. + */ + // @ElapsedRealtimeLong + long halEventReceivedMillis = -1; +} diff --git a/media/aidl/android/media/soundtrigger_middleware/RecognitionEventSys.aidl b/media/aidl/android/media/soundtrigger_middleware/RecognitionEventSys.aidl new file mode 100644 index 000000000000..84e327d5df8c --- /dev/null +++ b/media/aidl/android/media/soundtrigger_middleware/RecognitionEventSys.aidl @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.media.soundtrigger_middleware; + +import android.media.soundtrigger.RecognitionEvent; + +/** + * Wrapper to android.media.soundtrigger.RecognitionEvent providing additional fields used by the + * framework. + */ +parcelable RecognitionEventSys { + + RecognitionEvent recognitionEvent; + /** + * Timestamp of when the trigger event from SoundTriggerHal was received by the + * framework. + * + * <p>same units and timebase as {@link SystemClock#elapsedRealtime()}. + * The value will be -1 if the event was not generated from the HAL. + */ + // @ElapsedRealtimeLong + long halEventReceivedMillis = -1; +} diff --git a/packages/CredentialManager/src/com/android/credentialmanager/DataConverter.kt b/packages/CredentialManager/src/com/android/credentialmanager/DataConverter.kt index 00c2f1a63ed4..a35310cdad95 100644 --- a/packages/CredentialManager/src/com/android/credentialmanager/DataConverter.kt +++ b/packages/CredentialManager/src/com/android/credentialmanager/DataConverter.kt @@ -482,7 +482,8 @@ class CreateFlowUtils { CredentialType.PASSWORD, appLabel, context.getDrawable(R.drawable.ic_password_24) ?: return null, - preferImmediatelyAvailableCredentials = false, + preferImmediatelyAvailableCredentials = + createCredentialRequestJetpack.preferImmediatelyAvailableCredentials, appPreferredDefaultProviderId = appPreferredDefaultProviderId, userSetDefaultProviderIds = requestInfo.defaultProviderIds.toSet(), ) @@ -509,7 +510,8 @@ class CreateFlowUtils { appName = appLabel, typeIcon = displayInfo.credentialTypeIcon?.loadDrawable(context) ?: context.getDrawable(R.drawable.ic_other_sign_in_24) ?: return null, - preferImmediatelyAvailableCredentials = false, + preferImmediatelyAvailableCredentials = + createCredentialRequestJetpack.preferImmediatelyAvailableCredentials, appPreferredDefaultProviderId = appPreferredDefaultProviderId, userSetDefaultProviderIds = requestInfo.defaultProviderIds.toSet(), ) diff --git a/packages/SettingsProvider/test/src/android/provider/SettingsBackupTest.java b/packages/SettingsProvider/test/src/android/provider/SettingsBackupTest.java index 706666cbebab..2e49dd5eeba9 100644 --- a/packages/SettingsProvider/test/src/android/provider/SettingsBackupTest.java +++ b/packages/SettingsProvider/test/src/android/provider/SettingsBackupTest.java @@ -841,7 +841,8 @@ public class SettingsBackupTest { Settings.Secure.ACCESSIBILITY_SHOW_WINDOW_MAGNIFICATION_PROMPT, Settings.Secure.ACCESSIBILITY_FLOATING_MENU_MIGRATION_TOOLTIP_PROMPT, Settings.Secure.UI_TRANSLATION_ENABLED, - Settings.Secure.CREDENTIAL_SERVICE); + Settings.Secure.CREDENTIAL_SERVICE, + Settings.Secure.CREDENTIAL_SERVICE_PRIMARY); @Test public void systemSettingsBackedUpOrDenied() { diff --git a/packages/SystemUI/src/com/android/keyguard/KeyguardPINView.java b/packages/SystemUI/src/com/android/keyguard/KeyguardPINView.java index 03947542d21e..ecce83139a43 100644 --- a/packages/SystemUI/src/com/android/keyguard/KeyguardPINView.java +++ b/packages/SystemUI/src/com/android/keyguard/KeyguardPINView.java @@ -16,6 +16,7 @@ package com.android.keyguard; +import static com.android.internal.jank.InteractionJankMonitor.CUJ_LOCKSCREEN_PIN_APPEAR; import static com.android.internal.jank.InteractionJankMonitor.CUJ_LOCKSCREEN_PIN_DISAPPEAR; import static com.android.systemui.statusbar.policy.DevicePostureController.DEVICE_POSTURE_HALF_OPENED; import static com.android.systemui.statusbar.policy.DevicePostureController.DEVICE_POSTURE_UNKNOWN; @@ -184,6 +185,7 @@ public class KeyguardPINView extends KeyguardPinBasedInputView { } mAppearAnimator.setDuration(ANIMATION_DURATION); mAppearAnimator.addUpdateListener(animation -> animate(animation.getAnimatedFraction())); + mAppearAnimator.addListener(getAnimationListener(CUJ_LOCKSCREEN_PIN_APPEAR)); mAppearAnimator.start(); } diff --git a/packages/SystemUI/src/com/android/systemui/flags/Flags.kt b/packages/SystemUI/src/com/android/systemui/flags/Flags.kt index 1fafa3d97c48..d1d53cc9dd58 100644 --- a/packages/SystemUI/src/com/android/systemui/flags/Flags.kt +++ b/packages/SystemUI/src/com/android/systemui/flags/Flags.kt @@ -702,6 +702,5 @@ object Flags { // TODO(b/278761837): Tracking Bug @JvmField - val USE_NEW_ACTIVITY_STARTER = unreleasedFlag(2801, name = "use_new_activity_starter", - teamfood = true) + val USE_NEW_ACTIVITY_STARTER = releasedFlag(2801, name = "use_new_activity_starter") } diff --git a/packages/SystemUI/tests/src/com/android/systemui/biometrics/AuthContainerViewTest.kt b/packages/SystemUI/tests/src/com/android/systemui/biometrics/AuthContainerViewTest.kt index 6e37ee791bd2..a361bbc69ef3 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/biometrics/AuthContainerViewTest.kt +++ b/packages/SystemUI/tests/src/com/android/systemui/biometrics/AuthContainerViewTest.kt @@ -68,7 +68,6 @@ import org.mockito.Mockito.verify import org.mockito.Mockito.`when` as whenever import org.mockito.junit.MockitoJUnit -@Ignore("b/279650412") @RunWith(AndroidJUnit4::class) @RunWithLooper(setAsMainLooper = true) @SmallTest @@ -281,6 +280,7 @@ class AuthContainerViewTest : SysuiTestCase() { assertThat(authContainer!!.parent).isNull() } + @Ignore("b/279650412") @Test fun testActionUseDeviceCredential_sendsOnDeviceCredentialPressed() { val container = initializeFingerprintContainer( diff --git a/services/core/java/com/android/server/display/color/CctEvaluator.java b/services/core/java/com/android/server/display/color/CctEvaluator.java new file mode 100644 index 000000000000..878f7e5116b8 --- /dev/null +++ b/services/core/java/com/android/server/display/color/CctEvaluator.java @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.display.color; + +import android.animation.TypeEvaluator; +import android.util.Slog; + +import com.android.internal.annotations.VisibleForTesting; + +import java.util.Arrays; + +/** + * Interpolates between CCT values by a given step. + */ +class CctEvaluator implements TypeEvaluator<Integer> { + + private static final String TAG = "CctEvaluator"; + + /** + * The minimum input value, which will represent index 0 in the mValues array. Each + * subsequent input value is offset by this amount. + */ + private final int mIndexOffset; + /** + * Cached step values at each CCT value (offset by the {@link #mIndexOffset} above). For + * example, if the minimum CCT is 2000K (which is set to mIndexOffset), then the 0th index of + * this array is equivalent to the step value at 2000K, 1st index corresponds to 2001K, and so + * on. + */ + @VisibleForTesting + final int[] mStepsAtOffsetCcts; + /** + * Pre-computed stepped CCTs. These will be accessed frequently; the memory cost of caching them + * is well-spent. + */ + @VisibleForTesting + final int[] mSteppedCctsAtOffsetCcts; + + CctEvaluator(int min, int max, int[] cctRangeMinimums, int[] steps) { + final int delta = max - min + 1; + mStepsAtOffsetCcts = new int[delta]; + mSteppedCctsAtOffsetCcts = new int[delta]; + mIndexOffset = min; + + final int parallelArraysLength = cctRangeMinimums.length; + if (cctRangeMinimums.length != steps.length) { + Slog.e(TAG, + "Parallel arrays cctRangeMinimums and steps are different lengths; setting " + + "step of 1"); + setStepOfOne(); + } else if (parallelArraysLength == 0) { + Slog.e(TAG, "No cctRangeMinimums or steps are set; setting step of 1"); + setStepOfOne(); + } else { + int parallelArraysIndex = 0; + int index = 0; + int lastSteppedCct = Integer.MIN_VALUE; + while (index < delta) { + final int cct = index + mIndexOffset; + int nextParallelArraysIndex = parallelArraysIndex + 1; + while (nextParallelArraysIndex < parallelArraysLength + && cct >= cctRangeMinimums[nextParallelArraysIndex]) { + parallelArraysIndex = nextParallelArraysIndex; + nextParallelArraysIndex++; + } + mStepsAtOffsetCcts[index] = steps[parallelArraysIndex]; + if (lastSteppedCct == Integer.MIN_VALUE + || Math.abs(lastSteppedCct - cct) >= steps[parallelArraysIndex]) { + lastSteppedCct = cct; + } + mSteppedCctsAtOffsetCcts[index] = lastSteppedCct; + index++; + } + } + } + + @Override + public Integer evaluate(float fraction, Integer startValue, Integer endValue) { + final int cct = (int) (startValue + fraction * (endValue - startValue)); + final int index = cct - mIndexOffset; + if (index < 0 || index >= mSteppedCctsAtOffsetCcts.length) { + Slog.e(TAG, "steppedCctValueAt: returning same since invalid requested index=" + index); + return cct; + } + return mSteppedCctsAtOffsetCcts[index]; + } + + private void setStepOfOne() { + Arrays.fill(mStepsAtOffsetCcts, 1); + for (int i = 0; i < mSteppedCctsAtOffsetCcts.length; i++) { + mSteppedCctsAtOffsetCcts[i] = mIndexOffset + i; + } + } +} diff --git a/services/core/java/com/android/server/display/color/ColorDisplayService.java b/services/core/java/com/android/server/display/color/ColorDisplayService.java index 0284d9c68689..c0ea5fead9a5 100644 --- a/services/core/java/com/android/server/display/color/ColorDisplayService.java +++ b/services/core/java/com/android/server/display/color/ColorDisplayService.java @@ -180,6 +180,8 @@ public final class ColorDisplayService extends SystemService { */ private SparseIntArray mColorModeCompositionColorSpaces = null; + private final Object mCctTintApplierLock = new Object(); + public ColorDisplayService(Context context) { super(context); mHandler = new TintHandler(DisplayThread.get().getLooper()); @@ -698,6 +700,79 @@ public final class ColorDisplayService extends SystemService { } } + private void applyTintByCct(ColorTemperatureTintController tintController, boolean immediate) { + synchronized (mCctTintApplierLock) { + tintController.cancelAnimator(); + + final DisplayTransformManager dtm = getLocalService(DisplayTransformManager.class); + final int from = tintController.getAppliedCct(); + final int to = tintController.isActivated() ? tintController.getTargetCct() + : tintController.getDisabledCct(); + + if (immediate) { + Slog.d(TAG, tintController.getClass().getSimpleName() + + " applied immediately: toCct=" + to + " fromCct=" + from); + dtm.setColorMatrix(tintController.getLevel(), + tintController.computeMatrixForCct(to)); + tintController.setAppliedCct(to); + } else { + Slog.d(TAG, tintController.getClass().getSimpleName() + " animation started: toCct=" + + to + " fromCct=" + from); + ValueAnimator valueAnimator = ValueAnimator.ofInt(from, to); + tintController.setAnimator(valueAnimator); + final CctEvaluator evaluator = tintController.getEvaluator(); + if (evaluator != null) { + valueAnimator.setEvaluator(evaluator); + } + valueAnimator.setDuration(tintController.getTransitionDurationMilliseconds()); + valueAnimator.setInterpolator(AnimationUtils.loadInterpolator( + getContext(), android.R.interpolator.linear)); + valueAnimator.addUpdateListener((ValueAnimator animator) -> { + synchronized (mCctTintApplierLock) { + final int value = (int) animator.getAnimatedValue(); + if (value != tintController.getAppliedCct()) { + dtm.setColorMatrix(tintController.getLevel(), + tintController.computeMatrixForCct(value)); + tintController.setAppliedCct(value); + } + } + }); + valueAnimator.addListener(new AnimatorListenerAdapter() { + + private boolean mIsCancelled; + + @Override + public void onAnimationCancel(Animator animator) { + Slog.d(TAG, tintController.getClass().getSimpleName() + + " animation cancelled"); + mIsCancelled = true; + } + + @Override + public void onAnimationEnd(Animator animator) { + synchronized (mCctTintApplierLock) { + Slog.d(TAG, tintController.getClass().getSimpleName() + + " animation ended: wasCancelled=" + mIsCancelled + + " toCct=" + to + + " fromCct=" + from); + if (!mIsCancelled) { + // Ensure final color matrix is set at the end of the animation. + // If the animation is cancelled then don't set the final color + // matrix so the new animator can pick up from where this one left + // off. + dtm.setColorMatrix(tintController.getLevel(), + tintController.computeMatrixForCct(to)); + tintController.setAppliedCct(to); + } + tintController.setAnimator(null); + } + } + }); + valueAnimator.start(); + } + } + } + /** * Returns the first date time corresponding to the local time that occurs before the provided * date time. @@ -747,7 +822,7 @@ public final class ColorDisplayService extends SystemService { // If disabled, clear the tint. If enabled, do nothing more here and let the next // temperature update set the correct tint. - if (!activated) { + if (oldActivated && !activated) { mHandler.sendEmptyMessage(MSG_APPLY_DISPLAY_WHITE_BALANCE); } } @@ -1452,7 +1527,7 @@ public final class ColorDisplayService extends SystemService { public class ColorDisplayServiceInternal { /** Sets whether DWB should be allowed in the current state. */ - public void setDisplayWhiteBalanceAllowed(boolean allowed) { + public void setDisplayWhiteBalanceAllowed(boolean allowed) { mDisplayWhiteBalanceTintController.setAllowed(allowed); updateDisplayWhiteBalanceStatus(); } @@ -1464,8 +1539,8 @@ public final class ColorDisplayService extends SystemService { * @param cct the color temperature in Kelvin. */ public boolean setDisplayWhiteBalanceColorTemperature(int cct) { - // Update the transform matrix even if it can't be applied. - mDisplayWhiteBalanceTintController.setMatrix(cct); + // Update the transform target CCT even if it can't be applied. + mDisplayWhiteBalanceTintController.setTargetCct(cct); if (mDisplayWhiteBalanceTintController.isActivated()) { mHandler.sendEmptyMessage(MSG_APPLY_DISPLAY_WHITE_BALANCE); @@ -1601,7 +1676,7 @@ public final class ColorDisplayService extends SystemService { applyTint(mNightDisplayTintController, false); break; case MSG_APPLY_DISPLAY_WHITE_BALANCE: - applyTint(mDisplayWhiteBalanceTintController, false); + applyTintByCct(mDisplayWhiteBalanceTintController, false); break; } } diff --git a/services/core/java/com/android/server/display/color/ColorTemperatureTintController.java b/services/core/java/com/android/server/display/color/ColorTemperatureTintController.java new file mode 100644 index 000000000000..0fbd9d409986 --- /dev/null +++ b/services/core/java/com/android/server/display/color/ColorTemperatureTintController.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.display.color; + +abstract class ColorTemperatureTintController extends TintController { + + abstract int getAppliedCct(); + + abstract void setAppliedCct(int cct); + + abstract int getTargetCct(); + + abstract void setTargetCct(int cct); + + /** + * Returns the CCT value most closely associated with the "disabled" (identity) matrix for + * this device, to use as the target when deactivating this transform. + */ + abstract int getDisabledCct(); + + abstract float[] computeMatrixForCct(int cct); + + abstract CctEvaluator getEvaluator(); +} diff --git a/services/core/java/com/android/server/display/color/DisplayWhiteBalanceTintController.java b/services/core/java/com/android/server/display/color/DisplayWhiteBalanceTintController.java index 106ac0c0d478..bf0139f347f8 100644 --- a/services/core/java/com/android/server/display/color/DisplayWhiteBalanceTintController.java +++ b/services/core/java/com/android/server/display/color/DisplayWhiteBalanceTintController.java @@ -21,6 +21,7 @@ import static android.view.Display.DEFAULT_DISPLAY; import static com.android.server.display.color.DisplayTransformManager.LEVEL_COLOR_MATRIX_DISPLAY_WHITE_BALANCE; import android.annotation.NonNull; +import android.annotation.Nullable; import android.annotation.Size; import android.content.Context; import android.content.res.Resources; @@ -36,7 +37,7 @@ import com.android.internal.annotations.VisibleForTesting; import java.io.PrintWriter; -final class DisplayWhiteBalanceTintController extends TintController { +final class DisplayWhiteBalanceTintController extends ColorTemperatureTintController { // Three chromaticity coordinates per color: X, Y, and Z private static final int NUM_VALUES_PER_PRIMARY = 3; @@ -52,9 +53,11 @@ final class DisplayWhiteBalanceTintController extends TintController { private int mTemperatureDefault; @VisibleForTesting float[] mDisplayNominalWhiteXYZ = new float[NUM_VALUES_PER_PRIMARY]; + private int mDisplayNominalWhiteCct; @VisibleForTesting ColorSpace.Rgb mDisplayColorSpaceRGB; private float[] mChromaticAdaptationMatrix; + // The temperature currently represented in the matrix. @VisibleForTesting int mCurrentColorTemperature; private float[] mCurrentColorTemperatureXYZ; @@ -65,6 +68,9 @@ final class DisplayWhiteBalanceTintController extends TintController { private Boolean mIsAvailable; // This feature becomes disallowed if the device is in an unsupported strong/light state. private boolean mIsAllowed = true; + private int mTargetCct; + private int mAppliedCct; + private CctEvaluator mCctEvaluator; private final DisplayManagerInternal mDisplayManagerInternal; @@ -108,6 +114,9 @@ final class DisplayWhiteBalanceTintController extends TintController { displayNominalWhiteXYZ[i] = Float.parseFloat(nominalWhiteValues[i]); } + final int displayNominalWhiteCct = res.getInteger( + R.integer.config_displayWhiteBalanceDisplayNominalWhiteCct); + final int colorTemperatureMin = res.getInteger( R.integer.config_displayWhiteBalanceColorTemperatureMin); if (colorTemperatureMin <= 0) { @@ -124,19 +133,28 @@ final class DisplayWhiteBalanceTintController extends TintController { return; } - final int colorTemperature = res.getInteger( + final int defaultTemperature = res.getInteger( R.integer.config_displayWhiteBalanceColorTemperatureDefault); mTransitionDuration = res.getInteger( R.integer.config_displayWhiteBalanceTransitionTime); + int[] cctRangeMinimums = res.getIntArray( + R.array.config_displayWhiteBalanceDisplayRangeMinimums); + int[] steps = res.getIntArray(R.array.config_displayWhiteBalanceDisplaySteps); + synchronized (mLock) { mDisplayColorSpaceRGB = displayColorSpaceRGB; mDisplayNominalWhiteXYZ = displayNominalWhiteXYZ; + mDisplayNominalWhiteCct = displayNominalWhiteCct; + mTargetCct = mDisplayNominalWhiteCct; + mAppliedCct = mDisplayNominalWhiteCct; mTemperatureMin = colorTemperatureMin; mTemperatureMax = colorTemperatureMax; - mTemperatureDefault = colorTemperature; + mTemperatureDefault = defaultTemperature; mSetUp = true; + mCctEvaluator = new CctEvaluator(mTemperatureMin, mTemperatureMax, + cctRangeMinimums, steps); } setMatrix(mTemperatureDefault); @@ -144,8 +162,16 @@ final class DisplayWhiteBalanceTintController extends TintController { @Override public float[] getMatrix() { - return mSetUp && isActivated() ? mMatrixDisplayWhiteBalance - : ColorDisplayService.MATRIX_IDENTITY; + if (!mSetUp || !isActivated()) { + return ColorDisplayService.MATRIX_IDENTITY; + } + computeMatrixForCct(mAppliedCct); + return mMatrixDisplayWhiteBalance; + } + + @Override + public int getTargetCct() { + return mTargetCct; } /** @@ -174,6 +200,12 @@ final class DisplayWhiteBalanceTintController extends TintController { @Override public void setMatrix(int cct) { + setTargetCct(cct); + computeMatrixForCct(mTargetCct); + } + + @Override + public void setTargetCct(int cct) { if (!mSetUp) { Slog.w(ColorDisplayService.TAG, "Can't set display white balance temperature: uninitialized"); @@ -183,50 +215,93 @@ final class DisplayWhiteBalanceTintController extends TintController { if (cct < mTemperatureMin) { Slog.w(ColorDisplayService.TAG, "Requested display color temperature is below allowed minimum"); - cct = mTemperatureMin; + mTargetCct = mTemperatureMin; } else if (cct > mTemperatureMax) { Slog.w(ColorDisplayService.TAG, "Requested display color temperature is above allowed maximum"); - cct = mTemperatureMax; + mTargetCct = mTemperatureMax; + } else { + mTargetCct = cct; + } + } + + @Override + public int getDisabledCct() { + return mDisplayNominalWhiteCct; + } + + @Override + public float[] computeMatrixForCct(int cct) { + if (!mSetUp || cct == 0) { + Slog.w(ColorDisplayService.TAG, "Couldn't compute matrix for cct=" + cct); + return ColorDisplayService.MATRIX_IDENTITY; } synchronized (mLock) { mCurrentColorTemperature = cct; - // Adapt the display's nominal white point to match the requested CCT value - mCurrentColorTemperatureXYZ = ColorSpace.cctToXyz(cct); - - mChromaticAdaptationMatrix = - ColorSpace.chromaticAdaptation(ColorSpace.Adaptation.BRADFORD, - mDisplayNominalWhiteXYZ, mCurrentColorTemperatureXYZ); - - // Convert the adaptation matrix to RGB space - float[] result = mul3x3(mChromaticAdaptationMatrix, - mDisplayColorSpaceRGB.getTransform()); - result = mul3x3(mDisplayColorSpaceRGB.getInverseTransform(), result); - - // Normalize the transform matrix to peak white value in RGB space - final float adaptedMaxR = result[0] + result[3] + result[6]; - final float adaptedMaxG = result[1] + result[4] + result[7]; - final float adaptedMaxB = result[2] + result[5] + result[8]; - final float denum = Math.max(Math.max(adaptedMaxR, adaptedMaxG), adaptedMaxB); - - Matrix.setIdentityM(mMatrixDisplayWhiteBalance, 0); - for (int i = 0; i < result.length; i++) { - result[i] /= denum; - if (!isColorMatrixCoeffValid(result[i])) { - Slog.e(ColorDisplayService.TAG, "Invalid DWB color matrix"); - return; - } + if (cct == mDisplayNominalWhiteCct && !isActivated()) { + // DWB is finished turning off. Clear the matrix. + Matrix.setIdentityM(mMatrixDisplayWhiteBalance, 0); + } else { + computeMatrixForCctLocked(cct); } - java.lang.System.arraycopy(result, 0, mMatrixDisplayWhiteBalance, 0, 3); - java.lang.System.arraycopy(result, 3, mMatrixDisplayWhiteBalance, 4, 3); - java.lang.System.arraycopy(result, 6, mMatrixDisplayWhiteBalance, 8, 3); + Slog.d(ColorDisplayService.TAG, "computeDisplayWhiteBalanceMatrix: cct =" + cct + + " matrix =" + matrixToString(mMatrixDisplayWhiteBalance, 16)); + + return mMatrixDisplayWhiteBalance; } + } + + private void computeMatrixForCctLocked(int cct) { + // Adapt the display's nominal white point to match the requested CCT value + mCurrentColorTemperatureXYZ = ColorSpace.cctToXyz(cct); + + mChromaticAdaptationMatrix = + ColorSpace.chromaticAdaptation(ColorSpace.Adaptation.BRADFORD, + mDisplayNominalWhiteXYZ, mCurrentColorTemperatureXYZ); - Slog.d(ColorDisplayService.TAG, "setDisplayWhiteBalanceTemperatureMatrix: cct = " + cct - + " matrix = " + matrixToString(mMatrixDisplayWhiteBalance, 16)); + // Convert the adaptation matrix to RGB space + float[] result = mul3x3(mChromaticAdaptationMatrix, + mDisplayColorSpaceRGB.getTransform()); + result = mul3x3(mDisplayColorSpaceRGB.getInverseTransform(), result); + + // Normalize the transform matrix to peak white value in RGB space + final float adaptedMaxR = result[0] + result[3] + result[6]; + final float adaptedMaxG = result[1] + result[4] + result[7]; + final float adaptedMaxB = result[2] + result[5] + result[8]; + final float denum = Math.max(Math.max(adaptedMaxR, adaptedMaxG), adaptedMaxB); + + Matrix.setIdentityM(mMatrixDisplayWhiteBalance, 0); + + for (int i = 0; i < result.length; i++) { + result[i] /= denum; + if (!isColorMatrixCoeffValid(result[i])) { + Slog.e(ColorDisplayService.TAG, "Invalid DWB color matrix"); + return; + } + } + + java.lang.System.arraycopy(result, 0, mMatrixDisplayWhiteBalance, 0, 3); + java.lang.System.arraycopy(result, 3, mMatrixDisplayWhiteBalance, 4, 3); + java.lang.System.arraycopy(result, 6, mMatrixDisplayWhiteBalance, 8, 3); + } + + @Override + int getAppliedCct() { + return mAppliedCct; + } + + @Override + void setAppliedCct(int cct) { + mAppliedCct = cct; + } + + @Override + @Nullable + CctEvaluator getEvaluator() { + return mCctEvaluator; } @Override @@ -258,7 +333,10 @@ final class DisplayWhiteBalanceTintController extends TintController { pw.println(" mTemperatureMin = " + mTemperatureMin); pw.println(" mTemperatureMax = " + mTemperatureMax); pw.println(" mTemperatureDefault = " + mTemperatureDefault); + pw.println(" mDisplayNominalWhiteCct = " + mDisplayNominalWhiteCct); pw.println(" mCurrentColorTemperature = " + mCurrentColorTemperature); + pw.println(" mTargetCct = " + mTargetCct); + pw.println(" mAppliedCct = " + mAppliedCct); pw.println(" mCurrentColorTemperatureXYZ = " + matrixToString(mCurrentColorTemperatureXYZ, 3)); pw.println(" mDisplayColorSpaceRGB RGB-to-XYZ = " @@ -340,11 +418,7 @@ final class DisplayWhiteBalanceTintController extends TintController { } private boolean isColorMatrixCoeffValid(float coeff) { - if (Float.isNaN(coeff) || Float.isInfinite(coeff)) { - return false; - } - - return true; + return !Float.isNaN(coeff) && !Float.isInfinite(coeff); } private boolean isColorMatrixValid(float[] matrix) { @@ -352,8 +426,8 @@ final class DisplayWhiteBalanceTintController extends TintController { return false; } - for (int i = 0; i < matrix.length; i++) { - if (!isColorMatrixCoeffValid(matrix[i])) { + for (float value : matrix) { + if (!isColorMatrixCoeffValid(value)) { return false; } } diff --git a/services/core/java/com/android/server/display/color/TintController.java b/services/core/java/com/android/server/display/color/TintController.java index c53ac06c66e1..384333a574b5 100644 --- a/services/core/java/com/android/server/display/color/TintController.java +++ b/services/core/java/com/android/server/display/color/TintController.java @@ -16,6 +16,7 @@ package com.android.server.display.color; +import android.animation.ValueAnimator; import android.content.Context; import android.util.Slog; @@ -28,14 +29,14 @@ abstract class TintController { */ private static final long TRANSITION_DURATION = 3000L; - private ColorDisplayService.TintValueAnimator mAnimator; + private ValueAnimator mAnimator; private Boolean mIsActivated; - public ColorDisplayService.TintValueAnimator getAnimator() { + public ValueAnimator getAnimator() { return mAnimator; } - public void setAnimator(ColorDisplayService.TintValueAnimator animator) { + public void setAnimator(ValueAnimator animator) { mAnimator = animator; } diff --git a/services/tests/mockingservicestests/src/com/android/server/display/color/DisplayWhiteBalanceTintControllerTest.java b/services/tests/mockingservicestests/src/com/android/server/display/color/DisplayWhiteBalanceTintControllerTest.java index 941a3a419d59..3faf394fc534 100644 --- a/services/tests/mockingservicestests/src/com/android/server/display/color/DisplayWhiteBalanceTintControllerTest.java +++ b/services/tests/mockingservicestests/src/com/android/server/display/color/DisplayWhiteBalanceTintControllerTest.java @@ -72,18 +72,28 @@ public class DisplayWhiteBalanceTintControllerTest { mResources = InstrumentationRegistry.getContext().getResources(); // These Resources are common to all tests. - doReturn(mResources.getInteger(R.integer.config_displayWhiteBalanceColorTemperatureMin)) + doReturn(4000) .when(mMockedResources) .getInteger(R.integer.config_displayWhiteBalanceColorTemperatureMin); - doReturn(mResources.getInteger(R.integer.config_displayWhiteBalanceColorTemperatureMax)) + doReturn(8000) .when(mMockedResources) .getInteger(R.integer.config_displayWhiteBalanceColorTemperatureMax); - doReturn(mResources.getInteger(R.integer.config_displayWhiteBalanceColorTemperatureDefault)) + doReturn(6500) .when(mMockedResources) .getInteger(R.integer.config_displayWhiteBalanceColorTemperatureDefault); - doReturn(mResources.getStringArray(R.array.config_displayWhiteBalanceDisplayNominalWhite)) - .when(mMockedResources) - .getStringArray(R.array.config_displayWhiteBalanceDisplayNominalWhite); + doReturn(new String[] {"0.950456", "1.000000", "1.089058"}) + .when(mMockedResources) + .getStringArray(R.array.config_displayWhiteBalanceDisplayNominalWhite); + doReturn(6500) + .when(mMockedResources) + .getInteger(R.integer.config_displayWhiteBalanceDisplayNominalWhiteCct); + doReturn(new int[] {0}) + .when(mMockedResources) + .getIntArray(R.array.config_displayWhiteBalanceDisplaySteps); + doReturn(new int[] {20}) + .when(mMockedResources) + .getIntArray(R.array.config_displayWhiteBalanceDisplayRangeMinimums); + doReturn(mMockedResources).when(mMockedContext).getResources(); mDisplayToken = new Binder(); @@ -195,7 +205,7 @@ public class DisplayWhiteBalanceTintControllerTest { * Matrix should match the precalculated one for given cct and display primaries. */ @Test - public void displayWhiteBalance_validateTransformMatrix() { + public void displayWhiteBalance_getAndSetMatrix_validateTransformMatrix() { DisplayPrimaries displayPrimaries = new DisplayPrimaries(); displayPrimaries.red = new CieXyz(); displayPrimaries.red.X = 0.412315f; @@ -223,10 +233,12 @@ public class DisplayWhiteBalanceTintControllerTest { final int cct = 6500; mDisplayWhiteBalanceTintController.setMatrix(cct); + mDisplayWhiteBalanceTintController.setAppliedCct( + mDisplayWhiteBalanceTintController.getTargetCct()); + assertWithMessage("Failed to set temperature") .that(mDisplayWhiteBalanceTintController.mCurrentColorTemperature) .isEqualTo(cct); - float[] matrixDwb = mDisplayWhiteBalanceTintController.getMatrix(); final float[] expectedMatrixDwb = { 0.971848f, -0.001421f, 0.000491f, 0.0f, @@ -238,6 +250,54 @@ public class DisplayWhiteBalanceTintControllerTest { 1e-6f /* tolerance */); } + /** + * Matrix should match the precalculated one for given cct and display primaries. + */ + @Test + public void displayWhiteBalance_targetApplied_validateTransformMatrix() { + DisplayPrimaries displayPrimaries = new DisplayPrimaries(); + displayPrimaries.red = new CieXyz(); + displayPrimaries.red.X = 0.412315f; + displayPrimaries.red.Y = 0.212600f; + displayPrimaries.red.Z = 0.019327f; + displayPrimaries.green = new CieXyz(); + displayPrimaries.green.X = 0.357600f; + displayPrimaries.green.Y = 0.715200f; + displayPrimaries.green.Z = 0.119200f; + displayPrimaries.blue = new CieXyz(); + displayPrimaries.blue.X = 0.180500f; + displayPrimaries.blue.Y = 0.072200f; + displayPrimaries.blue.Z = 0.950633f; + displayPrimaries.white = new CieXyz(); + displayPrimaries.white.X = 0.950456f; + displayPrimaries.white.Y = 1.000000f; + displayPrimaries.white.Z = 1.089058f; + when(mDisplayManagerInternal.getDisplayNativePrimaries(DEFAULT_DISPLAY)) + .thenReturn(displayPrimaries); + + setUpTintController(); + assertWithMessage("Setup with valid SurfaceControl failed") + .that(mDisplayWhiteBalanceTintController.mSetUp) + .isTrue(); + + final int cct = 6500; + mDisplayWhiteBalanceTintController.setTargetCct(cct); + final float[] matrixDwb = mDisplayWhiteBalanceTintController.computeMatrixForCct(cct); + mDisplayWhiteBalanceTintController.setAppliedCct(cct); + + assertWithMessage("Failed to set temperature") + .that(mDisplayWhiteBalanceTintController.mCurrentColorTemperature) + .isEqualTo(cct); + final float[] expectedMatrixDwb = { + 0.971848f, -0.001421f, 0.000491f, 0.0f, + 0.028193f, 0.945798f, 0.003207f, 0.0f, + -0.000042f, -0.000989f, 0.988659f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + }; + assertArrayEquals("Unexpected DWB matrix", expectedMatrixDwb, matrixDwb, + 1e-6f /* tolerance */); + } + private void setUpTintController() { mDisplayWhiteBalanceTintController = new DisplayWhiteBalanceTintController( mDisplayManagerInternal); diff --git a/services/tests/servicestests/src/com/android/server/display/color/CctEvaluatorTest.java b/services/tests/servicestests/src/com/android/server/display/color/CctEvaluatorTest.java new file mode 100644 index 000000000000..b96666ae40a3 --- /dev/null +++ b/services/tests/servicestests/src/com/android/server/display/color/CctEvaluatorTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.display.color; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public class CctEvaluatorTest { + + @Test + public void noEntriesInParallelArrays_setsEverythingToOne() { + final CctEvaluator evaluator = new CctEvaluator(0, 5, new int[]{}, new int[]{}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo(new int[]{1, 1, 1, 1, 1, 1}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{0, 1, 2, 3, 4, 5}); + } + + @Test + public void unevenNumberOfEntriesInParallelArrays_setsEverythingToOne() { + final CctEvaluator evaluator = new CctEvaluator(0, 5, new int[]{0}, new int[]{}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo(new int[]{1, 1, 1, 1, 1, 1}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{0, 1, 2, 3, 4, 5}); + } + + @Test + public void singleEntryInParallelArray_computesCorrectly() { + final CctEvaluator evaluator = new CctEvaluator(0, 5, new int[]{0}, new int[]{2}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo(new int[]{2, 2, 2, 2, 2, 2}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{0, 0, 2, 2, 4, 4}); + } + + @Test + public void minimumIsBelowFirstRange_computesCorrectly() { + final CctEvaluator evaluator = new CctEvaluator(3000, 3005, new int[]{3002}, + new int[]{20}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo(new int[]{20, 20, 20, 20, 20, 20}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{3000, 3000, 3000, 3000, 3000, 3000}); + } + + @Test + public void minimumIsAboveFirstRange_computesCorrectly() { + final CctEvaluator evaluator = new CctEvaluator(3000, 3008, new int[]{3002}, + new int[]{20}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo( + new int[]{20, 20, 20, 20, 20, 20, 20, 20, 20}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000}); + } + + @Test + public void multipleStepsStartsAtThreshold_computesCorrectly() { + final CctEvaluator evaluator = new CctEvaluator(5, 20, new int[]{0, 4, 5, 10, 18}, + new int[]{11, 7, 2, 15, 9}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo( + new int[]{2, 2, 2, 2, 2, 15, 15, 15, 15, 15, 15, 15, 15, 9, 9, 9}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{5, 5, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 18, 18, 18}); + } + + @Test + public void multipleStepsStartsInBetween_computesCorrectly() { + final CctEvaluator evaluator = new CctEvaluator(4, 20, new int[]{0, 5, 10, 18}, + new int[]{14, 2, 15, 9}); + assertThat(evaluator.mStepsAtOffsetCcts).isEqualTo( + new int[]{14, 2, 2, 2, 2, 2, 15, 15, 15, 15, 15, 15, 15, 15, 9, 9, 9}); + assertThat(evaluator.mSteppedCctsAtOffsetCcts).isEqualTo( + new int[]{4, 4, 6, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 18, 18, 18}); + } +} diff --git a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java index 2d0755d00ba8..3ac9a2758e08 100644 --- a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java +++ b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java @@ -16,6 +16,8 @@ package com.android.server.soundtrigger_middleware; +import static com.google.common.truth.Truth.assertThat; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -34,12 +36,12 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.Status; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.HwParcel; import android.os.IBinder; import android.os.IHwBinder; @@ -617,13 +619,16 @@ public class SoundHw2CompatTest { final int handle = 85; final int status = android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.ABORT; - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); hwCallback.recognitionCallback(TestUtil.createRecognitionEvent_2_0(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED, + RecognitionEventSys lastEvent = eventCaptor.getValue(); + assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0); + TestUtil.validateRecognitionEvent(lastEvent.recognitionEvent, + RecognitionStatus.ABORTED, false); } @@ -631,14 +636,16 @@ public class SoundHw2CompatTest { final int handle = 92; final int status = android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.SUCCESS; - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); hwCallback.phraseRecognitionCallback( TestUtil.createPhraseRecognitionEvent_2_0(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(), + PhraseRecognitionEventSys lastEvent = eventCaptor.getValue(); + assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0); + TestUtil.validatePhraseRecognitionEvent(lastEvent.phraseRecognitionEvent, RecognitionStatus.SUCCESS, false); } verifyNoMoreInteractions(canonicalCallback); @@ -652,28 +659,34 @@ public class SoundHw2CompatTest { final int handle = 85; final int status = android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.ABORT; - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); hwCallback.recognitionCallback_2_1(TestUtil.createRecognitionEvent_2_1(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED, + RecognitionEventSys lastEvent = eventCaptor.getValue(); + assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0); + TestUtil.validateRecognitionEvent(lastEvent.recognitionEvent, + RecognitionStatus.ABORTED, false); } { final int handle = 87; final int status = 3; // FORCED; - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); hwCallback.recognitionCallback_2_1(TestUtil.createRecognitionEvent_2_1(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.FORCED, + RecognitionEventSys lastEvent = eventCaptor.getValue(); + assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0); + TestUtil.validateRecognitionEvent(lastEvent.recognitionEvent, + RecognitionStatus.FORCED, true); } @@ -681,28 +694,32 @@ public class SoundHw2CompatTest { final int handle = 92; final int status = android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.SUCCESS; - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); hwCallback.phraseRecognitionCallback_2_1( TestUtil.createPhraseRecognitionEvent_2_1(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(), + PhraseRecognitionEventSys lastEvent = eventCaptor.getValue(); + assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0); + TestUtil.validatePhraseRecognitionEvent(lastEvent.phraseRecognitionEvent, RecognitionStatus.SUCCESS, false); } { final int handle = 102; final int status = 3; // FORCED; - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); hwCallback.phraseRecognitionCallback_2_1( TestUtil.createPhraseRecognitionEvent_2_1(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(), + PhraseRecognitionEventSys lastEvent = eventCaptor.getValue(); + assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0); + TestUtil.validatePhraseRecognitionEvent(lastEvent.phraseRecognitionEvent, RecognitionStatus.FORCED, true); } verifyNoMoreInteractions(canonicalCallback); diff --git a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandlerTest.java b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandlerTest.java index 61989252d04d..9a59ede20e23 100644 --- a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandlerTest.java +++ b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandlerTest.java @@ -30,8 +30,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; +import android.media.soundtrigger_middleware.RecognitionEventSys; import androidx.annotation.NonNull; @@ -68,13 +68,14 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { mNotifier.setActive(true); verify(mUnderlying).stopRecognition(handle); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); Thread.sleep(50); verify(callback).recognitionCallback(eq(handle), eventCaptor.capture()); - RecognitionEvent event = eventCaptor.getValue(); - assertEquals(event.status, RecognitionStatus.ABORTED); - assertFalse(event.recognitionStillActive); + RecognitionEventSys event = eventCaptor.getValue(); + assertEquals(event.halEventReceivedMillis, -1); + assertEquals(event.recognitionEvent.status, RecognitionStatus.ABORTED); + assertFalse(event.recognitionEvent.recognitionStillActive); verifyZeroInteractions(mGlobalCallback); clearInvocations(callback, mUnderlying); @@ -116,8 +117,11 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { mNotifier.setActive(true); verify(mUnderlying, times(1)).stopRecognition(handle); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); mHandler.stopRecognition(handle); - verify(callback, times(1)).recognitionCallback(eq(handle), any()); + verify(callback).recognitionCallback(eq(handle), eventCaptor.capture()); + assertEquals(eventCaptor.getValue().halEventReceivedMillis, -1); } @Test(timeout = 200) @@ -133,19 +137,21 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { verify(mUnderlying).startRecognition(eq(handle), eq(101), eq(102), any()); doAnswer(invocation -> { - RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.ABORTED, + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent( + RecognitionStatus.ABORTED, false); + recognitionEventSys.halEventReceivedMillis = 12345; // Call the callback from a different thread to detect deadlocks by preventing recursive // locking from working. - runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event)); + runOnSeparateThread( + () -> modelCallback.recognitionCallback(handle, recognitionEventSys)); return null; }).when(mUnderlying).stopRecognition(handle); mHandler.stopRecognition(handle); verify(mUnderlying, times(1)).stopRecognition(handle); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); - verify(callback, atMost(1)).recognitionCallback(eq(handle), eventCaptor.capture()); + verify(callback, atMost(1)).recognitionCallback(eq(handle), any(RecognitionEventSys.class)); } @Test(timeout = 200) @@ -162,11 +168,15 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { doAnswer(invocation -> { // The stop request causes a callback to be flushed. - RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.FORCED, + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent( + RecognitionStatus.FORCED, true); + recognitionEventSys.halEventReceivedMillis = 12345; // Call the callback from a different thread to detect deadlocks by preventing recursive // locking from working. - runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event)); + runOnSeparateThread( + () -> modelCallback.recognitionCallback(handle, recognitionEventSys)); // While the HAL is processing the stop request, capture state becomes active. new Thread(() -> mNotifier.setActive(true)).start(); Thread.sleep(50); @@ -194,11 +204,15 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { doAnswer(invocation -> { // The stop request causes a callback to be flushed. - RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.FORCED, + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent( + RecognitionStatus.FORCED, true); + recognitionEventSys.halEventReceivedMillis = 12345; // Call the callback from a different thread to detect deadlocks by preventing recursive // locking from working. - runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event)); + runOnSeparateThread( + () -> modelCallback.recognitionCallback(handle, recognitionEventSys)); // While the HAL is processing the stop request, client requests stop. new Thread(() -> mHandler.stopRecognition(handle)).start(); Thread.sleep(50); @@ -223,23 +237,22 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { verify(mUnderlying).startRecognition(eq(handle), eq(101), eq(102), any()); doAnswer(invocation -> { - RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.SUCCESS, + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent( + RecognitionStatus.SUCCESS, false); + recognitionEventSys.halEventReceivedMillis = 12345; // Call the callback from a different thread to detect deadlocks by preventing recursive // locking from working. - runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event)); + runOnSeparateThread( + () -> modelCallback.recognitionCallback(handle, recognitionEventSys)); return null; }).when(mUnderlying).stopRecognition(handle); mNotifier.setActive(true); verify(mUnderlying, times(1)).stopRecognition(handle); Thread.sleep(50); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); - verify(callback, atMost(2)).recognitionCallback(eq(handle), eventCaptor.capture()); - RecognitionEvent lastEvent = eventCaptor.getValue(); - assertEquals(lastEvent.status, RecognitionStatus.ABORTED); - assertFalse(lastEvent.recognitionStillActive); + verify(callback, atMost(2)).recognitionCallback(eq(handle), any()); } @@ -256,11 +269,15 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { verify(mUnderlying).startRecognition(eq(handle), eq(101), eq(102), any()); doAnswer(invocation -> { - RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.FORCED, + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent( + RecognitionStatus.FORCED, true); + recognitionEventSys.halEventReceivedMillis = 12345; // Call the callback from a different thread to detect deadlocks by preventing recursive // locking from working. - runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event)); + runOnSeparateThread( + () -> modelCallback.recognitionCallback(handle, recognitionEventSys)); return null; }).when(mUnderlying).stopRecognition(handle); @@ -268,12 +285,7 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest { verify(mUnderlying, times(1)).stopRecognition(handle); Thread.sleep(50); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); - verify(callback, atMost(2)).recognitionCallback(eq(handle), eventCaptor.capture()); - RecognitionEvent lastEvent = eventCaptor.getValue(); - assertEquals(lastEvent.status, RecognitionStatus.ABORTED); - assertFalse(lastEvent.recognitionStillActive); + verify(callback, atMost(2)).recognitionCallback(eq(handle), any()); } private static void runOnSeparateThread(Runnable runnable) { diff --git a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java index 3bebc94fe0ed..5a2451f3b17e 100644 --- a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java +++ b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java @@ -30,18 +30,19 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import android.annotation.ElapsedRealtimeLong; import android.media.soundtrigger.ModelParameter; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; import android.os.RemoteException; import android.util.Pair; @@ -224,10 +225,12 @@ public class SoundTriggerMiddlewareImplTest { // Stop the recognition. stopRecognition(module, handle, hwHandle); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); - assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().status); + RecognitionEventSys lastEvent = eventCaptor.getValue(); + assertEquals(-1, lastEvent.halEventReceivedMillis); + assertEquals(RecognitionStatus.ABORTED, lastEvent.recognitionEvent.status); // Unload the model. unloadModel(module, handle, hwHandle); @@ -273,10 +276,12 @@ public class SoundTriggerMiddlewareImplTest { // Stop the recognition. stopRecognition(module, handle, hwHandle); - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101)); - assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().common.status); + PhraseRecognitionEventSys lastEvent = eventCaptor.getValue(); + assertEquals(-1, lastEvent.halEventReceivedMillis); + assertEquals(RecognitionStatus.ABORTED, lastEvent.phraseRecognitionEvent.common.status); // Unload the model. unloadModel(module, handle, hwHandle); @@ -299,11 +304,11 @@ public class SoundTriggerMiddlewareImplTest { { // Signal a capture from the driver (with "still active"). - RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, - RecognitionStatus.SUCCESS, true); + RecognitionEventSys event = hwCallback.sendRecognitionEvent(hwHandle, + RecognitionStatus.SUCCESS, true, 12345); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. @@ -312,11 +317,11 @@ public class SoundTriggerMiddlewareImplTest { { // Signal a capture from the driver (without "still active"). - RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, - RecognitionStatus.SUCCESS, false); + RecognitionEventSys event = hwCallback.sendRecognitionEvent(hwHandle, + RecognitionStatus.SUCCESS, false, 12345); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); verify(callback, times(2)).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. @@ -343,11 +348,11 @@ public class SoundTriggerMiddlewareImplTest { startRecognition(module, handle, hwHandle); // Signal a capture from the driver. - PhraseRecognitionEvent event = hwCallback.sendPhraseRecognitionEvent(hwHandle, - RecognitionStatus.SUCCESS, false); + PhraseRecognitionEventSys event = hwCallback.sendPhraseRecognitionEvent(hwHandle, + RecognitionStatus.SUCCESS, false, 12345); - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. @@ -377,11 +382,11 @@ public class SoundTriggerMiddlewareImplTest { verify(mHalDriver).forceRecognitionEvent(hwHandle); // Signal a capture from the driver. - RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, - RecognitionStatus.FORCED, true); + RecognitionEventSys event = hwCallback.sendRecognitionEvent(hwHandle, + RecognitionStatus.FORCED, true, 12345); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. @@ -445,11 +450,11 @@ public class SoundTriggerMiddlewareImplTest { verify(mHalDriver).forceRecognitionEvent(hwHandle); // Signal a capture from the driver. - PhraseRecognitionEvent event = hwCallback.sendPhraseRecognitionEvent(hwHandle, - RecognitionStatus.FORCED, true); + PhraseRecognitionEventSys event = hwCallback.sendPhraseRecognitionEvent(hwHandle, + RecognitionStatus.FORCED, true, 12345); - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. @@ -510,14 +515,16 @@ public class SoundTriggerMiddlewareImplTest { startRecognition(module, handle, hwHandle); // Abort. - hwCallback.sendRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false); + hwCallback.sendRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false, 12345); - ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); + ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + RecognitionEventSys.class); verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. - assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().status); + RecognitionEventSys lastEvent = eventCaptor.getValue(); + assertEquals(12345, lastEvent.halEventReceivedMillis); + assertEquals(RecognitionStatus.ABORTED, lastEvent.recognitionEvent.status); // Unload the model. unloadModel(module, handle, hwHandle); @@ -540,14 +547,16 @@ public class SoundTriggerMiddlewareImplTest { startRecognition(module, handle, hwHandle); // Abort. - hwCallback.sendPhraseRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false); + hwCallback.sendPhraseRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false, 12345); - ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass( - PhraseRecognitionEvent.class); + ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEventSys.class); verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101)); // Validate the event. - assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().common.status); + PhraseRecognitionEventSys lastEvent = eventCaptor.getValue(); + assertEquals(12345, lastEvent.halEventReceivedMillis); + assertEquals(RecognitionStatus.ABORTED, lastEvent.phraseRecognitionEvent.common.status); // Unload the model. unloadModel(module, handle, hwHandle); @@ -630,18 +639,24 @@ public class SoundTriggerMiddlewareImplTest { mCallback = callback; } - private RecognitionEvent sendRecognitionEvent(int hwHandle, @RecognitionStatus int status, - boolean recognitionStillActive) { - RecognitionEvent event = TestUtil.createRecognitionEvent(status, + private RecognitionEventSys sendRecognitionEvent(int hwHandle, + @RecognitionStatus int status, + boolean recognitionStillActive, @ElapsedRealtimeLong long halEventReceivedMillis) { + RecognitionEventSys event = new RecognitionEventSys(); + event.recognitionEvent = TestUtil.createRecognitionEvent(status, recognitionStillActive); + event.halEventReceivedMillis = halEventReceivedMillis; mCallback.recognitionCallback(hwHandle, event); return event; } - private PhraseRecognitionEvent sendPhraseRecognitionEvent(int hwHandle, - @RecognitionStatus int status, boolean recognitionStillActive) { - PhraseRecognitionEvent event = TestUtil.createPhraseRecognitionEvent(status, + private PhraseRecognitionEventSys sendPhraseRecognitionEvent(int hwHandle, + @RecognitionStatus int status, boolean recognitionStillActive, + @ElapsedRealtimeLong long halEventReceivedMillis) { + PhraseRecognitionEventSys event = new PhraseRecognitionEventSys(); + event.phraseRecognitionEvent = TestUtil.createPhraseRecognitionEvent(status, recognitionStillActive); + event.halEventReceivedMillis = halEventReceivedMillis; mCallback.phraseRecognitionCallback(hwHandle, event); return event; } diff --git a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLoggingLatencyTest.java b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLoggingLatencyTest.java index f8a068c7f121..cc357d76cb4a 100644 --- a/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLoggingLatencyTest.java +++ b/services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLoggingLatencyTest.java @@ -32,7 +32,7 @@ import android.media.soundtrigger.PhraseRecognitionExtra; import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger_middleware.ISoundTriggerCallback; -import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; import android.os.BatteryStatsInternal; import android.os.Process; import android.os.RemoteException; @@ -51,8 +51,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.util.Optional; - @RunWith(JUnit4.class) public class SoundTriggerMiddlewareLoggingLatencyTest { @@ -63,8 +61,6 @@ public class SoundTriggerMiddlewareLoggingLatencyTest { private ISoundTriggerMiddlewareInternal mDelegateMiddleware; @Mock private ISoundTriggerCallback mISoundTriggerCallback; - @Mock - private ISoundTriggerModule mSoundTriggerModule; private SoundTriggerMiddlewareLogging mSoundTriggerMiddlewareLogging; @Before @@ -109,7 +105,7 @@ public class SoundTriggerMiddlewareLoggingLatencyTest { verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture()); triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(), - RecognitionStatus.SUCCESS, Optional.of(100) /* keyphraseId */); + RecognitionStatus.SUCCESS, 100 /* keyphraseId */); assertThat(mLatencyTracker.getActiveActionStartTime( ACTION_SHOW_VOICE_INTERACTION)).isGreaterThan(-1); @@ -124,11 +120,11 @@ public class SoundTriggerMiddlewareLoggingLatencyTest { verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture()); triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(), - RecognitionStatus.SUCCESS, Optional.of(100) /* keyphraseId */); + RecognitionStatus.SUCCESS, 100 /* keyphraseId */); long firstTriggerSessionStartTime = mLatencyTracker.getActiveActionStartTime( ACTION_SHOW_VOICE_INTERACTION); triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(), - RecognitionStatus.SUCCESS, Optional.of(100) /* keyphraseId */); + RecognitionStatus.SUCCESS, 100 /* keyphraseId */); assertThat(mLatencyTracker.getActiveActionStartTime( ACTION_SHOW_VOICE_INTERACTION)).isGreaterThan(-1); assertThat(mLatencyTracker.getActiveActionStartTime( @@ -145,7 +141,7 @@ public class SoundTriggerMiddlewareLoggingLatencyTest { verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture()); triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(), - RecognitionStatus.ABORTED, Optional.of(100) /* keyphraseId */); + RecognitionStatus.ABORTED, 100 /* keyphraseId */); assertThat( mLatencyTracker.getActiveActionStartTime(ACTION_SHOW_VOICE_INTERACTION)).isEqualTo( @@ -162,7 +158,7 @@ public class SoundTriggerMiddlewareLoggingLatencyTest { verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture()); triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(), - RecognitionStatus.SUCCESS, Optional.empty() /* keyphraseId */); + RecognitionStatus.SUCCESS); assertThat( mLatencyTracker.getActiveActionStartTime(ACTION_SHOW_VOICE_INTERACTION)).isEqualTo( @@ -170,19 +166,27 @@ public class SoundTriggerMiddlewareLoggingLatencyTest { } private void triggerPhraseRecognitionEvent(ISoundTriggerCallback callback, - @RecognitionStatus int triggerEventStatus, Optional<Integer> optionalKeyphraseId) - throws RemoteException { + @RecognitionStatus int triggerEventStatus) throws RemoteException { + triggerPhraseRecognitionEvent(callback, triggerEventStatus, -1 /* keyphraseId */); + } + + private void triggerPhraseRecognitionEvent(ISoundTriggerCallback callback, + @RecognitionStatus int triggerEventStatus, int keyphraseId) throws RemoteException { // trigger a phrase recognition to start a latency tracker session PhraseRecognitionEvent successEventWithKeyphraseId = new PhraseRecognitionEvent(); successEventWithKeyphraseId.common = new RecognitionEvent(); successEventWithKeyphraseId.common.status = triggerEventStatus; - if (optionalKeyphraseId.isPresent()) { + if (keyphraseId > 0) { PhraseRecognitionExtra recognitionExtra = new PhraseRecognitionExtra(); - recognitionExtra.id = optionalKeyphraseId.get(); + recognitionExtra.id = keyphraseId; successEventWithKeyphraseId.phraseExtras = new PhraseRecognitionExtra[]{recognitionExtra}; } - callback.onPhraseRecognition(0 /* modelHandle */, successEventWithKeyphraseId, + PhraseRecognitionEventSys phraseRecognitionEventSys = new PhraseRecognitionEventSys(); + phraseRecognitionEventSys.phraseRecognitionEvent = successEventWithKeyphraseId; + phraseRecognitionEventSys.halEventReceivedMillis = 12345; + callback.onPhraseRecognition(0 /* modelHandle */, phraseRecognitionEventSys, 0 /* captureSession */); } + } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/AidlUtil.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/AidlUtil.java index f3457f5a221b..56a159eb465a 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/AidlUtil.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/AidlUtil.java @@ -21,6 +21,8 @@ import android.media.soundtrigger.PhraseRecognitionExtra; import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModelType; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; /** * Utilities for working with sound trigger related AIDL generated types. @@ -49,23 +51,29 @@ public class AidlUtil { /** * Creates a new generic abort event. + * * @return The new event. */ - static RecognitionEvent newAbortEvent() { - RecognitionEvent event = newEmptyRecognitionEvent(); - event.type = SoundModelType.GENERIC; - event.status = RecognitionStatus.ABORTED; - return event; + static RecognitionEventSys newAbortEvent() { + RecognitionEvent recognitionEvent = newEmptyRecognitionEvent(); + recognitionEvent.type = SoundModelType.GENERIC; + recognitionEvent.status = RecognitionStatus.ABORTED; + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = recognitionEvent; + return recognitionEventSys; } /** * Creates a new generic phrase event. + * * @return The new event. */ - static PhraseRecognitionEvent newAbortPhraseEvent() { - PhraseRecognitionEvent event = newEmptyPhraseRecognitionEvent(); - event.common.type = SoundModelType.KEYPHRASE; - event.common.status = RecognitionStatus.ABORTED; - return event; + static PhraseRecognitionEventSys newAbortPhraseEvent() { + PhraseRecognitionEvent recognitionEvent = newEmptyPhraseRecognitionEvent(); + recognitionEvent.common.type = SoundModelType.KEYPHRASE; + recognitionEvent.common.status = RecognitionStatus.ABORTED; + PhraseRecognitionEventSys phraseRecognitionEventSys = new PhraseRecognitionEventSys(); + phraseRecognitionEventSys.phraseRecognitionEvent = recognitionEvent; + return phraseRecognitionEventSys; } } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/ISoundTriggerHal.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/ISoundTriggerHal.java index 75206e69bc6a..6f4a94692c4b 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/ISoundTriggerHal.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/ISoundTriggerHal.java @@ -20,12 +20,12 @@ import android.hardware.soundtrigger3.ISoundTriggerHw; import android.hardware.soundtrigger3.ISoundTriggerHwCallback; import android.hardware.soundtrigger3.ISoundTriggerHwGlobalCallback; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.SoundModel; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.IBinder; /** @@ -173,14 +173,19 @@ interface ISoundTriggerHal { */ interface ModelCallback { /** - * @see ISoundTriggerHwCallback#recognitionCallback(int, RecognitionEvent) + * Decorated callback of + * {@link ISoundTriggerHwCallback#recognitionCallback(int, RecognitionEvent)} where + * {@link RecognitionEventSys} is decorating the returned {@link RecognitionEvent} */ - void recognitionCallback(int modelHandle, RecognitionEvent event); + void recognitionCallback(int modelHandle, RecognitionEventSys event); /** - * @see ISoundTriggerHwCallback#phraseRecognitionCallback(int, PhraseRecognitionEvent) + * Decorated callback of + * {@link ISoundTriggerHwCallback#phraseRecognitionCallback(int, PhraseRecognitionEvent)} + * where {@link PhraseRecognitionEventSys} is decorating the returned + * {@link PhraseRecognitionEvent} */ - void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEvent event); + void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEventSys event); /** * @see ISoundTriggerHwCallback#modelUnloaded(int) diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java index 8c7cabeee320..d8ef2b6640fd 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java @@ -19,14 +19,14 @@ package com.android.server.soundtrigger_middleware; import android.annotation.NonNull; import android.annotation.Nullable; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.SoundModelType; import android.media.soundtrigger.Status; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.IBinder; import java.util.HashSet; @@ -238,13 +238,13 @@ public class SoundTriggerHalConcurrentCaptureHandler implements ISoundTriggerHal } @Override - public void recognitionCallback(int modelHandle, RecognitionEvent event) { + public void recognitionCallback(int modelHandle, RecognitionEventSys event) { synchronized (mActiveModels) { if (!mActiveModels.contains(modelHandle)) { // Discard the event. return; } - if (!event.recognitionStillActive) { + if (!event.recognitionEvent.recognitionStillActive) { mActiveModels.remove(modelHandle); } // A recognition event must be the last one for its model, unless it indicates that @@ -255,13 +255,13 @@ public class SoundTriggerHalConcurrentCaptureHandler implements ISoundTriggerHal } @Override - public void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEvent event) { + public void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEventSys event) { synchronized (mActiveModels) { if (!mActiveModels.contains(modelHandle)) { // Discard the event. return; } - if (!event.common.recognitionStillActive) { + if (!event.phraseRecognitionEvent.common.recognitionStillActive) { mActiveModels.remove(modelHandle); } // A recognition event must be the last one for its model, unless it indicates that diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java index 24741e1caea9..bac24669696c 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java @@ -17,14 +17,14 @@ package com.android.server.soundtrigger_middleware; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.DeadObjectException; import android.os.IBinder; import android.util.Log; @@ -253,7 +253,7 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { } @Override - public void recognitionCallback(int model, RecognitionEvent event) { + public void recognitionCallback(int model, RecognitionEventSys event) { synchronized (mModelStates) { ModelState state = mModelStates.get(model); if (state == null || state == ModelState.INACTIVE) { @@ -261,15 +261,16 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { reboot(); return; } - if (event.recognitionStillActive && event.status != RecognitionStatus.SUCCESS - && event.status != RecognitionStatus.FORCED) { + if (event.recognitionEvent.recognitionStillActive + && event.recognitionEvent.status != RecognitionStatus.SUCCESS + && event.recognitionEvent.status != RecognitionStatus.FORCED) { Log.wtfStack(TAG, "recognitionStillActive is only allowed when the recognition status " + "is SUCCESS"); reboot(); return; } - if (!event.recognitionStillActive) { + if (!event.recognitionEvent.recognitionStillActive) { mModelStates.replace(model, ModelState.INACTIVE); } } @@ -278,7 +279,7 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { } @Override - public void phraseRecognitionCallback(int model, PhraseRecognitionEvent event) { + public void phraseRecognitionCallback(int model, PhraseRecognitionEventSys event) { synchronized (mModelStates) { ModelState state = mModelStates.get(model); if (state == null || state == ModelState.INACTIVE) { @@ -286,16 +287,16 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { reboot(); return; } - if (event.common.recognitionStillActive - && event.common.status != RecognitionStatus.SUCCESS - && event.common.status != RecognitionStatus.FORCED) { + if (event.phraseRecognitionEvent.common.recognitionStillActive + && event.phraseRecognitionEvent.common.status != RecognitionStatus.SUCCESS + && event.phraseRecognitionEvent.common.status != RecognitionStatus.FORCED) { Log.wtfStack(TAG, "recognitionStillActive is only allowed when the recognition status " + "is SUCCESS"); reboot(); return; } - if (!event.common.recognitionStillActive) { + if (!event.phraseRecognitionEvent.common.recognitionStillActive) { mModelStates.replace(model, ModelState.INACTIVE); } } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java index c67bdd76eee8..df2e9b41662b 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java @@ -25,9 +25,12 @@ import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.IBinder; import android.os.IHwBinder; import android.os.RemoteException; +import android.os.SystemClock; import android.system.OsConstants; import android.util.Log; @@ -570,16 +573,20 @@ final class SoundTriggerHw2Compat implements ISoundTriggerHal { public void recognitionCallback_2_1( android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent event, int cookie) { - mDelegate.recognitionCallback(event.header.model, - ConversionUtil.hidl2aidlRecognitionEvent(event)); + RecognitionEventSys eventSys = new RecognitionEventSys(); + eventSys.recognitionEvent = ConversionUtil.hidl2aidlRecognitionEvent(event); + eventSys.halEventReceivedMillis = SystemClock.elapsedRealtime(); + mDelegate.recognitionCallback(event.header.model, eventSys); } @Override public void phraseRecognitionCallback_2_1( android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent event, int cookie) { - mDelegate.phraseRecognitionCallback(event.common.header.model, - ConversionUtil.hidl2aidlPhraseRecognitionEvent(event)); + PhraseRecognitionEventSys eventSys = new PhraseRecognitionEventSys(); + eventSys.phraseRecognitionEvent = ConversionUtil.hidl2aidlPhraseRecognitionEvent(event); + eventSys.halEventReceivedMillis = SystemClock.elapsedRealtime(); + mDelegate.phraseRecognitionCallback(event.common.header.model, eventSys); } @Override diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java index 8bb5eb191858..b1165bb57628 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java @@ -29,9 +29,12 @@ import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.IBinder; import android.os.RemoteException; import android.os.ServiceSpecificException; +import android.os.SystemClock; public class SoundTriggerHw3Compat implements ISoundTriggerHal { private final @NonNull ISoundTriggerHw mDriver; @@ -244,14 +247,20 @@ public class SoundTriggerHw3Compat implements ISoundTriggerHal { public void phraseRecognitionCallback(int model, PhraseRecognitionEvent event) { // A FORCED status implies that recognition is still active after the event. event.common.recognitionStillActive |= event.common.status == RecognitionStatus.FORCED; - mDelegate.phraseRecognitionCallback(model, event); + PhraseRecognitionEventSys phraseRecognitionEventSys = new PhraseRecognitionEventSys(); + phraseRecognitionEventSys.phraseRecognitionEvent = event; + phraseRecognitionEventSys.halEventReceivedMillis = SystemClock.elapsedRealtimeNanos(); + mDelegate.phraseRecognitionCallback(model, phraseRecognitionEventSys); } @Override public void recognitionCallback(int model, RecognitionEvent event) { // A FORCED status implies that recognition is still active after the event. event.recognitionStillActive |= event.status == RecognitionStatus.FORCED; - mDelegate.recognitionCallback(model, event); + RecognitionEventSys recognitionEventSys = new RecognitionEventSys(); + recognitionEventSys.recognitionEvent = event; + recognitionEventSys.halEventReceivedMillis = SystemClock.elapsedRealtimeNanos(); + mDelegate.recognitionCallback(model, recognitionEventSys); } @Override diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLogging.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLogging.java index 0e796d1afbd6..2ee4e3cff02c 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLogging.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLogging.java @@ -16,12 +16,24 @@ package com.android.server.soundtrigger_middleware; -import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.*; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.DETACH; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.FORCE_RECOGNITION; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.GET_MODEL_PARAMETER; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.LOAD_MODEL; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.LOAD_PHRASE_MODEL; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.MODEL_UNLOADED; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.MODULE_DIED; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.QUERY_MODEL_PARAMETER; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.RECOGNITION; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.RESOURCES_AVAILABLE; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.SET_MODEL_PARAMETER; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.START_RECOGNITION; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.STOP_RECOGNITION; +import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.UNLOAD_MODEL; import static com.android.server.utils.EventLogger.Event.ALOGI; import static com.android.server.utils.EventLogger.Event.ALOGW; import android.annotation.NonNull; -import android.annotation.Nullable; import android.content.Context; import android.media.permission.Identity; import android.media.permission.IdentityContext; @@ -29,11 +41,12 @@ import android.media.soundtrigger.ModelParameterRange; import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; import android.os.BatteryStatsInternal; import android.os.IBinder; @@ -45,19 +58,18 @@ import com.android.internal.annotations.VisibleForTesting; import com.android.internal.util.ArrayUtils; import com.android.internal.util.LatencyTracker; import com.android.server.LocalServices; -import com.android.server.utils.EventLogger.Event; import com.android.server.utils.EventLogger; - +import com.android.server.utils.EventLogger.Event; import java.io.PrintWriter; import java.util.Arrays; +import java.util.Deque; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import java.util.Deque; /** @@ -370,7 +382,7 @@ public class SoundTriggerMiddlewareLogging implements ISoundTriggerMiddlewareInt } @Override - public void onRecognition(int modelHandle, RecognitionEvent event, int captureSession) + public void onRecognition(int modelHandle, RecognitionEventSys event, int captureSession) throws RemoteException { try { mBatteryStatsInternalSupplier.get().noteWakingSoundTrigger( @@ -388,13 +400,13 @@ public class SoundTriggerMiddlewareLogging implements ISoundTriggerMiddlewareInt } @Override - public void onPhraseRecognition(int modelHandle, PhraseRecognitionEvent event, + public void onPhraseRecognition(int modelHandle, PhraseRecognitionEventSys event, int captureSession) throws RemoteException { try { mBatteryStatsInternalSupplier.get().noteWakingSoundTrigger( SystemClock.elapsedRealtime(), mOriginatorIdentity.uid); - startKeyphraseEventLatencyTracking(event); + startKeyphraseEventLatencyTracking(event.phraseRecognitionEvent); mCallbackDelegate.onPhraseRecognition(modelHandle, event, captureSession); mEventLogger.enqueue(SessionEvent.createForVoid( RECOGNITION, modelHandle, event, captureSession) diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewarePermission.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewarePermission.java index 00cedd77414e..00b894e1c6b5 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewarePermission.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewarePermission.java @@ -27,15 +27,15 @@ import android.media.permission.Identity; import android.media.permission.IdentityContext; import android.media.permission.PermissionUtil; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; import android.os.IBinder; import android.os.RemoteException; @@ -307,16 +307,15 @@ public class SoundTriggerMiddlewarePermission implements ISoundTriggerMiddleware } @Override - public void onRecognition(int modelHandle, RecognitionEvent event, int captureSession) - throws RemoteException { + public void onRecognition(int modelHandle, RecognitionEventSys event, + int captureSession) throws RemoteException { enforcePermissions("Sound trigger recognition."); mDelegate.onRecognition(modelHandle, event, captureSession); } @Override - public void onPhraseRecognition(int modelHandle, PhraseRecognitionEvent event, - int captureSession) - throws RemoteException { + public void onPhraseRecognition(int modelHandle, PhraseRecognitionEventSys event, + int captureSession) throws RemoteException { enforcePermissions("Sound trigger phrase recognition."); mDelegate.onPhraseRecognition(modelHandle, event, captureSession); } diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java index 15c9ba923d3a..f208c03024b2 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java @@ -21,17 +21,17 @@ import android.annotation.Nullable; import android.media.permission.Identity; import android.media.permission.IdentityContext; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService; import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor; import android.os.IBinder; import android.os.RemoteException; @@ -710,8 +710,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware } } - class CallbackWrapper implements ISoundTriggerCallback, - IBinder.DeathRecipient { + class CallbackWrapper implements ISoundTriggerCallback, IBinder.DeathRecipient { private final ISoundTriggerCallback mCallback; CallbackWrapper(ISoundTriggerCallback callback) { @@ -728,11 +727,11 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware } @Override - public void onRecognition(int modelHandle, @NonNull RecognitionEvent event, + public void onRecognition(int modelHandle, @NonNull RecognitionEventSys event, int captureSession) { synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (!event.recognitionStillActive) { + if (!event.recognitionEvent.recognitionStillActive) { modelState.activityState = ModelState.Activity.LOADED; } } @@ -744,7 +743,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware Log.w(TAG, "Client callback exception.", e); synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (event.status != RecognitionStatus.FORCED) { + if (event.recognitionEvent.status != RecognitionStatus.FORCED) { modelState.activityState = ModelState.Activity.INTERCEPTED; // If we failed to deliver an actual event to the client, they would // never know to restart it whenever circumstances change. Thus, we @@ -758,10 +757,10 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware @Override public void onPhraseRecognition(int modelHandle, - @NonNull PhraseRecognitionEvent event, int captureSession) { + @NonNull PhraseRecognitionEventSys event, int captureSession) { synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (!event.common.recognitionStillActive) { + if (!event.phraseRecognitionEvent.common.recognitionStillActive) { modelState.activityState = ModelState.Activity.LOADED; } } @@ -773,7 +772,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware Log.w(TAG, "Client callback exception.", e); synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (!event.common.recognitionStillActive) { + if (!event.phraseRecognitionEvent.common.recognitionStillActive) { modelState.activityState = ModelState.Activity.INTERCEPTED; // If we failed to deliver an actual event to the client, they would // never know to restart it whenever circumstances change. Thus, we diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java index 6223b2e8ace4..84cec5592831 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java @@ -19,16 +19,16 @@ package com.android.server.soundtrigger_middleware; import android.annotation.NonNull; import android.annotation.Nullable; import android.media.soundtrigger.ModelParameterRange; -import android.media.soundtrigger.PhraseRecognitionEvent; import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; -import android.media.soundtrigger.RecognitionEvent; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.SoundModelType; import android.media.soundtrigger.Status; import android.media.soundtrigger_middleware.ISoundTriggerCallback; import android.media.soundtrigger_middleware.ISoundTriggerModule; +import android.media.soundtrigger_middleware.PhraseRecognitionEventSys; +import android.media.soundtrigger_middleware.RecognitionEventSys; import android.os.Binder; import android.os.IBinder; import android.os.RemoteException; @@ -499,10 +499,10 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo @Override public void recognitionCallback(int modelHandle, - @NonNull RecognitionEvent recognitionEvent) { + @NonNull RecognitionEventSys event) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { - if (!recognitionEvent.recognitionStillActive) { + if (!event.recognitionEvent.recognitionStillActive) { setState(ModelState.LOADED); } callback = mCallback; @@ -510,7 +510,7 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo // The callback must be invoked outside of the lock. try { if (callback != null) { - callback.onRecognition(mHandle, recognitionEvent, mSession.mSessionHandle); + callback.onRecognition(mHandle, event, mSession.mSessionHandle); } } catch (RemoteException e) { // We're not expecting any exceptions here. @@ -520,10 +520,10 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo @Override public void phraseRecognitionCallback(int modelHandle, - @NonNull PhraseRecognitionEvent phraseRecognitionEvent) { + @NonNull PhraseRecognitionEventSys event) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { - if (!phraseRecognitionEvent.common.recognitionStillActive) { + if (!event.phraseRecognitionEvent.common.recognitionStillActive) { setState(ModelState.LOADED); } callback = mCallback; @@ -532,8 +532,7 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo // The callback must be invoked outside of the lock. try { if (callback != null) { - mCallback.onPhraseRecognition(mHandle, phraseRecognitionEvent, - mSession.mSessionHandle); + mCallback.onPhraseRecognition(mHandle, event, mSession.mSessionHandle); } } catch (RemoteException e) { // We're not expecting any exceptions here. |