diff options
author | 2020-08-19 16:29:18 +0200 | |
---|---|---|
committer | 2020-09-30 14:44:01 +0000 | |
commit | cf47bdd8885c1436156b9a6f19a4c8c81eb71af5 (patch) | |
tree | 029494b60f7c49be69e7a70c1d6a66f9c87df299 | |
parent | 15b6144bd46327b4ef9f21d52ea48ee3fcfa54b0 (diff) |
Add system service for music recognition.
The client submits a RecognitionRequest via MusicRecognitionManager. System server opens an audio stream based on the request and sends it to a MusicRecognitionService (which is exposed by a system app). The result is passed back through system server to the original caller.
Test: tracked in b/169662646
Bug: 169403302
Change-Id: I4c7fd9d9d72ddd5678867fd037cab6198bff2c2d
22 files changed, 1305 insertions, 0 deletions
diff --git a/api/system-current.txt b/api/system-current.txt index 8009d0370dd8..25d5b598973f 100755 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -38,6 +38,7 @@ package android { field public static final String BIND_EXTERNAL_STORAGE_SERVICE = "android.permission.BIND_EXTERNAL_STORAGE_SERVICE"; field public static final String BIND_IMS_SERVICE = "android.permission.BIND_IMS_SERVICE"; field public static final String BIND_KEYGUARD_APPWIDGET = "android.permission.BIND_KEYGUARD_APPWIDGET"; + field public static final String BIND_MUSIC_RECOGNITION_SERVICE = "android.permission.BIND_MUSIC_RECOGNITION_SERVICE"; field public static final String BIND_NETWORK_RECOMMENDATION_SERVICE = "android.permission.BIND_NETWORK_RECOMMENDATION_SERVICE"; field public static final String BIND_NOTIFICATION_ASSISTANT_SERVICE = "android.permission.BIND_NOTIFICATION_ASSISTANT_SERVICE"; field public static final String BIND_PHONE_ACCOUNT_SUGGESTION_SERVICE = "android.permission.BIND_PHONE_ACCOUNT_SUGGESTION_SERVICE"; @@ -119,6 +120,7 @@ package android { field public static final String MANAGE_DEBUGGING = "android.permission.MANAGE_DEBUGGING"; field public static final String MANAGE_FACTORY_RESET_PROTECTION = "android.permission.MANAGE_FACTORY_RESET_PROTECTION"; field public static final String MANAGE_IPSEC_TUNNELS = "android.permission.MANAGE_IPSEC_TUNNELS"; + field public static final String MANAGE_MUSIC_RECOGNITION = "android.permission.MANAGE_MUSIC_RECOGNITION"; field public static final String MANAGE_ONE_TIME_PERMISSION_SESSIONS = "android.permission.MANAGE_ONE_TIME_PERMISSION_SESSIONS"; field public static final String MANAGE_ROLE_HOLDERS = "android.permission.MANAGE_ROLE_HOLDERS"; field public static final String MANAGE_ROLLBACKS = "android.permission.MANAGE_ROLLBACKS"; @@ -4528,6 +4530,58 @@ package android.media.audiopolicy { } +package android.media.musicrecognition { + + public class MusicRecognitionManager { + method @RequiresPermission(android.Manifest.permission.MANAGE_MUSIC_RECOGNITION) public void beginStreamingSearch(@NonNull android.media.musicrecognition.RecognitionRequest, @NonNull java.util.concurrent.Executor, @NonNull android.media.musicrecognition.MusicRecognitionManager.RecognitionCallback); + field public static final int RECOGNITION_FAILED_AUDIO_UNAVAILABLE = 7; // 0x7 + field public static final int RECOGNITION_FAILED_NOT_FOUND = 1; // 0x1 + field public static final int RECOGNITION_FAILED_NO_CONNECTIVITY = 2; // 0x2 + field public static final int RECOGNITION_FAILED_SERVICE_KILLED = 5; // 0x5 + field public static final int RECOGNITION_FAILED_SERVICE_UNAVAILABLE = 3; // 0x3 + field public static final int RECOGNITION_FAILED_TIMEOUT = 6; // 0x6 + field public static final int RECOGNITION_FAILED_UNKNOWN = -1; // 0xffffffff + } + + public static interface MusicRecognitionManager.RecognitionCallback { + method public void onAudioStreamClosed(); + method public void onRecognitionFailed(@NonNull android.media.musicrecognition.RecognitionRequest, int); + method public void onRecognitionSucceeded(@NonNull android.media.musicrecognition.RecognitionRequest, @NonNull android.media.MediaMetadata, @Nullable android.os.Bundle); + } + + public abstract class MusicRecognitionService extends android.app.Service { + ctor public MusicRecognitionService(); + method public abstract void onRecognize(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.media.musicrecognition.MusicRecognitionService.Callback); + } + + public static interface MusicRecognitionService.Callback { + method public void onRecognitionFailed(int); + method public void onRecognitionSucceeded(@NonNull android.media.MediaMetadata, @Nullable android.os.Bundle); + } + + public final class RecognitionRequest implements android.os.Parcelable { + method public int describeContents(); + method @NonNull public android.media.AudioAttributes getAudioAttributes(); + method @NonNull public android.media.AudioFormat getAudioFormat(); + method public int getCaptureSession(); + method public int getIgnoreBeginningFrames(); + method public int getMaxAudioLengthSeconds(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator<android.media.musicrecognition.RecognitionRequest> CREATOR; + } + + public static final class RecognitionRequest.Builder { + ctor public RecognitionRequest.Builder(); + method @NonNull public android.media.musicrecognition.RecognitionRequest build(); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setAudioAttributes(@NonNull android.media.AudioAttributes); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setCaptureSession(int); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setIgnoreBeginningFrames(int); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setMaxAudioLengthSeconds(int); + } + +} + package android.media.session { public final class MediaSessionManager { diff --git a/api/test-current.txt b/api/test-current.txt index af70562839ba..0488bc7a3088 100644 --- a/api/test-current.txt +++ b/api/test-current.txt @@ -2006,6 +2006,57 @@ package android.media.audiopolicy { } +package android.media.musicrecognition { + + public class MusicRecognitionManager { + field public static final int RECOGNITION_FAILED_AUDIO_UNAVAILABLE = 7; // 0x7 + field public static final int RECOGNITION_FAILED_NOT_FOUND = 1; // 0x1 + field public static final int RECOGNITION_FAILED_NO_CONNECTIVITY = 2; // 0x2 + field public static final int RECOGNITION_FAILED_SERVICE_KILLED = 5; // 0x5 + field public static final int RECOGNITION_FAILED_SERVICE_UNAVAILABLE = 3; // 0x3 + field public static final int RECOGNITION_FAILED_TIMEOUT = 6; // 0x6 + field public static final int RECOGNITION_FAILED_UNKNOWN = -1; // 0xffffffff + } + + public static interface MusicRecognitionManager.RecognitionCallback { + method public void onAudioStreamClosed(); + method public void onRecognitionFailed(@NonNull android.media.musicrecognition.RecognitionRequest, int); + method public void onRecognitionSucceeded(@NonNull android.media.musicrecognition.RecognitionRequest, @NonNull android.media.MediaMetadata, @Nullable android.os.Bundle); + } + + public abstract class MusicRecognitionService extends android.app.Service { + ctor public MusicRecognitionService(); + method public abstract void onRecognize(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.media.musicrecognition.MusicRecognitionService.Callback); + } + + public static interface MusicRecognitionService.Callback { + method public void onRecognitionFailed(int); + method public void onRecognitionSucceeded(@NonNull android.media.MediaMetadata, @Nullable android.os.Bundle); + } + + public final class RecognitionRequest implements android.os.Parcelable { + method public int describeContents(); + method @NonNull public android.media.AudioAttributes getAudioAttributes(); + method @NonNull public android.media.AudioFormat getAudioFormat(); + method public int getCaptureSession(); + method public int getIgnoreBeginningFrames(); + method public int getMaxAudioLengthSeconds(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator<android.media.musicrecognition.RecognitionRequest> CREATOR; + } + + public static final class RecognitionRequest.Builder { + ctor public RecognitionRequest.Builder(); + method @NonNull public android.media.musicrecognition.RecognitionRequest build(); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setAudioAttributes(@NonNull android.media.AudioAttributes); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setCaptureSession(int); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setIgnoreBeginningFrames(int); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setMaxAudioLengthSeconds(int); + } + +} + package android.media.tv { public final class TvInputManager { diff --git a/core/java/android/app/SystemServiceRegistry.java b/core/java/android/app/SystemServiceRegistry.java index 3b11b0d80da5..9ddc66f95591 100644 --- a/core/java/android/app/SystemServiceRegistry.java +++ b/core/java/android/app/SystemServiceRegistry.java @@ -106,6 +106,8 @@ import android.media.MediaRouter; import android.media.MediaTranscodeManager; import android.media.midi.IMidiManager; import android.media.midi.MidiManager; +import android.media.musicrecognition.IMusicRecognitionManager; +import android.media.musicrecognition.MusicRecognitionManager; import android.media.projection.MediaProjectionManager; import android.media.soundtrigger.SoundTriggerManager; import android.media.tv.ITvInputManager; @@ -1118,6 +1120,17 @@ public final class SystemServiceRegistry { return new AutofillManager(ctx.getOuterContext(), service); }}); + registerService(Context.MUSIC_RECOGNITION_SERVICE, MusicRecognitionManager.class, + new CachedServiceFetcher<MusicRecognitionManager>() { + @Override + public MusicRecognitionManager createService(ContextImpl ctx) { + IBinder b = ServiceManager.getService( + Context.MUSIC_RECOGNITION_SERVICE); + return new MusicRecognitionManager( + IMusicRecognitionManager.Stub.asInterface(b)); + } + }); + registerService(Context.CONTENT_CAPTURE_MANAGER_SERVICE, ContentCaptureManager.class, new CachedServiceFetcher<ContentCaptureManager>() { @Override diff --git a/core/java/android/content/Context.java b/core/java/android/content/Context.java index 98f78878ec15..e1c09d9c6aeb 100644 --- a/core/java/android/content/Context.java +++ b/core/java/android/content/Context.java @@ -4481,6 +4481,14 @@ public abstract class Context { public static final String SOUND_TRIGGER_MIDDLEWARE_SERVICE = "soundtrigger_middleware"; /** + * Used to access {@link MusicRecognitionManagerService}. + * + * @hide + * @see #getSystemService(String) + */ + public static final String MUSIC_RECOGNITION_SERVICE = "music_recognition"; + + /** * Official published name of the (internal) permission service. * * @see #getSystemService(String) diff --git a/core/res/AndroidManifest.xml b/core/res/AndroidManifest.xml index cdcb24b6a247..aab9fa7b44a4 100644 --- a/core/res/AndroidManifest.xml +++ b/core/res/AndroidManifest.xml @@ -3442,6 +3442,14 @@ <permission android:name="android.permission.BIND_CONTENT_SUGGESTIONS_SERVICE" android:protectionLevel="signature" /> + <!-- Must be declared by a android.service.musicrecognition.MusicRecognitionService, + to ensure that only the system can bind to it. + @SystemApi @hide This is not a third-party API (intended for OEMs and system apps). + <p>Protection level: signature + --> + <permission android:name="android.permission.BIND_MUSIC_RECOGNITION_SERVICE" + android:protectionLevel="signature" /> + <!-- Must be required by a android.service.autofill.augmented.AugmentedAutofillService, to ensure that only the system can bind to it. @SystemApi @hide This is not a third-party API (intended for OEMs and system apps). @@ -4799,6 +4807,11 @@ <permission android:name="android.permission.MANAGE_CONTENT_CAPTURE" android:protectionLevel="signature" /> + <!-- @SystemApi Allows an application to manage the music recognition service. + @hide <p>Not for use by third-party applications.</p> --> + <permission android:name="android.permission.MANAGE_MUSIC_RECOGNITION" + android:protectionLevel="signature" /> + <!-- @SystemApi Allows an application to manage the content suggestions service. @hide <p>Not for use by third-party applications.</p> --> <permission android:name="android.permission.MANAGE_CONTENT_SUGGESTIONS" diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml index 5f2e4f905b1c..7ff163c606f6 100644 --- a/core/res/res/values/config.xml +++ b/core/res/res/values/config.xml @@ -3635,6 +3635,8 @@ --> <string name="config_defaultContentSuggestionsService" translatable="false"></string> + <string name="config_defaultMusicRecognitionService" translatable="false"></string> + <!-- The package name for the default retail demo app. This package must be trusted, as it has the permissions to query the usage stats on the device. diff --git a/core/res/res/values/symbols.xml b/core/res/res/values/symbols.xml index 35ce780d3408..8134038bcbbe 100644 --- a/core/res/res/values/symbols.xml +++ b/core/res/res/values/symbols.xml @@ -3446,6 +3446,7 @@ <java-symbol type="string" name="config_defaultAugmentedAutofillService" /> <java-symbol type="string" name="config_defaultAppPredictionService" /> <java-symbol type="string" name="config_defaultContentSuggestionsService" /> + <java-symbol type="string" name="config_defaultMusicRecognitionService" /> <java-symbol type="string" name="config_defaultAttentionService" /> <java-symbol type="string" name="config_defaultSystemCaptionsService" /> <java-symbol type="string" name="config_defaultSystemCaptionsManagerService" /> diff --git a/media/java/android/media/musicrecognition/IMusicRecognitionManager.aidl b/media/java/android/media/musicrecognition/IMusicRecognitionManager.aidl new file mode 100644 index 000000000000..9e71afaa5023 --- /dev/null +++ b/media/java/android/media/musicrecognition/IMusicRecognitionManager.aidl @@ -0,0 +1,14 @@ +package android.media.musicrecognition; + +import android.media.musicrecognition.RecognitionRequest; +import android.os.IBinder; + +/** + * Used by {@link MusicRecognitionManager} to tell system server to begin open an audio stream to + * the designated lookup service. + * + * @hide + */ +interface IMusicRecognitionManager { + void beginRecognition(in RecognitionRequest recognitionRequest, in IBinder callback); +}
\ No newline at end of file diff --git a/media/java/android/media/musicrecognition/IMusicRecognitionManagerCallback.aidl b/media/java/android/media/musicrecognition/IMusicRecognitionManagerCallback.aidl new file mode 100644 index 000000000000..e70504f0bd07 --- /dev/null +++ b/media/java/android/media/musicrecognition/IMusicRecognitionManagerCallback.aidl @@ -0,0 +1,15 @@ +package android.media.musicrecognition; + +import android.os.Bundle; +import android.media.MediaMetadata; + +/** + * Callback used by system server to notify invoker of {@link MusicRecognitionManager} of the result + * + * @hide + */ +oneway interface IMusicRecognitionManagerCallback { + void onRecognitionSucceeded(in MediaMetadata result, in Bundle extras); + void onRecognitionFailed(int failureCode); + void onAudioStreamClosed(); +}
\ No newline at end of file diff --git a/media/java/android/media/musicrecognition/IMusicRecognitionService.aidl b/media/java/android/media/musicrecognition/IMusicRecognitionService.aidl new file mode 100644 index 000000000000..26543ed8bf8c --- /dev/null +++ b/media/java/android/media/musicrecognition/IMusicRecognitionService.aidl @@ -0,0 +1,18 @@ +package android.media.musicrecognition; + +import android.media.AudioFormat; +import android.os.ParcelFileDescriptor; +import android.os.IBinder; +import android.media.musicrecognition.IMusicRecognitionServiceCallback; + +/** + * Interface from the system to a {@link MusicRecognitionService}. + * + * @hide + */ +oneway interface IMusicRecognitionService { + void onAudioStreamStarted( + in ParcelFileDescriptor fd, + in AudioFormat audioFormat, + in IMusicRecognitionServiceCallback callback); +}
\ No newline at end of file diff --git a/media/java/android/media/musicrecognition/IMusicRecognitionServiceCallback.aidl b/media/java/android/media/musicrecognition/IMusicRecognitionServiceCallback.aidl new file mode 100644 index 000000000000..15215c4e15f1 --- /dev/null +++ b/media/java/android/media/musicrecognition/IMusicRecognitionServiceCallback.aidl @@ -0,0 +1,15 @@ +package android.media.musicrecognition; + +import android.os.Bundle; +import android.media.MediaMetadata; + +/** + * Interface from a {@MusicRecognitionService} the system. + * + * @hide + */ +oneway interface IMusicRecognitionServiceCallback { + void onRecognitionSucceeded(in MediaMetadata result, in Bundle extras); + + void onRecognitionFailed(int failureCode); +}
\ No newline at end of file diff --git a/media/java/android/media/musicrecognition/MusicRecognitionManager.java b/media/java/android/media/musicrecognition/MusicRecognitionManager.java new file mode 100644 index 000000000000..c7f55dfe1c76 --- /dev/null +++ b/media/java/android/media/musicrecognition/MusicRecognitionManager.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.musicrecognition; + +import static java.util.Objects.requireNonNull; + +import android.annotation.CallbackExecutor; +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.annotation.SystemApi; +import android.annotation.SystemService; +import android.annotation.TestApi; +import android.content.Context; +import android.media.MediaMetadata; +import android.os.Bundle; +import android.os.RemoteException; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.util.concurrent.Executor; + +/** + * System service that manages music recognition. + * + * @hide + */ +@SystemApi +@TestApi +@SystemService(Context.MUSIC_RECOGNITION_SERVICE) +public class MusicRecognitionManager { + + /** + * Error code provided by RecognitionCallback#onRecognitionFailed() + * + * @hide + */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(prefix = {"RECOGNITION_FAILED_"}, + value = {RECOGNITION_FAILED_UNKNOWN, + RECOGNITION_FAILED_NOT_FOUND, + RECOGNITION_FAILED_NO_CONNECTIVITY, + RECOGNITION_FAILED_SERVICE_UNAVAILABLE, + RECOGNITION_FAILED_SERVICE_KILLED, + RECOGNITION_FAILED_TIMEOUT, + RECOGNITION_FAILED_AUDIO_UNAVAILABLE}) + public @interface RecognitionFailureCode { + } + + /** Catchall error code. */ + public static final int RECOGNITION_FAILED_UNKNOWN = -1; + /** Recognition was performed but no result could be identified. */ + public static final int RECOGNITION_FAILED_NOT_FOUND = 1; + /** Recognition failed because the server couldn't be reached. */ + public static final int RECOGNITION_FAILED_NO_CONNECTIVITY = 2; + /** + * Recognition was not possible because the application which provides it is not available (for + * example, disabled). + */ + public static final int RECOGNITION_FAILED_SERVICE_UNAVAILABLE = 3; + /** Recognition failed because the recognizer was killed. */ + public static final int RECOGNITION_FAILED_SERVICE_KILLED = 5; + /** Recognition attempt timed out. */ + public static final int RECOGNITION_FAILED_TIMEOUT = 6; + /** Recognition failed due to an issue with obtaining an audio stream. */ + public static final int RECOGNITION_FAILED_AUDIO_UNAVAILABLE = 7; + + /** Callback interface for the caller of this api. */ + public interface RecognitionCallback { + /** + * Should be invoked by receiving app with the result of the search. + * + * @param recognitionRequest original request that started the recognition + * @param result result of the search + * @param extras extra data to be supplied back to the caller. Note that all + * executable parameters and file descriptors would be removed from the + * supplied bundle + */ + void onRecognitionSucceeded(@NonNull RecognitionRequest recognitionRequest, + @NonNull MediaMetadata result, @Nullable Bundle extras); + + /** + * Invoked when the search is not successful (possibly but not necessarily due to error). + * + * @param recognitionRequest original request that started the recognition + * @param failureCode failure code describing reason for failure + */ + void onRecognitionFailed(@NonNull RecognitionRequest recognitionRequest, + @RecognitionFailureCode int failureCode); + + /** + * Invoked by the system once the audio stream is closed either due to error, reaching the + * limit, or the remote service closing the stream. Always called per + * #beingStreamingSearch() invocation. + */ + void onAudioStreamClosed(); + } + + private final IMusicRecognitionManager mService; + + /** @hide */ + public MusicRecognitionManager(IMusicRecognitionManager service) { + mService = service; + } + + /** + * Constructs an {@link android.media.AudioRecord} from the given parameters and streams the + * audio bytes to the designated cloud lookup service. After the lookup is done, the given + * callback will be invoked by the system with the result or lack thereof. + * + * @param recognitionRequest audio parameters for the stream to search + * @param callbackExecutor where the callback is invoked + * @param callback invoked when the result is available + * + * @hide + */ + @SystemApi + @RequiresPermission(android.Manifest.permission.MANAGE_MUSIC_RECOGNITION) + public void beginStreamingSearch( + @NonNull RecognitionRequest recognitionRequest, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull RecognitionCallback callback) { + try { + mService.beginRecognition( + requireNonNull(recognitionRequest), + new MusicRecognitionCallbackWrapper( + requireNonNull(recognitionRequest), + requireNonNull(callback), + requireNonNull(callbackExecutor))); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + private final class MusicRecognitionCallbackWrapper extends + IMusicRecognitionManagerCallback.Stub { + + @NonNull + private final RecognitionRequest mRecognitionRequest; + @NonNull + private final RecognitionCallback mCallback; + @NonNull + private final Executor mCallbackExecutor; + + MusicRecognitionCallbackWrapper( + RecognitionRequest recognitionRequest, + RecognitionCallback callback, + Executor callbackExecutor) { + mRecognitionRequest = recognitionRequest; + mCallback = callback; + mCallbackExecutor = callbackExecutor; + } + + @Override + public void onRecognitionSucceeded(MediaMetadata result, Bundle extras) { + mCallbackExecutor.execute( + () -> mCallback.onRecognitionSucceeded(mRecognitionRequest, result, extras)); + } + + @Override + public void onRecognitionFailed(@RecognitionFailureCode int failureCode) { + mCallbackExecutor.execute( + () -> mCallback.onRecognitionFailed(mRecognitionRequest, failureCode)); + } + + @Override + public void onAudioStreamClosed() { + mCallbackExecutor.execute(mCallback::onAudioStreamClosed); + } + } +} diff --git a/media/java/android/media/musicrecognition/MusicRecognitionService.java b/media/java/android/media/musicrecognition/MusicRecognitionService.java new file mode 100644 index 000000000000..b75d2c4f1e50 --- /dev/null +++ b/media/java/android/media/musicrecognition/MusicRecognitionService.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.musicrecognition; + +import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.annotation.TestApi; +import android.app.Service; +import android.content.Intent; +import android.media.AudioFormat; +import android.media.MediaMetadata; +import android.os.Bundle; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.ParcelFileDescriptor; +import android.os.RemoteException; +import android.util.Log; + +/** + * Implemented by an app that wants to offer music search lookups. The system will start the + * service and stream up to 16 seconds of audio over the given file descriptor. + * + * @hide + */ +@SystemApi +@TestApi +public abstract class MusicRecognitionService extends Service { + + private static final String TAG = MusicRecognitionService.class.getSimpleName(); + + /** Callback for the result of the remote search. */ + public interface Callback { + /** + * Call this method to pass back a successful search result. + * + * @param result successful result of the search + * @param extras extra data to be supplied back to the caller. Note that all executable + * parameters and file descriptors would be removed from the supplied bundle + */ + void onRecognitionSucceeded(@NonNull MediaMetadata result, @Nullable Bundle extras); + + /** + * Call this method if the search does not find a result on an error occurred. + */ + void onRecognitionFailed(@MusicRecognitionManager.RecognitionFailureCode int failureCode); + } + + /** + * Action used to start this service. + * + * @hide + */ + public static final String ACTION_MUSIC_SEARCH_LOOKUP = + "android.service.musicrecognition.MUSIC_RECOGNITION"; + + private Handler mHandler; + private final IMusicRecognitionService mServiceInterface = + new IMusicRecognitionService.Stub() { + @Override + public void onAudioStreamStarted(ParcelFileDescriptor fd, + AudioFormat audioFormat, + IMusicRecognitionServiceCallback callback) { + mHandler.sendMessage( + obtainMessage(MusicRecognitionService.this::onRecognize, fd, + audioFormat, + new Callback() { + @Override + public void onRecognitionSucceeded( + @NonNull MediaMetadata result, + @Nullable Bundle extras) { + try { + callback.onRecognitionSucceeded(result, extras); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + + @Override + public void onRecognitionFailed(int failureCode) { + try { + callback.onRecognitionFailed(failureCode); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + })); + } + }; + + @Override + public void onCreate() { + super.onCreate(); + mHandler = new Handler(Looper.getMainLooper(), null, true); + } + + /** + * Read audio from this stream. You must invoke the callback whether the music is recognized or + * not. + * + * @param stream containing music to be recognized. Close when you are finished. + * @param audioFormat describes sample rate, channels and endianness of the stream + * @param callback to invoke after lookup is finished. Must always be called. + */ + public abstract void onRecognize(@NonNull ParcelFileDescriptor stream, + @NonNull AudioFormat audioFormat, + @NonNull Callback callback); + + /** + * @hide + */ + @Nullable + @Override + public IBinder onBind(@NonNull Intent intent) { + if (ACTION_MUSIC_SEARCH_LOOKUP.equals(intent.getAction())) { + return mServiceInterface.asBinder(); + } + Log.w(TAG, + "Tried to bind to wrong intent (should be " + ACTION_MUSIC_SEARCH_LOOKUP + ": " + + intent); + return null; + } +} diff --git a/media/java/android/media/musicrecognition/RecognitionRequest.aidl b/media/java/android/media/musicrecognition/RecognitionRequest.aidl new file mode 100644 index 000000000000..757b57010e23 --- /dev/null +++ b/media/java/android/media/musicrecognition/RecognitionRequest.aidl @@ -0,0 +1,18 @@ +/* Copyright 2020, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +package android.media.musicrecognition; + +parcelable RecognitionRequest;
\ No newline at end of file diff --git a/media/java/android/media/musicrecognition/RecognitionRequest.java b/media/java/android/media/musicrecognition/RecognitionRequest.java new file mode 100644 index 000000000000..65b2ccd37f79 --- /dev/null +++ b/media/java/android/media/musicrecognition/RecognitionRequest.java @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.musicrecognition; + +import static android.media.AudioAttributes.CONTENT_TYPE_MUSIC; +import static android.media.AudioFormat.ENCODING_PCM_16BIT; + +import static java.util.Objects.requireNonNull; + +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.annotation.TestApi; +import android.media.AudioAttributes; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.os.Parcel; +import android.os.Parcelable; + +/** + * Encapsulates parameters for making music recognition queries via {@link MusicRecognitionManager}. + * + * @hide + */ +@SystemApi +@TestApi +public final class RecognitionRequest implements Parcelable { + @NonNull private final AudioAttributes mAudioAttributes; + @NonNull private final AudioFormat mAudioFormat; + private final int mCaptureSession; + private final int mMaxAudioLengthSeconds; + private final int mIgnoreBeginningFrames; + + private RecognitionRequest(Builder b) { + mAudioAttributes = requireNonNull(b.mAudioAttributes); + mAudioFormat = requireNonNull(b.mAudioFormat); + mCaptureSession = b.mCaptureSession; + mMaxAudioLengthSeconds = b.mMaxAudioLengthSeconds; + mIgnoreBeginningFrames = b.mIgnoreBeginningFrames; + } + + @NonNull + public AudioAttributes getAudioAttributes() { + return mAudioAttributes; + } + + @NonNull + public AudioFormat getAudioFormat() { + return mAudioFormat; + } + + public int getCaptureSession() { + return mCaptureSession; + } + + @SuppressWarnings("MethodNameUnits") + public int getMaxAudioLengthSeconds() { + return mMaxAudioLengthSeconds; + } + + public int getIgnoreBeginningFrames() { + return mIgnoreBeginningFrames; + } + + /** + * Builder for constructing StreamSearchRequest objects. + * + * @hide + */ + @SystemApi + @TestApi + public static final class Builder { + private AudioFormat mAudioFormat = new AudioFormat.Builder() + .setSampleRate(16000) + .setEncoding(ENCODING_PCM_16BIT) + .build(); + private AudioAttributes mAudioAttributes = new AudioAttributes.Builder() + .setContentType(CONTENT_TYPE_MUSIC) + .build(); + private int mCaptureSession = MediaRecorder.AudioSource.MIC; + private int mMaxAudioLengthSeconds = 24; // Max enforced in system server. + private int mIgnoreBeginningFrames = 0; + + /** Attributes passed to the constructed {@link AudioRecord}. */ + @NonNull + public Builder setAudioAttributes(@NonNull AudioAttributes audioAttributes) { + mAudioAttributes = audioAttributes; + return this; + } + + /** AudioFormat passed to the constructed {@link AudioRecord}. */ + @NonNull + public Builder setAudioFormat(@NonNull AudioFormat audioFormat) { + mAudioFormat = audioFormat; + return this; + } + + /** Constant from {@link android.media.MediaRecorder.AudioSource}. */ + @NonNull + public Builder setCaptureSession(int captureSession) { + mCaptureSession = captureSession; + return this; + } + + /** Maximum number of seconds to stream from the audio source. */ + @NonNull + public Builder setMaxAudioLengthSeconds(int maxAudioLengthSeconds) { + mMaxAudioLengthSeconds = maxAudioLengthSeconds; + return this; + } + + /** Number of samples to drop from the start of the stream. */ + @NonNull + public Builder setIgnoreBeginningFrames(int ignoreBeginningFrames) { + mIgnoreBeginningFrames = ignoreBeginningFrames; + return this; + } + + /** Returns the constructed request. */ + @NonNull + public RecognitionRequest build() { + return new RecognitionRequest(this); + } + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + dest.writeParcelable(mAudioFormat, flags); + dest.writeParcelable(mAudioAttributes, flags); + dest.writeInt(mCaptureSession); + dest.writeInt(mMaxAudioLengthSeconds); + dest.writeInt(mIgnoreBeginningFrames); + } + + private RecognitionRequest(Parcel in) { + mAudioFormat = in.readParcelable(AudioFormat.class.getClassLoader()); + mAudioAttributes = in.readParcelable(AudioAttributes.class.getClassLoader()); + mCaptureSession = in.readInt(); + mMaxAudioLengthSeconds = in.readInt(); + mIgnoreBeginningFrames = in.readInt(); + } + + @NonNull public static final Creator<RecognitionRequest> CREATOR = + new Creator<RecognitionRequest>() { + + @Override + public RecognitionRequest createFromParcel(Parcel p) { + return new RecognitionRequest(p); + } + + @Override + public RecognitionRequest[] newArray(int size) { + return new RecognitionRequest[size]; + } + }; +} diff --git a/non-updatable-api/system-current.txt b/non-updatable-api/system-current.txt index 024f4acd8940..374f3613e48f 100644 --- a/non-updatable-api/system-current.txt +++ b/non-updatable-api/system-current.txt @@ -38,6 +38,7 @@ package android { field public static final String BIND_EXTERNAL_STORAGE_SERVICE = "android.permission.BIND_EXTERNAL_STORAGE_SERVICE"; field public static final String BIND_IMS_SERVICE = "android.permission.BIND_IMS_SERVICE"; field public static final String BIND_KEYGUARD_APPWIDGET = "android.permission.BIND_KEYGUARD_APPWIDGET"; + field public static final String BIND_MUSIC_RECOGNITION_SERVICE = "android.permission.BIND_MUSIC_RECOGNITION_SERVICE"; field public static final String BIND_NETWORK_RECOMMENDATION_SERVICE = "android.permission.BIND_NETWORK_RECOMMENDATION_SERVICE"; field public static final String BIND_NOTIFICATION_ASSISTANT_SERVICE = "android.permission.BIND_NOTIFICATION_ASSISTANT_SERVICE"; field public static final String BIND_PHONE_ACCOUNT_SUGGESTION_SERVICE = "android.permission.BIND_PHONE_ACCOUNT_SUGGESTION_SERVICE"; @@ -119,6 +120,7 @@ package android { field public static final String MANAGE_DEBUGGING = "android.permission.MANAGE_DEBUGGING"; field public static final String MANAGE_FACTORY_RESET_PROTECTION = "android.permission.MANAGE_FACTORY_RESET_PROTECTION"; field public static final String MANAGE_IPSEC_TUNNELS = "android.permission.MANAGE_IPSEC_TUNNELS"; + field public static final String MANAGE_MUSIC_RECOGNITION = "android.permission.MANAGE_MUSIC_RECOGNITION"; field public static final String MANAGE_ONE_TIME_PERMISSION_SESSIONS = "android.permission.MANAGE_ONE_TIME_PERMISSION_SESSIONS"; field public static final String MANAGE_ROLE_HOLDERS = "android.permission.MANAGE_ROLE_HOLDERS"; field public static final String MANAGE_ROLLBACKS = "android.permission.MANAGE_ROLLBACKS"; @@ -4468,6 +4470,58 @@ package android.media.audiopolicy { } +package android.media.musicrecognition { + + public class MusicRecognitionManager { + method @RequiresPermission(android.Manifest.permission.MANAGE_MUSIC_RECOGNITION) public void beginStreamingSearch(@NonNull android.media.musicrecognition.RecognitionRequest, @NonNull java.util.concurrent.Executor, @NonNull android.media.musicrecognition.MusicRecognitionManager.RecognitionCallback); + field public static final int RECOGNITION_FAILED_AUDIO_UNAVAILABLE = 7; // 0x7 + field public static final int RECOGNITION_FAILED_NOT_FOUND = 1; // 0x1 + field public static final int RECOGNITION_FAILED_NO_CONNECTIVITY = 2; // 0x2 + field public static final int RECOGNITION_FAILED_SERVICE_KILLED = 5; // 0x5 + field public static final int RECOGNITION_FAILED_SERVICE_UNAVAILABLE = 3; // 0x3 + field public static final int RECOGNITION_FAILED_TIMEOUT = 6; // 0x6 + field public static final int RECOGNITION_FAILED_UNKNOWN = -1; // 0xffffffff + } + + public static interface MusicRecognitionManager.RecognitionCallback { + method public void onAudioStreamClosed(); + method public void onRecognitionFailed(@NonNull android.media.musicrecognition.RecognitionRequest, int); + method public void onRecognitionSucceeded(@NonNull android.media.musicrecognition.RecognitionRequest, @NonNull android.media.MediaMetadata, @Nullable android.os.Bundle); + } + + public abstract class MusicRecognitionService extends android.app.Service { + ctor public MusicRecognitionService(); + method public abstract void onRecognize(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.media.musicrecognition.MusicRecognitionService.Callback); + } + + public static interface MusicRecognitionService.Callback { + method public void onRecognitionFailed(int); + method public void onRecognitionSucceeded(@NonNull android.media.MediaMetadata, @Nullable android.os.Bundle); + } + + public final class RecognitionRequest implements android.os.Parcelable { + method public int describeContents(); + method @NonNull public android.media.AudioAttributes getAudioAttributes(); + method @NonNull public android.media.AudioFormat getAudioFormat(); + method public int getCaptureSession(); + method public int getIgnoreBeginningFrames(); + method public int getMaxAudioLengthSeconds(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator<android.media.musicrecognition.RecognitionRequest> CREATOR; + } + + public static final class RecognitionRequest.Builder { + ctor public RecognitionRequest.Builder(); + method @NonNull public android.media.musicrecognition.RecognitionRequest build(); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setAudioAttributes(@NonNull android.media.AudioAttributes); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setCaptureSession(int); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setIgnoreBeginningFrames(int); + method @NonNull public android.media.musicrecognition.RecognitionRequest.Builder setMaxAudioLengthSeconds(int); + } + +} + package android.media.session { public final class MediaSessionManager { diff --git a/services/Android.bp b/services/Android.bp index ef52c2aff002..d3577efb7fba 100644 --- a/services/Android.bp +++ b/services/Android.bp @@ -28,6 +28,7 @@ filegroup { ":services.coverage-sources", ":services.devicepolicy-sources", ":services.midi-sources", + ":services.musicsearch-sources", ":services.net-sources", ":services.print-sources", ":services.profcollect-sources", @@ -71,6 +72,7 @@ java_library { "services.coverage", "services.devicepolicy", "services.midi", + "services.musicsearch", "services.net", "services.people", "services.print", diff --git a/services/java/com/android/server/SystemServer.java b/services/java/com/android/server/SystemServer.java index 97ae505b4fcf..0b35ba4f0953 100644 --- a/services/java/com/android/server/SystemServer.java +++ b/services/java/com/android/server/SystemServer.java @@ -278,6 +278,8 @@ public final class SystemServer { "com.android.server.autofill.AutofillManagerService"; private static final String CONTENT_CAPTURE_MANAGER_SERVICE_CLASS = "com.android.server.contentcapture.ContentCaptureManagerService"; + private static final String MUSIC_RECOGNITION_MANAGER_SERVICE_CLASS = + "com.android.server.musicrecognition.MusicRecognitionManagerService"; private static final String SYSTEM_CAPTIONS_MANAGER_SERVICE_CLASS = "com.android.server.systemcaptions.SystemCaptionsManagerService"; private static final String TIME_ZONE_RULES_MANAGER_SERVICE_CLASS = @@ -1402,6 +1404,17 @@ public final class SystemServer { t.traceEnd(); } + if (deviceHasConfigString(context, + R.string.config_defaultMusicRecognitionService)) { + t.traceBegin("StartMusicRecognitionManagerService"); + mSystemServiceManager.startService(MUSIC_RECOGNITION_MANAGER_SERVICE_CLASS); + t.traceEnd(); + } else { + Slog.d(TAG, + "MusicRecognitionManagerService not defined by OEM or disabled by flag"); + } + + startContentCaptureService(context, t); startAttentionService(context, t); diff --git a/services/musicrecognition/Android.bp b/services/musicrecognition/Android.bp new file mode 100644 index 000000000000..39b5bb688767 --- /dev/null +++ b/services/musicrecognition/Android.bp @@ -0,0 +1,13 @@ +filegroup { + name: "services.musicsearch-sources", + srcs: ["java/**/*.java"], + path: "java", + visibility: ["//frameworks/base/services"], +} + +java_library_static { + name: "services.musicsearch", + defaults: ["services_defaults"], + srcs: [":services.musicsearch-sources"], + libs: ["services.core", "app-compat-annotations"], +}
\ No newline at end of file diff --git a/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerPerUserService.java b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerPerUserService.java new file mode 100644 index 000000000000..e258ef009fb0 --- /dev/null +++ b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerPerUserService.java @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.musicrecognition; + +import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_KILLED; +import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_UNAVAILABLE; +import static android.media.musicrecognition.MusicRecognitionManager.RecognitionFailureCode; + +import android.Manifest; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.app.AppGlobals; +import android.content.ComponentName; +import android.content.pm.PackageManager; +import android.content.pm.ServiceInfo; +import android.media.AudioRecord; +import android.media.MediaMetadata; +import android.media.musicrecognition.IMusicRecognitionManagerCallback; +import android.media.musicrecognition.IMusicRecognitionServiceCallback; +import android.media.musicrecognition.RecognitionRequest; +import android.os.Bundle; +import android.os.IBinder; +import android.os.ParcelFileDescriptor; +import android.os.RemoteException; +import android.util.Pair; +import android.util.Slog; + +import com.android.internal.annotations.GuardedBy; +import com.android.server.infra.AbstractPerUserSystemService; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Handles per-user requests received by + * {@link MusicRecognitionManagerService}. Opens an audio stream from the + * dsp and writes it into a pipe to {@link RemoteMusicRecognitionService}. + */ +public final class MusicRecognitionManagerPerUserService extends + AbstractPerUserSystemService<MusicRecognitionManagerPerUserService, + MusicRecognitionManagerService> + implements RemoteMusicRecognitionService.Callbacks { + + private static final String TAG = MusicRecognitionManagerPerUserService.class.getSimpleName(); + // Number of bytes per sample of audio (which is a short). + private static final int BYTES_PER_SAMPLE = 2; + private static final int MAX_STREAMING_SECONDS = 24; + + @Nullable + @GuardedBy("mLock") + private RemoteMusicRecognitionService mRemoteService; + + private MusicRecognitionServiceCallback mRemoteServiceCallback = + new MusicRecognitionServiceCallback(); + private IMusicRecognitionManagerCallback mCallback; + + MusicRecognitionManagerPerUserService( + @NonNull MusicRecognitionManagerService primary, + @NonNull Object lock, int userId) { + super(primary, lock, userId); + } + + @NonNull + @GuardedBy("mLock") + @Override + protected ServiceInfo newServiceInfoLocked(@NonNull ComponentName serviceComponent) + throws PackageManager.NameNotFoundException { + ServiceInfo si; + try { + si = AppGlobals.getPackageManager().getServiceInfo(serviceComponent, + PackageManager.GET_META_DATA, mUserId); + } catch (RemoteException e) { + throw new PackageManager.NameNotFoundException( + "Could not get service for " + serviceComponent); + } + if (!Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE.equals(si.permission)) { + Slog.w(TAG, "MusicRecognitionService from '" + si.packageName + + "' does not require permission " + + Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE); + throw new SecurityException("Service does not require permission " + + Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE); + } + // TODO(b/158194857): check process which owns the service has RECORD_AUDIO permission. How? + return si; + } + + @GuardedBy("mLock") + @Nullable + private RemoteMusicRecognitionService ensureRemoteServiceLocked() { + if (mRemoteService == null) { + final String serviceName = getComponentNameLocked(); + if (serviceName == null) { + if (mMaster.verbose) { + Slog.v(TAG, "ensureRemoteServiceLocked(): not set"); + } + return null; + } + ComponentName serviceComponent = ComponentName.unflattenFromString(serviceName); + + mRemoteService = new RemoteMusicRecognitionService(getContext(), + serviceComponent, mUserId, this, + mRemoteServiceCallback, mMaster.isBindInstantServiceAllowed(), mMaster.verbose); + } + + return mRemoteService; + } + + /** + * Read audio from the given capture session using an AudioRecord and writes it to a + * ParcelFileDescriptor. + */ + @GuardedBy("mLock") + public void beginRecognitionLocked( + @NonNull RecognitionRequest recognitionRequest, + @NonNull IBinder callback) { + int maxAudioLengthSeconds = Math.min(recognitionRequest.getMaxAudioLengthSeconds(), + MAX_STREAMING_SECONDS); + mCallback = IMusicRecognitionManagerCallback.Stub.asInterface(callback); + AudioRecord audioRecord = createAudioRecord(recognitionRequest, maxAudioLengthSeconds); + + mRemoteService = ensureRemoteServiceLocked(); + if (mRemoteService == null) { + try { + mCallback.onRecognitionFailed( + RECOGNITION_FAILED_SERVICE_UNAVAILABLE); + } catch (RemoteException e) { + // Ignored. + } + return; + } + + Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); + if (clientPipe == null) { + try { + mCallback.onAudioStreamClosed(); + } catch (RemoteException ignored) { + // Ignored. + } + return; + } + ParcelFileDescriptor audioSink = clientPipe.second; + ParcelFileDescriptor clientRead = clientPipe.first; + + mMaster.mExecutorService.execute(() -> { + try (OutputStream fos = + new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) { + int halfSecondBufferSize = + audioRecord.getBufferSizeInFrames() / maxAudioLengthSeconds; + byte[] byteBuffer = new byte[halfSecondBufferSize]; + int bytesRead = 0; + int totalBytesRead = 0; + int ignoreBytes = + recognitionRequest.getIgnoreBeginningFrames() * BYTES_PER_SAMPLE; + audioRecord.startRecording(); + while (bytesRead >= 0 && totalBytesRead + < audioRecord.getBufferSizeInFrames() * BYTES_PER_SAMPLE) { + bytesRead = audioRecord.read(byteBuffer, 0, byteBuffer.length); + if (bytesRead > 0) { + totalBytesRead += bytesRead; + // If we are ignoring the first x bytes, update that counter. + if (ignoreBytes > 0) { + ignoreBytes -= bytesRead; + // If we've dipped negative, we've skipped through all ignored bytes + // and then some. Write out the bytes we shouldn't have skipped. + if (ignoreBytes < 0) { + fos.write(byteBuffer, bytesRead + ignoreBytes, -ignoreBytes); + } + } else { + fos.write(byteBuffer); + } + } + } + Slog.i(TAG, String.format("Streamed %s bytes from audio record", totalBytesRead)); + } catch (IOException e) { + Slog.e(TAG, "Audio streaming stopped.", e); + } finally { + audioRecord.release(); + try { + mCallback.onAudioStreamClosed(); + } catch (RemoteException ignored) { + // Ignored. + } + + } + }); + // Send the pipe down to the lookup service while we write to it asynchronously. + mRemoteService.writeAudioToPipe(clientRead, recognitionRequest.getAudioFormat()); + } + + /** + * Callback invoked by {@link android.service.musicrecognition.MusicRecognitionService} to pass + * back the music search result. + */ + private final class MusicRecognitionServiceCallback extends + IMusicRecognitionServiceCallback.Stub { + @Override + public void onRecognitionSucceeded(MediaMetadata result, Bundle extras) { + try { + sanitizeBundle(extras); + mCallback.onRecognitionSucceeded(result, extras); + } catch (RemoteException ignored) { + // Ignored. + } + } + + @Override + public void onRecognitionFailed(@RecognitionFailureCode int failureCode) { + try { + mCallback.onRecognitionFailed(failureCode); + } catch (RemoteException ignored) { + // Ignored. + } + } + } + + @Override + public void onServiceDied(@NonNull RemoteMusicRecognitionService service) { + try { + mCallback.onRecognitionFailed(RECOGNITION_FAILED_SERVICE_KILLED); + } catch (RemoteException e) { + // Ignored. + } + Slog.w(TAG, "remote service died: " + service); + } + + /** Establishes an audio stream from the DSP audio source. */ + private static AudioRecord createAudioRecord( + @NonNull RecognitionRequest recognitionRequest, + int maxAudioLengthSeconds) { + int sampleRate = recognitionRequest.getAudioFormat().getSampleRate(); + int bufferSize = getBufferSizeInBytes(sampleRate, maxAudioLengthSeconds); + return new AudioRecord(recognitionRequest.getAudioAttributes(), + recognitionRequest.getAudioFormat(), bufferSize, + recognitionRequest.getCaptureSession()); + } + + /** + * Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at + * {@code sampleRate} Hz, using the format returned by DSP audio capture. + */ + private static int getBufferSizeInBytes(int sampleRate, int bufferLengthSeconds) { + return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds; + } + + private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { + ParcelFileDescriptor[] fileDescriptors; + try { + fileDescriptors = ParcelFileDescriptor.createPipe(); + } catch (IOException e) { + Slog.e(TAG, "Failed to create audio stream pipe", e); + return null; + } + + if (fileDescriptors.length != 2) { + Slog.e(TAG, "Failed to create audio stream pipe, " + + "unexpected number of file descriptors"); + return null; + } + + if (!fileDescriptors[0].getFileDescriptor().valid() + || !fileDescriptors[1].getFileDescriptor().valid()) { + Slog.e(TAG, "Failed to create audio stream pipe, didn't " + + "receive a pair of valid file descriptors."); + return null; + } + + return Pair.create(fileDescriptors[0], fileDescriptors[1]); + } + + /** Removes remote objects from the bundle. */ + private static void sanitizeBundle(@Nullable Bundle bundle) { + if (bundle == null) { + return; + } + + for (String key : bundle.keySet()) { + Object o = bundle.get(key); + + if (o instanceof Bundle) { + sanitizeBundle((Bundle) o); + } else if (o instanceof IBinder || o instanceof ParcelFileDescriptor) { + bundle.remove(key); + } + } + } +} diff --git a/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerService.java b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerService.java new file mode 100644 index 000000000000..b4cb33795878 --- /dev/null +++ b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerService.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.musicrecognition; + +import static android.content.PermissionChecker.PERMISSION_GRANTED; +import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_UNAVAILABLE; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Context; +import android.media.musicrecognition.IMusicRecognitionManager; +import android.media.musicrecognition.IMusicRecognitionManagerCallback; +import android.media.musicrecognition.RecognitionRequest; +import android.os.Binder; +import android.os.IBinder; +import android.os.RemoteException; +import android.os.UserHandle; + +import com.android.server.infra.AbstractMasterSystemService; +import com.android.server.infra.FrameworkResourcesServiceNameResolver; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Service which allows a DSP audio event to be securely streamed to a designated {@link + * MusicRecognitionService}. + */ +public class MusicRecognitionManagerService extends + AbstractMasterSystemService<MusicRecognitionManagerService, + MusicRecognitionManagerPerUserService> { + + private static final String TAG = MusicRecognitionManagerService.class.getSimpleName(); + + private MusicRecognitionManagerStub mMusicRecognitionManagerStub; + final ExecutorService mExecutorService = Executors.newCachedThreadPool(); + + /** + * Initializes the system service. + * + * Subclasses must define a single argument constructor that accepts the context + * and passes it to super. + * + * @param context The system server context. + */ + public MusicRecognitionManagerService(@NonNull Context context) { + super(context, new FrameworkResourcesServiceNameResolver(context, + com.android.internal.R.string.config_defaultMusicRecognitionService), + /** disallowProperty */null); + } + + @Nullable + @Override + protected MusicRecognitionManagerPerUserService newServiceLocked(int resolvedUserId, + boolean disabled) { + return new MusicRecognitionManagerPerUserService(this, mLock, resolvedUserId); + } + + @Override + public void onStart() { + mMusicRecognitionManagerStub = new MusicRecognitionManagerStub(); + publishBinderService(Context.MUSIC_RECOGNITION_SERVICE, mMusicRecognitionManagerStub); + } + + private void enforceCaller(String func) { + Context ctx = getContext(); + if (ctx.checkCallingPermission(android.Manifest.permission.MANAGE_MUSIC_RECOGNITION) + == PERMISSION_GRANTED) { + return; + } + + String msg = "Permission Denial: " + func + " from pid=" + + Binder.getCallingPid() + + ", uid=" + Binder.getCallingUid() + + " doesn't hold " + android.Manifest.permission.MANAGE_MUSIC_RECOGNITION; + throw new SecurityException(msg); + } + + final class MusicRecognitionManagerStub extends IMusicRecognitionManager.Stub { + @Override + public void beginRecognition( + @NonNull RecognitionRequest recognitionRequest, + @NonNull IBinder callback) { + enforceCaller("beginRecognition"); + + synchronized (mLock) { + final MusicRecognitionManagerPerUserService service = getServiceForUserLocked( + UserHandle.getCallingUserId()); + if (service != null) { + service.beginRecognitionLocked(recognitionRequest, callback); + } else { + try { + IMusicRecognitionManagerCallback.Stub.asInterface(callback) + .onRecognitionFailed(RECOGNITION_FAILED_SERVICE_UNAVAILABLE); + } catch (RemoteException e) { + // ignored. + } + } + } + } + } +} diff --git a/services/musicrecognition/java/com/android/server/musicrecognition/RemoteMusicRecognitionService.java b/services/musicrecognition/java/com/android/server/musicrecognition/RemoteMusicRecognitionService.java new file mode 100644 index 000000000000..4814a821d525 --- /dev/null +++ b/services/musicrecognition/java/com/android/server/musicrecognition/RemoteMusicRecognitionService.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.musicrecognition; + +import android.annotation.NonNull; +import android.content.ComponentName; +import android.content.Context; +import android.media.AudioFormat; +import android.media.musicrecognition.IMusicRecognitionService; +import android.media.musicrecognition.IMusicRecognitionServiceCallback; +import android.media.musicrecognition.MusicRecognitionService; +import android.os.IBinder; +import android.os.ParcelFileDescriptor; +import android.text.format.DateUtils; + +import com.android.internal.infra.AbstractMultiplePendingRequestsRemoteService; + +/** Remote connection to an instance of {@link MusicRecognitionService}. */ +public class RemoteMusicRecognitionService extends + AbstractMultiplePendingRequestsRemoteService<RemoteMusicRecognitionService, + IMusicRecognitionService> { + + // Maximum time allotted for the remote service to return a result. Up to 24s of audio plus + // time to fingerprint and make rpcs. + private static final long TIMEOUT_IDLE_BIND_MILLIS = 40 * DateUtils.SECOND_IN_MILLIS; + + // Allows the remote service to send back a result. + private final IMusicRecognitionServiceCallback mServerCallback; + + public RemoteMusicRecognitionService(Context context, ComponentName serviceName, + int userId, MusicRecognitionManagerPerUserService perUserService, + IMusicRecognitionServiceCallback callback, + boolean bindInstantServiceAllowed, boolean verbose) { + super(context, MusicRecognitionService.ACTION_MUSIC_SEARCH_LOOKUP, serviceName, userId, + perUserService, + context.getMainThreadHandler(), + // Prevents the service from having its permissions stripped while in background. + Context.BIND_INCLUDE_CAPABILITIES | (bindInstantServiceAllowed + ? Context.BIND_ALLOW_INSTANT : 0), verbose, + /* initialCapacity= */ 1); + mServerCallback = callback; + } + + @NonNull + @Override + protected IMusicRecognitionService getServiceInterface(@NonNull IBinder service) { + return IMusicRecognitionService.Stub.asInterface(service); + } + + @Override + protected long getTimeoutIdleBindMillis() { + return TIMEOUT_IDLE_BIND_MILLIS; + } + + /** + * Required, but empty since we don't need to notify the callback implementation of the request + * results. + */ + interface Callbacks extends VultureCallback<RemoteMusicRecognitionService> {} + + /** + * Sends the given descriptor to the app's {@link MusicRecognitionService} to read the + * audio. + */ + public void writeAudioToPipe(@NonNull ParcelFileDescriptor fd, + @NonNull AudioFormat audioFormat) { + scheduleAsyncRequest( + binder -> binder.onAudioStreamStarted(fd, audioFormat, mServerCallback)); + } +} |