diff options
| author | 2024-12-10 16:31:09 +0000 | |
|---|---|---|
| committer | 2024-12-12 16:10:52 +0000 | |
| commit | 9f0ff89cc2f442869497d3c305475d5eaf3201be (patch) | |
| tree | c8bd62e783fe3619679bfe48074fbcd18fe0108a /packages/NeuralNetworks | |
| parent | d9f8adde933236154321e99dec18f78ba9b5f2d7 (diff) | |
Fork code changes into platform/ and module/ folders
The platform folder will have all changes reverted to
their state before the refactoring to avoid any new
behavior when flag is off or in intermediate releases
like 25q1.
Flag: build.release_ondevice_intelligence_module
Bug: 376427781
Change-Id: Ie2293837116684da833980f85b49099bfffe44bf
Diffstat (limited to 'packages/NeuralNetworks')
84 files changed, 6276 insertions, 0 deletions
diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/DownloadCallback.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/DownloadCallback.java index 95fb2888a3e9..95fb2888a3e9 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/DownloadCallback.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/DownloadCallback.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/Feature.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/Feature.aidl index 47cfb4a60dc4..47cfb4a60dc4 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/Feature.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/Feature.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/Feature.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/Feature.java index 88f4de2989e4..88f4de2989e4 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/Feature.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/Feature.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/FeatureDetails.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/FeatureDetails.aidl index c5b3532796cd..c5b3532796cd 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/FeatureDetails.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/FeatureDetails.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/FeatureDetails.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/FeatureDetails.java index 063cfb8c321e..063cfb8c321e 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/FeatureDetails.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/FeatureDetails.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ICancellationSignal.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ICancellationSignal.aidl index 1fe201f8f1f8..1fe201f8f1f8 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ICancellationSignal.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ICancellationSignal.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IDownloadCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IDownloadCallback.aidl index 2d7ea1a7b016..2d7ea1a7b016 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IDownloadCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IDownloadCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IFeatureCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IFeatureCallback.aidl index 2e056926e400..2e056926e400 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IFeatureCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IFeatureCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl index 8688028743d7..8688028743d7 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl index 7e5eb57bbc4a..7e5eb57bbc4a 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl index fac5ec6064f8..fac5ec6064f8 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IProcessingSignal.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IProcessingSignal.aidl index 03946eebd40b..03946eebd40b 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IProcessingSignal.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IProcessingSignal.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IRemoteCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IRemoteCallback.aidl index 6f07693dd39c..6f07693dd39c 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IRemoteCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IRemoteCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IResponseCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IResponseCallback.aidl index 270b600e2de5..270b600e2de5 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IResponseCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IResponseCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl index 3e902405f3e0..3e902405f3e0 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl index 958bef0a93e0..958bef0a93e0 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/InferenceInfo.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/InferenceInfo.aidl index 6f6325408979..6f6325408979 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/InferenceInfo.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/InferenceInfo.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/InferenceInfo.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/InferenceInfo.java index cae8db27a435..cae8db27a435 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/InferenceInfo.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/InferenceInfo.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java index 2881c9d217dc..2881c9d217dc 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java index 7d35dd7f2237..7d35dd7f2237 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java index dc0665a5cea7..dc0665a5cea7 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ProcessingCallback.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ProcessingCallback.java index e50d6b1fa97a..e50d6b1fa97a 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ProcessingCallback.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ProcessingCallback.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ProcessingSignal.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ProcessingSignal.java index 733f4fad96f4..733f4fad96f4 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/ProcessingSignal.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/ProcessingSignal.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java index 7ee2af7376ed..7ee2af7376ed 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/TokenInfo.aidl b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/TokenInfo.aidl index 599b337fd20f..599b337fd20f 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/TokenInfo.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/TokenInfo.aidl diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/TokenInfo.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/TokenInfo.java index 035cc4b365b5..035cc4b365b5 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/TokenInfo.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/TokenInfo.java diff --git a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/utils/BinderUtils.java b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/utils/BinderUtils.java index 2916f030e3d0..2916f030e3d0 100644 --- a/packages/NeuralNetworks/framework/java/android/app/ondeviceintelligence/utils/BinderUtils.java +++ b/packages/NeuralNetworks/framework/module/java/android/app/ondeviceintelligence/utils/BinderUtils.java diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl index cba18c1ef36d..cba18c1ef36d 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl index 504fdd9b17f9..504fdd9b17f9 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl index 7ead8690abb4..7ead8690abb4 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl index 32a8a6a70406..32a8a6a70406 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl index 253df890b198..253df890b198 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java index 6907e2bdf2b3..6907e2bdf2b3 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java diff --git a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java index 315dbaf919e5..315dbaf919e5 100644 --- a/packages/NeuralNetworks/framework/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java +++ b/packages/NeuralNetworks/framework/module/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/DownloadCallback.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/DownloadCallback.java new file mode 100644 index 000000000000..95fb2888a3e9 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/DownloadCallback.java @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.os.PersistableBundle; +import android.annotation.IntDef; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Callback functions used for feature downloading via the + * {@link OnDeviceIntelligenceManager#requestFeatureDownload}. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public interface DownloadCallback { + int DOWNLOAD_FAILURE_STATUS_UNKNOWN = 0; + + /** + * Sent when feature download could not succeed due to there being no available disk space on + * the device. + */ + int DOWNLOAD_FAILURE_STATUS_NOT_ENOUGH_DISK_SPACE = 1; + + /** + * Sent when feature download could not succeed due to a network error. + */ + int DOWNLOAD_FAILURE_STATUS_NETWORK_FAILURE = 2; + + /** + * Sent when feature download has been initiated already, hence no need to request download + * again. Caller can query {@link OnDeviceIntelligenceManager#getFeatureDetails} to check if + * download has been completed. + */ + int DOWNLOAD_FAILURE_STATUS_DOWNLOADING = 3; + + /** + * Sent when feature download did not start due to errors (e.g. remote exception of features not + * available). Caller can query {@link OnDeviceIntelligenceManager#getFeatureDetails} to check + * if feature-status is {@link FeatureDetails#FEATURE_STATUS_DOWNLOADABLE}. + */ + int DOWNLOAD_FAILURE_STATUS_UNAVAILABLE = 4; + + /** @hide */ + @IntDef(value = { + DOWNLOAD_FAILURE_STATUS_UNKNOWN, + DOWNLOAD_FAILURE_STATUS_NOT_ENOUGH_DISK_SPACE, + DOWNLOAD_FAILURE_STATUS_NETWORK_FAILURE, + DOWNLOAD_FAILURE_STATUS_DOWNLOADING, + DOWNLOAD_FAILURE_STATUS_UNAVAILABLE + }) + @Retention(RetentionPolicy.SOURCE) + @interface DownloadFailureStatus { + } + + /** + * Called when model download started properly. + * + * @param bytesToDownload the total bytes to be downloaded for this {@link Feature} + */ + default void onDownloadStarted(long bytesToDownload) { + } + + /** + * Called when model download failed. + * + * @param failureStatus the download failure status + * @param errorMessage the error message associated with the download failure + */ + void onDownloadFailed( + @DownloadFailureStatus int failureStatus, + @Nullable String errorMessage, + @NonNull PersistableBundle errorParams); + + /** + * Called when model download is in progress. + * + * @param totalBytesDownloaded the already downloaded bytes for this {@link Feature} + */ + default void onDownloadProgress(long totalBytesDownloaded) { + } + + /** + * Called when model download is completed. The remote implementation can populate any + * associated download params like file stats etc. in this callback to inform the client. + * + * @param downloadParams params containing info about the completed download. + */ + void onDownloadCompleted(@NonNull PersistableBundle downloadParams); +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/Feature.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/Feature.aidl new file mode 100644 index 000000000000..47cfb4a60dc4 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/Feature.aidl @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +/** + * @hide + */ +@JavaOnlyStableParcelable parcelable Feature; diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/Feature.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/Feature.java new file mode 100644 index 000000000000..88f4de2989e4 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/Feature.java @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.os.Parcel; +import android.os.Parcelable; +import android.os.PersistableBundle; + +import java.util.Objects; + +/** + * Represents a typical feature associated with on-device intelligence. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public final class Feature implements Parcelable { + private final int mId; + @Nullable + private final String mName; + @Nullable + private final String mModelName; + private final int mType; + private final int mVariant; + @NonNull + private final PersistableBundle mFeatureParams; + + /* package-private */ Feature( + int id, + @Nullable String name, + @Nullable String modelName, + int type, + int variant, + @NonNull PersistableBundle featureParams) { + this.mId = id; + this.mName = name; + this.mModelName = modelName; + this.mType = type; + this.mVariant = variant; + this.mFeatureParams = Objects.requireNonNull(featureParams, + "featureParams should be non-null."); + } + + /** Returns the unique and immutable identifier of this feature. */ + public int getId() { + return mId; + } + + /** Returns human-readable name of this feature. */ + public @Nullable String getName() { + return mName; + } + + /** Returns base model name of this feature. */ + public @Nullable String getModelName() { + return mModelName; + } + + /** Returns type identifier of this feature. */ + public int getType() { + return mType; + } + + /** Returns variant kind for this feature. */ + public int getVariant() { + return mVariant; + } + + public @NonNull PersistableBundle getFeatureParams() { + return mFeatureParams; + } + + @Override + public String toString() { + return "Feature { " + + "id = " + mId + ", " + + "name = " + mName + ", " + + "modelName = " + mModelName + ", " + + "type = " + mType + ", " + + "variant = " + mVariant + ", " + + "featureParams = " + mFeatureParams + + " }"; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + @SuppressWarnings("unchecked") + Feature that = (Feature) o; + //noinspection PointlessBooleanExpression + return true + && mId == that.mId + && java.util.Objects.equals(mName, that.mName) + && java.util.Objects.equals(mModelName, that.mModelName) + && mType == that.mType + && mVariant == that.mVariant + && java.util.Objects.equals(mFeatureParams, that.mFeatureParams); + } + + @Override + public int hashCode() { + int _hash = 1; + _hash = 31 * _hash + mId; + _hash = 31 * _hash + java.util.Objects.hashCode(mName); + _hash = 31 * _hash + java.util.Objects.hashCode(mModelName); + _hash = 31 * _hash + mType; + _hash = 31 * _hash + mVariant; + _hash = 31 * _hash + java.util.Objects.hashCode(mFeatureParams); + return _hash; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + byte flg = 0; + if (mName != null) flg |= 0x2; + if (mModelName != null) flg |= 0x4; + dest.writeByte(flg); + dest.writeInt(mId); + if (mName != null) dest.writeString(mName); + if (mModelName != null) dest.writeString(mModelName); + dest.writeInt(mType); + dest.writeInt(mVariant); + dest.writeTypedObject(mFeatureParams, flags); + } + + @Override + public int describeContents() { + return 0; + } + + /** @hide */ + @SuppressWarnings({"unchecked", "RedundantCast"}) + /* package-private */ Feature(@NonNull Parcel in) { + byte flg = in.readByte(); + int id = in.readInt(); + String name = (flg & 0x2) == 0 ? null : in.readString(); + String modelName = (flg & 0x4) == 0 ? null : in.readString(); + int type = in.readInt(); + int variant = in.readInt(); + PersistableBundle featureParams = (PersistableBundle) in.readTypedObject( + PersistableBundle.CREATOR); + + this.mId = id; + this.mName = name; + this.mModelName = modelName; + this.mType = type; + this.mVariant = variant; + this.mFeatureParams = featureParams; + } + + public static final @NonNull Parcelable.Creator<Feature> CREATOR + = new Parcelable.Creator<Feature>() { + @Override + public Feature[] newArray(int size) { + return new Feature[size]; + } + + @Override + public Feature createFromParcel(@NonNull Parcel in) { + return new Feature(in); + } + }; + + /** + * A builder for {@link Feature} + */ + @SuppressWarnings("WeakerAccess") + public static final class Builder { + private int mId; + private @Nullable String mName; + private @Nullable String mModelName; + private int mType; + private int mVariant; + private @NonNull PersistableBundle mFeatureParams; + + private long mBuilderFieldsSet = 0L; + + /** + * Provides a builder instance to create a feature for given id. + * + * @param id the unique identifier for the feature. + */ + public Builder(int id) { + mId = id; + mFeatureParams = new PersistableBundle(); + } + + public @NonNull Builder setName(@NonNull String value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x2; + mName = value; + return this; + } + + public @NonNull Builder setModelName(@NonNull String value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x4; + mModelName = value; + return this; + } + + public @NonNull Builder setType(int value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x8; + mType = value; + return this; + } + + public @NonNull Builder setVariant(int value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x10; + mVariant = value; + return this; + } + + public @NonNull Builder setFeatureParams(@NonNull PersistableBundle value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x20; + mFeatureParams = value; + return this; + } + + /** Builds the instance. This builder should not be touched after calling this! */ + public @NonNull Feature build() { + checkNotUsed(); + mBuilderFieldsSet |= 0x40; // Mark builder used + + Feature o = new Feature( + mId, + mName, + mModelName, + mType, + mVariant, + mFeatureParams); + return o; + } + + private void checkNotUsed() { + if ((mBuilderFieldsSet & 0x40) != 0) { + throw new IllegalStateException( + "This Builder should not be reused. Use a new Builder instance instead"); + } + } + } +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/FeatureDetails.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/FeatureDetails.aidl new file mode 100644 index 000000000000..c5b3532796cd --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/FeatureDetails.aidl @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +/** + * @hide + */ +@JavaOnlyStableParcelable parcelable FeatureDetails; diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/FeatureDetails.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/FeatureDetails.java new file mode 100644 index 000000000000..063cfb8c321e --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/FeatureDetails.java @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.Parcelable; +import android.os.PersistableBundle; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.text.MessageFormat; +import java.util.Objects; + +/** + * Represents a status of a requested {@link Feature}. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public final class FeatureDetails implements Parcelable { + @Status + private final int mFeatureStatus; + @NonNull + private final PersistableBundle mFeatureDetailParams; + + /** Invalid or unavailable {@code AiFeature}. */ + public static final int FEATURE_STATUS_UNAVAILABLE = 0; + + /** Feature can be downloaded on request. */ + public static final int FEATURE_STATUS_DOWNLOADABLE = 1; + + /** Feature is being downloaded. */ + public static final int FEATURE_STATUS_DOWNLOADING = 2; + + /** Feature is fully downloaded and ready to use. */ + public static final int FEATURE_STATUS_AVAILABLE = 3; + + /** Underlying service is unavailable and feature status cannot be fetched. */ + public static final int FEATURE_STATUS_SERVICE_UNAVAILABLE = 4; + + /** + * @hide + */ + @IntDef(value = { + FEATURE_STATUS_UNAVAILABLE, + FEATURE_STATUS_DOWNLOADABLE, + FEATURE_STATUS_DOWNLOADING, + FEATURE_STATUS_AVAILABLE, + FEATURE_STATUS_SERVICE_UNAVAILABLE + }) + @Target({ElementType.TYPE_USE, ElementType.METHOD, ElementType.PARAMETER, ElementType.FIELD}) + @Retention(RetentionPolicy.SOURCE) + public @interface Status { + } + + public FeatureDetails( + @Status int featureStatus, + @NonNull PersistableBundle featureDetailParams) { + this.mFeatureStatus = featureStatus; + this.mFeatureDetailParams = Objects.requireNonNull(featureDetailParams); + } + + public FeatureDetails( + @Status int featureStatus) { + this.mFeatureStatus = featureStatus; + this.mFeatureDetailParams = new PersistableBundle(); + } + + + /** + * Returns an integer value associated with the feature status. + */ + public @Status int getFeatureStatus() { + return mFeatureStatus; + } + + + /** + * Returns a persistable bundle contain any additional status related params. + */ + public @NonNull PersistableBundle getFeatureDetailParams() { + return mFeatureDetailParams; + } + + @Override + public String toString() { + return MessageFormat.format("FeatureDetails '{' status = {0}, " + + "persistableBundle = {1} '}'", + mFeatureStatus, + mFeatureDetailParams); + } + + @Override + public boolean equals(@android.annotation.Nullable Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + @SuppressWarnings("unchecked") + FeatureDetails that = (FeatureDetails) o; + return mFeatureStatus == that.mFeatureStatus + && java.util.Objects.equals(mFeatureDetailParams, that.mFeatureDetailParams); + } + + @Override + public int hashCode() { + int _hash = 1; + _hash = 31 * _hash + mFeatureStatus; + _hash = 31 * _hash + java.util.Objects.hashCode(mFeatureDetailParams); + return _hash; + } + + @Override + public void writeToParcel(@NonNull android.os.Parcel dest, int flags) { + dest.writeInt(mFeatureStatus); + dest.writeTypedObject(mFeatureDetailParams, flags); + } + + @Override + public int describeContents() { + return 0; + } + + /** @hide */ + @SuppressWarnings({"unchecked", "RedundantCast"}) + FeatureDetails(@NonNull android.os.Parcel in) { + int status = in.readInt(); + PersistableBundle persistableBundle = (PersistableBundle) in.readTypedObject( + PersistableBundle.CREATOR); + + this.mFeatureStatus = status; + this.mFeatureDetailParams = persistableBundle; + } + + + public static final @NonNull Parcelable.Creator<FeatureDetails> CREATOR = + new Parcelable.Creator<>() { + @Override + public FeatureDetails[] newArray(int size) { + return new FeatureDetails[size]; + } + + @Override + public FeatureDetails createFromParcel(@NonNull android.os.Parcel in) { + return new FeatureDetails(in); + } + }; + +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ICancellationSignal.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ICancellationSignal.aidl new file mode 100644 index 000000000000..1fe201f8f1f8 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ICancellationSignal.aidl @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +/** + * @hide + */ +oneway interface ICancellationSignal { + void cancel(); +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IDownloadCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IDownloadCallback.aidl new file mode 100644 index 000000000000..2d7ea1a7b016 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IDownloadCallback.aidl @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import android.os.PersistableBundle; + +/** + * Interface for Download callback to be passed onto service implementation, + * + * @hide + */ +oneway interface IDownloadCallback { + void onDownloadStarted(long bytesToDownload) = 1; + void onDownloadProgress(long bytesDownloaded) = 2; + void onDownloadFailed(int failureStatus, String errorMessage, in PersistableBundle errorParams) = 3; + void onDownloadCompleted(in PersistableBundle downloadParams) = 4; +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IFeatureCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IFeatureCallback.aidl new file mode 100644 index 000000000000..2e056926e400 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IFeatureCallback.aidl @@ -0,0 +1,14 @@ +package android.app.ondeviceintelligence; + +import android.app.ondeviceintelligence.Feature; +import android.os.PersistableBundle; + +/** + * Interface for receiving a feature for the given identifier. + * + * @hide + */ +oneway interface IFeatureCallback { + void onSuccess(in Feature result) = 1; + void onFailure(int errorCode, in String errorMessage, in PersistableBundle errorParams) = 2; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl new file mode 100644 index 000000000000..8688028743d7 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IFeatureDetailsCallback.aidl @@ -0,0 +1,14 @@ +package android.app.ondeviceintelligence; + +import android.app.ondeviceintelligence.FeatureDetails; +import android.os.PersistableBundle; + +/** + * Interface for receiving details about a given feature. . + * + * @hide + */ +oneway interface IFeatureDetailsCallback { + void onSuccess(in FeatureDetails result) = 1; + void onFailure(int errorCode, in String errorMessage, in PersistableBundle errorParams) = 2; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl new file mode 100644 index 000000000000..7e5eb57bbc4a --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IListFeaturesCallback.aidl @@ -0,0 +1,15 @@ +package android.app.ondeviceintelligence; + +import java.util.List; +import android.app.ondeviceintelligence.Feature; +import android.os.PersistableBundle; + +/** + * Interface for receiving list of supported features. + * + * @hide + */ +oneway interface IListFeaturesCallback { + void onSuccess(in List<Feature> result) = 1; + void onFailure(int errorCode, in String errorMessage, in PersistableBundle errorParams) = 2; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl new file mode 100644 index 000000000000..fac5ec6064f8 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IOnDeviceIntelligenceManager.aidl @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package android.app.ondeviceintelligence; + + import com.android.modules.utils.AndroidFuture; + import android.app.ondeviceintelligence.ICancellationSignal; + import android.os.ParcelFileDescriptor; + import android.os.PersistableBundle; + import android.os.RemoteCallback; + import android.os.Bundle; + import android.app.ondeviceintelligence.Feature; + import android.app.ondeviceintelligence.FeatureDetails; + import android.app.ondeviceintelligence.InferenceInfo; + import java.util.List; + import android.app.ondeviceintelligence.IDownloadCallback; + import android.app.ondeviceintelligence.IListFeaturesCallback; + import android.app.ondeviceintelligence.IFeatureCallback; + import android.app.ondeviceintelligence.IFeatureDetailsCallback; + import android.app.ondeviceintelligence.IResponseCallback; + import android.app.ondeviceintelligence.IStreamingResponseCallback; + import android.app.ondeviceintelligence.IProcessingSignal; + import android.app.ondeviceintelligence.ITokenInfoCallback; + + + /** + * Interface for a OnDeviceIntelligenceManager for managing OnDeviceIntelligenceService and OnDeviceSandboxedInferenceService. + * + * @hide + */ +interface IOnDeviceIntelligenceManager { + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void getVersion(in RemoteCallback remoteCallback) = 1; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void getFeature(in int featureId, in IFeatureCallback remoteCallback) = 2; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void listFeatures(in IListFeaturesCallback listFeaturesCallback) = 3; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void getFeatureDetails(in Feature feature, in IFeatureDetailsCallback featureDetailsCallback) = 4; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void requestFeatureDownload(in Feature feature, in AndroidFuture cancellationSignalFuture, in IDownloadCallback callback) = 5; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void requestTokenInfo(in Feature feature, in Bundle requestBundle, in AndroidFuture cancellationSignalFuture, + in ITokenInfoCallback tokenInfocallback) = 6; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void processRequest(in Feature feature, in Bundle requestBundle, int requestType, + in AndroidFuture cancellationSignalFuture, + in AndroidFuture processingSignalFuture, + in IResponseCallback responseCallback) = 7; + + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.USE_ON_DEVICE_INTELLIGENCE)") + void processRequestStreaming(in Feature feature, + in Bundle requestBundle, int requestType, in AndroidFuture cancellationSignalFuture, + in AndroidFuture processingSignalFuture, + in IStreamingResponseCallback streamingCallback) = 8; + + String getRemoteServicePackageName() = 9; + + List<InferenceInfo> getLatestInferenceInfo(long startTimeEpochMillis) = 10; + } diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IProcessingSignal.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IProcessingSignal.aidl new file mode 100644 index 000000000000..03946eebd40b --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IProcessingSignal.aidl @@ -0,0 +1,14 @@ +package android.app.ondeviceintelligence; + +import android.os.PersistableBundle; + +/** +* Signal to provide to the remote implementation in context of a given request or +* feature specific event. +* +* @hide +*/ + +oneway interface IProcessingSignal { + void sendSignal(in PersistableBundle actionParams) = 2; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IRemoteCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IRemoteCallback.aidl new file mode 100644 index 000000000000..6f07693dd39c --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IRemoteCallback.aidl @@ -0,0 +1,24 @@ +/* +* Copyright 2024, The Android Open Source Project +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package android.app.ondeviceintelligence; + +import android.os.Bundle; + +/* @hide */ +oneway interface IRemoteCallback { + void sendResult(in Bundle data); +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IResponseCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IResponseCallback.aidl new file mode 100644 index 000000000000..270b600e2de5 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IResponseCallback.aidl @@ -0,0 +1,16 @@ +package android.app.ondeviceintelligence; + +import android.os.PersistableBundle; +import android.os.Bundle; +import android.os.RemoteCallback; + +/** + * Interface for a IResponseCallback for receiving response from on-device intelligence service. + * + * @hide + */ +oneway interface IResponseCallback { + void onSuccess(in Bundle resultBundle) = 1; + void onFailure(int errorCode, in String errorMessage, in PersistableBundle errorParams) = 2; + void onDataAugmentRequest(in Bundle processedContent, in RemoteCallback responseCallback) = 3; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl new file mode 100644 index 000000000000..3e902405f3e0 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/IStreamingResponseCallback.aidl @@ -0,0 +1,18 @@ +package android.app.ondeviceintelligence; + +import android.os.PersistableBundle; +import android.os.RemoteCallback; +import android.os.Bundle; + + +/** + * This callback is a streaming variant of {@link IResponseCallback}. + * + * @hide + */ +oneway interface IStreamingResponseCallback { + void onNewContent(in Bundle processedResult) = 1; + void onSuccess(in Bundle result) = 2; + void onFailure(int errorCode, in String errorMessage, in PersistableBundle errorParams) = 3; + void onDataAugmentRequest(in Bundle processedContent, in RemoteCallback responseCallback) = 4; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl new file mode 100644 index 000000000000..958bef0a93e0 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ITokenInfoCallback.aidl @@ -0,0 +1,14 @@ +package android.app.ondeviceintelligence; + +import android.os.PersistableBundle; +import android.app.ondeviceintelligence.TokenInfo; + +/** + * Interface for receiving the token info of a request for a given feature. + * + * @hide + */ +oneway interface ITokenInfoCallback { + void onSuccess(in TokenInfo tokenInfo) = 1; + void onFailure(int errorCode, in String errorMessage, in PersistableBundle errorParams) = 2; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/InferenceInfo.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/InferenceInfo.aidl new file mode 100644 index 000000000000..6f6325408979 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/InferenceInfo.aidl @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +/** + * @hide + */ +@JavaOnlyStableParcelable parcelable InferenceInfo; diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/InferenceInfo.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/InferenceInfo.java new file mode 100644 index 000000000000..cae8db27a435 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/InferenceInfo.java @@ -0,0 +1,220 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE_MODULE; + +import android.annotation.CurrentTimeMillisLong; +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.Parcel; +import android.os.Parcelable; + +/** + * This class represents the information related to an inference event to track the resource usage + * as a function of inference time. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE_MODULE) +public final class InferenceInfo implements Parcelable { + + /** + * Uid for the caller app. + */ + private final int uid; + + /** + * Inference start time (milliseconds from the epoch time). + */ + private final long startTimeMs; + + /** + * Inference end time (milliseconds from the epoch time). + */ + private final long endTimeMs; + + /** + * Suspended time in milliseconds. + */ + private final long suspendedTimeMs; + + /** + * Constructs an InferenceInfo object with the specified parameters. + * + * @param uid Uid for the caller app. + * @param startTimeMs Inference start time (milliseconds from the epoch time). + * @param endTimeMs Inference end time (milliseconds from the epoch time). + * @param suspendedTimeMs Suspended time in milliseconds. + */ + InferenceInfo(int uid, long startTimeMs, long endTimeMs, + long suspendedTimeMs) { + this.uid = uid; + this.startTimeMs = startTimeMs; + this.endTimeMs = endTimeMs; + this.suspendedTimeMs = suspendedTimeMs; + } + + /** + * Constructs an InferenceInfo object from a Parcel. + * + * @param in The Parcel to read the object's data from. + */ + private InferenceInfo(@NonNull Parcel in) { + uid = in.readInt(); + startTimeMs = in.readLong(); + endTimeMs = in.readLong(); + suspendedTimeMs = in.readLong(); + } + + + /** + * Writes the object's data to the provided Parcel. + * + * @param dest The Parcel to write the object's data to. + * @param flags Additional flags about how the object should be written. + */ + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + dest.writeInt(uid); + dest.writeLong(startTimeMs); + dest.writeLong(endTimeMs); + dest.writeLong(suspendedTimeMs); + } + + /** + * Returns the UID for the caller app. + * + * @return the UID for the caller app. + */ + public int getUid() { + return uid; + } + + /** + * Returns the inference start time in milliseconds from the epoch time. + * + * @return the inference start time in milliseconds from the epoch time. + */ + @CurrentTimeMillisLong + public long getStartTimeMillis() { + return startTimeMs; + } + + /** + * Returns the inference end time in milliseconds from the epoch time. + * + * @return the inference end time in milliseconds from the epoch time. + */ + @CurrentTimeMillisLong + public long getEndTimeMillis() { + return endTimeMs; + } + + /** + * Returns the suspended time in milliseconds. + * + * @return the suspended time in milliseconds. + */ + @CurrentTimeMillisLong + public long getSuspendedTimeMillis() { + return suspendedTimeMs; + } + + @Override + public int describeContents() { + return 0; + } + + + public static final @android.annotation.NonNull Parcelable.Creator<InferenceInfo> CREATOR + = new Parcelable.Creator<InferenceInfo>() { + @Override + public InferenceInfo[] newArray(int size) { + return new InferenceInfo[size]; + } + + @Override + public InferenceInfo createFromParcel(@android.annotation.NonNull Parcel in) { + return new InferenceInfo(in); + } + }; + + /** + * Builder class for creating instances of {@link InferenceInfo}. + */ + public static final class Builder { + private final int uid; + private long startTimeMs; + private long endTimeMs; + private long suspendedTimeMs; + + /** + * Provides a builder instance to create a InferenceInfo for given caller uid. + * + * @param uid the caller uid associated with the inference info. + */ + public Builder(int uid) { + this.uid = uid; + } + + /** + * Sets the inference start time in milliseconds from the epoch time. + * + * @param startTimeMs the inference start time in milliseconds from the epoch time. + * @return the Builder instance. + */ + public @NonNull Builder setStartTimeMillis(@CurrentTimeMillisLong long startTimeMs) { + this.startTimeMs = startTimeMs; + return this; + } + + /** + * Sets the inference end time in milliseconds from the epoch time. + * + * @param endTimeMs the inference end time in milliseconds from the epoch time. + * @return the Builder instance. + */ + public @NonNull Builder setEndTimeMillis(@CurrentTimeMillisLong long endTimeMs) { + this.endTimeMs = endTimeMs; + return this; + } + + /** + * Sets the suspended time in milliseconds. + * + * @param suspendedTimeMs the suspended time in milliseconds. + * @return the Builder instance. + */ + public @NonNull Builder setSuspendedTimeMillis(@CurrentTimeMillisLong long suspendedTimeMs) { + this.suspendedTimeMs = suspendedTimeMs; + return this; + } + + /** + * Builds and returns an instance of {@link InferenceInfo}. + * + * @return an instance of {@link InferenceInfo}. + */ + public @NonNull InferenceInfo build() { + return new InferenceInfo(uid, startTimeMs, endTimeMs, + suspendedTimeMs); + } + } +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java new file mode 100644 index 000000000000..2881c9d217dc --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceException.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.PersistableBundle; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Exception type to be used for errors related to on-device intelligence system service with + * appropriate error code. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public class OnDeviceIntelligenceException extends Exception { + + public static final int PROCESSING_ERROR_UNKNOWN = 1; + + /** Request passed contains bad data for e.g. format. */ + public static final int PROCESSING_ERROR_BAD_DATA = 2; + + /** Bad request for inputs. */ + public static final int PROCESSING_ERROR_BAD_REQUEST = 3; + + /** Whole request was classified as not safe, and no response will be generated. */ + public static final int PROCESSING_ERROR_REQUEST_NOT_SAFE = 4; + + /** Underlying processing encountered an error and failed to compute results. */ + public static final int PROCESSING_ERROR_COMPUTE_ERROR = 5; + + /** Encountered an error while performing IPC */ + public static final int PROCESSING_ERROR_IPC_ERROR = 6; + + /** Request was cancelled either by user signal or by the underlying implementation. */ + public static final int PROCESSING_ERROR_CANCELLED = 7; + + /** Underlying processing in the remote implementation is not available. */ + public static final int PROCESSING_ERROR_NOT_AVAILABLE = 8; + + /** The service is currently busy. Callers should retry with exponential backoff. */ + public static final int PROCESSING_ERROR_BUSY = 9; + + /** Something went wrong with safety classification service. */ + public static final int PROCESSING_ERROR_SAFETY_ERROR = 10; + + /** Response generated was classified unsafe. */ + public static final int PROCESSING_ERROR_RESPONSE_NOT_SAFE = 11; + + /** Request is too large to be processed. */ + public static final int PROCESSING_ERROR_REQUEST_TOO_LARGE = 12; + + /** Inference suspended so that higher-priority inference can run. */ + public static final int PROCESSING_ERROR_SUSPENDED = 13; + + /** + * Underlying processing encountered an internal error, like a violated precondition + * . + */ + public static final int PROCESSING_ERROR_INTERNAL = 14; + + /** + * The processing was not able to be passed on to the remote implementation, as the + * service + * was unavailable. + */ + public static final int PROCESSING_ERROR_SERVICE_UNAVAILABLE = 15; + /** + * Error code returned when the OnDeviceIntelligenceManager service is unavailable. + */ + public static final int ON_DEVICE_INTELLIGENCE_SERVICE_UNAVAILABLE = 100; + + /** + * The connection to remote service failed and the processing state could not be updated. + */ + public static final int PROCESSING_UPDATE_STATUS_CONNECTION_FAILED = 200; + + + /** + * Error code associated with the on-device intelligence failure. + * + * @hide + */ + @IntDef( + value = { + PROCESSING_ERROR_UNKNOWN, + PROCESSING_ERROR_BAD_DATA, + PROCESSING_ERROR_BAD_REQUEST, + PROCESSING_ERROR_REQUEST_NOT_SAFE, + PROCESSING_ERROR_COMPUTE_ERROR, + PROCESSING_ERROR_IPC_ERROR, + PROCESSING_ERROR_CANCELLED, + PROCESSING_ERROR_NOT_AVAILABLE, + PROCESSING_ERROR_BUSY, + PROCESSING_ERROR_SAFETY_ERROR, + PROCESSING_ERROR_RESPONSE_NOT_SAFE, + PROCESSING_ERROR_REQUEST_TOO_LARGE, + PROCESSING_ERROR_SUSPENDED, + PROCESSING_ERROR_INTERNAL, + PROCESSING_ERROR_SERVICE_UNAVAILABLE, + ON_DEVICE_INTELLIGENCE_SERVICE_UNAVAILABLE, + PROCESSING_UPDATE_STATUS_CONNECTION_FAILED + }) + @Target({ElementType.TYPE_PARAMETER, ElementType.TYPE_USE}) + @Retention(RetentionPolicy.SOURCE) + @interface OnDeviceIntelligenceError { + } + + private final int mErrorCode; + private final PersistableBundle mErrorParams; + + /** Returns the error code of the exception. */ + public int getErrorCode() { + return mErrorCode; + } + + /** Returns the error params of the exception. */ + @NonNull + public PersistableBundle getErrorParams() { + return mErrorParams; + } + + /** + * Creates a new OnDeviceIntelligenceException with the specified error code, error message and + * error params. + * + * @param errorCode The error code. + * @param errorMessage The error message. + * @param errorParams The error params. + */ + public OnDeviceIntelligenceException( + @OnDeviceIntelligenceError int errorCode, @NonNull String errorMessage, + @NonNull PersistableBundle errorParams) { + super(errorMessage); + this.mErrorCode = errorCode; + this.mErrorParams = errorParams; + } + + /** + * Creates a new OnDeviceIntelligenceException with the specified error code and error params. + * + * @param errorCode The error code. + * @param errorParams The error params. + */ + public OnDeviceIntelligenceException( + @OnDeviceIntelligenceError int errorCode, + @NonNull PersistableBundle errorParams) { + this.mErrorCode = errorCode; + this.mErrorParams = errorParams; + } + + /** + * Creates a new OnDeviceIntelligenceException with the specified error code and error message. + * + * @param errorCode The error code. + * @param errorMessage The error message. + */ + public OnDeviceIntelligenceException( + @OnDeviceIntelligenceError int errorCode, @NonNull String errorMessage) { + super(errorMessage); + this.mErrorCode = errorCode; + this.mErrorParams = new PersistableBundle(); + } + + /** + * Creates a new OnDeviceIntelligenceException with the specified error code. + * + * @param errorCode The error code. + */ + public OnDeviceIntelligenceException( + @OnDeviceIntelligenceError int errorCode) { + this.mErrorCode = errorCode; + this.mErrorParams = new PersistableBundle(); + } +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java new file mode 100644 index 000000000000..7d35dd7f2237 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceFrameworkInitializer.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.SystemApi; +import android.app.SystemServiceRegistry; +import android.content.Context; + +/** + * Class for performing registration for OnDeviceIntelligence service. + * + * @hide + */ +@SystemApi(client = SystemApi.Client.MODULE_LIBRARIES) +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public class OnDeviceIntelligenceFrameworkInitializer { + private OnDeviceIntelligenceFrameworkInitializer() { + } + + /** + * Called by {@link SystemServiceRegistry}'s static initializer and registers + * OnDeviceIntelligence service to {@link Context}, so that {@link Context#getSystemService} can + * return them. + * + * @throws IllegalStateException if this is called from anywhere besides {@link + * SystemServiceRegistry} + */ + public static void registerServiceWrappers() { + SystemServiceRegistry.registerContextAwareService(Context.ON_DEVICE_INTELLIGENCE_SERVICE, + OnDeviceIntelligenceManager.class, + (context, serviceBinder) -> { + IOnDeviceIntelligenceManager manager = + IOnDeviceIntelligenceManager.Stub.asInterface(serviceBinder); + return new OnDeviceIntelligenceManager(context, manager); + }); + } +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java new file mode 100644 index 000000000000..dc0665a5cea7 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/OnDeviceIntelligenceManager.java @@ -0,0 +1,677 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE_MODULE; + +import android.Manifest; +import android.annotation.CallbackExecutor; +import android.annotation.CurrentTimeMillisLong; +import android.annotation.FlaggedApi; +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.annotation.SystemApi; +import android.annotation.SystemService; +import android.app.ondeviceintelligence.utils.BinderUtils; +import android.content.Context; +import android.graphics.Bitmap; +import android.os.Bundle; +import android.os.CancellationSignal; +import android.os.IBinder; +import android.os.OutcomeReceiver; +import android.os.PersistableBundle; +import android.os.RemoteCallback; +import android.os.RemoteException; +import android.system.OsConstants; +import android.util.Log; + +import com.android.modules.utils.AndroidFuture; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.function.LongConsumer; + +/** + * Allows granted apps to manage on-device intelligence service configured on the device. Typical + * calling pattern will be to query and setup a required feature before proceeding to request + * processing. + * + * The contracts in this Manager class are designed to be open-ended in general, to allow + * interoperability. Therefore, it is recommended that implementations of this system-service + * expose this API to the clients via a separate sdk or library which has more defined contract. + * + * @hide + */ +@SystemApi +@SystemService(Context.ON_DEVICE_INTELLIGENCE_SERVICE) +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public final class OnDeviceIntelligenceManager { + /** + * @hide + */ + public static final String API_VERSION_BUNDLE_KEY = "ApiVersionBundleKey"; + + /** + * @hide + */ + public static final String AUGMENT_REQUEST_CONTENT_BUNDLE_KEY = + "AugmentRequestContentBundleKey"; + + /** + * Timeout to be used for unbinding to the configured remote {@link + * android.service.ondeviceintelligence.OnDeviceIntelligenceService} if there are no requests in + * the queue. A value of -1 represents to never unbind. + * + * @hide + */ + public static final String ON_DEVICE_INTELLIGENCE_UNBIND_TIMEOUT_MS = + "on_device_intelligence_unbind_timeout_ms"; + + /** + * Timeout that represents maximum idle time before which a callback should be populated. + * + * @hide + */ + public static final String ON_DEVICE_INTELLIGENCE_IDLE_TIMEOUT_MS = + "on_device_intelligence_idle_timeout_ms"; + + /** + * Timeout to be used for unbinding to the configured remote {@link + * android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService} if there are no + * requests in the queue. A value of -1 represents to never unbind. + * + * @hide + */ + public static final String ON_DEVICE_INFERENCE_UNBIND_TIMEOUT_MS = + "on_device_inference_unbind_timeout_ms"; + + private static final String TAG = "OnDeviceIntelligence"; + private final Context mContext; + private final IOnDeviceIntelligenceManager mService; + + + /** + * @hide + */ + public OnDeviceIntelligenceManager(Context context, IOnDeviceIntelligenceManager service) { + mContext = context; + mService = service; + } + + /** + * Asynchronously get the version of the underlying remote implementation. + * + * @param versionConsumer consumer to populate the version of remote implementation. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void getVersion( + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull LongConsumer versionConsumer) { + try { + RemoteCallback callback = new RemoteCallback(result -> { + if (result == null) { + BinderUtils.withCleanCallingIdentity( + () -> callbackExecutor.execute(() -> versionConsumer.accept(0))); + } + long version = result.getLong(API_VERSION_BUNDLE_KEY); + BinderUtils.withCleanCallingIdentity( + () -> callbackExecutor.execute(() -> versionConsumer.accept(version))); + }); + mService.getVersion(callback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + + /** + * Get package name configured for providing the remote implementation for this system service. + */ + @Nullable + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public String getRemoteServicePackageName() { + String result; + try { + result = mService.getRemoteServicePackageName(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + return result; + } + + /** + * Asynchronously get feature for a given id. + * + * @param featureId the identifier pointing to the feature. + * @param featureReceiver callback to populate the feature object for given identifier. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void getFeature( + int featureId, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull OutcomeReceiver<Feature, OnDeviceIntelligenceException> featureReceiver) { + try { + IFeatureCallback callback = + new IFeatureCallback.Stub() { + @Override + public void onSuccess(Feature result) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> featureReceiver.onResult(result))); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> featureReceiver.onError( + new OnDeviceIntelligenceException( + errorCode, errorMessage, errorParams)))); + } + }; + mService.getFeature(featureId, callback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Asynchronously get a list of features that are supported for the caller. + * + * @param featureListReceiver callback to populate the list of features. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void listFeatures( + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull OutcomeReceiver<List<Feature>, OnDeviceIntelligenceException> featureListReceiver) { + try { + IListFeaturesCallback callback = + new IListFeaturesCallback.Stub() { + @Override + public void onSuccess(List<Feature> result) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> featureListReceiver.onResult(result))); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> featureListReceiver.onError( + new OnDeviceIntelligenceException( + errorCode, errorMessage, errorParams)))); + } + }; + mService.listFeatures(callback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * This method should be used to fetch details about a feature which need some additional + * computation, that can be inefficient to return in all calls to {@link #getFeature}. Callers + * and implementation can utilize the {@link Feature#getFeatureParams()} to pass hint on what + * details are expected by the caller. + * + * @param feature the feature to check status for. + * @param featureDetailsReceiver callback to populate the feature details to. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void getFeatureDetails(@NonNull Feature feature, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull OutcomeReceiver<FeatureDetails, OnDeviceIntelligenceException> featureDetailsReceiver) { + try { + IFeatureDetailsCallback callback = new IFeatureDetailsCallback.Stub() { + + @Override + public void onSuccess(FeatureDetails result) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> featureDetailsReceiver.onResult(result))); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> featureDetailsReceiver.onError( + new OnDeviceIntelligenceException(errorCode, + errorMessage, errorParams)))); + } + }; + mService.getFeatureDetails(feature, callback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * This method handles downloading all model and config files required to process requests + * sent against a given feature. The caller can listen to updates on the download status via + * the callback. + * + * Note: If a feature was already requested for downloaded previously, the onDownloadFailed + * callback would be invoked with {@link DownloadCallback#DOWNLOAD_FAILURE_STATUS_DOWNLOADING}. + * In such cases, clients should query the feature status via {@link #getFeatureDetails} to + * check on the feature's download status. + * + * @param feature feature to request download for. + * @param callback callback to populate updates about download status. + * @param cancellationSignal signal to invoke cancellation on the operation in the remote + * implementation. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void requestFeatureDownload(@NonNull Feature feature, + @Nullable CancellationSignal cancellationSignal, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull DownloadCallback callback) { + try { + IDownloadCallback downloadCallback = new IDownloadCallback.Stub() { + + @Override + public void onDownloadStarted(long bytesToDownload) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> callback.onDownloadStarted(bytesToDownload))); + } + + @Override + public void onDownloadProgress(long bytesDownloaded) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> callback.onDownloadProgress(bytesDownloaded))); + } + + @Override + public void onDownloadFailed(int failureStatus, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> callback.onDownloadFailed(failureStatus, errorMessage, + errorParams))); + } + + @Override + public void onDownloadCompleted(PersistableBundle downloadParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> callback.onDownloadCompleted(downloadParams))); + } + }; + + mService.requestFeatureDownload(feature, + configureRemoteCancellationFuture(cancellationSignal, callbackExecutor), + downloadCallback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + + /** + * The methods computes the token related information for a given request payload using the + * provided {@link Feature}. + * + * @param feature feature associated with the request. + * @param request request and associated params represented by the Bundle + * data. + * @param outcomeReceiver callback to populate the token info or exception in case of + * failure. + * @param cancellationSignal signal to invoke cancellation on the operation in the remote + * implementation. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void requestTokenInfo(@NonNull Feature feature, @NonNull @InferenceParams Bundle request, + @Nullable CancellationSignal cancellationSignal, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull OutcomeReceiver<TokenInfo, + OnDeviceIntelligenceException> outcomeReceiver) { + try { + ITokenInfoCallback callback = new ITokenInfoCallback.Stub() { + @Override + public void onSuccess(TokenInfo tokenInfo) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> outcomeReceiver.onResult(tokenInfo))); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> outcomeReceiver.onError( + new OnDeviceIntelligenceException( + errorCode, errorMessage, errorParams)))); + } + }; + + mService.requestTokenInfo(feature, request, + configureRemoteCancellationFuture(cancellationSignal, callbackExecutor), + callback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + + /** + * Asynchronously Process a request based on the associated params, to populate a + * response in + * {@link OutcomeReceiver#onResult} callback or failure callback status code if there + * was a + * failure. + * + * @param feature feature associated with the request. + * @param request request and associated params represented by the Bundle + * data. + * @param requestType type of request being sent for processing the content. + * @param cancellationSignal signal to invoke cancellation. + * @param processingSignal signal to send custom signals in the + * remote implementation. + * @param callbackExecutor executor to run the callback on. + * @param processingCallback callback to populate the response content and + * associated params. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void processRequest(@NonNull Feature feature, @NonNull @InferenceParams Bundle request, + @RequestType int requestType, + @Nullable CancellationSignal cancellationSignal, + @Nullable ProcessingSignal processingSignal, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull ProcessingCallback processingCallback) { + try { + IResponseCallback callback = new IResponseCallback.Stub() { + @Override + public void onSuccess(@InferenceParams Bundle result) { + BinderUtils.withCleanCallingIdentity(() -> { + callbackExecutor.execute(() -> processingCallback.onResult(result)); + }); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> processingCallback.onError( + new OnDeviceIntelligenceException( + errorCode, errorMessage, errorParams)))); + } + + @Override + public void onDataAugmentRequest(@NonNull @InferenceParams Bundle request, + @NonNull RemoteCallback contentCallback) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> processingCallback.onDataAugmentRequest(request, result -> { + Bundle bundle = new Bundle(); + bundle.putParcelable(AUGMENT_REQUEST_CONTENT_BUNDLE_KEY, result); + callbackExecutor.execute(() -> contentCallback.sendResult(bundle)); + }))); + } + }; + + + mService.processRequest(feature, request, requestType, + configureRemoteCancellationFuture(cancellationSignal, callbackExecutor), + configureRemoteProcessingSignalFuture(processingSignal, callbackExecutor), + callback); + + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Variation of {@link #processRequest} that asynchronously processes a request in a + * streaming + * fashion, where new content is pushed to caller in chunks via the + * {@link StreamingProcessingCallback#onPartialResult}. After the streaming is complete, + * the service should call {@link StreamingProcessingCallback#onResult} and can optionally + * populate the complete the full response {@link Bundle} as part of the callback in cases + * when the final response contains an enhanced aggregation of the contents already + * streamed. + * + * @param feature feature associated with the request. + * @param request request and associated params represented by the Bundle + * data. + * @param requestType type of request being sent for processing the content. + * @param cancellationSignal signal to invoke cancellation. + * @param processingSignal signal to send custom signals in the + * remote implementation. + * @param streamingProcessingCallback streaming callback to populate the response content and + * associated params. + * @param callbackExecutor executor to run the callback on. + */ + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void processRequestStreaming(@NonNull Feature feature, + @NonNull @InferenceParams Bundle request, + @RequestType int requestType, + @Nullable CancellationSignal cancellationSignal, + @Nullable ProcessingSignal processingSignal, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull StreamingProcessingCallback streamingProcessingCallback) { + try { + IStreamingResponseCallback callback = new IStreamingResponseCallback.Stub() { + @Override + public void onNewContent(@InferenceParams Bundle result) { + BinderUtils.withCleanCallingIdentity(() -> { + callbackExecutor.execute( + () -> streamingProcessingCallback.onPartialResult(result)); + }); + } + + @Override + public void onSuccess(@InferenceParams Bundle result) { + BinderUtils.withCleanCallingIdentity(() -> { + callbackExecutor.execute( + () -> streamingProcessingCallback.onResult(result)); + }); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) { + BinderUtils.withCleanCallingIdentity(() -> { + callbackExecutor.execute( + () -> streamingProcessingCallback.onError( + new OnDeviceIntelligenceException( + errorCode, errorMessage, errorParams))); + }); + } + + + @Override + public void onDataAugmentRequest(@NonNull @InferenceParams Bundle content, + @NonNull RemoteCallback contentCallback) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> streamingProcessingCallback.onDataAugmentRequest(content, + contentResponse -> { + Bundle bundle = new Bundle(); + bundle.putParcelable(AUGMENT_REQUEST_CONTENT_BUNDLE_KEY, + contentResponse); + callbackExecutor.execute( + () -> contentCallback.sendResult(bundle)); + }))); + } + }; + + mService.processRequestStreaming( + feature, request, requestType, + configureRemoteCancellationFuture(cancellationSignal, callbackExecutor), + configureRemoteProcessingSignalFuture(processingSignal, callbackExecutor), + callback); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * This is primarily intended to be used to attribute/blame on-device intelligence power usage, + * via the configured remote implementation, to its actual caller. + * + * @param startTimeEpochMillis epoch millis used to filter the InferenceInfo events. + * @return InferenceInfo events since the passed in startTimeEpochMillis. + */ + @RequiresPermission(Manifest.permission.DUMP) + @FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE_MODULE) + public @NonNull List<InferenceInfo> getLatestInferenceInfo(@CurrentTimeMillisLong long startTimeEpochMillis) { + try { + return mService.getLatestInferenceInfo(startTimeEpochMillis); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + + /** Request inference with provided Bundle and Params. */ + public static final int REQUEST_TYPE_INFERENCE = 0; + + /** + * Prepares the remote implementation environment for e.g.loading inference runtime etc + * .which + * are time consuming beforehand to remove overhead and allow quick processing of requests + * thereof. + */ + public static final int REQUEST_TYPE_PREPARE = 1; + + /** Request Embeddings of the passed-in Bundle. */ + public static final int REQUEST_TYPE_EMBEDDINGS = 2; + + /** + * @hide + */ + @IntDef(value = { + REQUEST_TYPE_INFERENCE, + REQUEST_TYPE_PREPARE, + REQUEST_TYPE_EMBEDDINGS + }) + @Target({ElementType.TYPE_USE, ElementType.METHOD, ElementType.PARAMETER, + ElementType.FIELD}) + @Retention(RetentionPolicy.SOURCE) + public @interface RequestType { + } + + /** + * {@link Bundle}s annotated with this type will be validated that they are in-effect read-only + * when passed via Binder IPC. Following restrictions apply : + * <ul> + * <li> {@link PersistableBundle}s are allowed.</li> + * <li> Any primitive types or their collections can be added as usual.</li> + * <li>IBinder objects should *not* be added.</li> + * <li>Parcelable data which has no active-objects, should be added as + * {@link Bundle#putByteArray}</li> + * <li>Parcelables have active-objects, only following types will be allowed</li> + * <ul> + * <li>{@link android.os.ParcelFileDescriptor} opened in + * {@link android.os.ParcelFileDescriptor#MODE_READ_ONLY}</li> + * </ul> + * </ul> + * + * In all other scenarios the system-server might throw a + * {@link android.os.BadParcelableException} if the Bundle validation fails. + * + * @hide + */ + @Target({ElementType.PARAMETER, ElementType.FIELD}) + public @interface StateParams { + } + + /** + * This is an extension of {@link StateParams} but for purpose of inference few other types are + * also allowed as read-only, as listed below. + * + * <li>{@link Bitmap} set as immutable.</li> + * <li>{@link android.database.CursorWindow}</li> + * <li>{@link android.os.SharedMemory} set to {@link OsConstants#PROT_READ}</li> + * </ul> + * </ul> + * + * In all other scenarios the system-server might throw a + * {@link android.os.BadParcelableException} if the Bundle validation fails. + * + * @hide + */ + @Target({ElementType.PARAMETER, ElementType.FIELD, ElementType.TYPE_USE}) + public @interface InferenceParams { + } + + /** + * This is an extension of {@link StateParams} with the exception that it allows writing + * {@link Bitmap} as part of the response. + * + * In all other scenarios the system-server might throw a + * {@link android.os.BadParcelableException} if the Bundle validation fails. + * + * @hide + */ + @Target({ElementType.PARAMETER, ElementType.FIELD}) + public @interface ResponseParams { + } + + @Nullable + private static AndroidFuture<IBinder> configureRemoteCancellationFuture( + @Nullable CancellationSignal cancellationSignal, + @NonNull Executor callbackExecutor) { + if (cancellationSignal == null) { + return null; + } + AndroidFuture<IBinder> cancellationFuture = new AndroidFuture<>(); + cancellationFuture.whenCompleteAsync( + (cancellationTransport, error) -> { + if (error != null || cancellationTransport == null) { + Log.e(TAG, "Unable to receive the remote cancellation signal.", error); + } else { + ICancellationSignal remoteCancellationSignal = + ICancellationSignal.Stub.asInterface(cancellationTransport); + cancellationSignal.setOnCancelListener( + () -> { + try { + remoteCancellationSignal.cancel(); + } catch (RemoteException e) { + Log.w(TAG, "Unable to propagate cancellation signal.", + e); + } + }); + } + }, callbackExecutor); + return cancellationFuture; + } + + @Nullable + private static AndroidFuture<IBinder> configureRemoteProcessingSignalFuture( + ProcessingSignal processingSignal, Executor executor) { + if (processingSignal == null) { + return null; + } + AndroidFuture<IBinder> processingSignalFuture = new AndroidFuture<>(); + processingSignalFuture.whenCompleteAsync( + (transport, error) -> { + if (error != null || transport == null) { + Log.e(TAG, "Unable to receive the remote processing signal.", error); + } else { + processingSignal.setRemote(IProcessingSignal.Stub.asInterface(transport)); + } + }, executor); + return processingSignalFuture; + } +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ProcessingCallback.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ProcessingCallback.java new file mode 100644 index 000000000000..e50d6b1fa97a --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ProcessingCallback.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.Bundle; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.InferenceParams; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.ResponseParams; + +import java.util.function.Consumer; + +/** + * Callback to populate the processed response or any error that occurred during the + * request processing. This callback also provides a method to request additional data to be + * augmented to the request-processing, using the partial response that was already + * processed in the remote implementation. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public interface ProcessingCallback { + /** + * Invoked when request has been processed and result is ready to be propagated to the + * caller. + * + * @param result Response to be passed as a result. + */ + void onResult(@NonNull @ResponseParams Bundle result); + + /** + * Called when the request processing fails. The failure details are indicated by the + * {@link OnDeviceIntelligenceException} passed as an argument to this method. + * + * @param error An exception with more details about the error that occurred. + */ + void onError(@NonNull OnDeviceIntelligenceException error); + + /** + * Callback to be invoked in cases where the remote service needs to perform retrieval or + * transformation operations based on a partially processed request, in order to augment the + * final response, by using the additional context sent via this callback. + * + * @param processedContent The content payload that should be used to augment ongoing request. + * @param contentConsumer The augmentation data that should be sent to remote + * service for further processing a request. Bundle passed in here is + * expected to be non-null or EMPTY when there is no response. + */ + default void onDataAugmentRequest( + @NonNull @ResponseParams Bundle processedContent, + @NonNull Consumer<@InferenceParams Bundle> contentConsumer) { + contentConsumer.accept(Bundle.EMPTY); + } +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ProcessingSignal.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ProcessingSignal.java new file mode 100644 index 000000000000..733f4fad96f4 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/ProcessingSignal.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.CallbackExecutor; +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.os.PersistableBundle; +import android.os.RemoteException; + +import com.android.internal.annotations.GuardedBy; + +import java.util.ArrayDeque; +import java.util.Objects; +import java.util.concurrent.Executor; + +/** + * A signal to perform orchestration actions on the inference and optionally receive a output about + * the result of the signal. This is an extension of {@link android.os.CancellationSignal}. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public final class ProcessingSignal { + private final Object mLock = new Object(); + + private static final int MAX_QUEUE_SIZE = 10; + + @GuardedBy("mLock") + private final ArrayDeque<PersistableBundle> mActionParamsQueue; + + @GuardedBy("mLock") + private IProcessingSignal mRemote; + + private OnProcessingSignalCallback mOnProcessingSignalCallback; + private Executor mExecutor; + + public ProcessingSignal() { + mActionParamsQueue = new ArrayDeque<>(MAX_QUEUE_SIZE); + } + + /** + * Interface definition for a callback to be invoked when processing signals are received. + */ + public interface OnProcessingSignalCallback { + /** + * Called when a custom signal was received. + * This method allows the receiver to provide logic to be executed based on the signal + * received. + * + * @param actionParams Parameters for the signal in the form of a {@link PersistableBundle}. + */ + + void onSignalReceived(@NonNull PersistableBundle actionParams); + } + + + /** + * Sends a custom signal with the provided parameters. If there are multiple concurrent + * requests to this method, the actionParams are queued in a blocking fashion, in the order they + * are received. + * + * It also signals the remote callback + * with the same params if already configured, if not the action is queued to be sent when a + * remote is configured. Similarly, on the receiver side, the callback will be invoked if + * already set, if not all actions are queued to be sent to callback when it is set. + * + * @param actionParams Parameters for the signal. + */ + public void sendSignal(@NonNull PersistableBundle actionParams) { + final OnProcessingSignalCallback callback; + final IProcessingSignal remote; + synchronized (mLock) { + if (mActionParamsQueue.size() > MAX_QUEUE_SIZE) { + throw new RuntimeException( + "Maximum actions that can be queued are : " + MAX_QUEUE_SIZE); + } + + mActionParamsQueue.add(actionParams); + callback = mOnProcessingSignalCallback; + remote = mRemote; + + if (callback != null) { + while (!mActionParamsQueue.isEmpty()) { + PersistableBundle params = mActionParamsQueue.removeFirst(); + mExecutor.execute( + () -> callback.onSignalReceived(params)); + } + } + if (remote != null) { + while (!mActionParamsQueue.isEmpty()) { + try { + remote.sendSignal(mActionParamsQueue.removeFirst()); + } catch (RemoteException e) { + throw new RuntimeException(e); + } + } + } + } + } + + + /** + * Sets the processing signal callback to be called when signals are received. + * + * This method is intended to be used by the recipient of a processing signal + * such as the remote implementation in + * {@link android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService} to handle + * processing signals while performing a long-running operation. This method is not + * intended to be used by the caller themselves. + * + * If {@link ProcessingSignal#sendSignal} has already been called, then the provided callback + * is invoked immediately and all previously queued actions are passed to remote signal. + * + * This method is guaranteed that the callback will not be called after it + * has been removed. + * + * @param callback The processing signal callback, or null to remove the current callback. + * @param executor Executor to the run the callback methods on. + */ + public void setOnProcessingSignalCallback( + @NonNull @CallbackExecutor Executor executor, + @Nullable OnProcessingSignalCallback callback) { + Objects.requireNonNull(executor); + synchronized (mLock) { + if (mOnProcessingSignalCallback == callback) { + return; + } + + mOnProcessingSignalCallback = callback; + mExecutor = executor; + if (callback == null || mActionParamsQueue.isEmpty()) { + return; + } + + while (!mActionParamsQueue.isEmpty()) { + PersistableBundle params = mActionParamsQueue.removeFirst(); + mExecutor.execute(() -> callback.onSignalReceived(params)); + } + } + } + + /** + * Sets the remote transport. + * + * If there are actions queued from {@link ProcessingSignal#sendSignal}, they are also + * sequentially sent to the configured remote. + * + * This method guarantees that the remote transport will not be called after it + * has been removed. + * + * @param remote The remote transport, or null to remove. + * @hide + */ + void setRemote(IProcessingSignal remote) { + synchronized (mLock) { + mRemote = remote; + if (mActionParamsQueue.isEmpty() || remote == null) { + return; + } + + while (!mActionParamsQueue.isEmpty()) { + try { + remote.sendSignal(mActionParamsQueue.removeFirst()); + } catch (RemoteException e) { + throw new RuntimeException("Failed to send action to remote signal", e); + } + } + } + } + + /** + * Creates a transport that can be returned back to the caller of + * a Binder function and subsequently used to dispatch a processing signal. + * + * @return The new processing signal transport. + * @hide + */ + public static IProcessingSignal createTransport() { + return new Transport(); + } + + /** + * Given a locally created transport, returns its associated processing signal. + * + * @param transport The locally created transport, or null if none. + * @return The associated processing signal, or null if none. + * @hide + */ + public static ProcessingSignal fromTransport(IProcessingSignal transport) { + if (transport instanceof Transport) { + return ((Transport) transport).mProcessingSignal; + } + return null; + } + + private static final class Transport extends IProcessingSignal.Stub { + final ProcessingSignal mProcessingSignal = new ProcessingSignal(); + + @Override + public void sendSignal(PersistableBundle actionParams) { + mProcessingSignal.sendSignal(actionParams); + } + } + +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java new file mode 100644 index 000000000000..7ee2af7376ed --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/StreamingProcessingCallback.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.Bundle; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.ResponseParams; + +/** + * Streaming variant of {@link ProcessingCallback} to populate response while processing a given + * request, possibly in chunks to provide a async processing behaviour to the caller. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public interface StreamingProcessingCallback extends ProcessingCallback { + /** + * Callback that would be invoked when a part of the response i.e. some response is + * already processed, and needs to be passed onto the caller. + */ + void onPartialResult(@NonNull @ResponseParams Bundle partialResult); +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/TokenInfo.aidl b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/TokenInfo.aidl new file mode 100644 index 000000000000..599b337fd20f --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/TokenInfo.aidl @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +/** + * @hide + */ +@JavaOnlyStableParcelable parcelable TokenInfo; diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/TokenInfo.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/TokenInfo.java new file mode 100644 index 000000000000..035cc4b365b5 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/TokenInfo.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.SystemApi; +import android.os.Parcel; +import android.os.Parcelable; +import android.os.PersistableBundle; + +/** + * This class is used to provide a token count response for the + * {@link OnDeviceIntelligenceManager#requestTokenInfo} outcome receiver. + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public final class TokenInfo implements Parcelable { + private final long mCount; + private final PersistableBundle mInfoParams; + + /** + * Construct a token count using the count value and associated params. + */ + public TokenInfo(long count, @NonNull PersistableBundle persistableBundle) { + this.mCount = count; + mInfoParams = persistableBundle; + } + + /** + * Construct a token count using the count value. + */ + public TokenInfo(long count) { + this.mCount = count; + this.mInfoParams = new PersistableBundle(); + } + + /** + * Returns the token count associated with a request payload. + */ + public long getCount() { + return mCount; + } + + /** + * Returns the params representing token info. + */ + @NonNull + public PersistableBundle getInfoParams() { + return mInfoParams; + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + dest.writeLong(mCount); + dest.writePersistableBundle(mInfoParams); + } + + public static final @NonNull Parcelable.Creator<TokenInfo> CREATOR + = new Parcelable.Creator<>() { + @Override + public TokenInfo[] newArray(int size) { + return new TokenInfo[size]; + } + + @Override + public TokenInfo createFromParcel(@NonNull Parcel in) { + return new TokenInfo(in.readLong(), in.readPersistableBundle()); + } + }; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/utils/BinderUtils.java b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/utils/BinderUtils.java new file mode 100644 index 000000000000..2916f030e3d0 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/app/ondeviceintelligence/utils/BinderUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.app.ondeviceintelligence.utils; + +import android.annotation.NonNull; +import android.os.Binder; + +import java.util.function.Supplier; + +/** + * Collection of utilities for {@link Binder} and related classes. + * @hide + */ +public class BinderUtils { + /** + * Convenience method for running the provided action enclosed in + * {@link Binder#clearCallingIdentity}/{@link Binder#restoreCallingIdentity} + * + * Any exception thrown by the given action will be caught and rethrown after the call to + * {@link Binder#restoreCallingIdentity} + * + * Note that this is copied from Binder#withCleanCallingIdentity with minor changes + * since it is not public. + * + * @hide + */ + public static final <T extends Exception> void withCleanCallingIdentity( + @NonNull ThrowingRunnable<T> action) throws T { + final long callingIdentity = Binder.clearCallingIdentity(); + try { + action.run(); + } finally { + Binder.restoreCallingIdentity(callingIdentity); + } + } + + /** + * Like a Runnable, but declared to throw an exception. + * + * @param <T> The exception class which is declared to be thrown. + */ + @FunctionalInterface + public interface ThrowingRunnable<T extends Exception> { + /** @see java.lang.Runnable */ + void run() throws T; + } + + /** + * Convenience method for running the provided action enclosed in + * {@link Binder#clearCallingIdentity}/{@link Binder#restoreCallingIdentity} returning the + * result. + * + * <p>Any exception thrown by the given action will be caught and rethrown after + * the call to {@link Binder#restoreCallingIdentity}. + * + * Note that this is copied from Binder#withCleanCallingIdentity with minor changes + * since it is not public. + * + * @hide + */ + public static final <T, E extends Exception> T withCleanCallingIdentity( + @NonNull ThrowingSupplier<T, E> action) throws E { + final long callingIdentity = Binder.clearCallingIdentity(); + try { + return action.get(); + } finally { + Binder.restoreCallingIdentity(callingIdentity); + } + } + + /** + * An equivalent of {@link Supplier} + * + * @param <T> The class which is declared to be returned. + * @param <E> The exception class which is declared to be thrown. + */ + @FunctionalInterface + public interface ThrowingSupplier<T, E extends Exception> { + /** @see java.util.function.Supplier */ + T get() throws E; + } +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl new file mode 100644 index 000000000000..cba18c1ef36d --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IOnDeviceIntelligenceService.aidl @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.ondeviceintelligence; + +import android.os.PersistableBundle; +import android.os.ParcelFileDescriptor; +import android.app.ondeviceintelligence.ICancellationSignal; +import android.os.RemoteCallback; +import android.app.ondeviceintelligence.IDownloadCallback; +import android.app.ondeviceintelligence.Feature; +import android.app.ondeviceintelligence.IFeatureCallback; +import android.app.ondeviceintelligence.IListFeaturesCallback; +import android.app.ondeviceintelligence.IFeatureDetailsCallback; +import com.android.modules.utils.AndroidFuture; +import android.service.ondeviceintelligence.IRemoteProcessingService; + + +/** + * Interface for a concrete implementation to provide on device intelligence services. + * + * @hide + */ +oneway interface IOnDeviceIntelligenceService { + void getVersion(in RemoteCallback remoteCallback); + void getFeature(int callerUid, int featureId, in IFeatureCallback featureCallback); + void listFeatures(int callerUid, in IListFeaturesCallback listFeaturesCallback); + void getFeatureDetails(int callerUid, in Feature feature, in IFeatureDetailsCallback featureDetailsCallback); + void getReadOnlyFileDescriptor(in String fileName, in AndroidFuture<ParcelFileDescriptor> future); + void getReadOnlyFeatureFileDescriptorMap(in Feature feature, in RemoteCallback remoteCallback); + void requestFeatureDownload(int callerUid, in Feature feature, + in AndroidFuture cancellationSignal, + in IDownloadCallback downloadCallback); + void registerRemoteServices(in IRemoteProcessingService remoteProcessingService); + void notifyInferenceServiceConnected(); + void notifyInferenceServiceDisconnected(); + void ready(); +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl new file mode 100644 index 000000000000..504fdd9b17f9 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IOnDeviceSandboxedInferenceService.aidl @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.ondeviceintelligence; + +import android.app.ondeviceintelligence.IStreamingResponseCallback; +import android.app.ondeviceintelligence.IResponseCallback; +import android.app.ondeviceintelligence.ITokenInfoCallback; +import android.app.ondeviceintelligence.IProcessingSignal; +import android.app.ondeviceintelligence.Feature; +import android.app.ondeviceintelligence.IRemoteCallback; +import android.app.ondeviceintelligence.ICancellationSignal; +import android.os.PersistableBundle; +import android.os.Bundle; +import com.android.modules.utils.AndroidFuture; +import android.service.ondeviceintelligence.IRemoteStorageService; +import android.service.ondeviceintelligence.IProcessingUpdateStatusCallback; + +/** + * Interface for a concrete implementation to provide on-device sandboxed inference. + * + * @hide + */ +oneway interface IOnDeviceSandboxedInferenceService { + void registerRemoteStorageService(in IRemoteStorageService storageService, + in IRemoteCallback remoteCallback) = 0; + void requestTokenInfo(int callerUid, in Feature feature, in Bundle request, + in AndroidFuture cancellationSignal, + in ITokenInfoCallback tokenInfoCallback) = 1; + void processRequest(int callerUid, in Feature feature, in Bundle request, in int requestType, + in AndroidFuture cancellationSignal, + in AndroidFuture processingSignal, + in IResponseCallback callback) = 2; + void processRequestStreaming(int callerUid, in Feature feature, in Bundle request, in int requestType, + in AndroidFuture cancellationSignal, + in AndroidFuture processingSignal, + in IStreamingResponseCallback callback) = 3; + void updateProcessingState(in Bundle processingState, + in IProcessingUpdateStatusCallback callback) = 4; +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl new file mode 100644 index 000000000000..7ead8690abb4 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IProcessingUpdateStatusCallback.aidl @@ -0,0 +1,14 @@ +package android.service.ondeviceintelligence; + +import android.os.PersistableBundle; + +/** + * Interface for receiving status from a updateProcessingState call from on-device intelligence + * service. + * + * @hide + */ +interface IProcessingUpdateStatusCallback { + void onSuccess(in PersistableBundle statusParams) = 1; + void onFailure(int errorCode, in String errorMessage) = 2; +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl new file mode 100644 index 000000000000..32a8a6a70406 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IRemoteProcessingService.aidl @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.ondeviceintelligence; + +import android.os.Bundle; +import android.service.ondeviceintelligence.IProcessingUpdateStatusCallback; + + +/** + * Interface for a concrete implementation to provide methods to update state of a remote service. + * + * @hide + */ +oneway interface IRemoteProcessingService { + void updateProcessingState(in Bundle processingState, + in IProcessingUpdateStatusCallback callback); +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl new file mode 100644 index 000000000000..253df890b198 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/IRemoteStorageService.aidl @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.ondeviceintelligence; + +import android.app.ondeviceintelligence.Feature; +import android.os.ParcelFileDescriptor; +import android.os.RemoteCallback; + +import com.android.modules.utils.AndroidFuture; + +/** + * Interface for a concrete implementation to provide access to storage read access + * for the isolated process. + * + * @hide + */ +oneway interface IRemoteStorageService { + void getReadOnlyFileDescriptor(in String filePath, in AndroidFuture<ParcelFileDescriptor> future); + void getReadOnlyFeatureFileDescriptorMap(in Feature feature, in RemoteCallback remoteCallback); +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java new file mode 100644 index 000000000000..6907e2bdf2b3 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/OnDeviceIntelligenceService.java @@ -0,0 +1,676 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.CallSuper; +import android.annotation.CallbackExecutor; +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SdkConstant; +import android.annotation.SystemApi; +import android.app.Service; +import android.app.ondeviceintelligence.DownloadCallback; +import android.app.ondeviceintelligence.Feature; +import android.app.ondeviceintelligence.FeatureDetails; +import android.app.ondeviceintelligence.ICancellationSignal; +import android.app.ondeviceintelligence.IDownloadCallback; +import android.app.ondeviceintelligence.IFeatureCallback; +import android.app.ondeviceintelligence.IFeatureDetailsCallback; +import android.app.ondeviceintelligence.IListFeaturesCallback; +import android.app.ondeviceintelligence.OnDeviceIntelligenceException; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.StateParams; +import android.app.ondeviceintelligence.utils.BinderUtils; +import android.content.Intent; +import android.os.Bundle; +import android.os.CancellationSignal; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.Message; +import android.os.OutcomeReceiver; +import android.os.ParcelFileDescriptor; +import android.os.PersistableBundle; +import android.os.RemoteCallback; +import android.os.RemoteException; +import android.util.Log; +import android.util.Slog; + +import com.android.modules.utils.AndroidFuture; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.function.Consumer; +import java.util.function.LongConsumer; + +/** + * Abstract base class for performing setup for on-device inference and providing file access to + * the isolated counter part {@link OnDeviceSandboxedInferenceService}. + * + * <p> A service that provides configuration and model files relevant to performing inference on + * device. The system's default OnDeviceIntelligenceService implementation is configured in + * {@code config_defaultOnDeviceIntelligenceService}. If this config has no value, a stub is + * returned. + * + * <p> Similar to {@link OnDeviceIntelligenceManager} class, the contracts in this service are + * defined to be open-ended in general, to allow interoperability. Therefore, it is recommended + * that implementations of this system-service expose this API to the clients via a library which + * has more defined contract.</p> + * <pre> + * {@literal + * <service android:name=".SampleOnDeviceIntelligenceService" + * android:permission="android.permission.BIND_ON_DEVICE_INTELLIGENCE_SERVICE"> + * </service>} + * </pre> + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public abstract class OnDeviceIntelligenceService extends Service { + private static final int MSG_ON_READY = 1; + private static final int MSG_GET_VERSION = 2; + private static final int MSG_LIST_FEATURES = 3; + private static final int MSG_GET_FEATURE = 4; + private static final int MSG_GET_FEATURE_DETAILS = 5; + private static final int MSG_DOWNLOAD_FEATURE = 6; + private static final int MSG_GET_READ_ONLY_FILE_DESCRIPTOR = 7; + private static final int MSG_GET_READ_ONLY_FEATURE_FILE_DESCRIPTOR_MAP = 8; + private static final int MSG_REGISTER_REMOTE_SERVICES = 9; + private static final int MSG_INFERENCE_SERVICE_CONNECTED = 10; + private static final int MSG_INFERENCE_SERVICE_DISCONNECTED = 11; + + private static final String TAG = OnDeviceIntelligenceService.class.getSimpleName(); + + private volatile IRemoteProcessingService mRemoteProcessingService; + private Handler mHandler; + + @CallSuper + @Override + public void onCreate() { + super.onCreate(); + mHandler = new Handler(Looper.getMainLooper()) { + @Override + public void handleMessage(@NonNull Message msg) { + switch (msg.what) { + case MSG_ON_READY: + OnDeviceIntelligenceService.this.onReady(); + break; + case MSG_GET_VERSION: + OnDeviceIntelligenceService.this.onGetVersion( + (LongConsumer) msg.obj); + break; + case MSG_LIST_FEATURES: + OnDeviceIntelligenceService.this.onListFeatures( + msg.arg1, + (OutcomeReceiver<List<Feature>, OnDeviceIntelligenceException>) msg.obj); + break; + case MSG_GET_FEATURE: + GetFeatureParams params = (GetFeatureParams) msg.obj; + OnDeviceIntelligenceService.this.onGetFeature( + msg.arg1, + msg.arg2, + params.callback); + break; + case MSG_GET_FEATURE_DETAILS: + FeatureDetailsParams detailsParams = (FeatureDetailsParams) msg.obj; + OnDeviceIntelligenceService.this.onGetFeatureDetails( + msg.arg1, + detailsParams.feature, + detailsParams.callback); + break; + case MSG_DOWNLOAD_FEATURE: + DownloadParams downloadParams = (DownloadParams) msg.obj; + OnDeviceIntelligenceService.this.onDownloadFeature( + msg.arg1, + downloadParams.feature, + downloadParams.cancellationSignal, + downloadParams.callback); + break; + case MSG_GET_READ_ONLY_FILE_DESCRIPTOR: + FileDescriptorParams fdParams = (FileDescriptorParams) msg.obj; + OnDeviceIntelligenceService.this.onGetReadOnlyFileDescriptor( + fdParams.fileName, + fdParams.future); + break; + case MSG_GET_READ_ONLY_FEATURE_FILE_DESCRIPTOR_MAP: + FeatureFileDescriptorParams ffdParams = + (FeatureFileDescriptorParams) msg.obj; + OnDeviceIntelligenceService.this.onGetReadOnlyFeatureFileDescriptorMap( + ffdParams.feature, + ffdParams.consumer); + break; + case MSG_REGISTER_REMOTE_SERVICES: + mRemoteProcessingService = (IRemoteProcessingService) msg.obj; + break; + case MSG_INFERENCE_SERVICE_CONNECTED: + OnDeviceIntelligenceService.this.onInferenceServiceConnected(); + break; + case MSG_INFERENCE_SERVICE_DISCONNECTED: + OnDeviceIntelligenceService.this.onInferenceServiceDisconnected(); + break; + } + } + }; + } + + + /** + * @hide + */ + @Nullable + @Override + public final IBinder onBind(@NonNull Intent intent) { + if (SERVICE_INTERFACE.equals(intent.getAction())) { + return new IOnDeviceIntelligenceService.Stub() { + /** {@inheritDoc} */ + @Override + public void ready() { + mHandler.sendEmptyMessage(MSG_ON_READY); + } + + @Override + public void getVersion(RemoteCallback remoteCallback) { + Objects.requireNonNull(remoteCallback); + Message msg = Message.obtain(mHandler, MSG_GET_VERSION, + (LongConsumer) (l -> { + Bundle b = new Bundle(); + b.putLong(OnDeviceIntelligenceManager.API_VERSION_BUNDLE_KEY, l); + remoteCallback.sendResult(b); + })); + mHandler.sendMessage(msg); + } + + @Override + public void listFeatures(int callerUid, + IListFeaturesCallback listFeaturesCallback) { + Objects.requireNonNull(listFeaturesCallback); + Message msg = Message.obtain(mHandler, MSG_LIST_FEATURES, + callerUid, 0, wrapListFeaturesCallback(listFeaturesCallback)); + mHandler.sendMessage(msg); + } + + @Override + public void getFeature(int callerUid, int id, IFeatureCallback featureCallback) { + Objects.requireNonNull(featureCallback); + Message msg = Message.obtain(mHandler, MSG_GET_FEATURE, + callerUid, id, + new GetFeatureParams(wrapFeatureCallback(featureCallback))); + mHandler.sendMessage(msg); + } + + + @Override + public void getFeatureDetails(int callerUid, Feature feature, + IFeatureDetailsCallback featureDetailsCallback) { + Objects.requireNonNull(feature); + Objects.requireNonNull(featureDetailsCallback); + Message msg = Message.obtain(mHandler, MSG_GET_FEATURE_DETAILS, + new FeatureDetailsParams(feature, + wrapFeatureDetailsCallback(featureDetailsCallback))); + msg.arg1 = callerUid; + mHandler.sendMessage(msg); + } + + @Override + public void requestFeatureDownload(int callerUid, Feature feature, + AndroidFuture cancellationSignalFuture, + IDownloadCallback downloadCallback) { + Objects.requireNonNull(feature); + Objects.requireNonNull(downloadCallback); + + CancellationSignal cancellationSignal = new CancellationSignal(); + if (cancellationSignalFuture != null) { + ICancellationSignal transport = new ICancellationSignal.Stub() { + @Override + public void cancel() { + cancellationSignal.cancel(); + } + }; + cancellationSignalFuture.complete(transport); + } + + Message msg = Message.obtain(mHandler, MSG_DOWNLOAD_FEATURE, + new DownloadParams(feature, + cancellationSignalFuture != null ? cancellationSignal : null, + wrapDownloadCallback(downloadCallback))); + msg.arg1 = callerUid; + mHandler.sendMessage(msg); + } + + @Override + public void getReadOnlyFileDescriptor(String fileName, + AndroidFuture<ParcelFileDescriptor> future) { + Objects.requireNonNull(fileName); + Objects.requireNonNull(future); + Message msg = Message.obtain(mHandler, MSG_GET_READ_ONLY_FILE_DESCRIPTOR, + new FileDescriptorParams(fileName, future)); + mHandler.sendMessage(msg); + } + + @Override + public void getReadOnlyFeatureFileDescriptorMap( + Feature feature, RemoteCallback remoteCallback) { + Objects.requireNonNull(feature); + Objects.requireNonNull(remoteCallback); + Message msg = Message.obtain(mHandler, + MSG_GET_READ_ONLY_FEATURE_FILE_DESCRIPTOR_MAP, + new FeatureFileDescriptorParams(feature, parcelFileDescriptorMap -> { + Bundle bundle = new Bundle(); + parcelFileDescriptorMap.forEach(bundle::putParcelable); + remoteCallback.sendResult(bundle); + tryClosePfds(parcelFileDescriptorMap.values()); + })); + mHandler.sendMessage(msg); + } + + @Override + public void registerRemoteServices( + IRemoteProcessingService remoteProcessingService) { + mRemoteProcessingService = remoteProcessingService; + } + + @Override + public void notifyInferenceServiceConnected() { + mHandler.sendEmptyMessage(MSG_INFERENCE_SERVICE_CONNECTED); + } + + @Override + public void notifyInferenceServiceDisconnected() { + mHandler.sendEmptyMessage(MSG_INFERENCE_SERVICE_DISCONNECTED); + } + }; + } + Slog.w(TAG, "Incorrect service interface, returning null."); + return null; + } + + /** + * The {@link Intent} that must be declared as handled by the service. To be supported, the + * service must also require the + * {@link android.Manifest.permission#BIND_ON_DEVICE_INTELLIGENCE_SERVICE} + * permission so that other applications can not abuse it. + */ + @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION) + public static final String SERVICE_INTERFACE = + "android.service.ondeviceintelligence.OnDeviceIntelligenceService"; + + // Parameter holder classes + private static class GetFeatureParams { + final OutcomeReceiver<Feature, OnDeviceIntelligenceException> callback; + + GetFeatureParams(OutcomeReceiver<Feature, OnDeviceIntelligenceException> callback) { + this.callback = callback; + } + } + + private static class FeatureDetailsParams { + final Feature feature; + final OutcomeReceiver<FeatureDetails, OnDeviceIntelligenceException> callback; + + FeatureDetailsParams(Feature feature, + OutcomeReceiver<FeatureDetails, OnDeviceIntelligenceException> callback) { + this.feature = feature; + this.callback = callback; + } + } + + private static class DownloadParams { + final Feature feature; + final CancellationSignal cancellationSignal; + final DownloadCallback callback; + + DownloadParams(Feature feature, CancellationSignal cancellationSignal, + DownloadCallback callback) { + this.feature = feature; + this.cancellationSignal = cancellationSignal; + this.callback = callback; + } + } + + private static class FileDescriptorParams { + final String fileName; + final AndroidFuture<ParcelFileDescriptor> future; + + FileDescriptorParams(String fileName, AndroidFuture<ParcelFileDescriptor> future) { + this.fileName = fileName; + this.future = future; + } + } + + private static class FeatureFileDescriptorParams { + final Feature feature; + final Consumer<Map<String, ParcelFileDescriptor>> consumer; + + FeatureFileDescriptorParams(Feature feature, + Consumer<Map<String, ParcelFileDescriptor>> consumer) { + this.feature = feature; + this.consumer = consumer; + } + } + + /** + * Using this signal to assertively a signal each time service binds successfully, used only in + * tests to get a signal that service instance is ready. This is needed because we cannot rely + * on {@link #onCreate} or {@link #onBind} to be invoke on each binding. + * + * @hide + */ + @SystemApi + public void onReady() { + } + + + /** + * Invoked when a new instance of the remote inference service is created. + * This method should be used as a signal to perform any initialization operations, for e.g. by + * invoking the {@link #updateProcessingState} method to initialize the remote processing + * service. + */ + public abstract void onInferenceServiceConnected(); + + + /** + * Invoked when an instance of the remote inference service is disconnected. + */ + public abstract void onInferenceServiceDisconnected(); + + + /** + * Invoked by the {@link OnDeviceIntelligenceService} inorder to send updates to the inference + * service if there is a state change to be performed. State change could be config updates, + * performing initialization or cleanup tasks in the remote inference service. + * The Bundle passed in here is expected to be read-only and will be rejected if it has any + * writable fields as detailed under {@link StateParams}. + * + * @param processingState the updated state to be applied. + * @param callbackExecutor executor to the run status callback on. + * @param statusReceiver receiver to get status of the update state operation. + */ + public final void updateProcessingState(@NonNull @StateParams Bundle processingState, + @NonNull @CallbackExecutor Executor callbackExecutor, + @NonNull OutcomeReceiver<PersistableBundle, OnDeviceIntelligenceException> statusReceiver) { + Objects.requireNonNull(callbackExecutor); + if (mRemoteProcessingService == null) { + throw new IllegalStateException("Remote processing service is unavailable."); + } + try { + mRemoteProcessingService.updateProcessingState(processingState, + new IProcessingUpdateStatusCallback.Stub() { + @Override + public void onSuccess(PersistableBundle result) { + BinderUtils.withCleanCallingIdentity(() -> { + callbackExecutor.execute( + () -> statusReceiver.onResult(result)); + }); + } + + @Override + public void onFailure(int errorCode, String errorMessage) { + BinderUtils.withCleanCallingIdentity(() -> callbackExecutor.execute( + () -> statusReceiver.onError( + new OnDeviceIntelligenceException( + errorCode, errorMessage)))); + } + }); + } catch (RemoteException e) { + Slog.e(TAG, "Error in updateProcessingState: " + e); + throw new RuntimeException(e); + } + } + + private OutcomeReceiver<Feature, + OnDeviceIntelligenceException> wrapFeatureCallback( + IFeatureCallback featureCallback) { + return new OutcomeReceiver<>() { + @Override + public void onResult(@NonNull Feature feature) { + try { + featureCallback.onSuccess(feature); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending feature: " + e); + } + } + + @Override + public void onError( + @NonNull OnDeviceIntelligenceException exception) { + try { + featureCallback.onFailure(exception.getErrorCode(), exception.getMessage(), + exception.getErrorParams()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending download feature: " + e); + } + } + }; + } + + private OutcomeReceiver<List<Feature>, + OnDeviceIntelligenceException> wrapListFeaturesCallback( + IListFeaturesCallback listFeaturesCallback) { + return new OutcomeReceiver<>() { + @Override + public void onResult(@NonNull List<Feature> features) { + try { + listFeaturesCallback.onSuccess(features); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending feature: " + e); + } + } + + @Override + public void onError( + @NonNull OnDeviceIntelligenceException exception) { + try { + listFeaturesCallback.onFailure(exception.getErrorCode(), exception.getMessage(), + exception.getErrorParams()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending download feature: " + e); + } + } + }; + } + + private OutcomeReceiver<FeatureDetails, + OnDeviceIntelligenceException> wrapFeatureDetailsCallback( + IFeatureDetailsCallback featureStatusCallback) { + return new OutcomeReceiver<>() { + @Override + public void onResult(FeatureDetails result) { + try { + featureStatusCallback.onSuccess(result); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending feature status: " + e); + } + } + + @Override + public void onError( + @NonNull OnDeviceIntelligenceException exception) { + try { + featureStatusCallback.onFailure(exception.getErrorCode(), + exception.getMessage(), exception.getErrorParams()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending feature status: " + e); + } + } + }; + } + + + private DownloadCallback wrapDownloadCallback(IDownloadCallback downloadCallback) { + return new DownloadCallback() { + @Override + public void onDownloadStarted(long bytesToDownload) { + try { + downloadCallback.onDownloadStarted(bytesToDownload); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending download status: " + e); + } + } + + @Override + public void onDownloadFailed(int failureStatus, + String errorMessage, @NonNull PersistableBundle errorParams) { + try { + downloadCallback.onDownloadFailed(failureStatus, errorMessage, errorParams); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending download status: " + e); + } + } + + @Override + public void onDownloadProgress(long totalBytesDownloaded) { + try { + downloadCallback.onDownloadProgress(totalBytesDownloaded); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending download status: " + e); + } + } + + @Override + public void onDownloadCompleted(@NonNull PersistableBundle persistableBundle) { + try { + downloadCallback.onDownloadCompleted(persistableBundle); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending download status: " + e); + } + } + }; + } + + private static void tryClosePfds(Collection<ParcelFileDescriptor> pfds) { + pfds.forEach(pfd -> { + try { + pfd.close(); + } catch (Exception e) { + Log.w(TAG, "Error closing FD", e); + } + }); + } + + private void onGetReadOnlyFileDescriptor(@NonNull String fileName, + @NonNull AndroidFuture<ParcelFileDescriptor> future) { + Slog.v(TAG, "onGetReadOnlyFileDescriptor " + fileName); + BinderUtils.withCleanCallingIdentity(() -> { + Slog.v(TAG, + "onGetReadOnlyFileDescriptor: " + fileName + " under internal app storage."); + File f = new File(getBaseContext().getFilesDir(), fileName); + if (!f.exists()) { + f = new File(fileName); + } + ParcelFileDescriptor pfd = null; + try { + pfd = ParcelFileDescriptor.open(f, ParcelFileDescriptor.MODE_READ_ONLY); + Slog.d(TAG, "Successfully opened a file with ParcelFileDescriptor."); + } catch (FileNotFoundException e) { + Slog.e(TAG, "Cannot open file. No ParcelFileDescriptor returned."); + future.completeExceptionally(e); + } finally { + future.complete(pfd); + if (pfd != null) { + try { + pfd.close(); + } catch (IOException e) { + Log.w(TAG, "Error closing FD", e); + } + } + } + }); + } + + /** + * Provide implementation for a scenario when caller wants to get all feature related + * file-descriptors that might be required for processing a request for the corresponding the + * feature. + * + * @param feature the feature for which files need to be opened. + * @param fileDescriptorMapConsumer callback to be populated with a map of file-path and + * corresponding ParcelDescriptor to be used in a remote + * service. + */ + public abstract void onGetReadOnlyFeatureFileDescriptorMap( + @NonNull Feature feature, + @NonNull Consumer<Map<String, ParcelFileDescriptor>> fileDescriptorMapConsumer); + + /** + * Request download for feature that is requested and listen to download progress updates. If + * the download completes successfully, success callback should be populated. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param feature the feature for which files need to be downlaoded. + * process. + * @param cancellationSignal signal to attach a listener to, and receive cancellation signals + * from thw client. + * @param downloadCallback callback to populate download updates for clients to listen on.. + */ + public abstract void onDownloadFeature( + int callerUid, @NonNull Feature feature, + @Nullable CancellationSignal cancellationSignal, + @NonNull DownloadCallback downloadCallback); + + /** + * Provide feature details for the passed in feature. Usually the client and remote + * implementation use the {@link Feature#getFeatureParams()} as a hint to communicate what + * details the client is looking for. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param feature the feature for which status needs to be known. + * @param featureDetailsCallback callback to populate the resulting feature status. + */ + public abstract void onGetFeatureDetails(int callerUid, @NonNull Feature feature, + @NonNull OutcomeReceiver<FeatureDetails, + OnDeviceIntelligenceException> featureDetailsCallback); + + + /** + * Get feature using the provided identifier to the remote implementation. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param featureCallback callback to populate the features list. + */ + public abstract void onGetFeature(int callerUid, int featureId, + @NonNull OutcomeReceiver<Feature, + OnDeviceIntelligenceException> featureCallback); + + /** + * List all features which are available in the remote implementation. The implementation might + * choose to provide only a certain list of features based on the caller. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param listFeaturesCallback callback to populate the features list. + */ + public abstract void onListFeatures(int callerUid, @NonNull OutcomeReceiver<List<Feature>, + OnDeviceIntelligenceException> listFeaturesCallback); + + /** + * Provides a long value representing the version of the remote implementation processing + * requests. + * + * @param versionConsumer consumer to populate the version. + */ + public abstract void onGetVersion(@NonNull LongConsumer versionConsumer); +} diff --git a/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java new file mode 100644 index 000000000000..315dbaf919e5 --- /dev/null +++ b/packages/NeuralNetworks/framework/platform/java/android/service/ondeviceintelligence/OnDeviceSandboxedInferenceService.java @@ -0,0 +1,734 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.ondeviceintelligence; + +import static android.app.ondeviceintelligence.OnDeviceIntelligenceManager.AUGMENT_REQUEST_CONTENT_BUNDLE_KEY; +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE; + +import android.annotation.CallSuper; +import android.annotation.CallbackExecutor; +import android.annotation.FlaggedApi; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SdkConstant; +import android.annotation.SuppressLint; +import android.annotation.SystemApi; +import android.app.Service; +import android.app.ondeviceintelligence.Feature; +import android.app.ondeviceintelligence.ICancellationSignal; +import android.app.ondeviceintelligence.IProcessingSignal; +import android.app.ondeviceintelligence.IRemoteCallback; +import android.app.ondeviceintelligence.IResponseCallback; +import android.app.ondeviceintelligence.IStreamingResponseCallback; +import android.app.ondeviceintelligence.ITokenInfoCallback; +import android.app.ondeviceintelligence.OnDeviceIntelligenceException; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.InferenceParams; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.StateParams; +import android.app.ondeviceintelligence.ProcessingCallback; +import android.app.ondeviceintelligence.ProcessingSignal; +import android.app.ondeviceintelligence.StreamingProcessingCallback; +import android.app.ondeviceintelligence.TokenInfo; +import android.content.Context; +import android.content.Intent; +import android.os.Bundle; +import android.os.CancellationSignal; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.Message; +import android.os.OutcomeReceiver; +import android.os.ParcelFileDescriptor; +import android.os.PersistableBundle; +import android.os.RemoteCallback; +import android.os.RemoteException; +import android.util.Log; +import android.util.Slog; + +import com.android.modules.utils.AndroidFuture; +import com.android.modules.utils.HandlerExecutor; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +/** + * Abstract base class for performing inference in a isolated process. This service exposes its + * methods via {@link android.app.ondeviceintelligence.OnDeviceIntelligenceManager}. + * + * <p> A service that provides methods to perform on-device inference both in streaming and + * non-streaming fashion. Also, provides a way to register a storage service that will be used to + * read-only access files from the {@link OnDeviceIntelligenceService} counterpart. </p> + * + * <p> Similar to {@link OnDeviceIntelligenceManager} class, the contracts in this service are + * defined to be open-ended in general, to allow interoperability. Therefore, it is recommended + * that implementations of this system-service expose this API to the clients via a library which + * has more defined contract.</p> + * + * <pre> + * {@literal + * <service android:name=".SampleSandboxedInferenceService" + * android:permission="android.permission.BIND_ONDEVICE_SANDBOXED_INFERENCE_SERVICE" + * android:isolatedProcess="true"> + * </service>} + * </pre> + * + * @hide + */ +@SystemApi +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE) +public abstract class OnDeviceSandboxedInferenceService extends Service { + private static final String TAG = OnDeviceSandboxedInferenceService.class.getSimpleName(); + + private static final int MSG_TOKEN_INFO_REQUEST = 1; + private static final int MSG_PROCESS_REQUEST_STREAMING = 2; + private static final int MSG_PROCESS_REQUEST = 3; + private static final int MSG_UPDATE_PROCESSING_STATE = 4; + + + /** + * @hide + */ + public static final String INFERENCE_INFO_BUNDLE_KEY = "inference_info"; + + /** + * The {@link Intent} that must be declared as handled by the service. To be supported, the + * service must also require the + * {@link android.Manifest.permission#BIND_ON_DEVICE_SANDBOXED_INFERENCE_SERVICE} + * permission so that other applications can not abuse it. + */ + @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION) + public static final String SERVICE_INTERFACE = + "android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService"; + + // TODO(339594686): make API + /** + * @hide + */ + public static final String REGISTER_MODEL_UPDATE_CALLBACK_BUNDLE_KEY = + "register_model_update_callback"; + /** + * @hide + */ + public static final String MODEL_LOADED_BUNDLE_KEY = "model_loaded"; + /** + * @hide + */ + public static final String MODEL_UNLOADED_BUNDLE_KEY = "model_unloaded"; + /** + * @hide + */ + public static final String MODEL_LOADED_BROADCAST_INTENT = + "android.service.ondeviceintelligence.MODEL_LOADED"; + /** + * @hide + */ + public static final String MODEL_UNLOADED_BROADCAST_INTENT = + "android.service.ondeviceintelligence.MODEL_UNLOADED"; + + /** + * @hide + */ + public static final String DEVICE_CONFIG_UPDATE_BUNDLE_KEY = "device_config_update"; + + private IRemoteStorageService mRemoteStorageService; + private Handler mHandler; + + @CallSuper + @Override + public void onCreate() { + super.onCreate(); + mHandler = new Handler(Looper.getMainLooper()) { + @Override + public void handleMessage(Message msg) { + switch (msg.what) { + case MSG_TOKEN_INFO_REQUEST: + TokenInfoParams params = (TokenInfoParams) msg.obj; + OnDeviceSandboxedInferenceService.this.onTokenInfoRequest( + msg.arg1, + params.feature, + params.request, + params.cancellationSignal, + params.callback); + break; + case MSG_PROCESS_REQUEST_STREAMING: + StreamingRequestParams streamParams = (StreamingRequestParams) msg.obj; + OnDeviceSandboxedInferenceService.this.onProcessRequestStreaming( + msg.arg1, + streamParams.feature, + streamParams.request, + msg.arg2, + streamParams.cancellationSignal, + streamParams.processingSignal, + streamParams.callback); + break; + case MSG_PROCESS_REQUEST: + RequestParams requestParams = (RequestParams) msg.obj; + OnDeviceSandboxedInferenceService.this.onProcessRequest( + msg.arg1, + requestParams.feature, + requestParams.request, + msg.arg2, + requestParams.cancellationSignal, + requestParams.processingSignal, + requestParams.callback); + break; + case MSG_UPDATE_PROCESSING_STATE: + UpdateStateParams stateParams = (UpdateStateParams) msg.obj; + OnDeviceSandboxedInferenceService.this.onUpdateProcessingState( + stateParams.processingState, + stateParams.callback); + break; + } + } + }; + } + + // Parameter holder classes + private static class TokenInfoParams { + final Feature feature; + final Bundle request; + final CancellationSignal cancellationSignal; + final OutcomeReceiver<TokenInfo, OnDeviceIntelligenceException> callback; + + TokenInfoParams(Feature feature, Bundle request, CancellationSignal cancellationSignal, + OutcomeReceiver<TokenInfo, OnDeviceIntelligenceException> callback) { + this.feature = feature; + this.request = request; + this.cancellationSignal = cancellationSignal; + this.callback = callback; + } + } + + private static class StreamingRequestParams { + final Feature feature; + final Bundle request; + final CancellationSignal cancellationSignal; + final ProcessingSignal processingSignal; + final StreamingProcessingCallback callback; + + StreamingRequestParams(Feature feature, Bundle request, + CancellationSignal cancellationSignal, ProcessingSignal processingSignal, + StreamingProcessingCallback callback) { + this.feature = feature; + this.request = request; + this.cancellationSignal = cancellationSignal; + this.processingSignal = processingSignal; + this.callback = callback; + } + } + + private static class RequestParams { + final Feature feature; + final Bundle request; + final CancellationSignal cancellationSignal; + final ProcessingSignal processingSignal; + final ProcessingCallback callback; + + RequestParams(Feature feature, Bundle request, + CancellationSignal cancellationSignal, ProcessingSignal processingSignal, + ProcessingCallback callback) { + this.feature = feature; + this.request = request; + this.cancellationSignal = cancellationSignal; + this.processingSignal = processingSignal; + this.callback = callback; + } + } + + private static class UpdateStateParams { + final Bundle processingState; + final OutcomeReceiver<PersistableBundle, OnDeviceIntelligenceException> callback; + + UpdateStateParams(Bundle processingState, + OutcomeReceiver<PersistableBundle, OnDeviceIntelligenceException> callback) { + this.processingState = processingState; + this.callback = callback; + } + } + + @Nullable + @Override + public final IBinder onBind(@NonNull Intent intent) { + if (SERVICE_INTERFACE.equals(intent.getAction())) { + return new IOnDeviceSandboxedInferenceService.Stub() { + @Override + public void registerRemoteStorageService(IRemoteStorageService storageService, + IRemoteCallback remoteCallback) throws RemoteException { + Objects.requireNonNull(storageService); + mRemoteStorageService = storageService; + remoteCallback.sendResult(Bundle.EMPTY); + } + + @Override + public void requestTokenInfo(int callerUid, Feature feature, Bundle request, + AndroidFuture cancellationSignalFuture, + ITokenInfoCallback tokenInfoCallback) { + Objects.requireNonNull(feature); + Objects.requireNonNull(tokenInfoCallback); + CancellationSignal cancellationSignal = new CancellationSignal(); + if (cancellationSignalFuture != null) { + ICancellationSignal transport = new ICancellationSignal.Stub() { + @Override + public void cancel() { + cancellationSignal.cancel(); + } + }; + cancellationSignalFuture.complete(transport); + } + + Message msg = Message.obtain(mHandler, MSG_TOKEN_INFO_REQUEST, + callerUid, 0, + new TokenInfoParams(feature, request, + cancellationSignalFuture != null ? cancellationSignal : null, + wrapTokenInfoCallback(tokenInfoCallback))); + mHandler.sendMessage(msg); + } + + @Override + public void processRequestStreaming(int callerUid, Feature feature, + Bundle request, int requestType, + AndroidFuture cancellationSignalFuture, + AndroidFuture processingSignalFuture, + IStreamingResponseCallback callback) { + Objects.requireNonNull(feature); + Objects.requireNonNull(callback); + + CancellationSignal cancellationSignal = new CancellationSignal(); + if (cancellationSignalFuture != null) { + ICancellationSignal transport = new ICancellationSignal.Stub() { + @Override + public void cancel() { + cancellationSignal.cancel(); + } + }; + cancellationSignalFuture.complete(transport); + } + IProcessingSignal processingSignalTransport = null; + if (processingSignalFuture != null) { + processingSignalTransport = ProcessingSignal.createTransport(); + processingSignalFuture.complete(processingSignalTransport); + } + + Message msg = Message.obtain(mHandler, MSG_PROCESS_REQUEST_STREAMING, + callerUid, requestType, + new StreamingRequestParams(feature, request, + cancellationSignalFuture != null ? cancellationSignal : null, + ProcessingSignal.fromTransport(processingSignalTransport), + wrapStreamingResponseCallback(callback))); + mHandler.sendMessage(msg); + } + + @Override + public void processRequest(int callerUid, Feature feature, + Bundle request, int requestType, + AndroidFuture cancellationSignalFuture, + AndroidFuture processingSignalFuture, + IResponseCallback callback) { + Objects.requireNonNull(feature); + Objects.requireNonNull(callback); + + CancellationSignal cancellationSignal = new CancellationSignal(); + if (cancellationSignalFuture != null) { + ICancellationSignal transport = new ICancellationSignal.Stub() { + @Override + public void cancel() { + cancellationSignal.cancel(); + } + }; + cancellationSignalFuture.complete(transport); + } + IProcessingSignal processingSignalTransport = null; + if (processingSignalFuture != null) { + processingSignalTransport = ProcessingSignal.createTransport(); + processingSignalFuture.complete(processingSignalTransport); + } + + Message msg = Message.obtain(mHandler, MSG_PROCESS_REQUEST, + callerUid, requestType, + new RequestParams(feature, request, + cancellationSignalFuture != null ? cancellationSignal : null, + ProcessingSignal.fromTransport(processingSignalTransport), + wrapResponseCallback(callback))); + mHandler.sendMessage(msg); + } + + @Override + public void updateProcessingState(Bundle processingState, + IProcessingUpdateStatusCallback callback) { + Objects.requireNonNull(processingState); + Objects.requireNonNull(callback); + + Message msg = Message.obtain(mHandler, MSG_UPDATE_PROCESSING_STATE, + new UpdateStateParams(processingState, + wrapOutcomeReceiver(callback))); + mHandler.sendMessage(msg); + } + }; + } + Slog.w(TAG, "Incorrect service interface, returning null."); + return null; + } + + /** + * Invoked when caller wants to obtain token info related to the payload in the passed + * content, associated with the provided feature. + * The expectation from the implementation is that when processing is complete, it + * should provide the token info in the {@link OutcomeReceiver#onResult}. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param feature feature which is associated with the request. + * @param request request that requires processing. + * @param cancellationSignal Cancellation Signal to receive cancellation events from client and + * configure a listener to. + * @param callback callback to populate failure or the token info for the provided + * request. + */ + @NonNull + public abstract void onTokenInfoRequest( + int callerUid, @NonNull Feature feature, + @NonNull @InferenceParams Bundle request, + @Nullable CancellationSignal cancellationSignal, + @NonNull OutcomeReceiver<TokenInfo, OnDeviceIntelligenceException> callback); + + /** + * Invoked when caller provides a request for a particular feature to be processed in a + * streaming manner. The expectation from the implementation is that when processing the + * request, + * it periodically populates the {@link StreamingProcessingCallback#onPartialResult} to + * continuously + * provide partial Bundle results for the caller to utilize. Optionally the implementation can + * provide the complete response in the {@link StreamingProcessingCallback#onResult} upon + * processing completion. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param feature feature which is associated with the request. + * @param request request that requires processing. + * @param requestType identifier representing the type of request. + * @param cancellationSignal Cancellation Signal to receive cancellation events from client and + * configure a listener to. + * @param processingSignal Signal to receive custom action instructions from client. + * @param callback callback to populate the partial responses, failure and optionally + * full response for the provided request. + */ + @NonNull + public abstract void onProcessRequestStreaming( + int callerUid, @NonNull Feature feature, + @NonNull @InferenceParams Bundle request, + @OnDeviceIntelligenceManager.RequestType int requestType, + @Nullable CancellationSignal cancellationSignal, + @Nullable ProcessingSignal processingSignal, + @NonNull StreamingProcessingCallback callback); + + /** + * Invoked when caller provides a request for a particular feature to be processed in one shot + * completely. + * The expectation from the implementation is that when processing the request is complete, it + * should + * provide the complete response in the {@link OutcomeReceiver#onResult}. + * + * @param callerUid UID of the caller that initiated this call chain. + * @param feature feature which is associated with the request. + * @param request request that requires processing. + * @param requestType identifier representing the type of request. + * @param cancellationSignal Cancellation Signal to receive cancellation events from client and + * configure a listener to. + * @param processingSignal Signal to receive custom action instructions from client. + * @param callback callback to populate failure and full response for the provided + * request. + */ + @NonNull + public abstract void onProcessRequest( + int callerUid, @NonNull Feature feature, + @NonNull @InferenceParams Bundle request, + @OnDeviceIntelligenceManager.RequestType int requestType, + @Nullable CancellationSignal cancellationSignal, + @Nullable ProcessingSignal processingSignal, + @NonNull ProcessingCallback callback); + + + /** + * Invoked when processing environment needs to be updated or refreshed with fresh + * configuration, files or state. + * + * @param processingState contains updated state and params that are to be applied to the + * processing environmment, + * @param callback callback to populate the update status and if there are params + * associated with the status. + */ + public abstract void onUpdateProcessingState(@NonNull @StateParams Bundle processingState, + @NonNull OutcomeReceiver<PersistableBundle, + OnDeviceIntelligenceException> callback); + + + /** + * Overrides {@link Context#openFileInput} to read files with the given file names under the + * internal app storage of the {@link OnDeviceIntelligenceService}, i.e., only files stored in + * {@link Context#getFilesDir()} can be opened. + */ + @Override + public final FileInputStream openFileInput(@NonNull String filename) throws + FileNotFoundException { + try { + AndroidFuture<ParcelFileDescriptor> future = new AndroidFuture<>(); + mRemoteStorageService.getReadOnlyFileDescriptor(filename, future); + ParcelFileDescriptor pfd = future.get(); + return new FileInputStream(pfd.getFileDescriptor()); + } catch (RemoteException | ExecutionException | InterruptedException e) { + Log.w(TAG, "Cannot open file due to remote service failure"); + throw new FileNotFoundException(e.getMessage()); + } + } + + /** + * Provides read-only access to the internal app storage via the + * {@link OnDeviceIntelligenceService}. This is an asynchronous alternative for + * {@link #openFileInput(String)}. + * + * @param fileName File name relative to the {@link Context#getFilesDir()}. + * @param resultConsumer Consumer to populate the corresponding file descriptor in. + */ + public final void getReadOnlyFileDescriptor(@NonNull String fileName, + @NonNull @CallbackExecutor Executor executor, + @NonNull Consumer<ParcelFileDescriptor> resultConsumer) throws FileNotFoundException { + AndroidFuture<ParcelFileDescriptor> future = new AndroidFuture<>(); + try { + mRemoteStorageService.getReadOnlyFileDescriptor(fileName, future); + } catch (RemoteException e) { + Log.w(TAG, "Cannot open file due to remote service failure"); + throw new FileNotFoundException(e.getMessage()); + } + future.whenCompleteAsync((pfd, err) -> { + if (err != null) { + Log.e(TAG, "Failure when reading file: " + fileName + err); + executor.execute(() -> resultConsumer.accept(null)); + } else { + executor.execute( + () -> resultConsumer.accept(pfd)); + } + }, executor); + } + + /** + * Provides access to all file streams required for feature via the + * {@link OnDeviceIntelligenceService}. + * + * @param feature Feature for which the associated files should be fetched. + * @param executor Executor to run the consumer callback on. + * @param resultConsumer Consumer to receive a map of filePath to the corresponding file input + * stream. + */ + public final void fetchFeatureFileDescriptorMap(@NonNull Feature feature, + @NonNull @CallbackExecutor Executor executor, + @NonNull Consumer<Map<String, ParcelFileDescriptor>> resultConsumer) { + try { + mRemoteStorageService.getReadOnlyFeatureFileDescriptorMap(feature, + wrapAsRemoteCallback(resultConsumer, executor)); + } catch (RemoteException e) { + throw new RuntimeException(e); + } + } + + + /** + * Returns the {@link Executor} to use for incoming IPC from request sender into your service + * implementation. For e.g. see + * {@link ProcessingCallback#onDataAugmentRequest(Bundle, + * Consumer)} where we use the executor to populate the consumer. + * <p> + * Override this method in your {@link OnDeviceSandboxedInferenceService} implementation to + * provide the executor you want to use for incoming IPC. + * + * @return the {@link Executor} to use for incoming IPC from {@link OnDeviceIntelligenceManager} + * to {@link OnDeviceSandboxedInferenceService}. + */ + @SuppressLint("OnNameExpected") + @NonNull + public Executor getCallbackExecutor() { + return new HandlerExecutor(Handler.createAsync(getMainLooper())); + } + + + private RemoteCallback wrapAsRemoteCallback( + @NonNull Consumer<Map<String, ParcelFileDescriptor>> resultConsumer, + @NonNull Executor executor) { + return new RemoteCallback(result -> { + if (result == null) { + executor.execute(() -> resultConsumer.accept(new HashMap<>())); + } else { + Map<String, ParcelFileDescriptor> pfdMap = new HashMap<>(); + result.keySet().forEach(key -> + pfdMap.put(key, result.getParcelable(key, + ParcelFileDescriptor.class))); + executor.execute(() -> resultConsumer.accept(pfdMap)); + } + }); + } + + private ProcessingCallback wrapResponseCallback( + IResponseCallback callback) { + return new ProcessingCallback() { + @Override + public void onResult(@NonNull Bundle result) { + try { + callback.onSuccess(result); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + } + } + + @Override + public void onError( + OnDeviceIntelligenceException exception) { + try { + callback.onFailure(exception.getErrorCode(), exception.getMessage(), + exception.getErrorParams()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + } + } + + @Override + public void onDataAugmentRequest(@NonNull Bundle content, + @NonNull Consumer<Bundle> contentCallback) { + try { + callback.onDataAugmentRequest(content, wrapRemoteCallback(contentCallback)); + + } catch (RemoteException e) { + Slog.e(TAG, "Error sending augment request: " + e); + } + } + }; + } + + private StreamingProcessingCallback wrapStreamingResponseCallback( + IStreamingResponseCallback callback) { + return new StreamingProcessingCallback() { + @Override + public void onPartialResult(@NonNull Bundle partialResult) { + try { + callback.onNewContent(partialResult); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + } + } + + @Override + public void onResult(@NonNull Bundle result) { + try { + callback.onSuccess(result); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + } + } + + @Override + public void onError( + OnDeviceIntelligenceException exception) { + try { + callback.onFailure(exception.getErrorCode(), exception.getMessage(), + exception.getErrorParams()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + } + } + + @Override + public void onDataAugmentRequest(@NonNull Bundle content, + @NonNull Consumer<Bundle> contentCallback) { + try { + callback.onDataAugmentRequest(content, wrapRemoteCallback(contentCallback)); + + } catch (RemoteException e) { + Slog.e(TAG, "Error sending augment request: " + e); + } + } + }; + } + + private RemoteCallback wrapRemoteCallback( + @NonNull Consumer<Bundle> contentCallback) { + return new RemoteCallback( + result -> { + if (result != null) { + getCallbackExecutor().execute(() -> contentCallback.accept( + result.getParcelable(AUGMENT_REQUEST_CONTENT_BUNDLE_KEY, + Bundle.class))); + } else { + getCallbackExecutor().execute( + () -> contentCallback.accept(null)); + } + }); + } + + private OutcomeReceiver<TokenInfo, OnDeviceIntelligenceException> wrapTokenInfoCallback( + ITokenInfoCallback tokenInfoCallback) { + return new OutcomeReceiver<>() { + @Override + public void onResult(TokenInfo tokenInfo) { + try { + tokenInfoCallback.onSuccess(tokenInfo); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + } + } + + @Override + public void onError( + OnDeviceIntelligenceException exception) { + try { + tokenInfoCallback.onFailure(exception.getErrorCode(), exception.getMessage(), + exception.getErrorParams()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending failure: " + e); + } + } + }; + } + + @NonNull + private static OutcomeReceiver<PersistableBundle, OnDeviceIntelligenceException> wrapOutcomeReceiver( + IProcessingUpdateStatusCallback callback) { + return new OutcomeReceiver<>() { + @Override + public void onResult(@NonNull PersistableBundle result) { + try { + callback.onSuccess(result); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending result: " + e); + + } + } + + @Override + public void onError( + @NonNull OnDeviceIntelligenceException error) { + try { + callback.onFailure(error.getErrorCode(), error.getMessage()); + } catch (RemoteException e) { + Slog.e(TAG, "Error sending exception details: " + e); + } + } + }; + } + +} diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/BundleUtil.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/BundleUtil.java index 2626cc880e09..2626cc880e09 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/BundleUtil.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/BundleUtil.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java index e8a1b322f661..e8a1b322f661 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java index 6badc53ae97e..6badc53ae97e 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java index a078f7542c11..a078f7542c11 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java index c641de8b47b1..c641de8b47b1 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java index 0c43a309c456..0c43a309c456 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java index 8c5d5a7ba736..8c5d5a7ba736 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java diff --git a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java index 249bcd37db5d..249bcd37db5d 100644 --- a/packages/NeuralNetworks/service/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java +++ b/packages/NeuralNetworks/service/module/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/BundleUtil.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/BundleUtil.java new file mode 100644 index 000000000000..2626cc880e09 --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/BundleUtil.java @@ -0,0 +1,415 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import static android.system.OsConstants.F_GETFL; +import static android.system.OsConstants.O_ACCMODE; +import static android.system.OsConstants.O_RDONLY; +import static android.system.OsConstants.PROT_READ; + +import android.annotation.SuppressLint; +import android.app.ondeviceintelligence.IResponseCallback; +import android.app.ondeviceintelligence.IStreamingResponseCallback; +import android.app.ondeviceintelligence.ITokenInfoCallback; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.InferenceParams; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.ResponseParams; +import android.app.ondeviceintelligence.OnDeviceIntelligenceManager.StateParams; +import android.app.ondeviceintelligence.TokenInfo; +import android.database.CursorWindow; +import android.graphics.Bitmap; +import android.os.BadParcelableException; +import android.os.Bundle; +import android.os.ParcelFileDescriptor; +import android.os.Parcelable; +import android.os.PersistableBundle; +import android.os.RemoteCallback; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.system.ErrnoException; +import android.system.Os; +import android.util.Log; + +import com.android.modules.utils.AndroidFuture; + +import java.util.concurrent.Executor; +import java.util.concurrent.TimeoutException; + +/** + * Util methods for ensuring the Bundle passed in various methods are read-only and restricted to + * some known types. + * + * @hide + */ +public class BundleUtil { + private static final String TAG = "BundleUtil"; + + /** + * Validation of the inference request payload as described in {@link InferenceParams} + * description. + * + * @throws BadParcelableException when the bundle does not meet the read-only requirements. + */ + public static void sanitizeInferenceParams( + @InferenceParams Bundle bundle) { + ensureValidBundle(bundle); + + if (!bundle.hasFileDescriptors()) { + return; //safe to exit if there are no FDs and Binders + } + + for (String key : bundle.keySet()) { + Object obj = bundle.get(key); + if (obj == null) { + /* Null value here could also mean deserializing a custom parcelable has failed, + * and since {@link Bundle} is marked as defusable in system-server - the + * {@link ClassNotFoundException} exception is swallowed and `null` is returned + * instead. We want to ensure cleanup of null entries in such case. + */ + bundle.putParcelable(key, null); + continue; + } + if (canMarshall(obj) || obj instanceof CursorWindow) { + continue; + } + if (obj instanceof Bundle) { + sanitizeInferenceParams((Bundle) obj); + } else if (obj instanceof ParcelFileDescriptor) { + validatePfdReadOnly((ParcelFileDescriptor) obj); + } else if (obj instanceof SharedMemory) { + ((SharedMemory) obj).setProtect(PROT_READ); + } else if (obj instanceof Bitmap) { + validateBitmap((Bitmap) obj); + } else if (obj instanceof Parcelable[]) { + validateParcelableArray((Parcelable[]) obj); + } else { + throw new BadParcelableException( + "Unsupported Parcelable type encountered in the Bundle: " + + obj.getClass().getSimpleName()); + } + } + } + + /** + * Validation of the inference request payload as described in {@link ResponseParams} + * description. + * + * @throws BadParcelableException when the bundle does not meet the read-only requirements. + */ + public static void sanitizeResponseParams( + @ResponseParams Bundle bundle) { + ensureValidBundle(bundle); + + if (!bundle.hasFileDescriptors()) { + return; //safe to exit if there are no FDs and Binders + } + + for (String key : bundle.keySet()) { + Object obj = bundle.get(key); + if (obj == null) { + /* Null value here could also mean deserializing a custom parcelable has failed, + * and since {@link Bundle} is marked as defusable in system-server - the + * {@link ClassNotFoundException} exception is swallowed and `null` is returned + * instead. We want to ensure cleanup of null entries in such case. + */ + bundle.putParcelable(key, null); + continue; + } + if (canMarshall(obj)) { + continue; + } + + if (obj instanceof Bundle) { + sanitizeResponseParams((Bundle) obj); + } else if (obj instanceof ParcelFileDescriptor) { + validatePfdReadOnly((ParcelFileDescriptor) obj); + } else if (obj instanceof Bitmap) { + validateBitmap((Bitmap) obj); + } else if (obj instanceof Parcelable[]) { + validateParcelableArray((Parcelable[]) obj); + } else { + throw new BadParcelableException( + "Unsupported Parcelable type encountered in the Bundle: " + + obj.getClass().getSimpleName()); + } + } + } + + /** + * Validation of the inference request payload as described in {@link StateParams} + * description. + * + * @throws BadParcelableException when the bundle does not meet the read-only requirements. + */ + public static void sanitizeStateParams( + @StateParams Bundle bundle) { + ensureValidBundle(bundle); + + if (!bundle.hasFileDescriptors()) { + return; //safe to exit if there are no FDs and Binders + } + + for (String key : bundle.keySet()) { + Object obj = bundle.get(key); + if (obj == null) { + /* Null value here could also mean deserializing a custom parcelable has failed, + * and since {@link Bundle} is marked as defusable in system-server - the + * {@link ClassNotFoundException} exception is swallowed and `null` is returned + * instead. We want to ensure cleanup of null entries in such case. + */ + bundle.putParcelable(key, null); + continue; + } + if (canMarshall(obj)) { + continue; + } + + if (obj instanceof ParcelFileDescriptor) { + validatePfdReadOnly((ParcelFileDescriptor) obj); + } else { + throw new BadParcelableException( + "Unsupported Parcelable type encountered in the Bundle: " + + obj.getClass().getSimpleName()); + } + } + } + + + public static IStreamingResponseCallback wrapWithValidation( + IStreamingResponseCallback streamingResponseCallback, + Executor resourceClosingExecutor, + AndroidFuture future, + InferenceInfoStore inferenceInfoStore) { + return new IStreamingResponseCallback.Stub() { + @Override + public void onNewContent(Bundle processedResult) throws RemoteException { + try { + sanitizeResponseParams(processedResult); + streamingResponseCallback.onNewContent(processedResult); + } finally { + resourceClosingExecutor.execute(() -> tryCloseResource(processedResult)); + } + } + + @Override + public void onSuccess(Bundle resultBundle) + throws RemoteException { + try { + sanitizeResponseParams(resultBundle); + streamingResponseCallback.onSuccess(resultBundle); + } finally { + inferenceInfoStore.addInferenceInfoFromBundle(resultBundle); + resourceClosingExecutor.execute(() -> tryCloseResource(resultBundle)); + future.complete(null); + } + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) throws RemoteException { + streamingResponseCallback.onFailure(errorCode, errorMessage, errorParams); + inferenceInfoStore.addInferenceInfoFromBundle(errorParams); + future.completeExceptionally(new TimeoutException()); + } + + @Override + public void onDataAugmentRequest(Bundle processedContent, + RemoteCallback remoteCallback) + throws RemoteException { + try { + sanitizeResponseParams(processedContent); + streamingResponseCallback.onDataAugmentRequest(processedContent, + new RemoteCallback( + augmentedData -> { + try { + sanitizeInferenceParams(augmentedData); + remoteCallback.sendResult(augmentedData); + } finally { + resourceClosingExecutor.execute( + () -> tryCloseResource(augmentedData)); + } + })); + } finally { + resourceClosingExecutor.execute(() -> tryCloseResource(processedContent)); + } + } + }; + } + + public static IResponseCallback wrapWithValidation(IResponseCallback responseCallback, + Executor resourceClosingExecutor, + AndroidFuture future, + InferenceInfoStore inferenceInfoStore) { + return new IResponseCallback.Stub() { + @Override + public void onSuccess(Bundle resultBundle) + throws RemoteException { + try { + sanitizeResponseParams(resultBundle); + responseCallback.onSuccess(resultBundle); + } finally { + inferenceInfoStore.addInferenceInfoFromBundle(resultBundle); + resourceClosingExecutor.execute(() -> tryCloseResource(resultBundle)); + future.complete(null); + } + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) throws RemoteException { + responseCallback.onFailure(errorCode, errorMessage, errorParams); + inferenceInfoStore.addInferenceInfoFromBundle(errorParams); + future.completeExceptionally(new TimeoutException()); + } + + @Override + public void onDataAugmentRequest(Bundle processedContent, + RemoteCallback remoteCallback) + throws RemoteException { + try { + sanitizeResponseParams(processedContent); + responseCallback.onDataAugmentRequest(processedContent, new RemoteCallback( + augmentedData -> { + try { + sanitizeInferenceParams(augmentedData); + remoteCallback.sendResult(augmentedData); + } finally { + resourceClosingExecutor.execute( + () -> tryCloseResource(augmentedData)); + } + })); + } finally { + resourceClosingExecutor.execute(() -> tryCloseResource(processedContent)); + } + } + }; + } + + + public static ITokenInfoCallback wrapWithValidation(ITokenInfoCallback responseCallback, + AndroidFuture future, + InferenceInfoStore inferenceInfoStore) { + return new ITokenInfoCallback.Stub() { + @Override + public void onSuccess(TokenInfo tokenInfo) throws RemoteException { + responseCallback.onSuccess(tokenInfo); + inferenceInfoStore.addInferenceInfoFromBundle(tokenInfo.getInfoParams()); + future.complete(null); + } + + @Override + public void onFailure(int errorCode, String errorMessage, PersistableBundle errorParams) + throws RemoteException { + responseCallback.onFailure(errorCode, errorMessage, errorParams); + inferenceInfoStore.addInferenceInfoFromBundle(errorParams); + future.completeExceptionally(new TimeoutException()); + } + }; + } + + private static boolean canMarshall(Object value) { + return (value instanceof byte[]) || (value instanceof Integer) || (value instanceof Long) || + (value instanceof Double) || (value instanceof String) || + (value instanceof int[]) || (value instanceof long[]) || + (value instanceof double[]) || (value instanceof String[]) || + (value instanceof PersistableBundle) || (value == null) || + (value instanceof Boolean) || (value instanceof boolean[]); + } + + @SuppressLint("NewApi") + private static void ensureValidBundle(Bundle bundle) { + if (bundle == null) { + throw new IllegalArgumentException("Request passed is expected to be non-null"); + } + + if (bundle.hasBinders() != Bundle.STATUS_BINDERS_NOT_PRESENT) { + throw new BadParcelableException("Bundle should not contain IBinder objects."); + } + } + + private static void validateParcelableArray(Parcelable[] parcelables) { + if (parcelables.length > 0 + && parcelables[0] instanceof ParcelFileDescriptor) { + // Safe to cast + validatePfdsReadOnly(parcelables); + } else if (parcelables.length > 0 + && parcelables[0] instanceof Bitmap) { + validateBitmapsImmutable(parcelables); + } else { + throw new BadParcelableException( + "Could not cast to any known parcelable array"); + } + } + + public static void validatePfdsReadOnly(Parcelable[] pfds) { + for (Parcelable pfd : pfds) { + validatePfdReadOnly((ParcelFileDescriptor) pfd); + } + } + + public static void validatePfdReadOnly(ParcelFileDescriptor pfd) { + if (pfd == null) { + return; + } + try { + int readMode = Os.fcntlInt(pfd.getFileDescriptor(), F_GETFL, 0) & O_ACCMODE; + if (readMode != O_RDONLY) { + throw new BadParcelableException( + "Bundle contains a parcel file descriptor which is not read-only."); + } + } catch (ErrnoException e) { + throw new BadParcelableException( + "Invalid File descriptor passed in the Bundle."); + } + } + + private static void validateBitmap(Bitmap obj) { + if (obj.isMutable()) { + throw new BadParcelableException( + "Encountered a mutable Bitmap in the Bundle at key : " + obj); + } + } + + private static void validateBitmapsImmutable(Parcelable[] bitmaps) { + for (Parcelable bitmap : bitmaps) { + validateBitmap((Bitmap) bitmap); + } + } + + public static void tryCloseResource(Bundle bundle) { + if (bundle == null || bundle.isEmpty() || !bundle.hasFileDescriptors()) { + return; + } + + for (String key : bundle.keySet()) { + Object obj = bundle.get(key); + + try { + // TODO(b/329898589) : This can be cleaned up after the flag passing is fixed. + if (obj instanceof ParcelFileDescriptor) { + ((ParcelFileDescriptor) obj).close(); + } else if (obj instanceof CursorWindow) { + ((CursorWindow) obj).close(); + } else if (obj instanceof SharedMemory) { + // TODO(b/331796886) : Shared memory should honour parcelable flags. + ((SharedMemory) obj).close(); + } + } catch (Exception e) { + Log.e(TAG, "Error closing resource with key: " + key, e); + } + } + } +} diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java new file mode 100644 index 000000000000..e8a1b322f661 --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/InferenceInfoStore.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import android.app.ondeviceintelligence.InferenceInfo; +import android.os.Bundle; +import android.os.PersistableBundle; +import android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService; +import android.util.Base64; +import android.util.Slog; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; +import java.util.TreeSet; + +/** + * @hide + */ +public class InferenceInfoStore { + private static final String TAG = "InferenceInfoStore"; + private final TreeSet<InferenceInfo> inferenceInfos; + private final long maxAgeMs; + + public InferenceInfoStore(long maxAgeMs) { + this.maxAgeMs = maxAgeMs; + this.inferenceInfos = new TreeSet<>( + Comparator.comparingLong(InferenceInfo::getStartTimeMillis)); + } + + public List<InferenceInfo> getLatestInferenceInfo(long startTimeEpochMillis) { + return inferenceInfos.stream().filter( + info -> info.getStartTimeMillis() > startTimeEpochMillis).toList(); + } + + public void addInferenceInfoFromBundle(PersistableBundle pb) { + if (!pb.containsKey(OnDeviceSandboxedInferenceService.INFERENCE_INFO_BUNDLE_KEY)) { + return; + } + + try { + String infoBytesBase64String = pb.getString( + OnDeviceSandboxedInferenceService.INFERENCE_INFO_BUNDLE_KEY); + if (infoBytesBase64String != null) { + byte[] infoBytes = Base64.decode(infoBytesBase64String, Base64.DEFAULT); + com.android.server.ondeviceintelligence.nano.InferenceInfo inferenceInfo = + com.android.server.ondeviceintelligence.nano.InferenceInfo.parseFrom( + infoBytes); + add(inferenceInfo); + } + } catch (IOException e) { + Slog.e(TAG, "Unable to parse InferenceInfo from the received bytes."); + } + } + + public void addInferenceInfoFromBundle(Bundle b) { + if (!b.containsKey(OnDeviceSandboxedInferenceService.INFERENCE_INFO_BUNDLE_KEY)) { + return; + } + + try { + byte[] infoBytes = b.getByteArray( + OnDeviceSandboxedInferenceService.INFERENCE_INFO_BUNDLE_KEY); + if (infoBytes != null) { + com.android.server.ondeviceintelligence.nano.InferenceInfo inferenceInfo = + com.android.server.ondeviceintelligence.nano.InferenceInfo.parseFrom( + infoBytes); + add(inferenceInfo); + } + } catch (IOException e) { + Slog.e(TAG, "Unable to parse InferenceInfo from the received bytes."); + } + } + + private synchronized void add(com.android.server.ondeviceintelligence.nano.InferenceInfo info) { + while (!inferenceInfos.isEmpty() + && System.currentTimeMillis() - inferenceInfos.first().getStartTimeMillis() + > maxAgeMs) { + inferenceInfos.pollFirst(); + } + inferenceInfos.add(toInferenceInfo(info)); + } + + private static InferenceInfo toInferenceInfo( + com.android.server.ondeviceintelligence.nano.InferenceInfo info) { + return new InferenceInfo.Builder(info.uid).setStartTimeMillis( + info.startTimeMs).setEndTimeMillis(info.endTimeMs).setSuspendedTimeMillis( + info.suspendedTimeMs).build(); + } +} diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java new file mode 100644 index 000000000000..6badc53ae97e --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerLocal.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import static android.app.ondeviceintelligence.flags.Flags.FLAG_ENABLE_ON_DEVICE_INTELLIGENCE_MODULE; + +import android.annotation.FlaggedApi; +import android.annotation.SystemApi; +import android.annotation.SystemApi.Client; + +/** + * Exposes APIs to {@code system_server} components outside of the module boundaries. + * <p> This API should be access using {@link com.android.server.LocalManagerRegistry}. </p> + * + * @hide + */ +@SystemApi(client = Client.SYSTEM_SERVER) +@FlaggedApi(FLAG_ENABLE_ON_DEVICE_INTELLIGENCE_MODULE) +public interface OnDeviceIntelligenceManagerLocal { + /** + * Gets the uid for the process that is currently hosting the + * {@link android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService} registered on + * the device. + */ + int getInferenceServiceUid(); +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java new file mode 100644 index 000000000000..a078f7542c11 --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceManagerService.java @@ -0,0 +1,1147 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import static android.app.ondeviceintelligence.OnDeviceIntelligenceManager.ON_DEVICE_INTELLIGENCE_IDLE_TIMEOUT_MS; +import static android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService.DEVICE_CONFIG_UPDATE_BUNDLE_KEY; +import static android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService.MODEL_LOADED_BROADCAST_INTENT; +import static android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService.MODEL_LOADED_BUNDLE_KEY; +import static android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService.MODEL_UNLOADED_BROADCAST_INTENT; +import static android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService.MODEL_UNLOADED_BUNDLE_KEY; +import static android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService.REGISTER_MODEL_UPDATE_CALLBACK_BUNDLE_KEY; + +import static com.android.server.ondeviceintelligence.BundleUtil.sanitizeInferenceParams; +import static com.android.server.ondeviceintelligence.BundleUtil.sanitizeStateParams; +import static com.android.server.ondeviceintelligence.BundleUtil.validatePfdReadOnly; +import static com.android.server.ondeviceintelligence.BundleUtil.wrapWithValidation; + +import android.Manifest; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.app.ondeviceintelligence.DownloadCallback; +import android.app.ondeviceintelligence.Feature; +import android.app.ondeviceintelligence.FeatureDetails; +import android.app.ondeviceintelligence.ICancellationSignal; +import android.app.ondeviceintelligence.IDownloadCallback; +import android.app.ondeviceintelligence.IFeatureCallback; +import android.app.ondeviceintelligence.IFeatureDetailsCallback; +import android.app.ondeviceintelligence.IListFeaturesCallback; +import android.app.ondeviceintelligence.IOnDeviceIntelligenceManager; +import android.app.ondeviceintelligence.IProcessingSignal; +import android.app.ondeviceintelligence.IRemoteCallback; +import android.app.ondeviceintelligence.IResponseCallback; +import android.app.ondeviceintelligence.IStreamingResponseCallback; +import android.app.ondeviceintelligence.ITokenInfoCallback; +import android.app.ondeviceintelligence.InferenceInfo; +import android.app.ondeviceintelligence.OnDeviceIntelligenceException; +import android.app.ondeviceintelligence.utils.BinderUtils; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ServiceInfo; +import android.content.res.Resources; +import android.os.Binder; +import android.os.Bundle; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.Message; +import android.os.ParcelFileDescriptor; +import android.os.PersistableBundle; +import android.os.RemoteCallback; +import android.os.RemoteException; +import android.os.UserHandle; +import android.provider.DeviceConfig; +import android.provider.Settings; +import android.service.ondeviceintelligence.IOnDeviceIntelligenceService; +import android.service.ondeviceintelligence.IOnDeviceSandboxedInferenceService; +import android.service.ondeviceintelligence.IProcessingUpdateStatusCallback; +import android.service.ondeviceintelligence.IRemoteProcessingService; +import android.service.ondeviceintelligence.IRemoteStorageService; +import android.service.ondeviceintelligence.OnDeviceIntelligenceService; +import android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService; +import android.text.TextUtils; +import android.util.Log; +import android.util.Slog; + +import com.android.internal.annotations.GuardedBy; +import com.android.modules.utils.AndroidFuture; +import com.android.modules.utils.BackgroundThread; +import com.android.modules.utils.ServiceConnector; +import com.android.server.LocalManagerRegistry; +import com.android.server.SystemService; +import com.android.server.ondeviceintelligence.callbacks.ListenableDownloadCallback; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * This is the system service for handling calls on the + * {@link android.app.ondeviceintelligence.OnDeviceIntelligenceManager}. This + * service holds connection references to the underlying remote services i.e. the isolated service + * {@link OnDeviceSandboxedInferenceService} and a regular + * service counter part {@link OnDeviceIntelligenceService}. + * + * Note: Both the remote services run under the SYSTEM user, as we cannot have separate instance of + * the Inference service for each user, due to possible high memory footprint. + * + * @hide + */ +public class OnDeviceIntelligenceManagerService extends SystemService { + + private static final String TAG = OnDeviceIntelligenceManagerService.class.getSimpleName(); + private static final String KEY_SERVICE_ENABLED = "service_enabled"; + + /** Handler message to {@link #resetTemporaryServices()} */ + private static final int MSG_RESET_TEMPORARY_SERVICE = 0; + /** Handler message to clean up temporary broadcast keys. */ + private static final int MSG_RESET_BROADCAST_KEYS = 1; + /** Handler message to clean up temporary config namespace. */ + private static final int MSG_RESET_CONFIG_NAMESPACE = 2; + + /** Default value in absence of {@link DeviceConfig} override. */ + private static final boolean DEFAULT_SERVICE_ENABLED = true; + private static final String NAMESPACE_ON_DEVICE_INTELLIGENCE = "ondeviceintelligence"; + + private static final String SYSTEM_PACKAGE = "android"; + private static final long MAX_AGE_MS = TimeUnit.HOURS.toMillis(3); + + + private final Executor resourceClosingExecutor = Executors.newCachedThreadPool(); + private final Executor callbackExecutor = Executors.newCachedThreadPool(); + private final Executor broadcastExecutor = Executors.newCachedThreadPool(); + private final Executor mConfigExecutor = Executors.newCachedThreadPool(); + + + private final Context mContext; + protected final Object mLock = new Object(); + + private final InferenceInfoStore mInferenceInfoStore; + private RemoteOnDeviceSandboxedInferenceService mRemoteInferenceService; + private RemoteOnDeviceIntelligenceService mRemoteOnDeviceIntelligenceService; + volatile boolean mIsServiceEnabled; + + @GuardedBy("mLock") + private int remoteInferenceServiceUid = -1; + + @GuardedBy("mLock") + private String[] mTemporaryServiceNames; + @GuardedBy("mLock") + private String[] mTemporaryBroadcastKeys; + @GuardedBy("mLock") + private String mBroadcastPackageName = SYSTEM_PACKAGE; + @GuardedBy("mLock") + private String mTemporaryConfigNamespace; + + private final DeviceConfig.OnPropertiesChangedListener mOnPropertiesChangedListener = + this::sendUpdatedConfig; + + + /** + * Handler used to reset the temporary service names. + */ + private Handler mTemporaryHandler; + private final @NonNull Handler mMainHandler = new Handler(Looper.getMainLooper()); + + + public OnDeviceIntelligenceManagerService(Context context) { + super(context); + mContext = context; + mTemporaryServiceNames = new String[0]; + mInferenceInfoStore = new InferenceInfoStore(MAX_AGE_MS); + } + + @Override + public void onStart() { + publishBinderService( + Context.ON_DEVICE_INTELLIGENCE_SERVICE, getOnDeviceIntelligenceManagerService(), + /* allowIsolated = */ true); + LocalManagerRegistry.addManager(OnDeviceIntelligenceManagerLocal.class, + this::getRemoteInferenceServiceUid); + } + + @Override + public void onBootPhase(int phase) { + if (phase == SystemService.PHASE_SYSTEM_SERVICES_READY) { + DeviceConfig.addOnPropertiesChangedListener( + NAMESPACE_ON_DEVICE_INTELLIGENCE, + BackgroundThread.getExecutor(), + (properties) -> onDeviceConfigChange(properties.getKeyset())); + + mIsServiceEnabled = isServiceEnabled(); + } + } + + private void onDeviceConfigChange(@NonNull Set<String> keys) { + if (keys.contains(KEY_SERVICE_ENABLED)) { + mIsServiceEnabled = isServiceEnabled(); + } + } + + private boolean isServiceEnabled() { + return DeviceConfig.getBoolean( + NAMESPACE_ON_DEVICE_INTELLIGENCE, + KEY_SERVICE_ENABLED, DEFAULT_SERVICE_ENABLED); + } + + private IBinder getOnDeviceIntelligenceManagerService() { + return new IOnDeviceIntelligenceManager.Stub() { + @Override + public String getRemoteServicePackageName() { + return OnDeviceIntelligenceManagerService.this.getRemoteConfiguredPackageName(); + } + + @Override + public List<InferenceInfo> getLatestInferenceInfo(long startTimeEpochMillis) { + mContext.enforceCallingPermission( + Manifest.permission.DUMP, TAG); + return OnDeviceIntelligenceManagerService.this.getLatestInferenceInfo( + startTimeEpochMillis); + } + + @Override + public void getVersion(RemoteCallback remoteCallback) { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal getVersion"); + Objects.requireNonNull(remoteCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + remoteCallback.sendResult(null); + return; + } + ensureRemoteIntelligenceServiceInitialized(/* throwServiceIfInvalid */ true); + mRemoteOnDeviceIntelligenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.getVersion(new RemoteCallback( + result -> { + remoteCallback.sendResult(result); + future.complete(null); + })); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + } + + @Override + public void getFeature(int id, IFeatureCallback featureCallback) + throws RemoteException { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal getFeatures"); + Objects.requireNonNull(featureCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + featureCallback.onFailure( + OnDeviceIntelligenceException.ON_DEVICE_INTELLIGENCE_SERVICE_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + return; + } + ensureRemoteIntelligenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + mRemoteOnDeviceIntelligenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.getFeature(callerUid, id, new IFeatureCallback.Stub() { + @Override + public void onSuccess(Feature result) throws RemoteException { + featureCallback.onSuccess(result); + future.complete(null); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) throws RemoteException { + featureCallback.onFailure(errorCode, errorMessage, errorParams); + future.completeExceptionally(new TimeoutException()); + } + }); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + } + + @Override + public void listFeatures(IListFeaturesCallback listFeaturesCallback) + throws RemoteException { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal getFeatures"); + Objects.requireNonNull(listFeaturesCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + listFeaturesCallback.onFailure( + OnDeviceIntelligenceException.ON_DEVICE_INTELLIGENCE_SERVICE_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + return; + } + ensureRemoteIntelligenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + mRemoteOnDeviceIntelligenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.listFeatures(callerUid, + new IListFeaturesCallback.Stub() { + @Override + public void onSuccess(List<Feature> result) + throws RemoteException { + listFeaturesCallback.onSuccess(result); + future.complete(null); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) + throws RemoteException { + listFeaturesCallback.onFailure(errorCode, errorMessage, + errorParams); + future.completeExceptionally(new TimeoutException()); + } + }); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + } + + @Override + public void getFeatureDetails(Feature feature, + IFeatureDetailsCallback featureDetailsCallback) + throws RemoteException { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal getFeatureStatus"); + Objects.requireNonNull(feature); + Objects.requireNonNull(featureDetailsCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + featureDetailsCallback.onFailure( + OnDeviceIntelligenceException.ON_DEVICE_INTELLIGENCE_SERVICE_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + return; + } + ensureRemoteIntelligenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + mRemoteOnDeviceIntelligenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.getFeatureDetails(callerUid, feature, + new IFeatureDetailsCallback.Stub() { + @Override + public void onSuccess(FeatureDetails result) + throws RemoteException { + future.complete(null); + featureDetailsCallback.onSuccess(result); + } + + @Override + public void onFailure(int errorCode, String errorMessage, + PersistableBundle errorParams) + throws RemoteException { + future.completeExceptionally(null); + featureDetailsCallback.onFailure(errorCode, + errorMessage, errorParams); + } + }); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + } + + @Override + public void requestFeatureDownload(Feature feature, + AndroidFuture cancellationSignalFuture, + IDownloadCallback downloadCallback) throws RemoteException { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal requestFeatureDownload"); + Objects.requireNonNull(feature); + Objects.requireNonNull(downloadCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + downloadCallback.onDownloadFailed( + DownloadCallback.DOWNLOAD_FAILURE_STATUS_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + } + ensureRemoteIntelligenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + mRemoteOnDeviceIntelligenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + ListenableDownloadCallback listenableDownloadCallback = + new ListenableDownloadCallback( + downloadCallback, + mMainHandler, future, getIdleTimeoutMs()); + service.requestFeatureDownload(callerUid, feature, + wrapCancellationFuture(cancellationSignalFuture), + listenableDownloadCallback); + return future; // this future has no timeout because, actual download + // might take long, but we fail early if there is no progress callbacks. + } + ); + } + + + @Override + public void requestTokenInfo(Feature feature, + Bundle request, + AndroidFuture cancellationSignalFuture, + ITokenInfoCallback tokenInfoCallback) throws RemoteException { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal requestTokenInfo"); + AndroidFuture<Void> result = null; + try { + Objects.requireNonNull(feature); + sanitizeInferenceParams(request); + Objects.requireNonNull(tokenInfoCallback); + + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + tokenInfoCallback.onFailure( + OnDeviceIntelligenceException.ON_DEVICE_INTELLIGENCE_SERVICE_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + } + ensureRemoteInferenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + result = mRemoteInferenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.requestTokenInfo(callerUid, feature, + request, + wrapCancellationFuture(cancellationSignalFuture), + wrapWithValidation(tokenInfoCallback, future, + mInferenceInfoStore)); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + result.whenCompleteAsync((c, e) -> BundleUtil.tryCloseResource(request), + resourceClosingExecutor); + } finally { + if (result == null) { + resourceClosingExecutor.execute(() -> BundleUtil.tryCloseResource(request)); + } + } + } + + @Override + public void processRequest(Feature feature, + Bundle request, + int requestType, + AndroidFuture cancellationSignalFuture, + AndroidFuture processingSignalFuture, + IResponseCallback responseCallback) + throws RemoteException { + AndroidFuture<Void> result = null; + try { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal processRequest"); + Objects.requireNonNull(feature); + sanitizeInferenceParams(request); + Objects.requireNonNull(responseCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + responseCallback.onFailure( + OnDeviceIntelligenceException.PROCESSING_ERROR_SERVICE_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + } + ensureRemoteInferenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + result = mRemoteInferenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.processRequest(callerUid, feature, + request, + requestType, + wrapCancellationFuture(cancellationSignalFuture), + wrapProcessingFuture(processingSignalFuture), + wrapWithValidation(responseCallback, + resourceClosingExecutor, future, + mInferenceInfoStore)); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + result.whenCompleteAsync((c, e) -> BundleUtil.tryCloseResource(request), + resourceClosingExecutor); + } finally { + if (result == null) { + resourceClosingExecutor.execute(() -> BundleUtil.tryCloseResource(request)); + } + } + } + + @Override + public void processRequestStreaming(Feature feature, + Bundle request, + int requestType, + AndroidFuture cancellationSignalFuture, + AndroidFuture processingSignalFuture, + IStreamingResponseCallback streamingCallback) throws RemoteException { + AndroidFuture<Void> result = null; + try { + Slog.i(TAG, "OnDeviceIntelligenceManagerInternal processRequestStreaming"); + Objects.requireNonNull(feature); + sanitizeInferenceParams(request); + Objects.requireNonNull(streamingCallback); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + if (!mIsServiceEnabled) { + Slog.w(TAG, "Service not available"); + streamingCallback.onFailure( + OnDeviceIntelligenceException.PROCESSING_ERROR_SERVICE_UNAVAILABLE, + "OnDeviceIntelligenceManagerService is unavailable", + PersistableBundle.EMPTY); + } + ensureRemoteInferenceServiceInitialized(/* throwServiceIfInvalid */ true); + int callerUid = Binder.getCallingUid(); + result = mRemoteInferenceService.postAsync( + service -> { + AndroidFuture future = new AndroidFuture(); + service.processRequestStreaming(callerUid, + feature, + request, requestType, + wrapCancellationFuture(cancellationSignalFuture), + wrapProcessingFuture(processingSignalFuture), + wrapWithValidation(streamingCallback, + resourceClosingExecutor, future, + mInferenceInfoStore)); + return future.orTimeout(getIdleTimeoutMs(), TimeUnit.MILLISECONDS); + }); + result.whenCompleteAsync((c, e) -> BundleUtil.tryCloseResource(request), + resourceClosingExecutor); + } finally { + if (result == null) { + resourceClosingExecutor.execute(() -> BundleUtil.tryCloseResource(request)); + } + } + } + + @Override + public int handleShellCommand(@NonNull ParcelFileDescriptor in, + @NonNull ParcelFileDescriptor out, @NonNull ParcelFileDescriptor err, + @NonNull String[] args) { + return new com.android.server.ondeviceintelligence.OnDeviceIntelligenceShellCommand( + OnDeviceIntelligenceManagerService.this).exec( + this, + in.getFileDescriptor(), + out.getFileDescriptor(), + err.getFileDescriptor(), + args); + } + }; + } + + private boolean ensureRemoteIntelligenceServiceInitialized(boolean throwIfServiceInvalid) { + synchronized (mLock) { + if (mRemoteOnDeviceIntelligenceService == null) { + String serviceName = getServiceNames()[0]; + if (!BinderUtils.withCleanCallingIdentity( + () -> validateServiceElevated(serviceName, false, + throwIfServiceInvalid))) { + return false; + } + mRemoteOnDeviceIntelligenceService = new RemoteOnDeviceIntelligenceService( + mContext, + ComponentName.unflattenFromString(serviceName), + UserHandle.SYSTEM.getIdentifier()); + mRemoteOnDeviceIntelligenceService.setServiceLifecycleCallbacks( + new ServiceConnector.ServiceLifecycleCallbacks<>() { + @Override + public void onConnected( + @NonNull IOnDeviceIntelligenceService service) { + try { + service.registerRemoteServices( + getRemoteProcessingService()); + service.ready(); + } catch (RemoteException ex) { + Slog.w(TAG, "Failed to send connected event", ex); + } + } + }); + } + } + return true; + } + + @NonNull + private IRemoteProcessingService.Stub getRemoteProcessingService() { + return new IRemoteProcessingService.Stub() { + @Override + public void updateProcessingState( + Bundle processingState, + IProcessingUpdateStatusCallback callback) { + callbackExecutor.execute(() -> { + AndroidFuture<Void> result = null; + try { + sanitizeStateParams(processingState); + if (ensureRemoteInferenceServiceInitialized(/* throwServiceIfInvalid */ + false)) { + result = mRemoteInferenceService.post( + service -> service.updateProcessingState( + processingState, callback)); + result.whenCompleteAsync( + (c, e) -> BundleUtil.tryCloseResource(processingState), + resourceClosingExecutor); + } else { + callback.onFailure( + OnDeviceIntelligenceException.PROCESSING_ERROR_SERVICE_UNAVAILABLE, + "Remote service cannot be initialized."); + } + } catch (RemoteException e) { + Slog.w("Failed to invoke updateProcessingState", e); + } finally { + if (result == null) { + resourceClosingExecutor.execute( + () -> BundleUtil.tryCloseResource(processingState)); + } + } + }); + } + }; + } + + private boolean ensureRemoteInferenceServiceInitialized(boolean throwIfServiceInvalid) { + synchronized (mLock) { + if (mRemoteInferenceService == null) { + String serviceName = getServiceNames()[1]; + if (!BinderUtils.withCleanCallingIdentity( + () -> validateServiceElevated(serviceName, true, throwIfServiceInvalid))) { + return false; + } + mRemoteInferenceService = new RemoteOnDeviceSandboxedInferenceService(mContext, + ComponentName.unflattenFromString(serviceName), + UserHandle.SYSTEM.getIdentifier()); + mRemoteInferenceService.setServiceLifecycleCallbacks( + new ServiceConnector.ServiceLifecycleCallbacks<>() { + @Override + public void onConnected( + @NonNull IOnDeviceSandboxedInferenceService service) { + try { + if (!ensureRemoteIntelligenceServiceInitialized( + /* throwServiceIfInvalid */ + false)) { + return; + } + service.registerRemoteStorageService( + getIRemoteStorageService(), new IRemoteCallback.Stub() { + @Override + public void sendResult(Bundle bundle) { + final int uid = Binder.getCallingUid(); + setRemoteInferenceServiceUid(uid); + } + }); + mRemoteOnDeviceIntelligenceService.run( + IOnDeviceIntelligenceService::notifyInferenceServiceConnected); + broadcastExecutor.execute( + () -> registerModelLoadingBroadcasts(service)); + mConfigExecutor.execute( + () -> registerDeviceConfigChangeListener()); + } catch (RemoteException ex) { + Slog.w(TAG, "Failed to send connected event", ex); + } + } + + @Override + public void onDisconnected( + @NonNull IOnDeviceSandboxedInferenceService service) { + if (!ensureRemoteIntelligenceServiceInitialized( + /* throwServiceIfInvalid */ + false)) { + return; + } + mRemoteOnDeviceIntelligenceService.run( + IOnDeviceIntelligenceService::notifyInferenceServiceDisconnected); + } + + @Override + public void onBinderDied() { + if (!ensureRemoteIntelligenceServiceInitialized( + /* throwServiceIfInvalid */ + false)) { + return; + } + mRemoteOnDeviceIntelligenceService.run( + IOnDeviceIntelligenceService::notifyInferenceServiceDisconnected); + } + }); + } + } + return true; + } + + private void registerModelLoadingBroadcasts(IOnDeviceSandboxedInferenceService service) { + String[] modelBroadcastKeys; + try { + modelBroadcastKeys = getBroadcastKeys(); + } catch (Resources.NotFoundException e) { + Slog.d(TAG, "Skipping model broadcasts as broadcast intents configured."); + return; + } + + Bundle bundle = new Bundle(); + bundle.putBoolean(REGISTER_MODEL_UPDATE_CALLBACK_BUNDLE_KEY, true); + try { + service.updateProcessingState(bundle, new IProcessingUpdateStatusCallback.Stub() { + @Override + public void onSuccess(PersistableBundle statusParams) { + Binder.clearCallingIdentity(); + synchronized (mLock) { + if (statusParams.containsKey(MODEL_LOADED_BUNDLE_KEY)) { + String modelLoadedBroadcastKey = modelBroadcastKeys[0]; + if (modelLoadedBroadcastKey != null + && !modelLoadedBroadcastKey.isEmpty()) { + final Intent intent = new Intent(modelLoadedBroadcastKey); + intent.setPackage(mBroadcastPackageName); + mContext.sendBroadcast(intent, + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE); + } + } else if (statusParams.containsKey(MODEL_UNLOADED_BUNDLE_KEY)) { + String modelUnloadedBroadcastKey = modelBroadcastKeys[1]; + if (modelUnloadedBroadcastKey != null + && !modelUnloadedBroadcastKey.isEmpty()) { + final Intent intent = new Intent(modelUnloadedBroadcastKey); + intent.setPackage(mBroadcastPackageName); + mContext.sendBroadcast(intent, + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE); + } + } + } + } + + @Override + public void onFailure(int errorCode, String errorMessage) { + Slog.e(TAG, "Failed to register model loading callback with status code", + new OnDeviceIntelligenceException(errorCode, errorMessage)); + } + }); + } catch (RemoteException e) { + Slog.e(TAG, "Failed to register model loading callback with status code", e); + } + } + + private void registerDeviceConfigChangeListener() { + Log.d(TAG, "registerDeviceConfigChangeListener"); + String configNamespace = getConfigNamespace(); + if (configNamespace.isEmpty()) { + Slog.e(TAG, "config_defaultOnDeviceIntelligenceDeviceConfigNamespace is empty"); + return; + } + DeviceConfig.addOnPropertiesChangedListener( + configNamespace, + mConfigExecutor, + mOnPropertiesChangedListener); + } + + private String getConfigNamespace() { + synchronized (mLock) { + if (mTemporaryConfigNamespace != null) { + return mTemporaryConfigNamespace; + } + return mContext.getResources().getString( + android.R.string.config_defaultOnDeviceIntelligenceDeviceConfigNamespace); + } + } + + private void sendUpdatedConfig( + DeviceConfig.Properties props) { + Log.d(TAG, "sendUpdatedConfig"); + + PersistableBundle persistableBundle = new PersistableBundle(); + for (String key : props.getKeyset()) { + persistableBundle.putString(key, props.getString(key, "")); + } + Bundle bundle = new Bundle(); + bundle.putParcelable(DEVICE_CONFIG_UPDATE_BUNDLE_KEY, persistableBundle); + if (!ensureRemoteIntelligenceServiceInitialized( + /* throwServiceIfInvalid */ + false)) { + return; + } + mRemoteInferenceService.run(service -> service.updateProcessingState(bundle, + new IProcessingUpdateStatusCallback.Stub() { + @Override + public void onSuccess(PersistableBundle result) { + Slog.d(TAG, "Config update successful." + result); + } + + @Override + public void onFailure(int errorCode, String errorMessage) { + Slog.e(TAG, "Config update failed with code [" + + String.valueOf(errorCode) + "] and message = " + errorMessage); + } + })); + } + + @NonNull + private IRemoteStorageService.Stub getIRemoteStorageService() { + return new IRemoteStorageService.Stub() { + @Override + public void getReadOnlyFileDescriptor( + String filePath, + AndroidFuture<ParcelFileDescriptor> future) { + if (!ensureRemoteIntelligenceServiceInitialized( + /* throwServiceIfInvalid */ + false)) { + future.completeExceptionally(new OnDeviceIntelligenceException( + OnDeviceIntelligenceException.PROCESSING_ERROR_NOT_AVAILABLE)); + return; + } + AndroidFuture<ParcelFileDescriptor> pfdFuture = new AndroidFuture<>(); + mRemoteOnDeviceIntelligenceService.run( + service -> service.getReadOnlyFileDescriptor( + filePath, pfdFuture)); + pfdFuture.whenCompleteAsync((pfd, error) -> { + try { + if (error != null) { + future.completeExceptionally(error); + } else { + validatePfdReadOnly(pfd); + future.complete(pfd); + } + } finally { + tryClosePfd(pfd); + } + }, callbackExecutor); + } + + @Override + public void getReadOnlyFeatureFileDescriptorMap( + Feature feature, + RemoteCallback remoteCallback) { + ensureRemoteIntelligenceServiceInitialized(/* throwServiceIfInvalid */ true); + mRemoteOnDeviceIntelligenceService.run( + service -> service.getReadOnlyFeatureFileDescriptorMap( + feature, + new RemoteCallback(result -> callbackExecutor.execute(() -> { + try { + if (result == null) { + remoteCallback.sendResult(null); + } + for (String key : result.keySet()) { + ParcelFileDescriptor pfd = result.getParcelable(key, + ParcelFileDescriptor.class); + validatePfdReadOnly(pfd); + } + remoteCallback.sendResult(result); + } finally { + resourceClosingExecutor.execute( + () -> BundleUtil.tryCloseResource(result)); + } + })))); + } + }; + } + + private boolean validateServiceElevated(String serviceName, boolean checkIsolated, + boolean throwIfServiceInvalid) { + try { + if (TextUtils.isEmpty(serviceName)) { + if (throwIfServiceInvalid) { + throw new IllegalStateException( + "Remote service is not configured to complete the request"); + } + return false; + } + ComponentName serviceComponent = ComponentName.unflattenFromString( + serviceName); + ServiceInfo serviceInfo = mContext.getPackageManager().getServiceInfo( + serviceComponent, + PackageManager.MATCH_DIRECT_BOOT_AWARE + | PackageManager.MATCH_DIRECT_BOOT_UNAWARE); + if (!checkIsolated) { + checkServiceRequiresPermission(serviceInfo, + Manifest.permission.BIND_ON_DEVICE_INTELLIGENCE_SERVICE); + return true; + } + + checkServiceRequiresPermission(serviceInfo, + Manifest.permission.BIND_ON_DEVICE_SANDBOXED_INFERENCE_SERVICE); + if (!isIsolatedService(serviceInfo)) { + throw new SecurityException( + "Call required an isolated service, but the configured service: " + + serviceName + ", is not isolated"); + } + } catch (PackageManager.NameNotFoundException e) { + if (throwIfServiceInvalid) { + throw new IllegalStateException( + "Remote service is not configured to complete the request."); + } + return false; + } catch (SecurityException e) { + if (throwIfServiceInvalid) { + throw e; + } + return false; + } + return true; + } + + private static void checkServiceRequiresPermission(ServiceInfo serviceInfo, + String requiredPermission) { + final String permission = serviceInfo.permission; + if (!requiredPermission.equals(permission)) { + throw new SecurityException(String.format( + "%s requires %s permission. Found %s permission", + serviceInfo, + requiredPermission, + serviceInfo.permission)); + } + } + + private static boolean isIsolatedService(@NonNull ServiceInfo serviceInfo) { + return (serviceInfo.flags & ServiceInfo.FLAG_ISOLATED_PROCESS) != 0 + && (serviceInfo.flags & ServiceInfo.FLAG_EXTERNAL_SERVICE) == 0; + } + + private List<InferenceInfo> getLatestInferenceInfo(long startTimeEpochMillis) { + return mInferenceInfoStore.getLatestInferenceInfo(startTimeEpochMillis); + } + + @Nullable + public String getRemoteConfiguredPackageName() { + try { + String[] serviceNames = getServiceNames(); + ComponentName componentName = ComponentName.unflattenFromString(serviceNames[1]); + if (componentName != null) { + return componentName.getPackageName(); + } + } catch (Resources.NotFoundException e) { + Slog.e(TAG, "Could not find resource", e); + } + + return null; + } + + + protected String[] getServiceNames() throws Resources.NotFoundException { + // TODO 329240495 : Consider a small class with explicit field names for the two services + synchronized (mLock) { + if (mTemporaryServiceNames != null && mTemporaryServiceNames.length == 2) { + return mTemporaryServiceNames; + } + } + return new String[]{mContext.getResources().getString( + android.R.string.config_defaultOnDeviceIntelligenceService), + mContext.getResources().getString( + android.R.string.config_defaultOnDeviceSandboxedInferenceService)}; + } + + protected String[] getBroadcastKeys() throws Resources.NotFoundException { + // TODO 329240495 : Consider a small class with explicit field names for the two services + synchronized (mLock) { + if (mTemporaryBroadcastKeys != null && mTemporaryBroadcastKeys.length == 2) { + return mTemporaryBroadcastKeys; + } + } + + return new String[]{MODEL_LOADED_BROADCAST_INTENT, MODEL_UNLOADED_BROADCAST_INTENT}; + } + + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void setTemporaryServices(@NonNull String[] componentNames, int durationMs) { + Objects.requireNonNull(componentNames); + enforceShellOnly(Binder.getCallingUid(), "setTemporaryServices"); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + synchronized (mLock) { + mTemporaryServiceNames = componentNames; + if (mRemoteInferenceService != null) { + mRemoteInferenceService.unbind(); + mRemoteInferenceService = null; + } + if (mRemoteOnDeviceIntelligenceService != null) { + mRemoteOnDeviceIntelligenceService.unbind(); + mRemoteOnDeviceIntelligenceService = null; + } + + if (durationMs != -1) { + getTemporaryHandler().sendEmptyMessageDelayed(MSG_RESET_TEMPORARY_SERVICE, + durationMs); + } + } + } + + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void setModelBroadcastKeys(@NonNull String[] broadcastKeys, String receiverPackageName, + int durationMs) { + Objects.requireNonNull(broadcastKeys); + enforceShellOnly(Binder.getCallingUid(), "setModelBroadcastKeys"); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + synchronized (mLock) { + mTemporaryBroadcastKeys = broadcastKeys; + mBroadcastPackageName = receiverPackageName; + if (durationMs != -1) { + getTemporaryHandler().sendEmptyMessageDelayed(MSG_RESET_BROADCAST_KEYS, durationMs); + } + } + } + + @RequiresPermission(Manifest.permission.USE_ON_DEVICE_INTELLIGENCE) + public void setTemporaryDeviceConfigNamespace(@NonNull String configNamespace, + int durationMs) { + Objects.requireNonNull(configNamespace); + enforceShellOnly(Binder.getCallingUid(), "setTemporaryDeviceConfigNamespace"); + mContext.enforceCallingPermission( + Manifest.permission.USE_ON_DEVICE_INTELLIGENCE, TAG); + synchronized (mLock) { + mTemporaryConfigNamespace = configNamespace; + if (durationMs != -1) { + getTemporaryHandler().sendEmptyMessageDelayed(MSG_RESET_CONFIG_NAMESPACE, + durationMs); + } + } + } + + /** + * Reset the temporary services set in CTS tests, this method is primarily used to only revert + * the changes caused by CTS tests. + */ + public void resetTemporaryServices() { + synchronized (mLock) { + if (mTemporaryHandler != null) { + mTemporaryHandler.removeMessages(MSG_RESET_TEMPORARY_SERVICE); + mTemporaryHandler = null; + } + + mRemoteInferenceService = null; + mRemoteOnDeviceIntelligenceService = null; + mTemporaryServiceNames = new String[0]; + } + } + + /** + * Throws if the caller is not of a shell (or root) UID. + * + * @param callingUid pass Binder.callingUid(). + */ + public static void enforceShellOnly(int callingUid, String message) { + if (callingUid == android.os.Process.SHELL_UID + || callingUid == android.os.Process.ROOT_UID) { + return; // okay + } + + throw new SecurityException(message + ": Only shell user can call it"); + } + + private AndroidFuture<IBinder> wrapCancellationFuture( + AndroidFuture future) { + if (future == null) { + return null; + } + AndroidFuture<IBinder> cancellationFuture = new AndroidFuture<>(); + cancellationFuture.whenCompleteAsync((c, e) -> { + if (e != null) { + Log.e(TAG, "Error forwarding ICancellationSignal to manager layer", e); + future.completeExceptionally(e); + } else { + future.complete(new ICancellationSignal.Stub() { + @Override + public void cancel() throws RemoteException { + ICancellationSignal.Stub.asInterface(c).cancel(); + } + }); + } + }); + return cancellationFuture; + } + + private AndroidFuture<IBinder> wrapProcessingFuture( + AndroidFuture future) { + if (future == null) { + return null; + } + AndroidFuture<IBinder> processingSignalFuture = new AndroidFuture<>(); + processingSignalFuture.whenCompleteAsync((c, e) -> { + if (e != null) { + future.completeExceptionally(e); + } else { + future.complete(new IProcessingSignal.Stub() { + @Override + public void sendSignal(PersistableBundle actionParams) throws RemoteException { + IProcessingSignal.Stub.asInterface(c).sendSignal(actionParams); + } + }); + } + }); + return processingSignalFuture; + } + + private static void tryClosePfd(ParcelFileDescriptor pfd) { + if (pfd != null) { + try { + pfd.close(); + } catch (IOException e) { + Log.e(TAG, "Failed to close parcel file descriptor ", e); + } + } + } + + private synchronized Handler getTemporaryHandler() { + if (mTemporaryHandler == null) { + mTemporaryHandler = new Handler(Looper.getMainLooper()) { + @Override + public void handleMessage(Message msg) { + synchronized (mLock) { + if (msg.what == MSG_RESET_TEMPORARY_SERVICE) { + resetTemporaryServices(); + } else if (msg.what == MSG_RESET_BROADCAST_KEYS) { + mTemporaryBroadcastKeys = null; + mBroadcastPackageName = SYSTEM_PACKAGE; + } else if (msg.what == MSG_RESET_CONFIG_NAMESPACE) { + mTemporaryConfigNamespace = null; + } else { + Slog.wtf(TAG, "invalid handler msg: " + msg); + } + } + } + }; + } + + return mTemporaryHandler; + } + + // Using #getLong here as the timeout settings are only applicable to the services running in + // SYSTEM user only. + @SuppressWarnings("NonUserGetterCalled") + private long getIdleTimeoutMs() { + return Settings.Secure.getLong(mContext.getContentResolver(), + ON_DEVICE_INTELLIGENCE_IDLE_TIMEOUT_MS, + TimeUnit.HOURS.toMillis(1)); + } + + private int getRemoteInferenceServiceUid() { + synchronized (mLock) { + return remoteInferenceServiceUid; + } + } + + private void setRemoteInferenceServiceUid(int remoteInferenceServiceUid) { + synchronized (mLock) { + this.remoteInferenceServiceUid = remoteInferenceServiceUid; + } + } +} diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java new file mode 100644 index 000000000000..c641de8b47b1 --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/OnDeviceIntelligenceShellCommand.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import android.annotation.NonNull; +import android.os.Binder; + +import com.android.modules.utils.BasicShellCommandHandler; + +import java.io.PrintWriter; +import java.util.Objects; + +/** + * @hide + */ +final class OnDeviceIntelligenceShellCommand extends BasicShellCommandHandler { + private static final String TAG = OnDeviceIntelligenceShellCommand.class.getSimpleName(); + + @NonNull + private final OnDeviceIntelligenceManagerService mService; + + OnDeviceIntelligenceShellCommand(@NonNull OnDeviceIntelligenceManagerService service) { + mService = service; + } + + @Override + public int onCommand(String cmd) { + if (cmd == null) { + return handleDefaultCommands(cmd); + } + + switch (cmd) { + case "set-temporary-services": + return setTemporaryServices(); + case "get-services": + return getConfiguredServices(); + case "set-model-broadcasts": + return setBroadcastKeys(); + case "set-deviceconfig-namespace": + return setDeviceConfigNamespace(); + default: + return handleDefaultCommands(cmd); + } + } + + @Override + public void onHelp() { + PrintWriter pw = getOutPrintWriter(); + pw.println("OnDeviceIntelligenceShellCommand commands: "); + pw.println(" help"); + pw.println(" Print this help text."); + pw.println(); + pw.println( + " set-temporary-services [IntelligenceServiceComponentName] " + + "[InferenceServiceComponentName] [DURATION]"); + pw.println(" Temporarily (for DURATION ms) changes the service implementations."); + pw.println(" To reset, call without any arguments."); + + pw.println(" get-services To get the names of services that are currently being used."); + pw.println( + " set-model-broadcasts [ModelLoadedBroadcastKey] [ModelUnloadedBroadcastKey] " + + "[ReceiverPackageName] " + + "[DURATION] To set the names of broadcast intent keys that are to be " + + "emitted for cts tests."); + pw.println( + " set-deviceconfig-namespace [DeviceConfigNamespace] " + + "[DURATION] To set the device config namespace " + + "to use for cts tests."); + } + + private int setTemporaryServices() { + final PrintWriter out = getOutPrintWriter(); + final String intelligenceServiceName = getNextArg(); + final String inferenceServiceName = getNextArg(); + + if (getRemainingArgsCount() == 0 && intelligenceServiceName == null + && inferenceServiceName == null) { + OnDeviceIntelligenceManagerService.enforceShellOnly(Binder.getCallingUid(), + "resetTemporaryServices"); + mService.resetTemporaryServices(); + out.println("OnDeviceIntelligenceManagerService temporary reset. "); + return 0; + } + + Objects.requireNonNull(intelligenceServiceName); + Objects.requireNonNull(inferenceServiceName); + final int duration = Integer.parseInt(getNextArgRequired()); + mService.setTemporaryServices( + new String[]{intelligenceServiceName, inferenceServiceName}, + duration); + out.println("OnDeviceIntelligenceService temporarily set to " + intelligenceServiceName + + " \n and \n OnDeviceTrustedInferenceService set to " + inferenceServiceName + + " for " + duration + "ms"); + return 0; + } + + private int getConfiguredServices() { + final PrintWriter out = getOutPrintWriter(); + String[] services = mService.getServiceNames(); + out.println("OnDeviceIntelligenceService set to : " + services[0] + + " \n and \n OnDeviceTrustedInferenceService set to : " + services[1]); + return 0; + } + + private int setBroadcastKeys() { + final PrintWriter out = getOutPrintWriter(); + final String modelLoadedKey = getNextArgRequired(); + final String modelUnloadedKey = getNextArgRequired(); + final String receiverPackageName = getNextArg(); + + final int duration = Integer.parseInt(getNextArgRequired()); + mService.setModelBroadcastKeys( + new String[]{modelLoadedKey, modelUnloadedKey}, receiverPackageName, duration); + out.println("OnDeviceIntelligence Model Loading broadcast keys temporarily set to " + + modelLoadedKey + + " \n and \n OnDeviceTrustedInferenceService set to " + modelUnloadedKey + + "\n and Package name set to : " + receiverPackageName + + " for " + duration + "ms"); + return 0; + } + + private int setDeviceConfigNamespace() { + final PrintWriter out = getOutPrintWriter(); + final String configNamespace = getNextArg(); + + final int duration = Integer.parseInt(getNextArgRequired()); + mService.setTemporaryDeviceConfigNamespace(configNamespace, duration); + out.println("OnDeviceIntelligence DeviceConfig Namespace temporarily set to " + + configNamespace + + " for " + duration + "ms"); + return 0; + } + +}
\ No newline at end of file diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java new file mode 100644 index 000000000000..0c43a309c456 --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/RemoteOnDeviceIntelligenceService.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import static android.app.ondeviceintelligence.OnDeviceIntelligenceManager.ON_DEVICE_INTELLIGENCE_UNBIND_TIMEOUT_MS; +import static android.content.Context.BIND_FOREGROUND_SERVICE; +import static android.content.Context.BIND_INCLUDE_CAPABILITIES; + +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.provider.Settings; +import android.service.ondeviceintelligence.IOnDeviceIntelligenceService; +import android.service.ondeviceintelligence.OnDeviceIntelligenceService; + +import com.android.modules.utils.ServiceConnector; + +import java.util.concurrent.TimeUnit; + +/** + * Manages the connection to the remote on-device intelligence service. Also, handles unbinding + * logic set by the service implementation via a Secure Settings flag. + * + * @hide + */ +public class RemoteOnDeviceIntelligenceService extends + ServiceConnector.Impl<IOnDeviceIntelligenceService> { + private static final long LONG_TIMEOUT = TimeUnit.HOURS.toMillis(4); + private static final String TAG = + RemoteOnDeviceIntelligenceService.class.getSimpleName(); + + RemoteOnDeviceIntelligenceService(Context context, ComponentName serviceName, + int userId) { + super(context, new Intent( + OnDeviceIntelligenceService.SERVICE_INTERFACE).setComponent(serviceName), + BIND_FOREGROUND_SERVICE | BIND_INCLUDE_CAPABILITIES, userId, + IOnDeviceIntelligenceService.Stub::asInterface); + + // Bind right away + connect(); + } + + @Override + protected long getRequestTimeoutMs() { + return LONG_TIMEOUT; + } + + // Using #getLong here as the timeout settings are only applicable to the services running in + // SYSTEM user only. + @Override + @SuppressWarnings("NonUserGetterCalled") + protected long getAutoDisconnectTimeoutMs() { + return Settings.Secure.getLong(mContext.getContentResolver(), + ON_DEVICE_INTELLIGENCE_UNBIND_TIMEOUT_MS, + TimeUnit.SECONDS.toMillis(30)); + } +} diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java new file mode 100644 index 000000000000..8c5d5a7ba736 --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/RemoteOnDeviceSandboxedInferenceService.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence; + +import static android.app.ondeviceintelligence.OnDeviceIntelligenceManager.ON_DEVICE_INFERENCE_UNBIND_TIMEOUT_MS; +import static android.content.Context.BIND_FOREGROUND_SERVICE; +import static android.content.Context.BIND_INCLUDE_CAPABILITIES; + +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.provider.Settings; +import android.service.ondeviceintelligence.IOnDeviceSandboxedInferenceService; +import android.service.ondeviceintelligence.OnDeviceSandboxedInferenceService; + +import com.android.modules.utils.ServiceConnector; + +import java.util.concurrent.TimeUnit; + + +/** + * Manages the connection to the remote on-device sand boxed inference service. Also, handles + * unbinding + * logic set by the service implementation via a SecureSettings flag. + * + * @hide + */ +public class RemoteOnDeviceSandboxedInferenceService extends + ServiceConnector.Impl<IOnDeviceSandboxedInferenceService> { + private static final long LONG_TIMEOUT = TimeUnit.HOURS.toMillis(1); + + /** + * Creates an instance of {@link ServiceConnector} + * + * See {@code protected} methods for optional parameters you can override. + * + * @param context to be used for {@link Context#bindServiceAsUser binding} and + * {@link Context#unbindService unbinding} + * @param userId to be used for {@link Context#bindServiceAsUser binding} + */ + RemoteOnDeviceSandboxedInferenceService(Context context, ComponentName serviceName, + int userId) { + super(context, new Intent( + OnDeviceSandboxedInferenceService.SERVICE_INTERFACE).setComponent(serviceName), + BIND_FOREGROUND_SERVICE | BIND_INCLUDE_CAPABILITIES, userId, + IOnDeviceSandboxedInferenceService.Stub::asInterface); + + // Bind right away + connect(); + } + + @Override + protected long getRequestTimeoutMs() { + return LONG_TIMEOUT; + } + + // Using #getLong here as the timeout settings are only applicable to the services running in + // SYSTEM user only. + @Override + @SuppressWarnings("NonUserGetterCalled") + protected long getAutoDisconnectTimeoutMs() { + return Settings.Secure.getLong(mContext.getContentResolver(), + ON_DEVICE_INFERENCE_UNBIND_TIMEOUT_MS, + TimeUnit.SECONDS.toMillis(30)); + } +} diff --git a/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java new file mode 100644 index 000000000000..249bcd37db5d --- /dev/null +++ b/packages/NeuralNetworks/service/platform/java/com/android/server/ondeviceintelligence/callbacks/ListenableDownloadCallback.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.ondeviceintelligence.callbacks; + +import android.app.ondeviceintelligence.IDownloadCallback; +import android.os.Handler; +import android.os.PersistableBundle; +import android.os.RemoteException; + +import com.android.modules.utils.AndroidFuture; + +import java.util.concurrent.TimeoutException; + +/** + * This class extends the {@link IDownloadCallback} and adds a timeout Runnable to the callback + * such that, in the case where the callback methods are not invoked, we do not have to wait for + * timeout based on {@link #onDownloadCompleted} which might take minutes or hours to complete in + * some cases. Instead, in such cases we rely on the remote service sending progress updates and if + * there are *no* progress callbacks in the duration of {@link #idleTimeoutMs}, we can assume the + * download will not complete and enabling faster cleanup. + * + * @hide + */ +public class ListenableDownloadCallback extends IDownloadCallback.Stub implements Runnable { + private final IDownloadCallback callback; + private final Handler handler; + private final AndroidFuture future; + private final long idleTimeoutMs; + + /** + * Constructor to create a ListenableDownloadCallback. + * + * @param callback callback to send download updates to caller. + * @param handler handler to schedule timeout runnable. + * @param future future to complete to signal the callback has reached a terminal state. + * @param idleTimeoutMs timeout within which download updates should be received. + */ + public ListenableDownloadCallback(IDownloadCallback callback, Handler handler, + AndroidFuture future, + long idleTimeoutMs) { + this.callback = callback; + this.handler = handler; + this.future = future; + this.idleTimeoutMs = idleTimeoutMs; + handler.postDelayed(this, + idleTimeoutMs); // init the timeout runnable in case no callback is ever invoked + } + + @Override + public void onDownloadStarted(long bytesToDownload) throws RemoteException { + callback.onDownloadStarted(bytesToDownload); + handler.removeCallbacks(this); + handler.postDelayed(this, idleTimeoutMs); + } + + @Override + public void onDownloadProgress(long bytesDownloaded) throws RemoteException { + callback.onDownloadProgress(bytesDownloaded); + handler.removeCallbacks(this); // remove previously queued timeout tasks. + handler.postDelayed(this, idleTimeoutMs); // queue fresh timeout task for next update. + } + + @Override + public void onDownloadFailed(int failureStatus, + String errorMessage, PersistableBundle errorParams) throws RemoteException { + callback.onDownloadFailed(failureStatus, errorMessage, errorParams); + handler.removeCallbacks(this); + future.completeExceptionally(new TimeoutException()); + } + + @Override + public void onDownloadCompleted( + android.os.PersistableBundle downloadParams) throws RemoteException { + callback.onDownloadCompleted(downloadParams); + handler.removeCallbacks(this); + future.complete(null); + } + + @Override + public void run() { + future.completeExceptionally( + new TimeoutException()); // complete the future as we haven't received updates + // for download progress. + } +}
\ No newline at end of file |