framework import
diff --git a/FaceShared/src/Android.bp b/FaceShared/src/Android.bp
index 9707493..7015623 100644
--- a/FaceShared/src/Android.bp
+++ b/FaceShared/src/Android.bp
@@ -3,27 +3,18 @@
sdk_version: "current",
srcs: ["main/java/**/*.java", "withoutGpu/java/**/*.java"],
manifest: "main/AndroidManifest.xml",
+ required: ["detect-class1.tflite", "mobile_face_net.tflite"],
static_libs: [
"tensorflowlite_java",
],
}
-android_library {
- name: "LMOFaceShared",
- sdk_version: "current",
- manifest: "main/AndroidManifest.xml",
- asset_dirs: ["main/assets"],
-
- static_libs: [
- "LMOFaceShared_java",
- ],
-}
-
prebuilt_etc {
name: "detect-class1.tflite",
src: "main/assets/detect-class1.tflite",
sub_dir: "face",
+ required: ["detect-class1.txt"],
}
prebuilt_etc {
@@ -36,6 +27,7 @@
name: "mobile_face_net.tflite",
src: "main/assets/mobile_face_net.tflite",
sub_dir: "face",
+ required: ["mobile_face_net.txt"],
}
prebuilt_etc {
diff --git a/adapter/Android.bp b/adapter/Android.bp
index 172f7ee..6e9b830 100644
--- a/adapter/Android.bp
+++ b/adapter/Android.bp
@@ -4,8 +4,10 @@
srcs: ["java/**/*.java"],
static_libs: [
"android.hardware.biometrics.face-V1.0-java",
+ "LMOFaceClient",
],
vintf_fragments: ["manifest_face_lmodroid.xml"],
+ required: ["LMOFaceHalAdapterService.rc"],
}
prebuilt_etc {
diff --git a/adapter/java/com/libremobileos/faceunlock/FaceHalAdapterService.java b/adapter/java/com/libremobileos/faceunlock/FaceHalAdapterService.java
index 06f592b..e5e2ff9 100644
--- a/adapter/java/com/libremobileos/faceunlock/FaceHalAdapterService.java
+++ b/adapter/java/com/libremobileos/faceunlock/FaceHalAdapterService.java
@@ -11,8 +11,8 @@
import android.os.ServiceManager;
import android.util.Log;
-import com.android.internal.libremobileos.faceunlock.IFaceHalService;
-import com.android.internal.libremobileos.faceunlock.IFaceHalServiceCallback;
+import com.libremobileos.faceunlock.client.IFaceHalService;
+import com.libremobileos.faceunlock.client.IFaceHalServiceCallback;
import java.util.ArrayList;
diff --git a/app/src/main/Android.bp b/app/src/main/Android.bp
index 9e54bce..27a7558 100644
--- a/app/src/main/Android.bp
+++ b/app/src/main/Android.bp
@@ -13,6 +13,7 @@
"androidx.annotation_annotation",
"androidx.cardview_cardview",
"LMOFaceShared_java",
+ "LMOFaceClient",
"android.hardware.biometrics.face-V1.0-java",
],
jni_libs: ["libtensorflowlite_jni"],
diff --git a/app/src/main/java/com/libremobileos/faceunlock/CameraActivity.java b/app/src/main/java/com/libremobileos/faceunlock/CameraActivity.java
index 0c1f786..f63638d 100644
--- a/app/src/main/java/com/libremobileos/faceunlock/CameraActivity.java
+++ b/app/src/main/java/com/libremobileos/faceunlock/CameraActivity.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2023 LibreMobileOS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package com.libremobileos.faceunlock;
import android.app.Activity;
diff --git a/app/src/main/java/com/libremobileos/faceunlock/RemoteFaceServiceClient.java b/app/src/main/java/com/libremobileos/faceunlock/RemoteFaceServiceClient.java
index 8e0b3d2..88b5116 100644
--- a/app/src/main/java/com/libremobileos/faceunlock/RemoteFaceServiceClient.java
+++ b/app/src/main/java/com/libremobileos/faceunlock/RemoteFaceServiceClient.java
@@ -1,6 +1,21 @@
+/*
+ * Copyright 2023 LibreMobileOS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package com.libremobileos.faceunlock;
-import android.content.Context;
import android.util.Base64;
import android.util.Log;
@@ -16,7 +31,6 @@
public abstract class RemoteFaceServiceClient {
public static final String FACE = "Face";
- public static final String SECURE = "secure";
public static void connect(String dir, Consumer<RemoteFaceServiceClient> callback) {
new Thread(() -> {
@@ -30,11 +44,12 @@
@Override
public boolean isSecure() {
- return false;
+ return false; // TODO
}
@Override
public void setSecure(boolean secure) {
+ // TODO
}
@Override
diff --git a/app/src/main/java/com/libremobileos/faceunlock/ScanActivity.java b/app/src/main/java/com/libremobileos/faceunlock/ScanActivity.java
index adce2cc..6d23635 100644
--- a/app/src/main/java/com/libremobileos/faceunlock/ScanActivity.java
+++ b/app/src/main/java/com/libremobileos/faceunlock/ScanActivity.java
@@ -16,7 +16,7 @@
package com.libremobileos.faceunlock;
-import static com.android.internal.libremobileos.faceunlock.FaceUnlockManager.SERVICE_NAME;
+import static com.libremobileos.faceunlock.client.FaceUnlockManager.SERVICE_NAME;
import android.annotation.SuppressLint;
import android.content.ComponentName;
@@ -39,7 +39,8 @@
import com.libremobileos.yifan.face.FaceDetector;
import com.libremobileos.yifan.face.FaceFinder;
import com.libremobileos.yifan.face.FaceScanner;
-import com.android.internal.libremobileos.faceunlock.IFaceUnlockManager;
+
+import com.libremobileos.faceunlock.client.IFaceUnlockManager;
import java.util.ArrayList;
import java.util.List;
diff --git a/framework/Android.bp b/framework/Android.bp
new file mode 100644
index 0000000..6353d1a
--- /dev/null
+++ b/framework/Android.bp
@@ -0,0 +1,21 @@
+java_library_static {
+ name: "LMOFaceClient",
+ platform_apis: true,
+ srcs: ["client/**/*.java", "client/**/*.aidl"],
+ aidl: {
+ include_dirs: [
+ "packages/apps/FaceUnlock/framework/client",
+ ],
+ },
+}
+
+java_library_static {
+ name: "LMOFaceServer",
+ platform_apis: true,
+ srcs: ["server/**/*.java"],
+ static_libs: [
+ "LMOFaceClient",
+ "LMOFaceShared_java",
+ "android.hardware.biometrics.face-V1.0-java",
+ ],
+}
\ No newline at end of file
diff --git a/framework/client/com/libremobileos/faceunlock/client/FaceUnlockManager.java b/framework/client/com/libremobileos/faceunlock/client/FaceUnlockManager.java
new file mode 100644
index 0000000..5230467
--- /dev/null
+++ b/framework/client/com/libremobileos/faceunlock/client/FaceUnlockManager.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.client;
+
+import android.os.RemoteException;
+import android.os.ServiceManager;
+
+public final class FaceUnlockManager {
+
+ public static final String SERVICE_NAME = "faceunlock";
+ private static final String TAG = "FaceUnlockManager";
+
+ private static FaceUnlockManager sFaceUnlockManager;
+ private IFaceUnlockManager mFaceUnlockManager;
+
+ private FaceUnlockManager() {
+ mFaceUnlockManager = IFaceUnlockManager.Stub.asInterface(
+ ServiceManager.getService(SERVICE_NAME));
+ if (mFaceUnlockManager == null)
+ throw new RuntimeException("Unable to get FaceUnlockService.");
+ }
+
+ public static FaceUnlockManager getInstance() {
+ if (sFaceUnlockManager != null)
+ return sFaceUnlockManager;
+ sFaceUnlockManager = new FaceUnlockManager();
+ return sFaceUnlockManager;
+ }
+
+ /**
+ * Send enroll result remainings to HAL.
+ */
+ public void enrollResult(int remaining) {
+ try {
+ mFaceUnlockManager.enrollResult(remaining);
+ } catch (RemoteException e) {
+ throw new RuntimeException("Failed when enrollResult(): " + e);
+ }
+ }
+
+ /**
+ * Send error code to HAL.
+ */
+ public void error(int error) {
+ try {
+ mFaceUnlockManager.error(error);
+ } catch (RemoteException e) {
+ throw new RuntimeException("Failed when error(): " + e);
+ }
+ }
+
+ /**
+ * Get face data dir storage path
+ */
+ public String getStorePath() {
+ try {
+ return mFaceUnlockManager.getStorePath();
+ } catch (RemoteException e) {
+ throw new RuntimeException("Failed when getStorePath(): " + e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/framework/client/com/libremobileos/faceunlock/client/IFaceHalService.aidl b/framework/client/com/libremobileos/faceunlock/client/IFaceHalService.aidl
new file mode 100644
index 0000000..33b396c
--- /dev/null
+++ b/framework/client/com/libremobileos/faceunlock/client/IFaceHalService.aidl
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.client;
+
+import com.libremobileos.faceunlock.client.IFaceHalServiceCallback;
+
+interface IFaceHalService {
+ long getDeviceId();
+
+ oneway void setCallback(in IFaceHalServiceCallback callback);
+
+ int setActiveUser(int userId, String storePath);
+
+ long generateChallenge(int timeout);
+
+ int enroll(in byte[] token, int timeout, in int[] disabledFeatures);
+
+ int revokeChallenge();
+
+ int setFeature(int feature, boolean enable, in byte[] token, int faceId);
+
+ boolean getFeature(int feature, int faceId);
+
+ long getAuthenticatorId();
+
+ int cancel();
+
+ int enumerate();
+
+ int remove(int faceId);
+
+ int authenticate(long operationId);
+
+ int userActivity();
+
+ int resetLockout(in byte[] token);
+}
\ No newline at end of file
diff --git a/framework/client/com/libremobileos/faceunlock/client/IFaceHalServiceCallback.aidl b/framework/client/com/libremobileos/faceunlock/client/IFaceHalServiceCallback.aidl
new file mode 100644
index 0000000..5713d98
--- /dev/null
+++ b/framework/client/com/libremobileos/faceunlock/client/IFaceHalServiceCallback.aidl
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.client;
+
+oneway interface IFaceHalServiceCallback {
+ void onEnrollResult(long deviceId, int faceId, int userId, int remaining);
+
+ void onAuthenticated(long deviceId, int faceId, int userId, in byte[] token);
+
+ void onAcquired(long deviceId, int userId, int acquiredInfo, int vendorCode);
+
+ void onError(long deviceId, int userId, int error, int vendorCode);
+
+ void onRemoved(long deviceId, in int[] faceIds, int userId);
+
+ void onEnumerate(long deviceId, in int[] faceIds, int userId);
+
+ void onLockoutChanged(long duration);
+}
\ No newline at end of file
diff --git a/framework/client/com/libremobileos/faceunlock/client/IFaceUnlockManager.aidl b/framework/client/com/libremobileos/faceunlock/client/IFaceUnlockManager.aidl
new file mode 100644
index 0000000..51d3a74
--- /dev/null
+++ b/framework/client/com/libremobileos/faceunlock/client/IFaceUnlockManager.aidl
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.client;
+
+interface IFaceUnlockManager {
+ void enrollResult(int remaining);
+ void error(int error);
+ String getStorePath();
+}
\ No newline at end of file
diff --git a/framework/server/com/libremobileos/faceunlock/server/CameraService.java b/framework/server/com/libremobileos/faceunlock/server/CameraService.java
new file mode 100644
index 0000000..9237544
--- /dev/null
+++ b/framework/server/com/libremobileos/faceunlock/server/CameraService.java
@@ -0,0 +1,385 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.server;
+
+import android.annotation.NonNull;
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.ImageFormat;
+import android.graphics.SurfaceTexture;
+import android.hardware.camera2.CameraAccessException;
+import android.hardware.camera2.CameraCaptureSession;
+import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraDevice;
+import android.hardware.camera2.CameraManager;
+import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.params.StreamConfigurationMap;
+import android.media.Image;
+import android.media.ImageReader;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.Trace;
+import android.text.TextUtils;
+import android.util.Log;
+import android.util.Size;
+import android.view.Display;
+import android.view.Surface;
+import android.view.WindowManager;
+
+import com.libremobileos.yifan.face.ImageUtils;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class CameraService implements ImageReader.OnImageAvailableListener {
+
+ private static final String TAG = "Camera2Service";
+
+ /**
+ * The camera preview size will be chosen to be the smallest frame by pixel size capable of
+ * containing a DESIRED_SIZE x DESIRED_SIZE square.
+ */
+ private static final int MINIMUM_PREVIEW_SIZE = 320;
+
+ private Handler mBackgroundHandler;
+ private HandlerThread mBackgroundThread;
+ private CameraDevice cameraDevice;
+ private CameraCaptureSession cameraCaptureSessions;
+ private CaptureRequest captureRequest;
+ private CaptureRequest.Builder captureRequestBuilder;
+ private ImageReader previewReader;
+ private final byte[][] yuvBytes = new byte[3][];
+ private int[] rgbBytes = null;
+ private boolean isProcessingFrame = false;
+ private int yRowStride;
+ private Runnable postInferenceCallback;
+ private Runnable imageConverter;
+ private Bitmap rgbFrameBitmap = null;
+ private Size previewSize;
+ private Size rotatedSize;
+ private final Context mContext;
+ private final CameraCallback mCallback;
+
+ protected final Size desiredInputSize = new Size(640, 480);
+ // The calculated actual processing width & height
+ protected int imageOrientation;
+
+ public interface CameraCallback {
+ void setupFaceRecognizer(Size bitmapSize, int rotation);
+
+ void processImage(Size previewSize, Size rotatedSize, Bitmap rgbBitmap, int rotation);
+ }
+
+ public CameraService(Context context, CameraCallback callback) {
+ mContext = context;
+ mCallback = callback;
+ }
+
+ private final CameraDevice.StateCallback stateCallback = new CameraDevice.StateCallback() {
+ @Override
+ public void onOpened(CameraDevice camera) {
+ //This is called when the camera is open
+ Log.e(TAG, "onOpened");
+ cameraDevice = camera;
+ createCameraPreview();
+ }
+
+ @Override
+ public void onDisconnected(CameraDevice camera) {
+ cameraDevice.close();
+ }
+
+ @Override
+ public void onError(CameraDevice camera, int error) {
+ cameraDevice.close();
+ cameraDevice = null;
+ }
+ };
+
+ public void startBackgroundThread() {
+ mBackgroundThread = new HandlerThread("Camera Background");
+ mBackgroundThread.start();
+ mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
+ }
+
+ public void stopBackgroundThread() {
+ closeCamera();
+ mBackgroundThread.quitSafely();
+ try {
+ mBackgroundThread.join();
+ mBackgroundThread = null;
+ mBackgroundHandler = null;
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void createCameraPreview() {
+ try {
+ previewReader =
+ ImageReader.newInstance(
+ previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2);
+
+ previewReader.setOnImageAvailableListener(this, mBackgroundHandler);
+ captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
+ captureRequestBuilder.addTarget(previewReader.getSurface());
+
+ cameraDevice.createCaptureSession(Collections.singletonList(previewReader.getSurface()),
+ new CameraCaptureSession.StateCallback() {
+ @Override
+ public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) {
+ //The camera is already closed
+ if (null == cameraDevice) {
+ return;
+ }
+ // When the session is ready, we start displaying the preview.
+ cameraCaptureSessions = cameraCaptureSession;
+ try {
+ // Auto focus should be continuous for camera preview.
+ captureRequestBuilder.set(
+ CaptureRequest.CONTROL_AF_MODE,
+ CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
+ // Flash is automatically enabled when necessary.
+ captureRequestBuilder.set(
+ CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH);
+
+ // Finally, we start displaying the camera preview.
+ captureRequest = captureRequestBuilder.build();
+ cameraCaptureSessions.setRepeatingRequest(
+ captureRequest, null, mBackgroundHandler);
+ } catch (final CameraAccessException e) {
+ Log.e(TAG, "Exception!", e);
+ }
+ }
+
+ @Override
+ public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) {
+ }
+ }, null);
+ } catch (CameraAccessException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /** Compares two {@code Size}s based on their areas. */
+ private static class CompareSizesByArea implements Comparator<Size> {
+ @Override
+ public int compare(final Size lhs, final Size rhs) {
+ // We cast here to ensure the multiplications won't overflow
+ return Long.signum(
+ (long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight());
+ }
+ }
+
+ /**
+ * Given {@code choices} of {@code Size}s supported by a camera, chooses the smallest one whose
+ * width and height are at least as large as the minimum of both, or an exact match if possible.
+ *
+ * @param choices The list of sizes that the camera supports for the intended output class
+ * @param width The minimum desired width
+ * @param height The minimum desired height
+ * @return The optimal {@code Size}, or an arbitrary one if none were big enough
+ */
+ private static Size chooseOptimalSize(final Size[] choices, final int width, final int height) {
+ final int minSize = Math.max(Math.min(width, height), MINIMUM_PREVIEW_SIZE);
+ final Size desiredSize = new Size(width, height);
+
+ // Collect the supported resolutions that are at least as big as the preview Surface
+ boolean exactSizeFound = false;
+ final List<Size> bigEnough = new ArrayList<>();
+ final List<Size> tooSmall = new ArrayList<>();
+ for (final Size option : choices) {
+ if (option.equals(desiredSize)) {
+ // Set the size but don't return yet so that remaining sizes will still be logged.
+ exactSizeFound = true;
+ }
+
+ if (option.getHeight() >= minSize && option.getWidth() >= minSize) {
+ bigEnough.add(option);
+ } else {
+ tooSmall.add(option);
+ }
+ }
+
+ Log.i(TAG, "Desired size: " + desiredSize + ", min size: " + minSize + "x" + minSize);
+ Log.i(TAG,"Valid preview sizes: [" + TextUtils.join(", ", bigEnough) + "]");
+ Log.i(TAG, "Rejected preview sizes: [" + TextUtils.join(", ", tooSmall) + "]");
+
+ if (exactSizeFound) {
+ Log.i(TAG, "Exact size match found.");
+ return desiredSize;
+ }
+
+ // Pick the smallest of those, assuming we found any
+ if (bigEnough.size() > 0) {
+ final Size chosenSize = Collections.min(bigEnough, new CompareSizesByArea());
+ Log.i(TAG, "Chosen size: " + chosenSize.getWidth() + "x" + chosenSize.getHeight());
+ return chosenSize;
+ } else {
+ Log.e(TAG, "Couldn't find any suitable preview size");
+ return choices[0];
+ }
+ }
+
+ public void openCamera() {
+ CameraManager manager = (CameraManager) mContext.getSystemService(Context.CAMERA_SERVICE);
+ Log.e(TAG, "is camera open");
+ try {
+ String cameraId = manager.getCameraIdList()[0];
+ for (String id : manager.getCameraIdList()) {
+ CameraCharacteristics characteristics = manager.getCameraCharacteristics(id);
+ if (characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT) {
+ cameraId = id;
+ break;
+ }
+ }
+ CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
+ StreamConfigurationMap map = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+ Integer sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
+
+ assert map != null;
+
+ // Danger, W.R.! Attempting to use too large a preview size could exceed the camera
+ // bus' bandwidth limitation, resulting in gorgeous previews but the storage of
+ // garbage capture data.
+ previewSize =
+ chooseOptimalSize(
+ map.getOutputSizes(SurfaceTexture.class),
+ desiredInputSize.getWidth(), desiredInputSize.getHeight());
+ rotatedSize = previewSize;
+
+ imageOrientation = sensorOrientation + getScreenOrientation();
+ rgbFrameBitmap = Bitmap.createBitmap(previewSize.getWidth(), previewSize.getHeight(), Bitmap.Config.ARGB_8888);
+
+ if (imageOrientation % 180 != 0) {
+ rotatedSize = new Size(previewSize.getHeight(), previewSize.getWidth());
+ }
+ mCallback.setupFaceRecognizer(rotatedSize, imageOrientation);
+
+ manager.openCamera(cameraId, stateCallback, null);
+ } catch (CameraAccessException | SecurityException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void closeCamera() {
+ if (null != cameraDevice) {
+ cameraDevice.close();
+ cameraDevice = null;
+ }
+ if (null != previewReader) {
+ previewReader.close();
+ previewReader = null;
+ }
+ }
+
+ private void fillBytes(final Image.Plane[] planes, final byte[][] yuvBytes) {
+ // Because of the variable row stride it's not possible to know in
+ // advance the actual necessary dimensions of the yuv planes.
+ for (int i = 0; i < planes.length; ++i) {
+ final ByteBuffer buffer = planes[i].getBuffer();
+ if (yuvBytes[i] == null) {
+ yuvBytes[i] = new byte[buffer.capacity()];
+ }
+ buffer.get(yuvBytes[i]);
+ }
+ }
+
+ private int[] getRgbBytes() {
+ imageConverter.run();
+ return rgbBytes;
+ }
+
+ private int getScreenOrientation() {
+ Display display = ((WindowManager) mContext.getSystemService(Context.WINDOW_SERVICE)).getDefaultDisplay();
+ switch (display.getRotation()) {
+ case Surface.ROTATION_270:
+ return 270;
+ case Surface.ROTATION_180:
+ return 180;
+ case Surface.ROTATION_90:
+ return 90;
+ default:
+ return 0;
+ }
+ }
+
+ public void readyForNextImage() {
+ if (postInferenceCallback != null) {
+ postInferenceCallback.run();
+ }
+ }
+
+ @Override
+ public void onImageAvailable(ImageReader reader) {
+ int previewWidth = previewSize.getWidth();
+ int previewHeight = previewSize.getHeight();
+
+ if (rgbBytes == null) {
+ rgbBytes = new int[previewWidth * previewHeight];
+ }
+ try {
+ final Image image = reader.acquireLatestImage();
+
+ if (image == null) {
+ return;
+ }
+
+ if (isProcessingFrame) {
+ image.close();
+ return;
+ }
+ isProcessingFrame = true;
+ Trace.beginSection("imageAvailable");
+ final Image.Plane[] planes = image.getPlanes();
+ fillBytes(planes, yuvBytes);
+ yRowStride = planes[0].getRowStride();
+ final int uvRowStride = planes[1].getRowStride();
+ final int uvPixelStride = planes[1].getPixelStride();
+
+ imageConverter =
+ () -> ImageUtils.convertYUV420ToARGB8888(
+ yuvBytes[0],
+ yuvBytes[1],
+ yuvBytes[2],
+ previewWidth,
+ previewHeight,
+ yRowStride,
+ uvRowStride,
+ uvPixelStride,
+ rgbBytes);
+
+ postInferenceCallback =
+ () -> {
+ image.close();
+ isProcessingFrame = false;
+ };
+
+ rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, previewHeight);
+
+ mCallback.processImage(previewSize, rotatedSize, rgbFrameBitmap, imageOrientation);
+ } catch (final Exception e) {
+ Log.e(TAG, "Exception!", e);
+ Trace.endSection();
+ return;
+ }
+ Trace.endSection();
+ }
+}
\ No newline at end of file
diff --git a/framework/server/com/libremobileos/faceunlock/server/FaceUnlockServer.java b/framework/server/com/libremobileos/faceunlock/server/FaceUnlockServer.java
new file mode 100644
index 0000000..1d114fb
--- /dev/null
+++ b/framework/server/com/libremobileos/faceunlock/server/FaceUnlockServer.java
@@ -0,0 +1,401 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.server;
+
+import static com.libremobileos.faceunlock.client.FaceUnlockManager.SERVICE_NAME;
+
+import android.content.om.IOverlayManager;
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.hardware.biometrics.face.V1_0.FaceAcquiredInfo;
+import android.hardware.biometrics.face.V1_0.Feature;
+import android.hardware.biometrics.face.V1_0.Status;
+import android.os.Handler;
+import android.os.IBinder;
+import android.os.Looper;
+import android.os.Message;
+import android.os.RemoteException;
+import android.os.ServiceManager;
+import android.util.Base64;
+import android.util.Log;
+import android.util.Size;
+
+import com.libremobileos.faceunlock.client.IFaceHalService;
+import com.libremobileos.faceunlock.client.IFaceHalServiceCallback;
+import com.libremobileos.faceunlock.client.IFaceUnlockManager;
+
+import com.libremobileos.yifan.face.DirectoryFaceStorageBackend;
+import com.libremobileos.yifan.face.FaceRecognizer;
+import com.libremobileos.yifan.face.FaceStorageBackend;
+import com.libremobileos.yifan.face.ImageUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Random;
+
+public class FaceUnlockServer {
+ private final String TAG = "FaceUnlockServer";
+ private final long kDeviceId = 123; // Arbitrary value.
+ private final int kFaceId = 100; // Arbitrary value.
+ private final boolean DEBUG = false;
+ private final String SETTINGS_OVERLAY_PACKAGE = "com.libremobileos.facedetect.settings.overlay";
+
+ private static final int MSG_CHALLENGE_TIMEOUT = 100;
+
+ private IFaceHalServiceCallback mCallback;
+ private FaceRecognizer faceRecognizer;
+ private FaceHandler mWorkHandler;
+ private Context mContext;
+ private long mChallenge = 0;
+ private int mChallengeCount = 0;
+ private boolean computingDetection = false;
+ private CameraService mCameraService;
+ private int mUserId = 0;
+ private String mStorePath = "/data/vendor_de/0/facedata";
+
+ private final IBinder mFaceUnlockHalBinder = new IFaceHalService.Stub() {
+ @Override
+ public long getDeviceId() {
+ return kDeviceId;
+ }
+
+ @Override
+ public void setCallback(IFaceHalServiceCallback clientCallback) {
+ if (DEBUG)
+ Log.d(TAG, "setCallback");
+
+ mCallback = clientCallback;
+
+ mWorkHandler.post(() -> {
+ IOverlayManager overlayManager = IOverlayManager.Stub.asInterface(
+ ServiceManager.getService("overlay" /* Context.OVERLAY_SERVICE */));
+ try {
+ overlayManager.setEnabledExclusiveInCategory(SETTINGS_OVERLAY_PACKAGE, -2 /* USER_CURRENT */);
+ } catch (Exception e) {
+ Log.e(TAG, "Failed to enable settings overlay", e);
+ }
+ });
+ }
+
+ @Override
+ public int setActiveUser(int userId, String storePath) {
+ if (DEBUG)
+ Log.d(TAG, "setActiveUser " + userId + " " + storePath);
+
+ mUserId = userId;
+ mStorePath = storePath;
+ File facesDir = new File(mStorePath + "/faces");
+ if (!facesDir.exists()) {
+ facesDir.mkdir();
+ }
+
+ return Status.OK;
+ }
+
+ @Override
+ public long generateChallenge(int challengeTimeoutSec) {
+ if (DEBUG)
+ Log.d(TAG, "generateChallenge + " + challengeTimeoutSec);
+
+ if (mChallengeCount <= 0 || mChallenge == 0) {
+ mChallenge = new Random().nextLong();
+ }
+ mChallengeCount += 1;
+ mWorkHandler.removeMessages(MSG_CHALLENGE_TIMEOUT);
+ mWorkHandler.sendEmptyMessageDelayed(MSG_CHALLENGE_TIMEOUT, challengeTimeoutSec * 1000L);
+
+ return mChallenge;
+ }
+
+ @Override
+ public int enroll(byte[] hat, int timeoutSec, int[] disabledFeatures) {
+ if (DEBUG)
+ Log.d(TAG, "enroll");
+
+ return Status.OK;
+ }
+
+ @Override
+ public int revokeChallenge() {
+ if (DEBUG)
+ Log.d(TAG, "revokeChallenge");
+
+ mChallengeCount -= 1;
+ if (mChallengeCount <= 0 && mChallenge != 0) {
+ mChallenge = 0;
+ mChallengeCount = 0;
+ mWorkHandler.removeMessages(MSG_CHALLENGE_TIMEOUT);
+ }
+ return Status.OK;
+ }
+
+ @Override
+ public int setFeature(int feature, boolean enabled, byte[] hat, int faceId) {
+ if (DEBUG)
+ Log.d(TAG, "setFeature " + feature + " " + enabled + " " + faceId);
+
+ // We don't do that here;
+
+ return Status.OK;
+ }
+
+ @Override
+ public boolean getFeature(int feature, int faceId) {
+ if (DEBUG)
+ Log.d(TAG, "getFeature " + feature + " " + faceId);
+
+ switch (feature) {
+ case Feature.REQUIRE_ATTENTION:
+ return false;
+ case Feature.REQUIRE_DIVERSITY:
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public long getAuthenticatorId() {
+ if (DEBUG)
+ Log.d(TAG, "getAuthenticatorId");
+
+ return 987; // Arbitrary value.
+ }
+
+ @Override
+ public int cancel() {
+ // Not sure what to do here.
+ mCameraService.closeCamera();
+ mCameraService.stopBackgroundThread();
+ return Status.OK;
+ }
+
+ @Override
+ public int enumerate() {
+ if (DEBUG)
+ Log.d(TAG, "enumerate");
+
+ mWorkHandler.post(() -> {
+ RemoteFaceServiceClient.connect(mStorePath, faced -> {
+ int[] faceIds = new int[1];
+ if (faced.isEnrolled()) {
+ faceIds[0] = kFaceId;
+ Log.d(TAG, "enumerate face added");
+ }
+ if (mCallback != null) {
+ try {
+ mCallback.onEnumerate(kDeviceId, faceIds, mUserId);
+ } catch (RemoteException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ });
+
+ return Status.OK;
+ }
+
+ @Override
+ public int remove(int faceId) {
+ if (DEBUG)
+ Log.d(TAG, "remove " + faceId);
+
+ mWorkHandler.post(() -> {
+ RemoteFaceServiceClient.connect(mStorePath, faced -> {
+ if ((faceId == kFaceId || faceId == 0) && faced.isEnrolled()) {
+ faced.unenroll();
+ int[] faceIds = new int[1];
+ faceIds[0] = faceId;
+ try {
+ if (mCallback != null)
+ mCallback.onRemoved(kDeviceId, faceIds, mUserId);
+ } catch (RemoteException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ });
+ return Status.OK;
+ }
+
+ @Override
+ public int authenticate(long operationId) {
+ if (DEBUG)
+ Log.d(TAG, "authenticate " + operationId);
+
+ mCameraService = new CameraService(mContext, faceCallback);
+ mWorkHandler.post(() -> {
+ mCameraService.startBackgroundThread();
+ mCameraService.openCamera();
+ });
+ return Status.OK;
+ }
+
+ @Override
+ public int userActivity() {
+ if (DEBUG)
+ Log.d(TAG, "userActivity");
+
+ return Status.OK;
+ }
+
+ @Override
+ public int resetLockout(byte[] hat) {
+ if (DEBUG)
+ Log.d(TAG, "resetLockout");
+
+ return Status.OK;
+ }
+ };
+
+ CameraService.CameraCallback faceCallback = new CameraService.CameraCallback() {
+ @Override
+ public void setupFaceRecognizer ( final Size bitmapSize, int rotation) {
+ // Store registered Faces
+ // example for in-memory: FaceStorageBackend faceStorage = new VolatileFaceStorageBackend();
+ // example for shared preferences: FaceStorageBackend faceStorage = new SharedPreferencesFaceStorageBackend(getSharedPreferences("faces", 0));
+ FaceStorageBackend faceStorage = new DirectoryFaceStorageBackend(new File(mStorePath + "/faces"));
+
+ // Create AI-based face detection
+ faceRecognizer = FaceRecognizer.create(mContext,
+ faceStorage, /* face data storage */
+ 0.6f, /* minimum confidence to consider object as face */
+ bitmapSize.getWidth(), /* bitmap width */
+ bitmapSize.getHeight(), /* bitmap height */
+ rotation,
+ 0.7f, /* maximum distance (to saved face model, not from camera) to track face */
+ 1, /* minimum model count to track face */
+ false, false, 4
+ );
+ }
+
+ @Override
+ public void processImage (Size previewSize, Size rotatedSize, Bitmap rgbBitmap,int rotation)
+ {
+ // No mutex needed as this method is not reentrant.
+ if (computingDetection) {
+ mCameraService.readyForNextImage();
+ return;
+ }
+ computingDetection = true;
+ List<FaceRecognizer.Face> data = faceRecognizer.recognize(rgbBitmap);
+ computingDetection = false;
+
+ // Camera is frontal so the image is flipped horizontally,
+ // so flip it again (and rotate Rect to match preview rotation)
+ Matrix flip = ImageUtils.getTransformationMatrix(previewSize.getWidth(), previewSize.getHeight(), rotatedSize.getWidth(), rotatedSize.getHeight(), rotation, false);
+ flip.preScale(1, -1, previewSize.getWidth() / 2f, previewSize.getHeight() / 2f);
+
+ for (FaceRecognizer.Face face : data) {
+ try {
+ if (mCallback != null) {
+ mCallback.onAcquired(kDeviceId, mUserId, FaceAcquiredInfo.GOOD, 0);
+ // Do we have any match?
+ if (face.isRecognized()) {
+ File f = new File(mStorePath, ".FACE_HAT");
+ try {
+ if (!f.exists()) {
+ throw new IOException("f.exists() == false");
+ }
+ if (!f.canRead()) {
+ throw new IOException("f.canRead() == false");
+ }
+ try (InputStream inputStream = new FileInputStream(f)) {
+ // https://stackoverflow.com/a/35446009
+ ByteArrayOutputStream result = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ for (int length; (length = inputStream.read(buffer)) != -1; ) {
+ result.write(buffer, 0, length);
+ }
+ // ignore the warning, api 33-only stuff right there :D
+ String base64hat = result.toString(StandardCharsets.UTF_8.name());
+ byte[] hat = Base64.decode(base64hat, Base64.URL_SAFE);
+ mCallback.onAuthenticated(kDeviceId, kFaceId, mUserId, hat);
+ }
+ } catch (IOException e) {
+ Log.e("Authentication", Log.getStackTraceString(e));
+ }
+ mCameraService.closeCamera();
+ mCameraService.stopBackgroundThread();
+ }
+ }
+ } catch (RemoteException e) {
+ e.printStackTrace();
+ }
+ }
+
+ mCameraService.readyForNextImage();
+ }
+ };
+
+ private class FaceHandler extends Handler {
+ public FaceHandler(Looper looper) {
+ super(looper);
+ }
+
+ @Override
+ public void handleMessage(Message message) {
+ if (message.what == MSG_CHALLENGE_TIMEOUT) {
+ mChallenge = 0;
+ mChallengeCount = 0;
+ }
+ }
+ }
+
+ public FaceUnlockServer(Context context, Looper serviceThreadLooper, BinderPublishCallback bpc) {
+ mContext = context;
+ mUserId = 0;
+ File facesDir = new File(mStorePath + "/faces");
+ if (!facesDir.exists()) {
+ facesDir.mkdir();
+ }
+ mWorkHandler = new FaceHandler(serviceThreadLooper);
+
+ bpc.publishBinderService(SERVICE_NAME, mFaceUnlockManagerBinder);
+ bpc.publishBinderService("faceunlockhal", mFaceUnlockHalBinder);
+ }
+
+ private final IBinder mFaceUnlockManagerBinder = new IFaceUnlockManager.Stub() {
+ @Override
+ public void enrollResult(int remaining) throws RemoteException {
+ if (mCallback != null) {
+ mCallback.onEnrollResult(kDeviceId, kFaceId, mUserId, remaining);
+ }
+ }
+
+ @Override
+ public void error(int error) throws RemoteException {
+ if (mCallback != null) {
+ mCallback.onError(kDeviceId, mUserId, error, 0);
+ }
+ }
+
+ @Override
+ public String getStorePath() {
+ return mStorePath;
+ }
+ };
+
+ public static interface BinderPublishCallback {
+ public void publishBinderService(String name, IBinder binder);
+ }
+}
\ No newline at end of file
diff --git a/framework/server/com/libremobileos/faceunlock/server/RemoteFaceServiceClient.java b/framework/server/com/libremobileos/faceunlock/server/RemoteFaceServiceClient.java
new file mode 100644
index 0000000..f01aeb1
--- /dev/null
+++ b/framework/server/com/libremobileos/faceunlock/server/RemoteFaceServiceClient.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2023 LibreMobileOS Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.libremobileos.faceunlock.server;
+
+import android.content.Context;
+import android.util.Base64;
+import android.util.Log;
+
+import com.libremobileos.yifan.face.DirectoryFaceStorageBackend;
+import com.libremobileos.yifan.face.FaceDataEncoder;
+import com.libremobileos.yifan.face.FaceStorageBackend;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.function.Consumer;
+
+public abstract class RemoteFaceServiceClient {
+ public static final String FACE = "Face";
+ public static final String SECURE = "secure";
+
+ public static void connect(String dir, Consumer<RemoteFaceServiceClient> callback) {
+ new Thread(() -> {
+ FaceStorageBackend s = new DirectoryFaceStorageBackend(new File(dir + "/faces"));
+ callback.accept(new RemoteFaceServiceClient() {
+
+ @Override
+ public boolean isEnrolled() {
+ return s.getNames().contains(FACE);
+ }
+
+ @Override
+ public boolean isSecure() {
+ return false;
+ }
+
+ @Override
+ public void setSecure(boolean secure) {
+ }
+
+ @Override
+ public boolean unenroll() {
+ boolean result = s.delete(FACE);
+ if (result) {
+ File f = new File(dir, ".FACE_HAT");
+ if (f.exists()) {
+ f.delete();
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public boolean enroll(String data, byte[] hat) {
+ boolean result = s.register(FACE, FaceDataEncoder.decode(data), true);
+ if (result) {
+ File f = new File(dir, ".FACE_HAT");
+ try {
+ if (f.exists()) {
+ f.delete();
+ } else {
+ if (!f.createNewFile())
+ throw new IOException("f.createNewFile() failed");
+ }
+ OutputStreamWriter hatOSW = new OutputStreamWriter(new FileOutputStream(f));
+ hatOSW.write(new String(Base64.encode(hat, Base64.URL_SAFE)));
+ hatOSW.close();
+ } catch (IOException e) {
+ Log.e("RemoteFaceServiceClient", "Failed to write HAT", e);
+ return false;
+ }
+ }
+ return result;
+ }
+ });
+ }).start();
+ }
+
+ public abstract boolean isEnrolled();
+ public abstract boolean isSecure();
+ public abstract void setSecure(boolean secure);
+ public abstract boolean unenroll();
+ public abstract boolean enroll(String data, byte[] hat);
+
+ public boolean enroll(float[][] data, byte[] hat) {
+ return enroll(FaceDataEncoder.encode(data), hat);
+ }
+}
\ No newline at end of file