Add Display Auto White Balance to ColorDisplayService

This change adds Display Auto White Balance to ColorDisplayService.
Given panel chromaticity data, a nominal white point, and a desired
correlated color temperature (CCT), this logic generates and applies a
matrix transform to shift the nominal white point to match the desired
CCT.

Bug 116516917

Change-Id: I07e341881a020a130944217fff372003edac0a8c

5 files changed, 265 insertions, 27 deletions

diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml
index 1feb59a52ad1..d40f01c7c960 100644
--- a/core/res/res/values/config.xml
+++ b/core/res/res/values/config.xml
@@ -929,9 +929,6 @@
          in hardware. -->
     <bool name="config_setColorTransformAccelerated">false</bool>
 
-    <!-- Boolean indicating whether display white balance is supported. -->
-    <bool name="config_displayWhiteBalanceAvailable">false</bool>
-
     <!-- Control whether Night display is available. This should only be enabled on devices
          that have a HWC implementation that can apply the matrix passed to setColorTransform
          without impacting power, performance, and app compatibility (e.g. protected content). -->
@@ -987,6 +984,44 @@
         <!-- B y-intercept --> <item>-0.198650895</item>
     </string-array>
 
+    <!-- Boolean indicating whether display white balance is supported. -->
+    <bool name="config_displayWhiteBalanceAvailable">false</bool>
+
+    <!-- Minimum color temperature, in Kelvin, supported by display white balance. -->
+    <integer name="config_displayWhiteBalanceColorTemperatureMin">4000</integer>
+
+    <!-- Maximum color temperature, in Kelvin, supported by display white balance. -->
+    <integer name="config_displayWhiteBalanceColorTemperatureMax">8000</integer>
+
+    <!-- Default color temperature, in Kelvin, used by display white balance. -->
+    <integer name="config_displayWhiteBalanceColorTemperatureDefault">6500</integer>
+
+    <!-- The display primaries, in CIE1931 XYZ color space, for display
+         white balance to use in its calculations. -->
+    <string-array name="config_displayWhiteBalanceDisplayPrimaries">
+        <!-- Red X -->   <item>0.412315</item>
+        <!-- Red Y -->   <item>0.212600</item>
+        <!-- Red Z -->   <item>0.019327</item>
+        <!-- Green X --> <item>0.357600</item>
+        <!-- Green Y --> <item>0.715200</item>
+        <!-- Green Z --> <item>0.119200</item>
+        <!-- Blue X -->  <item>0.180500</item>
+        <!-- Blue Y -->  <item>0.072200</item>
+        <!-- Blue Z -->  <item>0.950633</item>
+        <!-- White X --> <item>0.950456</item>
+        <!-- White Y --> <item>1.000000</item>
+        <!-- White Z --> <item>1.089058</item>
+    </string-array>
+
+    <!-- The nominal white coordinates, in CIE1931 XYZ color space, for Display White Balance to
+         use in its calculations. AWB will adapt this white point to the target ambient white
+         point. -->
+    <string-array name="config_displayWhiteBalanceDisplayNominalWhite">
+        <!-- Nominal White X --> <item>0.950456</item>
+        <!-- Nominal White Y --> <item>1.000000</item>
+        <!-- Nominal White Z --> <item>1.089058</item>
+    </string-array>
+
 
     <!-- Indicate available ColorDisplayController.COLOR_MODE_xxx. -->
     <integer-array name="config_availableColorModes">
diff --git a/core/res/res/values/symbols.xml b/core/res/res/values/symbols.xml
index 010accf19d18..a1ef0d5cc93b 100644
--- a/core/res/res/values/symbols.xml
+++ b/core/res/res/values/symbols.xml
@@ -3042,6 +3042,13 @@
   <java-symbol type="array" name="config_nightDisplayColorTemperatureCoefficientsNative" />
   <java-symbol type="array" name="config_availableColorModes" />
 
+  <java-symbol type="bool" name="config_displayWhiteBalanceAvailable" />
+  <java-symbol type="integer" name="config_displayWhiteBalanceColorTemperatureMin" />
+  <java-symbol type="integer" name="config_displayWhiteBalanceColorTemperatureMax" />
+  <java-symbol type="integer" name="config_displayWhiteBalanceColorTemperatureDefault" />
+  <java-symbol type="array" name="config_displayWhiteBalanceDisplayPrimaries" />
+  <java-symbol type="array" name="config_displayWhiteBalanceDisplayNominalWhite" />
+
   <!-- Default first user restrictions -->
   <java-symbol type="array" name="config_defaultFirstUserRestrictions" />
 
diff --git a/graphics/java/android/graphics/ColorSpace.java b/graphics/java/android/graphics/ColorSpace.java
index 95317a4d0bf2..9fa70a5ab19c 100644
--- a/graphics/java/android/graphics/ColorSpace.java
+++ b/graphics/java/android/graphics/ColorSpace.java
@@ -1661,10 +1661,12 @@ public abstract class ColorSpace {
      * @param rhs 3x3 matrix, as a non-null array of 9 floats
      * @return A new array of 9 floats containing the result of the multiplication
      *         of rhs by lhs
+     *
+     * @hide
      */
     @NonNull
     @Size(9)
-    private static float[] mul3x3(@NonNull @Size(9) float[] lhs, @NonNull @Size(9) float[] rhs) {
+    public static float[] mul3x3(@NonNull @Size(9) float[] lhs, @NonNull @Size(9) float[] rhs) {
         float[] r = new float[9];
         r[0] = lhs[0] * rhs[0] + lhs[3] * rhs[1] + lhs[6] * rhs[2];
         r[1] = lhs[1] * rhs[0] + lhs[4] * rhs[1] + lhs[7] * rhs[2];
diff --git a/services/core/java/com/android/server/display/ColorDisplayService.java b/services/core/java/com/android/server/display/ColorDisplayService.java
index b332c47ce281..2fe17d8f3c98 100644
--- a/services/core/java/com/android/server/display/ColorDisplayService.java
+++ b/services/core/java/com/android/server/display/ColorDisplayService.java
@@ -34,8 +34,10 @@ import android.content.Context;
 import android.content.Intent;
 import android.content.IntentFilter;
 import android.content.pm.PackageManager;
+import android.content.res.Resources;
 import android.database.ContentObserver;
 import android.hardware.display.ColorDisplayManager;
+import android.graphics.ColorSpace;
 import android.hardware.display.IColorDisplayManager;
 import android.net.Uri;
 import android.opengl.Matrix;
@@ -59,6 +61,7 @@ import com.android.server.twilight.TwilightListener;
 import com.android.server.twilight.TwilightManager;
 import com.android.server.twilight.TwilightState;
 
+import java.io.PrintWriter;
 import java.time.DateTimeException;
 import java.time.Instant;
 import java.time.LocalDateTime;
@@ -148,26 +151,204 @@ public final class ColorDisplayService extends SystemService {
     };
 
     private final TintController mDisplayWhiteBalanceTintController = new TintController() {
-
+        // Three chromaticity coordinates per color: X, Y, and Z
+        private final int NUM_VALUES_PER_PRIMARY = 3;
+        // Four colors: red, green, blue, and white
+        private final int NUM_DISPLAY_PRIMARIES_VALS = 4 * NUM_VALUES_PER_PRIMARY;
+
+        private final Object mLock = new Object();
+        private int mTemperatureMin;
+        private int mTemperatureMax;
+        private int mTemperatureDefault;
+        private float[] mDisplayNominalWhiteXYZ = new float[NUM_VALUES_PER_PRIMARY];
+        private ColorSpace.Rgb mDisplayColorSpaceRGB;
+        private float[] mChromaticAdaptationMatrix;
+        private int mCurrentColorTemperature;
+        private float[] mCurrentColorTemperatureXYZ;
+        private boolean mSetUp = false;
         private float[] mMatrixDisplayWhiteBalance = new float[16];
 
         @Override
         public void setUp(Context context, boolean needsLinear) {
+            mSetUp = false;
+
+            final Resources res = getContext().getResources();
+            final String[] displayPrimariesValues = res.getStringArray(
+                    R.array.config_displayWhiteBalanceDisplayPrimaries);
+            final String[] nominalWhiteValues = res.getStringArray(
+                    R.array.config_displayWhiteBalanceDisplayNominalWhite);
+
+            if (displayPrimariesValues.length != NUM_DISPLAY_PRIMARIES_VALS) {
+                Slog.e(TAG, "Unexpected display white balance primaries resource length " +
+                        displayPrimariesValues.length);
+                return;
+            }
+
+            if (nominalWhiteValues.length != NUM_VALUES_PER_PRIMARY) {
+                Slog.e(TAG, "Unexpected display white balance nominal white resource length " +
+                        nominalWhiteValues.length);
+                return;
+            }
+
+            float[] displayRedGreenBlueXYZ =
+                new float[NUM_DISPLAY_PRIMARIES_VALS - NUM_VALUES_PER_PRIMARY];
+            float[] displayWhiteXYZ = new float[NUM_VALUES_PER_PRIMARY];
+            for (int i = 0; i < displayRedGreenBlueXYZ.length; i++) {
+                displayRedGreenBlueXYZ[i] = Float.parseFloat(displayPrimariesValues[i]);
+            }
+            for (int i = 0; i < displayWhiteXYZ.length; i++) {
+                displayWhiteXYZ[i] = Float.parseFloat(
+                        displayPrimariesValues[displayRedGreenBlueXYZ.length + i]);
+            }
+
+            final ColorSpace.Rgb displayColorSpaceRGB = new ColorSpace.Rgb(
+                "Display Color Space",
+                displayRedGreenBlueXYZ,
+                displayWhiteXYZ,
+                2.2f // gamma, unused for display white balance
+            );
+
+            float[] displayNominalWhiteXYZ = new float[NUM_VALUES_PER_PRIMARY];
+            for (int i = 0; i < nominalWhiteValues.length; i++) {
+                displayNominalWhiteXYZ[i] = Float.parseFloat(nominalWhiteValues[i]);
+            }
+
+            final int colorTemperatureMin = res.getInteger(
+                    R.integer.config_displayWhiteBalanceColorTemperatureMin);
+            if (colorTemperatureMin <= 0) {
+                Slog.e(TAG, "display white balance minimum temperature must be greater than 0");
+                return;
+            }
+
+            final int colorTemperatureMax = res.getInteger(
+                    R.integer.config_displayWhiteBalanceColorTemperatureMax);
+            if (colorTemperatureMax < colorTemperatureMin) {
+                Slog.e(TAG, "display white balance max temp must be greater or equal to min");
+                return;
+            }
+
+            final int colorTemperature = res.getInteger(
+                    R.integer.config_displayWhiteBalanceColorTemperatureDefault);
+
+            synchronized (mLock) {
+                mDisplayColorSpaceRGB = displayColorSpaceRGB;
+                mDisplayNominalWhiteXYZ = displayNominalWhiteXYZ;
+                mTemperatureMin = colorTemperatureMin;
+                mTemperatureMax = colorTemperatureMax;
+                mTemperatureDefault = colorTemperature;
+                mSetUp = true;
+            }
+
+            setMatrix(mTemperatureDefault);
         }
 
         @Override
         public float[] getMatrix() {
-            return isActivated() ? mMatrixDisplayWhiteBalance : MATRIX_IDENTITY;
+            return mSetUp && isActivated() ? mMatrixDisplayWhiteBalance : MATRIX_IDENTITY;
         }
 
         @Override
         public void setMatrix(int cct) {
+            if (!mSetUp) {
+                Slog.w(TAG, "Can't set display white balance temperature: uninitialized");
+                return;
+            }
+
+            if (cct < mTemperatureMin) {
+                Slog.w(TAG, "Requested display color temperature is below allowed minimum");
+                cct = mTemperatureMin;
+            } else if (cct > mTemperatureMax) {
+                Slog.w(TAG, "Requested display color temperature is above allowed maximum");
+                cct = mTemperatureMax;
+            }
+
+            Slog.d(TAG, "setDisplayWhiteBalanceTemperatureMatrix: cct = " + cct);
+
+            synchronized (mLock) {
+                mCurrentColorTemperature = cct;
+
+                // Adapt the display's nominal white point to match the requested CCT value
+                mCurrentColorTemperatureXYZ = ColorSpace.cctToIlluminantdXyz(cct);
+
+                mChromaticAdaptationMatrix =
+                    ColorSpace.chromaticAdaptation(ColorSpace.Adaptation.BRADFORD,
+                            mDisplayNominalWhiteXYZ, mCurrentColorTemperatureXYZ);
+
+                // Convert the adaptation matrix to RGB space
+                float[] result = ColorSpace.mul3x3(mChromaticAdaptationMatrix,
+                        mDisplayColorSpaceRGB.getTransform());
+                result = ColorSpace.mul3x3(mDisplayColorSpaceRGB.getInverseTransform(), result);
+
+                // Normalize the transform matrix to peak white value in RGB space
+                final float adaptedMaxR = result[0] + result[3] + result[6];
+                final float adaptedMaxG = result[1] + result[4] + result[7];
+                final float adaptedMaxB = result[2] + result[5] + result[8];
+                final float denum = Math.max(Math.max(adaptedMaxR, adaptedMaxG), adaptedMaxB);
+                for (int i = 0; i < result.length; i++) {
+                    result[i] /= denum;
+                }
+
+                Matrix.setIdentityM(mMatrixDisplayWhiteBalance, 0);
+                java.lang.System.arraycopy(result, 0, mMatrixDisplayWhiteBalance, 0, 3);
+                java.lang.System.arraycopy(result, 3, mMatrixDisplayWhiteBalance, 4, 3);
+                java.lang.System.arraycopy(result, 6, mMatrixDisplayWhiteBalance, 8, 3);
+            }
         }
 
         @Override
         public int getLevel() {
             return LEVEL_COLOR_MATRIX_DISPLAY_WHITE_BALANCE;
         }
+
+        /**
+         * Format a given matrix into a string.
+         *
+         * @param matrix the matrix to format
+         * @param cols number of columns in the matrix
+         */
+        private String matrixToString(float[] matrix, int cols) {
+            if (matrix == null || cols <= 0) {
+                Slog.e(TAG, "Invalid arguments when formatting matrix to string");
+                return "";
+            }
+
+            StringBuilder sb = new StringBuilder("");
+            for (int i = 0; i < matrix.length; i++) {
+                if (i % cols == 0) {
+                    sb.append("\n    ");
+                }
+                sb.append(String.format("%9.6f ", matrix[i]));
+            }
+            return sb.toString();
+        }
+
+        @Override
+        public void dump(PrintWriter pw) {
+            synchronized (mLock) {
+                pw.println("ColorDisplayService");
+                pw.println("  mSetUp = " + mSetUp);
+
+                if (!mSetUp) {
+                    return;
+                }
+
+                pw.println("  isActivated = " + isActivated());
+                pw.println("  mTemperatureMin = " + mTemperatureMin);
+                pw.println("  mTemperatureMax = " + mTemperatureMax);
+                pw.println("  mTemperatureDefault = " + mTemperatureDefault);
+                pw.println("  mCurrentColorTemperature = " + mCurrentColorTemperature);
+                pw.println("  mCurrentColorTemperatureXYZ = " +
+                        matrixToString(mCurrentColorTemperatureXYZ, 3));
+                pw.println("  mDisplayColorSpaceRGB RGB-to-XYZ = " +
+                        matrixToString(mDisplayColorSpaceRGB.getTransform(), 3));
+                pw.println("  mChromaticAdaptationMatrix = " +
+                        matrixToString(mChromaticAdaptationMatrix, 3));
+                pw.println("  mDisplayColorSpaceRGB XYZ-to-RGB = " +
+                        matrixToString(mDisplayColorSpaceRGB.getInverseTransform(), 3));
+                pw.println("  mMatrixDisplayWhiteBalance = " +
+                        matrixToString(mMatrixDisplayWhiteBalance, 4));
+            }
+        }
     };
 
     private final TintController mGlobalSaturationTintController = new TintController() {
@@ -230,8 +411,6 @@ public final class ColorDisplayService extends SystemService {
 
     private NightDisplayAutoMode mNightDisplayAutoMode;
 
-    private Integer mDisplayWhiteBalanceColorTemperature;
-
     public ColorDisplayService(Context context) {
         super(context);
         mHandler = new TintHandler(Looper.getMainLooper());
@@ -363,7 +542,7 @@ public final class ColorDisplayService extends SystemService {
                                 onAccessibilityTransformChanged();
                                 break;
                             case Secure.DISPLAY_WHITE_BALANCE_ENABLED:
-                                onDisplayWhiteBalanceEnabled(isDisplayWhiteBalanceSettingEnabled());
+                                updateDisplayWhiteBalanceStatus();
                                 break;
                         }
                     }
@@ -416,14 +595,9 @@ public final class ColorDisplayService extends SystemService {
 
         if (ColorDisplayManager.isDisplayWhiteBalanceAvailable(getContext())) {
             // Prepare the display white balance transform matrix.
-            mDisplayWhiteBalanceTintController
-                    .setUp(getContext(), DisplayTransformManager.needsLinearColorMatrix());
-            if (mDisplayWhiteBalanceColorTemperature != null) {
-                mDisplayWhiteBalanceTintController
-                        .setMatrix(mDisplayWhiteBalanceColorTemperature);
-            }
+            mDisplayWhiteBalanceTintController.setUp(getContext(), true /* needsLinear */);
 
-            onDisplayWhiteBalanceEnabled(isDisplayWhiteBalanceSettingEnabled());
+            updateDisplayWhiteBalanceStatus();
         }
     }
 
@@ -460,6 +634,8 @@ public final class ColorDisplayService extends SystemService {
                 mNightDisplayAutoMode.onActivated(activated);
             }
 
+            updateDisplayWhiteBalanceStatus();
+
             applyTint(mNightDisplayTintController, false);
         }
     }
@@ -516,11 +692,7 @@ public final class ColorDisplayService extends SystemService {
                 .setUp(getContext(), DisplayTransformManager.needsLinearColorMatrix(mode));
         mNightDisplayTintController.setMatrix(mNightDisplayController.getColorTemperature());
 
-        mDisplayWhiteBalanceTintController
-                .setUp(getContext(), DisplayTransformManager.needsLinearColorMatrix(mode));
-        if (mDisplayWhiteBalanceColorTemperature != null) {
-            mDisplayWhiteBalanceTintController.setMatrix(mDisplayWhiteBalanceColorTemperature);
-        }
+        updateDisplayWhiteBalanceStatus();
 
         final DisplayTransformManager dtm = getLocalService(DisplayTransformManager.class);
         dtm.setColorMode(mode, mNightDisplayTintController.getMatrix());
@@ -611,10 +783,15 @@ public final class ColorDisplayService extends SystemService {
         return ldt.isBefore(compareTime) ? ldt.plusDays(1) : ldt;
     }
 
-    private void onDisplayWhiteBalanceEnabled(boolean enabled) {
-        mDisplayWhiteBalanceTintController.setActivated(enabled);
-        if (mDisplayWhiteBalanceListener != null) {
-            mDisplayWhiteBalanceListener.onDisplayWhiteBalanceStatusChanged(enabled);
+    private void updateDisplayWhiteBalanceStatus() {
+        boolean oldActivated = mDisplayWhiteBalanceTintController.isActivated();
+        mDisplayWhiteBalanceTintController.setActivated(isDisplayWhiteBalanceSettingEnabled() &&
+                !mNightDisplayTintController.isActivated() &&
+                DisplayTransformManager.needsLinearColorMatrix());
+        boolean activated = mDisplayWhiteBalanceTintController.isActivated();
+
+        if (mDisplayWhiteBalanceListener != null && oldActivated != activated) {
+            mDisplayWhiteBalanceListener.onDisplayWhiteBalanceStatusChanged(activated);
         }
     }
 
@@ -896,6 +1073,12 @@ public final class ColorDisplayService extends SystemService {
         }
 
         /**
+         * Dump debug information.
+         */
+        public void dump(PrintWriter pw) {
+        }
+
+        /**
          * Set up any constants needed for computing the matrix.
          */
         public abstract void setUp(Context context, boolean needsLinear);
@@ -929,11 +1112,10 @@ public final class ColorDisplayService extends SystemService {
          */
         public boolean setDisplayWhiteBalanceColorTemperature(int cct) {
             // Update the transform matrix even if it can't be applied.
-            mDisplayWhiteBalanceColorTemperature = cct;
             mDisplayWhiteBalanceTintController.setMatrix(cct);
 
             if (mDisplayWhiteBalanceTintController.isActivated()) {
-                applyTint(mDisplayWhiteBalanceTintController, true);
+                applyTint(mDisplayWhiteBalanceTintController, false);
                 return true;
             }
             return false;
@@ -946,6 +1128,10 @@ public final class ColorDisplayService extends SystemService {
             mDisplayWhiteBalanceListener = listener;
             return mDisplayWhiteBalanceTintController.isActivated();
         }
+
+        public void dump(PrintWriter pw) {
+            mDisplayWhiteBalanceTintController.dump(pw);
+        }
     }
 
     /**
diff --git a/services/core/java/com/android/server/display/DisplayManagerService.java b/services/core/java/com/android/server/display/DisplayManagerService.java
index 4a17c6591449..2340b77c4a3c 100644
--- a/services/core/java/com/android/server/display/DisplayManagerService.java
+++ b/services/core/java/com/android/server/display/DisplayManagerService.java
@@ -95,6 +95,7 @@ import com.android.server.SystemService;
 import com.android.server.UiThread;
 import com.android.server.wm.SurfaceAnimationThread;
 import com.android.server.wm.WindowManagerInternal;
+import com.android.server.display.ColorDisplayService.ColorDisplayServiceInternal;
 
 import java.io.FileDescriptor;
 import java.io.PrintWriter;
@@ -1459,6 +1460,13 @@ public final class DisplayManagerService extends SystemService {
 
             pw.println();
             mPersistentDataStore.dump(pw);
+
+            final ColorDisplayServiceInternal cds = LocalServices.getService(
+                    ColorDisplayServiceInternal.class);
+            if (cds != null) {
+                pw.println();
+                cds.dump(pw);
+            }
         }
     }