From 78fc4a3bf8bfa2c691e79d56cd7e8e496d0b28cd Mon Sep 17 00:00:00 2001 From: Ibrahim Yilmaz Date: Wed, 22 May 2024 15:56:59 +0000 Subject: [API Abuse - Text Consistency] Eliminate zero width, invisible formatting chars and consecutive spaces Some apps use different invisible characters like in their contents to prevent the content getting truncated or trimmed. Such character combinations can also look like multiple new lines when there are many and they can also prevent us removing consecutive new lines. Bug: 313439845 Test: NotificationBigTextNormalizer Flag: android.app.Flags.clean_up_spans_and_new_lines Change-Id: I36474427f3934450a7779f8e0afa619c6ad9a2b2 --- core/java/android/app/Notification.java | 8 +- .../android/internal/util/NewlineNormalizer.java | 39 ------ .../util/NotificationBigTextNormalizer.java | 123 +++++++++++++++++ .../internal/util/NewlineNormalizerTest.java | 71 ---------- .../util/NotificationBigTextNormalizerTest.java | 148 +++++++++++++++++++++ 5 files changed, 275 insertions(+), 114 deletions(-) delete mode 100644 core/java/com/android/internal/util/NewlineNormalizer.java create mode 100644 core/java/com/android/internal/util/NotificationBigTextNormalizer.java delete mode 100644 core/tests/utiltests/src/com/android/internal/util/NewlineNormalizerTest.java create mode 100644 core/tests/utiltests/src/com/android/internal/util/NotificationBigTextNormalizerTest.java diff --git a/core/java/android/app/Notification.java b/core/java/android/app/Notification.java index f8215203ec67..a732cc2f5b98 100644 --- a/core/java/android/app/Notification.java +++ b/core/java/android/app/Notification.java @@ -114,7 +114,7 @@ import com.android.internal.annotations.VisibleForTesting; import com.android.internal.graphics.ColorUtils; import com.android.internal.util.ArrayUtils; import com.android.internal.util.ContrastColorUtil; -import com.android.internal.util.NewlineNormalizer; +import com.android.internal.util.NotificationBigTextNormalizer; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -3186,12 +3186,12 @@ public class Notification implements Parcelable return cs.toString(); } - private static CharSequence cleanUpNewLines(@Nullable CharSequence charSequence) { + private static CharSequence normalizeBigText(@Nullable CharSequence charSequence) { if (charSequence == null) { return charSequence; } - return NewlineNormalizer.normalizeNewlines(charSequence.toString()); + return NotificationBigTextNormalizer.normalizeBigText(charSequence.toString()); } private static CharSequence removeTextSizeSpans(CharSequence charSequence) { @@ -8423,7 +8423,7 @@ public class Notification implements Parcelable // Replace the text with the big text, but only if the big text is not empty. CharSequence bigTextText = mBuilder.processLegacyText(mBigText); if (Flags.cleanUpSpansAndNewLines()) { - bigTextText = cleanUpNewLines(stripStyling(bigTextText)); + bigTextText = normalizeBigText(stripStyling(bigTextText)); } if (!TextUtils.isEmpty(bigTextText)) { p.text(bigTextText); diff --git a/core/java/com/android/internal/util/NewlineNormalizer.java b/core/java/com/android/internal/util/NewlineNormalizer.java deleted file mode 100644 index 0104d1f56f83..000000000000 --- a/core/java/com/android/internal/util/NewlineNormalizer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2024 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.internal.util; - - -import java.util.regex.Pattern; - -/** - * Utility class that replaces consecutive empty lines with single new line. - * @hide - */ -public class NewlineNormalizer { - - private static final Pattern MULTIPLE_NEWLINES = Pattern.compile("\\v(\\s*\\v)?"); - - // Private constructor to prevent instantiation - private NewlineNormalizer() {} - - /** - * Replaces consecutive newlines with a single newline in the input text. - */ - public static String normalizeNewlines(String text) { - return MULTIPLE_NEWLINES.matcher(text).replaceAll("\n"); - } -} diff --git a/core/java/com/android/internal/util/NotificationBigTextNormalizer.java b/core/java/com/android/internal/util/NotificationBigTextNormalizer.java new file mode 100644 index 000000000000..80d409500ef0 --- /dev/null +++ b/core/java/com/android/internal/util/NotificationBigTextNormalizer.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + + +import android.annotation.NonNull; +import android.os.Trace; + +import java.util.regex.Pattern; + +/** + * Utility class that normalizes BigText style Notification content. + * @hide + */ +public class NotificationBigTextNormalizer { + + private static final Pattern MULTIPLE_NEWLINES = Pattern.compile("\\v(\\s*\\v)?"); + private static final Pattern HORIZONTAL_WHITESPACES = Pattern.compile("\\h+"); + + // Private constructor to prevent instantiation + private NotificationBigTextNormalizer() {} + + /** + * Normalizes the given text by collapsing consecutive new lines into single one and cleaning + * up each line by removing zero-width characters, invisible formatting characters, and + * collapsing consecutive whitespace into single space. + */ + @NonNull + public static String normalizeBigText(@NonNull String text) { + try { + Trace.beginSection("NotifBigTextNormalizer#normalizeBigText"); + text = MULTIPLE_NEWLINES.matcher(text).replaceAll("\n"); + text = HORIZONTAL_WHITESPACES.matcher(text).replaceAll(" "); + text = normalizeLines(text); + return text; + } finally { + Trace.endSection(); + } + } + + /** + * Normalizes lines in a text by removing zero-width characters, invisible formatting + * characters, and collapsing consecutive whitespace into single space. + * + *

+ * This method processes the input text line by line. It eliminates zero-width + * characters (U+200B to U+200D, U+FEFF, U+034F), invisible formatting + * characters (U+2060 to U+2065, U+206A to U+206F, U+FFF9 to U+FFFB), + * and replaces any sequence of consecutive whitespace characters with a single space. + *

+ * + *

+ * Additionally, the method trims trailing whitespace from each line and removes any + * resulting empty lines. + *

+ */ + @NonNull + private static String normalizeLines(@NonNull String text) { + String[] lines = text.split("\n"); + final StringBuilder textSB = new StringBuilder(text.length()); + for (int i = 0; i < lines.length; i++) { + final String line = lines[i]; + final StringBuilder lineSB = new StringBuilder(line.length()); + boolean spaceSeen = false; + for (int j = 0; j < line.length(); j++) { + final char character = line.charAt(j); + + // Skip ZERO WIDTH characters + if ((character >= '\u200B' && character <= '\u200D') + || character == '\uFEFF' || character == '\u034F') { + continue; + } + // Skip INVISIBLE_FORMATTING_CHARACTERS + if ((character >= '\u2060' && character <= '\u2065') + || (character >= '\u206A' && character <= '\u206F') + || (character >= '\uFFF9' && character <= '\uFFFB')) { + continue; + } + + if (isSpace(character)) { + // eliminate consecutive spaces.... + if (!spaceSeen) { + lineSB.append(" "); + } + spaceSeen = true; + } else { + spaceSeen = false; + lineSB.append(character); + } + } + // trim line. + final String currentLine = lineSB.toString().trim(); + + // don't add empty lines after trim. + if (currentLine.length() > 0) { + if (textSB.length() > 0) { + textSB.append("\n"); + } + textSB.append(currentLine); + } + } + + return textSB.toString(); + } + + private static boolean isSpace(char ch) { + return ch != '\n' && Character.isSpaceChar(ch); + } +} diff --git a/core/tests/utiltests/src/com/android/internal/util/NewlineNormalizerTest.java b/core/tests/utiltests/src/com/android/internal/util/NewlineNormalizerTest.java deleted file mode 100644 index bcdac610a49d..000000000000 --- a/core/tests/utiltests/src/com/android/internal/util/NewlineNormalizerTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2024 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.internal.util; - -import static junit.framework.Assert.assertEquals; - - -import android.platform.test.annotations.DisabledOnRavenwood; -import android.platform.test.ravenwood.RavenwoodRule; - -import androidx.test.runner.AndroidJUnit4; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Test for {@link NewlineNormalizer} - * @hide - */ -@DisabledOnRavenwood(blockedBy = NewlineNormalizer.class) -@RunWith(AndroidJUnit4.class) -public class NewlineNormalizerTest { - - @Rule - public final RavenwoodRule mRavenwood = new RavenwoodRule(); - - @Test - public void testEmptyInput() { - assertEquals("", NewlineNormalizer.normalizeNewlines("")); - } - - @Test - public void testSingleNewline() { - assertEquals("\n", NewlineNormalizer.normalizeNewlines("\n")); - } - - @Test - public void testMultipleConsecutiveNewlines() { - assertEquals("\n", NewlineNormalizer.normalizeNewlines("\n\n\n\n\n")); - } - - @Test - public void testNewlinesWithSpacesAndTabs() { - String input = "Line 1\n \n \t \n\tLine 2"; - // Adjusted expected output to include the tab character - String expected = "Line 1\n\tLine 2"; - assertEquals(expected, NewlineNormalizer.normalizeNewlines(input)); - } - - @Test - public void testMixedNewlineCharacters() { - String input = "Line 1\r\nLine 2\u000BLine 3\fLine 4\u2028Line 5\u2029Line 6"; - String expected = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5\nLine 6"; - assertEquals(expected, NewlineNormalizer.normalizeNewlines(input)); - } -} diff --git a/core/tests/utiltests/src/com/android/internal/util/NotificationBigTextNormalizerTest.java b/core/tests/utiltests/src/com/android/internal/util/NotificationBigTextNormalizerTest.java new file mode 100644 index 000000000000..1f2e24aa8c68 --- /dev/null +++ b/core/tests/utiltests/src/com/android/internal/util/NotificationBigTextNormalizerTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import static junit.framework.Assert.assertEquals; + + +import android.platform.test.annotations.DisabledOnRavenwood; +import android.platform.test.ravenwood.RavenwoodRule; + +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Test for {@link NotificationBigTextNormalizer} + * @hide + */ +@DisabledOnRavenwood(blockedBy = NotificationBigTextNormalizer.class) +@RunWith(AndroidJUnit4.class) +public class NotificationBigTextNormalizerTest { + + @Rule + public final RavenwoodRule mRavenwood = new RavenwoodRule(); + + + @Test + public void testEmptyInput() { + assertEquals("", NotificationBigTextNormalizer.normalizeBigText("")); + } + + @Test + public void testSingleNewline() { + assertEquals("", NotificationBigTextNormalizer.normalizeBigText("\n")); + } + + @Test + public void testMultipleConsecutiveNewlines() { + assertEquals("", NotificationBigTextNormalizer.normalizeBigText("\n\n\n\n\n")); + } + + @Test + public void testNewlinesWithSpacesAndTabs() { + String input = "Line 1\n \n \t \n\tLine 2"; + // Adjusted expected output to include the tab character + String expected = "Line 1\nLine 2"; + assertEquals(expected, NotificationBigTextNormalizer.normalizeBigText(input)); + } + + @Test + public void testMixedNewlineCharacters() { + String input = "Line 1\r\nLine 2\u000BLine 3\fLine 4\u2028Line 5\u2029Line 6"; + String expected = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5\nLine 6"; + assertEquals(expected, NotificationBigTextNormalizer.normalizeBigText(input)); + } + + @Test + public void testConsecutiveSpaces() { + // Only spaces + assertEquals("This is a test.", NotificationBigTextNormalizer.normalizeBigText("This" + + " is a test.")); + // Zero width characters bw spaces. + assertEquals("This is a test.", NotificationBigTextNormalizer.normalizeBigText("This" + + "\u200B \u200B \u200B \u200B \u200B \u200B \u200B \u200Bis\uFEFF \uFEFF \uFEFF" + + " \uFEFFa \u034F \u034F \u034F \u034F \u034F \u034Ftest.")); + + // Invisible formatting characters bw spaces. + assertEquals("This is a test.", NotificationBigTextNormalizer.normalizeBigText("This" + + "\u2061 \u2061 \u2061 \u2061 \u2061 \u2061 \u2061 \u2061is\u206E \u206E \u206E" + + " \u206Ea \uFFFB \uFFFB \uFFFB \uFFFB \uFFFB \uFFFBtest.")); + // Non breakable spaces + assertEquals("This is a test.", NotificationBigTextNormalizer.normalizeBigText("This" + + "\u00A0\u00A0\u00A0\u00A0\u00A0\u00A0is\u2005 \u2005 \u2005" + + " \u2005a\u2005\u2005\u2005 \u2005\u2005\u2005test.")); + } + + @Test + public void testZeroWidthCharRemoval() { + // Test each character individually + char[] zeroWidthChars = { '\u200B', '\u200C', '\u200D', '\uFEFF', '\u034F' }; + + for (char c : zeroWidthChars) { + String input = "Test" + c + "string"; + String expected = "Teststring"; + assertEquals(expected, NotificationBigTextNormalizer.normalizeBigText(input)); + } + } + + @Test + public void testWhitespaceReplacement() { + assertEquals("This text has horizontal whitespace.", + NotificationBigTextNormalizer.normalizeBigText( + "This\ttext\thas\thorizontal\twhitespace.")); + assertEquals("This text has mixed whitespace.", + NotificationBigTextNormalizer.normalizeBigText( + "This text has \u00A0 mixed\u2009whitespace.")); + assertEquals("This text has leading and trailing whitespace.", + NotificationBigTextNormalizer.normalizeBigText( + "\t This text has leading and trailing whitespace. \n")); + } + + @Test + public void testInvisibleFormattingCharacterRemoval() { + // Test each character individually + char[] invisibleFormattingChars = { + '\u2060', '\u2061', '\u2062', '\u2063', '\u2064', '\u2065', + '\u206A', '\u206B', '\u206C', '\u206D', '\u206E', '\u206F', + '\uFFF9', '\uFFFA', '\uFFFB' + }; + + for (char c : invisibleFormattingChars) { + String input = "Test " + c + "string"; + String expected = "Test string"; + assertEquals(expected, NotificationBigTextNormalizer.normalizeBigText(input)); + } + } + @Test + public void testNonBreakSpaceReplacement() { + // Test each character individually + char[] nonBreakSpaces = { + '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', + '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', + '\u2008', '\u2009', '\u200A', '\u202F', '\u205F', '\u3000' + }; + + for (char c : nonBreakSpaces) { + String input = "Test" + c + "string"; + String expected = "Test string"; + assertEquals(expected, NotificationBigTextNormalizer.normalizeBigText(input)); + } + } +} -- cgit v1.2.3-59-g8ed1b