More efficient alternatives to ByteBuffer.
Some upcoming binary XML work needs to efficiently read and write
raw bytes, and we initially started using ByteBuffer. However, that
design had additional overhead since we were performing bounds checks
twice (once to fill/drain buffers, then again to parse data). In
addition, the upstream ByteBuffer makes per-byte method invocations
internally, instead of going directly the the buffer.
This change introduces FastDataInput/Output as local implementations
of DataInput/Output which are focused on performance. They also
handle fill/drain from an underlying Input/OutputStream, and the
included benchmarks show reading 3x faster and writing 2x faster:
timeRead_Upstream_mean: 5543730
timeRead_Local_mean: 1698602
timeWrite_Upstream_mean: 3731119
timeWrite_Local_mean: 1885983
We also use the new CharsetUtils methods to write UTF-8 values
directly without additional allocations whenever possible. This
requires using a non-movable buffer to avoid JNI overhead to gain
the 30% benchmarked performance wins.
Bug: 171832118
Test: atest CorePerfTests:com.android.internal.util.FastDataPerfTest
Test: atest FrameworksCoreTests:com.android.internal.util.FastDataTest
Change-Id: If28ee381adb528d03cc9851d78236d985b6ede16
diff --git a/apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java b/apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java
new file mode 100644
index 0000000..2700fff
--- /dev/null
+++ b/apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.internal.util;
+
+import android.perftests.utils.BenchmarkState;
+import android.perftests.utils.PerfStatusReporter;
+
+import androidx.test.filters.LargeTest;
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+@LargeTest
+@RunWith(AndroidJUnit4.class)
+public class FastDataPerfTest {
+ @Rule
+ public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter();
+
+ private static final int OUTPUT_SIZE = 64000;
+ private static final int BUFFER_SIZE = 4096;
+
+ @Test
+ public void timeWrite_Upstream() throws IOException {
+ final ByteArrayOutputStream os = new ByteArrayOutputStream(OUTPUT_SIZE);
+ final BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+ while (state.keepRunning()) {
+ os.reset();
+ final BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE);
+ final DataOutput out = new DataOutputStream(bos);
+ doWrite(out);
+ bos.flush();
+ }
+ }
+
+ @Test
+ public void timeWrite_Local() throws IOException {
+ final ByteArrayOutputStream os = new ByteArrayOutputStream(OUTPUT_SIZE);
+ final BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+ while (state.keepRunning()) {
+ os.reset();
+ final FastDataOutput out = new FastDataOutput(os, BUFFER_SIZE);
+ doWrite(out);
+ out.flush();
+ }
+ }
+
+ @Test
+ public void timeRead_Upstream() throws Exception {
+ final ByteArrayInputStream is = new ByteArrayInputStream(doWrite());
+ final BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+ while (state.keepRunning()) {
+ is.reset();
+ final BufferedInputStream bis = new BufferedInputStream(is, BUFFER_SIZE);
+ final DataInput in = new DataInputStream(bis);
+ doRead(in);
+ }
+ }
+
+ @Test
+ public void timeRead_Local() throws Exception {
+ final ByteArrayInputStream is = new ByteArrayInputStream(doWrite());
+ final BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+ while (state.keepRunning()) {
+ is.reset();
+ final DataInput in = new FastDataInput(is, BUFFER_SIZE);
+ doRead(in);
+ }
+ }
+
+ /**
+ * Since each iteration is around 64 bytes, we need to iterate many times to
+ * exercise the buffer logic.
+ */
+ private static final int REPEATS = 1000;
+
+ private static byte[] doWrite() throws IOException {
+ final ByteArrayOutputStream os = new ByteArrayOutputStream(OUTPUT_SIZE);
+ final DataOutput out = new DataOutputStream(os);
+ doWrite(out);
+ return os.toByteArray();
+ }
+
+ private static void doWrite(DataOutput out) throws IOException {
+ for (int i = 0; i < REPEATS; i++) {
+ out.writeByte(Byte.MAX_VALUE);
+ out.writeShort(Short.MAX_VALUE);
+ out.writeInt(Integer.MAX_VALUE);
+ out.writeLong(Long.MAX_VALUE);
+ out.writeFloat(Float.MAX_VALUE);
+ out.writeDouble(Double.MAX_VALUE);
+ out.writeUTF("com.example.typical_package_name");
+ }
+ }
+
+ private static void doRead(DataInput in) throws IOException {
+ for (int i = 0; i < REPEATS; i++) {
+ in.readByte();
+ in.readShort();
+ in.readInt();
+ in.readLong();
+ in.readFloat();
+ in.readDouble();
+ in.readUTF();
+ }
+ }
+}
diff --git a/core/java/com/android/internal/util/FastDataInput.java b/core/java/com/android/internal/util/FastDataInput.java
new file mode 100644
index 0000000..2e8cb47
--- /dev/null
+++ b/core/java/com/android/internal/util/FastDataInput.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.internal.util;
+
+import android.annotation.NonNull;
+
+import java.io.BufferedInputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Optimized implementation of {@link DataInput} which buffers data in memory
+ * from the underlying {@link InputStream}.
+ * <p>
+ * Benchmarks have demonstrated this class is 3x more efficient than using a
+ * {@link DataInputStream} with a {@link BufferedInputStream}.
+ */
+public class FastDataInput implements DataInput, Closeable {
+ private static final int MAX_UNSIGNED_SHORT = 65_535;
+
+ private final InputStream mIn;
+
+ private final byte[] mBuffer;
+ private final int mBufferCap;
+
+ private int mBufferPos;
+ private int mBufferLim;
+
+ /**
+ * Values that have been "interned" by {@link #readInternedUTF()}.
+ */
+ private int mStringRefCount = 0;
+ private String[] mStringRefs = new String[32];
+
+ public FastDataInput(@NonNull InputStream in, int bufferSize) {
+ mIn = Objects.requireNonNull(in);
+ if (bufferSize < 8) {
+ throw new IllegalArgumentException();
+ }
+
+ mBuffer = new byte[bufferSize];
+ mBufferCap = mBuffer.length;
+ }
+
+ private void fill(int need) throws IOException {
+ final int remain = mBufferLim - mBufferPos;
+ System.arraycopy(mBuffer, mBufferPos, mBuffer, 0, remain);
+ mBufferPos = 0;
+ mBufferLim = remain;
+ need -= remain;
+
+ while (need > 0) {
+ int c = mIn.read(mBuffer, mBufferLim, mBufferCap - mBufferLim);
+ if (c == -1) {
+ throw new EOFException();
+ } else {
+ mBufferLim += c;
+ need -= c;
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ mIn.close();
+ }
+
+ @Override
+ public void readFully(byte[] b) throws IOException {
+ readFully(b, 0, b.length);
+ }
+
+ @Override
+ public void readFully(byte[] b, int off, int len) throws IOException {
+ // Attempt to read directly from buffer space if there's enough room,
+ // otherwise fall back to chunking into place
+ if (mBufferCap >= len) {
+ if (mBufferLim - mBufferPos < len) fill(len);
+ System.arraycopy(mBuffer, mBufferPos, b, off, len);
+ mBufferPos += len;
+ } else {
+ final int remain = mBufferLim - mBufferPos;
+ System.arraycopy(mBuffer, mBufferPos, b, off, remain);
+ mBufferPos += remain;
+ off += remain;
+ len -= remain;
+
+ while (len > 0) {
+ int c = mIn.read(b, off, len);
+ if (c == -1) {
+ throw new EOFException();
+ } else {
+ off += c;
+ len -= c;
+ }
+ }
+ }
+ }
+
+ @Override
+ public String readUTF() throws IOException {
+ // Attempt to read directly from buffer space if there's enough room,
+ // otherwise fall back to chunking into place
+ final int len = readUnsignedShort();
+ if (mBufferCap >= len) {
+ if (mBufferLim - mBufferPos < len) fill(len);
+ final String res = new String(mBuffer, mBufferPos, len, StandardCharsets.UTF_8);
+ mBufferPos += len;
+ return res;
+ } else {
+ final byte[] tmp = new byte[len];
+ readFully(tmp, 0, tmp.length);
+ return new String(tmp, StandardCharsets.UTF_8);
+ }
+ }
+
+ /**
+ * Read a {@link String} value with the additional signal that the given
+ * value is a candidate for being canonicalized, similar to
+ * {@link String#intern()}.
+ * <p>
+ * Canonicalization is implemented by writing each unique string value once
+ * the first time it appears, and then writing a lightweight {@code short}
+ * reference when that string is written again in the future.
+ *
+ * @see FastDataOutput#writeInternedUTF(String)
+ */
+ public @NonNull String readInternedUTF() throws IOException {
+ final int ref = readUnsignedShort();
+ if (ref == MAX_UNSIGNED_SHORT) {
+ final String s = readUTF();
+
+ // We can only safely intern when we have remaining values; if we're
+ // full we at least sent the string value above
+ if (mStringRefCount < MAX_UNSIGNED_SHORT) {
+ if (mStringRefCount == mStringRefs.length) {
+ mStringRefs = Arrays.copyOf(mStringRefs,
+ mStringRefCount + (mStringRefCount >> 1));
+ }
+ mStringRefs[mStringRefCount++] = s;
+ }
+
+ return s;
+ } else {
+ return mStringRefs[ref];
+ }
+ }
+
+ @Override
+ public boolean readBoolean() throws IOException {
+ return readByte() != 0;
+ }
+
+ /**
+ * Returns the same decoded value as {@link #readByte()} but without
+ * actually consuming the underlying data.
+ */
+ public byte peekByte() throws IOException {
+ if (mBufferLim - mBufferPos < 1) fill(1);
+ return mBuffer[mBufferPos];
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ if (mBufferLim - mBufferPos < 1) fill(1);
+ return mBuffer[mBufferPos++];
+ }
+
+ @Override
+ public int readUnsignedByte() throws IOException {
+ return Byte.toUnsignedInt(readByte());
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ if (mBufferLim - mBufferPos < 2) fill(2);
+ return (short) (((mBuffer[mBufferPos++] & 0xff) << 8) |
+ ((mBuffer[mBufferPos++] & 0xff) << 0));
+ }
+
+ @Override
+ public int readUnsignedShort() throws IOException {
+ return Short.toUnsignedInt((short) readShort());
+ }
+
+ @Override
+ public char readChar() throws IOException {
+ return (char) readShort();
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ if (mBufferLim - mBufferPos < 4) fill(4);
+ return (((mBuffer[mBufferPos++] & 0xff) << 24) |
+ ((mBuffer[mBufferPos++] & 0xff) << 16) |
+ ((mBuffer[mBufferPos++] & 0xff) << 8) |
+ ((mBuffer[mBufferPos++] & 0xff) << 0));
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ if (mBufferLim - mBufferPos < 8) fill(8);
+ int h = ((mBuffer[mBufferPos++] & 0xff) << 24) |
+ ((mBuffer[mBufferPos++] & 0xff) << 16) |
+ ((mBuffer[mBufferPos++] & 0xff) << 8) |
+ ((mBuffer[mBufferPos++] & 0xff) << 0);
+ int l = ((mBuffer[mBufferPos++] & 0xff) << 24) |
+ ((mBuffer[mBufferPos++] & 0xff) << 16) |
+ ((mBuffer[mBufferPos++] & 0xff) << 8) |
+ ((mBuffer[mBufferPos++] & 0xff) << 0);
+ return (((long) h) << 32L) | ((long) l) & 0xffffffffL;
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ return Float.intBitsToFloat(readInt());
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ return Double.longBitsToDouble(readLong());
+ }
+
+ @Override
+ public int skipBytes(int n) throws IOException {
+ // Callers should read data piecemeal
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String readLine() throws IOException {
+ // Callers should read data piecemeal
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/core/java/com/android/internal/util/FastDataOutput.java b/core/java/com/android/internal/util/FastDataOutput.java
new file mode 100644
index 0000000..2530501
--- /dev/null
+++ b/core/java/com/android/internal/util/FastDataOutput.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.internal.util;
+
+import android.annotation.NonNull;
+import android.util.CharsetUtils;
+
+import dalvik.system.VMRuntime;
+
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.Flushable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Objects;
+
+/**
+ * Optimized implementation of {@link DataOutput} which buffers data in memory
+ * before flushing to the underlying {@link OutputStream}.
+ * <p>
+ * Benchmarks have demonstrated this class is 2x more efficient than using a
+ * {@link DataOutputStream} with a {@link BufferedOutputStream}.
+ */
+public class FastDataOutput implements DataOutput, Flushable, Closeable {
+ private static final int MAX_UNSIGNED_SHORT = 65_535;
+
+ private final OutputStream mOut;
+
+ private final byte[] mBuffer;
+ private final long mBufferPtr;
+ private final int mBufferCap;
+
+ private int mBufferPos;
+
+ /**
+ * Values that have been "interned" by {@link #writeInternedUTF(String)}.
+ */
+ private HashMap<String, Short> mStringRefs = new HashMap<>();
+
+ public FastDataOutput(@NonNull OutputStream out, int bufferSize) {
+ mOut = Objects.requireNonNull(out);
+ if (bufferSize < 8) {
+ throw new IllegalArgumentException();
+ }
+
+ mBuffer = (byte[]) VMRuntime.getRuntime().newNonMovableArray(byte.class, bufferSize);
+ mBufferPtr = VMRuntime.getRuntime().addressOf(mBuffer);
+ mBufferCap = mBuffer.length;
+ }
+
+ private void drain() throws IOException {
+ if (mBufferPos > 0) {
+ mOut.write(mBuffer, 0, mBufferPos);
+ mBufferPos = 0;
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ drain();
+ mOut.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ mOut.close();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ writeByte(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ if (mBufferCap < len) {
+ drain();
+ mOut.write(b, off, len);
+ } else {
+ if (mBufferCap - mBufferPos < len) drain();
+ System.arraycopy(b, off, mBuffer, mBufferPos, len);
+ mBufferPos += len;
+ }
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ // Attempt to write directly to buffer space if there's enough room,
+ // otherwise fall back to chunking into place
+ if (mBufferCap - mBufferPos < 2 + s.length()) drain();
+ final int res = CharsetUtils.toUtf8Bytes(s, mBufferPtr, mBufferPos + 2,
+ mBufferCap - mBufferPos - 2);
+ if (res >= 0) {
+ if (res > MAX_UNSIGNED_SHORT) {
+ throw new IOException("UTF-8 length too large: " + res);
+ }
+ writeShort(res);
+ mBufferPos += res;
+ } else {
+ final byte[] tmp = s.getBytes(StandardCharsets.UTF_8);
+ if (tmp.length > MAX_UNSIGNED_SHORT) {
+ throw new IOException("UTF-8 length too large: " + res);
+ }
+ writeShort(tmp.length);
+ write(tmp, 0, tmp.length);
+ }
+ }
+
+ /**
+ * Write a {@link String} value with the additional signal that the given
+ * value is a candidate for being canonicalized, similar to
+ * {@link String#intern()}.
+ * <p>
+ * Canonicalization is implemented by writing each unique string value once
+ * the first time it appears, and then writing a lightweight {@code short}
+ * reference when that string is written again in the future.
+ *
+ * @see FastDataInput#readInternedUTF()
+ */
+ public void writeInternedUTF(@NonNull String s) throws IOException {
+ Short ref = mStringRefs.get(s);
+ if (ref != null) {
+ writeShort(ref);
+ } else {
+ writeShort(MAX_UNSIGNED_SHORT);
+ writeUTF(s);
+
+ // We can only safely intern when we have remaining values; if we're
+ // full we at least sent the string value above
+ ref = (short) mStringRefs.size();
+ if (ref < MAX_UNSIGNED_SHORT) {
+ mStringRefs.put(s, ref);
+ }
+ }
+ }
+
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ writeByte(v ? 1 : 0);
+ }
+
+ @Override
+ public void writeByte(int v) throws IOException {
+ if (mBufferCap - mBufferPos < 1) drain();
+ mBuffer[mBufferPos++] = (byte) ((v >> 0) & 0xff);
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ if (mBufferCap - mBufferPos < 2) drain();
+ mBuffer[mBufferPos++] = (byte) ((v >> 8) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((v >> 0) & 0xff);
+ }
+
+ @Override
+ public void writeChar(int v) throws IOException {
+ writeShort((short) v);
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ if (mBufferCap - mBufferPos < 4) drain();
+ mBuffer[mBufferPos++] = (byte) ((v >> 24) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((v >> 16) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((v >> 8) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((v >> 0) & 0xff);
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ if (mBufferCap - mBufferPos < 8) drain();
+ int i = (int) (v >> 32);
+ mBuffer[mBufferPos++] = (byte) ((i >> 24) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((i >> 16) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((i >> 8) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((i >> 0) & 0xff);
+ i = (int) v;
+ mBuffer[mBufferPos++] = (byte) ((i >> 24) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((i >> 16) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((i >> 8) & 0xff);
+ mBuffer[mBufferPos++] = (byte) ((i >> 0) & 0xff);
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ writeInt(Float.floatToIntBits(v));
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ writeLong(Double.doubleToLongBits(v));
+ }
+
+ @Override
+ public void writeBytes(String s) throws IOException {
+ // Callers should use writeUTF()
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void writeChars(String s) throws IOException {
+ // Callers should use writeUTF()
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/core/tests/coretests/src/com/android/internal/util/FastDataTest.java b/core/tests/coretests/src/com/android/internal/util/FastDataTest.java
new file mode 100644
index 0000000..841d659
--- /dev/null
+++ b/core/tests/coretests/src/com/android/internal/util/FastDataTest.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.internal.util;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import android.annotation.NonNull;
+import android.util.ExceptionUtils;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.function.Consumer;
+
+@RunWith(AndroidJUnit4.class)
+public class FastDataTest {
+ private static final String TEST_SHORT_STRING = "a";
+ private static final String TEST_LONG_STRING = "com☃example☃typical☃package☃name";
+ private static final byte[] TEST_BYTES = TEST_LONG_STRING.getBytes(StandardCharsets.UTF_16LE);
+
+ @Test
+ public void testEndOfFile_Int() throws Exception {
+ try (FastDataInput in = new FastDataInput(new ByteArrayInputStream(
+ new byte[] { 1 }), 1000)) {
+ assertThrows(EOFException.class, () -> in.readInt());
+ }
+ try (FastDataInput in = new FastDataInput(new ByteArrayInputStream(
+ new byte[] { 1, 1, 1, 1 }), 1000)) {
+ assertEquals(1, in.readByte());
+ assertThrows(EOFException.class, () -> in.readInt());
+ }
+ }
+
+ @Test
+ public void testEndOfFile_String() throws Exception {
+ try (FastDataInput in = new FastDataInput(new ByteArrayInputStream(
+ new byte[] { 1 }), 1000)) {
+ assertThrows(EOFException.class, () -> in.readUTF());
+ }
+ try (FastDataInput in = new FastDataInput(new ByteArrayInputStream(
+ new byte[] { 1, 1, 1, 1 }), 1000)) {
+ assertThrows(EOFException.class, () -> in.readUTF());
+ }
+ }
+
+ @Test
+ public void testEndOfFile_Bytes_Small() throws Exception {
+ try (FastDataInput in = new FastDataInput(new ByteArrayInputStream(
+ new byte[] { 1, 1, 1, 1 }), 1000)) {
+ final byte[] tmp = new byte[10];
+ assertThrows(EOFException.class, () -> in.readFully(tmp));
+ }
+ try (FastDataInput in = new FastDataInput(new ByteArrayInputStream(
+ new byte[] { 1, 1, 1, 1 }), 1000)) {
+ final byte[] tmp = new byte[10_000];
+ assertThrows(EOFException.class, () -> in.readFully(tmp));
+ }
+ }
+
+ @Test
+ public void testUTF_Bounds() throws Exception {
+ final char[] buf = new char[65_535];
+ try (FastDataOutput out = new FastDataOutput(new ByteArrayOutputStream(), BOUNCE_SIZE)) {
+ // Writing simple string will fit fine
+ Arrays.fill(buf, '!');
+ final String simple = new String(buf);
+ out.writeUTF(simple);
+ out.writeInternedUTF(simple);
+
+ // Just one complex char will cause it to overflow
+ buf[0] = '☃';
+ final String complex = new String(buf);
+ assertThrows(IOException.class, () -> out.writeUTF(complex));
+ assertThrows(IOException.class, () -> out.writeInternedUTF(complex));
+ }
+ }
+
+ @Test
+ public void testBounce_Char() throws Exception {
+ doBounce((out) -> {
+ out.writeChar('\0');
+ out.writeChar('☃');
+ }, (in) -> {
+ assertEquals('\0', in.readChar());
+ assertEquals('☃', in.readChar());
+ });
+ }
+
+ @Test
+ public void testBounce_Short() throws Exception {
+ doBounce((out) -> {
+ out.writeShort(0);
+ out.writeShort((short) 0x0f0f);
+ out.writeShort((short) 0xf0f0);
+ out.writeShort(Short.MIN_VALUE);
+ out.writeShort(Short.MAX_VALUE);
+ }, (in) -> {
+ assertEquals(0, in.readShort());
+ assertEquals((short) 0x0f0f, in.readShort());
+ assertEquals((short) 0xf0f0, in.readShort());
+ assertEquals(Short.MIN_VALUE, in.readShort());
+ assertEquals(Short.MAX_VALUE, in.readShort());
+ });
+ }
+
+ @Test
+ public void testBounce_Int() throws Exception {
+ doBounce((out) -> {
+ out.writeInt(0);
+ out.writeInt(0x0f0f0f0f);
+ out.writeInt(0xf0f0f0f0);
+ out.writeInt(Integer.MIN_VALUE);
+ out.writeInt(Integer.MAX_VALUE);
+ }, (in) -> {
+ assertEquals(0, in.readInt());
+ assertEquals(0x0f0f0f0f, in.readInt());
+ assertEquals(0xf0f0f0f0, in.readInt());
+ assertEquals(Integer.MIN_VALUE, in.readInt());
+ assertEquals(Integer.MAX_VALUE, in.readInt());
+ });
+ }
+
+ @Test
+ public void testBounce_Long() throws Exception {
+ doBounce((out) -> {
+ out.writeLong(0);
+ out.writeLong(0x0f0f0f0f0f0f0f0fL);
+ out.writeLong(0xf0f0f0f0f0f0f0f0L);
+ out.writeLong(Long.MIN_VALUE);
+ out.writeLong(Long.MAX_VALUE);
+ }, (in) -> {
+ assertEquals(0, in.readLong());
+ assertEquals(0x0f0f0f0f0f0f0f0fL, in.readLong());
+ assertEquals(0xf0f0f0f0f0f0f0f0L, in.readLong());
+ assertEquals(Long.MIN_VALUE, in.readLong());
+ assertEquals(Long.MAX_VALUE, in.readLong());
+ });
+ }
+
+ @Test
+ public void testBounce_UTF() throws Exception {
+ doBounce((out) -> {
+ out.writeUTF("");
+ out.writeUTF("☃");
+ out.writeUTF("example");
+ }, (in) -> {
+ assertEquals("", in.readUTF());
+ assertEquals("☃", in.readUTF());
+ assertEquals("example", in.readUTF());
+ });
+ }
+
+ @Test
+ public void testBounce_UTF_Exact() throws Exception {
+ final char[] expectedBuf = new char[BOUNCE_SIZE];
+ Arrays.fill(expectedBuf, '!');
+ final String expected = new String(expectedBuf);
+
+ doBounce((out) -> {
+ out.writeUTF(expected);
+ }, (in) -> {
+ final String actual = in.readUTF();
+ assertEquals(expected.length(), actual.length());
+ assertEquals(expected, actual);
+ });
+ }
+
+ @Test
+ public void testBounce_UTF_Maximum() throws Exception {
+ final char[] expectedBuf = new char[65_535];
+ Arrays.fill(expectedBuf, '!');
+ final String expected = new String(expectedBuf);
+
+ doBounce((out) -> {
+ out.writeUTF(expected);
+ }, (in) -> {
+ final String actual = in.readUTF();
+ assertEquals(expected.length(), actual.length());
+ assertEquals(expected, actual);
+ }, 1);
+ }
+
+ @Test
+ public void testBounce_InternedUTF() throws Exception {
+ doBounce((out) -> {
+ out.writeInternedUTF("foo");
+ out.writeInternedUTF("bar");
+ out.writeInternedUTF("baz");
+ out.writeInternedUTF("bar");
+ out.writeInternedUTF("foo");
+ }, (in) -> {
+ assertEquals("foo", in.readInternedUTF());
+ assertEquals("bar", in.readInternedUTF());
+ assertEquals("baz", in.readInternedUTF());
+ assertEquals("bar", in.readInternedUTF());
+ assertEquals("foo", in.readInternedUTF());
+ });
+ }
+
+ /**
+ * Verify that when we overflow the maximum number of interned string
+ * references, we still transport the raw string values successfully.
+ */
+ @Test
+ public void testBounce_InternedUTF_Maximum() throws Exception {
+ final int num = 70_000;
+ doBounce((out) -> {
+ for (int i = 0; i < num; i++) {
+ out.writeInternedUTF("foo" + i);
+ }
+ }, (in) -> {
+ for (int i = 0; i < num; i++) {
+ assertEquals("foo" + i, in.readInternedUTF());
+ }
+ }, 1);
+ }
+
+ @Test
+ public void testBounce_Bytes() throws Exception {
+ doBounce((out) -> {
+ out.write(TEST_BYTES, 8, 32);
+ out.writeInt(64);
+ }, (in) -> {
+ final byte[] tmp = new byte[128];
+ in.readFully(tmp, 8, 32);
+ assertArrayEquals(Arrays.copyOfRange(TEST_BYTES, 8, 8 + 32),
+ Arrays.copyOfRange(tmp, 8, 8 + 32));
+ assertEquals(64, in.readInt());
+ });
+ }
+
+ @Test
+ public void testBounce_Mixed() throws Exception {
+ doBounce((out) -> {
+ out.writeBoolean(true);
+ out.writeBoolean(false);
+ out.writeByte(1);
+ out.writeShort(2);
+ out.writeInt(4);
+ out.writeUTF(TEST_SHORT_STRING);
+ out.writeUTF(TEST_LONG_STRING);
+ out.writeLong(8L);
+ out.writeFloat(16f);
+ out.writeDouble(32d);
+ }, (in) -> {
+ assertEquals(true, in.readBoolean());
+ assertEquals(false, in.readBoolean());
+ assertEquals(1, in.readByte());
+ assertEquals(2, in.readShort());
+ assertEquals(4, in.readInt());
+ assertEquals(TEST_SHORT_STRING, in.readUTF());
+ assertEquals(TEST_LONG_STRING, in.readUTF());
+ assertEquals(8L, in.readLong());
+ assertEquals(16f, in.readFloat(), 0.01);
+ assertEquals(32d, in.readDouble(), 0.01);
+ });
+ }
+
+ /**
+ * Buffer size to use for {@link #doBounce}; purposefully chosen to be a
+ * small prime number to help uncover edge cases.
+ */
+ private static final int BOUNCE_SIZE = 11;
+
+ /**
+ * Number of times to repeat message when bouncing; repeating is used to
+ * help uncover edge cases.
+ */
+ private static final int BOUNCE_REPEAT = 1_000;
+
+ /**
+ * Verify that some common data can be written and read back, effectively
+ * "bouncing" it through a serialized representation.
+ */
+ private static void doBounce(@NonNull ThrowingConsumer<FastDataOutput> out,
+ @NonNull ThrowingConsumer<FastDataInput> in) throws Exception {
+ doBounce(out, in, BOUNCE_REPEAT);
+ }
+
+ private static void doBounce(@NonNull ThrowingConsumer<FastDataOutput> out,
+ @NonNull ThrowingConsumer<FastDataInput> in, int count) throws Exception {
+ final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ final FastDataOutput outData = new FastDataOutput(outStream, BOUNCE_SIZE);
+ for (int i = 0; i < count; i++) {
+ out.accept(outData);
+ }
+ outData.flush();
+
+ final ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray());
+ final FastDataInput inData = new FastDataInput(inStream, BOUNCE_SIZE);
+ for (int i = 0; i < count; i++) {
+ in.accept(inData);
+ }
+ }
+
+ private static <T extends Exception> void assertThrows(Class<T> clazz, ThrowingRunnable r)
+ throws Exception {
+ try {
+ r.run();
+ fail("Expected " + clazz + " to be thrown");
+ } catch (Exception e) {
+ if (!clazz.isAssignableFrom(e.getClass())) {
+ throw e;
+ }
+ }
+ }
+
+ public interface ThrowingRunnable {
+ void run() throws Exception;
+ }
+
+ public interface ThrowingConsumer<T> extends Consumer<T> {
+ void acceptOrThrow(T t) throws Exception;
+
+ @Override
+ default void accept(T t) {
+ try {
+ acceptOrThrow(t);
+ } catch (Exception ex) {
+ throw ExceptionUtils.propagate(ex);
+ }
+ }
+ }
+}