Don't allow TTS engines to synthesize too for ahead.

For each synthesis request, we inspect the number of
bytes of audio that have not been written to the mixer yet
and if that is above a certain threshold (currently 500ms),
we block the synthesis thread.

bug:5214603
Change-Id: I24c64c48466bdb96ce47b34cee7da2523ee5f0eb

3 files changed, 86 insertions, 34 deletions

diff --git a/core/java/android/speech/tts/AudioPlaybackHandler.java b/core/java/android/speech/tts/AudioPlaybackHandler.java
index 89b6f321e756..f8d014232ba8 100644
--- a/core/java/android/speech/tts/AudioPlaybackHandler.java
+++ b/core/java/android/speech/tts/AudioPlaybackHandler.java
@@ -390,10 +390,10 @@ class AudioPlaybackHandler {
             audioTrack.play();
         }
         int count = 0;
-        while (count < bufferCopy.mLength) {
+        while (count < bufferCopy.mBytes.length) {
             // Note that we don't take bufferCopy.mOffset into account because
             // it is guaranteed to be 0.
-            int written = audioTrack.write(bufferCopy.mBytes, count, bufferCopy.mLength);
+            int written = audioTrack.write(bufferCopy.mBytes, count, bufferCopy.mBytes.length);
             if (written <= 0) {
                 break;
             }
@@ -453,7 +453,7 @@ class AudioPlaybackHandler {
         }
 
         final AudioTrack audioTrack = params.mAudioTrack;
-        final int bytesPerFrame = getBytesPerFrame(params.mAudioFormat);
+        final int bytesPerFrame = params.mBytesPerFrame;
         final int lengthInBytes = params.mBytesWritten;
         final int lengthInFrames = lengthInBytes / bytesPerFrame;
 
@@ -511,16 +511,6 @@ class AudioPlaybackHandler {
         return 0;
     }
 
-    static int getBytesPerFrame(int audioFormat) {
-        if (audioFormat == AudioFormat.ENCODING_PCM_8BIT) {
-            return 1;
-        } else if (audioFormat == AudioFormat.ENCODING_PCM_16BIT) {
-            return 2;
-        }
-
-        return -1;
-    }
-
     private static void setupVolume(AudioTrack audioTrack, float volume, float pan) {
         float vol = clip(volume, 0.0f, 1.0f);
         float panning = clip(pan, -1.0f, 1.0f);
diff --git a/core/java/android/speech/tts/PlaybackSynthesisCallback.java b/core/java/android/speech/tts/PlaybackSynthesisCallback.java
index 7dbf1acdaf1f..0cca06accd1f 100644
--- a/core/java/android/speech/tts/PlaybackSynthesisCallback.java
+++ b/core/java/android/speech/tts/PlaybackSynthesisCallback.java
@@ -85,6 +85,7 @@ class PlaybackSynthesisCallback extends AbstractSynthesisCallback {
         // Note that mLogger.mError might be true too at this point.
         mLogger.onStopped();
 
+        SynthesisMessageParams token = null;
         synchronized (mStateLock) {
             if (mStopped) {
                 Log.w(TAG, "stop() called twice");
@@ -97,9 +98,19 @@ class PlaybackSynthesisCallback extends AbstractSynthesisCallback {
                 // In all other cases, mAudioTrackHandler.stop() will
                 // result in onComplete being called.
                 mLogger.onWriteData();
+            } else {
+                token = mToken;
             }
             mStopped = true;
         }
+
+        if (token != null) {
+            // This might result in the synthesis thread being woken up, at which
+            // point it will write an additional buffer to the token - but we
+            // won't worry about that because the audio playback queue will be cleared
+            // soon after (see SynthHandler#stop(String).
+            token.clearBuffers();
+        }
     }
 
     @Override
@@ -155,18 +166,22 @@ class PlaybackSynthesisCallback extends AbstractSynthesisCallback {
                     + length + " bytes)");
         }
 
+        SynthesisMessageParams token = null;
         synchronized (mStateLock) {
             if (mToken == null || mStopped) {
                 return TextToSpeech.ERROR;
             }
-
-            // Sigh, another copy.
-            final byte[] bufferCopy = new byte[length];
-            System.arraycopy(buffer, offset, bufferCopy, 0, length);
-            mToken.addBuffer(bufferCopy);
-            mAudioTrackHandler.enqueueSynthesisDataAvailable(mToken);
+            token = mToken;
         }
 
+        // Sigh, another copy.
+        final byte[] bufferCopy = new byte[length];
+        System.arraycopy(buffer, offset, bufferCopy, 0, length);
+        // Might block on mToken.this, if there are too many buffers waiting to
+        // be consumed.
+        token.addBuffer(bufferCopy);
+        mAudioTrackHandler.enqueueSynthesisDataAvailable(token);
+
         mLogger.onEngineDataReceived();
 
         return TextToSpeech.SUCCESS;
@@ -176,6 +191,7 @@ class PlaybackSynthesisCallback extends AbstractSynthesisCallback {
     public int done() {
         if (DBG) Log.d(TAG, "done()");
 
+        SynthesisMessageParams token = null;
         synchronized (mStateLock) {
             if (mDone) {
                 Log.w(TAG, "Duplicate call to done()");
@@ -188,9 +204,12 @@ class PlaybackSynthesisCallback extends AbstractSynthesisCallback {
                 return TextToSpeech.ERROR;
             }
 
-            mAudioTrackHandler.enqueueSynthesisDone(mToken);
-            mLogger.onEngineComplete();
+            token = mToken;
         }
+
+        mAudioTrackHandler.enqueueSynthesisDone(token);
+        mLogger.onEngineComplete();
+
         return TextToSpeech.SUCCESS;
     }
 
diff --git a/core/java/android/speech/tts/SynthesisMessageParams.java b/core/java/android/speech/tts/SynthesisMessageParams.java
index 7da5daabdb6d..3e905d6cc5dd 100644
--- a/core/java/android/speech/tts/SynthesisMessageParams.java
+++ b/core/java/android/speech/tts/SynthesisMessageParams.java
@@ -15,6 +15,7 @@
  */
 package android.speech.tts;
 
+import android.media.AudioFormat;
 import android.media.AudioTrack;
 import android.speech.tts.TextToSpeechService.UtteranceCompletedDispatcher;
 
@@ -24,6 +25,8 @@ import java.util.LinkedList;
  * Params required to play back a synthesis request.
  */
 final class SynthesisMessageParams extends MessageParams {
+    private static final long MAX_UNCONSUMED_AUDIO_MS = 500;
+
     final int mStreamType;
     final int mSampleRateInHz;
     final int mAudioFormat;
@@ -32,10 +35,16 @@ final class SynthesisMessageParams extends MessageParams {
     final float mPan;
     final EventLogger mLogger;
 
+    final int mBytesPerFrame;
+
     volatile AudioTrack mAudioTrack;
-    // Not volatile, accessed only from the synthesis thread.
-    int mBytesWritten;
+    // Written by the synthesis thread, but read on the audio playback
+    // thread.
+    volatile int mBytesWritten;
+    // Not volatile, accessed only from the audio playback thread.
     int mAudioBufferSize;
+    // Always synchronized on "this".
+    int mUnconsumedBytes;
 
     private final LinkedList<ListEntry> mDataBufferList = new LinkedList<ListEntry>();
 
@@ -53,6 +62,8 @@ final class SynthesisMessageParams extends MessageParams {
         mPan = pan;
         mLogger = logger;
 
+        mBytesPerFrame = getBytesPerFrame(mAudioFormat) * mChannelCount;
+
         // initially null.
         mAudioTrack = null;
         mBytesWritten = 0;
@@ -64,18 +75,36 @@ final class SynthesisMessageParams extends MessageParams {
         return TYPE_SYNTHESIS;
     }
 
-    synchronized void addBuffer(byte[] buffer, int offset, int length) {
-        mDataBufferList.add(new ListEntry(buffer, offset, length));
+    synchronized void addBuffer(byte[] buffer) {
+        long unconsumedAudioMs = 0;
+
+        while ((unconsumedAudioMs = getUnconsumedAudioLengthMs()) > MAX_UNCONSUMED_AUDIO_MS) {
+            try {
+                wait();
+            } catch (InterruptedException ie) {
+                return;
+            }
+        }
+
+        mDataBufferList.add(new ListEntry(buffer));
+        mUnconsumedBytes += buffer.length;
     }
 
-    synchronized void addBuffer(byte[] buffer) {
-        mDataBufferList.add(new ListEntry(buffer, 0, buffer.length));
+    synchronized void clearBuffers() {
+        mDataBufferList.clear();
+        mUnconsumedBytes = 0;
+        notifyAll();
     }
 
     synchronized ListEntry getNextBuffer() {
-        return mDataBufferList.poll();
-    }
+        ListEntry entry = mDataBufferList.poll();
+        if (entry != null) {
+            mUnconsumedBytes -= entry.mBytes.length;
+            notifyAll();
+        }
 
+        return entry;
+    }
 
     void setAudioTrack(AudioTrack audioTrack) {
         mAudioTrack = audioTrack;
@@ -85,15 +114,29 @@ final class SynthesisMessageParams extends MessageParams {
         return mAudioTrack;
     }
 
+    // Must be called synchronized on this.
+    private long getUnconsumedAudioLengthMs() {
+        final int unconsumedFrames = mUnconsumedBytes / mBytesPerFrame;
+        final long estimatedTimeMs = unconsumedFrames * 1000 / mSampleRateInHz;
+
+        return estimatedTimeMs;
+    }
+
+    private static int getBytesPerFrame(int audioFormat) {
+        if (audioFormat == AudioFormat.ENCODING_PCM_8BIT) {
+            return 1;
+        } else if (audioFormat == AudioFormat.ENCODING_PCM_16BIT) {
+            return 2;
+        }
+
+        return -1;
+    }
+
     static final class ListEntry {
         final byte[] mBytes;
-        final int mOffset;
-        final int mLength;
 
-        ListEntry(byte[] bytes, int offset, int length) {
+        ListEntry(byte[] bytes) {
             mBytes = bytes;
-            mOffset = offset;
-            mLength = length;
         }
     }
 }