From 06572b9abd60a79deab00e69c7f8752ae9c77655 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 25 Feb 2011 09:13:53 -0800 Subject: DO NOT MERGE: Cherry picked hacks to better hide A/V sync issues. Squashed commit of the following: commit b1732e9b8bfea4c99bd07907c9aac9fcf1e8b264 Author: Andreas Huber Date: Thu Feb 24 12:05:40 2011 -0800 DO NOT MERGE: Alternate patch for late video issue: seek only the video ahead to the next syncframe after the current audio media position, leave audio untouched. Change-Id: I4f6eb4e577147bc12ed2e998bea299f4bcfaf936 related-to-bug: 3375737 commit 5e7282cf0b5c7a613da5e65fd6c8cb33a1058f8e Author: Andreas Huber Date: Fri Feb 11 13:09:36 2011 -0800 DO NOT MERGE: Start playing (and decoding) audio only after the first video frame has been decoded. if there's both audio and video content. This gives the video decoder an opportunity to fill its internal buffer queue at the start of playback. Change-Id: If17c4243546b1c27c8e5ee43941654d0e36f5ee5 related-to-bug: 3431702 Change-Id: Ic3a6be467ff7ad09da04b1d2bb1a692ee38002b4 --- media/libstagefright/AwesomePlayer.cpp | 201 +++++++++++++++++++-------- media/libstagefright/include/AwesomePlayer.h | 13 +- 2 files changed, 153 insertions(+), 61 deletions(-) diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp index 89b3dab46d73..1a8e86db31e0 100644 --- a/media/libstagefright/AwesomePlayer.cpp +++ b/media/libstagefright/AwesomePlayer.cpp @@ -52,7 +52,7 @@ #include "include/LiveSession.h" #define USE_SURFACE_ALLOC 1 -#define FRAME_DROP_FREQ 7 +#define FRAME_DROP_FREQ 0 namespace android { @@ -477,7 +477,7 @@ void AwesomePlayer::reset_l() { mTimeSourceDeltaUs = 0; mVideoTimeUs = 0; - mSeeking = false; + mSeeking = NO_SEEK; mSeekNotificationSent = false; mSeekTimeUs = 0; @@ -783,34 +783,33 @@ status_t AwesomePlayer::play_l() { mAudioPlayer = new AudioPlayer(mAudioSink, this); mAudioPlayer->setSource(mAudioSource); - // We've already started the MediaSource in order to enable - // the prefetcher to read its data. - status_t err = mAudioPlayer->start( - true /* sourceAlreadyStarted */); + mTimeSource = mAudioPlayer; - if (err != OK) { - delete mAudioPlayer; - mAudioPlayer = NULL; + deferredAudioSeek = true; - mFlags &= ~(PLAYING | FIRST_FRAME); + mWatchForAudioSeekComplete = false; + mWatchForAudioEOS = true; + } + } - if (mDecryptHandle != NULL) { - mDrmManagerClient->setPlaybackStatus(mDecryptHandle, - Playback::STOP, 0); - } + CHECK(!(mFlags & AUDIO_RUNNING)); - return err; - } + if (mVideoSource == NULL) { + status_t err = startAudioPlayer_l(); - mTimeSource = mAudioPlayer; + if (err != OK) { + delete mAudioPlayer; + mAudioPlayer = NULL; - deferredAudioSeek = true; + mFlags &= ~(PLAYING | FIRST_FRAME); - mWatchForAudioSeekComplete = false; - mWatchForAudioEOS = true; + if (mDecryptHandle != NULL) { + mDrmManagerClient->setPlaybackStatus( + mDecryptHandle, Playback::STOP, 0); + } + + return err; } - } else { - mAudioPlayer->resume(); } } @@ -842,6 +841,36 @@ status_t AwesomePlayer::play_l() { return OK; } +status_t AwesomePlayer::startAudioPlayer_l() { + CHECK(!(mFlags & AUDIO_RUNNING)); + + if (mAudioSource == NULL || mAudioPlayer == NULL) { + return OK; + } + + if (!(mFlags & AUDIOPLAYER_STARTED)) { + mFlags |= AUDIOPLAYER_STARTED; + + // We've already started the MediaSource in order to enable + // the prefetcher to read its data. + status_t err = mAudioPlayer->start( + true /* sourceAlreadyStarted */); + + if (err != OK) { + notifyListener_l(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err); + return err; + } + } else { + mAudioPlayer->resume(); + } + + mFlags |= AUDIO_RUNNING; + + mWatchForAudioEOS = true; + + return OK; +} + void AwesomePlayer::notifyVideoSize_l() { sp meta = mVideoSource->getFormat(); @@ -943,7 +972,7 @@ status_t AwesomePlayer::pause_l(bool at_eos) { cancelPlayerEvents(true /* keepBufferingGoing */); - if (mAudioPlayer != NULL) { + if (mAudioPlayer != NULL && (mFlags & AUDIO_RUNNING)) { if (at_eos) { // If we played the audio stream to completion we // want to make sure that all samples remaining in the audio @@ -952,6 +981,8 @@ status_t AwesomePlayer::pause_l(bool at_eos) { } else { mAudioPlayer->pause(); } + + mFlags &= ~AUDIO_RUNNING; } mFlags &= ~PLAYING; @@ -1009,7 +1040,7 @@ status_t AwesomePlayer::getPosition(int64_t *positionUs) { if (mRTSPController != NULL) { *positionUs = mRTSPController->getNormalPlayTimeUs(); } - else if (mSeeking) { + else if (mSeeking != NO_SEEK) { *positionUs = mSeekTimeUs; } else if (mVideoSource != NULL) { Mutex::Autolock autoLock(mMiscStateLock); @@ -1053,7 +1084,7 @@ status_t AwesomePlayer::seekTo_l(int64_t timeUs) { play_l(); } - mSeeking = true; + mSeeking = SEEK; mSeekNotificationSent = false; mSeekTimeUs = timeUs; mFlags &= ~(AT_EOS | AUDIO_AT_EOS | VIDEO_AT_EOS); @@ -1072,7 +1103,7 @@ status_t AwesomePlayer::seekTo_l(int64_t timeUs) { } void AwesomePlayer::seekAudioIfNecessary_l() { - if (mSeeking && mVideoSource == NULL && mAudioPlayer != NULL) { + if (mSeeking != NO_SEEK && mVideoSource == NULL && mAudioPlayer != NULL) { mAudioPlayer->seekTo(mSeekTimeUs); mWatchForAudioSeekComplete = true; @@ -1168,7 +1199,12 @@ status_t AwesomePlayer::initVideoDecoder(uint32_t flags) { } void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) { - if (!mSeeking) { + if (mSeeking == SEEK_VIDEO_ONLY) { + mSeeking = NO_SEEK; + return; + } + + if (mSeeking == NO_SEEK) { return; } @@ -1179,9 +1215,7 @@ void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) { // requested seek time instead. mAudioPlayer->seekTo(videoTimeUs < 0 ? mSeekTimeUs : videoTimeUs); - mAudioPlayer->resume(); mWatchForAudioSeekComplete = true; - mWatchForAudioEOS = true; } else if (!mSeekNotificationSent) { // If we're playing video only, report seek complete now, // otherwise audio player will notify us later. @@ -1189,7 +1223,7 @@ void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) { } mFlags |= FIRST_FRAME; - mSeeking = false; + mSeeking = NO_SEEK; mSeekNotificationSent = false; if (mDecryptHandle != NULL) { @@ -1209,13 +1243,13 @@ void AwesomePlayer::onVideoEvent() { } mVideoEventPending = false; - if (mSeeking) { + if (mSeeking != NO_SEEK) { if (mVideoBuffer) { mVideoBuffer->release(); mVideoBuffer = NULL; } - if (mCachedSource != NULL && mAudioSource != NULL) { + if (mSeeking == SEEK && mCachedSource != NULL && mAudioSource != NULL) { // We're going to seek the video source first, followed by // the audio source. // In order to avoid jumps in the DataSource offset caused by @@ -1224,8 +1258,10 @@ void AwesomePlayer::onVideoEvent() { // locations, we'll "pause" the audio source, causing it to // stop reading input data until a subsequent seek. - if (mAudioPlayer != NULL) { + if (mAudioPlayer != NULL && (mFlags & AUDIO_RUNNING)) { mAudioPlayer->pause(); + + mFlags &= ~AUDIO_RUNNING; } mAudioSource->pause(); } @@ -1233,11 +1269,14 @@ void AwesomePlayer::onVideoEvent() { if (!mVideoBuffer) { MediaSource::ReadOptions options; - if (mSeeking) { + if (mSeeking != NO_SEEK) { LOGV("seeking to %lld us (%.2f secs)", mSeekTimeUs, mSeekTimeUs / 1E6); options.setSeekTo( - mSeekTimeUs, MediaSource::ReadOptions::SEEK_CLOSEST_SYNC); + mSeekTimeUs, + mSeeking == SEEK_VIDEO_ONLY + ? MediaSource::ReadOptions::SEEK_NEXT_SYNC + : MediaSource::ReadOptions::SEEK_CLOSEST_SYNC); } for (;;) { status_t err = mVideoSource->read(&mVideoBuffer, &options); @@ -1261,7 +1300,7 @@ void AwesomePlayer::onVideoEvent() { // So video playback is complete, but we may still have // a seek request pending that needs to be applied // to the audio track. - if (mSeeking) { + if (mSeeking != NO_SEEK) { LOGV("video stream ended while seeking!"); } finishSeekIfNecessary(-1); @@ -1287,14 +1326,29 @@ void AwesomePlayer::onVideoEvent() { int64_t timeUs; CHECK(mVideoBuffer->meta_data()->findInt64(kKeyTime, &timeUs)); + if (mSeeking == SEEK_VIDEO_ONLY) { + if (mSeekTimeUs > timeUs) { + LOGI("XXX mSeekTimeUs = %lld us, timeUs = %lld us", + mSeekTimeUs, timeUs); + } + } + { Mutex::Autolock autoLock(mMiscStateLock); mVideoTimeUs = timeUs; } - bool wasSeeking = mSeeking; + SeekType wasSeeking = mSeeking; finishSeekIfNecessary(timeUs); + if (mAudioPlayer != NULL && !(mFlags & AUDIO_RUNNING)) { + status_t err = startAudioPlayer_l(); + if (err != OK) { + LOGE("Startung the audio player failed w/ err %d", err); + return; + } + } + TimeSource *ts = (mFlags & AUDIO_AT_EOS) ? &mSystemTimeSource : mTimeSource; if (mFlags & FIRST_FRAME) { @@ -1309,41 +1363,68 @@ void AwesomePlayer::onVideoEvent() { mTimeSourceDeltaUs = realTimeUs - mediaTimeUs; } - int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs; + if (wasSeeking == SEEK_VIDEO_ONLY) { + int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs; - int64_t latenessUs = nowUs - timeUs; + int64_t latenessUs = nowUs - timeUs; - if (wasSeeking) { - // Let's display the first frame after seeking right away. - latenessUs = 0; + if (latenessUs > 0) { + LOGI("after SEEK_VIDEO_ONLY we're late by %.2f secs", latenessUs / 1E6); + } } - if (mRTPSession != NULL) { - // We'll completely ignore timestamps for gtalk videochat - // and we'll play incoming video as fast as we get it. - latenessUs = 0; - } + if (wasSeeking == NO_SEEK) { + // Let's display the first frame after seeking right away. + + int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs; + + int64_t latenessUs = nowUs - timeUs; + + if (latenessUs > 500000ll + && mRTSPController == NULL + && mAudioPlayer != NULL + && mAudioPlayer->getMediaTimeMapping( + &realTimeUs, &mediaTimeUs)) { + LOGI("we're much too late (%.2f secs), video skipping ahead", + latenessUs / 1E6); - if (latenessUs > 40000) { - // We're more than 40ms late. - LOGV("we're late by %lld us (%.2f secs)", latenessUs, latenessUs / 1E6); - if ( mSinceLastDropped > FRAME_DROP_FREQ) - { - LOGV("we're late by %lld us (%.2f secs) dropping one after %d frames", latenessUs, latenessUs / 1E6, mSinceLastDropped); - mSinceLastDropped = 0; mVideoBuffer->release(); mVideoBuffer = NULL; + mSeeking = SEEK_VIDEO_ONLY; + mSeekTimeUs = mediaTimeUs; + postVideoEvent_l(); return; } - } - if (latenessUs < -10000) { - // We're more than 10ms early. + if (latenessUs > 40000) { + // We're more than 40ms late. - postVideoEvent_l(10000); - return; + LOGV("we're late by %lld us (%.2f secs)", + latenessUs, latenessUs / 1E6); + + if ( mSinceLastDropped > FRAME_DROP_FREQ) + { + LOGV("we're late by %lld us (%.2f secs) dropping one " + "after %d frames", + latenessUs, latenessUs / 1E6, mSinceLastDropped); + + mSinceLastDropped = 0; + mVideoBuffer->release(); + mVideoBuffer = NULL; + + postVideoEvent_l(); + return; + } + } + + if (latenessUs < -10000) { + // We're more than 10ms early. + + postVideoEvent_l(10000); + return; + } } if (mVideoRendererIsPreview || mVideoRenderer == NULL) { @@ -1424,7 +1505,7 @@ void AwesomePlayer::onCheckAudioStatus() { mSeekNotificationSent = true; } - mSeeking = false; + mSeeking = NO_SEEK; } status_t finalStatus; diff --git a/media/libstagefright/include/AwesomePlayer.h b/media/libstagefright/include/AwesomePlayer.h index 41ef181caef9..d5819a46ce43 100644 --- a/media/libstagefright/include/AwesomePlayer.h +++ b/media/libstagefright/include/AwesomePlayer.h @@ -119,6 +119,9 @@ private: // sufficient data to begin playback and finish the preparation phase // for good. PREPARING_CONNECTED = 2048, + + AUDIO_RUNNING = 8192, + AUDIOPLAYER_STARTED = 16384, }; mutable Mutex mLock; @@ -160,7 +163,13 @@ private: int64_t mTimeSourceDeltaUs; int64_t mVideoTimeUs; - bool mSeeking; + enum SeekType { + NO_SEEK, + SEEK, + SEEK_VIDEO_ONLY + }; + SeekType mSeeking; + bool mSeekNotificationSent; int64_t mSeekTimeUs; @@ -256,6 +265,8 @@ private: void finishSeekIfNecessary(int64_t videoTimeUs); void ensureCacheIsFetching_l(); + status_t startAudioPlayer_l(); + AwesomePlayer(const AwesomePlayer &); AwesomePlayer &operator=(const AwesomePlayer &); }; -- cgit v1.2.3-59-g8ed1b