blob: 39be40ddcc2e0356616de5964e8bbf082b4a14a9 [file] [log] [blame]
Andreas Huberf9334412010-12-15 15:17:42 -08001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
Dongwon Kang3dced072017-10-09 08:55:22 -070021#include "AWakeLock.h"
Andreas Huberf9334412010-12-15 15:17:42 -080022#include "NuPlayerRenderer.h"
Andy Hungf2c87b32016-04-07 19:49:29 -070023#include <algorithm>
Andy Hunga0b39712015-05-31 22:40:49 -070024#include <cutils/properties.h>
Andreas Huberf9334412010-12-15 15:17:42 -080025#include <media/stagefright/foundation/ADebug.h>
Andreas Huber5bc087c2010-12-23 10:27:40 -080026#include <media/stagefright/foundation/AMessage.h>
Ronghua Wua73d9e02014-10-08 15:13:29 -070027#include <media/stagefright/foundation/AUtils.h>
Wei Jia5833b6a2015-03-09 17:01:47 -070028#include <media/stagefright/MediaClock.h>
Ari Hausman-Cohen2b674bb2018-06-22 14:53:20 -070029#include <media/stagefright/MediaCodecConstants.h>
30#include <media/stagefright/MediaDefs.h>
Wei Jiabc2fb722014-07-08 16:37:57 -070031#include <media/stagefright/MediaErrors.h>
32#include <media/stagefright/MetaData.h>
Chong Zhang3b9eb1f2014-10-15 17:05:08 -070033#include <media/stagefright/Utils.h>
Lajos Molnara3725d72015-07-01 16:47:22 -070034#include <media/stagefright/VideoFrameScheduler.h>
Wonsik Kim7e34bf52016-08-23 00:09:18 +090035#include <media/MediaCodecBuffer.h>
Lajos Molnardc43dfa2014-05-07 15:33:04 -070036
Lajos Molnar09524832014-07-17 14:29:51 -070037#include <inttypes.h>
38
Andreas Huberf9334412010-12-15 15:17:42 -080039namespace android {
40
Andy Hunga0b39712015-05-31 22:40:49 -070041/*
42 * Example of common configuration settings in shell script form
43
44 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
45 adb shell setprop audio.offload.disable 1
46
47 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
48 adb shell setprop audio.offload.video 1
49
50 #Use audio callbacks for PCM data
51 adb shell setprop media.stagefright.audio.cbk 1
52
Andy Hung288da022015-05-31 22:55:59 -070053 #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
54 adb shell setprop media.stagefright.audio.deep 1
55
Andy Hung179652e2015-05-31 22:49:46 -070056 #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
57 adb shell setprop media.stagefright.audio.sink 1000
58
Andy Hunga0b39712015-05-31 22:40:49 -070059 * These configurations take effect for the next track played (not the current track).
60 */
61
62static inline bool getUseAudioCallbackSetting() {
63 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
64}
65
Andy Hung179652e2015-05-31 22:49:46 -070066static inline int32_t getAudioSinkPcmMsSetting() {
67 return property_get_int32(
68 "media.stagefright.audio.sink", 500 /* default_value */);
69}
70
Ronghua Wuf5b1db12014-09-09 10:11:08 -070071// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
72// is closed to allow the audio DSP to power down.
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -080073static const int64_t kOffloadPauseMaxUs = 10000000LL;
Ronghua Wuf5b1db12014-09-09 10:11:08 -070074
Wei Jia12b62652016-02-09 17:49:31 -080075// Maximum allowed delay from AudioSink, 1.5 seconds.
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -080076static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000LL;
Wei Jia12b62652016-02-09 17:49:31 -080077
Andy Hung528c8402016-01-12 12:39:34 -080078static const int64_t kMinimumAudioClockUpdatePeriodUs = 20 /* msec */ * 1000;
79
Wei Jiacc951512019-04-03 16:10:42 -070080// Default video frame display duration when only video exists.
81// Used to set max media time in MediaClock.
82static const int64_t kDefaultVideoFrameIntervalUs = 100000LL;
83
Andreas Huber714aa7b2011-09-13 08:28:38 -070084// static
Andy Hungf0e83642014-12-19 17:55:56 -080085const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
86 AUDIO_CHANNEL_NONE,
87 AUDIO_OUTPUT_FLAG_NONE,
88 AUDIO_FORMAT_INVALID,
89 0, // mNumChannels
90 0 // mSampleRate
91};
92
93// static
Andreas Huber714aa7b2011-09-13 08:28:38 -070094const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
95
Ari Hausman-Cohen2b674bb2018-06-22 14:53:20 -070096static audio_format_t constexpr audioFormatFromEncoding(int32_t pcmEncoding) {
97 switch (pcmEncoding) {
98 case kAudioEncodingPcmFloat:
99 return AUDIO_FORMAT_PCM_FLOAT;
100 case kAudioEncodingPcm16bit:
101 return AUDIO_FORMAT_PCM_16_BIT;
102 case kAudioEncodingPcm8bit:
103 return AUDIO_FORMAT_PCM_8_BIT; // TODO: do we want to support this?
104 default:
105 ALOGE("%s: Invalid encoding: %d", __func__, pcmEncoding);
106 return AUDIO_FORMAT_INVALID;
107 }
108}
109
Andreas Huberf9334412010-12-15 15:17:42 -0800110NuPlayer::Renderer::Renderer(
111 const sp<MediaPlayerBase::AudioSink> &sink,
Wei Jia0a68f662017-08-30 18:01:26 -0700112 const sp<MediaClock> &mediaClock,
Andreas Huberd5e56232013-03-12 11:01:43 -0700113 const sp<AMessage> &notify,
114 uint32_t flags)
Andreas Huberf9334412010-12-15 15:17:42 -0800115 : mAudioSink(sink),
Wei Jiad2f35de2016-02-17 16:48:06 -0800116 mUseVirtualAudioSink(false),
Andreas Huberf9334412010-12-15 15:17:42 -0800117 mNotify(notify),
Andreas Huberd5e56232013-03-12 11:01:43 -0700118 mFlags(flags),
Andreas Huberf9334412010-12-15 15:17:42 -0800119 mNumFramesWritten(0),
120 mDrainAudioQueuePending(false),
121 mDrainVideoQueuePending(false),
122 mAudioQueueGeneration(0),
123 mVideoQueueGeneration(0),
Wei Jia7b15cb32015-02-03 17:46:06 -0800124 mAudioDrainGeneration(0),
125 mVideoDrainGeneration(0),
Wei Jia7c8d0e02015-08-27 17:40:21 -0700126 mAudioEOSGeneration(0),
Wei Jia0a68f662017-08-30 18:01:26 -0700127 mMediaClock(mediaClock),
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700128 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
Ronghua Wua73d9e02014-10-08 15:13:29 -0700129 mAudioFirstAnchorTimeMediaUs(-1),
Ronghua Wueecb7802014-10-19 23:12:50 -0700130 mAnchorTimeMediaUs(-1),
Lajos Molnarf5926712014-10-21 09:36:43 -0700131 mAnchorNumFramesWritten(-1),
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -0800132 mVideoLateByUs(0LL),
Wei Jiadc293e32017-10-16 18:29:37 -0700133 mNextVideoTimeMediaUs(-1),
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800134 mHasAudio(false),
135 mHasVideo(false),
Chong Zhang7137ec72014-11-12 16:41:05 -0800136 mNotifyCompleteAudio(false),
137 mNotifyCompleteVideo(false),
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800138 mSyncQueues(false),
Andreas Huber714aa7b2011-09-13 08:28:38 -0700139 mPaused(false),
Andy Hungb03dcb32015-08-27 16:18:59 -0700140 mPauseDrainAudioAllowedUs(0),
Andy Hung09e0c362014-09-12 15:12:16 -0700141 mVideoSampleReceived(false),
James Dongf57b4ea2012-07-20 13:38:36 -0700142 mVideoRenderingStarted(false),
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700143 mVideoRenderingStartGeneration(0),
144 mAudioRenderingStartGeneration(0),
Andy Hungb12ea0b2015-09-03 15:13:01 -0700145 mRenderingDataDelivered(false),
Andy Hung528c8402016-01-12 12:39:34 -0800146 mNextAudioClockUpdateTimeUs(-1),
Wei Jia2995dc72015-07-24 16:00:33 -0700147 mLastAudioMediaTimeUs(-1),
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700148 mAudioOffloadPauseTimeoutGeneration(0),
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700149 mAudioTornDown(false),
Lajos Molnard5923402014-10-20 17:47:54 -0700150 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
Andy Hungf0e83642014-12-19 17:55:56 -0800151 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
Lajos Molnard5923402014-10-20 17:47:54 -0700152 mTotalBuffersQueued(0),
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800153 mLastAudioBufferDrained(0),
Andy Hunga0b39712015-05-31 22:40:49 -0700154 mUseAudioCallback(false),
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800155 mWakeLock(new AWakeLock()) {
Wei Jia0a68f662017-08-30 18:01:26 -0700156 CHECK(mediaClock != NULL);
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700157 mPlaybackRate = mPlaybackSettings.mSpeed;
158 mMediaClock->setPlaybackRate(mPlaybackRate);
Andreas Huberf9334412010-12-15 15:17:42 -0800159}
160
161NuPlayer::Renderer::~Renderer() {
Wei Jiabc2fb722014-07-08 16:37:57 -0700162 if (offloadingAudio()) {
163 mAudioSink->stop();
164 mAudioSink->flush();
165 mAudioSink->close();
166 }
Wei Jia4c74fde2016-08-26 16:00:52 -0700167
168 // Try to avoid racing condition in case callback is still on.
169 Mutex::Autolock autoLock(mLock);
Wei Jia9a3101b2016-11-08 14:34:24 -0800170 if (mUseAudioCallback) {
171 flushQueue(&mAudioQueue);
172 flushQueue(&mVideoQueue);
173 }
Wei Jia4c74fde2016-08-26 16:00:52 -0700174 mWakeLock.clear();
Wei Jia4c74fde2016-08-26 16:00:52 -0700175 mVideoScheduler.clear();
176 mNotify.clear();
177 mAudioSink.clear();
Andreas Huberf9334412010-12-15 15:17:42 -0800178}
179
180void NuPlayer::Renderer::queueBuffer(
181 bool audio,
Wonsik Kim7e34bf52016-08-23 00:09:18 +0900182 const sp<MediaCodecBuffer> &buffer,
Andreas Huberf9334412010-12-15 15:17:42 -0800183 const sp<AMessage> &notifyConsumed) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800184 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800185 msg->setInt32("queueGeneration", getQueueGeneration(audio));
Andreas Huberf9334412010-12-15 15:17:42 -0800186 msg->setInt32("audio", static_cast<int32_t>(audio));
Wonsik Kim7e34bf52016-08-23 00:09:18 +0900187 msg->setObject("buffer", buffer);
Andreas Huberf9334412010-12-15 15:17:42 -0800188 msg->setMessage("notifyConsumed", notifyConsumed);
189 msg->post();
190}
191
192void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
193 CHECK_NE(finalResult, (status_t)OK);
194
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800195 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800196 msg->setInt32("queueGeneration", getQueueGeneration(audio));
Andreas Huberf9334412010-12-15 15:17:42 -0800197 msg->setInt32("audio", static_cast<int32_t>(audio));
198 msg->setInt32("finalResult", finalResult);
199 msg->post();
200}
201
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700202status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
203 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
204 writeToAMessage(msg, rate);
205 sp<AMessage> response;
206 status_t err = msg->postAndAwaitResponse(&response);
207 if (err == OK && response != NULL) {
208 CHECK(response->findInt32("err", &err));
209 }
210 return err;
211}
212
213status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
214 if (rate.mSpeed == 0.f) {
215 onPause();
216 // don't call audiosink's setPlaybackRate if pausing, as pitch does not
217 // have to correspond to the any non-0 speed (e.g old speed). Keep
218 // settings nonetheless, using the old speed, in case audiosink changes.
219 AudioPlaybackRate newRate = rate;
220 newRate.mSpeed = mPlaybackSettings.mSpeed;
221 mPlaybackSettings = newRate;
222 return OK;
223 }
224
Wei Jia27ea08e2015-05-12 14:50:35 -0700225 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700226 status_t err = mAudioSink->setPlaybackRate(rate);
227 if (err != OK) {
228 return err;
229 }
230 }
231 mPlaybackSettings = rate;
232 mPlaybackRate = rate.mSpeed;
233 mMediaClock->setPlaybackRate(mPlaybackRate);
234 return OK;
235}
236
237status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
238 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
239 sp<AMessage> response;
240 status_t err = msg->postAndAwaitResponse(&response);
241 if (err == OK && response != NULL) {
242 CHECK(response->findInt32("err", &err));
243 if (err == OK) {
244 readFromAMessage(response, rate);
245 }
246 }
247 return err;
248}
249
250status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
Wei Jia27ea08e2015-05-12 14:50:35 -0700251 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700252 status_t err = mAudioSink->getPlaybackRate(rate);
253 if (err == OK) {
254 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
255 ALOGW("correcting mismatch in internal/external playback rate");
256 }
257 // get playback settings used by audiosink, as it may be
258 // slightly off due to audiosink not taking small changes.
259 mPlaybackSettings = *rate;
260 if (mPaused) {
261 rate->mSpeed = 0.f;
262 }
263 }
264 return err;
265 }
266 *rate = mPlaybackSettings;
267 return OK;
268}
269
270status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
271 sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
272 writeToAMessage(msg, sync, videoFpsHint);
273 sp<AMessage> response;
274 status_t err = msg->postAndAwaitResponse(&response);
275 if (err == OK && response != NULL) {
276 CHECK(response->findInt32("err", &err));
277 }
278 return err;
279}
280
281status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
282 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
283 return BAD_VALUE;
284 }
285 // TODO: support sync sources
286 return INVALID_OPERATION;
287}
288
289status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
290 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
291 sp<AMessage> response;
292 status_t err = msg->postAndAwaitResponse(&response);
293 if (err == OK && response != NULL) {
294 CHECK(response->findInt32("err", &err));
295 if (err == OK) {
296 readFromAMessage(response, sync, videoFps);
297 }
298 }
299 return err;
300}
301
302status_t NuPlayer::Renderer::onGetSyncSettings(
303 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
304 *sync = mSyncSettings;
305 *videoFps = -1.f;
306 return OK;
Wei Jia98160162015-02-04 17:01:11 -0800307}
308
Chong Zhang7137ec72014-11-12 16:41:05 -0800309void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
Andreas Huberf9334412010-12-15 15:17:42 -0800310 {
Wei Jia7b15cb32015-02-03 17:46:06 -0800311 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -0800312 if (audio) {
Chong Zhang7137ec72014-11-12 16:41:05 -0800313 mNotifyCompleteAudio |= notifyComplete;
Wei Jia3b0cd262015-07-13 11:28:10 -0700314 clearAudioFirstAnchorTime_l();
Wei Jia7b15cb32015-02-03 17:46:06 -0800315 ++mAudioQueueGeneration;
316 ++mAudioDrainGeneration;
Andreas Huberf9334412010-12-15 15:17:42 -0800317 } else {
Chong Zhang7137ec72014-11-12 16:41:05 -0800318 mNotifyCompleteVideo |= notifyComplete;
Wei Jia7b15cb32015-02-03 17:46:06 -0800319 ++mVideoQueueGeneration;
320 ++mVideoDrainGeneration;
Wei Jiacc951512019-04-03 16:10:42 -0700321 mNextVideoTimeMediaUs = -1;
Andreas Huberf9334412010-12-15 15:17:42 -0800322 }
Wei Jia7b15cb32015-02-03 17:46:06 -0800323
Wei Jia9a3101b2016-11-08 14:34:24 -0800324 mMediaClock->clearAnchor();
Wei Jia7b15cb32015-02-03 17:46:06 -0800325 mVideoLateByUs = 0;
326 mSyncQueues = false;
Andreas Huberf9334412010-12-15 15:17:42 -0800327 }
328
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800329 sp<AMessage> msg = new AMessage(kWhatFlush, this);
Andreas Huberf9334412010-12-15 15:17:42 -0800330 msg->setInt32("audio", static_cast<int32_t>(audio));
331 msg->post();
332}
333
334void NuPlayer::Renderer::signalTimeDiscontinuity() {
Wei Jia28a8a9f2014-08-18 11:29:50 -0700335}
336
337void NuPlayer::Renderer::signalDisableOffloadAudio() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800338 (new AMessage(kWhatDisableOffloadAudio, this))->post();
Wei Jia28a8a9f2014-08-18 11:29:50 -0700339}
340
Ronghua Wua10fd232014-11-06 16:15:20 -0800341void NuPlayer::Renderer::signalEnableOffloadAudio() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800342 (new AMessage(kWhatEnableOffloadAudio, this))->post();
Ronghua Wua10fd232014-11-06 16:15:20 -0800343}
344
Andreas Huberb4082222011-01-20 15:23:04 -0800345void NuPlayer::Renderer::pause() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800346 (new AMessage(kWhatPause, this))->post();
Andreas Huberb4082222011-01-20 15:23:04 -0800347}
348
349void NuPlayer::Renderer::resume() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800350 (new AMessage(kWhatResume, this))->post();
Andreas Huberb4082222011-01-20 15:23:04 -0800351}
352
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700353void NuPlayer::Renderer::setVideoFrameRate(float fps) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800354 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700355 msg->setFloat("frame-rate", fps);
356 msg->post();
357}
358
Wei Jia4ecbea32015-10-20 18:09:57 -0700359// Called on any threads without mLock acquired.
Wei Jia7b15cb32015-02-03 17:46:06 -0800360status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
Wei Jia4ecbea32015-10-20 18:09:57 -0700361 status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
362 if (result == OK) {
363 return result;
364 }
365
366 // MediaClock has not started yet. Try to start it if possible.
367 {
368 Mutex::Autolock autoLock(mLock);
369 if (mAudioFirstAnchorTimeMediaUs == -1) {
370 return result;
371 }
372
373 AudioTimestamp ts;
374 status_t res = mAudioSink->getTimestamp(ts);
375 if (res != OK) {
376 return result;
377 }
378
379 // AudioSink has rendered some frames.
380 int64_t nowUs = ALooper::GetNowUs();
zengjingdb131352018-03-19 17:23:49 +0800381 int64_t playedOutDurationUs = mAudioSink->getPlayedOutDurationUs(nowUs);
382 if (playedOutDurationUs == 0) {
383 *mediaUs = mAudioFirstAnchorTimeMediaUs;
384 return OK;
385 }
386 int64_t nowMediaUs = playedOutDurationUs + mAudioFirstAnchorTimeMediaUs;
Wei Jia4ecbea32015-10-20 18:09:57 -0700387 mMediaClock->updateAnchor(nowMediaUs, nowUs, -1);
388 }
389
Wei Jia98160162015-02-04 17:01:11 -0800390 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700391}
392
Wei Jia7b15cb32015-02-03 17:46:06 -0800393void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
394 mAudioFirstAnchorTimeMediaUs = -1;
395 mMediaClock->setStartingTimeMedia(-1);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700396}
397
Wei Jia7b15cb32015-02-03 17:46:06 -0800398void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
Ronghua Wua73d9e02014-10-08 15:13:29 -0700399 if (mAudioFirstAnchorTimeMediaUs == -1) {
400 mAudioFirstAnchorTimeMediaUs = mediaUs;
Wei Jia7b15cb32015-02-03 17:46:06 -0800401 mMediaClock->setStartingTimeMedia(mediaUs);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700402 }
403}
404
Wei Jia9a3101b2016-11-08 14:34:24 -0800405// Called on renderer looper.
406void NuPlayer::Renderer::clearAnchorTime() {
Wei Jia7b15cb32015-02-03 17:46:06 -0800407 mMediaClock->clearAnchor();
408 mAnchorTimeMediaUs = -1;
409 mAnchorNumFramesWritten = -1;
Ronghua Wua73d9e02014-10-08 15:13:29 -0700410}
411
412void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
Wei Jia7b15cb32015-02-03 17:46:06 -0800413 Mutex::Autolock autoLock(mLock);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700414 mVideoLateByUs = lateUs;
415}
416
417int64_t NuPlayer::Renderer::getVideoLateByUs() {
Wei Jia7b15cb32015-02-03 17:46:06 -0800418 Mutex::Autolock autoLock(mLock);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700419 return mVideoLateByUs;
420}
421
Andy Hung202bce12014-12-03 11:47:36 -0800422status_t NuPlayer::Renderer::openAudioSink(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700423 const sp<AMessage> &format,
424 bool offloadOnly,
425 bool hasVideo,
Andy Hung202bce12014-12-03 11:47:36 -0800426 uint32_t flags,
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +0530427 bool *isOffloaded,
428 bool isStreaming) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800429 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700430 msg->setMessage("format", format);
431 msg->setInt32("offload-only", offloadOnly);
432 msg->setInt32("has-video", hasVideo);
433 msg->setInt32("flags", flags);
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +0530434 msg->setInt32("isStreaming", isStreaming);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700435
436 sp<AMessage> response;
Leena Winterrowd66e4c4772015-09-10 15:41:08 -0700437 status_t postStatus = msg->postAndAwaitResponse(&response);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700438
Andy Hung202bce12014-12-03 11:47:36 -0800439 int32_t err;
Leena Winterrowd66e4c4772015-09-10 15:41:08 -0700440 if (postStatus != OK || response.get() == nullptr || !response->findInt32("err", &err)) {
Andy Hung202bce12014-12-03 11:47:36 -0800441 err = INVALID_OPERATION;
442 } else if (err == OK && isOffloaded != NULL) {
443 int32_t offload;
444 CHECK(response->findInt32("offload", &offload));
445 *isOffloaded = (offload != 0);
446 }
447 return err;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700448}
449
450void NuPlayer::Renderer::closeAudioSink() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800451 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700452
453 sp<AMessage> response;
454 msg->postAndAwaitResponse(&response);
455}
456
Wei Jia9a3101b2016-11-08 14:34:24 -0800457void NuPlayer::Renderer::changeAudioFormat(
458 const sp<AMessage> &format,
459 bool offloadOnly,
460 bool hasVideo,
461 uint32_t flags,
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +0530462 bool isStreaming,
Wei Jia9a3101b2016-11-08 14:34:24 -0800463 const sp<AMessage> &notify) {
464 sp<AMessage> meta = new AMessage;
465 meta->setMessage("format", format);
466 meta->setInt32("offload-only", offloadOnly);
467 meta->setInt32("has-video", hasVideo);
468 meta->setInt32("flags", flags);
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +0530469 meta->setInt32("isStreaming", isStreaming);
Wei Jia9a3101b2016-11-08 14:34:24 -0800470
471 sp<AMessage> msg = new AMessage(kWhatChangeAudioFormat, this);
472 msg->setInt32("queueGeneration", getQueueGeneration(true /* audio */));
473 msg->setMessage("notify", notify);
474 msg->setMessage("meta", meta);
475 msg->post();
476}
477
Andreas Huberf9334412010-12-15 15:17:42 -0800478void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
479 switch (msg->what()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700480 case kWhatOpenAudioSink:
481 {
482 sp<AMessage> format;
483 CHECK(msg->findMessage("format", &format));
484
485 int32_t offloadOnly;
486 CHECK(msg->findInt32("offload-only", &offloadOnly));
487
488 int32_t hasVideo;
489 CHECK(msg->findInt32("has-video", &hasVideo));
490
491 uint32_t flags;
492 CHECK(msg->findInt32("flags", (int32_t *)&flags));
493
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +0530494 uint32_t isStreaming;
495 CHECK(msg->findInt32("isStreaming", (int32_t *)&isStreaming));
496
497 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700498
499 sp<AMessage> response = new AMessage;
Andy Hung202bce12014-12-03 11:47:36 -0800500 response->setInt32("err", err);
501 response->setInt32("offload", offloadingAudio());
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700502
Lajos Molnar3f274362015-03-05 14:35:41 -0800503 sp<AReplyToken> replyID;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700504 CHECK(msg->senderAwaitsResponse(&replyID));
505 response->postReply(replyID);
506
507 break;
508 }
509
510 case kWhatCloseAudioSink:
511 {
Lajos Molnar3f274362015-03-05 14:35:41 -0800512 sp<AReplyToken> replyID;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700513 CHECK(msg->senderAwaitsResponse(&replyID));
514
515 onCloseAudioSink();
516
517 sp<AMessage> response = new AMessage;
518 response->postReply(replyID);
519 break;
520 }
521
Wei Jiabc2fb722014-07-08 16:37:57 -0700522 case kWhatStopAudioSink:
523 {
524 mAudioSink->stop();
525 break;
526 }
527
Wei Jia9a3101b2016-11-08 14:34:24 -0800528 case kWhatChangeAudioFormat:
529 {
530 int32_t queueGeneration;
531 CHECK(msg->findInt32("queueGeneration", &queueGeneration));
532
533 sp<AMessage> notify;
534 CHECK(msg->findMessage("notify", &notify));
535
536 if (offloadingAudio()) {
537 ALOGW("changeAudioFormat should NOT be called in offload mode");
538 notify->setInt32("err", INVALID_OPERATION);
539 notify->post();
540 break;
541 }
542
543 sp<AMessage> meta;
544 CHECK(msg->findMessage("meta", &meta));
545
546 if (queueGeneration != getQueueGeneration(true /* audio */)
547 || mAudioQueue.empty()) {
548 onChangeAudioFormat(meta, notify);
549 break;
550 }
551
552 QueueEntry entry;
553 entry.mNotifyConsumed = notify;
554 entry.mMeta = meta;
555
556 Mutex::Autolock autoLock(mLock);
557 mAudioQueue.push_back(entry);
558 postDrainAudioQueue_l();
559
560 break;
561 }
562
Andreas Huberf9334412010-12-15 15:17:42 -0800563 case kWhatDrainAudioQueue:
564 {
Wei Jia3ab25452015-06-10 09:37:47 -0700565 mDrainAudioQueuePending = false;
566
Andreas Huberf9334412010-12-15 15:17:42 -0800567 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800568 CHECK(msg->findInt32("drainGeneration", &generation));
569 if (generation != getDrainGeneration(true /* audio */)) {
Andreas Huberf9334412010-12-15 15:17:42 -0800570 break;
571 }
572
Andreas Huber078cfcf2011-09-15 12:25:04 -0700573 if (onDrainAudioQueue()) {
574 uint32_t numFramesPlayed;
575 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
576 (status_t)OK);
Andreas Huberf9334412010-12-15 15:17:42 -0800577
Wei Jia1fb35b72017-09-25 16:08:13 -0700578 // Handle AudioTrack race when start is immediately called after flush.
Andreas Huber078cfcf2011-09-15 12:25:04 -0700579 uint32_t numFramesPendingPlayout =
Wei Jia1fb35b72017-09-25 16:08:13 -0700580 (mNumFramesWritten > numFramesPlayed ?
581 mNumFramesWritten - numFramesPlayed : 0);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700582
583 // This is how long the audio sink will have data to
584 // play back.
585 int64_t delayUs =
586 mAudioSink->msecsPerFrame()
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -0800587 * numFramesPendingPlayout * 1000LL;
Wei Jiad9c2e9c2015-04-10 19:04:09 -0700588 if (mPlaybackRate > 1.0f) {
589 delayUs /= mPlaybackRate;
590 }
Andreas Huber078cfcf2011-09-15 12:25:04 -0700591
592 // Let's give it more data after about half that time
593 // has elapsed.
Andy Hungf2c87b32016-04-07 19:49:29 -0700594 delayUs /= 2;
595 // check the buffer size to estimate maximum delay permitted.
596 const int64_t maxDrainDelayUs = std::max(
597 mAudioSink->getBufferDurationInUs(), (int64_t)500000 /* half second */);
598 ALOGD_IF(delayUs > maxDrainDelayUs, "postDrainAudioQueue long delay: %lld > %lld",
599 (long long)delayUs, (long long)maxDrainDelayUs);
Wei Jia7b15cb32015-02-03 17:46:06 -0800600 Mutex::Autolock autoLock(mLock);
Andy Hungf2c87b32016-04-07 19:49:29 -0700601 postDrainAudioQueue_l(delayUs);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700602 }
Andreas Huberf9334412010-12-15 15:17:42 -0800603 break;
604 }
605
606 case kWhatDrainVideoQueue:
607 {
608 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800609 CHECK(msg->findInt32("drainGeneration", &generation));
610 if (generation != getDrainGeneration(false /* audio */)) {
Andreas Huberf9334412010-12-15 15:17:42 -0800611 break;
612 }
613
614 mDrainVideoQueuePending = false;
615
616 onDrainVideoQueue();
617
Wei Jia7b15cb32015-02-03 17:46:06 -0800618 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -0800619 break;
620 }
621
Lajos Molnard5923402014-10-20 17:47:54 -0700622 case kWhatPostDrainVideoQueue:
623 {
624 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800625 CHECK(msg->findInt32("drainGeneration", &generation));
626 if (generation != getDrainGeneration(false /* audio */)) {
Lajos Molnard5923402014-10-20 17:47:54 -0700627 break;
628 }
629
630 mDrainVideoQueuePending = false;
Wei Jia7b15cb32015-02-03 17:46:06 -0800631 postDrainVideoQueue();
Lajos Molnard5923402014-10-20 17:47:54 -0700632 break;
633 }
634
Andreas Huberf9334412010-12-15 15:17:42 -0800635 case kWhatQueueBuffer:
636 {
637 onQueueBuffer(msg);
638 break;
639 }
640
641 case kWhatQueueEOS:
642 {
643 onQueueEOS(msg);
644 break;
645 }
646
Wei Jia7c8d0e02015-08-27 17:40:21 -0700647 case kWhatEOS:
648 {
649 int32_t generation;
650 CHECK(msg->findInt32("audioEOSGeneration", &generation));
651 if (generation != mAudioEOSGeneration) {
652 break;
653 }
654 status_t finalResult;
655 CHECK(msg->findInt32("finalResult", &finalResult));
656 notifyEOS(true /* audio */, finalResult);
657 break;
658 }
659
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700660 case kWhatConfigPlayback:
Wei Jia98160162015-02-04 17:01:11 -0800661 {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700662 sp<AReplyToken> replyID;
663 CHECK(msg->senderAwaitsResponse(&replyID));
664 AudioPlaybackRate rate;
665 readFromAMessage(msg, &rate);
666 status_t err = onConfigPlayback(rate);
667 sp<AMessage> response = new AMessage;
668 response->setInt32("err", err);
669 response->postReply(replyID);
670 break;
671 }
672
673 case kWhatGetPlaybackSettings:
674 {
675 sp<AReplyToken> replyID;
676 CHECK(msg->senderAwaitsResponse(&replyID));
677 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
678 status_t err = onGetPlaybackSettings(&rate);
679 sp<AMessage> response = new AMessage;
680 if (err == OK) {
681 writeToAMessage(response, rate);
682 }
683 response->setInt32("err", err);
684 response->postReply(replyID);
685 break;
686 }
687
688 case kWhatConfigSync:
689 {
690 sp<AReplyToken> replyID;
691 CHECK(msg->senderAwaitsResponse(&replyID));
692 AVSyncSettings sync;
693 float videoFpsHint;
694 readFromAMessage(msg, &sync, &videoFpsHint);
695 status_t err = onConfigSync(sync, videoFpsHint);
696 sp<AMessage> response = new AMessage;
697 response->setInt32("err", err);
698 response->postReply(replyID);
699 break;
700 }
701
702 case kWhatGetSyncSettings:
703 {
704 sp<AReplyToken> replyID;
705 CHECK(msg->senderAwaitsResponse(&replyID));
706
707 ALOGV("kWhatGetSyncSettings");
708 AVSyncSettings sync;
709 float videoFps = -1.f;
710 status_t err = onGetSyncSettings(&sync, &videoFps);
711 sp<AMessage> response = new AMessage;
712 if (err == OK) {
713 writeToAMessage(response, sync, videoFps);
714 }
715 response->setInt32("err", err);
716 response->postReply(replyID);
Wei Jia98160162015-02-04 17:01:11 -0800717 break;
718 }
719
Andreas Huberf9334412010-12-15 15:17:42 -0800720 case kWhatFlush:
721 {
722 onFlush(msg);
723 break;
724 }
725
Wei Jiabc2fb722014-07-08 16:37:57 -0700726 case kWhatDisableOffloadAudio:
727 {
728 onDisableOffloadAudio();
729 break;
730 }
731
Ronghua Wua10fd232014-11-06 16:15:20 -0800732 case kWhatEnableOffloadAudio:
733 {
734 onEnableOffloadAudio();
735 break;
736 }
737
Andreas Huberb4082222011-01-20 15:23:04 -0800738 case kWhatPause:
739 {
740 onPause();
741 break;
742 }
743
744 case kWhatResume:
745 {
746 onResume();
747 break;
748 }
749
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700750 case kWhatSetVideoFrameRate:
751 {
752 float fps;
753 CHECK(msg->findFloat("frame-rate", &fps));
754 onSetVideoFrameRate(fps);
755 break;
756 }
757
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700758 case kWhatAudioTearDown:
Wei Jia3a2956d2014-07-22 16:01:33 -0700759 {
Wei Jiaa05f1e32016-03-25 16:31:22 -0700760 int32_t reason;
761 CHECK(msg->findInt32("reason", &reason));
762
763 onAudioTearDown((AudioTearDownReason)reason);
Wei Jia3a2956d2014-07-22 16:01:33 -0700764 break;
765 }
766
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700767 case kWhatAudioOffloadPauseTimeout:
768 {
769 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800770 CHECK(msg->findInt32("drainGeneration", &generation));
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700771 if (generation != mAudioOffloadPauseTimeoutGeneration) {
772 break;
773 }
Ronghua Wu08529172014-10-02 16:55:52 -0700774 ALOGV("Audio Offload tear down due to pause timeout.");
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700775 onAudioTearDown(kDueToTimeout);
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800776 mWakeLock->release();
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700777 break;
778 }
779
Andreas Huberf9334412010-12-15 15:17:42 -0800780 default:
781 TRESPASS();
782 break;
783 }
784}
785
Wei Jiabc2fb722014-07-08 16:37:57 -0700786void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
Andy Hung005e9d02015-05-31 23:16:07 -0700787 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
Andreas Huberf9334412010-12-15 15:17:42 -0800788 return;
789 }
790
791 if (mAudioQueue.empty()) {
792 return;
793 }
794
Andy Hungb03dcb32015-08-27 16:18:59 -0700795 // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
796 if (mPaused) {
797 const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
798 if (diffUs > delayUs) {
799 delayUs = diffUs;
800 }
801 }
802
Andreas Huberf9334412010-12-15 15:17:42 -0800803 mDrainAudioQueuePending = true;
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800804 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800805 msg->setInt32("drainGeneration", mAudioDrainGeneration);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700806 msg->post(delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800807}
808
Wei Jia7b15cb32015-02-03 17:46:06 -0800809void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
810 mAudioRenderingStartGeneration = mAudioDrainGeneration;
811 mVideoRenderingStartGeneration = mVideoDrainGeneration;
Andy Hungb12ea0b2015-09-03 15:13:01 -0700812 mRenderingDataDelivered = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700813}
814
Wei Jia7b15cb32015-02-03 17:46:06 -0800815void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
816 if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
817 mAudioRenderingStartGeneration == mAudioDrainGeneration) {
Andy Hungb12ea0b2015-09-03 15:13:01 -0700818 mRenderingDataDelivered = true;
819 if (mPaused) {
820 return;
821 }
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700822 mVideoRenderingStartGeneration = -1;
823 mAudioRenderingStartGeneration = -1;
824
825 sp<AMessage> notify = mNotify->dup();
826 notify->setInt32("what", kWhatMediaRenderingStart);
827 notify->post();
828 }
829}
830
Wei Jiabc2fb722014-07-08 16:37:57 -0700831// static
832size_t NuPlayer::Renderer::AudioSinkCallback(
833 MediaPlayerBase::AudioSink * /* audioSink */,
834 void *buffer,
835 size_t size,
836 void *cookie,
837 MediaPlayerBase::AudioSink::cb_event_t event) {
838 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
839
840 switch (event) {
841 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
842 {
843 return me->fillAudioBuffer(buffer, size);
844 break;
845 }
846
847 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
848 {
Andy Hunga0b39712015-05-31 22:40:49 -0700849 ALOGV("AudioSink::CB_EVENT_STREAM_END");
Wei Jia4c74fde2016-08-26 16:00:52 -0700850 me->notifyEOSCallback();
Wei Jiabc2fb722014-07-08 16:37:57 -0700851 break;
852 }
853
854 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
855 {
Andy Hunga0b39712015-05-31 22:40:49 -0700856 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
Wei Jiaa05f1e32016-03-25 16:31:22 -0700857 me->notifyAudioTearDown(kDueToError);
Wei Jiabc2fb722014-07-08 16:37:57 -0700858 break;
859 }
860 }
861
862 return 0;
863}
864
Wei Jia4c74fde2016-08-26 16:00:52 -0700865void NuPlayer::Renderer::notifyEOSCallback() {
866 Mutex::Autolock autoLock(mLock);
867
868 if (!mUseAudioCallback) {
869 return;
870 }
871
Wei Jiad214a562017-07-10 13:20:55 -0700872 notifyEOS_l(true /* audio */, ERROR_END_OF_STREAM);
Wei Jia4c74fde2016-08-26 16:00:52 -0700873}
874
Wei Jiabc2fb722014-07-08 16:37:57 -0700875size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
876 Mutex::Autolock autoLock(mLock);
877
Andy Hung85e48142015-05-31 23:04:15 -0700878 if (!mUseAudioCallback) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700879 return 0;
880 }
881
882 bool hasEOS = false;
883
884 size_t sizeCopied = 0;
Ronghua Wu3e5efb32014-08-18 16:27:08 -0700885 bool firstEntry = true;
Andy Hunga0b39712015-05-31 22:40:49 -0700886 QueueEntry *entry; // will be valid after while loop if hasEOS is set.
Wei Jiabc2fb722014-07-08 16:37:57 -0700887 while (sizeCopied < size && !mAudioQueue.empty()) {
Andy Hunga0b39712015-05-31 22:40:49 -0700888 entry = &*mAudioQueue.begin();
Wei Jiabc2fb722014-07-08 16:37:57 -0700889
890 if (entry->mBuffer == NULL) { // EOS
891 hasEOS = true;
892 mAudioQueue.erase(mAudioQueue.begin());
Wei Jiabc2fb722014-07-08 16:37:57 -0700893 break;
894 }
895
Ronghua Wu3e5efb32014-08-18 16:27:08 -0700896 if (firstEntry && entry->mOffset == 0) {
897 firstEntry = false;
Wei Jiabc2fb722014-07-08 16:37:57 -0700898 int64_t mediaTimeUs;
899 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andy Hunga0b39712015-05-31 22:40:49 -0700900 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
Wei Jia7b15cb32015-02-03 17:46:06 -0800901 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
Wei Jiabc2fb722014-07-08 16:37:57 -0700902 }
903
904 size_t copy = entry->mBuffer->size() - entry->mOffset;
905 size_t sizeRemaining = size - sizeCopied;
906 if (copy > sizeRemaining) {
907 copy = sizeRemaining;
908 }
909
910 memcpy((char *)buffer + sizeCopied,
911 entry->mBuffer->data() + entry->mOffset,
912 copy);
913
914 entry->mOffset += copy;
915 if (entry->mOffset == entry->mBuffer->size()) {
916 entry->mNotifyConsumed->post();
917 mAudioQueue.erase(mAudioQueue.begin());
918 entry = NULL;
919 }
920 sizeCopied += copy;
Wei Jia7b15cb32015-02-03 17:46:06 -0800921
922 notifyIfMediaRenderingStarted_l();
Wei Jiabc2fb722014-07-08 16:37:57 -0700923 }
924
Lajos Molnarf5926712014-10-21 09:36:43 -0700925 if (mAudioFirstAnchorTimeMediaUs >= 0) {
926 int64_t nowUs = ALooper::GetNowUs();
Wei Jia98160162015-02-04 17:01:11 -0800927 int64_t nowMediaUs =
Wei Jiac4ac8172015-10-21 10:35:48 -0700928 mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs);
Wei Jia7b15cb32015-02-03 17:46:06 -0800929 // we don't know how much data we are queueing for offloaded tracks.
Wei Jia98160162015-02-04 17:01:11 -0800930 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
Lajos Molnarf5926712014-10-21 09:36:43 -0700931 }
932
Andy Hunga0b39712015-05-31 22:40:49 -0700933 // for non-offloaded audio, we need to compute the frames written because
934 // there is no EVENT_STREAM_END notification. The frames written gives
935 // an estimate on the pending played out duration.
936 if (!offloadingAudio()) {
937 mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
Wei Jiabc2fb722014-07-08 16:37:57 -0700938 }
939
Andy Hunga0b39712015-05-31 22:40:49 -0700940 if (hasEOS) {
941 (new AMessage(kWhatStopAudioSink, this))->post();
942 // As there is currently no EVENT_STREAM_END callback notification for
943 // non-offloaded audio tracks, we need to post the EOS ourselves.
944 if (!offloadingAudio()) {
945 int64_t postEOSDelayUs = 0;
946 if (mAudioSink->needsTrailingPadding()) {
947 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
948 }
Wei Jiad214a562017-07-10 13:20:55 -0700949 ALOGV("fillAudioBuffer: notifyEOS_l "
Andy Hunga0b39712015-05-31 22:40:49 -0700950 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld",
951 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
Wei Jiad214a562017-07-10 13:20:55 -0700952 notifyEOS_l(true /* audio */, entry->mFinalResult, postEOSDelayUs);
Andy Hunga0b39712015-05-31 22:40:49 -0700953 }
954 }
Wei Jiabc2fb722014-07-08 16:37:57 -0700955 return sizeCopied;
956}
957
Chong Zhang9da0ce42015-06-11 17:18:58 -0700958void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
959 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
960 bool foundEOS = false;
961 while (it != mAudioQueue.end()) {
962 int32_t eos;
963 QueueEntry *entry = &*it++;
Wei Jia9a3101b2016-11-08 14:34:24 -0800964 if ((entry->mBuffer == nullptr && entry->mNotifyConsumed == nullptr)
Chong Zhang9da0ce42015-06-11 17:18:58 -0700965 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
966 itEOS = it;
967 foundEOS = true;
968 }
969 }
970
971 if (foundEOS) {
972 // post all replies before EOS and drop the samples
973 for (it = mAudioQueue.begin(); it != itEOS; it++) {
Wei Jia9a3101b2016-11-08 14:34:24 -0800974 if (it->mBuffer == nullptr) {
975 if (it->mNotifyConsumed == nullptr) {
976 // delay doesn't matter as we don't even have an AudioTrack
977 notifyEOS(true /* audio */, it->mFinalResult);
978 } else {
979 // TAG for re-opening audio sink.
980 onChangeAudioFormat(it->mMeta, it->mNotifyConsumed);
981 }
Chong Zhang9da0ce42015-06-11 17:18:58 -0700982 } else {
983 it->mNotifyConsumed->post();
984 }
985 }
986 mAudioQueue.erase(mAudioQueue.begin(), itEOS);
987 }
988}
989
Andreas Huber078cfcf2011-09-15 12:25:04 -0700990bool NuPlayer::Renderer::onDrainAudioQueue() {
Andy Hung58d315c2015-09-11 18:44:56 -0700991 // do not drain audio during teardown as queued buffers may be invalid.
992 if (mAudioTornDown) {
993 return false;
994 }
Wei Jia230b1882015-03-19 15:13:51 -0700995 // TODO: This call to getPosition checks if AudioTrack has been created
996 // in AudioSink before draining audio. If AudioTrack doesn't exist, then
997 // CHECKs on getPosition will fail.
998 // We still need to figure out why AudioTrack is not created when
999 // this function is called. One possible reason could be leftover
1000 // audio. Another possible place is to check whether decoder
1001 // has received INFO_FORMAT_CHANGED as the first buffer since
1002 // AudioSink is opened there, and possible interactions with flush
1003 // immediately after start. Investigate error message
1004 // "vorbis_dsp_synthesis returned -135", along with RTSP.
Andreas Huber078cfcf2011-09-15 12:25:04 -07001005 uint32_t numFramesPlayed;
Andreas Huber2bfdd422011-10-11 15:24:07 -07001006 if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
Chong Zhang9da0ce42015-06-11 17:18:58 -07001007 // When getPosition fails, renderer will not reschedule the draining
1008 // unless new samples are queued.
1009 // If we have pending EOS (or "eos" marker for discontinuities), we need
1010 // to post these now as NuPlayerDecoder might be waiting for it.
1011 drainAudioQueueUntilLastEOS();
1012
1013 ALOGW("onDrainAudioQueue(): audio sink is not ready");
Andreas Huber2bfdd422011-10-11 15:24:07 -07001014 return false;
1015 }
Andreas Huberc92fd242011-08-16 13:48:44 -07001016
Wei Jia230b1882015-03-19 15:13:51 -07001017#if 0
Andreas Huber078cfcf2011-09-15 12:25:04 -07001018 ssize_t numFramesAvailableToWrite =
1019 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
1020
Andreas Huber078cfcf2011-09-15 12:25:04 -07001021 if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
Steve Blockdf64d152012-01-04 20:05:49 +00001022 ALOGI("audio sink underrun");
Andreas Huber078cfcf2011-09-15 12:25:04 -07001023 } else {
Steve Block3856b092011-10-20 11:56:00 +01001024 ALOGV("audio queue has %d frames left to play",
Andreas Huber078cfcf2011-09-15 12:25:04 -07001025 mAudioSink->frameCount() - numFramesAvailableToWrite);
1026 }
1027#endif
1028
Andy Hung005e9d02015-05-31 23:16:07 -07001029 uint32_t prevFramesWritten = mNumFramesWritten;
Wei Jia7d3f4df2015-03-03 15:28:00 -08001030 while (!mAudioQueue.empty()) {
Andreas Huberc92fd242011-08-16 13:48:44 -07001031 QueueEntry *entry = &*mAudioQueue.begin();
1032
1033 if (entry->mBuffer == NULL) {
Wei Jia9a3101b2016-11-08 14:34:24 -08001034 if (entry->mNotifyConsumed != nullptr) {
1035 // TAG for re-open audio sink.
1036 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed);
1037 mAudioQueue.erase(mAudioQueue.begin());
1038 continue;
1039 }
1040
Andreas Huberc92fd242011-08-16 13:48:44 -07001041 // EOS
Haynes Mathew George62d4e072016-06-13 11:23:22 -07001042 if (mPaused) {
1043 // Do not notify EOS when paused.
1044 // This is needed to avoid switch to next clip while in pause.
1045 ALOGV("onDrainAudioQueue(): Do not notify EOS when paused");
1046 return false;
1047 }
1048
Ronghua Wu5095d702014-08-27 12:05:48 -07001049 int64_t postEOSDelayUs = 0;
1050 if (mAudioSink->needsTrailingPadding()) {
Lajos Molnar06ad1522014-08-28 07:27:44 -07001051 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
Ronghua Wu5095d702014-08-27 12:05:48 -07001052 }
1053 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
Wei Jia2995dc72015-07-24 16:00:33 -07001054 mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
Andreas Huberc92fd242011-08-16 13:48:44 -07001055
1056 mAudioQueue.erase(mAudioQueue.begin());
1057 entry = NULL;
Andy Hung34912322014-12-18 18:48:39 -08001058 if (mAudioSink->needsTrailingPadding()) {
1059 // If we're not in gapless playback (i.e. through setNextPlayer), we
1060 // need to stop the track here, because that will play out the last
1061 // little bit at the end of the file. Otherwise short files won't play.
1062 mAudioSink->stop();
1063 mNumFramesWritten = 0;
1064 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001065 return false;
Eric Laurent9b7d9502011-03-21 11:49:00 -07001066 }
1067
Wei Jia9a3101b2016-11-08 14:34:24 -08001068 mLastAudioBufferDrained = entry->mBufferOrdinal;
1069
Chong Zhang81636762015-04-21 09:33:47 -07001070 // ignore 0-sized buffer which could be EOS marker with no data
1071 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
Andreas Huberf9334412010-12-15 15:17:42 -08001072 int64_t mediaTimeUs;
1073 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andy Hunga0b39712015-05-31 22:40:49 -07001074 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
1075 mediaTimeUs / 1E6);
Ronghua Wueecb7802014-10-19 23:12:50 -07001076 onNewAudioMediaTime(mediaTimeUs);
Andreas Huberf9334412010-12-15 15:17:42 -08001077 }
1078
1079 size_t copy = entry->mBuffer->size() - entry->mOffset;
Andreas Huberf9334412010-12-15 15:17:42 -08001080
Wei Jia7d3f4df2015-03-03 15:28:00 -08001081 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
1082 copy, false /* blocking */);
Andy Hunga31335a2014-08-20 17:37:59 -07001083 if (written < 0) {
Andy Hung202bce12014-12-03 11:47:36 -08001084 // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
Wei Jia7d3f4df2015-03-03 15:28:00 -08001085 if (written == WOULD_BLOCK) {
Wei Jiadf809472015-06-09 11:21:13 -07001086 ALOGV("AudioSink write would block when writing %zu bytes", copy);
Wei Jia7d3f4df2015-03-03 15:28:00 -08001087 } else {
1088 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
Ronghua Wu7665f582015-09-02 10:15:30 -07001089 // This can only happen when AudioSink was opened with doNotReconnect flag set to
1090 // true, in which case the NuPlayer will handle the reconnect.
Wei Jiaa05f1e32016-03-25 16:31:22 -07001091 notifyAudioTearDown(kDueToError);
Wei Jia7d3f4df2015-03-03 15:28:00 -08001092 }
Andy Hung202bce12014-12-03 11:47:36 -08001093 break;
Andy Hunga31335a2014-08-20 17:37:59 -07001094 }
Andreas Huberf9334412010-12-15 15:17:42 -08001095
Andy Hunga31335a2014-08-20 17:37:59 -07001096 entry->mOffset += written;
Wei Jia03cee242016-05-26 15:22:59 -07001097 size_t remainder = entry->mBuffer->size() - entry->mOffset;
1098 if ((ssize_t)remainder < mAudioSink->frameSize()) {
1099 if (remainder > 0) {
1100 ALOGW("Corrupted audio buffer has fractional frames, discarding %zu bytes.",
1101 remainder);
1102 entry->mOffset += remainder;
1103 copy -= remainder;
1104 }
1105
Andreas Huberf9334412010-12-15 15:17:42 -08001106 entry->mNotifyConsumed->post();
1107 mAudioQueue.erase(mAudioQueue.begin());
Eric Laurent9b7d9502011-03-21 11:49:00 -07001108
Andreas Huberf9334412010-12-15 15:17:42 -08001109 entry = NULL;
1110 }
1111
Andy Hunga31335a2014-08-20 17:37:59 -07001112 size_t copiedFrames = written / mAudioSink->frameSize();
Andreas Huber078cfcf2011-09-15 12:25:04 -07001113 mNumFramesWritten += copiedFrames;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001114
Wei Jia7b15cb32015-02-03 17:46:06 -08001115 {
1116 Mutex::Autolock autoLock(mLock);
Wei Jiad005c5d2015-10-13 10:58:07 -07001117 int64_t maxTimeMedia;
1118 maxTimeMedia =
1119 mAnchorTimeMediaUs +
1120 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
1121 * 1000LL * mAudioSink->msecsPerFrame());
1122 mMediaClock->updateMaxTimeMedia(maxTimeMedia);
1123
Wei Jia7b15cb32015-02-03 17:46:06 -08001124 notifyIfMediaRenderingStarted_l();
1125 }
Andreas Huber43c3e6c2011-01-05 12:17:08 -08001126
Andy Hunga31335a2014-08-20 17:37:59 -07001127 if (written != (ssize_t)copy) {
1128 // A short count was received from AudioSink::write()
1129 //
Wei Jia7d3f4df2015-03-03 15:28:00 -08001130 // AudioSink write is called in non-blocking mode.
1131 // It may return with a short count when:
Andy Hunga31335a2014-08-20 17:37:59 -07001132 //
Wei Jia03cee242016-05-26 15:22:59 -07001133 // 1) Size to be copied is not a multiple of the frame size. Fractional frames are
1134 // discarded.
Wei Jia7d3f4df2015-03-03 15:28:00 -08001135 // 2) The data to be copied exceeds the available buffer in AudioSink.
1136 // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
1137 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
Andy Hunga31335a2014-08-20 17:37:59 -07001138
1139 // (Case 1)
1140 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it
1141 // needs to fail, as we should not carry over fractional frames between calls.
Colin Cross0bdcf662017-04-27 16:15:51 -07001142 CHECK_EQ(copy % mAudioSink->frameSize(), 0u);
Andy Hunga31335a2014-08-20 17:37:59 -07001143
Wei Jia7d3f4df2015-03-03 15:28:00 -08001144 // (Case 2, 3, 4)
Andy Hunga31335a2014-08-20 17:37:59 -07001145 // Return early to the caller.
1146 // Beware of calling immediately again as this may busy-loop if you are not careful.
Wei Jia7d3f4df2015-03-03 15:28:00 -08001147 ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
Andy Hunga31335a2014-08-20 17:37:59 -07001148 break;
1149 }
1150 }
Lajos Molnarf5926712014-10-21 09:36:43 -07001151
Andy Hung005e9d02015-05-31 23:16:07 -07001152 // calculate whether we need to reschedule another write.
1153 bool reschedule = !mAudioQueue.empty()
1154 && (!mPaused
1155 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
1156 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u",
1157 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
1158 return reschedule;
Andreas Huberf9334412010-12-15 15:17:42 -08001159}
1160
Wei Jia98160162015-02-04 17:01:11 -08001161int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
1162 int32_t sampleRate = offloadingAudio() ?
1163 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
Wei Jia4d7ac852015-08-31 18:33:14 -07001164 if (sampleRate == 0) {
1165 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
1166 return 0;
1167 }
hyoungjun.byun2c2050c2019-01-31 12:11:38 +09001168
1169 return (int64_t)(numFrames * 1000000LL / sampleRate);
Wei Jia98160162015-02-04 17:01:11 -08001170}
1171
1172// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
Lajos Molnar06ad1522014-08-28 07:27:44 -07001173int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
Wei Jia98160162015-02-04 17:01:11 -08001174 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
Wei Jiad2f35de2016-02-17 16:48:06 -08001175 if (mUseVirtualAudioSink) {
1176 int64_t nowUs = ALooper::GetNowUs();
1177 int64_t mediaUs;
1178 if (mMediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -08001179 return 0LL;
Wei Jiad2f35de2016-02-17 16:48:06 -08001180 } else {
1181 return writtenAudioDurationUs - (mediaUs - mAudioFirstAnchorTimeMediaUs);
1182 }
1183 }
Andy Hungffa36952017-08-17 10:41:51 -07001184
1185 const int64_t audioSinkPlayedUs = mAudioSink->getPlayedOutDurationUs(nowUs);
1186 int64_t pendingUs = writtenAudioDurationUs - audioSinkPlayedUs;
1187 if (pendingUs < 0) {
1188 // This shouldn't happen unless the timestamp is stale.
1189 ALOGW("%s: pendingUs %lld < 0, clamping to zero, potential resume after pause "
1190 "writtenAudioDurationUs: %lld, audioSinkPlayedUs: %lld",
1191 __func__, (long long)pendingUs,
1192 (long long)writtenAudioDurationUs, (long long)audioSinkPlayedUs);
1193 pendingUs = 0;
1194 }
1195 return pendingUs;
Ronghua Wu5095d702014-08-27 12:05:48 -07001196}
1197
Ronghua Wua73d9e02014-10-08 15:13:29 -07001198int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
Wei Jia98160162015-02-04 17:01:11 -08001199 int64_t realUs;
1200 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001201 // If failed to get current position, e.g. due to audio clock is
1202 // not ready, then just play out video immediately without delay.
Ronghua Wueecb7802014-10-19 23:12:50 -07001203 return nowUs;
Ronghua Wua73d9e02014-10-08 15:13:29 -07001204 }
Wei Jia98160162015-02-04 17:01:11 -08001205 return realUs;
Ronghua Wua73d9e02014-10-08 15:13:29 -07001206}
1207
Ronghua Wueecb7802014-10-19 23:12:50 -07001208void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001209 Mutex::Autolock autoLock(mLock);
Ronghua Wueecb7802014-10-19 23:12:50 -07001210 // TRICKY: vorbis decoder generates multiple frames with the same
1211 // timestamp, so only update on the first frame with a given timestamp
1212 if (mediaTimeUs == mAnchorTimeMediaUs) {
1213 return;
1214 }
Wei Jia7b15cb32015-02-03 17:46:06 -08001215 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
Wei Jia4ecbea32015-10-20 18:09:57 -07001216
Andy Hung528c8402016-01-12 12:39:34 -08001217 // mNextAudioClockUpdateTimeUs is -1 if we're waiting for audio sink to start
1218 if (mNextAudioClockUpdateTimeUs == -1) {
1219 AudioTimestamp ts;
1220 if (mAudioSink->getTimestamp(ts) == OK && ts.mPosition > 0) {
1221 mNextAudioClockUpdateTimeUs = 0; // start our clock updates
1222 }
1223 }
Wei Jia12b62652016-02-09 17:49:31 -08001224 int64_t nowUs = ALooper::GetNowUs();
Andy Hung528c8402016-01-12 12:39:34 -08001225 if (mNextAudioClockUpdateTimeUs >= 0) {
1226 if (nowUs >= mNextAudioClockUpdateTimeUs) {
1227 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1228 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
Wei Jiad2f35de2016-02-17 16:48:06 -08001229 mUseVirtualAudioSink = false;
Andy Hung528c8402016-01-12 12:39:34 -08001230 mNextAudioClockUpdateTimeUs = nowUs + kMinimumAudioClockUpdatePeriodUs;
1231 }
Wei Jia12b62652016-02-09 17:49:31 -08001232 } else {
1233 int64_t unused;
1234 if ((mMediaClock->getMediaTime(nowUs, &unused) != OK)
1235 && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten)
1236 > kMaxAllowedAudioSinkDelayUs)) {
1237 // Enough data has been sent to AudioSink, but AudioSink has not rendered
1238 // any data yet. Something is wrong with AudioSink, e.g., the device is not
1239 // connected to audio out.
1240 // Switch to system clock. This essentially creates a virtual AudioSink with
1241 // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten).
1242 // This virtual AudioSink renders audio data starting from the very first sample
1243 // and it's paced by system clock.
Wei Jia07af6512016-02-10 10:45:47 -08001244 ALOGW("AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock.");
Wei Jia12b62652016-02-09 17:49:31 -08001245 mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs);
Wei Jiad2f35de2016-02-17 16:48:06 -08001246 mUseVirtualAudioSink = true;
Wei Jia12b62652016-02-09 17:49:31 -08001247 }
Wei Jia4ecbea32015-10-20 18:09:57 -07001248 }
Wei Jia8edb3f82015-03-11 11:07:45 -07001249 mAnchorNumFramesWritten = mNumFramesWritten;
Wei Jia7b15cb32015-02-03 17:46:06 -08001250 mAnchorTimeMediaUs = mediaTimeUs;
Ronghua Wueecb7802014-10-19 23:12:50 -07001251}
1252
Wei Jia7b15cb32015-02-03 17:46:06 -08001253// Called without mLock acquired.
1254void NuPlayer::Renderer::postDrainVideoQueue() {
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001255 if (mDrainVideoQueuePending
Wei Jia7b15cb32015-02-03 17:46:06 -08001256 || getSyncQueues()
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001257 || (mPaused && mVideoSampleReceived)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001258 return;
1259 }
1260
1261 if (mVideoQueue.empty()) {
1262 return;
1263 }
1264
1265 QueueEntry &entry = *mVideoQueue.begin();
1266
Lajos Molnar1d15ab52015-03-04 16:46:34 -08001267 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
Wei Jia7b15cb32015-02-03 17:46:06 -08001268 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
Andreas Huberf9334412010-12-15 15:17:42 -08001269
Andreas Huberf9334412010-12-15 15:17:42 -08001270 if (entry.mBuffer == NULL) {
1271 // EOS doesn't carry a timestamp.
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001272 msg->post();
1273 mDrainVideoQueuePending = true;
1274 return;
1275 }
1276
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001277 int64_t nowUs = ALooper::GetNowUs();
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001278 if (mFlags & FLAG_REAL_TIME) {
Wei Jia17648f32017-09-19 13:16:30 -07001279 int64_t realTimeUs;
1280 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &realTimeUs));
Andreas Huberf9334412010-12-15 15:17:42 -08001281
Wei Jia17648f32017-09-19 13:16:30 -07001282 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
Lajos Molnard5923402014-10-20 17:47:54 -07001283
Wei Jia17648f32017-09-19 13:16:30 -07001284 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
Robert Shih00541e22015-12-18 17:04:00 -08001285
Wei Jia17648f32017-09-19 13:16:30 -07001286 int64_t delayUs = realTimeUs - nowUs;
1287
1288 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %lld", (long long)delayUs);
1289 // post 2 display refreshes before rendering is due
1290 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1291
1292 mDrainVideoQueuePending = true;
1293 return;
Andreas Huberf9334412010-12-15 15:17:42 -08001294 }
1295
Wei Jia17648f32017-09-19 13:16:30 -07001296 int64_t mediaTimeUs;
1297 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001298
Wei Jia17648f32017-09-19 13:16:30 -07001299 {
1300 Mutex::Autolock autoLock(mLock);
1301 if (mAnchorTimeMediaUs < 0) {
1302 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1303 mAnchorTimeMediaUs = mediaTimeUs;
1304 }
1305 }
Wei Jia1225b932019-01-04 12:44:38 -08001306 mNextVideoTimeMediaUs = mediaTimeUs;
Wei Jia17648f32017-09-19 13:16:30 -07001307 if (!mHasAudio) {
1308 // smooth out videos >= 10fps
Wei Jiacc951512019-04-03 16:10:42 -07001309 mMediaClock->updateMaxTimeMedia(mediaTimeUs + kDefaultVideoFrameIntervalUs);
Wei Jia17648f32017-09-19 13:16:30 -07001310 }
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001311
Wei Jia17648f32017-09-19 13:16:30 -07001312 if (!mVideoSampleReceived || mediaTimeUs < mAudioFirstAnchorTimeMediaUs) {
1313 msg->post();
1314 } else {
1315 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1316
1317 // post 2 display refreshes before rendering is due
1318 mMediaClock->addTimer(msg, mediaTimeUs, -twoVsyncsUs);
1319 }
Andreas Huberf9334412010-12-15 15:17:42 -08001320
1321 mDrainVideoQueuePending = true;
1322}
1323
1324void NuPlayer::Renderer::onDrainVideoQueue() {
1325 if (mVideoQueue.empty()) {
1326 return;
1327 }
1328
1329 QueueEntry *entry = &*mVideoQueue.begin();
1330
1331 if (entry->mBuffer == NULL) {
1332 // EOS
1333
Andreas Huberc92fd242011-08-16 13:48:44 -07001334 notifyEOS(false /* audio */, entry->mFinalResult);
Andreas Huberf9334412010-12-15 15:17:42 -08001335
1336 mVideoQueue.erase(mVideoQueue.begin());
1337 entry = NULL;
Andreas Huber3fe62152011-09-16 15:09:22 -07001338
Ronghua Wua73d9e02014-10-08 15:13:29 -07001339 setVideoLateByUs(0);
Andreas Huberf9334412010-12-15 15:17:42 -08001340 return;
1341 }
1342
Wei Jia25d696f2015-11-19 10:57:15 -08001343 int64_t nowUs = ALooper::GetNowUs();
Andreas Huberd5e56232013-03-12 11:01:43 -07001344 int64_t realTimeUs;
Wei Jia2995dc72015-07-24 16:00:33 -07001345 int64_t mediaTimeUs = -1;
Andreas Huberd5e56232013-03-12 11:01:43 -07001346 if (mFlags & FLAG_REAL_TIME) {
1347 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1348 } else {
Andreas Huberd5e56232013-03-12 11:01:43 -07001349 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andreas Huberf9334412010-12-15 15:17:42 -08001350
Ronghua Wua73d9e02014-10-08 15:13:29 -07001351 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
Andreas Huberd5e56232013-03-12 11:01:43 -07001352 }
Wei Jia17648f32017-09-19 13:16:30 -07001353 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
Andreas Huberd5e56232013-03-12 11:01:43 -07001354
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001355 bool tooLate = false;
Andreas Huber3fe62152011-09-16 15:09:22 -07001356
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001357 if (!mPaused) {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001358 setVideoLateByUs(nowUs - realTimeUs);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001359 tooLate = (mVideoLateByUs > 40000);
1360
1361 if (tooLate) {
1362 ALOGV("video late by %lld us (%.2f secs)",
Lajos Molnar6d339f12015-04-17 16:15:53 -07001363 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001364 } else {
Wei Jia98160162015-02-04 17:01:11 -08001365 int64_t mediaUs = 0;
1366 mMediaClock->getMediaTime(realTimeUs, &mediaUs);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001367 ALOGV("rendering video at media time %.2f secs",
1368 (mFlags & FLAG_REAL_TIME ? realTimeUs :
Wei Jia98160162015-02-04 17:01:11 -08001369 mediaUs) / 1E6);
Wei Jia2995dc72015-07-24 16:00:33 -07001370
1371 if (!(mFlags & FLAG_REAL_TIME)
1372 && mLastAudioMediaTimeUs != -1
1373 && mediaTimeUs > mLastAudioMediaTimeUs) {
1374 // If audio ends before video, video continues to drive media clock.
1375 // Also smooth out videos >= 10fps.
Wei Jiacc951512019-04-03 16:10:42 -07001376 mMediaClock->updateMaxTimeMedia(mediaTimeUs + kDefaultVideoFrameIntervalUs);
Wei Jia2995dc72015-07-24 16:00:33 -07001377 }
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001378 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001379 } else {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001380 setVideoLateByUs(0);
Ronghua Wueecb7802014-10-19 23:12:50 -07001381 if (!mVideoSampleReceived && !mHasAudio) {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001382 // This will ensure that the first frame after a flush won't be used as anchor
1383 // when renderer is in paused state, because resume can happen any time after seek.
Wei Jia9a3101b2016-11-08 14:34:24 -08001384 clearAnchorTime();
Wei Jia49966ff2014-10-08 18:44:45 -07001385 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001386 }
Andreas Huberf9334412010-12-15 15:17:42 -08001387
Wei Jia25d696f2015-11-19 10:57:15 -08001388 // Always render the first video frame while keeping stats on A/V sync.
1389 if (!mVideoSampleReceived) {
1390 realTimeUs = nowUs;
1391 tooLate = false;
1392 }
1393
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -08001394 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000LL);
Glenn Kasten683525b2011-11-04 18:05:35 -07001395 entry->mNotifyConsumed->setInt32("render", !tooLate);
Andreas Huberf9334412010-12-15 15:17:42 -08001396 entry->mNotifyConsumed->post();
1397 mVideoQueue.erase(mVideoQueue.begin());
1398 entry = NULL;
Andreas Huber43c3e6c2011-01-05 12:17:08 -08001399
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001400 mVideoSampleReceived = true;
James Dongf57b4ea2012-07-20 13:38:36 -07001401
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001402 if (!mPaused) {
1403 if (!mVideoRenderingStarted) {
1404 mVideoRenderingStarted = true;
1405 notifyVideoRenderingStart();
1406 }
Wei Jia7b15cb32015-02-03 17:46:06 -08001407 Mutex::Autolock autoLock(mLock);
1408 notifyIfMediaRenderingStarted_l();
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001409 }
Andreas Huberf9334412010-12-15 15:17:42 -08001410}
1411
James Dongf57b4ea2012-07-20 13:38:36 -07001412void NuPlayer::Renderer::notifyVideoRenderingStart() {
1413 sp<AMessage> notify = mNotify->dup();
1414 notify->setInt32("what", kWhatVideoRenderingStart);
1415 notify->post();
1416}
1417
Ronghua Wu5095d702014-08-27 12:05:48 -07001418void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
Wei Jiad214a562017-07-10 13:20:55 -07001419 Mutex::Autolock autoLock(mLock);
1420 notifyEOS_l(audio, finalResult, delayUs);
1421}
1422
1423void NuPlayer::Renderer::notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs) {
Wei Jia7c8d0e02015-08-27 17:40:21 -07001424 if (audio && delayUs > 0) {
1425 sp<AMessage> msg = new AMessage(kWhatEOS, this);
1426 msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1427 msg->setInt32("finalResult", finalResult);
1428 msg->post(delayUs);
1429 return;
1430 }
Andreas Huberf9334412010-12-15 15:17:42 -08001431 sp<AMessage> notify = mNotify->dup();
1432 notify->setInt32("what", kWhatEOS);
1433 notify->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huberc92fd242011-08-16 13:48:44 -07001434 notify->setInt32("finalResult", finalResult);
Ronghua Wu5095d702014-08-27 12:05:48 -07001435 notify->post(delayUs);
Wei Jiad214a562017-07-10 13:20:55 -07001436
1437 if (audio) {
1438 // Video might outlive audio. Clear anchor to enable video only case.
1439 mAnchorTimeMediaUs = -1;
Wei Jiaaec8d822017-08-25 15:27:57 -07001440 mHasAudio = false;
Wei Jiadc293e32017-10-16 18:29:37 -07001441 if (mNextVideoTimeMediaUs >= 0) {
1442 int64_t mediaUs = 0;
Wei Jia1225b932019-01-04 12:44:38 -08001443 int64_t nowUs = ALooper::GetNowUs();
1444 status_t result = mMediaClock->getMediaTime(nowUs, &mediaUs);
1445 if (result == OK) {
1446 if (mNextVideoTimeMediaUs > mediaUs) {
1447 mMediaClock->updateMaxTimeMedia(mNextVideoTimeMediaUs);
1448 }
1449 } else {
1450 mMediaClock->updateAnchor(
Wei Jiacc951512019-04-03 16:10:42 -07001451 mNextVideoTimeMediaUs, nowUs,
1452 mNextVideoTimeMediaUs + kDefaultVideoFrameIntervalUs);
Wei Jiadc293e32017-10-16 18:29:37 -07001453 }
1454 }
Wei Jiad214a562017-07-10 13:20:55 -07001455 }
Andreas Huberf9334412010-12-15 15:17:42 -08001456}
1457
Wei Jiaa05f1e32016-03-25 16:31:22 -07001458void NuPlayer::Renderer::notifyAudioTearDown(AudioTearDownReason reason) {
1459 sp<AMessage> msg = new AMessage(kWhatAudioTearDown, this);
1460 msg->setInt32("reason", reason);
1461 msg->post();
Wei Jia3a2956d2014-07-22 16:01:33 -07001462}
1463
Andreas Huberf9334412010-12-15 15:17:42 -08001464void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1465 int32_t audio;
1466 CHECK(msg->findInt32("audio", &audio));
1467
Wei Jia7b15cb32015-02-03 17:46:06 -08001468 if (dropBufferIfStale(audio, msg)) {
1469 return;
1470 }
1471
1472 if (audio) {
1473 mHasAudio = true;
1474 } else {
1475 mHasVideo = true;
1476 }
Ronghua Wua73d9e02014-10-08 15:13:29 -07001477
1478 if (mHasVideo) {
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001479 if (mVideoScheduler == NULL) {
1480 mVideoScheduler = new VideoFrameScheduler();
1481 mVideoScheduler->init();
1482 }
Andreas Huberbc7f5b22011-01-21 10:15:23 -08001483 }
1484
Wonsik Kim7e34bf52016-08-23 00:09:18 +09001485 sp<RefBase> obj;
1486 CHECK(msg->findObject("buffer", &obj));
1487 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
Andreas Huberf9334412010-12-15 15:17:42 -08001488
1489 sp<AMessage> notifyConsumed;
1490 CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1491
1492 QueueEntry entry;
1493 entry.mBuffer = buffer;
1494 entry.mNotifyConsumed = notifyConsumed;
1495 entry.mOffset = 0;
1496 entry.mFinalResult = OK;
Lajos Molnard5923402014-10-20 17:47:54 -07001497 entry.mBufferOrdinal = ++mTotalBuffersQueued;
Andreas Huberf9334412010-12-15 15:17:42 -08001498
1499 if (audio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001500 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -08001501 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -07001502 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001503 } else {
1504 mVideoQueue.push_back(entry);
Wei Jia7b15cb32015-02-03 17:46:06 -08001505 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -08001506 }
1507
Wei Jia7b15cb32015-02-03 17:46:06 -08001508 Mutex::Autolock autoLock(mLock);
Andreas Hubercb67cd12011-08-26 16:02:19 -07001509 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1510 return;
Andreas Huberf9334412010-12-15 15:17:42 -08001511 }
Andreas Hubercb67cd12011-08-26 16:02:19 -07001512
Wonsik Kim7e34bf52016-08-23 00:09:18 +09001513 sp<MediaCodecBuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1514 sp<MediaCodecBuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
Andreas Hubercb67cd12011-08-26 16:02:19 -07001515
1516 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1517 // EOS signalled on either queue.
Wei Jiabc2fb722014-07-08 16:37:57 -07001518 syncQueuesDone_l();
Andreas Hubercb67cd12011-08-26 16:02:19 -07001519 return;
1520 }
1521
1522 int64_t firstAudioTimeUs;
1523 int64_t firstVideoTimeUs;
1524 CHECK(firstAudioBuffer->meta()
1525 ->findInt64("timeUs", &firstAudioTimeUs));
1526 CHECK(firstVideoBuffer->meta()
1527 ->findInt64("timeUs", &firstVideoTimeUs));
1528
1529 int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1530
Steve Block3856b092011-10-20 11:56:00 +01001531 ALOGV("queueDiff = %.2f secs", diff / 1E6);
Andreas Hubercb67cd12011-08-26 16:02:19 -07001532
Chih-Hung Hsieh62309d52018-12-11 13:54:02 -08001533 if (diff > 100000LL) {
Andreas Hubercb67cd12011-08-26 16:02:19 -07001534 // Audio data starts More than 0.1 secs before video.
1535 // Drop some audio.
1536
1537 (*mAudioQueue.begin()).mNotifyConsumed->post();
1538 mAudioQueue.erase(mAudioQueue.begin());
1539 return;
1540 }
1541
Wei Jiabc2fb722014-07-08 16:37:57 -07001542 syncQueuesDone_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001543}
1544
Wei Jiabc2fb722014-07-08 16:37:57 -07001545void NuPlayer::Renderer::syncQueuesDone_l() {
Andreas Huberf9334412010-12-15 15:17:42 -08001546 if (!mSyncQueues) {
1547 return;
1548 }
1549
1550 mSyncQueues = false;
1551
1552 if (!mAudioQueue.empty()) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001553 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001554 }
1555
1556 if (!mVideoQueue.empty()) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001557 mLock.unlock();
1558 postDrainVideoQueue();
1559 mLock.lock();
Andreas Huberf9334412010-12-15 15:17:42 -08001560 }
1561}
1562
1563void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1564 int32_t audio;
1565 CHECK(msg->findInt32("audio", &audio));
1566
Wei Jia7b15cb32015-02-03 17:46:06 -08001567 if (dropBufferIfStale(audio, msg)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001568 return;
1569 }
1570
1571 int32_t finalResult;
1572 CHECK(msg->findInt32("finalResult", &finalResult));
1573
1574 QueueEntry entry;
1575 entry.mOffset = 0;
1576 entry.mFinalResult = finalResult;
1577
1578 if (audio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001579 Mutex::Autolock autoLock(mLock);
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001580 if (mAudioQueue.empty() && mSyncQueues) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001581 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001582 }
Andreas Huberf9334412010-12-15 15:17:42 -08001583 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -07001584 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001585 } else {
Wei Jia7b15cb32015-02-03 17:46:06 -08001586 if (mVideoQueue.empty() && getSyncQueues()) {
1587 Mutex::Autolock autoLock(mLock);
Wei Jiabc2fb722014-07-08 16:37:57 -07001588 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001589 }
Andreas Huberf9334412010-12-15 15:17:42 -08001590 mVideoQueue.push_back(entry);
Wei Jia7b15cb32015-02-03 17:46:06 -08001591 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -08001592 }
1593}
1594
1595void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
Chong Zhang7137ec72014-11-12 16:41:05 -08001596 int32_t audio, notifyComplete;
Andreas Huberf9334412010-12-15 15:17:42 -08001597 CHECK(msg->findInt32("audio", &audio));
1598
Wei Jia28a8a9f2014-08-18 11:29:50 -07001599 {
Wei Jia7b15cb32015-02-03 17:46:06 -08001600 Mutex::Autolock autoLock(mLock);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001601 if (audio) {
Chong Zhang7137ec72014-11-12 16:41:05 -08001602 notifyComplete = mNotifyCompleteAudio;
1603 mNotifyCompleteAudio = false;
Wei Jia2995dc72015-07-24 16:00:33 -07001604 mLastAudioMediaTimeUs = -1;
Wei Jiacc951512019-04-03 16:10:42 -07001605
1606 mHasAudio = false;
1607 if (mNextVideoTimeMediaUs >= 0) {
1608 int64_t nowUs = ALooper::GetNowUs();
1609 mMediaClock->updateAnchor(
1610 mNextVideoTimeMediaUs, nowUs,
1611 mNextVideoTimeMediaUs + kDefaultVideoFrameIntervalUs);
1612 }
Wei Jia28a8a9f2014-08-18 11:29:50 -07001613 } else {
Chong Zhang7137ec72014-11-12 16:41:05 -08001614 notifyComplete = mNotifyCompleteVideo;
1615 mNotifyCompleteVideo = false;
Wei Jia28a8a9f2014-08-18 11:29:50 -07001616 }
Wei Jia28a8a9f2014-08-18 11:29:50 -07001617
Wei Jia7b15cb32015-02-03 17:46:06 -08001618 // If we're currently syncing the queues, i.e. dropping audio while
1619 // aligning the first audio/video buffer times and only one of the
1620 // two queues has data, we may starve that queue by not requesting
1621 // more buffers from the decoder. If the other source then encounters
1622 // a discontinuity that leads to flushing, we'll never find the
1623 // corresponding discontinuity on the other queue.
1624 // Therefore we'll stop syncing the queues if at least one of them
1625 // is flushed.
1626 syncQueuesDone_l();
Wei Jiabc2fb722014-07-08 16:37:57 -07001627 }
Wei Jia9a3101b2016-11-08 14:34:24 -08001628 clearAnchorTime();
Andreas Huberf9334412010-12-15 15:17:42 -08001629
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001630 ALOGV("flushing %s", audio ? "audio" : "video");
Andreas Huberf9334412010-12-15 15:17:42 -08001631 if (audio) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001632 {
1633 Mutex::Autolock autoLock(mLock);
1634 flushQueue(&mAudioQueue);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001635
Wei Jia7b15cb32015-02-03 17:46:06 -08001636 ++mAudioDrainGeneration;
Wei Jia7c8d0e02015-08-27 17:40:21 -07001637 ++mAudioEOSGeneration;
Wei Jia7b15cb32015-02-03 17:46:06 -08001638 prepareForMediaRenderingStart_l();
Wei Jia28a8a9f2014-08-18 11:29:50 -07001639
Andy Hunga0b39712015-05-31 22:40:49 -07001640 // the frame count will be reset after flush.
1641 clearAudioFirstAnchorTime_l();
Wei Jiabc2fb722014-07-08 16:37:57 -07001642 }
Andreas Huberf9334412010-12-15 15:17:42 -08001643
Andreas Huberf9334412010-12-15 15:17:42 -08001644 mDrainAudioQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001645
Wei Jiabc2fb722014-07-08 16:37:57 -07001646 if (offloadingAudio()) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001647 mAudioSink->pause();
1648 mAudioSink->flush();
Andy Hung85e48142015-05-31 23:04:15 -07001649 if (!mPaused) {
1650 mAudioSink->start();
1651 }
Wei Jia9e7ed332015-05-01 16:35:43 -07001652 } else {
1653 mAudioSink->pause();
1654 mAudioSink->flush();
1655 // Call stop() to signal to the AudioSink to completely fill the
1656 // internal buffer before resuming playback.
Andy Hungb03dcb32015-08-27 16:18:59 -07001657 // FIXME: this is ignored after flush().
Wei Jia9e7ed332015-05-01 16:35:43 -07001658 mAudioSink->stop();
Andy Hung1d3556d2018-03-29 16:30:14 -07001659 if (!mPaused) {
Wei Jia9e7ed332015-05-01 16:35:43 -07001660 mAudioSink->start();
1661 }
1662 mNumFramesWritten = 0;
Wei Jiabc2fb722014-07-08 16:37:57 -07001663 }
Andy Hung528c8402016-01-12 12:39:34 -08001664 mNextAudioClockUpdateTimeUs = -1;
Andreas Huberf9334412010-12-15 15:17:42 -08001665 } else {
1666 flushQueue(&mVideoQueue);
1667
Andreas Huberf9334412010-12-15 15:17:42 -08001668 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001669
Lajos Molnarc851b5d2014-09-18 14:14:29 -07001670 if (mVideoScheduler != NULL) {
1671 mVideoScheduler->restart();
1672 }
1673
Wei Jia7b15cb32015-02-03 17:46:06 -08001674 Mutex::Autolock autoLock(mLock);
1675 ++mVideoDrainGeneration;
1676 prepareForMediaRenderingStart_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001677 }
1678
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001679 mVideoSampleReceived = false;
Chong Zhang7137ec72014-11-12 16:41:05 -08001680
1681 if (notifyComplete) {
1682 notifyFlushComplete(audio);
1683 }
Andreas Huberf9334412010-12-15 15:17:42 -08001684}
1685
1686void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1687 while (!queue->empty()) {
1688 QueueEntry *entry = &*queue->begin();
1689
1690 if (entry->mBuffer != NULL) {
1691 entry->mNotifyConsumed->post();
Wei Jia9a3101b2016-11-08 14:34:24 -08001692 } else if (entry->mNotifyConsumed != nullptr) {
1693 // Is it needed to open audio sink now?
1694 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed);
Andreas Huberf9334412010-12-15 15:17:42 -08001695 }
1696
1697 queue->erase(queue->begin());
1698 entry = NULL;
1699 }
1700}
1701
1702void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1703 sp<AMessage> notify = mNotify->dup();
1704 notify->setInt32("what", kWhatFlushComplete);
1705 notify->setInt32("audio", static_cast<int32_t>(audio));
1706 notify->post();
1707}
1708
Wei Jia7b15cb32015-02-03 17:46:06 -08001709bool NuPlayer::Renderer::dropBufferIfStale(
Andreas Huberf9334412010-12-15 15:17:42 -08001710 bool audio, const sp<AMessage> &msg) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001711 int32_t queueGeneration;
1712 CHECK(msg->findInt32("queueGeneration", &queueGeneration));
Andreas Huberf9334412010-12-15 15:17:42 -08001713
Wei Jia7b15cb32015-02-03 17:46:06 -08001714 if (queueGeneration == getQueueGeneration(audio)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001715 return false;
1716 }
1717
1718 sp<AMessage> notifyConsumed;
1719 if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1720 notifyConsumed->post();
1721 }
1722
1723 return true;
1724}
1725
Andreas Huber3831a062010-12-21 10:22:33 -08001726void NuPlayer::Renderer::onAudioSinkChanged() {
Wei Jiabc2fb722014-07-08 16:37:57 -07001727 if (offloadingAudio()) {
1728 return;
1729 }
Andreas Huber3831a062010-12-21 10:22:33 -08001730 CHECK(!mDrainAudioQueuePending);
1731 mNumFramesWritten = 0;
Wei Jia9a3101b2016-11-08 14:34:24 -08001732 mAnchorNumFramesWritten = -1;
Marco Nelissen4110c102012-03-29 09:31:28 -07001733 uint32_t written;
1734 if (mAudioSink->getFramesWritten(&written) == OK) {
1735 mNumFramesWritten = written;
1736 }
Andreas Huber3831a062010-12-21 10:22:33 -08001737}
1738
Wei Jiabc2fb722014-07-08 16:37:57 -07001739void NuPlayer::Renderer::onDisableOffloadAudio() {
1740 Mutex::Autolock autoLock(mLock);
1741 mFlags &= ~FLAG_OFFLOAD_AUDIO;
Wei Jia7b15cb32015-02-03 17:46:06 -08001742 ++mAudioDrainGeneration;
Robert Shihe1d70192015-07-23 17:54:13 -07001743 if (mAudioRenderingStartGeneration != -1) {
1744 prepareForMediaRenderingStart_l();
andysu36177952018-04-20 15:17:13 +08001745 // PauseTimeout is applied to offload mode only. Cancel pending timer.
1746 cancelAudioOffloadPauseTimeout();
Robert Shihe1d70192015-07-23 17:54:13 -07001747 }
Wei Jiabc2fb722014-07-08 16:37:57 -07001748}
1749
Ronghua Wua10fd232014-11-06 16:15:20 -08001750void NuPlayer::Renderer::onEnableOffloadAudio() {
1751 Mutex::Autolock autoLock(mLock);
1752 mFlags |= FLAG_OFFLOAD_AUDIO;
Wei Jia7b15cb32015-02-03 17:46:06 -08001753 ++mAudioDrainGeneration;
Robert Shihe1d70192015-07-23 17:54:13 -07001754 if (mAudioRenderingStartGeneration != -1) {
1755 prepareForMediaRenderingStart_l();
1756 }
Ronghua Wua10fd232014-11-06 16:15:20 -08001757}
1758
Andreas Huberb4082222011-01-20 15:23:04 -08001759void NuPlayer::Renderer::onPause() {
Rachad8592dbb2014-09-09 13:10:28 -07001760 if (mPaused) {
Rachad8592dbb2014-09-09 13:10:28 -07001761 return;
1762 }
Lajos Molnar6d339f12015-04-17 16:15:53 -07001763
Wei Jia28a8a9f2014-08-18 11:29:50 -07001764 {
1765 Mutex::Autolock autoLock(mLock);
Andy Hung005e9d02015-05-31 23:16:07 -07001766 // we do not increment audio drain generation so that we fill audio buffer during pause.
Wei Jia7b15cb32015-02-03 17:46:06 -08001767 ++mVideoDrainGeneration;
1768 prepareForMediaRenderingStart_l();
Wei Jia73ddd212014-08-29 16:33:49 -07001769 mPaused = true;
Wei Jia98160162015-02-04 17:01:11 -08001770 mMediaClock->setPlaybackRate(0.0);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001771 }
1772
Andreas Huberb4082222011-01-20 15:23:04 -08001773 mDrainAudioQueuePending = false;
Andreas Huberb4082222011-01-20 15:23:04 -08001774 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001775
Andy Hung84200732015-12-03 14:41:06 -08001776 // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1777 mAudioSink->pause();
1778 startAudioOffloadPauseTimeout();
Andreas Huberb4082222011-01-20 15:23:04 -08001779
Lajos Molnar6d339f12015-04-17 16:15:53 -07001780 ALOGV("now paused audio queue has %zu entries, video has %zu entries",
Andreas Huberea9d51b2011-11-30 09:53:40 -08001781 mAudioQueue.size(), mVideoQueue.size());
Andreas Huberb4082222011-01-20 15:23:04 -08001782}
1783
1784void NuPlayer::Renderer::onResume() {
Andreas Huberb58ce9f2011-11-28 16:27:35 -08001785 if (!mPaused) {
1786 return;
1787 }
Andreas Huberb4082222011-01-20 15:23:04 -08001788
Andy Hung84200732015-12-03 14:41:06 -08001789 // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1790 cancelAudioOffloadPauseTimeout();
1791 if (mAudioSink->ready()) {
Eric Laurent97c9f4f2015-08-11 18:04:14 -07001792 status_t err = mAudioSink->start();
1793 if (err != OK) {
Andy Hung58d315c2015-09-11 18:44:56 -07001794 ALOGE("cannot start AudioSink err %d", err);
Wei Jiaa05f1e32016-03-25 16:31:22 -07001795 notifyAudioTearDown(kDueToError);
Eric Laurent97c9f4f2015-08-11 18:04:14 -07001796 }
Andreas Huberb4082222011-01-20 15:23:04 -08001797 }
1798
Wei Jia7b15cb32015-02-03 17:46:06 -08001799 {
1800 Mutex::Autolock autoLock(mLock);
1801 mPaused = false;
Andy Hungb12ea0b2015-09-03 15:13:01 -07001802 // rendering started message may have been delayed if we were paused.
1803 if (mRenderingDataDelivered) {
1804 notifyIfMediaRenderingStarted_l();
1805 }
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001806 // configure audiosink as we did not do it when pausing
Wei Jia27ea08e2015-05-12 14:50:35 -07001807 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001808 mAudioSink->setPlaybackRate(mPlaybackSettings);
1809 }
1810
Wei Jia98160162015-02-04 17:01:11 -08001811 mMediaClock->setPlaybackRate(mPlaybackRate);
Andreas Huberb4082222011-01-20 15:23:04 -08001812
Wei Jia7b15cb32015-02-03 17:46:06 -08001813 if (!mAudioQueue.empty()) {
1814 postDrainAudioQueue_l();
1815 }
Andreas Huberb4082222011-01-20 15:23:04 -08001816 }
1817
1818 if (!mVideoQueue.empty()) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001819 postDrainVideoQueue();
Andreas Huberb4082222011-01-20 15:23:04 -08001820 }
1821}
1822
Lajos Molnarc851b5d2014-09-18 14:14:29 -07001823void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1824 if (mVideoScheduler == NULL) {
1825 mVideoScheduler = new VideoFrameScheduler();
1826 }
1827 mVideoScheduler->init(fps);
1828}
1829
Wei Jia7b15cb32015-02-03 17:46:06 -08001830int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1831 Mutex::Autolock autoLock(mLock);
1832 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1833}
1834
1835int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1836 Mutex::Autolock autoLock(mLock);
1837 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1838}
1839
1840bool NuPlayer::Renderer::getSyncQueues() {
1841 Mutex::Autolock autoLock(mLock);
1842 return mSyncQueues;
1843}
1844
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001845void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1846 if (mAudioTornDown) {
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001847 return;
1848 }
andysu36177952018-04-20 15:17:13 +08001849
1850 // TimeoutWhenPaused is only for offload mode.
1851 if (reason == kDueToTimeout && !offloadingAudio()) {
1852 return;
1853 }
1854
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001855 mAudioTornDown = true;
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001856
Ronghua Wua73d9e02014-10-08 15:13:29 -07001857 int64_t currentPositionUs;
Robert Shih1a5c8592015-08-04 18:07:44 -07001858 sp<AMessage> notify = mNotify->dup();
1859 if (getCurrentPosition(&currentPositionUs) == OK) {
1860 notify->setInt64("positionUs", currentPositionUs);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001861 }
Lajos Molnar06ad1522014-08-28 07:27:44 -07001862
Wei Jia3a2956d2014-07-22 16:01:33 -07001863 mAudioSink->stop();
1864 mAudioSink->flush();
1865
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001866 notify->setInt32("what", kWhatAudioTearDown);
Ronghua Wu08529172014-10-02 16:55:52 -07001867 notify->setInt32("reason", reason);
Wei Jia3a2956d2014-07-22 16:01:33 -07001868 notify->post();
1869}
1870
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001871void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1872 if (offloadingAudio()) {
Weiyin Jiang35d5af12015-01-28 16:14:02 +08001873 mWakeLock->acquire();
Lajos Molnar1d15ab52015-03-04 16:46:34 -08001874 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
Wei Jia7b15cb32015-02-03 17:46:06 -08001875 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001876 msg->post(kOffloadPauseMaxUs);
1877 }
1878}
1879
1880void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
Andy Hung21afe802016-04-21 14:52:05 -07001881 // We may have called startAudioOffloadPauseTimeout() without
1882 // the AudioSink open and with offloadingAudio enabled.
1883 //
1884 // When we cancel, it may be that offloadingAudio is subsequently disabled, so regardless
1885 // we always release the wakelock and increment the pause timeout generation.
1886 //
1887 // Note: The acquired wakelock prevents the device from suspending
1888 // immediately after offload pause (in case a resume happens shortly thereafter).
1889 mWakeLock->release(true);
1890 ++mAudioOffloadPauseTimeoutGeneration;
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001891}
1892
Andy Hung202bce12014-12-03 11:47:36 -08001893status_t NuPlayer::Renderer::onOpenAudioSink(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001894 const sp<AMessage> &format,
1895 bool offloadOnly,
1896 bool hasVideo,
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +05301897 uint32_t flags,
1898 bool isStreaming) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001899 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1900 offloadOnly, offloadingAudio());
1901 bool audioSinkChanged = false;
1902
1903 int32_t numChannels;
1904 CHECK(format->findInt32("channel-count", &numChannels));
1905
1906 int32_t channelMask;
1907 if (!format->findInt32("channel-mask", &channelMask)) {
1908 // signal to the AudioSink to derive the mask from count.
1909 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1910 }
1911
1912 int32_t sampleRate;
1913 CHECK(format->findInt32("sample-rate", &sampleRate));
1914
Ari Hausman-Cohen2b674bb2018-06-22 14:53:20 -07001915 // read pcm encoding from MediaCodec output format, if available
1916 int32_t pcmEncoding;
1917 audio_format_t audioFormat =
1918 format->findInt32(KEY_PCM_ENCODING, &pcmEncoding) ?
1919 audioFormatFromEncoding(pcmEncoding) : AUDIO_FORMAT_PCM_16_BIT;
1920
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001921 if (offloadingAudio()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001922 AString mime;
1923 CHECK(format->findString("mime", &mime));
1924 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1925
1926 if (err != OK) {
1927 ALOGE("Couldn't map mime \"%s\" to a valid "
1928 "audio_format", mime.c_str());
1929 onDisableOffloadAudio();
1930 } else {
1931 ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1932 mime.c_str(), audioFormat);
1933
1934 int avgBitRate = -1;
Lajos Molnar46f80162016-05-19 15:35:47 -07001935 format->findInt32("bitrate", &avgBitRate);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001936
1937 int32_t aacProfile = -1;
1938 if (audioFormat == AUDIO_FORMAT_AAC
1939 && format->findInt32("aac-profile", &aacProfile)) {
1940 // Redefine AAC format as per aac profile
1941 mapAACProfileToAudioFormat(
1942 audioFormat,
1943 aacProfile);
1944 }
1945
1946 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1947 offloadInfo.duration_us = -1;
1948 format->findInt64(
1949 "durationUs", &offloadInfo.duration_us);
1950 offloadInfo.sample_rate = sampleRate;
1951 offloadInfo.channel_mask = channelMask;
1952 offloadInfo.format = audioFormat;
1953 offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1954 offloadInfo.bit_rate = avgBitRate;
1955 offloadInfo.has_video = hasVideo;
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +05301956 offloadInfo.is_streaming = isStreaming;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001957
1958 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1959 ALOGV("openAudioSink: no change in offload mode");
1960 // no change from previous configuration, everything ok.
Andy Hung202bce12014-12-03 11:47:36 -08001961 return OK;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001962 }
Andy Hungf0e83642014-12-19 17:55:56 -08001963 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1964
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001965 ALOGV("openAudioSink: try to open AudioSink in offload mode");
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001966 uint32_t offloadFlags = flags;
1967 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1968 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001969 audioSinkChanged = true;
1970 mAudioSink->close();
Andy Hunga0b39712015-05-31 22:40:49 -07001971
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001972 err = mAudioSink->open(
1973 sampleRate,
1974 numChannels,
1975 (audio_channel_mask_t)channelMask,
1976 audioFormat,
Andy Hung179652e2015-05-31 22:49:46 -07001977 0 /* bufferCount - unused */,
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001978 &NuPlayer::Renderer::AudioSinkCallback,
1979 this,
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001980 (audio_output_flags_t)offloadFlags,
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001981 &offloadInfo);
1982
1983 if (err == OK) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001984 err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1985 }
1986
1987 if (err == OK) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001988 // If the playback is offloaded to h/w, we pass
1989 // the HAL some metadata information.
1990 // We don't want to do this for PCM because it
1991 // will be going through the AudioFlinger mixer
1992 // before reaching the hardware.
1993 // TODO
1994 mCurrentOffloadInfo = offloadInfo;
Andy Hung85e48142015-05-31 23:04:15 -07001995 if (!mPaused) { // for preview mode, don't start if paused
1996 err = mAudioSink->start();
1997 }
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001998 ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1999 }
2000 if (err != OK) {
2001 // Clean up, fall back to non offload mode.
2002 mAudioSink->close();
2003 onDisableOffloadAudio();
2004 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
2005 ALOGV("openAudioSink: offload failed");
Wei Jiaa05f1e32016-03-25 16:31:22 -07002006 if (offloadOnly) {
2007 notifyAudioTearDown(kForceNonOffload);
2008 }
Wei Jia3ab25452015-06-10 09:37:47 -07002009 } else {
2010 mUseAudioCallback = true; // offload mode transfers data through callback
2011 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002012 }
2013 }
2014 }
2015 if (!offloadOnly && !offloadingAudio()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002016 ALOGV("openAudioSink: open AudioSink in NON-offload mode");
Eric Laurentd88c3ca2014-10-28 10:52:11 -07002017 uint32_t pcmFlags = flags;
2018 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
Andy Hungf0e83642014-12-19 17:55:56 -08002019
2020 const PcmInfo info = {
2021 (audio_channel_mask_t)channelMask,
2022 (audio_output_flags_t)pcmFlags,
Ari Hausman-Cohen2b674bb2018-06-22 14:53:20 -07002023 audioFormat,
Andy Hungf0e83642014-12-19 17:55:56 -08002024 numChannels,
2025 sampleRate
2026 };
2027 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
2028 ALOGV("openAudioSink: no change in pcm mode");
2029 // no change from previous configuration, everything ok.
2030 return OK;
2031 }
2032
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002033 audioSinkChanged = true;
2034 mAudioSink->close();
2035 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
Andy Hunga0b39712015-05-31 22:40:49 -07002036 // Note: It is possible to set up the callback, but not use it to send audio data.
2037 // This requires a fix in AudioSink to explicitly specify the transfer mode.
2038 mUseAudioCallback = getUseAudioCallbackSetting();
Wei Jia3ab25452015-06-10 09:37:47 -07002039 if (mUseAudioCallback) {
2040 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
2041 }
Andy Hung179652e2015-05-31 22:49:46 -07002042
2043 // Compute the desired buffer size.
2044 // For callback mode, the amount of time before wakeup is about half the buffer size.
2045 const uint32_t frameCount =
2046 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
2047
Ronghua Wu7665f582015-09-02 10:15:30 -07002048 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
2049 // AudioSink. We don't want this when there's video because it will cause a video seek to
2050 // the previous I frame. But we do want this when there's only audio because it will give
2051 // NuPlayer a chance to switch from non-offload mode to offload mode.
2052 // So we only set doNotReconnect when there's no video.
2053 const bool doNotReconnect = !hasVideo;
Andy Hungff874dc2016-04-11 16:49:09 -07002054
2055 // We should always be able to set our playback settings if the sink is closed.
2056 LOG_ALWAYS_FATAL_IF(mAudioSink->setPlaybackRate(mPlaybackSettings) != OK,
2057 "onOpenAudioSink: can't set playback rate on closed sink");
Andy Hung202bce12014-12-03 11:47:36 -08002058 status_t err = mAudioSink->open(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002059 sampleRate,
2060 numChannels,
2061 (audio_channel_mask_t)channelMask,
Ari Hausman-Cohen2b674bb2018-06-22 14:53:20 -07002062 audioFormat,
Andy Hung179652e2015-05-31 22:49:46 -07002063 0 /* bufferCount - unused */,
Andy Hunga0b39712015-05-31 22:40:49 -07002064 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
2065 mUseAudioCallback ? this : NULL,
Ronghua Wufaeb0f22015-05-21 12:20:21 -07002066 (audio_output_flags_t)pcmFlags,
2067 NULL,
Ronghua Wu7665f582015-09-02 10:15:30 -07002068 doNotReconnect,
Andy Hung179652e2015-05-31 22:49:46 -07002069 frameCount);
Andy Hung202bce12014-12-03 11:47:36 -08002070 if (err != OK) {
2071 ALOGW("openAudioSink: non offloaded open failed status: %d", err);
Wei Jia4d7ac852015-08-31 18:33:14 -07002072 mAudioSink->close();
Andy Hungf0e83642014-12-19 17:55:56 -08002073 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
Andy Hung202bce12014-12-03 11:47:36 -08002074 return err;
2075 }
Andy Hungf0e83642014-12-19 17:55:56 -08002076 mCurrentPcmInfo = info;
Andy Hung005e9d02015-05-31 23:16:07 -07002077 if (!mPaused) { // for preview mode, don't start if paused
2078 mAudioSink->start();
2079 }
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002080 }
2081 if (audioSinkChanged) {
2082 onAudioSinkChanged();
2083 }
Ronghua Wufaeb0f22015-05-21 12:20:21 -07002084 mAudioTornDown = false;
Andy Hung202bce12014-12-03 11:47:36 -08002085 return OK;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002086}
2087
2088void NuPlayer::Renderer::onCloseAudioSink() {
2089 mAudioSink->close();
2090 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
Andy Hungf0e83642014-12-19 17:55:56 -08002091 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07002092}
2093
Wei Jia9a3101b2016-11-08 14:34:24 -08002094void NuPlayer::Renderer::onChangeAudioFormat(
2095 const sp<AMessage> &meta, const sp<AMessage> &notify) {
2096 sp<AMessage> format;
2097 CHECK(meta->findMessage("format", &format));
2098
2099 int32_t offloadOnly;
2100 CHECK(meta->findInt32("offload-only", &offloadOnly));
2101
2102 int32_t hasVideo;
2103 CHECK(meta->findInt32("has-video", &hasVideo));
2104
2105 uint32_t flags;
2106 CHECK(meta->findInt32("flags", (int32_t *)&flags));
2107
Dhananjay Kumarc387f2b2015-08-06 10:43:16 +05302108 uint32_t isStreaming;
2109 CHECK(meta->findInt32("isStreaming", (int32_t *)&isStreaming));
2110
2111 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming);
Wei Jia9a3101b2016-11-08 14:34:24 -08002112
2113 if (err != OK) {
2114 notify->setInt32("err", err);
2115 }
2116 notify->post();
2117}
2118
Andreas Huberf9334412010-12-15 15:17:42 -08002119} // namespace android