From 975c2cfdb944c123d15183eb2538aecb75a32af8 Mon Sep 17 00:00:00 2001 From: Timotej Lazar Date: Wed, 3 Jun 2020 11:53:29 +0200 Subject: [PATCH] Bink: add a queue for video frames Decode one or more video frames in advance. This prevents the input stream blocking while waiting for video packets to be drained, and makes the playback smoother. Also factor out reclamation of processed audio buffers, which is now independent from decoding new packets. --- src/bink.c | 270 ++++++++++++++++++++++++++++------------------------- 1 file changed, 144 insertions(+), 126 deletions(-) diff --git a/src/bink.c b/src/bink.c index 5bc99c5..b414658 100644 --- a/src/bink.c +++ b/src/bink.c @@ -19,7 +19,8 @@ extern float PlatVolumeToGain(int volume); //#define AL_CHECK() { int err = alGetError(); if(err!=AL_NO_ERROR) printf("%s:%d ALError %04x\n", __FILE__, __LINE__, err); } #define AL_CHECK() {} -#define FRAMEQUEUESIZE 4 +#define AUDIO_FRAMES 32 +#define VIDEO_FRAMES 4 struct binkMovie { AVFormatContext* avContext; @@ -29,12 +30,15 @@ struct binkMovie { AVCodecContext* videoCodecContext; AVFrame* videoFrame; struct SwsContext* videoScaleContext; - uint8_t* videoScalePicture[4]; int videoScaleLineSize[4]; uint videoScaleWidth; uint videoScaleHeight; enum AVPixelFormat videoScaleFormat; - float videoFrameDuration; + + uint8_t* videoFrames[VIDEO_FRAMES][4]; + int currentFrame; + int renderedFrames; + float frameDuration; int audioStreamIndex; AVCodecContext* audioCodecContext; @@ -43,8 +47,8 @@ struct binkMovie { BOOL alInited; ALuint alSource; - ALuint alBuffers[FRAMEQUEUESIZE]; - ALuint alFreeBuffers[FRAMEQUEUESIZE]; + ALuint alBuffers[AUDIO_FRAMES]; + ALuint alFreeBuffers[AUDIO_FRAMES]; ALuint alNumFreeBuffers; ALuint alNumChannels; ALenum alFormat; @@ -54,15 +58,23 @@ struct binkMovie { BOOL looping; }; -static void BinkRenderMovie(struct binkMovie* aMovie) +static int BinkRenderMovie(struct binkMovie* movie) { - if (aMovie && aMovie->videoFrame && aMovie->videoScalePicture[0]) { - DrawAvpMenuBink( - aMovie->videoScalePicture[0], - aMovie->videoFrame->width, - aMovie->videoFrame->height, - aMovie->videoScaleLineSize[0]); + const int t = SDL_GetTicks() - movie->timeStart; + int dt = 0; + while (movie->renderedFrames > 1 && + (dt = t - (movie->currentFrame + 1) * movie->frameDuration) >= 0) { + movie->currentFrame++; + movie->renderedFrames--; } + + if (movie->renderedFrames) { + DrawAvpMenuBink(movie->videoFrames[movie->currentFrame%VIDEO_FRAMES][0], + movie->videoFrame->width, + movie->videoFrame->height, + movie->videoScaleLineSize[0]); + } + return dt; } static void BinkInitMovieStruct(struct binkMovie* aMovie) @@ -79,7 +91,7 @@ static void BinkReleaseMovie(struct binkMovie* aMovie) if (aMovie->alInited) { alSourceStop(aMovie->alSource); alDeleteSources(1, &aMovie->alSource); - alDeleteBuffers(FRAMEQUEUESIZE, aMovie->alBuffers); + alDeleteBuffers(AUDIO_FRAMES, aMovie->alBuffers); if (aMovie->audioTempBuffer) free(aMovie->audioTempBuffer); } @@ -97,7 +109,8 @@ static void BinkReleaseMovie(struct binkMovie* aMovie) if (aMovie->videoScaleContext) { sws_freeContext(aMovie->videoScaleContext); - av_freep(&aMovie->videoScalePicture[0]); + for (int i = 0; i < VIDEO_FRAMES; i++) + av_freep(&aMovie->videoFrames[i][0]); } BinkInitMovieStruct(aMovie); @@ -105,9 +118,6 @@ static void BinkReleaseMovie(struct binkMovie* aMovie) static int DecodeVideoFrame(struct binkMovie* aMovie) { - if (avcodec_receive_frame(aMovie->videoCodecContext, aMovie->videoFrame) != 0) - return 0; - // Initialize scale context. if (aMovie->videoScaleContext == NULL) { if (aMovie->videoScaleWidth == 0) @@ -127,23 +137,42 @@ static int DecodeVideoFrame(struct binkMovie* aMovie) if (aMovie->videoScaleContext == NULL) return 0; - av_image_alloc(aMovie->videoScalePicture, aMovie->videoScaleLineSize, - aMovie->videoScaleWidth, aMovie->videoScaleHeight, - aMovie->videoScaleFormat, 1); + for (int i = 0; i < VIDEO_FRAMES; i++) { + av_image_alloc( + aMovie->videoFrames[i], aMovie->videoScaleLineSize, + aMovie->videoScaleWidth, aMovie->videoScaleHeight, + aMovie->videoScaleFormat, 1); + } } sws_scale(aMovie->videoScaleContext, (const uint8_t* const*)aMovie->videoFrame->data, aMovie->videoFrame->linesize, 0, aMovie->videoFrame->height, - aMovie->videoScalePicture, aMovie->videoScaleLineSize); + aMovie->videoFrames[(aMovie->currentFrame+aMovie->renderedFrames) % VIDEO_FRAMES], + aMovie->videoScaleLineSize); + aMovie->renderedFrames++; + return 1; +} + +// Reclaim completed audio buffers. +static int ProcessAudio(struct binkMovie* aMovie) +{ + if (!aMovie->alInited) + return 0; + + int processed = 0; + alGetSourcei(aMovie->alSource, AL_BUFFERS_PROCESSED, &processed); + if (processed > 0) { + ALuint buffers[AUDIO_FRAMES]; + alSourceUnqueueBuffers(aMovie->alSource, processed, buffers); + for (int i = 0; i < processed; i++) + aMovie->alFreeBuffers[aMovie->alNumFreeBuffers++] = buffers[i]; + } return 1; } static int DecodeAudioFrame(struct binkMovie* aMovie) { - if (avcodec_receive_frame(aMovie->audioCodecContext, aMovie->audioFrame) != 0) - return 0; - if (!SoundSys_IsOn()) return 0; @@ -170,95 +199,87 @@ static int DecodeAudioFrame(struct binkMovie* aMovie) if (aMovie->alNumChannels == 0) return 0; - // reclaim completed frames - int processed = 0; - alGetSourcei(aMovie->alSource, AL_BUFFERS_PROCESSED, &processed); - if (processed > 0) { - ALuint buffers[FRAMEQUEUESIZE]; - alSourceUnqueueBuffers(aMovie->alSource, processed, buffers); - AL_CHECK(); - for (int i = 0; i < processed; i++) - aMovie->alFreeBuffers[aMovie->alNumFreeBuffers++] = buffers[i]; - } + if (aMovie->alNumFreeBuffers == 0) + return 0; // queue this frame - if (aMovie->alNumFreeBuffers > 0) { - ALuint alBuffer = aMovie->alFreeBuffers[aMovie->alNumFreeBuffers-1]; + ALuint alBuffer = aMovie->alFreeBuffers[aMovie->alNumFreeBuffers-1]; - int sampleCount = aMovie->audioFrame->nb_samples * aMovie->alNumChannels; - uint dataSize = sampleCount * 2; // 16bit is deafult - void* data = (void*)aMovie->audioFrame->extended_data[0]; + int sampleCount = aMovie->audioFrame->nb_samples * aMovie->alNumChannels; + uint dataSize = sampleCount * 2; // 16bit is deafult + void* data = (void*)aMovie->audioFrame->extended_data[0]; - switch (aMovie->audioFrame->format) { - case AV_SAMPLE_FMT_U8: { - dataSize = sampleCount; - } break; + switch (aMovie->audioFrame->format) { + case AV_SAMPLE_FMT_U8: { + dataSize = sampleCount; + } break; - default: - case AV_SAMPLE_FMT_S16: { - /* - unsigned short* p = (unsigned short*) data; - for(int i=0; i>2; - */ - } break; - - case AV_SAMPLE_FMT_FLT: { - data = (void*)aMovie->audioTempBuffer; - short* tempBuf = (short*)aMovie->audioTempBuffer; - float* srcBuf = (float*)aMovie->audioFrame->extended_data[0]; - for (int i = 0; i < sampleCount; i++) { + default: + case AV_SAMPLE_FMT_S16: { + /* + unsigned short* p = (unsigned short*) data; + for(int i=0; i>2; + */ + } break; + + case AV_SAMPLE_FMT_FLT: { + data = (void*)aMovie->audioTempBuffer; + short* tempBuf = (short*)aMovie->audioTempBuffer; + float* srcBuf = (float*)aMovie->audioFrame->extended_data[0]; + for (int i = 0; i < sampleCount; i++) { + float val = srcBuf[i] * 32768; + if (val > 32767) + val = 32767; + if (val < -32768) + val = -32768; + tempBuf[i] = (short)val; + } + } break; + + case AV_SAMPLE_FMT_S32: { + data = (void*)aMovie->audioTempBuffer; + short* tempBuf = (short*)aMovie->audioTempBuffer; + unsigned int* srcBuf = (unsigned int*)aMovie->audioFrame->extended_data[0]; + for(int i = 0; i < sampleCount; i++) + tempBuf[i] = (short)(((*srcBuf - *srcBuf>>2) >> 16) & 0x0000FFFF); + } break; + + case AV_SAMPLE_FMT_FLTP: { + data = (void*)aMovie->audioTempBuffer; + short* tempBuf = (short*)aMovie->audioTempBuffer; + for (int i = 0; i < aMovie->audioFrame->nb_samples; i++) { + for (int j = 0; j < aMovie->alNumChannels; j++) { + float* srcBuf = (float*)aMovie->audioFrame->extended_data[j]; float val = srcBuf[i] * 32768; if (val > 32767) val = 32767; if (val < -32768) val = -32768; - tempBuf[i] = (short)val; + tempBuf[(i*aMovie->alNumChannels)+j] = (short)val; } - } break; - - case AV_SAMPLE_FMT_S32: { - data = (void*)aMovie->audioTempBuffer; - short* tempBuf = (short*)aMovie->audioTempBuffer; - unsigned int* srcBuf = (unsigned int*)aMovie->audioFrame->extended_data[0]; - for(int i = 0; i < sampleCount; i++) - tempBuf[i] = (short)(((*srcBuf - *srcBuf>>2) >> 16) & 0x0000FFFF); - } break; - - case AV_SAMPLE_FMT_FLTP: { - data = (void*)aMovie->audioTempBuffer; - short* tempBuf = (short*)aMovie->audioTempBuffer; - for (int i = 0; i < aMovie->audioFrame->nb_samples; i++) { - for (int j = 0; j < aMovie->alNumChannels; j++) { - float* srcBuf = (float*)aMovie->audioFrame->extended_data[j]; - float val = srcBuf[i] * 32768; - if (val > 32767) - val = 32767; - if (val < -32768) - val = -32768; - tempBuf[(i*aMovie->alNumChannels)+j] = (short)val; - } - } - } break; } + } break; + } - alSourceStop(aMovie->alSource); - - alBufferData(alBuffer, aMovie->alFormat, data, dataSize - 16, aMovie->alSampleRate); - AL_CHECK(); + alBufferData(alBuffer, aMovie->alFormat, data, dataSize - 16, aMovie->alSampleRate); + AL_CHECK(); - alSourceQueueBuffers(aMovie->alSource, 1, &alBuffer); - AL_CHECK(); + alSourceQueueBuffers(aMovie->alSource, 1, &alBuffer); + AL_CHECK(); - float vx, vy, vz; - alGetListener3f(AL_VELOCITY, &vx, &vy, &vz); - alSource3f(aMovie->alSource, AL_VELOCITY, vx, vy, vz); + float vx, vy, vz; + alGetListener3f(AL_VELOCITY, &vx, &vy, &vz); + alSource3f(aMovie->alSource, AL_VELOCITY, vx, vy, vz); + int state = 0; + alGetSourcei(aMovie->alSource, AL_SOURCE_STATE, &state); + if (state != AL_PLAYING) alSourcePlay(aMovie->alSource); - aMovie->alNumFreeBuffers--; - aMovie->alFreeBuffers[aMovie->alNumFreeBuffers] = 0; - } + aMovie->alNumFreeBuffers--; + aMovie->alFreeBuffers[aMovie->alNumFreeBuffers] = 0; + return 1; } @@ -281,7 +302,7 @@ static int ReadPacket(struct binkMovie* aMovie) } // Send the (possibly buffered) packet to decoder. - int ret = 0; + int ret = AVERROR(EAGAIN); if (aMovie->packet.stream_index == aMovie->videoStreamIndex) ret = avcodec_send_packet(aMovie->videoCodecContext, &aMovie->packet); else if (aMovie->packet.stream_index == aMovie->audioStreamIndex) @@ -335,8 +356,9 @@ static int BinkStartMovie(struct binkMovie* aMovie, const char* aFilename, aMovie->videoCodecContext = context; aMovie->videoStreamIndex = i; aMovie->videoFrame = av_frame_alloc(); - aMovie->videoFrameDuration = + aMovie->frameDuration = 1000.0f * (float)stream->time_base.num / (float)stream->time_base.den; + aMovie->timeStart = SDL_GetTicks(); numStreams++; } else if (aMovie->audioStreamIndex < 0 && context->codec_type == AVMEDIA_TYPE_AUDIO) { aMovie->audioCodecContext = context; @@ -352,8 +374,8 @@ static int BinkStartMovie(struct binkMovie* aMovie, const char* aFilename, alSourcef(aMovie->alSource, AL_PITCH, 1.0); alSourcef(aMovie->alSource, AL_GAIN, 1.0); - alGenBuffers(FRAMEQUEUESIZE, aMovie->alBuffers); - aMovie->alNumFreeBuffers = FRAMEQUEUESIZE; + alGenBuffers(AUDIO_FRAMES, aMovie->alBuffers); + aMovie->alNumFreeBuffers = AUDIO_FRAMES; for (int i = 0; i < aMovie->alNumFreeBuffers; i++) aMovie->alFreeBuffers[i] = aMovie->alBuffers[i]; @@ -369,7 +391,7 @@ static int BinkStartMovie(struct binkMovie* aMovie, const char* aFilename, } if (!aFmvFlag) { - for (int i = 0; i < (FRAMEQUEUESIZE-1) * numStreams; i++) + for (int i = 0; i < (AUDIO_FRAMES + VIDEO_FRAMES) / 2; i++) ReadPacket(aMovie); } @@ -381,29 +403,18 @@ static int BinkUpdateMovie(struct binkMovie* aMovie) if(!aMovie->avContext) return 0; - const int t = SDL_GetTicks(); const int eof = !ReadPacket(aMovie); int playing = 0; - if (aMovie->videoStreamIndex >= 0) { - if (aMovie->videoFrame->pts == 0) - aMovie->timeStart = t - aMovie->videoFrameDuration; - if (t - aMovie->timeStart >= aMovie->videoFrame->pts * aMovie->videoFrameDuration) + if (aMovie->videoStreamIndex >= 0 && aMovie->renderedFrames < VIDEO_FRAMES) + if (avcodec_receive_frame(aMovie->videoCodecContext, aMovie->videoFrame) == 0) playing += DecodeVideoFrame(aMovie); - } - if (aMovie->audioStreamIndex >= 0) { - int processedBuffers = 0; - alGetSourcei(aMovie->alSource, AL_BUFFERS_PROCESSED, &processedBuffers); - while (!aMovie->alInited || aMovie->alNumFreeBuffers>0 || processedBuffers>0) { - if (DecodeAudioFrame(aMovie)) - playing += 1; - else - break; - } - } + if (aMovie->audioStreamIndex >= 0 && aMovie->alNumFreeBuffers > 0) + if (avcodec_receive_frame(aMovie->audioCodecContext, aMovie->audioFrame) == 0) + playing += DecodeAudioFrame(aMovie); - return !eof || playing; + return eof && !playing ? -1 : playing; } void PlayBinkedFMV(char* filenamePtr, int volume) @@ -411,10 +422,15 @@ void PlayBinkedFMV(char* filenamePtr, int volume) struct binkMovie movie; if (BinkStartMovie(&movie, filenamePtr, FALSE, FALSE, FALSE)) { alSourcef(movie.alSource, AL_GAIN, PlatVolumeToGain(volume)); - while (BinkUpdateMovie(&movie)) { - BinkRenderMovie(&movie); + int updated = 0; + while ((updated = BinkUpdateMovie(&movie)) >= 0) { + ProcessAudio(&movie); + int dt = BinkRenderMovie(&movie); FlipBuffers(); - SDL_Delay(4); // don’t just burn it + + // don’t just burn it + if (!updated && dt > 0) + SDL_Delay(dt); } BinkReleaseMovie(&movie); } @@ -433,7 +449,8 @@ void StartMenuBackgroundBink() int PlayMenuBackgroundBink() { ClearScreenToBlack(); - if (BinkUpdateMovie(&menuBackgroundMovie)) { + if (BinkUpdateMovie(&menuBackgroundMovie) >= 0) { + ProcessAudio(&menuBackgroundMovie); BinkRenderMovie(&menuBackgroundMovie); return 1; } @@ -468,7 +485,7 @@ int PlayMusicBink(int volume) return 1; alSourcef(musicMovie.alSource, AL_GAIN, PlatVolumeToGain(volume)); - for (int i = 0; i < musicMovie.avContext->nb_streams * FRAMEQUEUESIZE; i++) { + for (int i = 0; i < musicMovie.avContext->nb_streams * AUDIO_FRAMES; i++) { int processedBuffers = 0; alGetSourcei(musicMovie.alSource, AL_BUFFERS_PROCESSED, &processedBuffers); if (processedBuffers + musicMovie.alNumFreeBuffers > 0) @@ -526,10 +543,11 @@ void CloseBinkFMV(FMVHandle aFmvHandle) char* GetBinkFMVImage(FMVHandle aFmvHandle) { if (aFmvHandle == 0) - return 0; + return NULL; struct binkMovie* movie = (struct binkMovie*)aFmvHandle; - if(!movie->videoScaleContext) - return 0; - return movie->videoScalePicture[0]; + if (!movie->videoScaleContext) + return NULL; + return movie->renderedFrames ? + movie->videoFrames[movie->currentFrame%VIDEO_FRAMES][0] : NULL; }