jetomit/avp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Bink: add a queue for video frames

Browse source 
Decode one or more video frames in advance. This prevents the input
stream blocking while waiting for video packets to be drained, and
makes the playback smoother.

Also factor out reclamation of processed audio buffers, which is now
independent from decoding new packets.
This commit is contained in:
Timotej Lazar 2020-06-03 11:53:29 +02:00
parent b9e2075c7e
commit 975c2cfdb9
1 changed files with 144 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

270
src/bink.c
View file

@ -19,7 +19,8 @@ extern float PlatVolumeToGain(int volume);
//#define AL_CHECK() { int err = alGetError(); if(err!=AL_NO_ERROR) printf("%s:%d ALError %04x\n", __FILE__, __LINE__, err); } //#define AL_CHECK() { int err = alGetError(); if(err!=AL_NO_ERROR) printf("%s:%d ALError %04x\n", __FILE__, __LINE__, err); }
#define AL_CHECK() {} #define AL_CHECK() {}
#define FRAMEQUEUESIZE 4 #define AUDIO_FRAMES 32
#define VIDEO_FRAMES 4
struct binkMovie { struct binkMovie {
AVFormatContext* avContext; AVFormatContext* avContext;
@ -29,12 +30,15 @@ struct binkMovie {
AVCodecContext* videoCodecContext; AVCodecContext* videoCodecContext;
AVFrame* videoFrame; AVFrame* videoFrame;
struct SwsContext* videoScaleContext; struct SwsContext* videoScaleContext;
uint8_t* videoScalePicture[4];
int videoScaleLineSize[4]; int videoScaleLineSize[4];
uint videoScaleWidth; uint videoScaleWidth;
uint videoScaleHeight; uint videoScaleHeight;
enum AVPixelFormat videoScaleFormat; enum AVPixelFormat videoScaleFormat;
float videoFrameDuration;
uint8_t* videoFrames[VIDEO_FRAMES][4];
int currentFrame;
int renderedFrames;
float frameDuration;
int audioStreamIndex; int audioStreamIndex;
AVCodecContext* audioCodecContext; AVCodecContext* audioCodecContext;
@ -43,8 +47,8 @@ struct binkMovie {
BOOL alInited; BOOL alInited;
ALuint alSource; ALuint alSource;
ALuint alBuffers[FRAMEQUEUESIZE]; ALuint alBuffers[AUDIO_FRAMES];
ALuint alFreeBuffers[FRAMEQUEUESIZE]; ALuint alFreeBuffers[AUDIO_FRAMES];
ALuint alNumFreeBuffers; ALuint alNumFreeBuffers;
ALuint alNumChannels; ALuint alNumChannels;
ALenum alFormat; ALenum alFormat;
@ -54,15 +58,23 @@ struct binkMovie {
BOOL looping; BOOL looping;
}; };
static void BinkRenderMovie(struct binkMovie* aMovie) static int BinkRenderMovie(struct binkMovie* movie)
{ {
if (aMovie && aMovie->videoFrame && aMovie->videoScalePicture[0]) { const int t = SDL_GetTicks() - movie->timeStart;
DrawAvpMenuBink( int dt = 0;
aMovie->videoScalePicture[0], while (movie->renderedFrames > 1 &&
aMovie->videoFrame->width, (dt = t - (movie->currentFrame + 1) * movie->frameDuration) >= 0) {
aMovie->videoFrame->height, movie->currentFrame++;
aMovie->videoScaleLineSize[0]); movie->renderedFrames--;
} }
if (movie->renderedFrames) {
DrawAvpMenuBink(movie->videoFrames[movie->currentFrame%VIDEO_FRAMES][0],
movie->videoFrame->width,
movie->videoFrame->height,
movie->videoScaleLineSize[0]);
}
return dt;
} }
static void BinkInitMovieStruct(struct binkMovie* aMovie) static void BinkInitMovieStruct(struct binkMovie* aMovie)
@ -79,7 +91,7 @@ static void BinkReleaseMovie(struct binkMovie* aMovie)
if (aMovie->alInited) { if (aMovie->alInited) {
alSourceStop(aMovie->alSource); alSourceStop(aMovie->alSource);
alDeleteSources(1, &aMovie->alSource); alDeleteSources(1, &aMovie->alSource);
alDeleteBuffers(FRAMEQUEUESIZE, aMovie->alBuffers); alDeleteBuffers(AUDIO_FRAMES, aMovie->alBuffers);
if (aMovie->audioTempBuffer) if (aMovie->audioTempBuffer)
free(aMovie->audioTempBuffer); free(aMovie->audioTempBuffer);
} }
@ -97,7 +109,8 @@ static void BinkReleaseMovie(struct binkMovie* aMovie)
if (aMovie->videoScaleContext) { if (aMovie->videoScaleContext) {
sws_freeContext(aMovie->videoScaleContext); sws_freeContext(aMovie->videoScaleContext);
av_freep(&aMovie->videoScalePicture[0]); for (int i = 0; i < VIDEO_FRAMES; i++)
av_freep(&aMovie->videoFrames[i][0]);
} }
BinkInitMovieStruct(aMovie); BinkInitMovieStruct(aMovie);
@ -105,9 +118,6 @@ static void BinkReleaseMovie(struct binkMovie* aMovie)
static int DecodeVideoFrame(struct binkMovie* aMovie) static int DecodeVideoFrame(struct binkMovie* aMovie)
{ {
if (avcodec_receive_frame(aMovie->videoCodecContext, aMovie->videoFrame) != 0)
return 0;
// Initialize scale context. // Initialize scale context.
if (aMovie->videoScaleContext == NULL) { if (aMovie->videoScaleContext == NULL) {
if (aMovie->videoScaleWidth == 0) if (aMovie->videoScaleWidth == 0)
@ -127,23 +137,42 @@ static int DecodeVideoFrame(struct binkMovie* aMovie)
if (aMovie->videoScaleContext == NULL) if (aMovie->videoScaleContext == NULL)
return 0; return 0;
av_image_alloc(aMovie->videoScalePicture, aMovie->videoScaleLineSize, for (int i = 0; i < VIDEO_FRAMES; i++) {
aMovie->videoScaleWidth, aMovie->videoScaleHeight, av_image_alloc(
aMovie->videoScaleFormat, 1); aMovie->videoFrames[i], aMovie->videoScaleLineSize,
aMovie->videoScaleWidth, aMovie->videoScaleHeight,
aMovie->videoScaleFormat, 1);
}
} }
sws_scale(aMovie->videoScaleContext, sws_scale(aMovie->videoScaleContext,
(const uint8_t* const*)aMovie->videoFrame->data, (const uint8_t* const*)aMovie->videoFrame->data,
aMovie->videoFrame->linesize, 0, aMovie->videoFrame->height, aMovie->videoFrame->linesize, 0, aMovie->videoFrame->height,
aMovie->videoScalePicture, aMovie->videoScaleLineSize); aMovie->videoFrames[(aMovie->currentFrame+aMovie->renderedFrames) % VIDEO_FRAMES],
aMovie->videoScaleLineSize);
aMovie->renderedFrames++;
return 1;
}
// Reclaim completed audio buffers.
static int ProcessAudio(struct binkMovie* aMovie)
{
if (!aMovie->alInited)
return 0;
int processed = 0;
alGetSourcei(aMovie->alSource, AL_BUFFERS_PROCESSED, &processed);
if (processed > 0) {
ALuint buffers[AUDIO_FRAMES];
alSourceUnqueueBuffers(aMovie->alSource, processed, buffers);
for (int i = 0; i < processed; i++)
aMovie->alFreeBuffers[aMovie->alNumFreeBuffers++] = buffers[i];
}
return 1; return 1;
} }
static int DecodeAudioFrame(struct binkMovie* aMovie) static int DecodeAudioFrame(struct binkMovie* aMovie)
{ {
if (avcodec_receive_frame(aMovie->audioCodecContext, aMovie->audioFrame) != 0)
return 0;
if (!SoundSys_IsOn()) if (!SoundSys_IsOn())
return 0; return 0;
@ -170,95 +199,87 @@ static int DecodeAudioFrame(struct binkMovie* aMovie)
if (aMovie->alNumChannels == 0) if (aMovie->alNumChannels == 0)
return 0; return 0;
// reclaim completed frames if (aMovie->alNumFreeBuffers == 0)
int processed = 0; return 0;
alGetSourcei(aMovie->alSource, AL_BUFFERS_PROCESSED, &processed);
if (processed > 0) {
ALuint buffers[FRAMEQUEUESIZE];
alSourceUnqueueBuffers(aMovie->alSource, processed, buffers);
AL_CHECK();
for (int i = 0; i < processed; i++)
aMovie->alFreeBuffers[aMovie->alNumFreeBuffers++] = buffers[i];
}
// queue this frame // queue this frame
if (aMovie->alNumFreeBuffers > 0) { ALuint alBuffer = aMovie->alFreeBuffers[aMovie->alNumFreeBuffers-1];
ALuint alBuffer = aMovie->alFreeBuffers[aMovie->alNumFreeBuffers-1];
int sampleCount = aMovie->audioFrame->nb_samples * aMovie->alNumChannels; int sampleCount = aMovie->audioFrame->nb_samples * aMovie->alNumChannels;
uint dataSize = sampleCount * 2; // 16bit is deafult uint dataSize = sampleCount * 2; // 16bit is deafult
void* data = (void*)aMovie->audioFrame->extended_data[0]; void* data = (void*)aMovie->audioFrame->extended_data[0];
switch (aMovie->audioFrame->format) { switch (aMovie->audioFrame->format) {
case AV_SAMPLE_FMT_U8: { case AV_SAMPLE_FMT_U8: {
dataSize = sampleCount; dataSize = sampleCount;
} break; } break;
default: default:
case AV_SAMPLE_FMT_S16: { case AV_SAMPLE_FMT_S16: {
/* /*
unsigned short* p = (unsigned short*) data; unsigned short* p = (unsigned short*) data;
for(int i=0; i<sampleCount; i++) for(int i=0; i<sampleCount; i++)
p[i] -= p[i]>>2; p[i] -= p[i]>>2;
*/ */
} break; } break;
case AV_SAMPLE_FMT_FLT: { case AV_SAMPLE_FMT_FLT: {
data = (void*)aMovie->audioTempBuffer; data = (void*)aMovie->audioTempBuffer;
short* tempBuf = (short*)aMovie->audioTempBuffer; short* tempBuf = (short*)aMovie->audioTempBuffer;
float* srcBuf = (float*)aMovie->audioFrame->extended_data[0]; float* srcBuf = (float*)aMovie->audioFrame->extended_data[0];
for (int i = 0; i < sampleCount; i++) { for (int i = 0; i < sampleCount; i++) {
float val = srcBuf[i] * 32768;
if (val > 32767)
val = 32767;
if (val < -32768)
val = -32768;
tempBuf[i] = (short)val;
}
} break;
case AV_SAMPLE_FMT_S32: {
data = (void*)aMovie->audioTempBuffer;
short* tempBuf = (short*)aMovie->audioTempBuffer;
unsigned int* srcBuf = (unsigned int*)aMovie->audioFrame->extended_data[0];
for(int i = 0; i < sampleCount; i++)
tempBuf[i] = (short)(((*srcBuf - *srcBuf>>2) >> 16) & 0x0000FFFF);
} break;
case AV_SAMPLE_FMT_FLTP: {
data = (void*)aMovie->audioTempBuffer;
short* tempBuf = (short*)aMovie->audioTempBuffer;
for (int i = 0; i < aMovie->audioFrame->nb_samples; i++) {
for (int j = 0; j < aMovie->alNumChannels; j++) {
float* srcBuf = (float*)aMovie->audioFrame->extended_data[j];
float val = srcBuf[i] * 32768; float val = srcBuf[i] * 32768;
if (val > 32767) if (val > 32767)
val = 32767; val = 32767;
if (val < -32768) if (val < -32768)
val = -32768; val = -32768;
tempBuf[i] = (short)val; tempBuf[(i*aMovie->alNumChannels)+j] = (short)val;
} }
} break;
case AV_SAMPLE_FMT_S32: {
data = (void*)aMovie->audioTempBuffer;
short* tempBuf = (short*)aMovie->audioTempBuffer;
unsigned int* srcBuf = (unsigned int*)aMovie->audioFrame->extended_data[0];
for(int i = 0; i < sampleCount; i++)
tempBuf[i] = (short)(((*srcBuf - *srcBuf>>2) >> 16) & 0x0000FFFF);
} break;
case AV_SAMPLE_FMT_FLTP: {
data = (void*)aMovie->audioTempBuffer;
short* tempBuf = (short*)aMovie->audioTempBuffer;
for (int i = 0; i < aMovie->audioFrame->nb_samples; i++) {
for (int j = 0; j < aMovie->alNumChannels; j++) {
float* srcBuf = (float*)aMovie->audioFrame->extended_data[j];
float val = srcBuf[i] * 32768;
if (val > 32767)
val = 32767;
if (val < -32768)
val = -32768;
tempBuf[(i*aMovie->alNumChannels)+j] = (short)val;
}
}
} break;
} }
} break;
}
alSourceStop(aMovie->alSource); alBufferData(alBuffer, aMovie->alFormat, data, dataSize - 16, aMovie->alSampleRate);
AL_CHECK();
alBufferData(alBuffer, aMovie->alFormat, data, dataSize - 16, aMovie->alSampleRate);
AL_CHECK();
alSourceQueueBuffers(aMovie->alSource, 1, &alBuffer); alSourceQueueBuffers(aMovie->alSource, 1, &alBuffer);
AL_CHECK(); AL_CHECK();
float vx, vy, vz; float vx, vy, vz;
alGetListener3f(AL_VELOCITY, &vx, &vy, &vz); alGetListener3f(AL_VELOCITY, &vx, &vy, &vz);
alSource3f(aMovie->alSource, AL_VELOCITY, vx, vy, vz); alSource3f(aMovie->alSource, AL_VELOCITY, vx, vy, vz);
int state = 0;
alGetSourcei(aMovie->alSource, AL_SOURCE_STATE, &state);
if (state != AL_PLAYING)
alSourcePlay(aMovie->alSource); alSourcePlay(aMovie->alSource);
aMovie->alNumFreeBuffers--; aMovie->alNumFreeBuffers--;
aMovie->alFreeBuffers[aMovie->alNumFreeBuffers] = 0; aMovie->alFreeBuffers[aMovie->alNumFreeBuffers] = 0;
}
return 1; return 1;
} }
@ -281,7 +302,7 @@ static int ReadPacket(struct binkMovie* aMovie)
} }
// Send the (possibly buffered) packet to decoder. // Send the (possibly buffered) packet to decoder.
int ret = 0; int ret = AVERROR(EAGAIN);
if (aMovie->packet.stream_index == aMovie->videoStreamIndex) if (aMovie->packet.stream_index == aMovie->videoStreamIndex)
ret = avcodec_send_packet(aMovie->videoCodecContext, &aMovie->packet); ret = avcodec_send_packet(aMovie->videoCodecContext, &aMovie->packet);
else if (aMovie->packet.stream_index == aMovie->audioStreamIndex) else if (aMovie->packet.stream_index == aMovie->audioStreamIndex)
@ -335,8 +356,9 @@ static int BinkStartMovie(struct binkMovie* aMovie, const char* aFilename,
aMovie->videoCodecContext = context; aMovie->videoCodecContext = context;
aMovie->videoStreamIndex = i; aMovie->videoStreamIndex = i;
aMovie->videoFrame = av_frame_alloc(); aMovie->videoFrame = av_frame_alloc();
aMovie->videoFrameDuration = aMovie->frameDuration =
1000.0f * (float)stream->time_base.num / (float)stream->time_base.den; 1000.0f * (float)stream->time_base.num / (float)stream->time_base.den;
aMovie->timeStart = SDL_GetTicks();
numStreams++; numStreams++;
} else if (aMovie->audioStreamIndex < 0 && context->codec_type == AVMEDIA_TYPE_AUDIO) { } else if (aMovie->audioStreamIndex < 0 && context->codec_type == AVMEDIA_TYPE_AUDIO) {
aMovie->audioCodecContext = context; aMovie->audioCodecContext = context;
@ -352,8 +374,8 @@ static int BinkStartMovie(struct binkMovie* aMovie, const char* aFilename,
alSourcef(aMovie->alSource, AL_PITCH, 1.0); alSourcef(aMovie->alSource, AL_PITCH, 1.0);
alSourcef(aMovie->alSource, AL_GAIN, 1.0); alSourcef(aMovie->alSource, AL_GAIN, 1.0);
alGenBuffers(FRAMEQUEUESIZE, aMovie->alBuffers); alGenBuffers(AUDIO_FRAMES, aMovie->alBuffers);
aMovie->alNumFreeBuffers = FRAMEQUEUESIZE; aMovie->alNumFreeBuffers = AUDIO_FRAMES;
for (int i = 0; i < aMovie->alNumFreeBuffers; i++) for (int i = 0; i < aMovie->alNumFreeBuffers; i++)
aMovie->alFreeBuffers[i] = aMovie->alBuffers[i]; aMovie->alFreeBuffers[i] = aMovie->alBuffers[i];
@ -369,7 +391,7 @@ static int BinkStartMovie(struct binkMovie* aMovie, const char* aFilename,
} }
if (!aFmvFlag) { if (!aFmvFlag) {
for (int i = 0; i < (FRAMEQUEUESIZE-1) * numStreams; i++) for (int i = 0; i < (AUDIO_FRAMES + VIDEO_FRAMES) / 2; i++)
ReadPacket(aMovie); ReadPacket(aMovie);
} }
@ -381,29 +403,18 @@ static int BinkUpdateMovie(struct binkMovie* aMovie)
if(!aMovie->avContext) if(!aMovie->avContext)
return 0; return 0;
const int t = SDL_GetTicks();
const int eof = !ReadPacket(aMovie); const int eof = !ReadPacket(aMovie);
int playing = 0; int playing = 0;
if (aMovie->videoStreamIndex >= 0) { if (aMovie->videoStreamIndex >= 0 && aMovie->renderedFrames < VIDEO_FRAMES)
if (aMovie->videoFrame->pts == 0) if (avcodec_receive_frame(aMovie->videoCodecContext, aMovie->videoFrame) == 0)
aMovie->timeStart = t - aMovie->videoFrameDuration;
if (t - aMovie->timeStart >= aMovie->videoFrame->pts * aMovie->videoFrameDuration)
playing += DecodeVideoFrame(aMovie); playing += DecodeVideoFrame(aMovie);
}
if (aMovie->audioStreamIndex >= 0) { if (aMovie->audioStreamIndex >= 0 && aMovie->alNumFreeBuffers > 0)
int processedBuffers = 0; if (avcodec_receive_frame(aMovie->audioCodecContext, aMovie->audioFrame) == 0)
alGetSourcei(aMovie->alSource, AL_BUFFERS_PROCESSED, &processedBuffers); playing += DecodeAudioFrame(aMovie);
while (!aMovie->alInited || aMovie->alNumFreeBuffers>0 || processedBuffers>0) {
if (DecodeAudioFrame(aMovie))
playing += 1;
else
break;
}
}
return !eof || playing; return eof && !playing ? -1 : playing;
} }
void PlayBinkedFMV(char* filenamePtr, int volume) void PlayBinkedFMV(char* filenamePtr, int volume)
@ -411,10 +422,15 @@ void PlayBinkedFMV(char* filenamePtr, int volume)
struct binkMovie movie; struct binkMovie movie;
if (BinkStartMovie(&movie, filenamePtr, FALSE, FALSE, FALSE)) { if (BinkStartMovie(&movie, filenamePtr, FALSE, FALSE, FALSE)) {
alSourcef(movie.alSource, AL_GAIN, PlatVolumeToGain(volume)); alSourcef(movie.alSource, AL_GAIN, PlatVolumeToGain(volume));
while (BinkUpdateMovie(&movie)) { int updated = 0;
BinkRenderMovie(&movie); while ((updated = BinkUpdateMovie(&movie)) >= 0) {
ProcessAudio(&movie);
int dt = BinkRenderMovie(&movie);
FlipBuffers(); FlipBuffers();
SDL_Delay(4); // don’t just burn it
// don’t just burn it
if (!updated && dt > 0)
SDL_Delay(dt);
} }
BinkReleaseMovie(&movie); BinkReleaseMovie(&movie);
} }
@ -433,7 +449,8 @@ void StartMenuBackgroundBink()
int PlayMenuBackgroundBink() int PlayMenuBackgroundBink()
{ {
ClearScreenToBlack(); ClearScreenToBlack();
if (BinkUpdateMovie(&menuBackgroundMovie)) { if (BinkUpdateMovie(&menuBackgroundMovie) >= 0) {
ProcessAudio(&menuBackgroundMovie);
BinkRenderMovie(&menuBackgroundMovie); BinkRenderMovie(&menuBackgroundMovie);
return 1; return 1;
} }
@ -468,7 +485,7 @@ int PlayMusicBink(int volume)
return 1; return 1;
alSourcef(musicMovie.alSource, AL_GAIN, PlatVolumeToGain(volume)); alSourcef(musicMovie.alSource, AL_GAIN, PlatVolumeToGain(volume));
for (int i = 0; i < musicMovie.avContext->nb_streams * FRAMEQUEUESIZE; i++) { for (int i = 0; i < musicMovie.avContext->nb_streams * AUDIO_FRAMES; i++) {
int processedBuffers = 0; int processedBuffers = 0;
alGetSourcei(musicMovie.alSource, AL_BUFFERS_PROCESSED, &processedBuffers); alGetSourcei(musicMovie.alSource, AL_BUFFERS_PROCESSED, &processedBuffers);
if (processedBuffers + musicMovie.alNumFreeBuffers > 0) if (processedBuffers + musicMovie.alNumFreeBuffers > 0)
@ -526,10 +543,11 @@ void CloseBinkFMV(FMVHandle aFmvHandle)
char* GetBinkFMVImage(FMVHandle aFmvHandle) char* GetBinkFMVImage(FMVHandle aFmvHandle)
{ {
if (aFmvHandle == 0) if (aFmvHandle == 0)
return 0; return NULL;
struct binkMovie* movie = (struct binkMovie*)aFmvHandle; struct binkMovie* movie = (struct binkMovie*)aFmvHandle;
if(!movie->videoScaleContext) if (!movie->videoScaleContext)
return 0; return NULL;
return movie->videoScalePicture[0]; return movie->renderedFrames ?
movie->videoFrames[movie->currentFrame%VIDEO_FRAMES][0] : NULL;
} }