Add intermediate frame in video buffer

There were only two frames simultaneously:
 - one used by the decoder;
 - one used by the renderer.

When the decoder finished decoding a frame, it swapped it with the
rendering frame.

Adding a third frame provides several benefits:
 - the decoder do not have to wait for the renderer to release the
   mutex;
 - it simplifies the video_buffer API;
 - it makes the rendering frame valid until the next call to
   video_buffer_take_rendering_frame(), which will be useful for
   swscaling on window resize.
This commit is contained in:
Romain Vimont 2021-02-01 09:38:11 +01:00
parent c53bd4d8b6
commit c0c4ba7009
3 changed files with 70 additions and 32 deletions

View file

@ -453,15 +453,12 @@ update_texture(struct screen *screen, const AVFrame *frame) {
bool bool
screen_update_frame(struct screen *screen, struct video_buffer *vb) { screen_update_frame(struct screen *screen, struct video_buffer *vb) {
sc_mutex_lock(&vb->mutex); const AVFrame *frame = video_buffer_take_rendering_frame(vb);
const AVFrame *frame = video_buffer_consume_rendered_frame(vb);
struct size new_frame_size = {frame->width, frame->height}; struct size new_frame_size = {frame->width, frame->height};
if (!prepare_for_frame(screen, new_frame_size)) { if (!prepare_for_frame(screen, new_frame_size)) {
sc_mutex_unlock(&vb->mutex);
return false; return false;
} }
update_texture(screen, frame); update_texture(screen, frame);
sc_mutex_unlock(&vb->mutex);
screen_render(screen, false); screen_render(screen, false);
return true; return true;

View file

@ -16,19 +16,24 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
goto error_0; goto error_0;
} }
vb->pending_frame = av_frame_alloc();
if (!vb->pending_frame) {
goto error_1;
}
vb->rendering_frame = av_frame_alloc(); vb->rendering_frame = av_frame_alloc();
if (!vb->rendering_frame) { if (!vb->rendering_frame) {
goto error_1; goto error_2;
} }
bool ok = sc_mutex_init(&vb->mutex); bool ok = sc_mutex_init(&vb->mutex);
if (!ok) { if (!ok) {
goto error_2; goto error_3;
} }
vb->render_expired_frames = render_expired_frames; vb->render_expired_frames = render_expired_frames;
if (render_expired_frames) { if (render_expired_frames) {
ok = sc_cond_init(&vb->rendering_frame_consumed_cond); ok = sc_cond_init(&vb->pending_frame_consumed_cond);
if (!ok) { if (!ok) {
sc_mutex_destroy(&vb->mutex); sc_mutex_destroy(&vb->mutex);
goto error_2; goto error_2;
@ -40,12 +45,14 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
// there is initially no rendering frame, so consider it has already been // there is initially no rendering frame, so consider it has already been
// consumed // consumed
vb->rendering_frame_consumed = true; vb->pending_frame_consumed = true;
return true; return true;
error_2: error_3:
av_frame_free(&vb->rendering_frame); av_frame_free(&vb->rendering_frame);
error_2:
av_frame_free(&vb->pending_frame);
error_1: error_1:
av_frame_free(&vb->decoding_frame); av_frame_free(&vb->decoding_frame);
error_0: error_0:
@ -55,19 +62,28 @@ error_0:
void void
video_buffer_destroy(struct video_buffer *vb) { video_buffer_destroy(struct video_buffer *vb) {
if (vb->render_expired_frames) { if (vb->render_expired_frames) {
sc_cond_destroy(&vb->rendering_frame_consumed_cond); sc_cond_destroy(&vb->pending_frame_consumed_cond);
} }
sc_mutex_destroy(&vb->mutex); sc_mutex_destroy(&vb->mutex);
av_frame_free(&vb->rendering_frame); av_frame_free(&vb->rendering_frame);
av_frame_free(&vb->pending_frame);
av_frame_free(&vb->decoding_frame); av_frame_free(&vb->decoding_frame);
} }
static void static void
video_buffer_swap_frames(struct video_buffer *vb) { video_buffer_swap_decoding_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex); sc_mutex_assert(&vb->mutex);
AVFrame *tmp = vb->decoding_frame; AVFrame *tmp = vb->decoding_frame;
vb->decoding_frame = vb->rendering_frame; vb->decoding_frame = vb->pending_frame;
vb->rendering_frame = tmp; vb->pending_frame = tmp;
}
static void
video_buffer_swap_rendering_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex);
AVFrame *tmp = vb->rendering_frame;
vb->rendering_frame = vb->pending_frame;
vb->pending_frame = tmp;
} }
void void
@ -76,31 +92,38 @@ video_buffer_offer_decoded_frame(struct video_buffer *vb,
sc_mutex_lock(&vb->mutex); sc_mutex_lock(&vb->mutex);
if (vb->render_expired_frames) { if (vb->render_expired_frames) {
// wait for the current (expired) frame to be consumed // wait for the current (expired) frame to be consumed
while (!vb->rendering_frame_consumed && !vb->interrupted) { while (!vb->pending_frame_consumed && !vb->interrupted) {
sc_cond_wait(&vb->rendering_frame_consumed_cond, &vb->mutex); sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex);
} }
} else if (!vb->rendering_frame_consumed) { } else if (!vb->pending_frame_consumed) {
fps_counter_add_skipped_frame(vb->fps_counter); fps_counter_add_skipped_frame(vb->fps_counter);
} }
video_buffer_swap_frames(vb); video_buffer_swap_decoding_frame(vb);
*previous_frame_skipped = !vb->rendering_frame_consumed; *previous_frame_skipped = !vb->pending_frame_consumed;
vb->rendering_frame_consumed = false; vb->pending_frame_consumed = false;
sc_mutex_unlock(&vb->mutex); sc_mutex_unlock(&vb->mutex);
} }
const AVFrame * const AVFrame *
video_buffer_consume_rendered_frame(struct video_buffer *vb) { video_buffer_take_rendering_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex); sc_mutex_lock(&vb->mutex);
assert(!vb->rendering_frame_consumed); assert(!vb->pending_frame_consumed);
vb->rendering_frame_consumed = true; vb->pending_frame_consumed = true;
fps_counter_add_rendered_frame(vb->fps_counter); fps_counter_add_rendered_frame(vb->fps_counter);
video_buffer_swap_rendering_frame(vb);
if (vb->render_expired_frames) { if (vb->render_expired_frames) {
// unblock video_buffer_offer_decoded_frame() // unblock video_buffer_offer_decoded_frame()
sc_cond_signal(&vb->rendering_frame_consumed_cond); sc_cond_signal(&vb->pending_frame_consumed_cond);
} }
sc_mutex_unlock(&vb->mutex);
// rendering_frame is only written from this thread, no need to lock
return vb->rendering_frame; return vb->rendering_frame;
} }
@ -111,6 +134,6 @@ video_buffer_interrupt(struct video_buffer *vb) {
vb->interrupted = true; vb->interrupted = true;
sc_mutex_unlock(&vb->mutex); sc_mutex_unlock(&vb->mutex);
// wake up blocking wait // wake up blocking wait
sc_cond_signal(&vb->rendering_frame_consumed_cond); sc_cond_signal(&vb->pending_frame_consumed_cond);
} }
} }

View file

@ -11,14 +11,35 @@
// forward declarations // forward declarations
typedef struct AVFrame AVFrame; typedef struct AVFrame AVFrame;
/**
* There are 3 frames in memory:
* - one frame is held by the decoder (decoding_frame)
* - one frame is held by the renderer (rendering_frame)
* - one frame is shared between the decoder and the renderer (pending_frame)
*
* The decoder decodes a packet into the decoding_frame (it may takes time).
*
* Once the frame is decoded, it calls video_buffer_offer_decoded_frame(),
* which swaps the decoding and pending frames.
*
* When the renderer is notified that a new frame is available, it calls
* video_buffer_take_rendering_frame() to retrieve it, which swaps the pending
* and rendering frames. The frame is valid until the next call, without
* blocking the decoder.
*/
struct video_buffer { struct video_buffer {
AVFrame *decoding_frame; AVFrame *decoding_frame;
AVFrame *pending_frame;
AVFrame *rendering_frame; AVFrame *rendering_frame;
sc_mutex mutex; sc_mutex mutex;
bool render_expired_frames; bool render_expired_frames;
bool interrupted; bool interrupted;
sc_cond rendering_frame_consumed_cond;
bool rendering_frame_consumed; sc_cond pending_frame_consumed_cond;
bool pending_frame_consumed;
struct fps_counter *fps_counter; struct fps_counter *fps_counter;
}; };
@ -30,18 +51,15 @@ void
video_buffer_destroy(struct video_buffer *vb); video_buffer_destroy(struct video_buffer *vb);
// set the decoded frame as ready for rendering // set the decoded frame as ready for rendering
// this function locks vb->mutex during its execution
// the output flag is set to report whether the previous frame has been skipped // the output flag is set to report whether the previous frame has been skipped
void void
video_buffer_offer_decoded_frame(struct video_buffer *vb, video_buffer_offer_decoded_frame(struct video_buffer *vb,
bool *previous_frame_skipped); bool *previous_frame_skipped);
// mark the rendering frame as consumed and return it // mark the rendering frame as consumed and return it
// MUST be called with vb->mutex locked!!! // the frame is valid until the next call to this function
// the caller is expected to render the returned frame to some texture before
// unlocking vb->mutex
const AVFrame * const AVFrame *
video_buffer_consume_rendered_frame(struct video_buffer *vb); video_buffer_take_rendering_frame(struct video_buffer *vb);
// wake up and avoid any blocking call // wake up and avoid any blocking call
void void