Add intermediate frame in video buffer

There were only two frames simultaneously:
 - one used by the decoder;
 - one used by the renderer.

When the decoder finished decoding a frame, it swapped it with the
rendering frame.

Adding a third frame provides several benefits:
 - the decoder do not have to wait for the renderer to release the
   mutex;
 - it simplifies the video_buffer API;
 - it makes the rendering frame valid until the next call to
   video_buffer_take_rendering_frame(), which will be useful for
   swscaling on window resize.
This commit is contained in:
Romain Vimont 2021-02-01 09:38:11 +01:00
parent c53bd4d8b6
commit c0c4ba7009
3 changed files with 70 additions and 32 deletions

View file

@ -453,15 +453,12 @@ update_texture(struct screen *screen, const AVFrame *frame) {
bool
screen_update_frame(struct screen *screen, struct video_buffer *vb) {
sc_mutex_lock(&vb->mutex);
const AVFrame *frame = video_buffer_consume_rendered_frame(vb);
const AVFrame *frame = video_buffer_take_rendering_frame(vb);
struct size new_frame_size = {frame->width, frame->height};
if (!prepare_for_frame(screen, new_frame_size)) {
sc_mutex_unlock(&vb->mutex);
return false;
}
update_texture(screen, frame);
sc_mutex_unlock(&vb->mutex);
screen_render(screen, false);
return true;

View file

@ -16,19 +16,24 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
goto error_0;
}
vb->pending_frame = av_frame_alloc();
if (!vb->pending_frame) {
goto error_1;
}
vb->rendering_frame = av_frame_alloc();
if (!vb->rendering_frame) {
goto error_1;
goto error_2;
}
bool ok = sc_mutex_init(&vb->mutex);
if (!ok) {
goto error_2;
goto error_3;
}
vb->render_expired_frames = render_expired_frames;
if (render_expired_frames) {
ok = sc_cond_init(&vb->rendering_frame_consumed_cond);
ok = sc_cond_init(&vb->pending_frame_consumed_cond);
if (!ok) {
sc_mutex_destroy(&vb->mutex);
goto error_2;
@ -40,12 +45,14 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
// there is initially no rendering frame, so consider it has already been
// consumed
vb->rendering_frame_consumed = true;
vb->pending_frame_consumed = true;
return true;
error_2:
error_3:
av_frame_free(&vb->rendering_frame);
error_2:
av_frame_free(&vb->pending_frame);
error_1:
av_frame_free(&vb->decoding_frame);
error_0:
@ -55,19 +62,28 @@ error_0:
void
video_buffer_destroy(struct video_buffer *vb) {
if (vb->render_expired_frames) {
sc_cond_destroy(&vb->rendering_frame_consumed_cond);
sc_cond_destroy(&vb->pending_frame_consumed_cond);
}
sc_mutex_destroy(&vb->mutex);
av_frame_free(&vb->rendering_frame);
av_frame_free(&vb->pending_frame);
av_frame_free(&vb->decoding_frame);
}
static void
video_buffer_swap_frames(struct video_buffer *vb) {
video_buffer_swap_decoding_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex);
AVFrame *tmp = vb->decoding_frame;
vb->decoding_frame = vb->rendering_frame;
vb->rendering_frame = tmp;
vb->decoding_frame = vb->pending_frame;
vb->pending_frame = tmp;
}
static void
video_buffer_swap_rendering_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex);
AVFrame *tmp = vb->rendering_frame;
vb->rendering_frame = vb->pending_frame;
vb->pending_frame = tmp;
}
void
@ -76,31 +92,38 @@ video_buffer_offer_decoded_frame(struct video_buffer *vb,
sc_mutex_lock(&vb->mutex);
if (vb->render_expired_frames) {
// wait for the current (expired) frame to be consumed
while (!vb->rendering_frame_consumed && !vb->interrupted) {
sc_cond_wait(&vb->rendering_frame_consumed_cond, &vb->mutex);
while (!vb->pending_frame_consumed && !vb->interrupted) {
sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex);
}
} else if (!vb->rendering_frame_consumed) {
} else if (!vb->pending_frame_consumed) {
fps_counter_add_skipped_frame(vb->fps_counter);
}
video_buffer_swap_frames(vb);
video_buffer_swap_decoding_frame(vb);
*previous_frame_skipped = !vb->rendering_frame_consumed;
vb->rendering_frame_consumed = false;
*previous_frame_skipped = !vb->pending_frame_consumed;
vb->pending_frame_consumed = false;
sc_mutex_unlock(&vb->mutex);
}
const AVFrame *
video_buffer_consume_rendered_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex);
assert(!vb->rendering_frame_consumed);
vb->rendering_frame_consumed = true;
video_buffer_take_rendering_frame(struct video_buffer *vb) {
sc_mutex_lock(&vb->mutex);
assert(!vb->pending_frame_consumed);
vb->pending_frame_consumed = true;
fps_counter_add_rendered_frame(vb->fps_counter);
video_buffer_swap_rendering_frame(vb);
if (vb->render_expired_frames) {
// unblock video_buffer_offer_decoded_frame()
sc_cond_signal(&vb->rendering_frame_consumed_cond);
sc_cond_signal(&vb->pending_frame_consumed_cond);
}
sc_mutex_unlock(&vb->mutex);
// rendering_frame is only written from this thread, no need to lock
return vb->rendering_frame;
}
@ -111,6 +134,6 @@ video_buffer_interrupt(struct video_buffer *vb) {
vb->interrupted = true;
sc_mutex_unlock(&vb->mutex);
// wake up blocking wait
sc_cond_signal(&vb->rendering_frame_consumed_cond);
sc_cond_signal(&vb->pending_frame_consumed_cond);
}
}

View file

@ -11,14 +11,35 @@
// forward declarations
typedef struct AVFrame AVFrame;
/**
* There are 3 frames in memory:
* - one frame is held by the decoder (decoding_frame)
* - one frame is held by the renderer (rendering_frame)
* - one frame is shared between the decoder and the renderer (pending_frame)
*
* The decoder decodes a packet into the decoding_frame (it may takes time).
*
* Once the frame is decoded, it calls video_buffer_offer_decoded_frame(),
* which swaps the decoding and pending frames.
*
* When the renderer is notified that a new frame is available, it calls
* video_buffer_take_rendering_frame() to retrieve it, which swaps the pending
* and rendering frames. The frame is valid until the next call, without
* blocking the decoder.
*/
struct video_buffer {
AVFrame *decoding_frame;
AVFrame *pending_frame;
AVFrame *rendering_frame;
sc_mutex mutex;
bool render_expired_frames;
bool interrupted;
sc_cond rendering_frame_consumed_cond;
bool rendering_frame_consumed;
sc_cond pending_frame_consumed_cond;
bool pending_frame_consumed;
struct fps_counter *fps_counter;
};
@ -30,18 +51,15 @@ void
video_buffer_destroy(struct video_buffer *vb);
// set the decoded frame as ready for rendering
// this function locks vb->mutex during its execution
// the output flag is set to report whether the previous frame has been skipped
void
video_buffer_offer_decoded_frame(struct video_buffer *vb,
bool *previous_frame_skipped);
// mark the rendering frame as consumed and return it
// MUST be called with vb->mutex locked!!!
// the caller is expected to render the returned frame to some texture before
// unlocking vb->mutex
// the frame is valid until the next call to this function
const AVFrame *
video_buffer_consume_rendered_frame(struct video_buffer *vb);
video_buffer_take_rendering_frame(struct video_buffer *vb);
// wake up and avoid any blocking call
void