From c0c4ba700904d54f087f6cc5a2e5821124ff42db Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Mon, 1 Feb 2021 09:38:11 +0100 Subject: [PATCH] Add intermediate frame in video buffer There were only two frames simultaneously: - one used by the decoder; - one used by the renderer. When the decoder finished decoding a frame, it swapped it with the rendering frame. Adding a third frame provides several benefits: - the decoder do not have to wait for the renderer to release the mutex; - it simplifies the video_buffer API; - it makes the rendering frame valid until the next call to video_buffer_take_rendering_frame(), which will be useful for swscaling on window resize. --- app/src/screen.c | 5 +--- app/src/video_buffer.c | 65 ++++++++++++++++++++++++++++-------------- app/src/video_buffer.h | 32 ++++++++++++++++----- 3 files changed, 70 insertions(+), 32 deletions(-) diff --git a/app/src/screen.c b/app/src/screen.c index 0f8a2226..5ec416f9 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -453,15 +453,12 @@ update_texture(struct screen *screen, const AVFrame *frame) { bool screen_update_frame(struct screen *screen, struct video_buffer *vb) { - sc_mutex_lock(&vb->mutex); - const AVFrame *frame = video_buffer_consume_rendered_frame(vb); + const AVFrame *frame = video_buffer_take_rendering_frame(vb); struct size new_frame_size = {frame->width, frame->height}; if (!prepare_for_frame(screen, new_frame_size)) { - sc_mutex_unlock(&vb->mutex); return false; } update_texture(screen, frame); - sc_mutex_unlock(&vb->mutex); screen_render(screen, false); return true; diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c index 95cfd755..4c8dd5da 100644 --- a/app/src/video_buffer.c +++ b/app/src/video_buffer.c @@ -16,19 +16,24 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter, goto error_0; } + vb->pending_frame = av_frame_alloc(); + if (!vb->pending_frame) { + goto error_1; + } + vb->rendering_frame = av_frame_alloc(); if (!vb->rendering_frame) { - goto error_1; + goto error_2; } bool ok = sc_mutex_init(&vb->mutex); if (!ok) { - goto error_2; + goto error_3; } vb->render_expired_frames = render_expired_frames; if (render_expired_frames) { - ok = sc_cond_init(&vb->rendering_frame_consumed_cond); + ok = sc_cond_init(&vb->pending_frame_consumed_cond); if (!ok) { sc_mutex_destroy(&vb->mutex); goto error_2; @@ -40,12 +45,14 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter, // there is initially no rendering frame, so consider it has already been // consumed - vb->rendering_frame_consumed = true; + vb->pending_frame_consumed = true; return true; -error_2: +error_3: av_frame_free(&vb->rendering_frame); +error_2: + av_frame_free(&vb->pending_frame); error_1: av_frame_free(&vb->decoding_frame); error_0: @@ -55,19 +62,28 @@ error_0: void video_buffer_destroy(struct video_buffer *vb) { if (vb->render_expired_frames) { - sc_cond_destroy(&vb->rendering_frame_consumed_cond); + sc_cond_destroy(&vb->pending_frame_consumed_cond); } sc_mutex_destroy(&vb->mutex); av_frame_free(&vb->rendering_frame); + av_frame_free(&vb->pending_frame); av_frame_free(&vb->decoding_frame); } static void -video_buffer_swap_frames(struct video_buffer *vb) { +video_buffer_swap_decoding_frame(struct video_buffer *vb) { sc_mutex_assert(&vb->mutex); AVFrame *tmp = vb->decoding_frame; - vb->decoding_frame = vb->rendering_frame; - vb->rendering_frame = tmp; + vb->decoding_frame = vb->pending_frame; + vb->pending_frame = tmp; +} + +static void +video_buffer_swap_rendering_frame(struct video_buffer *vb) { + sc_mutex_assert(&vb->mutex); + AVFrame *tmp = vb->rendering_frame; + vb->rendering_frame = vb->pending_frame; + vb->pending_frame = tmp; } void @@ -76,31 +92,38 @@ video_buffer_offer_decoded_frame(struct video_buffer *vb, sc_mutex_lock(&vb->mutex); if (vb->render_expired_frames) { // wait for the current (expired) frame to be consumed - while (!vb->rendering_frame_consumed && !vb->interrupted) { - sc_cond_wait(&vb->rendering_frame_consumed_cond, &vb->mutex); + while (!vb->pending_frame_consumed && !vb->interrupted) { + sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex); } - } else if (!vb->rendering_frame_consumed) { + } else if (!vb->pending_frame_consumed) { fps_counter_add_skipped_frame(vb->fps_counter); } - video_buffer_swap_frames(vb); + video_buffer_swap_decoding_frame(vb); - *previous_frame_skipped = !vb->rendering_frame_consumed; - vb->rendering_frame_consumed = false; + *previous_frame_skipped = !vb->pending_frame_consumed; + vb->pending_frame_consumed = false; sc_mutex_unlock(&vb->mutex); } const AVFrame * -video_buffer_consume_rendered_frame(struct video_buffer *vb) { - sc_mutex_assert(&vb->mutex); - assert(!vb->rendering_frame_consumed); - vb->rendering_frame_consumed = true; +video_buffer_take_rendering_frame(struct video_buffer *vb) { + sc_mutex_lock(&vb->mutex); + assert(!vb->pending_frame_consumed); + vb->pending_frame_consumed = true; + fps_counter_add_rendered_frame(vb->fps_counter); + + video_buffer_swap_rendering_frame(vb); + if (vb->render_expired_frames) { // unblock video_buffer_offer_decoded_frame() - sc_cond_signal(&vb->rendering_frame_consumed_cond); + sc_cond_signal(&vb->pending_frame_consumed_cond); } + sc_mutex_unlock(&vb->mutex); + + // rendering_frame is only written from this thread, no need to lock return vb->rendering_frame; } @@ -111,6 +134,6 @@ video_buffer_interrupt(struct video_buffer *vb) { vb->interrupted = true; sc_mutex_unlock(&vb->mutex); // wake up blocking wait - sc_cond_signal(&vb->rendering_frame_consumed_cond); + sc_cond_signal(&vb->pending_frame_consumed_cond); } } diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h index fbab046b..3e52caa7 100644 --- a/app/src/video_buffer.h +++ b/app/src/video_buffer.h @@ -11,14 +11,35 @@ // forward declarations typedef struct AVFrame AVFrame; +/** + * There are 3 frames in memory: + * - one frame is held by the decoder (decoding_frame) + * - one frame is held by the renderer (rendering_frame) + * - one frame is shared between the decoder and the renderer (pending_frame) + * + * The decoder decodes a packet into the decoding_frame (it may takes time). + * + * Once the frame is decoded, it calls video_buffer_offer_decoded_frame(), + * which swaps the decoding and pending frames. + * + * When the renderer is notified that a new frame is available, it calls + * video_buffer_take_rendering_frame() to retrieve it, which swaps the pending + * and rendering frames. The frame is valid until the next call, without + * blocking the decoder. + */ + struct video_buffer { AVFrame *decoding_frame; + AVFrame *pending_frame; AVFrame *rendering_frame; + sc_mutex mutex; bool render_expired_frames; bool interrupted; - sc_cond rendering_frame_consumed_cond; - bool rendering_frame_consumed; + + sc_cond pending_frame_consumed_cond; + bool pending_frame_consumed; + struct fps_counter *fps_counter; }; @@ -30,18 +51,15 @@ void video_buffer_destroy(struct video_buffer *vb); // set the decoded frame as ready for rendering -// this function locks vb->mutex during its execution // the output flag is set to report whether the previous frame has been skipped void video_buffer_offer_decoded_frame(struct video_buffer *vb, bool *previous_frame_skipped); // mark the rendering frame as consumed and return it -// MUST be called with vb->mutex locked!!! -// the caller is expected to render the returned frame to some texture before -// unlocking vb->mutex +// the frame is valid until the next call to this function const AVFrame * -video_buffer_consume_rendered_frame(struct video_buffer *vb); +video_buffer_take_rendering_frame(struct video_buffer *vb); // wake up and avoid any blocking call void