Display and control your Android device
Add intermediate frame in video buffer
Romain Vimontc0c4ba700904
diff --git a/app/src/screen.c b/app/src/screen.cindex 0f8a2226..5ec416f9 100644--- a/app/src/screen.c+++ b/app/src/screen.c@@ -453,15 +453,12 @@ update_texture(struct screen *screen, const AVFrame *frame) {boolscreen_update_frame(struct screen *screen, struct video_buffer *vb) {- sc_mutex_lock(&vb->mutex);- const AVFrame *frame = video_buffer_consume_rendered_frame(vb);+ const AVFrame *frame = video_buffer_take_rendering_frame(vb);struct size new_frame_size = {frame->width, frame->height};if (!prepare_for_frame(screen, new_frame_size)) {- sc_mutex_unlock(&vb->mutex);return false;}update_texture(screen, frame);- sc_mutex_unlock(&vb->mutex);screen_render(screen, false);return true;diff --git a/app/src/video_buffer.c b/app/src/video_buffer.cindex 95cfd755..4c8dd5da 100644--- a/app/src/video_buffer.c+++ b/app/src/video_buffer.c@@ -16,19 +16,24 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,goto error_0;}+ vb->pending_frame = av_frame_alloc();+ if (!vb->pending_frame) {+ goto error_1;+ }+vb->rendering_frame = av_frame_alloc();if (!vb->rendering_frame) {- goto error_1;+ goto error_2;}bool ok = sc_mutex_init(&vb->mutex);if (!ok) {- goto error_2;+ goto error_3;}vb->render_expired_frames = render_expired_frames;if (render_expired_frames) {- ok = sc_cond_init(&vb->rendering_frame_consumed_cond);+ ok = sc_cond_init(&vb->pending_frame_consumed_cond);if (!ok) {sc_mutex_destroy(&vb->mutex);goto error_2;@@ -40,12 +45,14 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,// there is initially no rendering frame, so consider it has already been// consumed- vb->rendering_frame_consumed = true;+ vb->pending_frame_consumed = true;return true;-error_2:+error_3:av_frame_free(&vb->rendering_frame);+error_2:+ av_frame_free(&vb->pending_frame);error_1:av_frame_free(&vb->decoding_frame);error_0:@@ -55,19 +62,28 @@ error_0:voidvideo_buffer_destroy(struct video_buffer *vb) {if (vb->render_expired_frames) {- sc_cond_destroy(&vb->rendering_frame_consumed_cond);+ sc_cond_destroy(&vb->pending_frame_consumed_cond);}sc_mutex_destroy(&vb->mutex);av_frame_free(&vb->rendering_frame);+ av_frame_free(&vb->pending_frame);av_frame_free(&vb->decoding_frame);}static void-video_buffer_swap_frames(struct video_buffer *vb) {+video_buffer_swap_decoding_frame(struct video_buffer *vb) {sc_mutex_assert(&vb->mutex);AVFrame *tmp = vb->decoding_frame;- vb->decoding_frame = vb->rendering_frame;- vb->rendering_frame = tmp;+ vb->decoding_frame = vb->pending_frame;+ vb->pending_frame = tmp;+}++static void+video_buffer_swap_rendering_frame(struct video_buffer *vb) {+ sc_mutex_assert(&vb->mutex);+ AVFrame *tmp = vb->rendering_frame;+ vb->rendering_frame = vb->pending_frame;+ vb->pending_frame = tmp;}void@@ -76,31 +92,38 @@ video_buffer_offer_decoded_frame(struct video_buffer *vb,sc_mutex_lock(&vb->mutex);if (vb->render_expired_frames) {// wait for the current (expired) frame to be consumed- while (!vb->rendering_frame_consumed && !vb->interrupted) {- sc_cond_wait(&vb->rendering_frame_consumed_cond, &vb->mutex);+ while (!vb->pending_frame_consumed && !vb->interrupted) {+ sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex);}- } else if (!vb->rendering_frame_consumed) {+ } else if (!vb->pending_frame_consumed) {fps_counter_add_skipped_frame(vb->fps_counter);}- video_buffer_swap_frames(vb);+ video_buffer_swap_decoding_frame(vb);- *previous_frame_skipped = !vb->rendering_frame_consumed;- vb->rendering_frame_consumed = false;+ *previous_frame_skipped = !vb->pending_frame_consumed;+ vb->pending_frame_consumed = false;sc_mutex_unlock(&vb->mutex);}const AVFrame *-video_buffer_consume_rendered_frame(struct video_buffer *vb) {- sc_mutex_assert(&vb->mutex);- assert(!vb->rendering_frame_consumed);- vb->rendering_frame_consumed = true;+video_buffer_take_rendering_frame(struct video_buffer *vb) {+ sc_mutex_lock(&vb->mutex);+ assert(!vb->pending_frame_consumed);+ vb->pending_frame_consumed = true;+fps_counter_add_rendered_frame(vb->fps_counter);++ video_buffer_swap_rendering_frame(vb);+if (vb->render_expired_frames) {// unblock video_buffer_offer_decoded_frame()- sc_cond_signal(&vb->rendering_frame_consumed_cond);+ sc_cond_signal(&vb->pending_frame_consumed_cond);}+ sc_mutex_unlock(&vb->mutex);++ // rendering_frame is only written from this thread, no need to lockreturn vb->rendering_frame;}@@ -111,6 +134,6 @@ video_buffer_interrupt(struct video_buffer *vb) {vb->interrupted = true;sc_mutex_unlock(&vb->mutex);// wake up blocking wait- sc_cond_signal(&vb->rendering_frame_consumed_cond);+ sc_cond_signal(&vb->pending_frame_consumed_cond);}}diff --git a/app/src/video_buffer.h b/app/src/video_buffer.hindex fbab046b..3e52caa7 100644--- a/app/src/video_buffer.h+++ b/app/src/video_buffer.h@@ -11,14 +11,35 @@// forward declarationstypedef struct AVFrame AVFrame;+/**+ * There are 3 frames in memory:+ * - one frame is held by the decoder (decoding_frame)+ * - one frame is held by the renderer (rendering_frame)+ * - one frame is shared between the decoder and the renderer (pending_frame)+ *+ * The decoder decodes a packet into the decoding_frame (it may takes time).+ *+ * Once the frame is decoded, it calls video_buffer_offer_decoded_frame(),+ * which swaps the decoding and pending frames.+ *+ * When the renderer is notified that a new frame is available, it calls+ * video_buffer_take_rendering_frame() to retrieve it, which swaps the pending+ * and rendering frames. The frame is valid until the next call, without+ * blocking the decoder.+ */+struct video_buffer {AVFrame *decoding_frame;+ AVFrame *pending_frame;AVFrame *rendering_frame;+sc_mutex mutex;bool render_expired_frames;bool interrupted;- sc_cond rendering_frame_consumed_cond;- bool rendering_frame_consumed;++ sc_cond pending_frame_consumed_cond;+ bool pending_frame_consumed;+struct fps_counter *fps_counter;};@@ -30,18 +51,15 @@ voidvideo_buffer_destroy(struct video_buffer *vb);// set the decoded frame as ready for rendering-// this function locks vb->mutex during its execution// the output flag is set to report whether the previous frame has been skippedvoidvideo_buffer_offer_decoded_frame(struct video_buffer *vb,bool *previous_frame_skipped);// mark the rendering frame as consumed and return it-// MUST be called with vb->mutex locked!!!-// the caller is expected to render the returned frame to some texture before-// unlocking vb->mutex+// the frame is valid until the next call to this functionconst AVFrame *-video_buffer_consume_rendered_frame(struct video_buffer *vb);+video_buffer_take_rendering_frame(struct video_buffer *vb);// wake up and avoid any blocking callvoid