Skip to content

Display and control your Android device

Add intermediate frame in video buffer

Romain Vimontc0c4ba700904
diff --git a/app/src/screen.c b/app/src/screen.c
index 0f8a2226..5ec416f9 100644
--- a/app/src/screen.c
+++ b/app/src/screen.c
@@ -453,15 +453,12 @@ update_texture(struct screen *screen, const AVFrame *frame) {
bool
screen_update_frame(struct screen *screen, struct video_buffer *vb) {
- sc_mutex_lock(&vb->mutex);
- const AVFrame *frame = video_buffer_consume_rendered_frame(vb);
+ const AVFrame *frame = video_buffer_take_rendering_frame(vb);
struct size new_frame_size = {frame->width, frame->height};
if (!prepare_for_frame(screen, new_frame_size)) {
- sc_mutex_unlock(&vb->mutex);
return false;
}
update_texture(screen, frame);
- sc_mutex_unlock(&vb->mutex);
screen_render(screen, false);
return true;
diff --git a/app/src/video_buffer.c b/app/src/video_buffer.c
index 95cfd755..4c8dd5da 100644
--- a/app/src/video_buffer.c
+++ b/app/src/video_buffer.c
@@ -16,19 +16,24 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
goto error_0;
}
+ vb->pending_frame = av_frame_alloc();
+ if (!vb->pending_frame) {
+ goto error_1;
+ }
+
vb->rendering_frame = av_frame_alloc();
if (!vb->rendering_frame) {
- goto error_1;
+ goto error_2;
}
bool ok = sc_mutex_init(&vb->mutex);
if (!ok) {
- goto error_2;
+ goto error_3;
}
vb->render_expired_frames = render_expired_frames;
if (render_expired_frames) {
- ok = sc_cond_init(&vb->rendering_frame_consumed_cond);
+ ok = sc_cond_init(&vb->pending_frame_consumed_cond);
if (!ok) {
sc_mutex_destroy(&vb->mutex);
goto error_2;
@@ -40,12 +45,14 @@ video_buffer_init(struct video_buffer *vb, struct fps_counter *fps_counter,
// there is initially no rendering frame, so consider it has already been
// consumed
- vb->rendering_frame_consumed = true;
+ vb->pending_frame_consumed = true;
return true;
-error_2:
+error_3:
av_frame_free(&vb->rendering_frame);
+error_2:
+ av_frame_free(&vb->pending_frame);
error_1:
av_frame_free(&vb->decoding_frame);
error_0:
@@ -55,19 +62,28 @@ error_0:
void
video_buffer_destroy(struct video_buffer *vb) {
if (vb->render_expired_frames) {
- sc_cond_destroy(&vb->rendering_frame_consumed_cond);
+ sc_cond_destroy(&vb->pending_frame_consumed_cond);
}
sc_mutex_destroy(&vb->mutex);
av_frame_free(&vb->rendering_frame);
+ av_frame_free(&vb->pending_frame);
av_frame_free(&vb->decoding_frame);
}
static void
-video_buffer_swap_frames(struct video_buffer *vb) {
+video_buffer_swap_decoding_frame(struct video_buffer *vb) {
sc_mutex_assert(&vb->mutex);
AVFrame *tmp = vb->decoding_frame;
- vb->decoding_frame = vb->rendering_frame;
- vb->rendering_frame = tmp;
+ vb->decoding_frame = vb->pending_frame;
+ vb->pending_frame = tmp;
+}
+
+static void
+video_buffer_swap_rendering_frame(struct video_buffer *vb) {
+ sc_mutex_assert(&vb->mutex);
+ AVFrame *tmp = vb->rendering_frame;
+ vb->rendering_frame = vb->pending_frame;
+ vb->pending_frame = tmp;
}
void
@@ -76,31 +92,38 @@ video_buffer_offer_decoded_frame(struct video_buffer *vb,
sc_mutex_lock(&vb->mutex);
if (vb->render_expired_frames) {
// wait for the current (expired) frame to be consumed
- while (!vb->rendering_frame_consumed && !vb->interrupted) {
- sc_cond_wait(&vb->rendering_frame_consumed_cond, &vb->mutex);
+ while (!vb->pending_frame_consumed && !vb->interrupted) {
+ sc_cond_wait(&vb->pending_frame_consumed_cond, &vb->mutex);
}
- } else if (!vb->rendering_frame_consumed) {
+ } else if (!vb->pending_frame_consumed) {
fps_counter_add_skipped_frame(vb->fps_counter);
}
- video_buffer_swap_frames(vb);
+ video_buffer_swap_decoding_frame(vb);
- *previous_frame_skipped = !vb->rendering_frame_consumed;
- vb->rendering_frame_consumed = false;
+ *previous_frame_skipped = !vb->pending_frame_consumed;
+ vb->pending_frame_consumed = false;
sc_mutex_unlock(&vb->mutex);
}
const AVFrame *
-video_buffer_consume_rendered_frame(struct video_buffer *vb) {
- sc_mutex_assert(&vb->mutex);
- assert(!vb->rendering_frame_consumed);
- vb->rendering_frame_consumed = true;
+video_buffer_take_rendering_frame(struct video_buffer *vb) {
+ sc_mutex_lock(&vb->mutex);
+ assert(!vb->pending_frame_consumed);
+ vb->pending_frame_consumed = true;
+
fps_counter_add_rendered_frame(vb->fps_counter);
+
+ video_buffer_swap_rendering_frame(vb);
+
if (vb->render_expired_frames) {
// unblock video_buffer_offer_decoded_frame()
- sc_cond_signal(&vb->rendering_frame_consumed_cond);
+ sc_cond_signal(&vb->pending_frame_consumed_cond);
}
+ sc_mutex_unlock(&vb->mutex);
+
+ // rendering_frame is only written from this thread, no need to lock
return vb->rendering_frame;
}
@@ -111,6 +134,6 @@ video_buffer_interrupt(struct video_buffer *vb) {
vb->interrupted = true;
sc_mutex_unlock(&vb->mutex);
// wake up blocking wait
- sc_cond_signal(&vb->rendering_frame_consumed_cond);
+ sc_cond_signal(&vb->pending_frame_consumed_cond);
}
}
diff --git a/app/src/video_buffer.h b/app/src/video_buffer.h
index fbab046b..3e52caa7 100644
--- a/app/src/video_buffer.h
+++ b/app/src/video_buffer.h
@@ -11,14 +11,35 @@
// forward declarations
typedef struct AVFrame AVFrame;
+/**
+ * There are 3 frames in memory:
+ * - one frame is held by the decoder (decoding_frame)
+ * - one frame is held by the renderer (rendering_frame)
+ * - one frame is shared between the decoder and the renderer (pending_frame)
+ *
+ * The decoder decodes a packet into the decoding_frame (it may takes time).
+ *
+ * Once the frame is decoded, it calls video_buffer_offer_decoded_frame(),
+ * which swaps the decoding and pending frames.
+ *
+ * When the renderer is notified that a new frame is available, it calls
+ * video_buffer_take_rendering_frame() to retrieve it, which swaps the pending
+ * and rendering frames. The frame is valid until the next call, without
+ * blocking the decoder.
+ */
+
struct video_buffer {
AVFrame *decoding_frame;
+ AVFrame *pending_frame;
AVFrame *rendering_frame;
+
sc_mutex mutex;
bool render_expired_frames;
bool interrupted;
- sc_cond rendering_frame_consumed_cond;
- bool rendering_frame_consumed;
+
+ sc_cond pending_frame_consumed_cond;
+ bool pending_frame_consumed;
+
struct fps_counter *fps_counter;
};
@@ -30,18 +51,15 @@ void
video_buffer_destroy(struct video_buffer *vb);
// set the decoded frame as ready for rendering
-// this function locks vb->mutex during its execution
// the output flag is set to report whether the previous frame has been skipped
void
video_buffer_offer_decoded_frame(struct video_buffer *vb,
bool *previous_frame_skipped);
// mark the rendering frame as consumed and return it
-// MUST be called with vb->mutex locked!!!
-// the caller is expected to render the returned frame to some texture before
-// unlocking vb->mutex
+// the frame is valid until the next call to this function
const AVFrame *
-video_buffer_consume_rendered_frame(struct video_buffer *vb);
+video_buffer_take_rendering_frame(struct video_buffer *vb);
// wake up and avoid any blocking call
void