Skip to content

Display and control your Android device

Make the video regulator apply backpressure

Romain Vimont394ca01c3c4e
diff --git a/app/src/decoder.c b/app/src/decoder.c
index aa1da273..27bc0f5c 100644
--- a/app/src/decoder.c
+++ b/app/src/decoder.c
@@ -91,6 +91,9 @@ sc_decoder_close(struct sc_decoder *decoder) {
static enum sc_sink_result
sc_decoder_process_packet(struct sc_decoder *decoder, const AVPacket *packet) {
+ // Do not decode more packets before the frame sinks are ready
+ sc_frame_source_sinks_apply_backpressure(&decoder->frame_source);
+
int ret = avcodec_send_packet(decoder->ctx, packet);
if (ret < 0 && ret != AVERROR(EAGAIN)) {
LOGE("Decoder '%s': could not send video packet: %s",
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
index b6885430..eef79506 100644
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@@ -48,6 +48,8 @@
#endif
#include "video_regulator.h"
+#define SC_BACKPRESSURE_THRESHOLD 8 // in-flight frames in video regulator
+
struct scrcpy {
struct sc_server server;
struct sc_screen screen;
@@ -846,8 +848,18 @@ aoa_complete:
if (options->video_playback) {
struct sc_frame_source *src = &s->video_decoder.frame_source;
if (options->video_buffer) {
+ uint32_t backpressure_threshold = SC_BACKPRESSURE_THRESHOLD;
+#ifdef HAVE_V4L2
+ if (options->v4l2_device && options->v4l2_buffer
+ && options->v4l2_buffer < options->video_buffer) {
+ // Disable the backpressure threshold for this video
+ // regulator, it will be handled by the v4l2 video regulator
+ backpressure_threshold = 0; // disabled
+ }
+#endif
if (!sc_video_regulator_init(&s->video_regulator,
- options->video_buffer, true)) {
+ options->video_buffer, true,
+ backpressure_threshold)) {
goto end;
}
video_regulator_initialized = true;
@@ -874,8 +886,16 @@ aoa_complete:
struct sc_frame_source *src = &s->video_decoder.frame_source;
if (options->v4l2_buffer) {
+ uint32_t backpressure_threshold = SC_BACKPRESSURE_THRESHOLD;
+ if (options->video_playback && options->video_buffer
+ && options->video_buffer <= options->v4l2_buffer) {
+ // Disable the backpressure threshold for this video
+ // regulator, it will be handled by the display video regulator
+ backpressure_threshold = 0; // disabled
+ }
if (!sc_video_regulator_init(&s->v4l2_regulator,
- options->v4l2_buffer, true)) {
+ options->v4l2_buffer,
+ true, backpressure_threshold)) {
goto end;
}
v4l2_regulator_initialized = true;
@@ -996,6 +1016,14 @@ end:
if (audio_decoder_started) {
sc_decoder_stop(&s->audio_decoder);
}
+ if (video_regulator_initialized) {
+ sc_video_regulator_stop(&s->video_regulator);
+ }
+#ifdef HAVE_V4L2
+ if (v4l2_regulator_initialized) {
+ sc_video_regulator_stop(&s->v4l2_regulator);
+ }
+#endif
if (recorder_initialized) {
sc_recorder_stop(&s->recorder);
}
diff --git a/app/src/trait/frame_sink.h b/app/src/trait/frame_sink.h
index 478c1f7c..19e19652 100644
--- a/app/src/trait/frame_sink.h
+++ b/app/src/trait/frame_sink.h
@@ -36,6 +36,13 @@ struct sc_frame_sink_ops {
enum sc_sink_result
(*push_session)(struct sc_frame_sink *sink,
const struct sc_stream_session *session);
+
+ /**
+ * Optional method to wait until the frame sink signals it is ready to
+ * accept more frames.
+ */
+ void
+ (*apply_backpressure)(struct sc_frame_sink *sink);
};
#endif
diff --git a/app/src/trait/frame_source.c b/app/src/trait/frame_source.c
index 737c85c4..a9ab0016 100644
--- a/app/src/trait/frame_source.c
+++ b/app/src/trait/frame_source.c
@@ -78,3 +78,14 @@ sc_frame_source_sinks_push_session(struct sc_frame_source *source,
return SC_SINK_OK;
}
+
+void
+sc_frame_source_sinks_apply_backpressure(struct sc_frame_source *source) {
+ assert(source->sink_count);
+ for (unsigned i = 0; i < source->sink_count; ++i) {
+ struct sc_frame_sink *sink = source->sinks[i];
+ if (sink->ops->apply_backpressure) {
+ sink->ops->apply_backpressure(sink);
+ }
+ }
+}
diff --git a/app/src/trait/frame_source.h b/app/src/trait/frame_source.h
index ad65d282..36ad02c4 100644
--- a/app/src/trait/frame_source.h
+++ b/app/src/trait/frame_source.h
@@ -42,4 +42,7 @@ enum sc_sink_result
sc_frame_source_sinks_push_session(struct sc_frame_source *source,
const struct sc_stream_session *session);
+void
+sc_frame_source_sinks_apply_backpressure(struct sc_frame_source *source);
+
#endif
diff --git a/app/src/video_regulator.c b/app/src/video_regulator.c
index 1d147f55..611c40b8 100644
--- a/app/src/video_regulator.c
+++ b/app/src/video_regulator.c
@@ -62,6 +62,7 @@ run_buffering(void *data) {
}
struct sc_delayed_packet dpacket = sc_vecdeque_pop(&vr->queue);
+ sc_cond_signal(&vr->backpressure_cond);
enum sc_sink_result result;
if (dpacket.type == SC_DELAYED_PACKET_TYPE_FRAME) {
@@ -111,6 +112,7 @@ run_buffering(void *data) {
sc_mutex_lock(&vr->mutex);
// Prevent to push any new packet
vr->stopped = true;
+ sc_cond_signal(&vr->backpressure_cond);
sc_mutex_unlock(&vr->mutex);
goto stopped;
}
@@ -140,7 +142,6 @@ sc_video_regulator_frame_sink_open(struct sc_frame_sink *sink,
sc_clock_init(&vr->clock);
sc_vecdeque_init(&vr->queue);
- vr->stopped = false;
if (!sc_frame_source_sinks_open(&vr->frame_source, ctx, session)) {
return false;
@@ -160,12 +161,7 @@ static void
sc_video_regulator_frame_sink_close(struct sc_frame_sink *sink) {
struct sc_video_regulator *vr = DOWNCAST(sink);
- sc_mutex_lock(&vr->mutex);
- vr->stopped = true;
- sc_cond_signal(&vr->queue_cond);
- sc_cond_signal(&vr->wait_cond);
- sc_mutex_unlock(&vr->mutex);
-
+ sc_video_regulator_stop(vr);
sc_thread_join(&vr->thread, NULL);
sc_frame_source_sinks_close(&vr->frame_source);
@@ -258,9 +254,29 @@ sc_video_regulator_frame_sink_push_session(struct sc_frame_sink *sink,
return SC_SINK_OK;
}
+static void
+sc_video_regulator_frame_sink_apply_backpressure(struct sc_frame_sink *sink) {
+ struct sc_video_regulator *vr = DOWNCAST(sink);
+
+ if (!vr->backpressure_threshold) {
+ // Backpressure disabled
+ return;
+ }
+
+ sc_mutex_lock(&vr->mutex);
+ // This not only counts queued frames, but also queued session packets.
+ // But session packets are rare and consumed quickly, so keep it simple.
+ while (!vr->stopped
+ && sc_vecdeque_size(&vr->queue) > vr->backpressure_threshold) {
+ sc_cond_wait(&vr->backpressure_cond, &vr->mutex);
+ }
+ sc_mutex_unlock(&vr->mutex);
+}
+
bool
sc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,
- bool first_frame_asap) {
+ bool first_frame_asap,
+ uint32_t backpressure_threshold) {
assert(delay > 0);
bool ok = sc_mutex_init(&vr->mutex);
@@ -278,8 +294,15 @@ sc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,
goto error_destroy_queue_cond;
}
+ ok = sc_cond_init(&vr->backpressure_cond);
+ if (!ok) {
+ goto error_destroy_wait_cond;
+ }
+
vr->delay = delay;
vr->first_frame_asap = first_frame_asap;
+ vr->backpressure_threshold = backpressure_threshold;
+ vr->stopped = false;
sc_frame_source_init(&vr->frame_source);
@@ -288,12 +311,15 @@ sc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,
.close = sc_video_regulator_frame_sink_close,
.push = sc_video_regulator_frame_sink_push,
.push_session = sc_video_regulator_frame_sink_push_session,
+ .apply_backpressure = sc_video_regulator_frame_sink_apply_backpressure,
};
vr->frame_sink.ops = &ops;
return true;
+error_destroy_wait_cond:
+ sc_cond_destroy(&vr->wait_cond);
error_destroy_queue_cond:
sc_cond_destroy(&vr->queue_cond);
error_destroy_mutex:
@@ -302,8 +328,19 @@ error_destroy_mutex:
return false;
}
+void
+sc_video_regulator_stop(struct sc_video_regulator *vr) {
+ sc_mutex_lock(&vr->mutex);
+ vr->stopped = true;
+ sc_cond_signal(&vr->queue_cond);
+ sc_cond_signal(&vr->wait_cond);
+ sc_cond_signal(&vr->backpressure_cond);
+ sc_mutex_unlock(&vr->mutex);
+}
+
void
sc_video_regulator_destroy(struct sc_video_regulator *vr) {
+ sc_cond_destroy(&vr->backpressure_cond);
sc_cond_destroy(&vr->wait_cond);
sc_cond_destroy(&vr->queue_cond);
sc_mutex_destroy(&vr->mutex);
diff --git a/app/src/video_regulator.h b/app/src/video_regulator.h
index 43333c86..d5fe8ae9 100644
--- a/app/src/video_regulator.h
+++ b/app/src/video_regulator.h
@@ -42,11 +42,13 @@ struct sc_video_regulator {
sc_tick delay;
bool first_frame_asap;
+ uint32_t backpressure_threshold;
sc_thread thread;
sc_mutex mutex;
sc_cond queue_cond;
sc_cond wait_cond;
+ sc_cond backpressure_cond;
struct sc_clock clock;
struct sc_delayed_packet_queue queue;
@@ -63,11 +65,22 @@ struct sc_video_regulator_callbacks {
*
* \param delay a (strictly) positive delay
* \param first_frame_asap if true, do not delay the first frame (useful for
- a video stream).
+ * a video stream).
+ * \param backpressure_threshold the max number of queued items to make
+ * apply_backpressure() return (0 to disable).
*/
bool
sc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,
- bool first_frame_asap);
+ bool first_frame_asap, uint32_t backpressure_threshold);
+
+/**
+ * Stop the video regulator.
+ *
+ * Reject any new frame, and interrupt any blocking call to wake up the
+ * producer.
+ */
+void
+sc_video_regulator_stop(struct sc_video_regulator *vr);
void
sc_video_regulator_destroy(struct sc_video_regulator *vr);