Display and control your Android device
Make the video regulator apply backpressure
Romain Vimont394ca01c3c4e
diff --git a/app/src/decoder.c b/app/src/decoder.cindex aa1da273..27bc0f5c 100644--- a/app/src/decoder.c+++ b/app/src/decoder.c@@ -91,6 +91,9 @@ sc_decoder_close(struct sc_decoder *decoder) {static enum sc_sink_resultsc_decoder_process_packet(struct sc_decoder *decoder, const AVPacket *packet) {+ // Do not decode more packets before the frame sinks are ready+ sc_frame_source_sinks_apply_backpressure(&decoder->frame_source);+int ret = avcodec_send_packet(decoder->ctx, packet);if (ret < 0 && ret != AVERROR(EAGAIN)) {LOGE("Decoder '%s': could not send video packet: %s",diff --git a/app/src/scrcpy.c b/app/src/scrcpy.cindex b6885430..eef79506 100644--- a/app/src/scrcpy.c+++ b/app/src/scrcpy.c@@ -48,6 +48,8 @@#endif#include "video_regulator.h"+#define SC_BACKPRESSURE_THRESHOLD 8 // in-flight frames in video regulator+struct scrcpy {struct sc_server server;struct sc_screen screen;@@ -846,8 +848,18 @@ aoa_complete:if (options->video_playback) {struct sc_frame_source *src = &s->video_decoder.frame_source;if (options->video_buffer) {+ uint32_t backpressure_threshold = SC_BACKPRESSURE_THRESHOLD;+#ifdef HAVE_V4L2+ if (options->v4l2_device && options->v4l2_buffer+ && options->v4l2_buffer < options->video_buffer) {+ // Disable the backpressure threshold for this video+ // regulator, it will be handled by the v4l2 video regulator+ backpressure_threshold = 0; // disabled+ }+#endifif (!sc_video_regulator_init(&s->video_regulator,- options->video_buffer, true)) {+ options->video_buffer, true,+ backpressure_threshold)) {goto end;}video_regulator_initialized = true;@@ -874,8 +886,16 @@ aoa_complete:struct sc_frame_source *src = &s->video_decoder.frame_source;if (options->v4l2_buffer) {+ uint32_t backpressure_threshold = SC_BACKPRESSURE_THRESHOLD;+ if (options->video_playback && options->video_buffer+ && options->video_buffer <= options->v4l2_buffer) {+ // Disable the backpressure threshold for this video+ // regulator, it will be handled by the display video regulator+ backpressure_threshold = 0; // disabled+ }if (!sc_video_regulator_init(&s->v4l2_regulator,- options->v4l2_buffer, true)) {+ options->v4l2_buffer,+ true, backpressure_threshold)) {goto end;}v4l2_regulator_initialized = true;@@ -996,6 +1016,14 @@ end:if (audio_decoder_started) {sc_decoder_stop(&s->audio_decoder);}+ if (video_regulator_initialized) {+ sc_video_regulator_stop(&s->video_regulator);+ }+#ifdef HAVE_V4L2+ if (v4l2_regulator_initialized) {+ sc_video_regulator_stop(&s->v4l2_regulator);+ }+#endifif (recorder_initialized) {sc_recorder_stop(&s->recorder);}diff --git a/app/src/trait/frame_sink.h b/app/src/trait/frame_sink.hindex 478c1f7c..19e19652 100644--- a/app/src/trait/frame_sink.h+++ b/app/src/trait/frame_sink.h@@ -36,6 +36,13 @@ struct sc_frame_sink_ops {enum sc_sink_result(*push_session)(struct sc_frame_sink *sink,const struct sc_stream_session *session);++ /**+ * Optional method to wait until the frame sink signals it is ready to+ * accept more frames.+ */+ void+ (*apply_backpressure)(struct sc_frame_sink *sink);};#endifdiff --git a/app/src/trait/frame_source.c b/app/src/trait/frame_source.cindex 737c85c4..a9ab0016 100644--- a/app/src/trait/frame_source.c+++ b/app/src/trait/frame_source.c@@ -78,3 +78,14 @@ sc_frame_source_sinks_push_session(struct sc_frame_source *source,return SC_SINK_OK;}++void+sc_frame_source_sinks_apply_backpressure(struct sc_frame_source *source) {+ assert(source->sink_count);+ for (unsigned i = 0; i < source->sink_count; ++i) {+ struct sc_frame_sink *sink = source->sinks[i];+ if (sink->ops->apply_backpressure) {+ sink->ops->apply_backpressure(sink);+ }+ }+}diff --git a/app/src/trait/frame_source.h b/app/src/trait/frame_source.hindex ad65d282..36ad02c4 100644--- a/app/src/trait/frame_source.h+++ b/app/src/trait/frame_source.h@@ -42,4 +42,7 @@ enum sc_sink_resultsc_frame_source_sinks_push_session(struct sc_frame_source *source,const struct sc_stream_session *session);+void+sc_frame_source_sinks_apply_backpressure(struct sc_frame_source *source);+#endifdiff --git a/app/src/video_regulator.c b/app/src/video_regulator.cindex 1d147f55..611c40b8 100644--- a/app/src/video_regulator.c+++ b/app/src/video_regulator.c@@ -62,6 +62,7 @@ run_buffering(void *data) {}struct sc_delayed_packet dpacket = sc_vecdeque_pop(&vr->queue);+ sc_cond_signal(&vr->backpressure_cond);enum sc_sink_result result;if (dpacket.type == SC_DELAYED_PACKET_TYPE_FRAME) {@@ -111,6 +112,7 @@ run_buffering(void *data) {sc_mutex_lock(&vr->mutex);// Prevent to push any new packetvr->stopped = true;+ sc_cond_signal(&vr->backpressure_cond);sc_mutex_unlock(&vr->mutex);goto stopped;}@@ -140,7 +142,6 @@ sc_video_regulator_frame_sink_open(struct sc_frame_sink *sink,sc_clock_init(&vr->clock);sc_vecdeque_init(&vr->queue);- vr->stopped = false;if (!sc_frame_source_sinks_open(&vr->frame_source, ctx, session)) {return false;@@ -160,12 +161,7 @@ static voidsc_video_regulator_frame_sink_close(struct sc_frame_sink *sink) {struct sc_video_regulator *vr = DOWNCAST(sink);- sc_mutex_lock(&vr->mutex);- vr->stopped = true;- sc_cond_signal(&vr->queue_cond);- sc_cond_signal(&vr->wait_cond);- sc_mutex_unlock(&vr->mutex);-+ sc_video_regulator_stop(vr);sc_thread_join(&vr->thread, NULL);sc_frame_source_sinks_close(&vr->frame_source);@@ -258,9 +254,29 @@ sc_video_regulator_frame_sink_push_session(struct sc_frame_sink *sink,return SC_SINK_OK;}+static void+sc_video_regulator_frame_sink_apply_backpressure(struct sc_frame_sink *sink) {+ struct sc_video_regulator *vr = DOWNCAST(sink);++ if (!vr->backpressure_threshold) {+ // Backpressure disabled+ return;+ }++ sc_mutex_lock(&vr->mutex);+ // This not only counts queued frames, but also queued session packets.+ // But session packets are rare and consumed quickly, so keep it simple.+ while (!vr->stopped+ && sc_vecdeque_size(&vr->queue) > vr->backpressure_threshold) {+ sc_cond_wait(&vr->backpressure_cond, &vr->mutex);+ }+ sc_mutex_unlock(&vr->mutex);+}+boolsc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,- bool first_frame_asap) {+ bool first_frame_asap,+ uint32_t backpressure_threshold) {assert(delay > 0);bool ok = sc_mutex_init(&vr->mutex);@@ -278,8 +294,15 @@ sc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,goto error_destroy_queue_cond;}+ ok = sc_cond_init(&vr->backpressure_cond);+ if (!ok) {+ goto error_destroy_wait_cond;+ }+vr->delay = delay;vr->first_frame_asap = first_frame_asap;+ vr->backpressure_threshold = backpressure_threshold;+ vr->stopped = false;sc_frame_source_init(&vr->frame_source);@@ -288,12 +311,15 @@ sc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,.close = sc_video_regulator_frame_sink_close,.push = sc_video_regulator_frame_sink_push,.push_session = sc_video_regulator_frame_sink_push_session,+ .apply_backpressure = sc_video_regulator_frame_sink_apply_backpressure,};vr->frame_sink.ops = &ops;return true;+error_destroy_wait_cond:+ sc_cond_destroy(&vr->wait_cond);error_destroy_queue_cond:sc_cond_destroy(&vr->queue_cond);error_destroy_mutex:@@ -302,8 +328,19 @@ error_destroy_mutex:return false;}+void+sc_video_regulator_stop(struct sc_video_regulator *vr) {+ sc_mutex_lock(&vr->mutex);+ vr->stopped = true;+ sc_cond_signal(&vr->queue_cond);+ sc_cond_signal(&vr->wait_cond);+ sc_cond_signal(&vr->backpressure_cond);+ sc_mutex_unlock(&vr->mutex);+}+voidsc_video_regulator_destroy(struct sc_video_regulator *vr) {+ sc_cond_destroy(&vr->backpressure_cond);sc_cond_destroy(&vr->wait_cond);sc_cond_destroy(&vr->queue_cond);sc_mutex_destroy(&vr->mutex);diff --git a/app/src/video_regulator.h b/app/src/video_regulator.hindex 43333c86..d5fe8ae9 100644--- a/app/src/video_regulator.h+++ b/app/src/video_regulator.h@@ -42,11 +42,13 @@ struct sc_video_regulator {sc_tick delay;bool first_frame_asap;+ uint32_t backpressure_threshold;sc_thread thread;sc_mutex mutex;sc_cond queue_cond;sc_cond wait_cond;+ sc_cond backpressure_cond;struct sc_clock clock;struct sc_delayed_packet_queue queue;@@ -63,11 +65,22 @@ struct sc_video_regulator_callbacks {** \param delay a (strictly) positive delay* \param first_frame_asap if true, do not delay the first frame (useful for- a video stream).+ * a video stream).+ * \param backpressure_threshold the max number of queued items to make+ * apply_backpressure() return (0 to disable).*/boolsc_video_regulator_init(struct sc_video_regulator *vr, sc_tick delay,- bool first_frame_asap);+ bool first_frame_asap, uint32_t backpressure_threshold);++/**+ * Stop the video regulator.+ *+ * Reject any new frame, and interrupt any blocking call to wake up the+ * producer.+ */+void+sc_video_regulator_stop(struct sc_video_regulator *vr);voidsc_video_regulator_destroy(struct sc_video_regulator *vr);