Display and control your Android device
Use separate threads for decoders
Romain Vimontd99a9e3279e5
diff --git a/app/src/decoder.c b/app/src/decoder.cindex 5bf98692..aa1da273 100644--- a/app/src/decoder.c+++ b/app/src/decoder.c@@ -1,6 +1,7 @@#include "decoder.h"#include <errno.h>+#include <stdio.h>#include <libavcodec/packet.h>#include <libavutil/avutil.h>#include <libavutil/error.h>@@ -71,19 +72,25 @@ error_free_context:static voidsc_decoder_close(struct sc_decoder *decoder) {+ sc_mutex_lock(&decoder->mutex);+ decoder->stopped = true;+ sc_cond_signal(&decoder->cond);++ // Wait until the decoder thread finished using the ctx, frame and the frame+ // sinks+ while (!decoder->ended) {+ sc_cond_wait(&decoder->cond, &decoder->mutex);+ }++ sc_mutex_unlock(&decoder->mutex);+sc_frame_source_sinks_close(&decoder->frame_source);av_frame_free(&decoder->frame);avcodec_free_context(&decoder->ctx);}static enum sc_sink_result-sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {- bool is_config = packet->pts == AV_NOPTS_VALUE;- if (is_config) {- // nothing to do- return SC_SINK_OK;- }-+sc_decoder_process_packet(struct sc_decoder *decoder, const AVPacket *packet) {int ret = avcodec_send_packet(decoder->ctx, packet);if (ret < 0 && ret != AVERROR(EAGAIN)) {LOGE("Decoder '%s': could not send video packet: %s",@@ -143,11 +150,168 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {return SC_SINK_OK;}+static void+sc_decoder_queue_clear(struct sc_decoder_queue *queue) {+ while (!sc_vecdeque_is_empty(queue)) {+ struct sc_decoder_packet *dp = sc_vecdeque_popref(queue);+ if (dp->type == SC_DECODER_PACKET_TYPE_AV_PACKET) {+ av_packet_free(&dp->packet);+ }+ }+}++static bool+sc_decoder_decode(struct sc_decoder *decoder) {+ for (;;) {+ sc_mutex_lock(&decoder->mutex);+ while (!decoder->stopped && sc_vecdeque_is_empty(&decoder->queue)) {+ sc_cond_wait(&decoder->cond, &decoder->mutex);+ }+ if (decoder->stopped) {+ sc_mutex_unlock(&decoder->mutex);+ return true;+ }++ struct sc_decoder_packet dp = sc_vecdeque_pop(&decoder->queue);+ sc_mutex_unlock(&decoder->mutex);++ enum sc_sink_result result;+ if (dp.type == SC_DECODER_PACKET_TYPE_AV_PACKET) {+ result = sc_decoder_process_packet(decoder, dp.packet);+ av_packet_free(&dp.packet);+ } else {+ assert(dp.type == SC_DECODER_PACKET_TYPE_SESSION);+ decoder->session = dp.session;+ enum sc_sink_result push_result =+ sc_frame_source_sinks_push_session(&decoder->frame_source,+ &dp.session);+ // Not a decoder error+ result = push_result == SC_SINK_OK ? SC_SINK_OK : SC_SINK_STOPPED;+ }++ if (result == SC_SINK_STOPPED) {+ // No error+ return true;+ }++ if (result == SC_SINK_KO) {+ return false;+ }+ }+ return true;+}++static int+run_decoder(void *data) {+ struct sc_decoder *decoder = data;++ bool success = sc_decoder_decode(decoder);++ sc_mutex_lock(&decoder->mutex);+ // Prevent the producer from pushing any new packet+ decoder->stopped = true;+ // Discard pending packets+ sc_decoder_queue_clear(&decoder->queue);++ decoder->ended = true;+ sc_cond_signal(&decoder->cond);++ sc_mutex_unlock(&decoder->mutex);++ if (!success) {+ LOGE("Decoding (%s) failed", decoder->name);+ }++ LOGD("Decoder (%s) thread ended", decoder->name);++ decoder->cbs->on_ended(decoder, success, decoder->cbs_userdata);++ return 0;+}++static AVPacket *+sc_decoder_packet_ref(const AVPacket *packet) {+ AVPacket *p = av_packet_alloc();+ if (!p) {+ LOG_OOM();+ return NULL;+ }++ if (av_packet_ref(p, packet)) {+ LOG_OOM();+ av_packet_free(&p);+ return NULL;+ }++ return p;+}++static enum sc_sink_result+sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {+ bool is_config = packet->pts == AV_NOPTS_VALUE;+ if (is_config) {+ // nothing to do+ return SC_SINK_OK;+ }++ sc_mutex_lock(&decoder->mutex);++ if (decoder->stopped) {+ // reject any new packet+ sc_mutex_unlock(&decoder->mutex);+ return SC_SINK_STOPPED;+ }++ AVPacket *p = sc_decoder_packet_ref(packet);+ if (!p) {+ sc_mutex_unlock(&decoder->mutex);+ return SC_SINK_KO;+ }++ struct sc_decoder_packet *dp =+ sc_vecdeque_push_uninitialized(&decoder->queue);+ if (!dp) {+ LOG_OOM();+ sc_mutex_unlock(&decoder->mutex);+ av_packet_free(&p);+ return SC_SINK_KO;+ }++ dp->type = SC_DECODER_PACKET_TYPE_AV_PACKET;+ dp->packet = p;++ sc_cond_signal(&decoder->cond);++ sc_mutex_unlock(&decoder->mutex);+ return SC_SINK_OK;+}+static enum sc_sink_resultsc_decoder_push_session(struct sc_decoder *decoder,const struct sc_stream_session *session) {- decoder->session = *session;- return sc_frame_source_sinks_push_session(&decoder->frame_source, session);+ sc_mutex_lock(&decoder->mutex);++ if (decoder->stopped) {+ // reject any new packet+ sc_mutex_unlock(&decoder->mutex);+ return SC_SINK_STOPPED;+ }++ struct sc_decoder_packet *dp =+ sc_vecdeque_push_uninitialized(&decoder->queue);+ if (!dp) {+ LOG_OOM();+ sc_mutex_unlock(&decoder->mutex);+ return SC_SINK_KO;+ }++ dp->type = SC_DECODER_PACKET_TYPE_SESSION;+ dp->session = *session;++ sc_cond_signal(&decoder->cond);++ sc_mutex_unlock(&decoder->mutex);+ return SC_SINK_OK;}static bool@@ -178,13 +342,33 @@ sc_decoder_packet_sink_push_session(struct sc_packet_sink *sink,return sc_decoder_push_session(decoder, session);}-void-sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque) {+bool+sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,+ const struct sc_decoder_callbacks *cbs, void *cbs_userdata) {+ bool ok = sc_mutex_init(&decoder->mutex);+ if (!ok) {+ return false;+ }++ ok = sc_cond_init(&decoder->cond);+ if (!ok) {+ sc_mutex_destroy(&decoder->mutex);+ return false;+ }+decoder->name = name; // statically allocatedsc_frame_source_init(&decoder->frame_source);decoder->copy_opaque = copy_opaque;+ sc_vecdeque_init(&decoder->queue);+ decoder->stopped = false;+ decoder->ended = false;++ assert(cbs && cbs->on_ended);+ decoder->cbs = cbs;+ decoder->cbs_userdata = cbs_userdata;+static const struct sc_packet_sink_ops ops = {.open = sc_decoder_packet_sink_open,.close = sc_decoder_packet_sink_close,@@ -193,4 +377,42 @@ sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque)};decoder->packet_sink.ops = &ops;++ return true;+}++bool+sc_decoder_start(struct sc_decoder *decoder) {+ char thread_name[16];+ // "scrcpy-vid-dec" or "scrcpy-aud-dec"+ snprintf(thread_name, sizeof(thread_name), "scrcpy-%.3s-dec",+ decoder->name);+ bool ok = sc_thread_create(&decoder->thread, run_decoder, thread_name,+ decoder);+ if (!ok) {+ LOGE("Could not start decoder (%s) thread", decoder->name);+ return false;+ }++ return true;+}++void+sc_decoder_stop(struct sc_decoder *decoder) {+ sc_mutex_lock(&decoder->mutex);+ decoder->stopped = true;+ sc_cond_signal(&decoder->cond);+ sc_mutex_unlock(&decoder->mutex);+}++void+sc_decoder_join(struct sc_decoder *decoder) {+ sc_thread_join(&decoder->thread, NULL);+}++void+sc_decoder_destroy(struct sc_decoder *decoder) {+ sc_vecdeque_destroy(&decoder->queue);+ sc_cond_destroy(&decoder->cond);+ sc_mutex_destroy(&decoder->mutex);}diff --git a/app/src/decoder.h b/app/src/decoder.hindex 4585c7b5..da5724d3 100644--- a/app/src/decoder.h+++ b/app/src/decoder.h@@ -8,6 +8,23 @@#include "coords.h"#include "trait/frame_source.h"#include "trait/packet_sink.h"+#include "util/thread.h"+#include "util/vecdeque.h"++enum sc_decoder_packet_type {+ SC_DECODER_PACKET_TYPE_AV_PACKET,+ SC_DECODER_PACKET_TYPE_SESSION,+};++struct sc_decoder_packet {+ enum sc_decoder_packet_type type;+ union {+ struct sc_stream_session session;+ AVPacket *packet;+ };+};++struct sc_decoder_queue SC_VECDEQUE(struct sc_decoder_packet);struct sc_decoder {struct sc_packet_sink packet_sink; // packet sink trait@@ -16,17 +33,46 @@ struct sc_decoder {const char *name; // must be statically allocated (e.g. a string literal)bool copy_opaque;+ sc_thread thread;+ sc_mutex mutex;+ sc_cond cond;+ // set on sc_decoder_stop(), packet_sink close or decoding failure+ bool stopped;+ // set by the decoder thread when it no longer uses the resources+ bool ended;+ struct sc_decoder_queue queue;+AVCodecContext *ctx;AVFrame *frame;struct sc_stream_session session; // only initialized for video streamstruct sc_size frame_size;++ const struct sc_decoder_callbacks *cbs;+ void *cbs_userdata;+};++struct sc_decoder_callbacks {+ void (*on_ended)(struct sc_decoder *decoder, bool success, void *userdata);};// The name must be statically allocated (e.g. a string literal)//// copy_opaque: Set to true to forward the AVPacket.opaque_ref to the AVFrame+bool+sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,+ const struct sc_decoder_callbacks *cbs, void *cbs_userdata);++bool+sc_decoder_start(struct sc_decoder *decoder);++void+sc_decoder_stop(struct sc_decoder *decoder);++void+sc_decoder_join(struct sc_decoder *decoder);+void-sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque);+sc_decoder_destroy(struct sc_decoder *decoder);#endifdiff --git a/app/src/events.h b/app/src/events.hindex d71d8f0d..9f42e29f 100644--- a/app/src/events.h+++ b/app/src/events.h@@ -15,6 +15,7 @@ enum {SC_EVENT_SERVER_CONNECTION_FAILED,SC_EVENT_SERVER_CONNECTED,SC_EVENT_DEMUXER_ERROR,+ SC_EVENT_DECODER_ERROR,SC_EVENT_RECORDER_ERROR,SC_EVENT_TIME_LIMIT_REACHED,SC_EVENT_CONTROLLER_ERROR,diff --git a/app/src/scrcpy.c b/app/src/scrcpy.cindex aacd39f1..5e756319 100644--- a/app/src/scrcpy.c+++ b/app/src/scrcpy.c@@ -130,6 +130,9 @@ event_loop(struct scrcpy *s, bool has_screen) {case SC_EVENT_CONTROLLER_ERROR:LOGE("Controller error");return SCRCPY_EXIT_FAILURE;+ case SC_EVENT_DECODER_ERROR:+ LOGE("Decoder error");+ return SCRCPY_EXIT_FAILURE;case SC_EVENT_RECORDER_ERROR:LOGE("Recorder error");return SCRCPY_EXIT_FAILURE;@@ -179,6 +182,16 @@ await_for_server(bool *connected) {return false;}+static void+sc_decoder_on_ended(struct sc_decoder *decoder, bool success, void *userdata) {+ (void) decoder;+ (void) userdata;++ if (!success) {+ sc_push_event(SC_EVENT_DECODER_ERROR);+ }+}+static voidsc_recorder_on_ended(struct sc_recorder *recorder, bool success,void *userdata) {@@ -345,6 +358,10 @@ scrcpy(struct scrcpy_options *options) {#endifbool video_demuxer_started = false;bool audio_demuxer_started = false;+ bool video_decoder_initialized = false;+ bool video_decoder_started = false;+ bool audio_decoder_initialized = false;+ bool audio_decoder_started = false;#ifdef HAVE_USBbool aoa_hid_initialized = false;bool keyboard_aoa_initialized = false;@@ -552,6 +569,10 @@ scrcpy(struct scrcpy_options *options) {false, &audio_demuxer_cbs, options);}+ static const struct sc_decoder_callbacks decoder_cbs = {+ .on_ended = sc_decoder_on_ended,+ };+bool needs_video_decoder = options->video_playback;bool needs_audio_decoder = options->audio_playback;#ifdef HAVE_V4L2@@ -561,14 +582,35 @@ scrcpy(struct scrcpy_options *options) {// If a video buffer is present, then the recv date must be forwarded// from the AVPacket to the AVFramebool copy_opaque = has_video_buffer;- sc_decoder_init(&s->video_decoder, "video", copy_opaque);+ if (!sc_decoder_init(&s->video_decoder, "video", copy_opaque,+ &decoder_cbs, NULL)) {+ goto end;+ }+ video_decoder_initialized = true;+sc_packet_source_add_sink(&s->video_demuxer.packet_source,&s->video_decoder.packet_sink);++ if (!sc_decoder_start(&s->video_decoder)) {+ goto end;+ }+ video_decoder_started = true;}+if (needs_audio_decoder) {- sc_decoder_init(&s->audio_decoder, "audio", false);+ if (!sc_decoder_init(&s->audio_decoder, "audio", false,+ &decoder_cbs, NULL)) {+ goto end;+ }+ audio_decoder_initialized = true;+sc_packet_source_add_sink(&s->audio_demuxer.packet_source,&s->audio_decoder.packet_sink);++ if (!sc_decoder_start(&s->audio_decoder)) {+ goto end;+ }+ audio_decoder_started = true;}if (options->record_filename) {@@ -940,6 +982,12 @@ end:if (file_pusher_initialized) {sc_file_pusher_stop(&s->file_pusher);}+ if (video_decoder_started) {+ sc_decoder_stop(&s->video_decoder);+ }+ if (audio_decoder_started) {+ sc_decoder_stop(&s->audio_decoder);+ }if (recorder_initialized) {sc_recorder_stop(&s->recorder);}@@ -959,10 +1007,18 @@ end:LOGD("Quit...");// Close the window immediately, because sc_screen_destroy() may only be- // called once the video demuxer thread is joined (it may take time)+ // called once the video decoder thread is joined (it may take time)sc_screen_hide_window(&s->screen);}+ if (video_decoder_started) {+ sc_decoder_join(&s->video_decoder);+ }++ if (audio_decoder_started) {+ sc_decoder_join(&s->audio_decoder);+ }+if (timeout_started) {sc_timeout_join(&s->timeout);}@@ -1009,6 +1065,14 @@ end:}#endif+ if (video_decoder_initialized) {+ sc_decoder_destroy(&s->video_decoder);+ }++ if (audio_decoder_initialized) {+ sc_decoder_destroy(&s->audio_decoder);+ }+// Destroy the screen only after the video demuxer is guaranteed to be// finished, because otherwise the screen could receive new frames after// destruction