Skip to content

Display and control your Android device

Use separate threads for decoders

Romain Vimontd99a9e3279e5
diff --git a/app/src/decoder.c b/app/src/decoder.c
index 5bf98692..aa1da273 100644
--- a/app/src/decoder.c
+++ b/app/src/decoder.c
@@ -1,6 +1,7 @@
#include "decoder.h"
#include <errno.h>
+#include <stdio.h>
#include <libavcodec/packet.h>
#include <libavutil/avutil.h>
#include <libavutil/error.h>
@@ -71,19 +72,25 @@ error_free_context:
static void
sc_decoder_close(struct sc_decoder *decoder) {
+ sc_mutex_lock(&decoder->mutex);
+ decoder->stopped = true;
+ sc_cond_signal(&decoder->cond);
+
+ // Wait until the decoder thread finished using the ctx, frame and the frame
+ // sinks
+ while (!decoder->ended) {
+ sc_cond_wait(&decoder->cond, &decoder->mutex);
+ }
+
+ sc_mutex_unlock(&decoder->mutex);
+
sc_frame_source_sinks_close(&decoder->frame_source);
av_frame_free(&decoder->frame);
avcodec_free_context(&decoder->ctx);
}
static enum sc_sink_result
-sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {
- bool is_config = packet->pts == AV_NOPTS_VALUE;
- if (is_config) {
- // nothing to do
- return SC_SINK_OK;
- }
-
+sc_decoder_process_packet(struct sc_decoder *decoder, const AVPacket *packet) {
int ret = avcodec_send_packet(decoder->ctx, packet);
if (ret < 0 && ret != AVERROR(EAGAIN)) {
LOGE("Decoder '%s': could not send video packet: %s",
@@ -143,11 +150,168 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {
return SC_SINK_OK;
}
+static void
+sc_decoder_queue_clear(struct sc_decoder_queue *queue) {
+ while (!sc_vecdeque_is_empty(queue)) {
+ struct sc_decoder_packet *dp = sc_vecdeque_popref(queue);
+ if (dp->type == SC_DECODER_PACKET_TYPE_AV_PACKET) {
+ av_packet_free(&dp->packet);
+ }
+ }
+}
+
+static bool
+sc_decoder_decode(struct sc_decoder *decoder) {
+ for (;;) {
+ sc_mutex_lock(&decoder->mutex);
+ while (!decoder->stopped && sc_vecdeque_is_empty(&decoder->queue)) {
+ sc_cond_wait(&decoder->cond, &decoder->mutex);
+ }
+ if (decoder->stopped) {
+ sc_mutex_unlock(&decoder->mutex);
+ return true;
+ }
+
+ struct sc_decoder_packet dp = sc_vecdeque_pop(&decoder->queue);
+ sc_mutex_unlock(&decoder->mutex);
+
+ enum sc_sink_result result;
+ if (dp.type == SC_DECODER_PACKET_TYPE_AV_PACKET) {
+ result = sc_decoder_process_packet(decoder, dp.packet);
+ av_packet_free(&dp.packet);
+ } else {
+ assert(dp.type == SC_DECODER_PACKET_TYPE_SESSION);
+ decoder->session = dp.session;
+ enum sc_sink_result push_result =
+ sc_frame_source_sinks_push_session(&decoder->frame_source,
+ &dp.session);
+ // Not a decoder error
+ result = push_result == SC_SINK_OK ? SC_SINK_OK : SC_SINK_STOPPED;
+ }
+
+ if (result == SC_SINK_STOPPED) {
+ // No error
+ return true;
+ }
+
+ if (result == SC_SINK_KO) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static int
+run_decoder(void *data) {
+ struct sc_decoder *decoder = data;
+
+ bool success = sc_decoder_decode(decoder);
+
+ sc_mutex_lock(&decoder->mutex);
+ // Prevent the producer from pushing any new packet
+ decoder->stopped = true;
+ // Discard pending packets
+ sc_decoder_queue_clear(&decoder->queue);
+
+ decoder->ended = true;
+ sc_cond_signal(&decoder->cond);
+
+ sc_mutex_unlock(&decoder->mutex);
+
+ if (!success) {
+ LOGE("Decoding (%s) failed", decoder->name);
+ }
+
+ LOGD("Decoder (%s) thread ended", decoder->name);
+
+ decoder->cbs->on_ended(decoder, success, decoder->cbs_userdata);
+
+ return 0;
+}
+
+static AVPacket *
+sc_decoder_packet_ref(const AVPacket *packet) {
+ AVPacket *p = av_packet_alloc();
+ if (!p) {
+ LOG_OOM();
+ return NULL;
+ }
+
+ if (av_packet_ref(p, packet)) {
+ LOG_OOM();
+ av_packet_free(&p);
+ return NULL;
+ }
+
+ return p;
+}
+
+static enum sc_sink_result
+sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {
+ bool is_config = packet->pts == AV_NOPTS_VALUE;
+ if (is_config) {
+ // nothing to do
+ return SC_SINK_OK;
+ }
+
+ sc_mutex_lock(&decoder->mutex);
+
+ if (decoder->stopped) {
+ // reject any new packet
+ sc_mutex_unlock(&decoder->mutex);
+ return SC_SINK_STOPPED;
+ }
+
+ AVPacket *p = sc_decoder_packet_ref(packet);
+ if (!p) {
+ sc_mutex_unlock(&decoder->mutex);
+ return SC_SINK_KO;
+ }
+
+ struct sc_decoder_packet *dp =
+ sc_vecdeque_push_uninitialized(&decoder->queue);
+ if (!dp) {
+ LOG_OOM();
+ sc_mutex_unlock(&decoder->mutex);
+ av_packet_free(&p);
+ return SC_SINK_KO;
+ }
+
+ dp->type = SC_DECODER_PACKET_TYPE_AV_PACKET;
+ dp->packet = p;
+
+ sc_cond_signal(&decoder->cond);
+
+ sc_mutex_unlock(&decoder->mutex);
+ return SC_SINK_OK;
+}
+
static enum sc_sink_result
sc_decoder_push_session(struct sc_decoder *decoder,
const struct sc_stream_session *session) {
- decoder->session = *session;
- return sc_frame_source_sinks_push_session(&decoder->frame_source, session);
+ sc_mutex_lock(&decoder->mutex);
+
+ if (decoder->stopped) {
+ // reject any new packet
+ sc_mutex_unlock(&decoder->mutex);
+ return SC_SINK_STOPPED;
+ }
+
+ struct sc_decoder_packet *dp =
+ sc_vecdeque_push_uninitialized(&decoder->queue);
+ if (!dp) {
+ LOG_OOM();
+ sc_mutex_unlock(&decoder->mutex);
+ return SC_SINK_KO;
+ }
+
+ dp->type = SC_DECODER_PACKET_TYPE_SESSION;
+ dp->session = *session;
+
+ sc_cond_signal(&decoder->cond);
+
+ sc_mutex_unlock(&decoder->mutex);
+ return SC_SINK_OK;
}
static bool
@@ -178,13 +342,33 @@ sc_decoder_packet_sink_push_session(struct sc_packet_sink *sink,
return sc_decoder_push_session(decoder, session);
}
-void
-sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque) {
+bool
+sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,
+ const struct sc_decoder_callbacks *cbs, void *cbs_userdata) {
+ bool ok = sc_mutex_init(&decoder->mutex);
+ if (!ok) {
+ return false;
+ }
+
+ ok = sc_cond_init(&decoder->cond);
+ if (!ok) {
+ sc_mutex_destroy(&decoder->mutex);
+ return false;
+ }
+
decoder->name = name; // statically allocated
sc_frame_source_init(&decoder->frame_source);
decoder->copy_opaque = copy_opaque;
+ sc_vecdeque_init(&decoder->queue);
+ decoder->stopped = false;
+ decoder->ended = false;
+
+ assert(cbs && cbs->on_ended);
+ decoder->cbs = cbs;
+ decoder->cbs_userdata = cbs_userdata;
+
static const struct sc_packet_sink_ops ops = {
.open = sc_decoder_packet_sink_open,
.close = sc_decoder_packet_sink_close,
@@ -193,4 +377,42 @@ sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque)
};
decoder->packet_sink.ops = &ops;
+
+ return true;
+}
+
+bool
+sc_decoder_start(struct sc_decoder *decoder) {
+ char thread_name[16];
+ // "scrcpy-vid-dec" or "scrcpy-aud-dec"
+ snprintf(thread_name, sizeof(thread_name), "scrcpy-%.3s-dec",
+ decoder->name);
+ bool ok = sc_thread_create(&decoder->thread, run_decoder, thread_name,
+ decoder);
+ if (!ok) {
+ LOGE("Could not start decoder (%s) thread", decoder->name);
+ return false;
+ }
+
+ return true;
+}
+
+void
+sc_decoder_stop(struct sc_decoder *decoder) {
+ sc_mutex_lock(&decoder->mutex);
+ decoder->stopped = true;
+ sc_cond_signal(&decoder->cond);
+ sc_mutex_unlock(&decoder->mutex);
+}
+
+void
+sc_decoder_join(struct sc_decoder *decoder) {
+ sc_thread_join(&decoder->thread, NULL);
+}
+
+void
+sc_decoder_destroy(struct sc_decoder *decoder) {
+ sc_vecdeque_destroy(&decoder->queue);
+ sc_cond_destroy(&decoder->cond);
+ sc_mutex_destroy(&decoder->mutex);
}
diff --git a/app/src/decoder.h b/app/src/decoder.h
index 4585c7b5..da5724d3 100644
--- a/app/src/decoder.h
+++ b/app/src/decoder.h
@@ -8,6 +8,23 @@
#include "coords.h"
#include "trait/frame_source.h"
#include "trait/packet_sink.h"
+#include "util/thread.h"
+#include "util/vecdeque.h"
+
+enum sc_decoder_packet_type {
+ SC_DECODER_PACKET_TYPE_AV_PACKET,
+ SC_DECODER_PACKET_TYPE_SESSION,
+};
+
+struct sc_decoder_packet {
+ enum sc_decoder_packet_type type;
+ union {
+ struct sc_stream_session session;
+ AVPacket *packet;
+ };
+};
+
+struct sc_decoder_queue SC_VECDEQUE(struct sc_decoder_packet);
struct sc_decoder {
struct sc_packet_sink packet_sink; // packet sink trait
@@ -16,17 +33,46 @@ struct sc_decoder {
const char *name; // must be statically allocated (e.g. a string literal)
bool copy_opaque;
+ sc_thread thread;
+ sc_mutex mutex;
+ sc_cond cond;
+ // set on sc_decoder_stop(), packet_sink close or decoding failure
+ bool stopped;
+ // set by the decoder thread when it no longer uses the resources
+ bool ended;
+ struct sc_decoder_queue queue;
+
AVCodecContext *ctx;
AVFrame *frame;
struct sc_stream_session session; // only initialized for video stream
struct sc_size frame_size;
+
+ const struct sc_decoder_callbacks *cbs;
+ void *cbs_userdata;
+};
+
+struct sc_decoder_callbacks {
+ void (*on_ended)(struct sc_decoder *decoder, bool success, void *userdata);
};
// The name must be statically allocated (e.g. a string literal)
//
// copy_opaque: Set to true to forward the AVPacket.opaque_ref to the AVFrame
+bool
+sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,
+ const struct sc_decoder_callbacks *cbs, void *cbs_userdata);
+
+bool
+sc_decoder_start(struct sc_decoder *decoder);
+
+void
+sc_decoder_stop(struct sc_decoder *decoder);
+
+void
+sc_decoder_join(struct sc_decoder *decoder);
+
void
-sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque);
+sc_decoder_destroy(struct sc_decoder *decoder);
#endif
diff --git a/app/src/events.h b/app/src/events.h
index d71d8f0d..9f42e29f 100644
--- a/app/src/events.h
+++ b/app/src/events.h
@@ -15,6 +15,7 @@ enum {
SC_EVENT_SERVER_CONNECTION_FAILED,
SC_EVENT_SERVER_CONNECTED,
SC_EVENT_DEMUXER_ERROR,
+ SC_EVENT_DECODER_ERROR,
SC_EVENT_RECORDER_ERROR,
SC_EVENT_TIME_LIMIT_REACHED,
SC_EVENT_CONTROLLER_ERROR,
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
index aacd39f1..5e756319 100644
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@@ -130,6 +130,9 @@ event_loop(struct scrcpy *s, bool has_screen) {
case SC_EVENT_CONTROLLER_ERROR:
LOGE("Controller error");
return SCRCPY_EXIT_FAILURE;
+ case SC_EVENT_DECODER_ERROR:
+ LOGE("Decoder error");
+ return SCRCPY_EXIT_FAILURE;
case SC_EVENT_RECORDER_ERROR:
LOGE("Recorder error");
return SCRCPY_EXIT_FAILURE;
@@ -179,6 +182,16 @@ await_for_server(bool *connected) {
return false;
}
+static void
+sc_decoder_on_ended(struct sc_decoder *decoder, bool success, void *userdata) {
+ (void) decoder;
+ (void) userdata;
+
+ if (!success) {
+ sc_push_event(SC_EVENT_DECODER_ERROR);
+ }
+}
+
static void
sc_recorder_on_ended(struct sc_recorder *recorder, bool success,
void *userdata) {
@@ -345,6 +358,10 @@ scrcpy(struct scrcpy_options *options) {
#endif
bool video_demuxer_started = false;
bool audio_demuxer_started = false;
+ bool video_decoder_initialized = false;
+ bool video_decoder_started = false;
+ bool audio_decoder_initialized = false;
+ bool audio_decoder_started = false;
#ifdef HAVE_USB
bool aoa_hid_initialized = false;
bool keyboard_aoa_initialized = false;
@@ -552,6 +569,10 @@ scrcpy(struct scrcpy_options *options) {
false, &audio_demuxer_cbs, options);
}
+ static const struct sc_decoder_callbacks decoder_cbs = {
+ .on_ended = sc_decoder_on_ended,
+ };
+
bool needs_video_decoder = options->video_playback;
bool needs_audio_decoder = options->audio_playback;
#ifdef HAVE_V4L2
@@ -561,14 +582,35 @@ scrcpy(struct scrcpy_options *options) {
// If a video buffer is present, then the recv date must be forwarded
// from the AVPacket to the AVFrame
bool copy_opaque = has_video_buffer;
- sc_decoder_init(&s->video_decoder, "video", copy_opaque);
+ if (!sc_decoder_init(&s->video_decoder, "video", copy_opaque,
+ &decoder_cbs, NULL)) {
+ goto end;
+ }
+ video_decoder_initialized = true;
+
sc_packet_source_add_sink(&s->video_demuxer.packet_source,
&s->video_decoder.packet_sink);
+
+ if (!sc_decoder_start(&s->video_decoder)) {
+ goto end;
+ }
+ video_decoder_started = true;
}
+
if (needs_audio_decoder) {
- sc_decoder_init(&s->audio_decoder, "audio", false);
+ if (!sc_decoder_init(&s->audio_decoder, "audio", false,
+ &decoder_cbs, NULL)) {
+ goto end;
+ }
+ audio_decoder_initialized = true;
+
sc_packet_source_add_sink(&s->audio_demuxer.packet_source,
&s->audio_decoder.packet_sink);
+
+ if (!sc_decoder_start(&s->audio_decoder)) {
+ goto end;
+ }
+ audio_decoder_started = true;
}
if (options->record_filename) {
@@ -940,6 +982,12 @@ end:
if (file_pusher_initialized) {
sc_file_pusher_stop(&s->file_pusher);
}
+ if (video_decoder_started) {
+ sc_decoder_stop(&s->video_decoder);
+ }
+ if (audio_decoder_started) {
+ sc_decoder_stop(&s->audio_decoder);
+ }
if (recorder_initialized) {
sc_recorder_stop(&s->recorder);
}
@@ -959,10 +1007,18 @@ end:
LOGD("Quit...");
// Close the window immediately, because sc_screen_destroy() may only be
- // called once the video demuxer thread is joined (it may take time)
+ // called once the video decoder thread is joined (it may take time)
sc_screen_hide_window(&s->screen);
}
+ if (video_decoder_started) {
+ sc_decoder_join(&s->video_decoder);
+ }
+
+ if (audio_decoder_started) {
+ sc_decoder_join(&s->audio_decoder);
+ }
+
if (timeout_started) {
sc_timeout_join(&s->timeout);
}
@@ -1009,6 +1065,14 @@ end:
}
#endif
+ if (video_decoder_initialized) {
+ sc_decoder_destroy(&s->video_decoder);
+ }
+
+ if (audio_decoder_initialized) {
+ sc_decoder_destroy(&s->audio_decoder);
+ }
+
// Destroy the screen only after the video demuxer is guaranteed to be
// finished, because otherwise the screen could receive new frames after
// destruction