Skip to content

Display and control your Android device

Use the packet reception date for video buffering

Romain Vimont8e9d566ee780
diff --git a/app/src/decoder.c b/app/src/decoder.c
index 9b3cec83..187eea72 100644
--- a/app/src/decoder.c
+++ b/app/src/decoder.c
@@ -31,6 +31,11 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec,
decoder->ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
+ if (decoder->copy_opaque) {
+ // Propagate AVPacket.opaque_ref (the recv_date) to the decoded AVFrame
+ decoder->ctx->flags |= AV_CODEC_FLAG_COPY_OPAQUE;
+ }
+
r = avcodec_open2(decoder->ctx, codec, NULL);
if (r < 0) {
LOGE("Decoder '%s': could not open codec", decoder->name);
@@ -175,10 +180,12 @@ sc_decoder_packet_sink_push_session(struct sc_packet_sink *sink,
}
void
-sc_decoder_init(struct sc_decoder *decoder, const char *name) {
+sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque) {
decoder->name = name; // statically allocated
sc_frame_source_init(&decoder->frame_source);
+ decoder->copy_opaque = copy_opaque;
+
static const struct sc_packet_sink_ops ops = {
.open = sc_decoder_packet_sink_open,
.close = sc_decoder_packet_sink_close,
diff --git a/app/src/decoder.h b/app/src/decoder.h
index 4a2f24e0..4585c7b5 100644
--- a/app/src/decoder.h
+++ b/app/src/decoder.h
@@ -14,6 +14,7 @@ struct sc_decoder {
struct sc_frame_source frame_source; // frame source trait
const char *name; // must be statically allocated (e.g. a string literal)
+ bool copy_opaque;
AVCodecContext *ctx;
AVFrame *frame;
@@ -23,7 +24,9 @@ struct sc_decoder {
};
// The name must be statically allocated (e.g. a string literal)
+//
+// copy_opaque: Set to true to forward the AVPacket.opaque_ref to the AVFrame
void
-sc_decoder_init(struct sc_decoder *decoder, const char *name);
+sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque);
#endif
diff --git a/app/src/demuxer.c b/app/src/demuxer.c
index 7e946de7..19a81a33 100644
--- a/app/src/demuxer.c
+++ b/app/src/demuxer.c
@@ -154,6 +154,20 @@ sc_demuxer_recv_packet(struct sc_demuxer *demuxer, const uint8_t *header,
return false;
}
+ if (demuxer->set_recv_date) {
+ sc_tick recv_date = sc_tick_now();
+
+ // Store the recv date as an opaque ref
+ packet->opaque_ref = av_buffer_alloc(sizeof(sc_tick));
+ if (!packet->opaque_ref) {
+ av_packet_unref(packet);
+ LOG_OOM();
+ return false;
+ }
+
+ *(sc_tick *) packet->opaque_ref->data = recv_date;
+ }
+
if (pts_flags & SC_PACKET_FLAG_CONFIG) {
packet->pts = AV_NOPTS_VALUE;
} else {
@@ -342,6 +356,7 @@ end:
void
sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket,
+ bool set_recv_date,
const struct sc_demuxer_callbacks *cbs, void *cbs_userdata) {
assert(socket != SC_SOCKET_NONE);
@@ -349,6 +364,8 @@ sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket,
demuxer->socket = socket;
sc_packet_source_init(&demuxer->packet_source);
+ demuxer->set_recv_date = set_recv_date;
+
assert(cbs && cbs->on_ended);
demuxer->cbs = cbs;
diff --git a/app/src/demuxer.h b/app/src/demuxer.h
index 2b7cb703..417cb2d4 100644
--- a/app/src/demuxer.h
+++ b/app/src/demuxer.h
@@ -13,6 +13,7 @@ struct sc_demuxer {
struct sc_packet_source packet_source; // packet source trait
const char *name; // must be statically allocated (e.g. a string literal)
+ bool set_recv_date;
sc_socket socket;
sc_thread thread;
@@ -35,6 +36,7 @@ struct sc_demuxer_callbacks {
// The name must be statically allocated (e.g. a string literal)
void
sc_demuxer_init(struct sc_demuxer *demuxer, const char *name, sc_socket socket,
+ bool set_recv_date,
const struct sc_demuxer_callbacks *cbs, void *cbs_userdata);
bool
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
index f5b644a8..aacd39f1 100644
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@@ -529,12 +529,19 @@ scrcpy(struct scrcpy_options *options) {
file_pusher_initialized = true;
}
+ bool has_video_buffer = options->video_buffer;
+#ifdef HAVE_V4L2
+ has_video_buffer |= options->v4l2_buffer;
+#endif
+
if (options->video) {
static const struct sc_demuxer_callbacks video_demuxer_cbs = {
.on_ended = sc_video_demuxer_on_ended,
};
+ // If a video buffer is present, then the recv date must be set
+ bool set_recv_date = has_video_buffer;
sc_demuxer_init(&s->video_demuxer, "video", s->server.video_socket,
- &video_demuxer_cbs, NULL);
+ set_recv_date, &video_demuxer_cbs, NULL);
}
if (options->audio) {
@@ -542,7 +549,7 @@ scrcpy(struct scrcpy_options *options) {
.on_ended = sc_audio_demuxer_on_ended,
};
sc_demuxer_init(&s->audio_demuxer, "audio", s->server.audio_socket,
- &audio_demuxer_cbs, options);
+ false, &audio_demuxer_cbs, options);
}
bool needs_video_decoder = options->video_playback;
@@ -551,12 +558,15 @@ scrcpy(struct scrcpy_options *options) {
needs_video_decoder |= !!options->v4l2_device;
#endif
if (needs_video_decoder) {
- sc_decoder_init(&s->video_decoder, "video");
+ // If a video buffer is present, then the recv date must be forwarded
+ // from the AVPacket to the AVFrame
+ bool copy_opaque = has_video_buffer;
+ sc_decoder_init(&s->video_decoder, "video", copy_opaque);
sc_packet_source_add_sink(&s->video_demuxer.packet_source,
&s->video_decoder.packet_sink);
}
if (needs_audio_decoder) {
- sc_decoder_init(&s->audio_decoder, "audio");
+ sc_decoder_init(&s->audio_decoder, "audio", false);
sc_packet_source_add_sink(&s->audio_demuxer.packet_source,
&s->audio_decoder.packet_sink);
}
diff --git a/app/src/video_regulator.c b/app/src/video_regulator.c
index a4d85e2d..2c0f1416 100644
--- a/app/src/video_regulator.c
+++ b/app/src/video_regulator.c
@@ -210,8 +210,13 @@ sc_video_regulator_frame_sink_push(struct sc_frame_sink *sink,
return false;
}
+ assert(frame->opaque_ref);
+ assert(frame->opaque_ref->size == sizeof(sc_tick));
+ // Retrieve the recv date set on the AVPacket by the demuxer
+ sc_tick recv_date = *(const sc_tick *) frame->opaque_ref->data;
+
sc_tick pts = SC_TICK_FROM_US(frame->pts);
- sc_clock_update(&vr->clock, sc_tick_now(), pts);
+ sc_clock_update(&vr->clock, recv_date, pts);
sc_cond_signal(&vr->wait_cond);
if (vr->first_frame_asap && vr->clock.range == 1) {