Display and control your Android device
Add session metadata for the video stream
Romain Vimont78cba1b7c29f
diff --git a/app/src/audio_player.c b/app/src/audio_player.cindex 3e88ea45..270e29d4 100644--- a/app/src/audio_player.c+++ b/app/src/audio_player.c@@ -47,7 +47,10 @@ sc_audio_player_frame_sink_push(struct sc_frame_sink *sink,static boolsc_audio_player_frame_sink_open(struct sc_frame_sink *sink,- const AVCodecContext *ctx) {+ const AVCodecContext *ctx,+ const struct sc_stream_session *session) {+ (void) session;+struct sc_audio_player *ap = DOWNCAST(sink);#ifdef SCRCPY_LAVU_HAS_CHLAYOUTdiff --git a/app/src/decoder.c b/app/src/decoder.cindex 4d0a1daf..e2269b2b 100644--- a/app/src/decoder.c+++ b/app/src/decoder.c@@ -10,14 +10,15 @@#define DOWNCAST(SINK) container_of(SINK, struct sc_decoder, packet_sink)static bool-sc_decoder_open(struct sc_decoder *decoder, AVCodecContext *ctx) {+sc_decoder_open(struct sc_decoder *decoder, AVCodecContext *ctx,+ const struct sc_stream_session *session) {decoder->frame = av_frame_alloc();if (!decoder->frame) {LOG_OOM();return false;}- if (!sc_frame_source_sinks_open(&decoder->frame_source, ctx)) {+ if (!sc_frame_source_sinks_open(&decoder->frame_source, ctx, session)) {av_frame_free(&decoder->frame);return false;}@@ -74,9 +75,16 @@ sc_decoder_push(struct sc_decoder *decoder, const AVPacket *packet) {}static bool-sc_decoder_packet_sink_open(struct sc_packet_sink *sink, AVCodecContext *ctx) {+sc_decoder_push_session(struct sc_decoder *decoder,+ const struct sc_stream_session *session) {+ return sc_frame_source_sinks_push_session(&decoder->frame_source, session);+}++static bool+sc_decoder_packet_sink_open(struct sc_packet_sink *sink, AVCodecContext *ctx,+ const struct sc_stream_session *session) {struct sc_decoder *decoder = DOWNCAST(sink);- return sc_decoder_open(decoder, ctx);+ return sc_decoder_open(decoder, ctx, session);}static void@@ -92,6 +100,14 @@ sc_decoder_packet_sink_push(struct sc_packet_sink *sink,return sc_decoder_push(decoder, packet);}+static bool+sc_decoder_packet_sink_push_session(struct sc_packet_sink *sink,+ const struct sc_stream_session *session) {++ struct sc_decoder *decoder = DOWNCAST(sink);+ return sc_decoder_push_session(decoder, session);+}+voidsc_decoder_init(struct sc_decoder *decoder, const char *name) {decoder->name = name; // statically allocated@@ -101,6 +117,7 @@ sc_decoder_init(struct sc_decoder *decoder, const char *name) {.open = sc_decoder_packet_sink_open,.close = sc_decoder_packet_sink_close,.push = sc_decoder_packet_sink_push,+ .push_session = sc_decoder_packet_sink_push_session,};decoder->packet_sink.ops = &ops;diff --git a/app/src/delay_buffer.c b/app/src/delay_buffer.cindex f75c6f72..2a6fcdb3 100644--- a/app/src/delay_buffer.c+++ b/app/src/delay_buffer.c@@ -109,7 +109,8 @@ stopped:static boolsc_delay_buffer_frame_sink_open(struct sc_frame_sink *sink,- const AVCodecContext *ctx) {+ const AVCodecContext *ctx,+ const struct sc_stream_session *session) {struct sc_delay_buffer *db = DOWNCAST(sink);(void) ctx;@@ -132,7 +133,7 @@ sc_delay_buffer_frame_sink_open(struct sc_frame_sink *sink,sc_vecdeque_init(&db->queue);db->stopped = false;- if (!sc_frame_source_sinks_open(&db->frame_source, ctx)) {+ if (!sc_frame_source_sinks_open(&db->frame_source, ctx, session)) {goto error_destroy_wait_cond;}diff --git a/app/src/demuxer.c b/app/src/demuxer.cindex 885cd6ee..73a4bb2f 100644--- a/app/src/demuxer.c+++ b/app/src/demuxer.c@@ -11,8 +11,8 @@#define SC_PACKET_HEADER_SIZE 12-#define SC_PACKET_FLAG_CONFIG (UINT64_C(1) << 63)-#define SC_PACKET_FLAG_KEY_FRAME (UINT64_C(1) << 62)+#define SC_PACKET_FLAG_CONFIG (UINT64_C(1) << 62)+#define SC_PACKET_FLAG_KEY_FRAME (UINT64_C(1) << 61)#define SC_PACKET_PTS_MASK (SC_PACKET_FLAG_KEY_FRAME - 1)@@ -63,48 +63,75 @@ sc_demuxer_recv_codec_id(struct sc_demuxer *demuxer, uint32_t *codec_id) {return true;}-static bool-sc_demuxer_recv_video_size(struct sc_demuxer *demuxer, uint32_t *width,- uint32_t *height) {- uint8_t data[8];- ssize_t r = net_recv_all(demuxer->socket, data, 8);- if (r < 8) {- return false;- }-- *width = sc_read32be(data);- *height = sc_read32be(data + 4);- return true;-}--static bool-sc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) {+static inline bool+sc_demuxer_recv_header(struct sc_demuxer *demuxer,+ uint8_t buf[static SC_PACKET_HEADER_SIZE]) {// The video and audio streams contain a sequence of raw packets (as// provided by MediaCodec), each prefixed with a "meta" header.//- // The "meta" header length is 12 bytes:+ // The "meta" header length is 12 bytes.+ //+ //+ // If the MSB is 1, then it is a session packet (for a video stream only),+ // which only contains a 12-byte header:+ //+ // byte 0 byte 1 byte 2 byte 3+ // 10000000 00000000 00000000 00000000+ // ^<-------------------------------->+ // | padding+ // `- session packet flag+ //+ // byte 4 byte 5 byte 6 byte 7 byte 8 byte 9 byte 10 byte 11+ // ........ ........ ........ ........ ........ ........ ........ ........+ // <---------------------------------> <--------------------------------->+ // video width video height+ //+ //+ // If the MSB is 0, then it is a media packet, comprised of a 12-byte header+ // followed by <packet_size> bytes containing the packet/frame:+ //// [. . . . . . . .|. . . .]. . . . . . . . . . . . . . . ...// <-------------> <-----> <-----------------------------...// PTS packet raw packet// size//- // It is followed by <packet_size> bytes containing the packet/frame.- //// The most significant bits of the PTS are used for packet flags://- // byte 7 byte 6 byte 5 byte 4 byte 3 byte 2 byte 1 byte 0- // CK...... ........ ........ ........ ........ ........ ........ ........- // ^^<------------------------------------------------------------------->- // || PTS- // | `- key frame- // `-- config packet+ // byte 0 byte 1 byte 2 byte 3 byte 4 byte 5 byte 6 byte 7+ // 0CK..... ........ ........ ........ ........ ........ ........ ........+ // ^^^<------------------------------------------------------------------>+ // ||| PTS+ // || `- key frame+ // | `-- config packet+ // `--- media packet flag+ //+ // byte 8 byte 9 byte 10 byte 11+ // ........ ........ ........ ........ ........ ........ . . .+ // <---------------------------------> <---------------- . . .+ // packet size raw packet+ //+ ssize_t r = net_recv_all(demuxer->socket, buf, SC_PACKET_HEADER_SIZE);+ assert(r <= SC_PACKET_HEADER_SIZE);+ return r == SC_PACKET_HEADER_SIZE;+}- uint8_t header[SC_PACKET_HEADER_SIZE];- ssize_t r = net_recv_all(demuxer->socket, header, SC_PACKET_HEADER_SIZE);- if (r < SC_PACKET_HEADER_SIZE) {- return false;- }+static bool+sc_demuxer_is_session(const uint8_t *header) {+ return header[0] & 0x80;+}+static void+sc_demuxer_parse_session(const uint8_t *header,+ struct sc_stream_session *session) {+ assert(sc_demuxer_is_session(header));+ session->video.width = sc_read32be(&header[4]);+ session->video.height = sc_read32be(&header[8]);+}++static bool+sc_demuxer_recv_packet(struct sc_demuxer *demuxer, const uint8_t *header,+ AVPacket *packet) {+ assert(!sc_demuxer_is_session(header));uint64_t pts_flags = sc_read64be(header);uint32_t len = sc_read32be(&header[8]);assert(len);@@ -114,7 +141,7 @@ sc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) {return false;}- r = net_recv_all(demuxer->socket, packet->data, len);+ ssize_t r = net_recv_all(demuxer->socket, packet->data, len);if (r < 0 || ((uint32_t) r) < len) {av_packet_unref(packet);return false;@@ -187,17 +214,28 @@ run_demuxer(void *data) {codec_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;+ uint8_t header[SC_PACKET_HEADER_SIZE];+ struct sc_stream_session session_data;++ struct sc_stream_session *session = NULL;if (codec->type == AVMEDIA_TYPE_VIDEO) {- uint32_t width;- uint32_t height;- ok = sc_demuxer_recv_video_size(demuxer, &width, &height);+ bool ok = sc_demuxer_recv_header(demuxer, header);if (!ok) {goto finally_free_context;}- codec_ctx->width = width;- codec_ctx->height = height;+ if (!sc_demuxer_is_session(header)) {+ LOGE("Unexpected packet (not a session header)");+ goto finally_free_context;+ }++ session = &session_data;+ sc_demuxer_parse_session(header, session);++ codec_ctx->width = session_data.video.width;+ codec_ctx->height = session_data.video.height;codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;+} else {// Hardcoded audio properties#ifdef SCRCPY_LAVU_HAS_CHLAYOUT@@ -219,7 +257,8 @@ run_demuxer(void *data) {goto finally_free_context;}- if (!sc_packet_source_sinks_open(&demuxer->packet_source, codec_ctx)) {+ if (!sc_packet_source_sinks_open(&demuxer->packet_source, codec_ctx,+ session)) {goto finally_free_context;}@@ -241,27 +280,39 @@ run_demuxer(void *data) {}for (;;) {- bool ok = sc_demuxer_recv_packet(demuxer, packet);+ bool ok = sc_demuxer_recv_header(demuxer, header);if (!ok) {// end of streamstatus = SC_DEMUXER_STATUS_EOS;break;}- if (must_merge_config_packet) {- // Prepend any config packet to the next media packet- ok = sc_packet_merger_merge(&merger, packet);+ if (sc_demuxer_is_session(header)) {+ sc_demuxer_parse_session(header, &session_data);+ ok = sc_packet_source_sinks_push_session(&demuxer->packet_source,+ &session_data);if (!ok) {- av_packet_unref(packet);+ // The sink already logged its concrete errorbreak;}- }+ } else {+ sc_demuxer_recv_packet(demuxer, header, packet);++ if (must_merge_config_packet) {+ // Prepend any config packet to the next media packet+ ok = sc_packet_merger_merge(&merger, packet);+ if (!ok) {+ av_packet_unref(packet);+ break;+ }+ }- ok = sc_packet_source_sinks_push(&demuxer->packet_source, packet);- av_packet_unref(packet);- if (!ok) {- // The sink already logged its concrete error- break;+ ok = sc_packet_source_sinks_push(&demuxer->packet_source, packet);+ av_packet_unref(packet);+ if (!ok) {+ // The sink already logged its concrete error+ break;+ }}}diff --git a/app/src/recorder.c b/app/src/recorder.cindex c26f8f2d..f173d72e 100644--- a/app/src/recorder.c+++ b/app/src/recorder.c@@ -541,7 +541,10 @@ sc_recorder_set_orientation(AVStream *stream, enum sc_orientation orientation) {static boolsc_recorder_video_packet_sink_open(struct sc_packet_sink *sink,- AVCodecContext *ctx) {+ AVCodecContext *ctx,+ const struct sc_stream_session *session) {+ (void) session;+struct sc_recorder *recorder = DOWNCAST_VIDEO(sink);// only written from this thread, no need to lockassert(!recorder->video_init);@@ -635,7 +638,10 @@ sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink,static boolsc_recorder_audio_packet_sink_open(struct sc_packet_sink *sink,- AVCodecContext *ctx) {+ AVCodecContext *ctx,+ const struct sc_stream_session *session) {+ (void) session;+struct sc_recorder *recorder = DOWNCAST_AUDIO(sink);assert(recorder->audio);// only written from this thread, no need to lockdiff --git a/app/src/screen.c b/app/src/screen.cindex 48b7886e..1e4ce0eb 100644--- a/app/src/screen.c+++ b/app/src/screen.c@@ -231,9 +231,11 @@ event_watcher(void *data, SDL_Event *event) {static boolsc_screen_frame_sink_open(struct sc_frame_sink *sink,- const AVCodecContext *ctx) {+ const AVCodecContext *ctx,+ const struct sc_stream_session *session) {assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P);(void) ctx;+ (void) session;struct sc_screen *screen = DOWNCAST(sink);(void) screen;diff --git a/app/src/trait/frame_sink.h b/app/src/trait/frame_sink.hindex 67be4d46..dc94b1fc 100644--- a/app/src/trait/frame_sink.h+++ b/app/src/trait/frame_sink.h@@ -6,6 +6,8 @@#include <stdbool.h>#include <libavcodec/avcodec.h>+#include "trait/packet_sink.h"+/*** Frame sink trait.*@@ -17,9 +19,16 @@ struct sc_frame_sink {struct sc_frame_sink_ops {/* The codec context is valid until the sink is closed */- bool (*open)(struct sc_frame_sink *sink, const AVCodecContext *ctx);+ bool (*open)(struct sc_frame_sink *sink, const AVCodecContext *ctx,+ const struct sc_stream_session *session);void (*close)(struct sc_frame_sink *sink);bool (*push)(struct sc_frame_sink *sink, const AVFrame *frame);++ /**+ * Optional callback to be notified of a new stream session.+ */+ bool (*push_session)(struct sc_frame_sink *sink,+ const struct sc_stream_session *session);};#endifdiff --git a/app/src/trait/frame_source.c b/app/src/trait/frame_source.cindex 56848309..d34e628b 100644--- a/app/src/trait/frame_source.c+++ b/app/src/trait/frame_source.c@@ -27,11 +27,12 @@ sc_frame_source_sinks_close_firsts(struct sc_frame_source *source,boolsc_frame_source_sinks_open(struct sc_frame_source *source,- const AVCodecContext *ctx) {+ const AVCodecContext *ctx,+ const struct sc_stream_session *session) {assert(source->sink_count);for (unsigned i = 0; i < source->sink_count; ++i) {struct sc_frame_sink *sink = source->sinks[i];- if (!sink->ops->open(sink, ctx)) {+ if (!sink->ops->open(sink, ctx, session)) {sc_frame_source_sinks_close_firsts(source, i);return false;}@@ -59,3 +60,18 @@ sc_frame_source_sinks_push(struct sc_frame_source *source,return true;}++bool+sc_frame_source_sinks_push_session(struct sc_frame_source *source,+ const struct sc_stream_session *session) {+ assert(source->sink_count);+ for (unsigned i = 0; i < source->sink_count; ++i) {+ struct sc_frame_sink *sink = source->sinks[i];+ if (sink->ops->push_session &&+ !sink->ops->push_session(sink, session)) {+ return false;+ }+ }++ return true;+}diff --git a/app/src/trait/frame_source.h b/app/src/trait/frame_source.hindex cb1ef905..0a1ea8f1 100644--- a/app/src/trait/frame_source.h+++ b/app/src/trait/frame_source.h@@ -28,7 +28,8 @@ sc_frame_source_add_sink(struct sc_frame_source *source,boolsc_frame_source_sinks_open(struct sc_frame_source *source,- const AVCodecContext *ctx);+ const AVCodecContext *ctx,+ const struct sc_stream_session *session);voidsc_frame_source_sinks_close(struct sc_frame_source *source);@@ -37,4 +38,8 @@ boolsc_frame_source_sinks_push(struct sc_frame_source *source,const AVFrame *frame);+bool+sc_frame_source_sinks_push_session(struct sc_frame_source *source,+ const struct sc_stream_session *session);+#endifdiff --git a/app/src/trait/packet_sink.h b/app/src/trait/packet_sink.hindex e12dea12..db742b2f 100644--- a/app/src/trait/packet_sink.h+++ b/app/src/trait/packet_sink.h@@ -15,12 +15,28 @@ struct sc_packet_sink {const struct sc_packet_sink_ops *ops;};+struct sc_stream_session_video {+ uint32_t width;+ uint32_t height;+};++struct sc_stream_session {+ struct sc_stream_session_video video;+};+struct sc_packet_sink_ops {/* The codec context is valid until the sink is closed */- bool (*open)(struct sc_packet_sink *sink, AVCodecContext *ctx);+ bool (*open)(struct sc_packet_sink *sink, AVCodecContext *ctx,+ const struct sc_stream_session *session);void (*close)(struct sc_packet_sink *sink);bool (*push)(struct sc_packet_sink *sink, const AVPacket *packet);+ /**+ * Optional callback to be notified of a new stream session.+ */+ bool (*push_session)(struct sc_packet_sink *sink,+ const struct sc_stream_session *session);+/*/* Called when the input stream has been disabled at runtime.*diff --git a/app/src/trait/packet_source.c b/app/src/trait/packet_source.cindex 0a2c6c4d..49cb6d14 100644--- a/app/src/trait/packet_source.c+++ b/app/src/trait/packet_source.c@@ -27,11 +27,12 @@ sc_packet_source_sinks_close_firsts(struct sc_packet_source *source,boolsc_packet_source_sinks_open(struct sc_packet_source *source,- AVCodecContext *ctx) {+ AVCodecContext *ctx,+ const struct sc_stream_session *session) {assert(source->sink_count);for (unsigned i = 0; i < source->sink_count; ++i) {struct sc_packet_sink *sink = source->sinks[i];- if (!sink->ops->open(sink, ctx)) {+ if (!sink->ops->open(sink, ctx, session)) {sc_packet_source_sinks_close_firsts(source, i);return false;}@@ -60,6 +61,20 @@ sc_packet_source_sinks_push(struct sc_packet_source *source,return true;}+bool+sc_packet_source_sinks_push_session(struct sc_packet_source *source,+ const struct sc_stream_session *session) {+ assert(source->sink_count);+ for (unsigned i = 0; i < source->sink_count; ++i) {+ struct sc_packet_sink *sink = source->sinks[i];+ if (!sink->ops->push_session(sink, session)) {+ return false;+ }+ }++ return true;+}+voidsc_packet_source_sinks_disable(struct sc_packet_source *source) {assert(source->sink_count);diff --git a/app/src/trait/packet_source.h b/app/src/trait/packet_source.hindex 8788021a..07be4970 100644--- a/app/src/trait/packet_source.h+++ b/app/src/trait/packet_source.h@@ -28,7 +28,8 @@ sc_packet_source_add_sink(struct sc_packet_source *source,boolsc_packet_source_sinks_open(struct sc_packet_source *source,- AVCodecContext *ctx);+ AVCodecContext *ctx,+ const struct sc_stream_session *session);voidsc_packet_source_sinks_close(struct sc_packet_source *source);@@ -37,6 +38,10 @@ boolsc_packet_source_sinks_push(struct sc_packet_source *source,const AVPacket *packet);+bool+sc_packet_source_sinks_push_session(struct sc_packet_source *source,+ const struct sc_stream_session *session);+voidsc_packet_source_sinks_disable(struct sc_packet_source *source);diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.cindex da9e02ef..92551149 100644--- a/app/src/v4l2_sink.c+++ b/app/src/v4l2_sink.c@@ -146,9 +146,11 @@ run_v4l2_sink(void *data) {}static bool-sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {+sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx,+ const struct sc_stream_session *session) {assert(ctx->pix_fmt == AV_PIX_FMT_YUV420P);(void) ctx;+ (void) session;bool ok = sc_frame_buffer_init(&vs->fb);if (!ok) {@@ -326,9 +328,10 @@ sc_v4l2_sink_push(struct sc_v4l2_sink *vs, const AVFrame *frame) {}static bool-sc_v4l2_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx) {+sc_v4l2_frame_sink_open(struct sc_frame_sink *sink, const AVCodecContext *ctx,+ const struct sc_stream_session *session) {struct sc_v4l2_sink *vs = DOWNCAST(sink);- return sc_v4l2_sink_open(vs, ctx);+ return sc_v4l2_sink_open(vs, ctx, session);}static voiddiff --git a/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/device/Streamer.javaindex 93c5077b..73d28292 100644--- a/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java+++ b/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java@@ -14,8 +14,9 @@ import java.util.Arrays;public final class Streamer {- private static final long PACKET_FLAG_CONFIG = 1L << 63;- private static final long PACKET_FLAG_KEY_FRAME = 1L << 62;+ private static final long PACKET_FLAG_SESSION = 1L << 63;+ private static final long PACKET_FLAG_CONFIG = 1L << 62;+ private static final long PACKET_FLAG_KEY_FRAME = 1L << 61;private final FileDescriptor fd;private final Codec codec;@@ -44,12 +45,10 @@ public final class Streamer {}}- public void writeVideoHeader(Size videoSize) throws IOException {+ public void writeVideoHeader() throws IOException {if (sendStreamMeta) {- ByteBuffer buffer = ByteBuffer.allocate(12);+ ByteBuffer buffer = ByteBuffer.allocate(4);buffer.putInt(codec.getId());- buffer.putInt(videoSize.getWidth());- buffer.putInt(videoSize.getHeight());buffer.flip();IO.writeFully(fd, buffer);}@@ -89,6 +88,18 @@ public final class Streamer {writePacket(codecBuffer, pts, config, keyFrame);}+ public void writeSessionMeta(int width, int height) throws IOException {+ if (sendStreamMeta) {+ headerBuffer.clear();++ headerBuffer.putInt((int) (PACKET_FLAG_SESSION >> 32)); // Set the first bit to 1+ headerBuffer.putInt(width);+ headerBuffer.putInt(height);+ headerBuffer.flip();+ IO.writeFully(fd, headerBuffer);+ }+ }+private void writeFrameMeta(FileDescriptor fd, int packetSize, long pts, boolean config, boolean keyFrame) throws IOException {headerBuffer.clear();diff --git a/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java b/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.javaindex 236a5f48..461ba472 100644--- a/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java+++ b/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java@@ -71,16 +71,13 @@ public class SurfaceEncoder implements AsyncProcessor {try {boolean alive;- boolean headerWritten = false;++ streamer.writeVideoHeader();do {reset.consumeReset(); // If a capture reset was requested, it is implicitly fulfilledcapture.prepare();Size size = capture.getSize();- if (!headerWritten) {- streamer.writeVideoHeader(size);- headerWritten = true;- }format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth());format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight());@@ -107,6 +104,7 @@ public class SurfaceEncoder implements AsyncProcessor {boolean resetRequested = reset.consumeReset();if (!resetRequested) {// If a reset is requested during encode(), it will interrupt the encoding by an EOS+ streamer.writeSessionMeta(size.getWidth(), size.getHeight());encode(mediaCodec, streamer);}// The capture might have been closed internally (for example if the camera is disconnected)