Display and control your Android device
Pass video size as codec metadata
Romain Vimont238ab872ba0e
diff --git a/app/src/demuxer.c b/app/src/demuxer.cindex eabcb81e..96303155 100644--- a/app/src/demuxer.c+++ b/app/src/demuxer.c@@ -57,6 +57,20 @@ sc_demuxer_recv_codec_id(struct sc_demuxer *demuxer, uint32_t *codec_id) {return true;}+static bool+sc_demuxer_recv_video_size(struct sc_demuxer *demuxer, uint32_t *width,+ uint32_t *height) {+ uint8_t data[8];+ ssize_t r = net_recv_all(demuxer->socket, data, 8);+ if (r < 8) {+ return false;+ }++ *width = sc_read32be(data);+ *height = sc_read32be(data + 4);+ return true;+}+static boolsc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) {// The video stream contains raw packets, without time information. When we@@ -169,7 +183,15 @@ run_demuxer(void *data) {codec_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;if (codec->type == AVMEDIA_TYPE_VIDEO) {- // Hardcoded video properties+ uint32_t width;+ uint32_t height;+ ok = sc_demuxer_recv_video_size(demuxer, &width, &height);+ if (!ok) {+ goto finally_free_context;+ }++ codec_ctx->width = width;+ codec_ctx->height = height;codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;} else {// Hardcoded audio propertiesdiff --git a/app/src/events.h b/app/src/events.hindex 0a45b652..609e3198 100644--- a/app/src/events.h+++ b/app/src/events.h@@ -5,3 +5,4 @@#define SC_EVENT_USB_DEVICE_DISCONNECTED (SDL_USEREVENT + 4)#define SC_EVENT_DEMUXER_ERROR (SDL_USEREVENT + 5)#define SC_EVENT_RECORDER_ERROR (SDL_USEREVENT + 6)+#define SC_EVENT_SCREEN_INIT_SIZE (SDL_USEREVENT + 7)diff --git a/app/src/recorder.c b/app/src/recorder.cindex e8484256..2fc95eca 100644--- a/app/src/recorder.c+++ b/app/src/recorder.c@@ -479,9 +479,6 @@ sc_recorder_video_packet_sink_open(struct sc_packet_sink *sink,return false;}- stream->codecpar->width = recorder->declared_frame_size.width;- stream->codecpar->height = recorder->declared_frame_size.height;-recorder->video_stream_index = stream->index;recorder->video_init = true;@@ -643,7 +640,6 @@ sc_recorder_audio_packet_sink_disable(struct sc_packet_sink *sink) {boolsc_recorder_init(struct sc_recorder *recorder, const char *filename,enum sc_record_format format, bool audio,- struct sc_size declared_frame_size,const struct sc_recorder_callbacks *cbs, void *cbs_userdata) {recorder->filename = strdup(filename);if (!recorder->filename) {@@ -679,7 +675,6 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename,recorder->audio_stream_index = -1;recorder->format = format;- recorder->declared_frame_size = declared_frame_size;assert(cbs && cbs->on_ended);recorder->cbs = cbs;diff --git a/app/src/recorder.h b/app/src/recorder.hindex 35758db7..41b8db65 100644--- a/app/src/recorder.h+++ b/app/src/recorder.h@@ -31,7 +31,6 @@ struct sc_recorder {char *filename;enum sc_record_format format;AVFormatContext *ctx;- struct sc_size declared_frame_size;sc_thread thread;sc_mutex mutex;@@ -61,7 +60,6 @@ struct sc_recorder_callbacks {boolsc_recorder_init(struct sc_recorder *recorder, const char *filename,enum sc_record_format format, bool audio,- struct sc_size declared_frame_size,const struct sc_recorder_callbacks *cbs, void *cbs_userdata);booldiff --git a/app/src/scrcpy.c b/app/src/scrcpy.cindex 09a8f918..9e5ec6f0 100644--- a/app/src/scrcpy.c+++ b/app/src/scrcpy.c@@ -473,7 +473,7 @@ scrcpy(struct scrcpy_options *options) {};if (!sc_recorder_init(&s->recorder, options->record_filename,options->record_format, options->audio,- info->frame_size, &recorder_cbs, NULL)) {+ &recorder_cbs, NULL)) {goto end;}recorder_initialized = true;@@ -660,7 +660,6 @@ aoa_hid_end:.clipboard_autosync = options->clipboard_autosync,.shortcut_mods = &options->shortcut_mods,.window_title = window_title,- .frame_size = info->frame_size,.always_on_top = options->always_on_top,.window_x = options->window_x,.window_y = options->window_y,@@ -697,8 +696,7 @@ aoa_hid_end:#ifdef HAVE_V4L2if (options->v4l2_device) {- if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device,- info->frame_size)) {+ if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device)) {goto end;}diff --git a/app/src/screen.c b/app/src/screen.cindex 56463711..f74fd8a5 100644--- a/app/src/screen.c+++ b/app/src/screen.c@@ -239,7 +239,7 @@ sc_screen_update_content_rect(struct sc_screen *screen) {}}-static inline SDL_Texture *+static boolcreate_texture(struct sc_screen *screen) {SDL_Renderer *renderer = screen->renderer;struct sc_size size = screen->frame_size;@@ -247,7 +247,8 @@ create_texture(struct sc_screen *screen) {SDL_TEXTUREACCESS_STREAMING,size.width, size.height);if (!texture) {- return NULL;+ LOGE("Could not create texture: %s", SDL_GetError());+ return false;}if (screen->mipmaps) {@@ -263,7 +264,8 @@ create_texture(struct sc_screen *screen) {SDL_GL_UnbindTexture(texture);}- return texture;+ screen->texture = texture;+ return true;}// render the texture to the renderer@@ -335,7 +337,25 @@ sc_screen_frame_sink_open(struct sc_frame_sink *sink,(void) ctx;struct sc_screen *screen = DOWNCAST(sink);- (void) screen;++ assert(ctx->width > 0 && ctx->width <= 0xFFFF);+ assert(ctx->height > 0 && ctx->height <= 0xFFFF);+ // screen->frame_size is never used before the event is pushed, and the+ // event acts as a memory barrier so it is safe without mutex+ screen->frame_size.width = ctx->width;+ screen->frame_size.height = ctx->height;++ static SDL_Event event = {+ .type = SC_EVENT_SCREEN_INIT_SIZE,+ };++ // Post the event on the UI thread (the texture must be created from there)+ int ret = SDL_PushEvent(&event);+ if (ret < 0) {+ LOGW("Could not post init size event: %s", SDL_GetError());+ return false;+ }+#ifndef NDEBUGscreen->open = true;#endif@@ -410,14 +430,10 @@ sc_screen_init(struct sc_screen *screen,goto error_destroy_frame_buffer;}- screen->frame_size = params->frame_size;screen->rotation = params->rotation;if (screen->rotation) {LOGI("Initial display rotation set to %u", screen->rotation);}- struct sc_size content_size =- get_rotated_size(screen->frame_size, screen->rotation);- screen->content_size = content_size;uint32_t window_flags = SDL_WINDOW_HIDDEN| SDL_WINDOW_RESIZABLE@@ -485,18 +501,10 @@ sc_screen_init(struct sc_screen *screen,LOGW("Could not load icon");}- LOGI("Initial texture: %" PRIu16 "x%" PRIu16, params->frame_size.width,- params->frame_size.height);- screen->texture = create_texture(screen);- if (!screen->texture) {- LOGE("Could not create texture: %s", SDL_GetError());- goto error_destroy_renderer;- }-screen->frame = av_frame_alloc();if (!screen->frame) {LOG_OOM();- goto error_destroy_texture;+ goto error_destroy_renderer;}struct sc_input_manager_params im_params = {@@ -531,8 +539,6 @@ sc_screen_init(struct sc_screen *screen,return true;-error_destroy_texture:- SDL_DestroyTexture(screen->texture);error_destroy_renderer:SDL_DestroyRenderer(screen->renderer);error_destroy_window:@@ -591,7 +597,9 @@ sc_screen_destroy(struct sc_screen *screen) {assert(!screen->open);#endifav_frame_free(&screen->frame);- SDL_DestroyTexture(screen->texture);+ if (screen->texture) {+ SDL_DestroyTexture(screen->texture);+ }SDL_DestroyRenderer(screen->renderer);SDL_DestroyWindow(screen->window);sc_fps_counter_destroy(&screen->fps_counter);@@ -655,6 +663,23 @@ sc_screen_set_rotation(struct sc_screen *screen, unsigned rotation) {sc_screen_render(screen, true);}+static bool+sc_screen_init_size(struct sc_screen *screen) {+ // Before first frame+ assert(!screen->has_frame);+ assert(!screen->texture);++ // The requested size is passed via screen->frame_size++ struct sc_size content_size =+ get_rotated_size(screen->frame_size, screen->rotation);+ screen->content_size = content_size;++ LOGI("Initial texture: %" PRIu16 "x%" PRIu16,+ screen->frame_size.width, screen->frame_size.height);+ return create_texture(screen);+}+// recreate the texture and resize the window if the frame size has changedstatic boolprepare_for_frame(struct sc_screen *screen, struct sc_size new_frame_size) {@@ -673,11 +698,7 @@ prepare_for_frame(struct sc_screen *screen, struct sc_size new_frame_size) {LOGI("New texture: %" PRIu16 "x%" PRIu16,screen->frame_size.width, screen->frame_size.height);- screen->texture = create_texture(screen);- if (!screen->texture) {- LOGE("Could not create texture: %s", SDL_GetError());- return false;- }+ return create_texture(screen);}return true;@@ -795,6 +816,14 @@ sc_screen_handle_event(struct sc_screen *screen, SDL_Event *event) {bool relative_mode = sc_screen_is_relative_mode(screen);switch (event->type) {+ case SC_EVENT_SCREEN_INIT_SIZE:+ // The initial size is passed via screen->frame_size+ bool ok = sc_screen_init_size(screen);+ if (!ok) {+ LOGE("Could not initialize screen size");+ return false;+ }+ return true;case SC_EVENT_NEW_FRAME: {bool ok = sc_screen_update_frame(screen);if (!ok) {diff --git a/app/src/screen.h b/app/src/screen.hindex 57927894..4fca04d8 100644--- a/app/src/screen.h+++ b/app/src/screen.h@@ -78,7 +78,6 @@ struct sc_screen_params {const struct sc_shortcut_mods *shortcut_mods;const char *window_title;- struct sc_size frame_size;bool always_on_top;int16_t window_x; // accepts SC_WINDOW_POSITION_UNDEFINEDdiff --git a/app/src/server.c b/app/src/server.cindex 7b503427..8c4e9a95 100644--- a/app/src/server.c+++ b/app/src/server.c@@ -441,9 +441,9 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params,static booldevice_read_info(struct sc_intr *intr, sc_socket device_socket,struct sc_server_info *info) {- unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH + 4];+ unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH];ssize_t r = net_recv_all_intr(intr, device_socket, buf, sizeof(buf));- if (r < SC_DEVICE_NAME_FIELD_LENGTH + 4) {+ if (r < SC_DEVICE_NAME_FIELD_LENGTH) {LOGE("Could not retrieve device information");return false;}@@ -451,9 +451,6 @@ device_read_info(struct sc_intr *intr, sc_socket device_socket,buf[SC_DEVICE_NAME_FIELD_LENGTH - 1] = '\0';memcpy(info->device_name, (char *) buf, sizeof(info->device_name));- unsigned char *fields = &buf[SC_DEVICE_NAME_FIELD_LENGTH];- info->frame_size.width = sc_read16be(fields);- info->frame_size.height = sc_read16be(&fields[2]);return true;}diff --git a/app/src/server.h b/app/src/server.hindex 8edf2666..c425856b 100644--- a/app/src/server.h+++ b/app/src/server.h@@ -18,7 +18,6 @@#define SC_DEVICE_NAME_FIELD_LENGTH 64struct sc_server_info {char device_name[SC_DEVICE_NAME_FIELD_LENGTH];- struct sc_size frame_size;};struct sc_server_params {diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.cindex c6714d18..717d2bd5 100644--- a/app/src/v4l2_sink.c+++ b/app/src/v4l2_sink.c@@ -210,9 +210,6 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {goto error_avformat_free_context;}- ostream->codecpar->width = vs->frame_size.width;- ostream->codecpar->height = vs->frame_size.height;-int ret = avio_open(&vs->format_ctx->pb, vs->device_name, AVIO_FLAG_WRITE);if (ret < 0) {LOGE("Failed to open output device: %s", vs->device_name);@@ -226,8 +223,8 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {goto error_avio_close;}- vs->encoder_ctx->width = vs->frame_size.width;- vs->encoder_ctx->height = vs->frame_size.height;+ vs->encoder_ctx->width = ctx->width;+ vs->encoder_ctx->height = ctx->height;vs->encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;vs->encoder_ctx->time_base.num = 1;vs->encoder_ctx->time_base.den = 1;@@ -343,16 +340,13 @@ sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {}bool-sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,- struct sc_size frame_size) {+sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name) {vs->device_name = strdup(device_name);if (!vs->device_name) {LOGE("Could not strdup v4l2 device name");return false;}- vs->frame_size = frame_size;-static const struct sc_frame_sink_ops ops = {.open = sc_v4l2_frame_sink_open,.close = sc_v4l2_frame_sink_close,diff --git a/app/src/v4l2_sink.h b/app/src/v4l2_sink.hindex 789e31c3..365a739d 100644--- a/app/src/v4l2_sink.h+++ b/app/src/v4l2_sink.h@@ -19,7 +19,6 @@ struct sc_v4l2_sink {AVCodecContext *encoder_ctx;char *device_name;- struct sc_size frame_size;sc_thread thread;sc_mutex mutex;@@ -33,8 +32,7 @@ struct sc_v4l2_sink {};bool-sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,- struct sc_size frame_size);+sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name);voidsc_v4l2_sink_destroy(struct sc_v4l2_sink *vs);diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.javaindex 0ba424ca..24d685c5 100644--- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java+++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java@@ -101,7 +101,7 @@ public final class AudioEncoder implements AsyncProcessor {}private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {- streamer.writeHeader();+ streamer.writeAudioHeader();while (!Thread.currentThread().isInterrupted()) {OutputTask task = outputTasks.take();diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java b/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.javaindex 2e483daa..4b1b5bd0 100644--- a/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java+++ b/server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java@@ -26,7 +26,7 @@ public final class AudioRawRecorder implements AsyncProcessor {try {capture.start();- streamer.writeHeader();+ streamer.writeAudioHeader();while (!Thread.currentThread().isInterrupted()) {buffer.position(0);int r = capture.read(buffer, READ_SIZE, bufferInfo);diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.javaindex 3e743621..4bfff726 100644--- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java+++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java@@ -122,18 +122,14 @@ public final class DesktopConnection implements Closeable {}}- public void sendDeviceMeta(String deviceName, int width, int height) throws IOException {- byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH + 4];+ public void sendDeviceMeta(String deviceName) throws IOException {+ byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH];byte[] deviceNameBytes = deviceName.getBytes(StandardCharsets.UTF_8);int len = StringUtils.getUtf8TruncationIndex(deviceNameBytes, DEVICE_NAME_FIELD_LENGTH - 1);System.arraycopy(deviceNameBytes, 0, buffer, 0, len);// byte[] are always 0-initialized in java, no need to set '\0' explicitly- buffer[DEVICE_NAME_FIELD_LENGTH] = (byte) (width >> 8);- buffer[DEVICE_NAME_FIELD_LENGTH + 1] = (byte) width;- buffer[DEVICE_NAME_FIELD_LENGTH + 2] = (byte) (height >> 8);- buffer[DEVICE_NAME_FIELD_LENGTH + 3] = (byte) height;IO.writeFully(videoFd, buffer, 0, buffer.length);}diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.javaindex f5f996ba..015cc993 100644--- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java+++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java@@ -66,7 +66,7 @@ public class ScreenEncoder implements Device.RotationListener {IBinder display = createDisplay();device.setRotationListener(this);- streamer.writeHeader();+ streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());boolean alive;try {diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.javaindex 2ece7415..5800487d 100644--- a/server/src/main/java/com/genymobile/scrcpy/Server.java+++ b/server/src/main/java/com/genymobile/scrcpy/Server.java@@ -96,8 +96,7 @@ public final class Server {try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) {if (options.getSendDeviceMeta()) {- Size videoSize = device.getScreenInfo().getVideoSize();- connection.sendDeviceMeta(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight());+ connection.sendDeviceMeta(Device.getDeviceName());}if (control) {diff --git a/server/src/main/java/com/genymobile/scrcpy/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/Streamer.javaindex f099cf4f..39f74fb6 100644--- a/server/src/main/java/com/genymobile/scrcpy/Streamer.java+++ b/server/src/main/java/com/genymobile/scrcpy/Streamer.java@@ -30,8 +30,7 @@ public final class Streamer {public Codec getCodec() {return codec;}-- public void writeHeader() throws IOException {+ public void writeAudioHeader() throws IOException {if (sendCodecMeta) {ByteBuffer buffer = ByteBuffer.allocate(4);buffer.putInt(codec.getId());@@ -40,6 +39,17 @@ public final class Streamer {}}+ public void writeVideoHeader(Size videoSize) throws IOException {+ if (sendCodecMeta) {+ ByteBuffer buffer = ByteBuffer.allocate(12);+ buffer.putInt(codec.getId());+ buffer.putInt(videoSize.getWidth());+ buffer.putInt(videoSize.getHeight());+ buffer.flip();+ IO.writeFully(fd, buffer);+ }+ }+public void writeDisableStream(boolean error) throws IOException {// Writing a specific code as codec-id means that the device disables the stream// code 0: it explicitly disables the stream (because it could not capture audio), scrcpy should continue mirroring video only