Display and control your Android device
Wait SET_CLIPBOARD ack before Ctrl+v via HID
Romain Vimont5d17bcf1bc38
diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.cindex 4c0b2bda..6fd32610 100644--- a/app/src/aoa_hid.c+++ b/app/src/aoa_hid.c@@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,hid_event->accessory_id = accessory_id;hid_event->buffer = buffer;hid_event->size = buffer_size;- hid_event->delay = 0;+ hid_event->ack_to_wait = SC_SEQUENCE_INVALID;}void@@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {}bool-sc_aoa_init(struct sc_aoa *aoa, const char *serial) {+sc_aoa_init(struct sc_aoa *aoa, const char *serial,+ struct sc_acksync *acksync) {+ assert(acksync);+cbuf_init(&aoa->queue);if (!sc_mutex_init(&aoa->mutex)) {@@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) {}aoa->stopped = false;+ aoa->acksync = acksync;return true;}@@ -332,23 +336,28 @@ run_aoa_thread(void *data) {assert(non_empty);(void) non_empty;- assert(event.delay >= 0);- if (event.delay) {- // Wait during the specified delay before injecting the HID event- sc_tick deadline = sc_tick_now() + event.delay;- bool timed_out = false;- while (!aoa->stopped && !timed_out) {- timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex,- deadline);- }- if (aoa->stopped) {- sc_mutex_unlock(&aoa->mutex);+ uint64_t ack_to_wait = event.ack_to_wait;+ sc_mutex_unlock(&aoa->mutex);++ if (ack_to_wait != SC_SEQUENCE_INVALID) {+ LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait);+ // Do not block the loop indefinitely if the ack never comes (it should+ // never happen)+ sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500);+ enum sc_acksync_wait_result result =+ sc_acksync_wait(aoa->acksync, ack_to_wait, deadline);++ if (result == SC_ACKSYNC_WAIT_TIMEOUT) {+ LOGW("Ack not received after 500ms, discarding HID event");+ sc_hid_event_destroy(&event);+ continue;+ } else if (result == SC_ACKSYNC_WAIT_INTR) {+ // stopped+ sc_hid_event_destroy(&event);break;}}- sc_mutex_unlock(&aoa->mutex);-bool ok = sc_aoa_send_hid_event(aoa, &event);sc_hid_event_destroy(&event);if (!ok) {@@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) {aoa->stopped = true;sc_cond_signal(&aoa->event_cond);sc_mutex_unlock(&aoa->mutex);++ sc_acksync_interrupt(aoa->acksync);}voiddiff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.hindex 24cef502..e8fb9708 100644--- a/app/src/aoa_hid.h+++ b/app/src/aoa_hid.h@@ -6,6 +6,7 @@#include <libusb-1.0/libusb.h>+#include "util/acksync.h"#include "util/cbuf.h"#include "util/thread.h"#include "util/tick.h"@@ -14,7 +15,7 @@ struct sc_hid_event {uint16_t accessory_id;unsigned char *buffer;uint16_t size;- sc_tick delay;+ uint64_t ack_to_wait;};// Takes ownership of buffer@@ -36,10 +37,12 @@ struct sc_aoa {sc_cond event_cond;bool stopped;struct sc_hid_event_queue queue;++ struct sc_acksync *acksync;};bool-sc_aoa_init(struct sc_aoa *aoa, const char *serial);+sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync);voidsc_aoa_destroy(struct sc_aoa *aoa);diff --git a/app/src/controller.c b/app/src/controller.cindex e486ea72..6cf3d20e 100644--- a/app/src/controller.c+++ b/app/src/controller.c@@ -5,10 +5,11 @@#include "util/log.h"bool-controller_init(struct controller *controller, sc_socket control_socket) {+controller_init(struct controller *controller, sc_socket control_socket,+ struct sc_acksync *acksync) {cbuf_init(&controller->queue);- bool ok = receiver_init(&controller->receiver, control_socket);+ bool ok = receiver_init(&controller->receiver, control_socket, acksync);if (!ok) {return false;}diff --git a/app/src/controller.h b/app/src/controller.hindex e7004131..00267878 100644--- a/app/src/controller.h+++ b/app/src/controller.h@@ -7,6 +7,7 @@#include "control_msg.h"#include "receiver.h"+#include "util/acksync.h"#include "util/cbuf.h"#include "util/net.h"#include "util/thread.h"@@ -24,7 +25,8 @@ struct controller {};bool-controller_init(struct controller *controller, sc_socket control_socket);+controller_init(struct controller *controller, sc_socket control_socket,+ struct sc_acksync *acksync);voidcontroller_destroy(struct controller *controller);diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.cindex 2809ccd6..2afc09b2 100644--- a/app/src/hid_keyboard.c+++ b/app/src/hid_keyboard.c@@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) {static voidsc_key_processor_process_key(struct sc_key_processor *kp,const SDL_KeyboardEvent *event,- bool device_clipboard_set) {+ uint64_t ack_to_wait) {if (event->repeat) {// In USB HID protocol, key repeat is handled by the host (Android), so// just ignore key repeat here.@@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp,}}- if (device_clipboard_set) {+ if (ack_to_wait) {// Ctrl+v is pressed, so clipboard synchronization has been- // requested. Wait a bit so that the clipboard is set before- // injecting Ctrl+v via HID, otherwise it would paste the old- // clipboard content.- hid_event.delay = SC_TICK_FROM_MS(5);+ // requested. Wait until clipboard synchronization is acknowledged+ // by the server, otherwise it could paste the old clipboard+ // content.+ hid_event.ack_to_wait = ack_to_wait;}if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {@@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {.process_text = sc_key_processor_process_text,};+ // Clipboard synchronization is requested over the control socket, while HID+ // events are sent over AOA, so it must wait for clipboard synchronization+ // to be acknowledged by the device before injecting Ctrl+v.+ kb->key_processor.async_paste = true;kb->key_processor.ops = &ops;return true;diff --git a/app/src/input_manager.c b/app/src/input_manager.cindex e7afee77..f1715b79 100644--- a/app/src/input_manager.c+++ b/app/src/input_manager.c@@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller,im->last_keycode = SDLK_UNKNOWN;im->last_mod = 0;im->key_repeat = 0;++ im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID}static void@@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) {}static bool-set_device_clipboard(struct controller *controller, bool paste) {+set_device_clipboard(struct controller *controller, bool paste,+ uint64_t sequence) {char *text = SDL_GetClipboardText();if (!text) {LOGW("Could not get clipboard text: %s", SDL_GetError());@@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) {struct control_msg msg;msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD;- msg.set_clipboard.sequence = 0; // unused for now+ msg.set_clipboard.sequence = sequence;msg.set_clipboard.text = text_dup;msg.set_clipboard.paste = paste;@@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im,// inject the text as input eventsclipboard_paste(controller);} else {- // store the text in the device clipboard and paste- set_device_clipboard(controller, true);+ // store the text in the device clipboard and paste,+ // without requesting an acknowledgment+ set_device_clipboard(controller, true,+ SC_SEQUENCE_INVALID);}}return;@@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im,return;}+ uint64_t ack_to_wait = SC_SEQUENCE_INVALID;bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat;if (is_ctrl_v) {if (im->legacy_paste) {@@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im,clipboard_paste(controller);return;}++ // Request an acknowledgement only if necessary+ uint64_t sequence = im->kp->async_paste ? im->next_sequence+ : SC_SEQUENCE_INVALID;+// Synchronize the computer clipboard to the device clipboard before// sending Ctrl+v, to allow seamless copy-paste.- bool ok = set_device_clipboard(controller, false);+ bool ok = set_device_clipboard(controller, false, sequence);if (!ok) {LOGW("Clipboard could not be synchronized, Ctrl+v not injected");return;}++ if (im->kp->async_paste) {+ // The key processor must wait for this ack before injecting Ctrl+v+ ack_to_wait = sequence;+ // Increment only when the request succeeded+ ++im->next_sequence;+ }}- im->kp->ops->process_key(im->kp, event, is_ctrl_v);+ im->kp->ops->process_key(im->kp, event, ack_to_wait);}static voiddiff --git a/app/src/input_manager.h b/app/src/input_manager.hindex f018f98a..22d77381 100644--- a/app/src/input_manager.h+++ b/app/src/input_manager.h@@ -38,6 +38,8 @@ struct input_manager {unsigned key_repeat;SDL_Keycode last_keycode;uint16_t last_mod;++ uint64_t next_sequence; // used for request acknowledgements};voiddiff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.cindex e59b5520..4112fd4c 100644--- a/app/src/keyboard_inject.c+++ b/app/src/keyboard_inject.c@@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to,static voidsc_key_processor_process_key(struct sc_key_processor *kp,const SDL_KeyboardEvent *event,- bool device_clipboard_set) {+ uint64_t ack_to_wait) {// The device clipboard synchronization and the key event messages are// serialized, there is nothing special to do to ensure that the clipboard// is set before injecting Ctrl+v.- (void) device_clipboard_set;+ (void) ack_to_wait;struct sc_keyboard_inject *ki = DOWNCAST(kp);@@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki,.process_text = sc_key_processor_process_text,};+ // Key injection and clipboard synchronization are serialized+ ki->key_processor.async_paste = false;ki->key_processor.ops = &ops;}diff --git a/app/src/receiver.c b/app/src/receiver.cindex 52e6034d..eeb206f1 100644--- a/app/src/receiver.c+++ b/app/src/receiver.c@@ -7,12 +7,16 @@#include "util/log.h"bool-receiver_init(struct receiver *receiver, sc_socket control_socket) {+receiver_init(struct receiver *receiver, sc_socket control_socket,+ struct sc_acksync *acksync) {bool ok = sc_mutex_init(&receiver->mutex);if (!ok) {return false;}+receiver->control_socket = control_socket;+ receiver->acksync = acksync;+return true;}@@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) {}static void-process_msg(struct device_msg *msg) {+process_msg(struct receiver *receiver, struct device_msg *msg) {switch (msg->type) {case DEVICE_MSG_TYPE_CLIPBOARD: {char *current = SDL_GetClipboardText();@@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) {break;}case DEVICE_MSG_TYPE_ACK_CLIPBOARD:- // TODO+ assert(receiver->acksync);+ LOGD("Ack device clipboard sequence=%" PRIu64_,+ msg->ack_clipboard.sequence);+ sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);break;}}static ssize_t-process_msgs(const unsigned char *buf, size_t len) {+process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) {size_t head = 0;for (;;) {struct device_msg msg;@@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) {return head;}- process_msg(&msg);+ process_msg(receiver, &msg);device_msg_destroy(&msg);head += r;@@ -84,7 +91,7 @@ run_receiver(void *data) {}head += r;- ssize_t consumed = process_msgs(buf, head);+ ssize_t consumed = process_msgs(receiver, buf, head);if (consumed == -1) {// an error occurredbreak;diff --git a/app/src/receiver.h b/app/src/receiver.hindex 99f128a4..3c4e8c64 100644--- a/app/src/receiver.h+++ b/app/src/receiver.h@@ -5,6 +5,7 @@#include <stdbool.h>+#include "util/acksync.h"#include "util/net.h"#include "util/thread.h"@@ -14,10 +15,13 @@ struct receiver {sc_socket control_socket;sc_thread thread;sc_mutex mutex;++ struct sc_acksync *acksync;};bool-receiver_init(struct receiver *receiver, sc_socket control_socket);+receiver_init(struct receiver *receiver, sc_socket control_socket,+ struct sc_acksync *acksync);voidreceiver_destroy(struct receiver *receiver);diff --git a/app/src/scrcpy.c b/app/src/scrcpy.cindex 61087681..04239bf5 100644--- a/app/src/scrcpy.c+++ b/app/src/scrcpy.c@@ -27,6 +27,7 @@#include "screen.h"#include "server.h"#include "stream.h"+#include "util/acksync.h"#include "util/log.h"#include "util/net.h"#ifdef HAVE_V4L2@@ -46,6 +47,8 @@ struct scrcpy {struct file_handler file_handler;#ifdef HAVE_AOA_HIDstruct sc_aoa aoa;+ // sequence/ack helper to synchronize clipboard and Ctrl+v via HID+ struct sc_acksync acksync;#endifunion {struct sc_keyboard_inject keyboard_inject;@@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) {bool controller_started = false;bool screen_initialized = false;+ struct sc_acksync *acksync = NULL;+struct sc_server_params params = {.serial = options->serial,.log_level = options->log_level,@@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) {}if (options->control) {- if (!controller_init(&s->controller, s->server.control_socket)) {+#ifdef HAVE_AOA_HID+ if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {+ bool ok = sc_acksync_init(&s->acksync);+ if (!ok) {+ goto end;+ }++ acksync = &s->acksync;+ }+#endif+ if (!controller_init(&s->controller, s->server.control_socket,+ acksync)) {goto end;}controller_initialized = true;@@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) {#ifdef HAVE_AOA_HIDbool aoa_hid_ok = false;- bool ok = sc_aoa_init(&s->aoa, serial);+ bool ok = sc_aoa_init(&s->aoa, serial, acksync);if (!ok) {goto aoa_hid_end;}@@ -586,6 +602,9 @@ end:sc_hid_keyboard_destroy(&s->keyboard_hid);sc_aoa_stop(&s->aoa);}+ if (acksync) {+ sc_acksync_destroy(acksync);+ }#endifif (controller_started) {controller_stop(&s->controller);diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.hindex 1f8132f2..f4afe27b 100644--- a/app/src/trait/key_processor.h+++ b/app/src/trait/key_processor.h@@ -14,6 +14,15 @@* Component able to process and inject keys should implement this trait.*/struct sc_key_processor {+ /**+ * Set by the implementation to indicate that it must explicitly wait for+ * the clipboard to be set on the device before injecting Ctrl+v to avoid+ * race conditions. If it is set, the input_manager will pass a valid+ * ack_to_wait to process_key() in case of clipboard synchronization+ * resulting of the key event.+ */+ bool async_paste;+const struct sc_key_processor_ops *ops;};@@ -22,13 +31,14 @@ struct sc_key_processor_ops {/*** Process the keyboard event*- * The flag `device_clipboard_set` indicates that the input manager sent a- * control message to synchronize the device clipboard as a result of this- * key event.+ * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates+ * the acknowledgement number to wait for before injecting this event.+ * This allows to ensure that the device clipboard is set before injecting+ * Ctrl+v on the device.*/void(*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event,- bool device_clipboard_set);+ uint64_t ack_to_wait);void(*process_text)(struct sc_key_processor *kp,diff --git a/app/src/util/acksync.h b/app/src/util/acksync.hindex 1fd34444..58ab1b35 100644--- a/app/src/util/acksync.h+++ b/app/src/util/acksync.h@@ -9,6 +9,11 @@/*** Helper to wait for acknowledgments+ *+ * In practice, it is used to wait for device clipboard acknowledgement from the+ * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the+ * content of the old device clipboard (if Ctrl+v was injected before the+ * clipboard content was actually set).*/struct sc_acksync {sc_mutex mutex;