Skip to content

Display and control your Android device

Wait SET_CLIPBOARD ack before Ctrl+v via HID

Romain Vimont5d17bcf1bc38
diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c
index 4c0b2bda..6fd32610 100644
--- a/app/src/aoa_hid.c
+++ b/app/src/aoa_hid.c
@@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
hid_event->accessory_id = accessory_id;
hid_event->buffer = buffer;
hid_event->size = buffer_size;
- hid_event->delay = 0;
+ hid_event->ack_to_wait = SC_SEQUENCE_INVALID;
}
void
@@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
}
bool
-sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
+sc_aoa_init(struct sc_aoa *aoa, const char *serial,
+ struct sc_acksync *acksync) {
+ assert(acksync);
+
cbuf_init(&aoa->queue);
if (!sc_mutex_init(&aoa->mutex)) {
@@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
}
aoa->stopped = false;
+ aoa->acksync = acksync;
return true;
}
@@ -332,23 +336,28 @@ run_aoa_thread(void *data) {
assert(non_empty);
(void) non_empty;
- assert(event.delay >= 0);
- if (event.delay) {
- // Wait during the specified delay before injecting the HID event
- sc_tick deadline = sc_tick_now() + event.delay;
- bool timed_out = false;
- while (!aoa->stopped && !timed_out) {
- timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex,
- deadline);
- }
- if (aoa->stopped) {
- sc_mutex_unlock(&aoa->mutex);
+ uint64_t ack_to_wait = event.ack_to_wait;
+ sc_mutex_unlock(&aoa->mutex);
+
+ if (ack_to_wait != SC_SEQUENCE_INVALID) {
+ LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait);
+ // Do not block the loop indefinitely if the ack never comes (it should
+ // never happen)
+ sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500);
+ enum sc_acksync_wait_result result =
+ sc_acksync_wait(aoa->acksync, ack_to_wait, deadline);
+
+ if (result == SC_ACKSYNC_WAIT_TIMEOUT) {
+ LOGW("Ack not received after 500ms, discarding HID event");
+ sc_hid_event_destroy(&event);
+ continue;
+ } else if (result == SC_ACKSYNC_WAIT_INTR) {
+ // stopped
+ sc_hid_event_destroy(&event);
break;
}
}
- sc_mutex_unlock(&aoa->mutex);
-
bool ok = sc_aoa_send_hid_event(aoa, &event);
sc_hid_event_destroy(&event);
if (!ok) {
@@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) {
aoa->stopped = true;
sc_cond_signal(&aoa->event_cond);
sc_mutex_unlock(&aoa->mutex);
+
+ sc_acksync_interrupt(aoa->acksync);
}
void
diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h
index 24cef502..e8fb9708 100644
--- a/app/src/aoa_hid.h
+++ b/app/src/aoa_hid.h
@@ -6,6 +6,7 @@
#include <libusb-1.0/libusb.h>
+#include "util/acksync.h"
#include "util/cbuf.h"
#include "util/thread.h"
#include "util/tick.h"
@@ -14,7 +15,7 @@ struct sc_hid_event {
uint16_t accessory_id;
unsigned char *buffer;
uint16_t size;
- sc_tick delay;
+ uint64_t ack_to_wait;
};
// Takes ownership of buffer
@@ -36,10 +37,12 @@ struct sc_aoa {
sc_cond event_cond;
bool stopped;
struct sc_hid_event_queue queue;
+
+ struct sc_acksync *acksync;
};
bool
-sc_aoa_init(struct sc_aoa *aoa, const char *serial);
+sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync);
void
sc_aoa_destroy(struct sc_aoa *aoa);
diff --git a/app/src/controller.c b/app/src/controller.c
index e486ea72..6cf3d20e 100644
--- a/app/src/controller.c
+++ b/app/src/controller.c
@@ -5,10 +5,11 @@
#include "util/log.h"
bool
-controller_init(struct controller *controller, sc_socket control_socket) {
+controller_init(struct controller *controller, sc_socket control_socket,
+ struct sc_acksync *acksync) {
cbuf_init(&controller->queue);
- bool ok = receiver_init(&controller->receiver, control_socket);
+ bool ok = receiver_init(&controller->receiver, control_socket, acksync);
if (!ok) {
return false;
}
diff --git a/app/src/controller.h b/app/src/controller.h
index e7004131..00267878 100644
--- a/app/src/controller.h
+++ b/app/src/controller.h
@@ -7,6 +7,7 @@
#include "control_msg.h"
#include "receiver.h"
+#include "util/acksync.h"
#include "util/cbuf.h"
#include "util/net.h"
#include "util/thread.h"
@@ -24,7 +25,8 @@ struct controller {
};
bool
-controller_init(struct controller *controller, sc_socket control_socket);
+controller_init(struct controller *controller, sc_socket control_socket,
+ struct sc_acksync *acksync);
void
controller_destroy(struct controller *controller);
diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c
index 2809ccd6..2afc09b2 100644
--- a/app/src/hid_keyboard.c
+++ b/app/src/hid_keyboard.c
@@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) {
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const SDL_KeyboardEvent *event,
- bool device_clipboard_set) {
+ uint64_t ack_to_wait) {
if (event->repeat) {
// In USB HID protocol, key repeat is handled by the host (Android), so
// just ignore key repeat here.
@@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
}
}
- if (device_clipboard_set) {
+ if (ack_to_wait) {
// Ctrl+v is pressed, so clipboard synchronization has been
- // requested. Wait a bit so that the clipboard is set before
- // injecting Ctrl+v via HID, otherwise it would paste the old
- // clipboard content.
- hid_event.delay = SC_TICK_FROM_MS(5);
+ // requested. Wait until clipboard synchronization is acknowledged
+ // by the server, otherwise it could paste the old clipboard
+ // content.
+ hid_event.ack_to_wait = ack_to_wait;
}
if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
@@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
.process_text = sc_key_processor_process_text,
};
+ // Clipboard synchronization is requested over the control socket, while HID
+ // events are sent over AOA, so it must wait for clipboard synchronization
+ // to be acknowledged by the device before injecting Ctrl+v.
+ kb->key_processor.async_paste = true;
kb->key_processor.ops = &ops;
return true;
diff --git a/app/src/input_manager.c b/app/src/input_manager.c
index e7afee77..f1715b79 100644
--- a/app/src/input_manager.c
+++ b/app/src/input_manager.c
@@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller,
im->last_keycode = SDLK_UNKNOWN;
im->last_mod = 0;
im->key_repeat = 0;
+
+ im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID
}
static void
@@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) {
}
static bool
-set_device_clipboard(struct controller *controller, bool paste) {
+set_device_clipboard(struct controller *controller, bool paste,
+ uint64_t sequence) {
char *text = SDL_GetClipboardText();
if (!text) {
LOGW("Could not get clipboard text: %s", SDL_GetError());
@@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) {
struct control_msg msg;
msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD;
- msg.set_clipboard.sequence = 0; // unused for now
+ msg.set_clipboard.sequence = sequence;
msg.set_clipboard.text = text_dup;
msg.set_clipboard.paste = paste;
@@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im,
// inject the text as input events
clipboard_paste(controller);
} else {
- // store the text in the device clipboard and paste
- set_device_clipboard(controller, true);
+ // store the text in the device clipboard and paste,
+ // without requesting an acknowledgment
+ set_device_clipboard(controller, true,
+ SC_SEQUENCE_INVALID);
}
}
return;
@@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im,
return;
}
+ uint64_t ack_to_wait = SC_SEQUENCE_INVALID;
bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat;
if (is_ctrl_v) {
if (im->legacy_paste) {
@@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im,
clipboard_paste(controller);
return;
}
+
+ // Request an acknowledgement only if necessary
+ uint64_t sequence = im->kp->async_paste ? im->next_sequence
+ : SC_SEQUENCE_INVALID;
+
// Synchronize the computer clipboard to the device clipboard before
// sending Ctrl+v, to allow seamless copy-paste.
- bool ok = set_device_clipboard(controller, false);
+ bool ok = set_device_clipboard(controller, false, sequence);
if (!ok) {
LOGW("Clipboard could not be synchronized, Ctrl+v not injected");
return;
}
+
+ if (im->kp->async_paste) {
+ // The key processor must wait for this ack before injecting Ctrl+v
+ ack_to_wait = sequence;
+ // Increment only when the request succeeded
+ ++im->next_sequence;
+ }
}
- im->kp->ops->process_key(im->kp, event, is_ctrl_v);
+ im->kp->ops->process_key(im->kp, event, ack_to_wait);
}
static void
diff --git a/app/src/input_manager.h b/app/src/input_manager.h
index f018f98a..22d77381 100644
--- a/app/src/input_manager.h
+++ b/app/src/input_manager.h
@@ -38,6 +38,8 @@ struct input_manager {
unsigned key_repeat;
SDL_Keycode last_keycode;
uint16_t last_mod;
+
+ uint64_t next_sequence; // used for request acknowledgements
};
void
diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.c
index e59b5520..4112fd4c 100644
--- a/app/src/keyboard_inject.c
+++ b/app/src/keyboard_inject.c
@@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to,
static void
sc_key_processor_process_key(struct sc_key_processor *kp,
const SDL_KeyboardEvent *event,
- bool device_clipboard_set) {
+ uint64_t ack_to_wait) {
// The device clipboard synchronization and the key event messages are
// serialized, there is nothing special to do to ensure that the clipboard
// is set before injecting Ctrl+v.
- (void) device_clipboard_set;
+ (void) ack_to_wait;
struct sc_keyboard_inject *ki = DOWNCAST(kp);
@@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki,
.process_text = sc_key_processor_process_text,
};
+ // Key injection and clipboard synchronization are serialized
+ ki->key_processor.async_paste = false;
ki->key_processor.ops = &ops;
}
diff --git a/app/src/receiver.c b/app/src/receiver.c
index 52e6034d..eeb206f1 100644
--- a/app/src/receiver.c
+++ b/app/src/receiver.c
@@ -7,12 +7,16 @@
#include "util/log.h"
bool
-receiver_init(struct receiver *receiver, sc_socket control_socket) {
+receiver_init(struct receiver *receiver, sc_socket control_socket,
+ struct sc_acksync *acksync) {
bool ok = sc_mutex_init(&receiver->mutex);
if (!ok) {
return false;
}
+
receiver->control_socket = control_socket;
+ receiver->acksync = acksync;
+
return true;
}
@@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) {
}
static void
-process_msg(struct device_msg *msg) {
+process_msg(struct receiver *receiver, struct device_msg *msg) {
switch (msg->type) {
case DEVICE_MSG_TYPE_CLIPBOARD: {
char *current = SDL_GetClipboardText();
@@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) {
break;
}
case DEVICE_MSG_TYPE_ACK_CLIPBOARD:
- // TODO
+ assert(receiver->acksync);
+ LOGD("Ack device clipboard sequence=%" PRIu64_,
+ msg->ack_clipboard.sequence);
+ sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
break;
}
}
static ssize_t
-process_msgs(const unsigned char *buf, size_t len) {
+process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) {
size_t head = 0;
for (;;) {
struct device_msg msg;
@@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) {
return head;
}
- process_msg(&msg);
+ process_msg(receiver, &msg);
device_msg_destroy(&msg);
head += r;
@@ -84,7 +91,7 @@ run_receiver(void *data) {
}
head += r;
- ssize_t consumed = process_msgs(buf, head);
+ ssize_t consumed = process_msgs(receiver, buf, head);
if (consumed == -1) {
// an error occurred
break;
diff --git a/app/src/receiver.h b/app/src/receiver.h
index 99f128a4..3c4e8c64 100644
--- a/app/src/receiver.h
+++ b/app/src/receiver.h
@@ -5,6 +5,7 @@
#include <stdbool.h>
+#include "util/acksync.h"
#include "util/net.h"
#include "util/thread.h"
@@ -14,10 +15,13 @@ struct receiver {
sc_socket control_socket;
sc_thread thread;
sc_mutex mutex;
+
+ struct sc_acksync *acksync;
};
bool
-receiver_init(struct receiver *receiver, sc_socket control_socket);
+receiver_init(struct receiver *receiver, sc_socket control_socket,
+ struct sc_acksync *acksync);
void
receiver_destroy(struct receiver *receiver);
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
index 61087681..04239bf5 100644
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@@ -27,6 +27,7 @@
#include "screen.h"
#include "server.h"
#include "stream.h"
+#include "util/acksync.h"
#include "util/log.h"
#include "util/net.h"
#ifdef HAVE_V4L2
@@ -46,6 +47,8 @@ struct scrcpy {
struct file_handler file_handler;
#ifdef HAVE_AOA_HID
struct sc_aoa aoa;
+ // sequence/ack helper to synchronize clipboard and Ctrl+v via HID
+ struct sc_acksync acksync;
#endif
union {
struct sc_keyboard_inject keyboard_inject;
@@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) {
bool controller_started = false;
bool screen_initialized = false;
+ struct sc_acksync *acksync = NULL;
+
struct sc_server_params params = {
.serial = options->serial,
.log_level = options->log_level,
@@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) {
}
if (options->control) {
- if (!controller_init(&s->controller, s->server.control_socket)) {
+#ifdef HAVE_AOA_HID
+ if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
+ bool ok = sc_acksync_init(&s->acksync);
+ if (!ok) {
+ goto end;
+ }
+
+ acksync = &s->acksync;
+ }
+#endif
+ if (!controller_init(&s->controller, s->server.control_socket,
+ acksync)) {
goto end;
}
controller_initialized = true;
@@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) {
#ifdef HAVE_AOA_HID
bool aoa_hid_ok = false;
- bool ok = sc_aoa_init(&s->aoa, serial);
+ bool ok = sc_aoa_init(&s->aoa, serial, acksync);
if (!ok) {
goto aoa_hid_end;
}
@@ -586,6 +602,9 @@ end:
sc_hid_keyboard_destroy(&s->keyboard_hid);
sc_aoa_stop(&s->aoa);
}
+ if (acksync) {
+ sc_acksync_destroy(acksync);
+ }
#endif
if (controller_started) {
controller_stop(&s->controller);
diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h
index 1f8132f2..f4afe27b 100644
--- a/app/src/trait/key_processor.h
+++ b/app/src/trait/key_processor.h
@@ -14,6 +14,15 @@
* Component able to process and inject keys should implement this trait.
*/
struct sc_key_processor {
+ /**
+ * Set by the implementation to indicate that it must explicitly wait for
+ * the clipboard to be set on the device before injecting Ctrl+v to avoid
+ * race conditions. If it is set, the input_manager will pass a valid
+ * ack_to_wait to process_key() in case of clipboard synchronization
+ * resulting of the key event.
+ */
+ bool async_paste;
+
const struct sc_key_processor_ops *ops;
};
@@ -22,13 +31,14 @@ struct sc_key_processor_ops {
/**
* Process the keyboard event
*
- * The flag `device_clipboard_set` indicates that the input manager sent a
- * control message to synchronize the device clipboard as a result of this
- * key event.
+ * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates
+ * the acknowledgement number to wait for before injecting this event.
+ * This allows to ensure that the device clipboard is set before injecting
+ * Ctrl+v on the device.
*/
void
(*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event,
- bool device_clipboard_set);
+ uint64_t ack_to_wait);
void
(*process_text)(struct sc_key_processor *kp,
diff --git a/app/src/util/acksync.h b/app/src/util/acksync.h
index 1fd34444..58ab1b35 100644
--- a/app/src/util/acksync.h
+++ b/app/src/util/acksync.h
@@ -9,6 +9,11 @@
/**
* Helper to wait for acknowledgments
+ *
+ * In practice, it is used to wait for device clipboard acknowledgement from the
+ * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the
+ * content of the old device clipboard (if Ctrl+v was injected before the
+ * clipboard content was actually set).
*/
struct sc_acksync {
sc_mutex mutex;