Skip to content

Display and control your Android device

Add support for USB HID keyboard over AOAv2

Alynx Zhou207082977a01
diff --git a/BUILD.md b/BUILD.md
index 69475f2c..5edb5a21 100644
--- a/BUILD.md
+++ b/BUILD.md
@@ -14,7 +14,8 @@ First, you need to install the required packages:
# for Debian/Ubuntu
sudo apt install ffmpeg libsdl2-2.0-0 adb wget \
gcc git pkg-config meson ninja-build libsdl2-dev \
- libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
+ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \
+ libusb-1.0-0 libusb-dev
```
Then clone the repo and execute the installation script
@@ -88,11 +89,12 @@ Install the required packages from your package manager.
```bash
# runtime dependencies
-sudo apt install ffmpeg libsdl2-2.0-0 adb
+sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0
# client build dependencies
sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \
- libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
+ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \
+ libusb-dev
# server build dependencies
sudo apt install openjdk-11-jdk
@@ -114,7 +116,7 @@ pip3 install meson
sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm
# client build dependencies
-sudo dnf install SDL2-devel ffms2-devel meson gcc make
+sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make
# server build dependencies
sudo dnf install java-devel
diff --git a/README.md b/README.md
index 84dd5ff5..fdd74087 100644
--- a/README.md
+++ b/README.md
@@ -673,6 +673,39 @@ content (if supported by the app) relative to the center of the screen.
Concretely, scrcpy generates additional touch events from a "virtual finger" at
a location inverted through the center of the screen.
+#### Physical keyboard simulation (HID)
+
+By default, scrcpy uses Android key or text injection: it works everywhere, but
+is limited to ASCII.
+
+On Linux, scrcpy can simulate a USB physical keyboard on Android to provide a
+better input experience (using [USB HID over AOAv2][hid-aoav2]): the virtual
+keyboard is disabled and it works for all characters and IME.
+
+[hid-aoav2]: https://source.android.com/devices/accessories/aoa2#hid-support
+
+However, it only works if the device is connected by USB, and is currently only
+supported on Linux.
+
+To enable this mode:
+
+```bash
+scrcpy --hid-keyboard
+scrcpy -K # short version
+```
+
+If it fails for some reason (for example because the device is not connected via
+USB), it automatically fallbacks to the default mode (with a log in the
+console). This allows to use the same command line options when connected over
+USB and TCP/IP.
+
+In this mode, raw key events (scancodes) are sent to the device, independently
+of the host key mapping. Therefore, if your keyboard layout does not match, it
+must be configured on the Android device, in Settings → System → Languages and
+input → [Physical keyboard].
+
+[Physical keyboard]: https://github.com/Genymobile/scrcpy/pull/2632#issuecomment-923756915
+
#### Text injection preference
@@ -692,6 +725,9 @@ scrcpy --prefer-text
(but this will break keyboard behavior in games)
+This option has no effect on HID keyboard (all key events are sent as
+scancodes in this mode).
+
[textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input
[prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343
@@ -707,6 +743,9 @@ To avoid forwarding repeated key events:
scrcpy --no-key-repeat
```
+This option has no effect on HID keyboard (key repeat is handled by Android
+directly in this mode).
+
#### Right-click and middle-click
diff --git a/app/meson.build b/app/meson.build
index 46d3994f..f82d37b3 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -42,6 +42,14 @@ if v4l2_support
src += [ 'src/v4l2_sink.c' ]
endif
+aoa_hid_support = host_machine.system() == 'linux'
+if aoa_hid_support
+ src += [
+ 'src/aoa_hid.c',
+ 'src/hid_keyboard.c',
+ ]
+endif
+
check_functions = [
'strdup'
]
@@ -62,8 +70,11 @@ if not get_option('crossbuild_windows')
dependencies += dependency('libavdevice')
endif
-else
+ if aoa_hid_support
+ dependencies += dependency('libusb-1.0')
+ endif
+else
# cross-compile mingw32 build (from Linux to Windows)
prebuilt_sdl2 = meson.get_cross_property('prebuilt_sdl2')
sdl2_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_sdl2 + '/bin'
@@ -140,6 +151,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == '
# enable V4L2 support (linux only)
conf.set('HAVE_V4L2', v4l2_support)
+# enable HID over AOA support (linux only)
+conf.set('HAVE_AOA_HID', aoa_hid_support)
+
configure_file(configuration: conf, output: 'config.h')
src_dir = include_directories('src')
diff --git a/app/scrcpy.1 b/app/scrcpy.1
index 1b69a065..46db6e1d 100644
--- a/app/scrcpy.1
+++ b/app/scrcpy.1
@@ -82,6 +82,14 @@ Start in fullscreen.
.B \-h, \-\-help
Print this help.
+.TP
+.B \-K, \-\-hid\-keyboard
+Simulate a physical keyboard by using HID over AOAv2.
+
+This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.
+
+It may only work over USB, and is currently only supported on Linux.
+
.TP
.B \-\-legacy\-paste
Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).
diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c
new file mode 100644
index 00000000..7fc0f34c
--- /dev/null
+++ b/app/src/aoa_hid.c
@@ -0,0 +1,368 @@
+#include "util/log.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+#include "aoa_hid.h"
+
+// See <https://source.android.com/devices/accessories/aoa2#hid-support>.
+#define ACCESSORY_REGISTER_HID 54
+#define ACCESSORY_SET_HID_REPORT_DESC 56
+#define ACCESSORY_SEND_HID_EVENT 57
+#define ACCESSORY_UNREGISTER_HID 55
+
+#define DEFAULT_TIMEOUT 1000
+
+static void
+sc_hid_event_log(const struct sc_hid_event *event) {
+ // HID Event: [00] FF FF FF FF...
+ assert(event->size);
+ unsigned buffer_size = event->size * 3 + 1;
+ char *buffer = malloc(buffer_size);
+ if (!buffer) {
+ return;
+ }
+ for (unsigned i = 0; i < event->size; ++i) {
+ snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]);
+ }
+ LOGV("HID Event: [%d]%s", event->accessory_id, buffer);
+ free(buffer);
+}
+
+void
+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
+ unsigned char *buffer, uint16_t buffer_size) {
+ hid_event->accessory_id = accessory_id;
+ hid_event->buffer = buffer;
+ hid_event->size = buffer_size;
+}
+
+void
+sc_hid_event_destroy(struct sc_hid_event *hid_event) {
+ free(hid_event->buffer);
+}
+
+static inline void
+log_libusb_error(enum libusb_error errcode) {
+ LOGW("libusb error: %s", libusb_strerror(errcode));
+}
+
+static bool
+accept_device(libusb_device *device, const char *serial) {
+ // do not log any USB error in this function, it is expected that many USB
+ // devices available on the computer have permission restrictions
+
+ struct libusb_device_descriptor desc;
+ libusb_get_device_descriptor(device, &desc);
+
+ if (!desc.iSerialNumber) {
+ return false;
+ }
+
+ libusb_device_handle *handle;
+ int result = libusb_open(device, &handle);
+ if (result < 0) {
+ return false;
+ }
+
+ char buffer[128];
+ result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber,
+ (unsigned char *) buffer,
+ sizeof(buffer));
+ libusb_close(handle);
+ if (result < 0) {
+ return false;
+ }
+
+ buffer[sizeof(buffer) - 1] = '\0'; // just in case
+
+ // accept the device if its serial matches
+ return !strcmp(buffer, serial);
+}
+
+static libusb_device *
+sc_aoa_find_usb_device(const char *serial) {
+ if (!serial) {
+ return NULL;
+ }
+
+ libusb_device **list;
+ libusb_device *result = NULL;
+ ssize_t count = libusb_get_device_list(NULL, &list);
+ if (count < 0) {
+ log_libusb_error((enum libusb_error) count);
+ return NULL;
+ }
+
+ for (size_t i = 0; i < (size_t) count; ++i) {
+ libusb_device *device = list[i];
+
+ if (accept_device(device, serial)) {
+ result = libusb_ref_device(device);
+ break;
+ }
+ }
+ libusb_free_device_list(list, 1);
+ return result;
+}
+
+static int
+sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
+ int result = libusb_open(device, handle);
+ if (result < 0) {
+ log_libusb_error((enum libusb_error) result);
+ return result;
+ }
+ return 0;
+}
+
+bool
+sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
+ cbuf_init(&aoa->queue);
+
+ if (!sc_mutex_init(&aoa->mutex)) {
+ return false;
+ }
+
+ if (!sc_cond_init(&aoa->event_cond)) {
+ sc_mutex_destroy(&aoa->mutex);
+ return false;
+ }
+
+ if (libusb_init(&aoa->usb_context) != LIBUSB_SUCCESS) {
+ sc_cond_destroy(&aoa->event_cond);
+ sc_mutex_destroy(&aoa->mutex);
+ return false;
+ }
+
+ aoa->usb_device = sc_aoa_find_usb_device(serial);
+ if (!aoa->usb_device) {
+ LOGW("USB device of serial %s not found", serial);
+ libusb_exit(aoa->usb_context);
+ sc_mutex_destroy(&aoa->mutex);
+ sc_cond_destroy(&aoa->event_cond);
+ return false;
+ }
+
+ if (sc_aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) {
+ LOGW("Open USB handle failed");
+ libusb_unref_device(aoa->usb_device);
+ libusb_exit(aoa->usb_context);
+ sc_cond_destroy(&aoa->event_cond);
+ sc_mutex_destroy(&aoa->mutex);
+ return false;
+ }
+
+ aoa->stopped = false;
+
+ return true;
+}
+
+void
+sc_aoa_destroy(struct sc_aoa *aoa) {
+ // Destroy remaining events
+ struct sc_hid_event event;
+ while (cbuf_take(&aoa->queue, &event)) {
+ sc_hid_event_destroy(&event);
+ }
+
+ libusb_close(aoa->usb_handle);
+ libusb_unref_device(aoa->usb_device);
+ libusb_exit(aoa->usb_context);
+ sc_cond_destroy(&aoa->event_cond);
+ sc_mutex_destroy(&aoa->mutex);
+}
+
+static bool
+sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id,
+ uint16_t report_desc_size) {
+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+ uint8_t request = ACCESSORY_REGISTER_HID;
+ // <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+ // value (arg0): accessory assigned ID for the HID device
+ // index (arg1): total length of the HID report descriptor
+ uint16_t value = accessory_id;
+ uint16_t index = report_desc_size;
+ unsigned char *buffer = NULL;
+ uint16_t length = 0;
+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+ value, index, buffer, length,
+ DEFAULT_TIMEOUT);
+ if (result < 0) {
+ log_libusb_error((enum libusb_error) result);
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id,
+ const unsigned char *report_desc,
+ uint16_t report_desc_size) {
+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+ uint8_t request = ACCESSORY_SET_HID_REPORT_DESC;
+ /**
+ * If the HID descriptor is longer than the endpoint zero max packet size,
+ * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC
+ * commands. The data for the descriptor must be sent sequentially
+ * if multiple packets are needed.
+ * <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+ *
+ * libusb handles packet abstraction internally, so we don't need to care
+ * about bMaxPacketSize0 here.
+ *
+ * See <https://libusb.sourceforge.io/api-1.0/libusb_packetoverflow.html>
+ */
+ // value (arg0): accessory assigned ID for the HID device
+ // index (arg1): offset of data (buffer) in descriptor
+ uint16_t value = accessory_id;
+ uint16_t index = 0;
+ // libusb_control_transfer expects a pointer to non-const
+ unsigned char *buffer = (unsigned char *) report_desc;
+ uint16_t length = report_desc_size;
+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+ value, index, buffer, length,
+ DEFAULT_TIMEOUT);
+ if (result < 0) {
+ log_libusb_error((enum libusb_error) result);
+ return false;
+ }
+
+ return true;
+}
+
+bool
+sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,
+ const unsigned char *report_desc, uint16_t report_desc_size) {
+ bool ok = sc_aoa_register_hid(aoa, accessory_id, report_desc_size);
+ if (!ok) {
+ return false;
+ }
+
+ ok = sc_aoa_set_hid_report_desc(aoa, accessory_id, report_desc,
+ report_desc_size);
+ if (!ok) {
+ if (!sc_aoa_unregister_hid(aoa, accessory_id)) {
+ LOGW("Could not unregister HID");
+ }
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+ uint8_t request = ACCESSORY_SEND_HID_EVENT;
+ // <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+ // value (arg0): accessory assigned ID for the HID device
+ // index (arg1): 0 (unused)
+ uint16_t value = event->accessory_id;
+ uint16_t index = 0;
+ unsigned char *buffer = event->buffer;
+ uint16_t length = event->size;
+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+ value, index, buffer, length,
+ DEFAULT_TIMEOUT);
+ if (result < 0) {
+ log_libusb_error((enum libusb_error) result);
+ return false;
+ }
+
+ return true;
+}
+
+bool
+sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) {
+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+ uint8_t request = ACCESSORY_UNREGISTER_HID;
+ // <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+ // value (arg0): accessory assigned ID for the HID device
+ // index (arg1): 0
+ uint16_t value = accessory_id;
+ uint16_t index = 0;
+ unsigned char *buffer = NULL;
+ uint16_t length = 0;
+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+ value, index, buffer, length,
+ DEFAULT_TIMEOUT);
+ if (result < 0) {
+ log_libusb_error((enum libusb_error) result);
+ return false;
+ }
+
+ return true;
+}
+
+bool
+sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
+ if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
+ sc_hid_event_log(event);
+ }
+
+ sc_mutex_lock(&aoa->mutex);
+ bool was_empty = cbuf_is_empty(&aoa->queue);
+ bool res = cbuf_push(&aoa->queue, *event);
+ if (was_empty) {
+ sc_cond_signal(&aoa->event_cond);
+ }
+ sc_mutex_unlock(&aoa->mutex);
+ return res;
+}
+
+static int
+run_aoa_thread(void *data) {
+ struct sc_aoa *aoa = data;
+
+ for (;;) {
+ sc_mutex_lock(&aoa->mutex);
+ while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) {
+ sc_cond_wait(&aoa->event_cond, &aoa->mutex);
+ }
+ if (aoa->stopped) {
+ // Stop immediately, do not process further events
+ sc_mutex_unlock(&aoa->mutex);
+ break;
+ }
+ struct sc_hid_event event;
+ bool non_empty = cbuf_take(&aoa->queue, &event);
+ assert(non_empty);
+ (void) non_empty;
+ sc_mutex_unlock(&aoa->mutex);
+
+ bool ok = sc_aoa_send_hid_event(aoa, &event);
+ sc_hid_event_destroy(&event);
+ if (!ok) {
+ LOGW("Could not send HID event to USB device");
+ }
+ }
+ return 0;
+}
+
+bool
+sc_aoa_start(struct sc_aoa *aoa) {
+ LOGD("Starting AOA thread");
+
+ bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa);
+ if (!ok) {
+ LOGC("Could not start AOA thread");
+ return false;
+ }
+
+ return true;
+}
+
+void
+sc_aoa_stop(struct sc_aoa *aoa) {
+ sc_mutex_lock(&aoa->mutex);
+ aoa->stopped = true;
+ sc_cond_signal(&aoa->event_cond);
+ sc_mutex_unlock(&aoa->mutex);
+}
+
+void
+sc_aoa_join(struct sc_aoa *aoa) {
+ sc_thread_join(&aoa->thread, NULL);
+}
diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h
new file mode 100644
index 00000000..11b879ce
--- /dev/null
+++ b/app/src/aoa_hid.h
@@ -0,0 +1,65 @@
+#ifndef SC_AOA_HID_H
+#define SC_AOA_HID_H
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <libusb-1.0/libusb.h>
+
+#include "scrcpy.h"
+#include "util/cbuf.h"
+#include "util/thread.h"
+
+struct sc_hid_event {
+ uint16_t accessory_id;
+ unsigned char *buffer;
+ uint16_t size;
+};
+
+// Takes ownership of buffer
+void
+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
+ unsigned char *buffer, uint16_t buffer_size);
+
+void
+sc_hid_event_destroy(struct sc_hid_event *hid_event);
+
+struct sc_hid_event_queue CBUF(struct sc_hid_event, 64);
+
+struct sc_aoa {
+ libusb_context *usb_context;
+ libusb_device *usb_device;
+ libusb_device_handle *usb_handle;
+ sc_thread thread;
+ sc_mutex mutex;
+ sc_cond event_cond;
+ bool stopped;
+ struct sc_hid_event_queue queue;
+};
+
+bool
+sc_aoa_init(struct sc_aoa *aoa, const char *serial);
+
+void
+sc_aoa_destroy(struct sc_aoa *aoa);
+
+bool
+sc_aoa_start(struct sc_aoa *aoa);
+
+void
+sc_aoa_stop(struct sc_aoa *aoa);
+
+void
+sc_aoa_join(struct sc_aoa *aoa);
+
+bool
+sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,
+ const unsigned char *report_desc, uint16_t report_desc_size);
+
+bool
+sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id);
+
+bool
+sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event);
+
+#endif
diff --git a/app/src/cli.c b/app/src/cli.c
index d22096ca..a0a508af 100644
--- a/app/src/cli.c
+++ b/app/src/cli.c
@@ -76,6 +76,14 @@ scrcpy_print_usage(const char *arg0) {
" -f, --fullscreen\n"
" Start in fullscreen.\n"
"\n"
+ " -K, --hid-keyboard\n"
+ " Simulate a physical keyboard by using HID over AOAv2.\n"
+ " It provides a better experience for IME users, and allows to\n"
+ " generate non-ASCII characters, contrary to the default\n"
+ " injection method.\n"
+ " It may only work over USB, and is currently only supported\n"
+ " on Linux.\n"
+ "\n"
" -h, --help\n"
" Print this help.\n"
"\n"
@@ -738,6 +746,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
OPT_FORWARD_ALL_CLICKS},
{"fullscreen", no_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
+ {"hid-keyboard", no_argument, NULL, 'K'},
{"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE},
{"lock-video-orientation", optional_argument, NULL,
OPT_LOCK_VIDEO_ORIENTATION},
@@ -784,7 +793,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
optind = 0; // reset to start from the first argument in tests
int c;
- while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w",
+ while ((c = getopt_long(argc, argv, "b:c:fF:hKm:nNp:r:s:StTvV:w",
long_options, NULL)) != -1) {
switch (c) {
case 'b':
@@ -817,6 +826,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
case 'h':
args->help = true;
break;
+ case 'K':
+ opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
+ break;
case OPT_MAX_FPS:
if (!parse_max_fps(optarg, &opts->max_fps)) {
return false;
diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c
new file mode 100644
index 00000000..2599e1d2
--- /dev/null
+++ b/app/src/hid_keyboard.c
@@ -0,0 +1,302 @@
+#include "hid_keyboard.h"
+
+#include <assert.h>
+#include <SDL2/SDL_events.h>
+
+#include "util/log.h"
+
+/** Downcast key processor to hid_keyboard */
+#define DOWNCAST(KP) container_of(KP, struct sc_hid_keyboard, key_processor)
+
+#define HID_KEYBOARD_ACCESSORY_ID 1
+
+#define HID_MODIFIER_NONE 0x00
+#define HID_MODIFIER_LEFT_CONTROL (1 << 0)
+#define HID_MODIFIER_LEFT_SHIFT (1 << 1)
+#define HID_MODIFIER_LEFT_ALT (1 << 2)
+#define HID_MODIFIER_LEFT_GUI (1 << 3)
+#define HID_MODIFIER_RIGHT_CONTROL (1 << 4)
+#define HID_MODIFIER_RIGHT_SHIFT (1 << 5)
+#define HID_MODIFIER_RIGHT_ALT (1 << 6)
+#define HID_MODIFIER_RIGHT_GUI (1 << 7)
+
+#define HID_KEYBOARD_INDEX_MODIFIER 0
+#define HID_KEYBOARD_INDEX_KEYS 2
+
+// USB HID protocol says 6 keys in an event is the requirement for BIOS
+// keyboard support, though OS could support more keys via modifying the report
+// desc. 6 should be enough for scrcpy.
+#define HID_KEYBOARD_MAX_KEYS 6
+#define HID_KEYBOARD_EVENT_SIZE (2 + HID_KEYBOARD_MAX_KEYS)
+
+#define HID_RESERVED 0x00
+#define HID_ERROR_ROLL_OVER 0x01
+
+/**
+ * For HID over AOAv2, only report descriptor is needed.
+ *
+ * The specification is available here:
+ * <https://www.usb.org/sites/default/files/hid1_11.pdf>
+ *
+ * In particular, read:
+ * - 6.2.2 Report Descriptor
+ * - Appendix B.1 Protocol 1 (Keyboard)
+ * - Appendix C: Keyboard Implementation
+ *
+ * Normally a basic HID keyboard uses 8 bytes:
+ * Modifier Reserved Key Key Key Key Key Key
+ *
+ * You can dump your device's report descriptor with:
+ *
+ * sudo usbhid-dump -m vid:pid -e descriptor
+ *
+ * (change vid:pid' to your device's vendor ID and product ID).
+ */
+static const unsigned char keyboard_report_desc[] = {
+ // Usage Page (Generic Desktop)
+ 0x05, 0x01,
+ // Usage (Keyboard)
+ 0x09, 0x06,
+
+ // Collection (Application)
+ 0xA1, 0x01,
+
+ // Usage Page (Key Codes)
+ 0x05, 0x07,
+ // Usage Minimum (224)
+ 0x19, 0xE0,
+ // Usage Maximum (231)
+ 0x29, 0xE7,
+ // Logical Minimum (0)
+ 0x15, 0x00,
+ // Logical Maximum (1)
+ 0x25, 0x01,
+ // Report Size (1)
+ 0x75, 0x01,
+ // Report Count (8)
+ 0x95, 0x08,
+ // Input (Data, Variable, Absolute): Modifier byte
+ 0x81, 0x02,
+
+ // Report Size (8)
+ 0x75, 0x08,
+ // Report Count (1)
+ 0x95, 0x01,
+ // Input (Constant): Reserved byte
+ 0x81, 0x01,
+
+ // Usage Page (LEDs)
+ 0x05, 0x08,
+ // Usage Minimum (1)
+ 0x19, 0x01,
+ // Usage Maximum (5)
+ 0x29, 0x05,
+ // Report Size (1)
+ 0x75, 0x01,
+ // Report Count (5)
+ 0x95, 0x05,
+ // Output (Data, Variable, Absolute): LED report
+ 0x91, 0x02,
+
+ // Report Size (3)
+ 0x75, 0x03,
+ // Report Count (1)
+ 0x95, 0x01,
+ // Output (Constant): LED report padding
+ 0x91, 0x01,
+
+ // Usage Page (Key Codes)
+ 0x05, 0x07,
+ // Usage Minimum (0)
+ 0x19, 0x00,
+ // Usage Maximum (101)
+ 0x29, SC_HID_KEYBOARD_KEYS - 1,
+ // Logical Minimum (0)
+ 0x15, 0x00,
+ // Logical Maximum(101)
+ 0x25, SC_HID_KEYBOARD_KEYS - 1,
+ // Report Size (8)
+ 0x75, 0x08,
+ // Report Count (6)
+ 0x95, HID_KEYBOARD_MAX_KEYS,
+ // Input (Data, Array): Keys
+ 0x81, 0x00,
+
+ // End Collection
+ 0xC0
+};
+
+static unsigned char
+sdl_keymod_to_hid_modifiers(SDL_Keymod mod) {
+ unsigned char modifiers = HID_MODIFIER_NONE;
+ if (mod & KMOD_LCTRL) {
+ modifiers |= HID_MODIFIER_LEFT_CONTROL;
+ }
+ if (mod & KMOD_LSHIFT) {
+ modifiers |= HID_MODIFIER_LEFT_SHIFT;
+ }
+ if (mod & KMOD_LALT) {
+ modifiers |= HID_MODIFIER_LEFT_ALT;
+ }
+ if (mod & KMOD_LGUI) {
+ modifiers |= HID_MODIFIER_LEFT_GUI;
+ }
+ if (mod & KMOD_RCTRL) {
+ modifiers |= HID_MODIFIER_RIGHT_CONTROL;
+ }
+ if (mod & KMOD_RSHIFT) {
+ modifiers |= HID_MODIFIER_RIGHT_SHIFT;
+ }
+ if (mod & KMOD_RALT) {
+ modifiers |= HID_MODIFIER_RIGHT_ALT;
+ }
+ if (mod & KMOD_RGUI) {
+ modifiers |= HID_MODIFIER_RIGHT_GUI;
+ }
+ return modifiers;
+}
+
+static bool
+sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) {
+ unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE);
+ if (!buffer) {
+ return false;
+ }
+
+ buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE;
+ buffer[1] = HID_RESERVED;
+ memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS);
+
+ sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, buffer,
+ HID_KEYBOARD_EVENT_SIZE);
+ return true;
+}
+
+static inline bool
+scancode_is_modifier(SDL_Scancode scancode) {
+ return scancode >= SDL_SCANCODE_LCTRL && scancode <= SDL_SCANCODE_RGUI;
+}
+
+static bool
+convert_hid_keyboard_event(struct sc_hid_keyboard *kb,
+ struct sc_hid_event *hid_event,
+ const SDL_KeyboardEvent *event) {
+ SDL_Scancode scancode = event->keysym.scancode;
+ assert(scancode >= 0);
+
+ // SDL also generates events when only modifiers are pressed, we cannot
+ // ignore them totally, for example press 'a' first then press 'Control',
+ // if we ignore 'Control' event, only 'a' is sent.
+ if (scancode >= SC_HID_KEYBOARD_KEYS && !scancode_is_modifier(scancode)) {
+ // Scancode to ignore
+ return false;
+ }
+
+ if (!sc_hid_keyboard_event_init(hid_event)) {
+ LOGW("Could not initialize HID keyboard event");
+ return false;
+ }
+
+ unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod);
+
+ if (scancode < SC_HID_KEYBOARD_KEYS) {
+ // Pressed is true and released is false
+ kb->keys[scancode] = (event->type == SDL_KEYDOWN);
+ LOGV("keys[%02x] = %s", scancode,
+ kb->keys[scancode] ? "true" : "false");
+ }
+
+ hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers;
+
+ unsigned char *keys_buffer = &hid_event->buffer[HID_KEYBOARD_INDEX_KEYS];
+ // Re-calculate pressed keys every time
+ int keys_pressed_count = 0;
+ for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) {
+ if (kb->keys[i]) {
+ // USB HID protocol says that if keys exceeds report count, a
+ // phantom state should be reported
+ if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) {
+ // Pantom state:
+ // - Modifiers
+ // - Reserved
+ // - ErrorRollOver * HID_MAX_KEYS
+ memset(keys_buffer, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS);
+ goto end;
+ }
+
+ keys_buffer[keys_pressed_count] = i;
+ ++keys_pressed_count;
+ }
+ }
+
+end:
+ LOGV("hid keyboard: key %-4s scancode=%02x (%u) mod=%02x",
+ event->type == SDL_KEYDOWN ? "down" : "up", event->keysym.scancode,
+ event->keysym.scancode, modifiers);
+
+ return true;
+}
+
+static void
+sc_key_processor_process_key(struct sc_key_processor *kp,
+ const SDL_KeyboardEvent *event) {
+ if (event->repeat) {
+ // In USB HID protocol, key repeat is handled by the host (Android), so
+ // just ignore key repeat here.
+ return;
+ }
+
+ struct sc_hid_keyboard *kb = DOWNCAST(kp);
+
+ struct sc_hid_event hid_event;
+ // Not all keys are supported, just ignore unsupported keys
+ if (convert_hid_keyboard_event(kb, &hid_event, event)) {
+ if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
+ sc_hid_event_destroy(&hid_event);
+ LOGW("Could request HID event");
+ }
+ }
+}
+
+static void
+sc_key_processor_process_text(struct sc_key_processor *kp,
+ const SDL_TextInputEvent *event) {
+ (void) kp;
+ (void) event;
+
+ // Never forward text input via HID (all the keys are injected separately)
+}
+
+bool
+sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
+ kb->aoa = aoa;
+
+ bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID,
+ keyboard_report_desc,
+ ARRAY_LEN(keyboard_report_desc));
+ if (!ok) {
+ LOGW("Register HID keyboard failed");
+ return false;
+ }
+
+ // Reset all states
+ memset(kb->keys, false, SC_HID_KEYBOARD_KEYS);
+
+ static const struct sc_key_processor_ops ops = {
+ .process_key = sc_key_processor_process_key,
+ .process_text = sc_key_processor_process_text,
+ };
+
+ kb->key_processor.ops = &ops;
+
+ return true;
+}
+
+void
+sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb) {
+ // Unregister HID keyboard so the soft keyboard shows again on Android
+ bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID);
+ if (!ok) {
+ LOGW("Could not unregister HID keyboard");
+ }
+}
diff --git a/app/src/hid_keyboard.h b/app/src/hid_keyboard.h
new file mode 100644
index 00000000..d8276cad
--- /dev/null
+++ b/app/src/hid_keyboard.h
@@ -0,0 +1,42 @@
+#ifndef SC_HID_KEYBOARD_H
+#define SC_HID_KEYBOARD_H
+
+#include "common.h"
+
+#include <stdbool.h>
+
+#include "aoa_hid.h"
+#include "trait/key_processor.h"
+
+// See "SDL2/SDL_scancode.h".
+// Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB
+// HID protocol.
+// 0x65 is Application, typically AT-101 Keyboard ends here.
+#define SC_HID_KEYBOARD_KEYS 0x66
+
+/**
+ * HID keyboard events are sequence-based, every time keyboard state changes
+ * it sends an array of currently pressed keys, the host is responsible for
+ * compare events and determine which key becomes pressed and which key becomes
+ * released. In order to convert SDL_KeyboardEvent to HID events, we first use
+ * an array of keys to save each keys' state. And when a SDL_KeyboardEvent was
+ * emitted, we updated our state, and then we use a loop to generate HID
+ * events. The sequence of array elements is unimportant and when too much keys
+ * pressed at the same time (more than report count), we should generate
+ * phantom state. Don't forget that modifiers should be updated too, even for
+ * phantom state.
+ */
+struct sc_hid_keyboard {
+ struct sc_key_processor key_processor; // key processor trait
+
+ struct sc_aoa *aoa;
+ bool keys[SC_HID_KEYBOARD_KEYS];
+};
+
+bool
+sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa);
+
+void
+sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb);
+
+#endif
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
index 50250e2c..c4041562 100644
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@@ -18,6 +18,9 @@
#include "events.h"
#include "file_handler.h"
#include "input_manager.h"
+#ifdef HAVE_AOA_HID
+# include "hid_keyboard.h"
+#endif
#include "keyboard_inject.h"
#include "mouse_inject.h"
#include "recorder.h"
@@ -41,7 +44,15 @@ struct scrcpy {
#endif
struct controller controller;
struct file_handler file_handler;
- struct sc_keyboard_inject keyboard_inject;
+#ifdef HAVE_AOA_HID
+ struct sc_aoa aoa;
+#endif
+ union {
+ struct sc_keyboard_inject keyboard_inject;
+#ifdef HAVE_AOA_HID
+ struct sc_hid_keyboard keyboard_hid;
+#endif
+ };
struct sc_mouse_inject mouse_inject;
struct input_manager input_manager;
};
@@ -243,7 +254,7 @@ stream_on_eos(struct stream *stream, void *userdata) {
}
bool
-scrcpy(const struct scrcpy_options *options) {
+scrcpy(struct scrcpy_options *options) {
static struct scrcpy scrcpy;
struct scrcpy *s = &scrcpy;
@@ -260,6 +271,9 @@ scrcpy(const struct scrcpy_options *options) {
bool v4l2_sink_initialized = false;
#endif
bool stream_started = false;
+#ifdef HAVE_AOA_HID
+ bool aoa_hid_initialized = false;
+#endif
bool controller_initialized = false;
bool controller_started = false;
bool screen_initialized = false;
@@ -419,8 +433,50 @@ scrcpy(const struct scrcpy_options *options) {
struct sc_mouse_processor *mp = NULL;
if (options->control) {
- sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options);
- kp = &s->keyboard_inject.key_processor;
+ if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
+#ifdef HAVE_AOA_HID
+ bool aoa_hid_ok = false;
+ if (!sc_aoa_init(&s->aoa, options->serial)) {
+ goto aoa_hid_end;
+ }
+
+ if (!sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
+ sc_aoa_destroy(&s->aoa);
+ goto aoa_hid_end;
+ }
+
+ if (!sc_aoa_start(&s->aoa)) {
+ sc_hid_keyboard_destroy(&s->keyboard_hid);
+ sc_aoa_destroy(&s->aoa);
+ goto aoa_hid_end;
+ }
+
+ aoa_hid_ok = true;
+ kp = &s->keyboard_hid.key_processor;
+
+ aoa_hid_initialized = true;
+
+aoa_hid_end:
+ if (!aoa_hid_ok) {
+ LOGE("Failed to enable HID over AOA, "
+ "fallback to default keyboard injection method "
+ "(-K/--hid-keyboard ignored)");
+ options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+ }
+#else
+ LOGE("HID over AOA is not supported on this platform, "
+ "fallback to default keyboard injection method "
+ "(-K/--hid-keyboard ignored)");
+ options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+#endif
+ }
+
+ // keyboard_input_mode may have been reset if HID mode failed
+ if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
+ sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
+ options);
+ kp = &s->keyboard_inject.key_processor;
+ }
sc_mouse_inject_init(&s->mouse_inject, &s->controller, &s->screen);
mp = &s->mouse_inject.mouse_processor;
@@ -439,6 +495,12 @@ scrcpy(const struct scrcpy_options *options) {
end:
// The stream is not stopped explicitly, because it will stop by itself on
// end-of-stream
+#ifdef HAVE_AOA_HID
+ if (aoa_hid_initialized) {
+ sc_hid_keyboard_destroy(&s->keyboard_hid);
+ sc_aoa_stop(&s->aoa);
+ }
+#endif
if (controller_started) {
controller_stop(&s->controller);
}
@@ -466,6 +528,13 @@ end:
}
#endif
+#ifdef HAVE_AOA_HID
+ if (aoa_hid_initialized) {
+ sc_aoa_join(&s->aoa);
+ sc_aoa_destroy(&s->aoa);
+ }
+#endif
+
// Destroy the screen only after the stream is guaranteed to be finished,
// because otherwise the screen could receive new frames after destruction
if (screen_initialized) {
diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h
index 8b76fb25..8cf4a917 100644
--- a/app/src/scrcpy.h
+++ b/app/src/scrcpy.h
@@ -33,6 +33,11 @@ enum sc_lock_video_orientation {
SC_LOCK_VIDEO_ORIENTATION_3,
};
+enum sc_keyboard_input_mode {
+ SC_KEYBOARD_INPUT_MODE_INJECT,
+ SC_KEYBOARD_INPUT_MODE_HID,
+};
+
#define SC_MAX_SHORTCUT_MODS 8
enum sc_shortcut_mod {
@@ -68,6 +73,7 @@ struct scrcpy_options {
const char *v4l2_device;
enum sc_log_level log_level;
enum sc_record_format record_format;
+ enum sc_keyboard_input_mode keyboard_input_mode;
struct sc_port_range port_range;
struct sc_shortcut_mods shortcut_mods;
uint16_t max_size;
@@ -112,6 +118,7 @@ struct scrcpy_options {
.v4l2_device = NULL, \
.log_level = SC_LOG_LEVEL_INFO, \
.record_format = SC_RECORD_FORMAT_AUTO, \
+ .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, \
.port_range = { \
.first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \
.last = DEFAULT_LOCAL_PORT_RANGE_LAST, \
@@ -151,6 +158,6 @@ struct scrcpy_options {
}
bool
-scrcpy(const struct scrcpy_options *options);
+scrcpy(struct scrcpy_options *options);
#endif