Display and control your Android device
Add support for USB HID keyboard over AOAv2
Alynx Zhou207082977a01
diff --git a/BUILD.md b/BUILD.mdindex 69475f2c..5edb5a21 100644--- a/BUILD.md+++ b/BUILD.md@@ -14,7 +14,8 @@ First, you need to install the required packages:# for Debian/Ubuntusudo apt install ffmpeg libsdl2-2.0-0 adb wget \gcc git pkg-config meson ninja-build libsdl2-dev \- libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev+ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \+ libusb-1.0-0 libusb-dev```Then clone the repo and execute the installation script@@ -88,11 +89,12 @@ Install the required packages from your package manager.```bash# runtime dependencies-sudo apt install ffmpeg libsdl2-2.0-0 adb+sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0# client build dependenciessudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \- libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev+ libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \+ libusb-dev# server build dependenciessudo apt install openjdk-11-jdk@@ -114,7 +116,7 @@ pip3 install mesonsudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm# client build dependencies-sudo dnf install SDL2-devel ffms2-devel meson gcc make+sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make# server build dependenciessudo dnf install java-develdiff --git a/README.md b/README.mdindex 84dd5ff5..fdd74087 100644--- a/README.md+++ b/README.md@@ -673,6 +673,39 @@ content (if supported by the app) relative to the center of the screen.Concretely, scrcpy generates additional touch events from a "virtual finger" ata location inverted through the center of the screen.+#### Physical keyboard simulation (HID)++By default, scrcpy uses Android key or text injection: it works everywhere, but+is limited to ASCII.++On Linux, scrcpy can simulate a USB physical keyboard on Android to provide a+better input experience (using [USB HID over AOAv2][hid-aoav2]): the virtual+keyboard is disabled and it works for all characters and IME.++[hid-aoav2]: https://source.android.com/devices/accessories/aoa2#hid-support++However, it only works if the device is connected by USB, and is currently only+supported on Linux.++To enable this mode:++```bash+scrcpy --hid-keyboard+scrcpy -K # short version+```++If it fails for some reason (for example because the device is not connected via+USB), it automatically fallbacks to the default mode (with a log in the+console). This allows to use the same command line options when connected over+USB and TCP/IP.++In this mode, raw key events (scancodes) are sent to the device, independently+of the host key mapping. Therefore, if your keyboard layout does not match, it+must be configured on the Android device, in Settings → System → Languages and+input → [Physical keyboard].++[Physical keyboard]: https://github.com/Genymobile/scrcpy/pull/2632#issuecomment-923756915+#### Text injection preference@@ -692,6 +725,9 @@ scrcpy --prefer-text(but this will break keyboard behavior in games)+This option has no effect on HID keyboard (all key events are sent as+scancodes in this mode).+[textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input[prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343@@ -707,6 +743,9 @@ To avoid forwarding repeated key events:scrcpy --no-key-repeat```+This option has no effect on HID keyboard (key repeat is handled by Android+directly in this mode).+#### Right-click and middle-clickdiff --git a/app/meson.build b/app/meson.buildindex 46d3994f..f82d37b3 100644--- a/app/meson.build+++ b/app/meson.build@@ -42,6 +42,14 @@ if v4l2_supportsrc += [ 'src/v4l2_sink.c' ]endif+aoa_hid_support = host_machine.system() == 'linux'+if aoa_hid_support+ src += [+ 'src/aoa_hid.c',+ 'src/hid_keyboard.c',+ ]+endif+check_functions = ['strdup']@@ -62,8 +70,11 @@ if not get_option('crossbuild_windows')dependencies += dependency('libavdevice')endif-else+ if aoa_hid_support+ dependencies += dependency('libusb-1.0')+ endif+else# cross-compile mingw32 build (from Linux to Windows)prebuilt_sdl2 = meson.get_cross_property('prebuilt_sdl2')sdl2_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_sdl2 + '/bin'@@ -140,6 +151,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == '# enable V4L2 support (linux only)conf.set('HAVE_V4L2', v4l2_support)+# enable HID over AOA support (linux only)+conf.set('HAVE_AOA_HID', aoa_hid_support)+configure_file(configuration: conf, output: 'config.h')src_dir = include_directories('src')diff --git a/app/scrcpy.1 b/app/scrcpy.1index 1b69a065..46db6e1d 100644--- a/app/scrcpy.1+++ b/app/scrcpy.1@@ -82,6 +82,14 @@ Start in fullscreen..B \-h, \-\-helpPrint this help.+.TP+.B \-K, \-\-hid\-keyboard+Simulate a physical keyboard by using HID over AOAv2.++This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.++It may only work over USB, and is currently only supported on Linux.+.TP.B \-\-legacy\-pasteInject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.cnew file mode 100644index 00000000..7fc0f34c--- /dev/null+++ b/app/src/aoa_hid.c@@ -0,0 +1,368 @@+#include "util/log.h"++#include <assert.h>+#include <stdio.h>++#include "aoa_hid.h"++// See <https://source.android.com/devices/accessories/aoa2#hid-support>.+#define ACCESSORY_REGISTER_HID 54+#define ACCESSORY_SET_HID_REPORT_DESC 56+#define ACCESSORY_SEND_HID_EVENT 57+#define ACCESSORY_UNREGISTER_HID 55++#define DEFAULT_TIMEOUT 1000++static void+sc_hid_event_log(const struct sc_hid_event *event) {+ // HID Event: [00] FF FF FF FF...+ assert(event->size);+ unsigned buffer_size = event->size * 3 + 1;+ char *buffer = malloc(buffer_size);+ if (!buffer) {+ return;+ }+ for (unsigned i = 0; i < event->size; ++i) {+ snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]);+ }+ LOGV("HID Event: [%d]%s", event->accessory_id, buffer);+ free(buffer);+}++void+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,+ unsigned char *buffer, uint16_t buffer_size) {+ hid_event->accessory_id = accessory_id;+ hid_event->buffer = buffer;+ hid_event->size = buffer_size;+}++void+sc_hid_event_destroy(struct sc_hid_event *hid_event) {+ free(hid_event->buffer);+}++static inline void+log_libusb_error(enum libusb_error errcode) {+ LOGW("libusb error: %s", libusb_strerror(errcode));+}++static bool+accept_device(libusb_device *device, const char *serial) {+ // do not log any USB error in this function, it is expected that many USB+ // devices available on the computer have permission restrictions++ struct libusb_device_descriptor desc;+ libusb_get_device_descriptor(device, &desc);++ if (!desc.iSerialNumber) {+ return false;+ }++ libusb_device_handle *handle;+ int result = libusb_open(device, &handle);+ if (result < 0) {+ return false;+ }++ char buffer[128];+ result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber,+ (unsigned char *) buffer,+ sizeof(buffer));+ libusb_close(handle);+ if (result < 0) {+ return false;+ }++ buffer[sizeof(buffer) - 1] = '\0'; // just in case++ // accept the device if its serial matches+ return !strcmp(buffer, serial);+}++static libusb_device *+sc_aoa_find_usb_device(const char *serial) {+ if (!serial) {+ return NULL;+ }++ libusb_device **list;+ libusb_device *result = NULL;+ ssize_t count = libusb_get_device_list(NULL, &list);+ if (count < 0) {+ log_libusb_error((enum libusb_error) count);+ return NULL;+ }++ for (size_t i = 0; i < (size_t) count; ++i) {+ libusb_device *device = list[i];++ if (accept_device(device, serial)) {+ result = libusb_ref_device(device);+ break;+ }+ }+ libusb_free_device_list(list, 1);+ return result;+}++static int+sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {+ int result = libusb_open(device, handle);+ if (result < 0) {+ log_libusb_error((enum libusb_error) result);+ return result;+ }+ return 0;+}++bool+sc_aoa_init(struct sc_aoa *aoa, const char *serial) {+ cbuf_init(&aoa->queue);++ if (!sc_mutex_init(&aoa->mutex)) {+ return false;+ }++ if (!sc_cond_init(&aoa->event_cond)) {+ sc_mutex_destroy(&aoa->mutex);+ return false;+ }++ if (libusb_init(&aoa->usb_context) != LIBUSB_SUCCESS) {+ sc_cond_destroy(&aoa->event_cond);+ sc_mutex_destroy(&aoa->mutex);+ return false;+ }++ aoa->usb_device = sc_aoa_find_usb_device(serial);+ if (!aoa->usb_device) {+ LOGW("USB device of serial %s not found", serial);+ libusb_exit(aoa->usb_context);+ sc_mutex_destroy(&aoa->mutex);+ sc_cond_destroy(&aoa->event_cond);+ return false;+ }++ if (sc_aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) {+ LOGW("Open USB handle failed");+ libusb_unref_device(aoa->usb_device);+ libusb_exit(aoa->usb_context);+ sc_cond_destroy(&aoa->event_cond);+ sc_mutex_destroy(&aoa->mutex);+ return false;+ }++ aoa->stopped = false;++ return true;+}++void+sc_aoa_destroy(struct sc_aoa *aoa) {+ // Destroy remaining events+ struct sc_hid_event event;+ while (cbuf_take(&aoa->queue, &event)) {+ sc_hid_event_destroy(&event);+ }++ libusb_close(aoa->usb_handle);+ libusb_unref_device(aoa->usb_device);+ libusb_exit(aoa->usb_context);+ sc_cond_destroy(&aoa->event_cond);+ sc_mutex_destroy(&aoa->mutex);+}++static bool+sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id,+ uint16_t report_desc_size) {+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;+ uint8_t request = ACCESSORY_REGISTER_HID;+ // <https://source.android.com/devices/accessories/aoa2.html#hid-support>+ // value (arg0): accessory assigned ID for the HID device+ // index (arg1): total length of the HID report descriptor+ uint16_t value = accessory_id;+ uint16_t index = report_desc_size;+ unsigned char *buffer = NULL;+ uint16_t length = 0;+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,+ value, index, buffer, length,+ DEFAULT_TIMEOUT);+ if (result < 0) {+ log_libusb_error((enum libusb_error) result);+ return false;+ }++ return true;+}++static bool+sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id,+ const unsigned char *report_desc,+ uint16_t report_desc_size) {+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;+ uint8_t request = ACCESSORY_SET_HID_REPORT_DESC;+ /**+ * If the HID descriptor is longer than the endpoint zero max packet size,+ * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC+ * commands. The data for the descriptor must be sent sequentially+ * if multiple packets are needed.+ * <https://source.android.com/devices/accessories/aoa2.html#hid-support>+ *+ * libusb handles packet abstraction internally, so we don't need to care+ * about bMaxPacketSize0 here.+ *+ * See <https://libusb.sourceforge.io/api-1.0/libusb_packetoverflow.html>+ */+ // value (arg0): accessory assigned ID for the HID device+ // index (arg1): offset of data (buffer) in descriptor+ uint16_t value = accessory_id;+ uint16_t index = 0;+ // libusb_control_transfer expects a pointer to non-const+ unsigned char *buffer = (unsigned char *) report_desc;+ uint16_t length = report_desc_size;+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,+ value, index, buffer, length,+ DEFAULT_TIMEOUT);+ if (result < 0) {+ log_libusb_error((enum libusb_error) result);+ return false;+ }++ return true;+}++bool+sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,+ const unsigned char *report_desc, uint16_t report_desc_size) {+ bool ok = sc_aoa_register_hid(aoa, accessory_id, report_desc_size);+ if (!ok) {+ return false;+ }++ ok = sc_aoa_set_hid_report_desc(aoa, accessory_id, report_desc,+ report_desc_size);+ if (!ok) {+ if (!sc_aoa_unregister_hid(aoa, accessory_id)) {+ LOGW("Could not unregister HID");+ }+ return false;+ }++ return true;+}++static bool+sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;+ uint8_t request = ACCESSORY_SEND_HID_EVENT;+ // <https://source.android.com/devices/accessories/aoa2.html#hid-support>+ // value (arg0): accessory assigned ID for the HID device+ // index (arg1): 0 (unused)+ uint16_t value = event->accessory_id;+ uint16_t index = 0;+ unsigned char *buffer = event->buffer;+ uint16_t length = event->size;+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,+ value, index, buffer, length,+ DEFAULT_TIMEOUT);+ if (result < 0) {+ log_libusb_error((enum libusb_error) result);+ return false;+ }++ return true;+}++bool+sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) {+ uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;+ uint8_t request = ACCESSORY_UNREGISTER_HID;+ // <https://source.android.com/devices/accessories/aoa2.html#hid-support>+ // value (arg0): accessory assigned ID for the HID device+ // index (arg1): 0+ uint16_t value = accessory_id;+ uint16_t index = 0;+ unsigned char *buffer = NULL;+ uint16_t length = 0;+ int result = libusb_control_transfer(aoa->usb_handle, request_type, request,+ value, index, buffer, length,+ DEFAULT_TIMEOUT);+ if (result < 0) {+ log_libusb_error((enum libusb_error) result);+ return false;+ }++ return true;+}++bool+sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {+ if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {+ sc_hid_event_log(event);+ }++ sc_mutex_lock(&aoa->mutex);+ bool was_empty = cbuf_is_empty(&aoa->queue);+ bool res = cbuf_push(&aoa->queue, *event);+ if (was_empty) {+ sc_cond_signal(&aoa->event_cond);+ }+ sc_mutex_unlock(&aoa->mutex);+ return res;+}++static int+run_aoa_thread(void *data) {+ struct sc_aoa *aoa = data;++ for (;;) {+ sc_mutex_lock(&aoa->mutex);+ while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) {+ sc_cond_wait(&aoa->event_cond, &aoa->mutex);+ }+ if (aoa->stopped) {+ // Stop immediately, do not process further events+ sc_mutex_unlock(&aoa->mutex);+ break;+ }+ struct sc_hid_event event;+ bool non_empty = cbuf_take(&aoa->queue, &event);+ assert(non_empty);+ (void) non_empty;+ sc_mutex_unlock(&aoa->mutex);++ bool ok = sc_aoa_send_hid_event(aoa, &event);+ sc_hid_event_destroy(&event);+ if (!ok) {+ LOGW("Could not send HID event to USB device");+ }+ }+ return 0;+}++bool+sc_aoa_start(struct sc_aoa *aoa) {+ LOGD("Starting AOA thread");++ bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa);+ if (!ok) {+ LOGC("Could not start AOA thread");+ return false;+ }++ return true;+}++void+sc_aoa_stop(struct sc_aoa *aoa) {+ sc_mutex_lock(&aoa->mutex);+ aoa->stopped = true;+ sc_cond_signal(&aoa->event_cond);+ sc_mutex_unlock(&aoa->mutex);+}++void+sc_aoa_join(struct sc_aoa *aoa) {+ sc_thread_join(&aoa->thread, NULL);+}diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.hnew file mode 100644index 00000000..11b879ce--- /dev/null+++ b/app/src/aoa_hid.h@@ -0,0 +1,65 @@+#ifndef SC_AOA_HID_H+#define SC_AOA_HID_H++#include <stdint.h>+#include <stdbool.h>++#include <libusb-1.0/libusb.h>++#include "scrcpy.h"+#include "util/cbuf.h"+#include "util/thread.h"++struct sc_hid_event {+ uint16_t accessory_id;+ unsigned char *buffer;+ uint16_t size;+};++// Takes ownership of buffer+void+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,+ unsigned char *buffer, uint16_t buffer_size);++void+sc_hid_event_destroy(struct sc_hid_event *hid_event);++struct sc_hid_event_queue CBUF(struct sc_hid_event, 64);++struct sc_aoa {+ libusb_context *usb_context;+ libusb_device *usb_device;+ libusb_device_handle *usb_handle;+ sc_thread thread;+ sc_mutex mutex;+ sc_cond event_cond;+ bool stopped;+ struct sc_hid_event_queue queue;+};++bool+sc_aoa_init(struct sc_aoa *aoa, const char *serial);++void+sc_aoa_destroy(struct sc_aoa *aoa);++bool+sc_aoa_start(struct sc_aoa *aoa);++void+sc_aoa_stop(struct sc_aoa *aoa);++void+sc_aoa_join(struct sc_aoa *aoa);++bool+sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,+ const unsigned char *report_desc, uint16_t report_desc_size);++bool+sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id);++bool+sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event);++#endifdiff --git a/app/src/cli.c b/app/src/cli.cindex d22096ca..a0a508af 100644--- a/app/src/cli.c+++ b/app/src/cli.c@@ -76,6 +76,14 @@ scrcpy_print_usage(const char *arg0) {" -f, --fullscreen\n"" Start in fullscreen.\n""\n"+ " -K, --hid-keyboard\n"+ " Simulate a physical keyboard by using HID over AOAv2.\n"+ " It provides a better experience for IME users, and allows to\n"+ " generate non-ASCII characters, contrary to the default\n"+ " injection method.\n"+ " It may only work over USB, and is currently only supported\n"+ " on Linux.\n"+ "\n"" -h, --help\n"" Print this help.\n""\n"@@ -738,6 +746,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {OPT_FORWARD_ALL_CLICKS},{"fullscreen", no_argument, NULL, 'f'},{"help", no_argument, NULL, 'h'},+ {"hid-keyboard", no_argument, NULL, 'K'},{"legacy-paste", no_argument, NULL, OPT_LEGACY_PASTE},{"lock-video-orientation", optional_argument, NULL,OPT_LOCK_VIDEO_ORIENTATION},@@ -784,7 +793,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {optind = 0; // reset to start from the first argument in testsint c;- while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w",+ while ((c = getopt_long(argc, argv, "b:c:fF:hKm:nNp:r:s:StTvV:w",long_options, NULL)) != -1) {switch (c) {case 'b':@@ -817,6 +826,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {case 'h':args->help = true;break;+ case 'K':+ opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;+ break;case OPT_MAX_FPS:if (!parse_max_fps(optarg, &opts->max_fps)) {return false;diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.cnew file mode 100644index 00000000..2599e1d2--- /dev/null+++ b/app/src/hid_keyboard.c@@ -0,0 +1,302 @@+#include "hid_keyboard.h"++#include <assert.h>+#include <SDL2/SDL_events.h>++#include "util/log.h"++/** Downcast key processor to hid_keyboard */+#define DOWNCAST(KP) container_of(KP, struct sc_hid_keyboard, key_processor)++#define HID_KEYBOARD_ACCESSORY_ID 1++#define HID_MODIFIER_NONE 0x00+#define HID_MODIFIER_LEFT_CONTROL (1 << 0)+#define HID_MODIFIER_LEFT_SHIFT (1 << 1)+#define HID_MODIFIER_LEFT_ALT (1 << 2)+#define HID_MODIFIER_LEFT_GUI (1 << 3)+#define HID_MODIFIER_RIGHT_CONTROL (1 << 4)+#define HID_MODIFIER_RIGHT_SHIFT (1 << 5)+#define HID_MODIFIER_RIGHT_ALT (1 << 6)+#define HID_MODIFIER_RIGHT_GUI (1 << 7)++#define HID_KEYBOARD_INDEX_MODIFIER 0+#define HID_KEYBOARD_INDEX_KEYS 2++// USB HID protocol says 6 keys in an event is the requirement for BIOS+// keyboard support, though OS could support more keys via modifying the report+// desc. 6 should be enough for scrcpy.+#define HID_KEYBOARD_MAX_KEYS 6+#define HID_KEYBOARD_EVENT_SIZE (2 + HID_KEYBOARD_MAX_KEYS)++#define HID_RESERVED 0x00+#define HID_ERROR_ROLL_OVER 0x01++/**+ * For HID over AOAv2, only report descriptor is needed.+ *+ * The specification is available here:+ * <https://www.usb.org/sites/default/files/hid1_11.pdf>+ *+ * In particular, read:+ * - 6.2.2 Report Descriptor+ * - Appendix B.1 Protocol 1 (Keyboard)+ * - Appendix C: Keyboard Implementation+ *+ * Normally a basic HID keyboard uses 8 bytes:+ * Modifier Reserved Key Key Key Key Key Key+ *+ * You can dump your device's report descriptor with:+ *+ * sudo usbhid-dump -m vid:pid -e descriptor+ *+ * (change vid:pid' to your device's vendor ID and product ID).+ */+static const unsigned char keyboard_report_desc[] = {+ // Usage Page (Generic Desktop)+ 0x05, 0x01,+ // Usage (Keyboard)+ 0x09, 0x06,++ // Collection (Application)+ 0xA1, 0x01,++ // Usage Page (Key Codes)+ 0x05, 0x07,+ // Usage Minimum (224)+ 0x19, 0xE0,+ // Usage Maximum (231)+ 0x29, 0xE7,+ // Logical Minimum (0)+ 0x15, 0x00,+ // Logical Maximum (1)+ 0x25, 0x01,+ // Report Size (1)+ 0x75, 0x01,+ // Report Count (8)+ 0x95, 0x08,+ // Input (Data, Variable, Absolute): Modifier byte+ 0x81, 0x02,++ // Report Size (8)+ 0x75, 0x08,+ // Report Count (1)+ 0x95, 0x01,+ // Input (Constant): Reserved byte+ 0x81, 0x01,++ // Usage Page (LEDs)+ 0x05, 0x08,+ // Usage Minimum (1)+ 0x19, 0x01,+ // Usage Maximum (5)+ 0x29, 0x05,+ // Report Size (1)+ 0x75, 0x01,+ // Report Count (5)+ 0x95, 0x05,+ // Output (Data, Variable, Absolute): LED report+ 0x91, 0x02,++ // Report Size (3)+ 0x75, 0x03,+ // Report Count (1)+ 0x95, 0x01,+ // Output (Constant): LED report padding+ 0x91, 0x01,++ // Usage Page (Key Codes)+ 0x05, 0x07,+ // Usage Minimum (0)+ 0x19, 0x00,+ // Usage Maximum (101)+ 0x29, SC_HID_KEYBOARD_KEYS - 1,+ // Logical Minimum (0)+ 0x15, 0x00,+ // Logical Maximum(101)+ 0x25, SC_HID_KEYBOARD_KEYS - 1,+ // Report Size (8)+ 0x75, 0x08,+ // Report Count (6)+ 0x95, HID_KEYBOARD_MAX_KEYS,+ // Input (Data, Array): Keys+ 0x81, 0x00,++ // End Collection+ 0xC0+};++static unsigned char+sdl_keymod_to_hid_modifiers(SDL_Keymod mod) {+ unsigned char modifiers = HID_MODIFIER_NONE;+ if (mod & KMOD_LCTRL) {+ modifiers |= HID_MODIFIER_LEFT_CONTROL;+ }+ if (mod & KMOD_LSHIFT) {+ modifiers |= HID_MODIFIER_LEFT_SHIFT;+ }+ if (mod & KMOD_LALT) {+ modifiers |= HID_MODIFIER_LEFT_ALT;+ }+ if (mod & KMOD_LGUI) {+ modifiers |= HID_MODIFIER_LEFT_GUI;+ }+ if (mod & KMOD_RCTRL) {+ modifiers |= HID_MODIFIER_RIGHT_CONTROL;+ }+ if (mod & KMOD_RSHIFT) {+ modifiers |= HID_MODIFIER_RIGHT_SHIFT;+ }+ if (mod & KMOD_RALT) {+ modifiers |= HID_MODIFIER_RIGHT_ALT;+ }+ if (mod & KMOD_RGUI) {+ modifiers |= HID_MODIFIER_RIGHT_GUI;+ }+ return modifiers;+}++static bool+sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) {+ unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE);+ if (!buffer) {+ return false;+ }++ buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE;+ buffer[1] = HID_RESERVED;+ memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS);++ sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, buffer,+ HID_KEYBOARD_EVENT_SIZE);+ return true;+}++static inline bool+scancode_is_modifier(SDL_Scancode scancode) {+ return scancode >= SDL_SCANCODE_LCTRL && scancode <= SDL_SCANCODE_RGUI;+}++static bool+convert_hid_keyboard_event(struct sc_hid_keyboard *kb,+ struct sc_hid_event *hid_event,+ const SDL_KeyboardEvent *event) {+ SDL_Scancode scancode = event->keysym.scancode;+ assert(scancode >= 0);++ // SDL also generates events when only modifiers are pressed, we cannot+ // ignore them totally, for example press 'a' first then press 'Control',+ // if we ignore 'Control' event, only 'a' is sent.+ if (scancode >= SC_HID_KEYBOARD_KEYS && !scancode_is_modifier(scancode)) {+ // Scancode to ignore+ return false;+ }++ if (!sc_hid_keyboard_event_init(hid_event)) {+ LOGW("Could not initialize HID keyboard event");+ return false;+ }++ unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod);++ if (scancode < SC_HID_KEYBOARD_KEYS) {+ // Pressed is true and released is false+ kb->keys[scancode] = (event->type == SDL_KEYDOWN);+ LOGV("keys[%02x] = %s", scancode,+ kb->keys[scancode] ? "true" : "false");+ }++ hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers;++ unsigned char *keys_buffer = &hid_event->buffer[HID_KEYBOARD_INDEX_KEYS];+ // Re-calculate pressed keys every time+ int keys_pressed_count = 0;+ for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) {+ if (kb->keys[i]) {+ // USB HID protocol says that if keys exceeds report count, a+ // phantom state should be reported+ if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) {+ // Pantom state:+ // - Modifiers+ // - Reserved+ // - ErrorRollOver * HID_MAX_KEYS+ memset(keys_buffer, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS);+ goto end;+ }++ keys_buffer[keys_pressed_count] = i;+ ++keys_pressed_count;+ }+ }++end:+ LOGV("hid keyboard: key %-4s scancode=%02x (%u) mod=%02x",+ event->type == SDL_KEYDOWN ? "down" : "up", event->keysym.scancode,+ event->keysym.scancode, modifiers);++ return true;+}++static void+sc_key_processor_process_key(struct sc_key_processor *kp,+ const SDL_KeyboardEvent *event) {+ if (event->repeat) {+ // In USB HID protocol, key repeat is handled by the host (Android), so+ // just ignore key repeat here.+ return;+ }++ struct sc_hid_keyboard *kb = DOWNCAST(kp);++ struct sc_hid_event hid_event;+ // Not all keys are supported, just ignore unsupported keys+ if (convert_hid_keyboard_event(kb, &hid_event, event)) {+ if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {+ sc_hid_event_destroy(&hid_event);+ LOGW("Could request HID event");+ }+ }+}++static void+sc_key_processor_process_text(struct sc_key_processor *kp,+ const SDL_TextInputEvent *event) {+ (void) kp;+ (void) event;++ // Never forward text input via HID (all the keys are injected separately)+}++bool+sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {+ kb->aoa = aoa;++ bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID,+ keyboard_report_desc,+ ARRAY_LEN(keyboard_report_desc));+ if (!ok) {+ LOGW("Register HID keyboard failed");+ return false;+ }++ // Reset all states+ memset(kb->keys, false, SC_HID_KEYBOARD_KEYS);++ static const struct sc_key_processor_ops ops = {+ .process_key = sc_key_processor_process_key,+ .process_text = sc_key_processor_process_text,+ };++ kb->key_processor.ops = &ops;++ return true;+}++void+sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb) {+ // Unregister HID keyboard so the soft keyboard shows again on Android+ bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID);+ if (!ok) {+ LOGW("Could not unregister HID keyboard");+ }+}diff --git a/app/src/hid_keyboard.h b/app/src/hid_keyboard.hnew file mode 100644index 00000000..d8276cad--- /dev/null+++ b/app/src/hid_keyboard.h@@ -0,0 +1,42 @@+#ifndef SC_HID_KEYBOARD_H+#define SC_HID_KEYBOARD_H++#include "common.h"++#include <stdbool.h>++#include "aoa_hid.h"+#include "trait/key_processor.h"++// See "SDL2/SDL_scancode.h".+// Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB+// HID protocol.+// 0x65 is Application, typically AT-101 Keyboard ends here.+#define SC_HID_KEYBOARD_KEYS 0x66++/**+ * HID keyboard events are sequence-based, every time keyboard state changes+ * it sends an array of currently pressed keys, the host is responsible for+ * compare events and determine which key becomes pressed and which key becomes+ * released. In order to convert SDL_KeyboardEvent to HID events, we first use+ * an array of keys to save each keys' state. And when a SDL_KeyboardEvent was+ * emitted, we updated our state, and then we use a loop to generate HID+ * events. The sequence of array elements is unimportant and when too much keys+ * pressed at the same time (more than report count), we should generate+ * phantom state. Don't forget that modifiers should be updated too, even for+ * phantom state.+ */+struct sc_hid_keyboard {+ struct sc_key_processor key_processor; // key processor trait++ struct sc_aoa *aoa;+ bool keys[SC_HID_KEYBOARD_KEYS];+};++bool+sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa);++void+sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb);++#endifdiff --git a/app/src/scrcpy.c b/app/src/scrcpy.cindex 50250e2c..c4041562 100644--- a/app/src/scrcpy.c+++ b/app/src/scrcpy.c@@ -18,6 +18,9 @@#include "events.h"#include "file_handler.h"#include "input_manager.h"+#ifdef HAVE_AOA_HID+# include "hid_keyboard.h"+#endif#include "keyboard_inject.h"#include "mouse_inject.h"#include "recorder.h"@@ -41,7 +44,15 @@ struct scrcpy {#endifstruct controller controller;struct file_handler file_handler;- struct sc_keyboard_inject keyboard_inject;+#ifdef HAVE_AOA_HID+ struct sc_aoa aoa;+#endif+ union {+ struct sc_keyboard_inject keyboard_inject;+#ifdef HAVE_AOA_HID+ struct sc_hid_keyboard keyboard_hid;+#endif+ };struct sc_mouse_inject mouse_inject;struct input_manager input_manager;};@@ -243,7 +254,7 @@ stream_on_eos(struct stream *stream, void *userdata) {}bool-scrcpy(const struct scrcpy_options *options) {+scrcpy(struct scrcpy_options *options) {static struct scrcpy scrcpy;struct scrcpy *s = &scrcpy;@@ -260,6 +271,9 @@ scrcpy(const struct scrcpy_options *options) {bool v4l2_sink_initialized = false;#endifbool stream_started = false;+#ifdef HAVE_AOA_HID+ bool aoa_hid_initialized = false;+#endifbool controller_initialized = false;bool controller_started = false;bool screen_initialized = false;@@ -419,8 +433,50 @@ scrcpy(const struct scrcpy_options *options) {struct sc_mouse_processor *mp = NULL;if (options->control) {- sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options);- kp = &s->keyboard_inject.key_processor;+ if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {+#ifdef HAVE_AOA_HID+ bool aoa_hid_ok = false;+ if (!sc_aoa_init(&s->aoa, options->serial)) {+ goto aoa_hid_end;+ }++ if (!sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {+ sc_aoa_destroy(&s->aoa);+ goto aoa_hid_end;+ }++ if (!sc_aoa_start(&s->aoa)) {+ sc_hid_keyboard_destroy(&s->keyboard_hid);+ sc_aoa_destroy(&s->aoa);+ goto aoa_hid_end;+ }++ aoa_hid_ok = true;+ kp = &s->keyboard_hid.key_processor;++ aoa_hid_initialized = true;++aoa_hid_end:+ if (!aoa_hid_ok) {+ LOGE("Failed to enable HID over AOA, "+ "fallback to default keyboard injection method "+ "(-K/--hid-keyboard ignored)");+ options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;+ }+#else+ LOGE("HID over AOA is not supported on this platform, "+ "fallback to default keyboard injection method "+ "(-K/--hid-keyboard ignored)");+ options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;+#endif+ }++ // keyboard_input_mode may have been reset if HID mode failed+ if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {+ sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,+ options);+ kp = &s->keyboard_inject.key_processor;+ }sc_mouse_inject_init(&s->mouse_inject, &s->controller, &s->screen);mp = &s->mouse_inject.mouse_processor;@@ -439,6 +495,12 @@ scrcpy(const struct scrcpy_options *options) {end:// The stream is not stopped explicitly, because it will stop by itself on// end-of-stream+#ifdef HAVE_AOA_HID+ if (aoa_hid_initialized) {+ sc_hid_keyboard_destroy(&s->keyboard_hid);+ sc_aoa_stop(&s->aoa);+ }+#endifif (controller_started) {controller_stop(&s->controller);}@@ -466,6 +528,13 @@ end:}#endif+#ifdef HAVE_AOA_HID+ if (aoa_hid_initialized) {+ sc_aoa_join(&s->aoa);+ sc_aoa_destroy(&s->aoa);+ }+#endif+// Destroy the screen only after the stream is guaranteed to be finished,// because otherwise the screen could receive new frames after destructionif (screen_initialized) {diff --git a/app/src/scrcpy.h b/app/src/scrcpy.hindex 8b76fb25..8cf4a917 100644--- a/app/src/scrcpy.h+++ b/app/src/scrcpy.h@@ -33,6 +33,11 @@ enum sc_lock_video_orientation {SC_LOCK_VIDEO_ORIENTATION_3,};+enum sc_keyboard_input_mode {+ SC_KEYBOARD_INPUT_MODE_INJECT,+ SC_KEYBOARD_INPUT_MODE_HID,+};+#define SC_MAX_SHORTCUT_MODS 8enum sc_shortcut_mod {@@ -68,6 +73,7 @@ struct scrcpy_options {const char *v4l2_device;enum sc_log_level log_level;enum sc_record_format record_format;+ enum sc_keyboard_input_mode keyboard_input_mode;struct sc_port_range port_range;struct sc_shortcut_mods shortcut_mods;uint16_t max_size;@@ -112,6 +118,7 @@ struct scrcpy_options {.v4l2_device = NULL, \.log_level = SC_LOG_LEVEL_INFO, \.record_format = SC_RECORD_FORMAT_AUTO, \+ .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, \.port_range = { \.first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \.last = DEFAULT_LOCAL_PORT_RANGE_LAST, \@@ -151,6 +158,6 @@ struct scrcpy_options {}bool-scrcpy(const struct scrcpy_options *options);+scrcpy(struct scrcpy_options *options);#endif