Display and control your Android device
Add support for HID mouse
Romain Vimontcba84f69993e
diff --git a/app/meson.build b/app/meson.buildindex 38393f0c..cee261bb 100644--- a/app/meson.build+++ b/app/meson.build@@ -77,6 +77,7 @@ if aoa_hid_supportsrc += ['src/aoa_hid.c','src/hid_keyboard.c',+ 'src/hid_mouse.c',]endifdiff --git a/app/scrcpy.1 b/app/scrcpy.1index 971c8a19..4d02982c 100644--- a/app/scrcpy.1+++ b/app/scrcpy.1@@ -96,6 +96,8 @@ The keyboard layout must be configured (once and for all) on the device, via SetHowever, the option is only available when the HID keyboard is enabled (or a physical keyboard is connected).+Also see \fB\-\-hid\-mouse\fR.+.TP.B \-\-legacy\-pasteInject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).@@ -120,6 +122,18 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimensionDefault is 0 (unlimited).+.TP+.B \-M, \-\-hid\-mouse+Simulate a physical mouse by using HID over AOAv2.++In this mode, the computer mouse is captured to control the device directly (relative mouse mode).++LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer.++It may only work over USB, and is currently only supported on Linux.++Also see \fB\-\-hid\-keyboard\fR.+.TP.B \-\-no\-clipboard\-autosyncBy default, scrcpy automatically synchronizes the computer clipboard to the device clipboard before injecting Ctrl+v, and the device clipboard to the computer clipboard whenever it changes.diff --git a/app/src/cli.c b/app/src/cli.cindex ec53e5ec..3fcf94eb 100644--- a/app/src/cli.c+++ b/app/src/cli.c@@ -178,7 +178,8 @@ static const struct sc_option options[] = {"directly: `adb shell am start -a ""android.settings.HARD_KEYBOARD_SETTINGS`.\n""However, the option is only available when the HID keyboard "- "is enabled (or a physical keyboard is connected).",+ "is enabled (or a physical keyboard is connected).\n"+ "Also see --hid-mouse.",},{.shortopt = 'h',@@ -214,6 +215,18 @@ static const struct sc_option options[] = {.text = "Limit the frame rate of screen capture (officially supported ""since Android 10, but may work on earlier versions).",},+ {+ .shortopt = 'M',+ .longopt = "hid-mouse",+ .text = "Simulate a physical mouse by using HID over AOAv2.\n"+ "In this mode, the computer mouse is captured to control the "+ "device directly (relative mouse mode).\n"+ "LAlt, LSuper or RSuper toggle the capture mode, to give "+ "control of the mouse back to the computer.\n"+ "It may only work over USB, and is currently only supported "+ "on Linux.\n"+ "Also see --hid-keyboard.",+ },{.shortopt = 'm',.longopt = "max-size",@@ -1315,6 +1328,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],return false;}break;+ case 'M':+#ifdef HAVE_AOA_HID+ opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID;+#else+ LOGE("HID over AOA (-M/--hid-mouse) is not supported on this"+ "platform. It is only available on Linux.");+ return false;+#endif+ break;case OPT_LOCK_VIDEO_ORIENTATION:if (!parse_lock_video_orientation(optarg,&opts->lock_video_orientation)) {diff --git a/app/src/hid_mouse.c b/app/src/hid_mouse.cnew file mode 100644index 00000000..0e26c7c3--- /dev/null+++ b/app/src/hid_mouse.c@@ -0,0 +1,267 @@+#include "hid_mouse.h"++#include <assert.h>++#include "input_events.h"+#include "util/log.h"++/** Downcast mouse processor to hid_mouse */+#define DOWNCAST(MP) container_of(MP, struct sc_hid_mouse, mouse_processor)++#define HID_MOUSE_ACCESSORY_ID 2++// 1 byte for buttons + padding, 1 byte for X position, 1 byte for Y position+#define HID_MOUSE_EVENT_SIZE 4++/**+ * Mouse descriptor from the specification:+ * <https://www.usb.org/sites/default/files/hid1_11.pdf>+ *+ * Appendix E (p71): §E.10 Report Descriptor (Mouse)+ *+ * The usage tags (like Wheel) are listed in "HID Usage Tables":+ * <https://www.usb.org/sites/default/files/documents/hut1_12v2.pdf>+ * §4 Generic Desktop Page (0x01) (p26)+ */+static const unsigned char mouse_report_desc[] = {+ // Usage Page (Generic Desktop)+ 0x05, 0x01,+ // Usage (Mouse)+ 0x09, 0x02,++ // Collection (Application)+ 0xA1, 0x01,++ // Usage (Pointer)+ 0x09, 0x01,++ // Collection (Physical)+ 0xA1, 0x00,++ // Usage Page (Buttons)+ 0x05, 0x09,++ // Usage Minimum (1)+ 0x19, 0x01,+ // Usage Maximum (5)+ 0x29, 0x05,+ // Logical Minimum (0)+ 0x15, 0x00,+ // Logical Maximum (1)+ 0x25, 0x01,+ // Report Count (5)+ 0x95, 0x05,+ // Report Size (1)+ 0x75, 0x01,+ // Input (Data, Variable, Absolute): 5 buttons bits+ 0x81, 0x02,++ // Report Count (1)+ 0x95, 0x01,+ // Report Size (3)+ 0x75, 0x03,+ // Input (Constant): 3 bits padding+ 0x81, 0x01,++ // Usage Page (Generic Desktop)+ 0x05, 0x01,+ // Usage (X)+ 0x09, 0x30,+ // Usage (Y)+ 0x09, 0x31,+ // Usage (Wheel)+ 0x09, 0x38,+ // Local Minimum (-127)+ 0x15, 0x81,+ // Local Maximum (127)+ 0x25, 0x7F,+ // Report Size (8)+ 0x75, 0x08,+ // Report Count (3)+ 0x95, 0x03,+ // Input (Data, Variable, Relative): 3 position bytes (X, Y, Wheel)+ 0x81, 0x06,++ // End Collection+ 0xC0,++ // End Collection+ 0xC0,+};++/**+ * A mouse HID event is 3 bytes long:+ *+ * - byte 0: buttons state+ * - byte 1: relative x motion (signed byte from -127 to 127)+ * - byte 2: relative y motion (signed byte from -127 to 127)+ *+ * 7 6 5 4 3 2 1 0+ * +---------------++ * byte 0: |0 0 0 . . . . .| buttons state+ * +---------------++ * ^ ^ ^ ^ ^+ * | | | | `- left button+ * | | | `--- right button+ * | | `----- middle button+ * | `------- button 4+ * `--------- button 5+ *+ * +---------------++ * byte 1: |. . . . . . . .| relative x motion+ * +---------------++ * byte 2: |. . . . . . . .| relative y motion+ * +---------------++ * byte 3: |. . . . . . . .| wheel motion (-1, 0 or 1)+ * +---------------++ *+ * As an example, here is the report for a motion of (x=5, y=-4) with left+ * button pressed:+ *+ * +---------------++ * |0 0 0 0 0 0 0 1| left button pressed+ * +---------------++ * |0 0 0 0 0 1 0 1| horizontal motion (x = 5)+ * +---------------++ * |1 1 1 1 1 1 0 0| relative y motion (y = -4)+ * +---------------++ * |0 0 0 0 0 0 0 0| wheel motion+ * +---------------++ */++static bool+sc_hid_mouse_event_init(struct sc_hid_event *hid_event) {+ unsigned char *buffer = calloc(1, HID_MOUSE_EVENT_SIZE);+ if (!buffer) {+ LOG_OOM();+ return false;+ }++ sc_hid_event_init(hid_event, HID_MOUSE_ACCESSORY_ID, buffer,+ HID_MOUSE_EVENT_SIZE);+ return true;+}++static unsigned char+buttons_state_to_hid_buttons(uint8_t buttons_state) {+ unsigned char c = 0;+ if (buttons_state & SC_MOUSE_BUTTON_LEFT) {+ c |= 1 << 0;+ }+ if (buttons_state & SC_MOUSE_BUTTON_RIGHT) {+ c |= 1 << 1;+ }+ if (buttons_state & SC_MOUSE_BUTTON_MIDDLE) {+ c |= 1 << 2;+ }+ if (buttons_state & SC_MOUSE_BUTTON_X1) {+ c |= 1 << 3;+ }+ if (buttons_state & SC_MOUSE_BUTTON_X2) {+ c |= 1 << 4;+ }+ return c;+}++static void+sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp,+ const struct sc_mouse_motion_event *event) {+ struct sc_hid_mouse *mouse = DOWNCAST(mp);++ struct sc_hid_event hid_event;+ if (!sc_hid_mouse_event_init(&hid_event)) {+ return;+ }++ unsigned char *buffer = hid_event.buffer;+ buffer[0] = buttons_state_to_hid_buttons(event->buttons_state);+ buffer[1] = CLAMP(event->xrel, -127, 127);+ buffer[2] = CLAMP(event->yrel, -127, 127);+ buffer[3] = 0; // wheel coordinates only used for scrolling++ if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) {+ sc_hid_event_destroy(&hid_event);+ LOGW("Could request HID event");+ }+}++static void+sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp,+ const struct sc_mouse_click_event *event) {+ struct sc_hid_mouse *mouse = DOWNCAST(mp);++ struct sc_hid_event hid_event;+ if (!sc_hid_mouse_event_init(&hid_event)) {+ return;+ }++ unsigned char *buffer = hid_event.buffer;+ buffer[0] = buttons_state_to_hid_buttons(event->buttons_state);+ buffer[1] = 0; // no x motion+ buffer[2] = 0; // no y motion+ buffer[3] = 0; // wheel coordinates only used for scrolling++ if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) {+ sc_hid_event_destroy(&hid_event);+ LOGW("Could request HID event");+ }+}++static void+sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp,+ const struct sc_mouse_scroll_event *event) {+ struct sc_hid_mouse *mouse = DOWNCAST(mp);++ struct sc_hid_event hid_event;+ if (!sc_hid_mouse_event_init(&hid_event)) {+ return;+ }++ unsigned char *buffer = hid_event.buffer;+ buffer[0] = 0; // buttons state irrelevant (and unknown)+ buffer[1] = 0; // no x motion+ buffer[2] = 0; // no y motion+ // In practice, vscroll is always -1, 0 or 1, but in theory other values+ // are possible+ buffer[3] = CLAMP(event->vscroll, -127, 127);+ // Horizontal scrolling ignored++ if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) {+ sc_hid_event_destroy(&hid_event);+ LOGW("Could request HID event");+ }+}++bool+sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa) {+ mouse->aoa = aoa;++ bool ok = sc_aoa_setup_hid(aoa, HID_MOUSE_ACCESSORY_ID, mouse_report_desc,+ ARRAY_LEN(mouse_report_desc));+ if (!ok) {+ LOGW("Register HID mouse failed");+ return false;+ }++ static const struct sc_mouse_processor_ops ops = {+ .process_mouse_motion = sc_mouse_processor_process_mouse_motion,+ .process_mouse_click = sc_mouse_processor_process_mouse_click,+ .process_mouse_scroll = sc_mouse_processor_process_mouse_scroll,+ // Touch events not supported (coordinates are not relative)+ .process_touch = NULL,+ };++ mouse->mouse_processor.ops = &ops;++ mouse->mouse_processor.relative_mode = true;++ return true;+}++void+sc_hid_mouse_destroy(struct sc_hid_mouse *mouse) {+ bool ok = sc_aoa_unregister_hid(mouse->aoa, HID_MOUSE_ACCESSORY_ID);+ if (!ok) {+ LOGW("Could not unregister HID");+ }+}diff --git a/app/src/hid_mouse.h b/app/src/hid_mouse.hnew file mode 100644index 00000000..2819b1ff--- /dev/null+++ b/app/src/hid_mouse.h@@ -0,0 +1,23 @@+#ifndef HID_MOUSE_H+#define HID_MOUSE_H++#include "common.h"++#include <stdbool.h>++#include "aoa_hid.h"+#include "trait/mouse_processor.h"++struct sc_hid_mouse {+ struct sc_mouse_processor mouse_processor; // mouse processor trait++ struct sc_aoa *aoa;+};++bool+sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa);++void+sc_hid_mouse_destroy(struct sc_hid_mouse *mouse);++#endifdiff --git a/app/src/options.h b/app/src/options.hindex 533f4a3b..b2c69664 100644--- a/app/src/options.h+++ b/app/src/options.h@@ -38,6 +38,11 @@ enum sc_keyboard_input_mode {SC_KEYBOARD_INPUT_MODE_HID,};+enum sc_mouse_input_mode {+ SC_MOUSE_INPUT_MODE_INJECT,+ SC_MOUSE_INPUT_MODE_HID,+};+enum sc_key_inject_mode {// Inject special keys, letters and space as key events.// Inject numbers and punctuation as text events.@@ -90,6 +95,7 @@ struct scrcpy_options {enum sc_log_level log_level;enum sc_record_format record_format;enum sc_keyboard_input_mode keyboard_input_mode;+ enum sc_mouse_input_mode mouse_input_mode;struct sc_port_range port_range;uint32_t tunnel_host;uint16_t tunnel_port;diff --git a/app/src/scrcpy.c b/app/src/scrcpy.cindex 5907c91e..3e50aaca 100644--- a/app/src/scrcpy.c+++ b/app/src/scrcpy.c@@ -19,6 +19,7 @@#include "file_handler.h"#ifdef HAVE_AOA_HID# include "hid_keyboard.h"+# include "hid_mouse.h"#endif#include "keyboard_inject.h"#include "mouse_inject.h"@@ -55,7 +56,12 @@ struct scrcpy {struct sc_hid_keyboard keyboard_hid;#endif};- struct sc_mouse_inject mouse_inject;+ union {+ struct sc_mouse_inject mouse_inject;+#ifdef HAVE_AOA_HID+ struct sc_hid_mouse mouse_hid;+#endif+ };};static inline void@@ -330,6 +336,7 @@ scrcpy(struct scrcpy_options *options) {#ifdef HAVE_AOA_HIDbool aoa_hid_initialized = false;bool hid_keyboard_initialized = false;+ bool hid_mouse_initialized = false;#endifbool controller_initialized = false;bool controller_started = false;@@ -451,7 +458,9 @@ scrcpy(struct scrcpy_options *options) {#ifdef HAVE_AOA_HIDbool use_hid_keyboard =options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;- if (use_hid_keyboard) {+ bool use_hid_mouse =+ options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;+ if (use_hid_keyboard || use_hid_mouse) {bool ok = sc_acksync_init(&s->acksync);if (!ok) {goto end;@@ -473,7 +482,16 @@ scrcpy(struct scrcpy_options *options) {}}- bool need_aoa = hid_keyboard_initialized;+ if (use_hid_mouse) {+ if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) {+ hid_mouse_initialized = true;+ mp = &s->mouse_hid.mouse_processor;+ } else {+ LOGE("Could not initialized HID mouse");+ }+ }++ bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized;if (!need_aoa || !sc_aoa_start(&s->aoa)) {sc_acksync_destroy(&s->acksync);@@ -491,6 +509,10 @@ aoa_hid_end:sc_hid_keyboard_destroy(&s->keyboard_hid);hid_keyboard_initialized = false;}+ if (hid_mouse_initialized) {+ sc_hid_mouse_destroy(&s->mouse_hid);+ hid_mouse_initialized = false;+ }}if (use_hid_keyboard && !hid_keyboard_initialized) {@@ -498,9 +520,16 @@ aoa_hid_end:"(-K/--hid-keyboard ignored)");options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;}++ if (use_hid_mouse && !hid_mouse_initialized) {+ LOGE("Fallback to default mouse injection method "+ "(-M/--hid-mouse ignored)");+ options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT;+ }}#elseassert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID);+ assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID);#endif// keyboard_input_mode may have been reset if HID mode failed@@ -510,8 +539,11 @@ aoa_hid_end:kp = &s->keyboard_inject.key_processor;}- sc_mouse_inject_init(&s->mouse_inject, &s->controller);- mp = &s->mouse_inject.mouse_processor;+ // mouse_input_mode may have been reset if HID mode failed+ if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) {+ sc_mouse_inject_init(&s->mouse_inject, &s->controller);+ mp = &s->mouse_inject.mouse_processor;+ }if (!controller_init(&s->controller, s->server.control_socket,acksync)) {