From cba84f69993e3d7bdbb0e44493b70c0a14689487 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sun, 26 Dec 2021 22:32:51 +0100 Subject: [PATCH] Add support for HID mouse --- app/meson.build | 1 + app/scrcpy.1 | 14 +++ app/src/cli.c | 24 +++- app/src/hid_mouse.c | 267 ++++++++++++++++++++++++++++++++++++++++++++ app/src/hid_mouse.h | 23 ++++ app/src/options.h | 6 + app/src/scrcpy.c | 42 ++++++- 7 files changed, 371 insertions(+), 6 deletions(-) create mode 100644 app/src/hid_mouse.c create mode 100644 app/src/hid_mouse.h diff --git a/app/meson.build b/app/meson.build index 38393f0c..cee261bb 100644 --- a/app/meson.build +++ b/app/meson.build @@ -77,6 +77,7 @@ if aoa_hid_support src += [ 'src/aoa_hid.c', 'src/hid_keyboard.c', + 'src/hid_mouse.c', ] endif diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 971c8a19..4d02982c 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -96,6 +96,8 @@ The keyboard layout must be configured (once and for all) on the device, via Set However, the option is only available when the HID keyboard is enabled (or a physical keyboard is connected). +Also see \fB\-\-hid\-mouse\fR. + .TP .B \-\-legacy\-paste Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v). @@ -120,6 +122,18 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimension Default is 0 (unlimited). +.TP +.B \-M, \-\-hid\-mouse +Simulate a physical mouse by using HID over AOAv2. + +In this mode, the computer mouse is captured to control the device directly (relative mouse mode). + +LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer. + +It may only work over USB, and is currently only supported on Linux. + +Also see \fB\-\-hid\-keyboard\fR. + .TP .B \-\-no\-clipboard\-autosync By default, scrcpy automatically synchronizes the computer clipboard to the device clipboard before injecting Ctrl+v, and the device clipboard to the computer clipboard whenever it changes. diff --git a/app/src/cli.c b/app/src/cli.c index ec53e5ec..3fcf94eb 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -178,7 +178,8 @@ static const struct sc_option options[] = { "directly: `adb shell am start -a " "android.settings.HARD_KEYBOARD_SETTINGS`.\n" "However, the option is only available when the HID keyboard " - "is enabled (or a physical keyboard is connected).", + "is enabled (or a physical keyboard is connected).\n" + "Also see --hid-mouse.", }, { .shortopt = 'h', @@ -214,6 +215,18 @@ static const struct sc_option options[] = { .text = "Limit the frame rate of screen capture (officially supported " "since Android 10, but may work on earlier versions).", }, + { + .shortopt = 'M', + .longopt = "hid-mouse", + .text = "Simulate a physical mouse by using HID over AOAv2.\n" + "In this mode, the computer mouse is captured to control the " + "device directly (relative mouse mode).\n" + "LAlt, LSuper or RSuper toggle the capture mode, to give " + "control of the mouse back to the computer.\n" + "It may only work over USB, and is currently only supported " + "on Linux.\n" + "Also see --hid-keyboard.", + }, { .shortopt = 'm', .longopt = "max-size", @@ -1315,6 +1328,15 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[], return false; } break; + case 'M': +#ifdef HAVE_AOA_HID + opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID; +#else + LOGE("HID over AOA (-M/--hid-mouse) is not supported on this" + "platform. It is only available on Linux."); + return false; +#endif + break; case OPT_LOCK_VIDEO_ORIENTATION: if (!parse_lock_video_orientation(optarg, &opts->lock_video_orientation)) { diff --git a/app/src/hid_mouse.c b/app/src/hid_mouse.c new file mode 100644 index 00000000..0e26c7c3 --- /dev/null +++ b/app/src/hid_mouse.c @@ -0,0 +1,267 @@ +#include "hid_mouse.h" + +#include + +#include "input_events.h" +#include "util/log.h" + +/** Downcast mouse processor to hid_mouse */ +#define DOWNCAST(MP) container_of(MP, struct sc_hid_mouse, mouse_processor) + +#define HID_MOUSE_ACCESSORY_ID 2 + +// 1 byte for buttons + padding, 1 byte for X position, 1 byte for Y position +#define HID_MOUSE_EVENT_SIZE 4 + +/** + * Mouse descriptor from the specification: + * + * + * Appendix E (p71): §E.10 Report Descriptor (Mouse) + * + * The usage tags (like Wheel) are listed in "HID Usage Tables": + * + * §4 Generic Desktop Page (0x01) (p26) + */ +static const unsigned char mouse_report_desc[] = { + // Usage Page (Generic Desktop) + 0x05, 0x01, + // Usage (Mouse) + 0x09, 0x02, + + // Collection (Application) + 0xA1, 0x01, + + // Usage (Pointer) + 0x09, 0x01, + + // Collection (Physical) + 0xA1, 0x00, + + // Usage Page (Buttons) + 0x05, 0x09, + + // Usage Minimum (1) + 0x19, 0x01, + // Usage Maximum (5) + 0x29, 0x05, + // Logical Minimum (0) + 0x15, 0x00, + // Logical Maximum (1) + 0x25, 0x01, + // Report Count (5) + 0x95, 0x05, + // Report Size (1) + 0x75, 0x01, + // Input (Data, Variable, Absolute): 5 buttons bits + 0x81, 0x02, + + // Report Count (1) + 0x95, 0x01, + // Report Size (3) + 0x75, 0x03, + // Input (Constant): 3 bits padding + 0x81, 0x01, + + // Usage Page (Generic Desktop) + 0x05, 0x01, + // Usage (X) + 0x09, 0x30, + // Usage (Y) + 0x09, 0x31, + // Usage (Wheel) + 0x09, 0x38, + // Local Minimum (-127) + 0x15, 0x81, + // Local Maximum (127) + 0x25, 0x7F, + // Report Size (8) + 0x75, 0x08, + // Report Count (3) + 0x95, 0x03, + // Input (Data, Variable, Relative): 3 position bytes (X, Y, Wheel) + 0x81, 0x06, + + // End Collection + 0xC0, + + // End Collection + 0xC0, +}; + +/** + * A mouse HID event is 3 bytes long: + * + * - byte 0: buttons state + * - byte 1: relative x motion (signed byte from -127 to 127) + * - byte 2: relative y motion (signed byte from -127 to 127) + * + * 7 6 5 4 3 2 1 0 + * +---------------+ + * byte 0: |0 0 0 . . . . .| buttons state + * +---------------+ + * ^ ^ ^ ^ ^ + * | | | | `- left button + * | | | `--- right button + * | | `----- middle button + * | `------- button 4 + * `--------- button 5 + * + * +---------------+ + * byte 1: |. . . . . . . .| relative x motion + * +---------------+ + * byte 2: |. . . . . . . .| relative y motion + * +---------------+ + * byte 3: |. . . . . . . .| wheel motion (-1, 0 or 1) + * +---------------+ + * + * As an example, here is the report for a motion of (x=5, y=-4) with left + * button pressed: + * + * +---------------+ + * |0 0 0 0 0 0 0 1| left button pressed + * +---------------+ + * |0 0 0 0 0 1 0 1| horizontal motion (x = 5) + * +---------------+ + * |1 1 1 1 1 1 0 0| relative y motion (y = -4) + * +---------------+ + * |0 0 0 0 0 0 0 0| wheel motion + * +---------------+ + */ + +static bool +sc_hid_mouse_event_init(struct sc_hid_event *hid_event) { + unsigned char *buffer = calloc(1, HID_MOUSE_EVENT_SIZE); + if (!buffer) { + LOG_OOM(); + return false; + } + + sc_hid_event_init(hid_event, HID_MOUSE_ACCESSORY_ID, buffer, + HID_MOUSE_EVENT_SIZE); + return true; +} + +static unsigned char +buttons_state_to_hid_buttons(uint8_t buttons_state) { + unsigned char c = 0; + if (buttons_state & SC_MOUSE_BUTTON_LEFT) { + c |= 1 << 0; + } + if (buttons_state & SC_MOUSE_BUTTON_RIGHT) { + c |= 1 << 1; + } + if (buttons_state & SC_MOUSE_BUTTON_MIDDLE) { + c |= 1 << 2; + } + if (buttons_state & SC_MOUSE_BUTTON_X1) { + c |= 1 << 3; + } + if (buttons_state & SC_MOUSE_BUTTON_X2) { + c |= 1 << 4; + } + return c; +} + +static void +sc_mouse_processor_process_mouse_motion(struct sc_mouse_processor *mp, + const struct sc_mouse_motion_event *event) { + struct sc_hid_mouse *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + if (!sc_hid_mouse_event_init(&hid_event)) { + return; + } + + unsigned char *buffer = hid_event.buffer; + buffer[0] = buttons_state_to_hid_buttons(event->buttons_state); + buffer[1] = CLAMP(event->xrel, -127, 127); + buffer[2] = CLAMP(event->yrel, -127, 127); + buffer[3] = 0; // wheel coordinates only used for scrolling + + if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { + sc_hid_event_destroy(&hid_event); + LOGW("Could request HID event"); + } +} + +static void +sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp, + const struct sc_mouse_click_event *event) { + struct sc_hid_mouse *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + if (!sc_hid_mouse_event_init(&hid_event)) { + return; + } + + unsigned char *buffer = hid_event.buffer; + buffer[0] = buttons_state_to_hid_buttons(event->buttons_state); + buffer[1] = 0; // no x motion + buffer[2] = 0; // no y motion + buffer[3] = 0; // wheel coordinates only used for scrolling + + if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { + sc_hid_event_destroy(&hid_event); + LOGW("Could request HID event"); + } +} + +static void +sc_mouse_processor_process_mouse_scroll(struct sc_mouse_processor *mp, + const struct sc_mouse_scroll_event *event) { + struct sc_hid_mouse *mouse = DOWNCAST(mp); + + struct sc_hid_event hid_event; + if (!sc_hid_mouse_event_init(&hid_event)) { + return; + } + + unsigned char *buffer = hid_event.buffer; + buffer[0] = 0; // buttons state irrelevant (and unknown) + buffer[1] = 0; // no x motion + buffer[2] = 0; // no y motion + // In practice, vscroll is always -1, 0 or 1, but in theory other values + // are possible + buffer[3] = CLAMP(event->vscroll, -127, 127); + // Horizontal scrolling ignored + + if (!sc_aoa_push_hid_event(mouse->aoa, &hid_event)) { + sc_hid_event_destroy(&hid_event); + LOGW("Could request HID event"); + } +} + +bool +sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa) { + mouse->aoa = aoa; + + bool ok = sc_aoa_setup_hid(aoa, HID_MOUSE_ACCESSORY_ID, mouse_report_desc, + ARRAY_LEN(mouse_report_desc)); + if (!ok) { + LOGW("Register HID mouse failed"); + return false; + } + + static const struct sc_mouse_processor_ops ops = { + .process_mouse_motion = sc_mouse_processor_process_mouse_motion, + .process_mouse_click = sc_mouse_processor_process_mouse_click, + .process_mouse_scroll = sc_mouse_processor_process_mouse_scroll, + // Touch events not supported (coordinates are not relative) + .process_touch = NULL, + }; + + mouse->mouse_processor.ops = &ops; + + mouse->mouse_processor.relative_mode = true; + + return true; +} + +void +sc_hid_mouse_destroy(struct sc_hid_mouse *mouse) { + bool ok = sc_aoa_unregister_hid(mouse->aoa, HID_MOUSE_ACCESSORY_ID); + if (!ok) { + LOGW("Could not unregister HID"); + } +} diff --git a/app/src/hid_mouse.h b/app/src/hid_mouse.h new file mode 100644 index 00000000..2819b1ff --- /dev/null +++ b/app/src/hid_mouse.h @@ -0,0 +1,23 @@ +#ifndef HID_MOUSE_H +#define HID_MOUSE_H + +#include "common.h" + +#include + +#include "aoa_hid.h" +#include "trait/mouse_processor.h" + +struct sc_hid_mouse { + struct sc_mouse_processor mouse_processor; // mouse processor trait + + struct sc_aoa *aoa; +}; + +bool +sc_hid_mouse_init(struct sc_hid_mouse *mouse, struct sc_aoa *aoa); + +void +sc_hid_mouse_destroy(struct sc_hid_mouse *mouse); + +#endif diff --git a/app/src/options.h b/app/src/options.h index 533f4a3b..b2c69664 100644 --- a/app/src/options.h +++ b/app/src/options.h @@ -38,6 +38,11 @@ enum sc_keyboard_input_mode { SC_KEYBOARD_INPUT_MODE_HID, }; +enum sc_mouse_input_mode { + SC_MOUSE_INPUT_MODE_INJECT, + SC_MOUSE_INPUT_MODE_HID, +}; + enum sc_key_inject_mode { // Inject special keys, letters and space as key events. // Inject numbers and punctuation as text events. @@ -90,6 +95,7 @@ struct scrcpy_options { enum sc_log_level log_level; enum sc_record_format record_format; enum sc_keyboard_input_mode keyboard_input_mode; + enum sc_mouse_input_mode mouse_input_mode; struct sc_port_range port_range; uint32_t tunnel_host; uint16_t tunnel_port; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 5907c91e..3e50aaca 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -19,6 +19,7 @@ #include "file_handler.h" #ifdef HAVE_AOA_HID # include "hid_keyboard.h" +# include "hid_mouse.h" #endif #include "keyboard_inject.h" #include "mouse_inject.h" @@ -55,7 +56,12 @@ struct scrcpy { struct sc_hid_keyboard keyboard_hid; #endif }; - struct sc_mouse_inject mouse_inject; + union { + struct sc_mouse_inject mouse_inject; +#ifdef HAVE_AOA_HID + struct sc_hid_mouse mouse_hid; +#endif + }; }; static inline void @@ -330,6 +336,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_AOA_HID bool aoa_hid_initialized = false; bool hid_keyboard_initialized = false; + bool hid_mouse_initialized = false; #endif bool controller_initialized = false; bool controller_started = false; @@ -451,7 +458,9 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_AOA_HID bool use_hid_keyboard = options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID; - if (use_hid_keyboard) { + bool use_hid_mouse = + options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID; + if (use_hid_keyboard || use_hid_mouse) { bool ok = sc_acksync_init(&s->acksync); if (!ok) { goto end; @@ -473,7 +482,16 @@ scrcpy(struct scrcpy_options *options) { } } - bool need_aoa = hid_keyboard_initialized; + if (use_hid_mouse) { + if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) { + hid_mouse_initialized = true; + mp = &s->mouse_hid.mouse_processor; + } else { + LOGE("Could not initialized HID mouse"); + } + } + + bool need_aoa = hid_keyboard_initialized || hid_mouse_initialized; if (!need_aoa || !sc_aoa_start(&s->aoa)) { sc_acksync_destroy(&s->acksync); @@ -491,6 +509,10 @@ aoa_hid_end: sc_hid_keyboard_destroy(&s->keyboard_hid); hid_keyboard_initialized = false; } + if (hid_mouse_initialized) { + sc_hid_mouse_destroy(&s->mouse_hid); + hid_mouse_initialized = false; + } } if (use_hid_keyboard && !hid_keyboard_initialized) { @@ -498,9 +520,16 @@ aoa_hid_end: "(-K/--hid-keyboard ignored)"); options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT; } + + if (use_hid_mouse && !hid_mouse_initialized) { + LOGE("Fallback to default mouse injection method " + "(-M/--hid-mouse ignored)"); + options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT; + } } #else assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID); + assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID); #endif // keyboard_input_mode may have been reset if HID mode failed @@ -510,8 +539,11 @@ aoa_hid_end: kp = &s->keyboard_inject.key_processor; } - sc_mouse_inject_init(&s->mouse_inject, &s->controller); - mp = &s->mouse_inject.mouse_processor; + // mouse_input_mode may have been reset if HID mode failed + if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) { + sc_mouse_inject_init(&s->mouse_inject, &s->controller); + mp = &s->mouse_inject.mouse_processor; + } if (!controller_init(&s->controller, s->server.control_socket, acksync)) {