Add support for USB HID keyboard over AOAv2

This provides a better input experience, by simulating a physical
keyboard. It converts SDL keyboard events to proper HID events, and send
them over AOAv2.

This is a rewriting and bugfix of the origin code from @amosbird:
<https://github.com/Genymobile/scrcpy/issues/279#issuecomment-453819354>

The feature is enabled the command line option -K or --hid-keyboard,
and is only available on Linux, over USB.

Refs <https://source.android.com/devices/accessories/aoa2#hid-support>
Refs <https://www.usb.org/sites/default/files/hid1_11.pdf>

PR #2632 <https://github.com/Genymobile/scrcpy/pull/2632>

Signed-off-by: Romain Vimont <rom@rom1v.com>

BUILD.md

@@ -14,7 +14,8 @@ First, you need to install the required packages:
 # for Debian/Ubuntu
 sudo apt install ffmpeg libsdl2-2.0-0 adb wget \
                  gcc git pkg-config meson ninja-build libsdl2-dev \
-                 libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
+                 libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \
+                 libusb-1.0-0 libusb-dev
 ```
 
 Then clone the repo and execute the installation script
@@ -88,11 +89,12 @@ Install the required packages from your package manager.
 
 ```bash
 # runtime dependencies
-sudo apt install ffmpeg libsdl2-2.0-0 adb
+sudo apt install ffmpeg libsdl2-2.0-0 adb libusb-1.0-0
 
 # client build dependencies
 sudo apt install gcc git pkg-config meson ninja-build libsdl2-dev \
-                 libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev
+                 libavcodec-dev libavdevice-dev libavformat-dev libavutil-dev \
+                 libusb-dev
 
 # server build dependencies
 sudo apt install openjdk-11-jdk
@@ -114,7 +116,7 @@ pip3 install meson
 sudo dnf install https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm
 
 # client build dependencies
-sudo dnf install SDL2-devel ffms2-devel meson gcc make
+sudo dnf install SDL2-devel ffms2-devel libusb-devel meson gcc make
 
 # server build dependencies
 sudo dnf install java-devel

README.md

@@ -673,6 +673,39 @@ content (if supported by the app) relative to the center of the screen.
 Concretely, scrcpy generates additional touch events from a "virtual finger" at
 a location inverted through the center of the screen.
 
+#### Physical keyboard simulation (HID)
+
+By default, scrcpy uses Android key or text injection: it works everywhere, but
+is limited to ASCII.
+
+On Linux, scrcpy can simulate a USB physical keyboard on Android to provide a
+better input experience (using [USB HID over AOAv2][hid-aoav2]): the virtual
+keyboard is disabled and it works for all characters and IME.
+
+[hid-aoav2]: https://source.android.com/devices/accessories/aoa2#hid-support
+
+However, it only works if the device is connected by USB, and is currently only
+supported on Linux.
+
+To enable this mode:
+
+```bash
+scrcpy --hid-keyboard
+scrcpy -K  # short version
+```
+
+If it fails for some reason (for example because the device is not connected via
+USB), it automatically fallbacks to the default mode (with a log in the
+console). This allows to use the same command line options when connected over
+USB and TCP/IP.
+
+In this mode, raw key events (scancodes) are sent to the device, independently
+of the host key mapping. Therefore, if your keyboard layout does not match, it
+must be configured on the Android device, in Settings → System → Languages and
+input → [Physical keyboard].
+
+[Physical keyboard]: https://github.com/Genymobile/scrcpy/pull/2632#issuecomment-923756915
+
 
 #### Text injection preference
 
@@ -692,6 +725,9 @@ scrcpy --prefer-text
 
 (but this will break keyboard behavior in games)
 
+This option has no effect on HID keyboard (all key events are sent as
+scancodes in this mode).
+
 [textevents]: https://blog.rom1v.com/2018/03/introducing-scrcpy/#handle-text-input
 [prefertext]: https://github.com/Genymobile/scrcpy/issues/650#issuecomment-512945343
 
@@ -707,6 +743,9 @@ To avoid forwarding repeated key events:
 scrcpy --no-key-repeat
 ```
 
+This option has no effect on HID keyboard (key repeat is handled by Android
+directly in this mode).
+
 
 #### Right-click and middle-click
 

app/meson.build

@@ -42,6 +42,14 @@ if v4l2_support
     src += [ 'src/v4l2_sink.c' ]
 endif
 
+aoa_hid_support = host_machine.system() == 'linux'
+if aoa_hid_support
+    src += [
+        'src/aoa_hid.c',
+        'src/hid_keyboard.c',
+    ]
+endif
+
 check_functions = [
     'strdup'
 ]
@@ -62,8 +70,11 @@ if not get_option('crossbuild_windows')
         dependencies += dependency('libavdevice')
     endif
 
-else
+    if aoa_hid_support
+        dependencies += dependency('libusb-1.0')
+    endif
 
+else
     # cross-compile mingw32 build (from Linux to Windows)
     prebuilt_sdl2 = meson.get_cross_property('prebuilt_sdl2')
     sdl2_bin_dir = meson.current_source_dir() + '/../prebuilt-deps/' + prebuilt_sdl2 + '/bin'
@@ -140,6 +151,9 @@ conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == '
 # enable V4L2 support (linux only)
 conf.set('HAVE_V4L2', v4l2_support)
 
+# enable HID over AOA support (linux only)
+conf.set('HAVE_AOA_HID', aoa_hid_support)
+
 configure_file(configuration: conf, output: 'config.h')
 
 src_dir = include_directories('src')

app/scrcpy.1

@@ -82,6 +82,14 @@ Start in fullscreen.
 .B \-h, \-\-help
 Print this help.
 
+.TP
+.B \-K, \-\-hid\-keyboard
+Simulate a physical keyboard by using HID over AOAv2.
+
+This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.
+
+It may only work over USB, and is currently only supported on Linux.
+
 .TP
 .B \-\-legacy\-paste
 Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v).

app/src/aoa_hid.c

@@ -0,0 +1,368 @@
+#include "util/log.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+#include "aoa_hid.h"
+
+// See <https://source.android.com/devices/accessories/aoa2#hid-support>.
+#define ACCESSORY_REGISTER_HID 54
+#define ACCESSORY_SET_HID_REPORT_DESC 56
+#define ACCESSORY_SEND_HID_EVENT 57
+#define ACCESSORY_UNREGISTER_HID 55
+
+#define DEFAULT_TIMEOUT 1000
+
+static void
+sc_hid_event_log(const struct sc_hid_event *event) {
+    // HID Event: [00] FF FF FF FF...
+    assert(event->size);
+    unsigned buffer_size = event->size * 3 + 1;
+    char *buffer = malloc(buffer_size);
+    if (!buffer) {
+        return;
+    }
+    for (unsigned i = 0; i < event->size; ++i) {
+        snprintf(buffer + i * 3, 4, " %02x", event->buffer[i]);
+    }
+    LOGV("HID Event: [%d]%s", event->accessory_id, buffer);
+    free(buffer);
+}
+
+void
+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
+                  unsigned char *buffer, uint16_t buffer_size) {
+    hid_event->accessory_id = accessory_id;
+    hid_event->buffer = buffer;
+    hid_event->size = buffer_size;
+}
+
+void
+sc_hid_event_destroy(struct sc_hid_event *hid_event) {
+    free(hid_event->buffer);
+}
+
+static inline void
+log_libusb_error(enum libusb_error errcode) {
+    LOGW("libusb error: %s", libusb_strerror(errcode));
+}
+
+static bool
+accept_device(libusb_device *device, const char *serial) {
+    // do not log any USB error in this function, it is expected that many USB
+    // devices available on the computer have permission restrictions
+
+    struct libusb_device_descriptor desc;
+    libusb_get_device_descriptor(device, &desc);
+
+    if (!desc.iSerialNumber) {
+        return false;
+    }
+
+    libusb_device_handle *handle;
+    int result = libusb_open(device, &handle);
+    if (result < 0) {
+        return false;
+    }
+
+    char buffer[128];
+    result = libusb_get_string_descriptor_ascii(handle, desc.iSerialNumber,
+                                                (unsigned char *) buffer,
+                                                sizeof(buffer));
+    libusb_close(handle);
+    if (result < 0) {
+        return false;
+    }
+
+    buffer[sizeof(buffer) - 1] = '\0'; // just in case
+
+    // accept the device if its serial matches
+    return !strcmp(buffer, serial);
+}
+
+static libusb_device *
+sc_aoa_find_usb_device(const char *serial) {
+    if (!serial) {
+        return NULL;
+    }
+
+    libusb_device **list;
+    libusb_device *result = NULL;
+    ssize_t count = libusb_get_device_list(NULL, &list);
+    if (count < 0) {
+        log_libusb_error((enum libusb_error) count);
+        return NULL;
+    }
+
+    for (size_t i = 0; i < (size_t) count; ++i) {
+        libusb_device *device = list[i];
+
+        if (accept_device(device, serial)) {
+            result = libusb_ref_device(device);
+            break;
+        }
+    }
+    libusb_free_device_list(list, 1);
+    return result;
+}
+
+static int
+sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) {
+    int result = libusb_open(device, handle);
+    if (result < 0) {
+        log_libusb_error((enum libusb_error) result);
+        return result;
+    }
+    return 0;
+}
+
+bool
+sc_aoa_init(struct sc_aoa *aoa, const char *serial) {
+    cbuf_init(&aoa->queue);
+
+    if (!sc_mutex_init(&aoa->mutex)) {
+        return false;
+    }
+
+    if (!sc_cond_init(&aoa->event_cond)) {
+        sc_mutex_destroy(&aoa->mutex);
+        return false;
+    }
+
+    if (libusb_init(&aoa->usb_context) != LIBUSB_SUCCESS) {
+        sc_cond_destroy(&aoa->event_cond);
+        sc_mutex_destroy(&aoa->mutex);
+        return false;
+    }
+
+    aoa->usb_device = sc_aoa_find_usb_device(serial);
+    if (!aoa->usb_device) {
+        LOGW("USB device of serial %s not found", serial);
+        libusb_exit(aoa->usb_context);
+        sc_mutex_destroy(&aoa->mutex);
+        sc_cond_destroy(&aoa->event_cond);
+        return false;
+    }
+
+    if (sc_aoa_open_usb_handle(aoa->usb_device, &aoa->usb_handle) < 0) {
+        LOGW("Open USB handle failed");
+        libusb_unref_device(aoa->usb_device);
+        libusb_exit(aoa->usb_context);
+        sc_cond_destroy(&aoa->event_cond);
+        sc_mutex_destroy(&aoa->mutex);
+        return false;
+    }
+
+    aoa->stopped = false;
+
+    return true;
+}
+
+void
+sc_aoa_destroy(struct sc_aoa *aoa) {
+    // Destroy remaining events
+    struct sc_hid_event event;
+    while (cbuf_take(&aoa->queue, &event)) {
+        sc_hid_event_destroy(&event);
+    }
+
+    libusb_close(aoa->usb_handle);
+    libusb_unref_device(aoa->usb_device);
+    libusb_exit(aoa->usb_context);
+    sc_cond_destroy(&aoa->event_cond);
+    sc_mutex_destroy(&aoa->mutex);
+}
+
+static bool
+sc_aoa_register_hid(struct sc_aoa *aoa, uint16_t accessory_id,
+                    uint16_t report_desc_size) {
+    uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+    uint8_t request = ACCESSORY_REGISTER_HID;
+    // <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+    // value (arg0): accessory assigned ID for the HID device
+    // index (arg1): total length of the HID report descriptor
+    uint16_t value = accessory_id;
+    uint16_t index = report_desc_size;
+    unsigned char *buffer = NULL;
+    uint16_t length = 0;
+    int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+                                         value, index, buffer, length,
+                                         DEFAULT_TIMEOUT);
+    if (result < 0) {
+        log_libusb_error((enum libusb_error) result);
+        return false;
+    }
+
+    return true;
+}
+
+static bool
+sc_aoa_set_hid_report_desc(struct sc_aoa *aoa, uint16_t accessory_id,
+                           const unsigned char *report_desc,
+                           uint16_t report_desc_size) {
+    uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+    uint8_t request = ACCESSORY_SET_HID_REPORT_DESC;
+    /**
+     * If the HID descriptor is longer than the endpoint zero max packet size,
+     * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC
+     * commands. The data for the descriptor must be sent sequentially
+     * if multiple packets are needed.
+     * <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+     *
+     * libusb handles packet abstraction internally, so we don't need to care
+     * about bMaxPacketSize0 here.
+     *
+     * See <https://libusb.sourceforge.io/api-1.0/libusb_packetoverflow.html>
+     */
+    // value (arg0): accessory assigned ID for the HID device
+    // index (arg1): offset of data (buffer) in descriptor
+    uint16_t value = accessory_id;
+    uint16_t index = 0;
+    // libusb_control_transfer expects a pointer to non-const
+    unsigned char *buffer = (unsigned char *) report_desc;
+    uint16_t length = report_desc_size;
+    int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+                                         value, index, buffer, length,
+                                         DEFAULT_TIMEOUT);
+    if (result < 0) {
+        log_libusb_error((enum libusb_error) result);
+        return false;
+    }
+
+    return true;
+}
+
+bool
+sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,
+                 const unsigned char *report_desc, uint16_t report_desc_size) {
+    bool ok = sc_aoa_register_hid(aoa, accessory_id, report_desc_size);
+    if (!ok) {
+        return false;
+    }
+
+    ok = sc_aoa_set_hid_report_desc(aoa, accessory_id, report_desc,
+                                    report_desc_size);
+    if (!ok) {
+        if (!sc_aoa_unregister_hid(aoa, accessory_id)) {
+            LOGW("Could not unregister HID");
+        }
+        return false;
+    }
+
+    return true;
+}
+
+static bool
+sc_aoa_send_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
+    uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+    uint8_t request = ACCESSORY_SEND_HID_EVENT;
+    // <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+    // value (arg0): accessory assigned ID for the HID device
+    // index (arg1): 0 (unused)
+    uint16_t value = event->accessory_id;
+    uint16_t index = 0;
+    unsigned char *buffer = event->buffer;
+    uint16_t length = event->size;
+    int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+                                         value, index, buffer, length,
+                                         DEFAULT_TIMEOUT);
+    if (result < 0) {
+        log_libusb_error((enum libusb_error) result);
+        return false;
+    }
+
+    return true;
+}
+
+bool
+sc_aoa_unregister_hid(struct sc_aoa *aoa, const uint16_t accessory_id) {
+    uint8_t request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR;
+    uint8_t request = ACCESSORY_UNREGISTER_HID;
+    // <https://source.android.com/devices/accessories/aoa2.html#hid-support>
+    // value (arg0): accessory assigned ID for the HID device
+    // index (arg1): 0
+    uint16_t value = accessory_id;
+    uint16_t index = 0;
+    unsigned char *buffer = NULL;
+    uint16_t length = 0;
+    int result = libusb_control_transfer(aoa->usb_handle, request_type, request,
+                                         value, index, buffer, length,
+                                         DEFAULT_TIMEOUT);
+    if (result < 0) {
+        log_libusb_error((enum libusb_error) result);
+        return false;
+    }
+
+    return true;
+}
+
+bool
+sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event) {
+    if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
+        sc_hid_event_log(event);
+    }
+
+    sc_mutex_lock(&aoa->mutex);
+    bool was_empty = cbuf_is_empty(&aoa->queue);
+    bool res = cbuf_push(&aoa->queue, *event);
+    if (was_empty) {
+        sc_cond_signal(&aoa->event_cond);
+    }
+    sc_mutex_unlock(&aoa->mutex);
+    return res;
+}
+
+static int
+run_aoa_thread(void *data) {
+    struct sc_aoa *aoa = data;
+
+    for (;;) {
+        sc_mutex_lock(&aoa->mutex);
+        while (!aoa->stopped && cbuf_is_empty(&aoa->queue)) {
+            sc_cond_wait(&aoa->event_cond, &aoa->mutex);
+        }
+        if (aoa->stopped) {
+            // Stop immediately, do not process further events
+            sc_mutex_unlock(&aoa->mutex);
+            break;
+        }
+        struct sc_hid_event event;
+        bool non_empty = cbuf_take(&aoa->queue, &event);
+        assert(non_empty);
+        (void) non_empty;
+        sc_mutex_unlock(&aoa->mutex);
+
+        bool ok = sc_aoa_send_hid_event(aoa, &event);
+        sc_hid_event_destroy(&event);
+        if (!ok) {
+            LOGW("Could not send HID event to USB device");
+        }
+    }
+    return 0;
+}
+
+bool
+sc_aoa_start(struct sc_aoa *aoa) {
+    LOGD("Starting AOA thread");
+
+    bool ok = sc_thread_create(&aoa->thread, run_aoa_thread, "aoa_thread", aoa);
+    if (!ok) {
+        LOGC("Could not start AOA thread");
+        return false;
+    }
+
+    return true;
+}
+
+void
+sc_aoa_stop(struct sc_aoa *aoa) {
+    sc_mutex_lock(&aoa->mutex);
+    aoa->stopped = true;
+    sc_cond_signal(&aoa->event_cond);
+    sc_mutex_unlock(&aoa->mutex);
+}
+
+void
+sc_aoa_join(struct sc_aoa *aoa) {
+    sc_thread_join(&aoa->thread, NULL);
+}

app/src/aoa_hid.h

@@ -0,0 +1,65 @@
+#ifndef SC_AOA_HID_H
+#define SC_AOA_HID_H
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <libusb-1.0/libusb.h>
+
+#include "scrcpy.h"
+#include "util/cbuf.h"
+#include "util/thread.h"
+
+struct sc_hid_event {
+    uint16_t accessory_id;
+    unsigned char *buffer;
+    uint16_t size;
+};
+
+// Takes ownership of buffer
+void
+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
+                  unsigned char *buffer, uint16_t buffer_size);
+
+void
+sc_hid_event_destroy(struct sc_hid_event *hid_event);
+
+struct sc_hid_event_queue CBUF(struct sc_hid_event, 64);
+
+struct sc_aoa {
+    libusb_context *usb_context;
+    libusb_device *usb_device;
+    libusb_device_handle *usb_handle;
+    sc_thread thread;
+    sc_mutex mutex;
+    sc_cond event_cond;
+    bool stopped;
+    struct sc_hid_event_queue queue;
+};
+
+bool
+sc_aoa_init(struct sc_aoa *aoa, const char *serial);
+
+void
+sc_aoa_destroy(struct sc_aoa *aoa);
+
+bool
+sc_aoa_start(struct sc_aoa *aoa);
+
+void
+sc_aoa_stop(struct sc_aoa *aoa);
+
+void
+sc_aoa_join(struct sc_aoa *aoa);
+
+bool
+sc_aoa_setup_hid(struct sc_aoa *aoa, uint16_t accessory_id,
+              const unsigned char *report_desc, uint16_t report_desc_size);
+
+bool
+sc_aoa_unregister_hid(struct sc_aoa *aoa, uint16_t accessory_id);
+
+bool
+sc_aoa_push_hid_event(struct sc_aoa *aoa, const struct sc_hid_event *event);
+
+#endif

app/src/cli.c

@@ -76,6 +76,14 @@ scrcpy_print_usage(const char *arg0) {
         "    -f, --fullscreen\n"
         "        Start in fullscreen.\n"
         "\n"
+        "    -K, --hid-keyboard\n"
+        "        Simulate a physical keyboard by using HID over AOAv2.\n"
+        "        It provides a better experience for IME users, and allows to\n"
+        "        generate non-ASCII characters, contrary to the default\n"
+        "        injection method.\n"
+        "        It may only work over USB, and is currently only supported\n"
+        "        on Linux.\n"
+        "\n"
         "    -h, --help\n"
         "        Print this help.\n"
         "\n"
@@ -738,6 +746,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
                                                   OPT_FORWARD_ALL_CLICKS},
         {"fullscreen",             no_argument,       NULL, 'f'},
         {"help",                   no_argument,       NULL, 'h'},
+        {"hid-keyboard",           no_argument,       NULL, 'K'},
         {"legacy-paste",           no_argument,       NULL, OPT_LEGACY_PASTE},
         {"lock-video-orientation", optional_argument, NULL,
                                                   OPT_LOCK_VIDEO_ORIENTATION},
@@ -784,7 +793,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
     optind = 0; // reset to start from the first argument in tests
 
     int c;
-    while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTvV:w",
+    while ((c = getopt_long(argc, argv, "b:c:fF:hKm:nNp:r:s:StTvV:w",
                             long_options, NULL)) != -1) {
         switch (c) {
             case 'b':
@@ -817,6 +826,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) {
             case 'h':
                 args->help = true;
                 break;
+            case 'K':
+                opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
+                break;
             case OPT_MAX_FPS:
                 if (!parse_max_fps(optarg, &opts->max_fps)) {
                     return false;

app/src/hid_keyboard.c

@@ -0,0 +1,302 @@
+#include "hid_keyboard.h"
+
+#include <assert.h>
+#include <SDL2/SDL_events.h>
+
+#include "util/log.h"
+
+/** Downcast key processor to hid_keyboard */
+#define DOWNCAST(KP) container_of(KP, struct sc_hid_keyboard, key_processor)
+
+#define HID_KEYBOARD_ACCESSORY_ID 1
+
+#define HID_MODIFIER_NONE 0x00
+#define HID_MODIFIER_LEFT_CONTROL (1 << 0)
+#define HID_MODIFIER_LEFT_SHIFT (1 << 1)
+#define HID_MODIFIER_LEFT_ALT (1 << 2)
+#define HID_MODIFIER_LEFT_GUI (1 << 3)
+#define HID_MODIFIER_RIGHT_CONTROL (1 << 4)
+#define HID_MODIFIER_RIGHT_SHIFT (1 << 5)
+#define HID_MODIFIER_RIGHT_ALT (1 << 6)
+#define HID_MODIFIER_RIGHT_GUI (1 << 7)
+
+#define HID_KEYBOARD_INDEX_MODIFIER 0
+#define HID_KEYBOARD_INDEX_KEYS 2
+
+// USB HID protocol says 6 keys in an event is the requirement for BIOS
+// keyboard support, though OS could support more keys via modifying the report
+// desc. 6 should be enough for scrcpy.
+#define HID_KEYBOARD_MAX_KEYS 6
+#define HID_KEYBOARD_EVENT_SIZE (2 + HID_KEYBOARD_MAX_KEYS)
+
+#define HID_RESERVED 0x00
+#define HID_ERROR_ROLL_OVER 0x01
+
+/**
+ * For HID over AOAv2, only report descriptor is needed.
+ *
+ * The specification is available here:
+ * <https://www.usb.org/sites/default/files/hid1_11.pdf>
+ *
+ * In particular, read:
+ *  - 6.2.2 Report Descriptor
+ *  - Appendix B.1 Protocol 1 (Keyboard)
+ *  - Appendix C: Keyboard Implementation
+ *
+ * Normally a basic HID keyboard uses 8 bytes:
+ *     Modifier Reserved Key Key Key Key Key Key
+ *
+ * You can dump your device's report descriptor with:
+ *
+ *     sudo usbhid-dump -m vid:pid -e descriptor
+ *
+ * (change vid:pid' to your device's vendor ID and product ID).
+ */
+static const unsigned char keyboard_report_desc[]  = {
+    // Usage Page (Generic Desktop)
+    0x05, 0x01,
+    // Usage (Keyboard)
+    0x09, 0x06,
+
+    // Collection (Application)
+    0xA1, 0x01,
+
+    // Usage Page (Key Codes)
+    0x05, 0x07,
+    // Usage Minimum (224)
+    0x19, 0xE0,
+     // Usage Maximum (231)
+    0x29, 0xE7,
+    // Logical Minimum (0)
+    0x15, 0x00,
+    // Logical Maximum (1)
+    0x25, 0x01,
+    // Report Size (1)
+    0x75, 0x01,
+    // Report Count (8)
+    0x95, 0x08,
+    // Input (Data, Variable, Absolute): Modifier byte
+    0x81, 0x02,
+
+    // Report Size (8)
+    0x75, 0x08,
+    // Report Count (1)
+    0x95, 0x01,
+    // Input (Constant): Reserved byte
+    0x81, 0x01,
+
+    // Usage Page (LEDs)
+    0x05, 0x08,
+    // Usage Minimum (1)
+    0x19, 0x01,
+    // Usage Maximum (5)
+    0x29, 0x05,
+    // Report Size (1)
+    0x75, 0x01,
+    // Report Count (5)
+    0x95, 0x05,
+    // Output (Data, Variable, Absolute): LED report
+    0x91, 0x02,
+
+    // Report Size (3)
+    0x75, 0x03,
+    // Report Count (1)
+    0x95, 0x01,
+    // Output (Constant): LED report padding
+    0x91, 0x01,
+
+    // Usage Page (Key Codes)
+    0x05, 0x07,
+    // Usage Minimum (0)
+    0x19, 0x00,
+    // Usage Maximum (101)
+    0x29, SC_HID_KEYBOARD_KEYS - 1,
+    // Logical Minimum (0)
+    0x15, 0x00,
+    // Logical Maximum(101)
+    0x25, SC_HID_KEYBOARD_KEYS - 1,
+    // Report Size (8)
+    0x75, 0x08,
+    // Report Count (6)
+    0x95, HID_KEYBOARD_MAX_KEYS,
+    // Input (Data, Array): Keys
+    0x81, 0x00,
+
+    // End Collection
+    0xC0
+};
+
+static unsigned char
+sdl_keymod_to_hid_modifiers(SDL_Keymod mod) {
+    unsigned char modifiers = HID_MODIFIER_NONE;
+    if (mod & KMOD_LCTRL) {
+        modifiers |= HID_MODIFIER_LEFT_CONTROL;
+    }
+    if (mod & KMOD_LSHIFT) {
+        modifiers |= HID_MODIFIER_LEFT_SHIFT;
+    }
+    if (mod & KMOD_LALT) {
+        modifiers |= HID_MODIFIER_LEFT_ALT;
+    }
+    if (mod & KMOD_LGUI) {
+        modifiers |= HID_MODIFIER_LEFT_GUI;
+    }
+    if (mod & KMOD_RCTRL) {
+        modifiers |= HID_MODIFIER_RIGHT_CONTROL;
+    }
+    if (mod & KMOD_RSHIFT) {
+        modifiers |= HID_MODIFIER_RIGHT_SHIFT;
+    }
+    if (mod & KMOD_RALT) {
+        modifiers |= HID_MODIFIER_RIGHT_ALT;
+    }
+    if (mod & KMOD_RGUI) {
+        modifiers |= HID_MODIFIER_RIGHT_GUI;
+    }
+    return modifiers;
+}
+
+static bool
+sc_hid_keyboard_event_init(struct sc_hid_event *hid_event) {
+    unsigned char *buffer = malloc(HID_KEYBOARD_EVENT_SIZE);
+    if (!buffer) {
+        return false;
+    }
+
+    buffer[HID_KEYBOARD_INDEX_MODIFIER] = HID_MODIFIER_NONE;
+    buffer[1] = HID_RESERVED;
+    memset(&buffer[HID_KEYBOARD_INDEX_KEYS], 0, HID_KEYBOARD_MAX_KEYS);
+
+    sc_hid_event_init(hid_event, HID_KEYBOARD_ACCESSORY_ID, buffer,
+                      HID_KEYBOARD_EVENT_SIZE);
+    return true;
+}
+
+static inline bool
+scancode_is_modifier(SDL_Scancode scancode) {
+    return scancode >= SDL_SCANCODE_LCTRL && scancode <= SDL_SCANCODE_RGUI;
+}
+
+static bool
+convert_hid_keyboard_event(struct sc_hid_keyboard *kb,
+                           struct sc_hid_event *hid_event,
+                           const SDL_KeyboardEvent *event) {
+    SDL_Scancode scancode = event->keysym.scancode;
+    assert(scancode >= 0);
+
+    // SDL also generates events when only modifiers are pressed, we cannot
+    // ignore them totally, for example press 'a' first then press 'Control',
+    // if we ignore 'Control' event, only 'a' is sent.
+    if (scancode >= SC_HID_KEYBOARD_KEYS && !scancode_is_modifier(scancode)) {
+        // Scancode to ignore
+        return false;
+    }
+
+    if (!sc_hid_keyboard_event_init(hid_event)) {
+        LOGW("Could not initialize HID keyboard event");
+        return false;
+    }
+
+    unsigned char modifiers = sdl_keymod_to_hid_modifiers(event->keysym.mod);
+
+    if (scancode < SC_HID_KEYBOARD_KEYS) {
+        // Pressed is true and released is false
+        kb->keys[scancode] = (event->type == SDL_KEYDOWN);
+        LOGV("keys[%02x] = %s", scancode,
+             kb->keys[scancode] ? "true" : "false");
+    }
+
+    hid_event->buffer[HID_KEYBOARD_INDEX_MODIFIER] = modifiers;
+
+    unsigned char *keys_buffer = &hid_event->buffer[HID_KEYBOARD_INDEX_KEYS];
+    // Re-calculate pressed keys every time
+    int keys_pressed_count = 0;
+    for (int i = 0; i < SC_HID_KEYBOARD_KEYS; ++i) {
+        if (kb->keys[i]) {
+            // USB HID protocol says that if keys exceeds report count, a
+            // phantom state should be reported
+            if (keys_pressed_count >= HID_KEYBOARD_MAX_KEYS) {
+                // Pantom state:
+                //  - Modifiers
+                //  - Reserved
+                //  - ErrorRollOver * HID_MAX_KEYS
+                memset(keys_buffer, HID_ERROR_ROLL_OVER, HID_KEYBOARD_MAX_KEYS);
+                goto end;
+            }
+
+            keys_buffer[keys_pressed_count] = i;
+            ++keys_pressed_count;
+        }
+    }
+
+end:
+    LOGV("hid keyboard: key %-4s scancode=%02x (%u) mod=%02x",
+         event->type == SDL_KEYDOWN ? "down" : "up", event->keysym.scancode,
+         event->keysym.scancode, modifiers);
+
+    return true;
+}
+
+static void
+sc_key_processor_process_key(struct sc_key_processor *kp,
+                             const SDL_KeyboardEvent *event) {
+    if (event->repeat) {
+        // In USB HID protocol, key repeat is handled by the host (Android), so
+        // just ignore key repeat here.
+        return;
+    }
+
+    struct sc_hid_keyboard *kb = DOWNCAST(kp);
+
+    struct sc_hid_event hid_event;
+    // Not all keys are supported, just ignore unsupported keys
+    if (convert_hid_keyboard_event(kb, &hid_event, event)) {
+        if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) {
+            sc_hid_event_destroy(&hid_event);
+            LOGW("Could request HID event");
+        }
+    }
+}
+
+static void
+sc_key_processor_process_text(struct sc_key_processor *kp,
+                              const SDL_TextInputEvent *event) {
+    (void) kp;
+    (void) event;
+
+    // Never forward text input via HID (all the keys are injected separately)
+}
+
+bool
+sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) {
+    kb->aoa = aoa;
+
+    bool ok = sc_aoa_setup_hid(aoa, HID_KEYBOARD_ACCESSORY_ID,
+                               keyboard_report_desc,
+                               ARRAY_LEN(keyboard_report_desc));
+    if (!ok) {
+        LOGW("Register HID keyboard failed");
+        return false;
+    }
+
+    // Reset all states
+    memset(kb->keys, false, SC_HID_KEYBOARD_KEYS);
+
+    static const struct sc_key_processor_ops ops = {
+        .process_key = sc_key_processor_process_key,
+        .process_text = sc_key_processor_process_text,
+    };
+
+    kb->key_processor.ops = &ops;
+
+    return true;
+}
+
+void
+sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb) {
+    // Unregister HID keyboard so the soft keyboard shows again on Android
+    bool ok = sc_aoa_unregister_hid(kb->aoa, HID_KEYBOARD_ACCESSORY_ID);
+    if (!ok) {
+        LOGW("Could not unregister HID keyboard");
+    }
+}

app/src/hid_keyboard.h

@@ -0,0 +1,42 @@
+#ifndef SC_HID_KEYBOARD_H
+#define SC_HID_KEYBOARD_H
+
+#include "common.h"
+
+#include <stdbool.h>
+
+#include "aoa_hid.h"
+#include "trait/key_processor.h"
+
+// See "SDL2/SDL_scancode.h".
+// Maybe SDL_Keycode is used by most people, but SDL_Scancode is taken from USB
+// HID protocol.
+// 0x65 is Application, typically AT-101 Keyboard ends here.
+#define SC_HID_KEYBOARD_KEYS 0x66
+
+/**
+ * HID keyboard events are sequence-based, every time keyboard state changes
+ * it sends an array of currently pressed keys, the host is responsible for
+ * compare events and determine which key becomes pressed and which key becomes
+ * released. In order to convert SDL_KeyboardEvent to HID events, we first use
+ * an array of keys to save each keys' state. And when a SDL_KeyboardEvent was
+ * emitted, we updated our state, and then we use a loop to generate HID
+ * events. The sequence of array elements is unimportant and when too much keys
+ * pressed at the same time (more than report count), we should generate
+ * phantom state. Don't forget that modifiers should be updated too, even for
+ * phantom state.
+ */
+struct sc_hid_keyboard {
+    struct sc_key_processor key_processor; // key processor trait
+
+    struct sc_aoa *aoa;
+    bool keys[SC_HID_KEYBOARD_KEYS];
+};
+
+bool
+sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa);
+
+void
+sc_hid_keyboard_destroy(struct sc_hid_keyboard *kb);
+
+#endif

app/src/scrcpy.c

@@ -18,6 +18,9 @@
 #include "events.h"
 #include "file_handler.h"
 #include "input_manager.h"
+#ifdef HAVE_AOA_HID
+# include "hid_keyboard.h"
+#endif
 #include "keyboard_inject.h"
 #include "mouse_inject.h"
 #include "recorder.h"
@@ -41,7 +44,15 @@ struct scrcpy {
 #endif
     struct controller controller;
     struct file_handler file_handler;
-    struct sc_keyboard_inject keyboard_inject;
+#ifdef HAVE_AOA_HID
+    struct sc_aoa aoa;
+#endif
+    union {
+        struct sc_keyboard_inject keyboard_inject;
+#ifdef HAVE_AOA_HID
+        struct sc_hid_keyboard keyboard_hid;
+#endif
+    };
     struct sc_mouse_inject mouse_inject;
     struct input_manager input_manager;
 };
@@ -243,7 +254,7 @@ stream_on_eos(struct stream *stream, void *userdata) {
 }
 
 bool
-scrcpy(const struct scrcpy_options *options) {
+scrcpy(struct scrcpy_options *options) {
     static struct scrcpy scrcpy;
     struct scrcpy *s = &scrcpy;
 
@@ -260,6 +271,9 @@ scrcpy(const struct scrcpy_options *options) {
     bool v4l2_sink_initialized = false;
 #endif
     bool stream_started = false;
+#ifdef HAVE_AOA_HID
+    bool aoa_hid_initialized = false;
+#endif
     bool controller_initialized = false;
     bool controller_started = false;
     bool screen_initialized = false;
@@ -419,8 +433,50 @@ scrcpy(const struct scrcpy_options *options) {
     struct sc_mouse_processor *mp = NULL;
 
     if (options->control) {
-        sc_keyboard_inject_init(&s->keyboard_inject, &s->controller, options);
+        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) {
-        kp = &s->keyboard_inject.key_processor;
+#ifdef HAVE_AOA_HID
+            bool aoa_hid_ok = false;
+            if (!sc_aoa_init(&s->aoa, options->serial)) {
+                goto aoa_hid_end;
+            }
+
+            if (!sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
+                sc_aoa_destroy(&s->aoa);
+                goto aoa_hid_end;
+            }
+
+            if (!sc_aoa_start(&s->aoa)) {
+                sc_hid_keyboard_destroy(&s->keyboard_hid);
+                sc_aoa_destroy(&s->aoa);
+                goto aoa_hid_end;
+            }
+
+            aoa_hid_ok = true;
+            kp = &s->keyboard_hid.key_processor;
+
+            aoa_hid_initialized = true;
+
+aoa_hid_end:
+            if (!aoa_hid_ok) {
+                LOGE("Failed to enable HID over AOA, "
+                     "fallback to default keyboard injection method "
+                     "(-K/--hid-keyboard ignored)");
+                options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+            }
+#else
+            LOGE("HID over AOA is not supported on this platform, "
+                 "fallback to default keyboard injection method "
+                 "(-K/--hid-keyboard ignored)");
+            options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+#endif
+        }
+
+        // keyboard_input_mode may have been reset if HID mode failed
+        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
+            sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
+                                    options);
+            kp = &s->keyboard_inject.key_processor;
+        }
 
         sc_mouse_inject_init(&s->mouse_inject, &s->controller, &s->screen);
         mp = &s->mouse_inject.mouse_processor;
@@ -439,6 +495,12 @@ scrcpy(const struct scrcpy_options *options) {
 end:
     // The stream is not stopped explicitly, because it will stop by itself on
     // end-of-stream
+#ifdef HAVE_AOA_HID
+    if (aoa_hid_initialized) {
+        sc_hid_keyboard_destroy(&s->keyboard_hid);
+        sc_aoa_stop(&s->aoa);
+    }
+#endif
     if (controller_started) {
         controller_stop(&s->controller);
     }
@@ -466,6 +528,13 @@ end:
     }
 #endif
 
+#ifdef HAVE_AOA_HID
+    if (aoa_hid_initialized) {
+        sc_aoa_join(&s->aoa);
+        sc_aoa_destroy(&s->aoa);
+    }
+#endif
+
     // Destroy the screen only after the stream is guaranteed to be finished,
     // because otherwise the screen could receive new frames after destruction
     if (screen_initialized) {

app/src/scrcpy.h

@@ -33,6 +33,11 @@ enum sc_lock_video_orientation {
     SC_LOCK_VIDEO_ORIENTATION_3,
 };
 
+enum sc_keyboard_input_mode {
+    SC_KEYBOARD_INPUT_MODE_INJECT,
+    SC_KEYBOARD_INPUT_MODE_HID,
+};
+
 #define SC_MAX_SHORTCUT_MODS 8
 
 enum sc_shortcut_mod {
@@ -68,6 +73,7 @@ struct scrcpy_options {
     const char *v4l2_device;
     enum sc_log_level log_level;
     enum sc_record_format record_format;
+    enum sc_keyboard_input_mode keyboard_input_mode;
     struct sc_port_range port_range;
     struct sc_shortcut_mods shortcut_mods;
     uint16_t max_size;
@@ -112,6 +118,7 @@ struct scrcpy_options {
     .v4l2_device = NULL, \
     .log_level = SC_LOG_LEVEL_INFO, \
     .record_format = SC_RECORD_FORMAT_AUTO, \
+    .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT, \
     .port_range = { \
         .first = DEFAULT_LOCAL_PORT_RANGE_FIRST, \
         .last = DEFAULT_LOCAL_PORT_RANGE_LAST, \
@@ -151,6 +158,6 @@ struct scrcpy_options {
 }
 
 bool
-scrcpy(const struct scrcpy_options *options);
+scrcpy(struct scrcpy_options *options);
 
 #endif