diff --git a/app/src/aoa_hid.c b/app/src/aoa_hid.c index 4c0b2bda..6fd32610 100644 --- a/app/src/aoa_hid.c +++ b/app/src/aoa_hid.c @@ -35,7 +35,7 @@ sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id, hid_event->accessory_id = accessory_id; hid_event->buffer = buffer; hid_event->size = buffer_size; - hid_event->delay = 0; + hid_event->ack_to_wait = SC_SEQUENCE_INVALID; } void @@ -118,7 +118,10 @@ sc_aoa_open_usb_handle(libusb_device *device, libusb_device_handle **handle) { } bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial) { +sc_aoa_init(struct sc_aoa *aoa, const char *serial, + struct sc_acksync *acksync) { + assert(acksync); + cbuf_init(&aoa->queue); if (!sc_mutex_init(&aoa->mutex)) { @@ -155,6 +158,7 @@ sc_aoa_init(struct sc_aoa *aoa, const char *serial) { } aoa->stopped = false; + aoa->acksync = acksync; return true; } @@ -332,23 +336,28 @@ run_aoa_thread(void *data) { assert(non_empty); (void) non_empty; - assert(event.delay >= 0); - if (event.delay) { - // Wait during the specified delay before injecting the HID event - sc_tick deadline = sc_tick_now() + event.delay; - bool timed_out = false; - while (!aoa->stopped && !timed_out) { - timed_out = !sc_cond_timedwait(&aoa->event_cond, &aoa->mutex, - deadline); - } - if (aoa->stopped) { - sc_mutex_unlock(&aoa->mutex); + uint64_t ack_to_wait = event.ack_to_wait; + sc_mutex_unlock(&aoa->mutex); + + if (ack_to_wait != SC_SEQUENCE_INVALID) { + LOGD("Waiting ack from server sequence=%" PRIu64_, ack_to_wait); + // Do not block the loop indefinitely if the ack never comes (it should + // never happen) + sc_tick deadline = sc_tick_now() + SC_TICK_FROM_MS(500); + enum sc_acksync_wait_result result = + sc_acksync_wait(aoa->acksync, ack_to_wait, deadline); + + if (result == SC_ACKSYNC_WAIT_TIMEOUT) { + LOGW("Ack not received after 500ms, discarding HID event"); + sc_hid_event_destroy(&event); + continue; + } else if (result == SC_ACKSYNC_WAIT_INTR) { + // stopped + sc_hid_event_destroy(&event); break; } } - sc_mutex_unlock(&aoa->mutex); - bool ok = sc_aoa_send_hid_event(aoa, &event); sc_hid_event_destroy(&event); if (!ok) { @@ -377,6 +386,8 @@ sc_aoa_stop(struct sc_aoa *aoa) { aoa->stopped = true; sc_cond_signal(&aoa->event_cond); sc_mutex_unlock(&aoa->mutex); + + sc_acksync_interrupt(aoa->acksync); } void diff --git a/app/src/aoa_hid.h b/app/src/aoa_hid.h index 24cef502..e8fb9708 100644 --- a/app/src/aoa_hid.h +++ b/app/src/aoa_hid.h @@ -6,6 +6,7 @@ #include +#include "util/acksync.h" #include "util/cbuf.h" #include "util/thread.h" #include "util/tick.h" @@ -14,7 +15,7 @@ struct sc_hid_event { uint16_t accessory_id; unsigned char *buffer; uint16_t size; - sc_tick delay; + uint64_t ack_to_wait; }; // Takes ownership of buffer @@ -36,10 +37,12 @@ struct sc_aoa { sc_cond event_cond; bool stopped; struct sc_hid_event_queue queue; + + struct sc_acksync *acksync; }; bool -sc_aoa_init(struct sc_aoa *aoa, const char *serial); +sc_aoa_init(struct sc_aoa *aoa, const char *serial, struct sc_acksync *acksync); void sc_aoa_destroy(struct sc_aoa *aoa); diff --git a/app/src/controller.c b/app/src/controller.c index e486ea72..6cf3d20e 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -5,10 +5,11 @@ #include "util/log.h" bool -controller_init(struct controller *controller, sc_socket control_socket) { +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync) { cbuf_init(&controller->queue); - bool ok = receiver_init(&controller->receiver, control_socket); + bool ok = receiver_init(&controller->receiver, control_socket, acksync); if (!ok) { return false; } diff --git a/app/src/controller.h b/app/src/controller.h index e7004131..00267878 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -7,6 +7,7 @@ #include "control_msg.h" #include "receiver.h" +#include "util/acksync.h" #include "util/cbuf.h" #include "util/net.h" #include "util/thread.h" @@ -24,7 +25,8 @@ struct controller { }; bool -controller_init(struct controller *controller, sc_socket control_socket); +controller_init(struct controller *controller, sc_socket control_socket, + struct sc_acksync *acksync); void controller_destroy(struct controller *controller); diff --git a/app/src/hid_keyboard.c b/app/src/hid_keyboard.c index 2809ccd6..2afc09b2 100644 --- a/app/src/hid_keyboard.c +++ b/app/src/hid_keyboard.c @@ -279,7 +279,7 @@ push_mod_lock_state(struct sc_hid_keyboard *kb, uint16_t sdl_mod) { static void sc_key_processor_process_key(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set) { + uint64_t ack_to_wait) { if (event->repeat) { // In USB HID protocol, key repeat is handled by the host (Android), so // just ignore key repeat here. @@ -299,12 +299,12 @@ sc_key_processor_process_key(struct sc_key_processor *kp, } } - if (device_clipboard_set) { + if (ack_to_wait) { // Ctrl+v is pressed, so clipboard synchronization has been - // requested. Wait a bit so that the clipboard is set before - // injecting Ctrl+v via HID, otherwise it would paste the old - // clipboard content. - hid_event.delay = SC_TICK_FROM_MS(5); + // requested. Wait until clipboard synchronization is acknowledged + // by the server, otherwise it could paste the old clipboard + // content. + hid_event.ack_to_wait = ack_to_wait; } if (!sc_aoa_push_hid_event(kb->aoa, &hid_event)) { @@ -345,6 +345,10 @@ sc_hid_keyboard_init(struct sc_hid_keyboard *kb, struct sc_aoa *aoa) { .process_text = sc_key_processor_process_text, }; + // Clipboard synchronization is requested over the control socket, while HID + // events are sent over AOA, so it must wait for clipboard synchronization + // to be acknowledged by the device before injecting Ctrl+v. + kb->key_processor.async_paste = true; kb->key_processor.ops = &ops; return true; diff --git a/app/src/input_manager.c b/app/src/input_manager.c index e7afee77..f1715b79 100644 --- a/app/src/input_manager.c +++ b/app/src/input_manager.c @@ -82,6 +82,8 @@ input_manager_init(struct input_manager *im, struct controller *controller, im->last_keycode = SDLK_UNKNOWN; im->last_mod = 0; im->key_repeat = 0; + + im->next_sequence = 1; // 0 is reserved for SC_SEQUENCE_INVALID } static void @@ -209,7 +211,8 @@ collapse_panels(struct controller *controller) { } static bool -set_device_clipboard(struct controller *controller, bool paste) { +set_device_clipboard(struct controller *controller, bool paste, + uint64_t sequence) { char *text = SDL_GetClipboardText(); if (!text) { LOGW("Could not get clipboard text: %s", SDL_GetError()); @@ -225,7 +228,7 @@ set_device_clipboard(struct controller *controller, bool paste) { struct control_msg msg; msg.type = CONTROL_MSG_TYPE_SET_CLIPBOARD; - msg.set_clipboard.sequence = 0; // unused for now + msg.set_clipboard.sequence = sequence; msg.set_clipboard.text = text_dup; msg.set_clipboard.paste = paste; @@ -461,8 +464,10 @@ input_manager_process_key(struct input_manager *im, // inject the text as input events clipboard_paste(controller); } else { - // store the text in the device clipboard and paste - set_device_clipboard(controller, true); + // store the text in the device clipboard and paste, + // without requesting an acknowledgment + set_device_clipboard(controller, true, + SC_SEQUENCE_INVALID); } } return; @@ -511,6 +516,7 @@ input_manager_process_key(struct input_manager *im, return; } + uint64_t ack_to_wait = SC_SEQUENCE_INVALID; bool is_ctrl_v = ctrl && !shift && keycode == SDLK_v && down && !repeat; if (is_ctrl_v) { if (im->legacy_paste) { @@ -518,16 +524,28 @@ input_manager_process_key(struct input_manager *im, clipboard_paste(controller); return; } + + // Request an acknowledgement only if necessary + uint64_t sequence = im->kp->async_paste ? im->next_sequence + : SC_SEQUENCE_INVALID; + // Synchronize the computer clipboard to the device clipboard before // sending Ctrl+v, to allow seamless copy-paste. - bool ok = set_device_clipboard(controller, false); + bool ok = set_device_clipboard(controller, false, sequence); if (!ok) { LOGW("Clipboard could not be synchronized, Ctrl+v not injected"); return; } + + if (im->kp->async_paste) { + // The key processor must wait for this ack before injecting Ctrl+v + ack_to_wait = sequence; + // Increment only when the request succeeded + ++im->next_sequence; + } } - im->kp->ops->process_key(im->kp, event, is_ctrl_v); + im->kp->ops->process_key(im->kp, event, ack_to_wait); } static void diff --git a/app/src/input_manager.h b/app/src/input_manager.h index f018f98a..22d77381 100644 --- a/app/src/input_manager.h +++ b/app/src/input_manager.h @@ -38,6 +38,8 @@ struct input_manager { unsigned key_repeat; SDL_Keycode last_keycode; uint16_t last_mod; + + uint64_t next_sequence; // used for request acknowledgements }; void diff --git a/app/src/keyboard_inject.c b/app/src/keyboard_inject.c index e59b5520..4112fd4c 100644 --- a/app/src/keyboard_inject.c +++ b/app/src/keyboard_inject.c @@ -189,11 +189,11 @@ convert_input_key(const SDL_KeyboardEvent *from, struct control_msg *to, static void sc_key_processor_process_key(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set) { + uint64_t ack_to_wait) { // The device clipboard synchronization and the key event messages are // serialized, there is nothing special to do to ensure that the clipboard // is set before injecting Ctrl+v. - (void) device_clipboard_set; + (void) ack_to_wait; struct sc_keyboard_inject *ki = DOWNCAST(kp); @@ -256,5 +256,7 @@ sc_keyboard_inject_init(struct sc_keyboard_inject *ki, .process_text = sc_key_processor_process_text, }; + // Key injection and clipboard synchronization are serialized + ki->key_processor.async_paste = false; ki->key_processor.ops = &ops; } diff --git a/app/src/receiver.c b/app/src/receiver.c index 52e6034d..eeb206f1 100644 --- a/app/src/receiver.c +++ b/app/src/receiver.c @@ -7,12 +7,16 @@ #include "util/log.h" bool -receiver_init(struct receiver *receiver, sc_socket control_socket) { +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync) { bool ok = sc_mutex_init(&receiver->mutex); if (!ok) { return false; } + receiver->control_socket = control_socket; + receiver->acksync = acksync; + return true; } @@ -22,7 +26,7 @@ receiver_destroy(struct receiver *receiver) { } static void -process_msg(struct device_msg *msg) { +process_msg(struct receiver *receiver, struct device_msg *msg) { switch (msg->type) { case DEVICE_MSG_TYPE_CLIPBOARD: { char *current = SDL_GetClipboardText(); @@ -38,13 +42,16 @@ process_msg(struct device_msg *msg) { break; } case DEVICE_MSG_TYPE_ACK_CLIPBOARD: - // TODO + assert(receiver->acksync); + LOGD("Ack device clipboard sequence=%" PRIu64_, + msg->ack_clipboard.sequence); + sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence); break; } } static ssize_t -process_msgs(const unsigned char *buf, size_t len) { +process_msgs(struct receiver *receiver, const unsigned char *buf, size_t len) { size_t head = 0; for (;;) { struct device_msg msg; @@ -56,7 +63,7 @@ process_msgs(const unsigned char *buf, size_t len) { return head; } - process_msg(&msg); + process_msg(receiver, &msg); device_msg_destroy(&msg); head += r; @@ -84,7 +91,7 @@ run_receiver(void *data) { } head += r; - ssize_t consumed = process_msgs(buf, head); + ssize_t consumed = process_msgs(receiver, buf, head); if (consumed == -1) { // an error occurred break; diff --git a/app/src/receiver.h b/app/src/receiver.h index 99f128a4..3c4e8c64 100644 --- a/app/src/receiver.h +++ b/app/src/receiver.h @@ -5,6 +5,7 @@ #include +#include "util/acksync.h" #include "util/net.h" #include "util/thread.h" @@ -14,10 +15,13 @@ struct receiver { sc_socket control_socket; sc_thread thread; sc_mutex mutex; + + struct sc_acksync *acksync; }; bool -receiver_init(struct receiver *receiver, sc_socket control_socket); +receiver_init(struct receiver *receiver, sc_socket control_socket, + struct sc_acksync *acksync); void receiver_destroy(struct receiver *receiver); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 61087681..04239bf5 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,6 +27,7 @@ #include "screen.h" #include "server.h" #include "stream.h" +#include "util/acksync.h" #include "util/log.h" #include "util/net.h" #ifdef HAVE_V4L2 @@ -46,6 +47,8 @@ struct scrcpy { struct file_handler file_handler; #ifdef HAVE_AOA_HID struct sc_aoa aoa; + // sequence/ack helper to synchronize clipboard and Ctrl+v via HID + struct sc_acksync acksync; #endif union { struct sc_keyboard_inject keyboard_inject; @@ -340,6 +343,8 @@ scrcpy(struct scrcpy_options *options) { bool controller_started = false; bool screen_initialized = false; + struct sc_acksync *acksync = NULL; + struct sc_server_params params = { .serial = options->serial, .log_level = options->log_level, @@ -445,7 +450,18 @@ scrcpy(struct scrcpy_options *options) { } if (options->control) { - if (!controller_init(&s->controller, s->server.control_socket)) { +#ifdef HAVE_AOA_HID + if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID) { + bool ok = sc_acksync_init(&s->acksync); + if (!ok) { + goto end; + } + + acksync = &s->acksync; + } +#endif + if (!controller_init(&s->controller, s->server.control_socket, + acksync)) { goto end; } controller_initialized = true; @@ -521,7 +537,7 @@ scrcpy(struct scrcpy_options *options) { #ifdef HAVE_AOA_HID bool aoa_hid_ok = false; - bool ok = sc_aoa_init(&s->aoa, serial); + bool ok = sc_aoa_init(&s->aoa, serial, acksync); if (!ok) { goto aoa_hid_end; } @@ -586,6 +602,9 @@ end: sc_hid_keyboard_destroy(&s->keyboard_hid); sc_aoa_stop(&s->aoa); } + if (acksync) { + sc_acksync_destroy(acksync); + } #endif if (controller_started) { controller_stop(&s->controller); diff --git a/app/src/trait/key_processor.h b/app/src/trait/key_processor.h index 1f8132f2..f4afe27b 100644 --- a/app/src/trait/key_processor.h +++ b/app/src/trait/key_processor.h @@ -14,6 +14,15 @@ * Component able to process and inject keys should implement this trait. */ struct sc_key_processor { + /** + * Set by the implementation to indicate that it must explicitly wait for + * the clipboard to be set on the device before injecting Ctrl+v to avoid + * race conditions. If it is set, the input_manager will pass a valid + * ack_to_wait to process_key() in case of clipboard synchronization + * resulting of the key event. + */ + bool async_paste; + const struct sc_key_processor_ops *ops; }; @@ -22,13 +31,14 @@ struct sc_key_processor_ops { /** * Process the keyboard event * - * The flag `device_clipboard_set` indicates that the input manager sent a - * control message to synchronize the device clipboard as a result of this - * key event. + * The `sequence` number (if different from `SC_SEQUENCE_INVALID`) indicates + * the acknowledgement number to wait for before injecting this event. + * This allows to ensure that the device clipboard is set before injecting + * Ctrl+v on the device. */ void (*process_key)(struct sc_key_processor *kp, const SDL_KeyboardEvent *event, - bool device_clipboard_set); + uint64_t ack_to_wait); void (*process_text)(struct sc_key_processor *kp, diff --git a/app/src/util/acksync.h b/app/src/util/acksync.h index 1fd34444..58ab1b35 100644 --- a/app/src/util/acksync.h +++ b/app/src/util/acksync.h @@ -9,6 +9,11 @@ /** * Helper to wait for acknowledgments + * + * In practice, it is used to wait for device clipboard acknowledgement from the + * server before injecting Ctrl+v via AOA HID, in order to avoid pasting the + * content of the old device clipboard (if Ctrl+v was injected before the + * clipboard content was actually set). */ struct sc_acksync { sc_mutex mutex;