From ec71a3f66ab48c2fdd1728753acc09edbd4db570 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Tue, 28 May 2019 21:03:54 +0200 Subject: [PATCH] Use two sockets for video and control The socket used the device-to-computer direction to stream the video and the computer-to-device direction to send control events. Some features, like copy-paste from device to computer, require to send non-video data from the device to the computer. To make them possible, use two sockets: - one for streaming the video from the device to the client; - one for control/events in both directions. --- app/src/controller.c | 6 +- app/src/controller.h | 4 +- app/src/scrcpy.c | 8 +- app/src/server.c | 30 ++++++-- app/src/server.h | 6 +- .../genymobile/scrcpy/DesktopConnection.java | 74 +++++++++++-------- .../java/com/genymobile/scrcpy/Server.java | 2 +- 7 files changed, 81 insertions(+), 49 deletions(-) diff --git a/app/src/controller.c b/app/src/controller.c index 30118218..5e74daba 100644 --- a/app/src/controller.c +++ b/app/src/controller.c @@ -7,7 +7,7 @@ #include "log.h" bool -controller_init(struct controller *controller, socket_t video_socket) { +controller_init(struct controller *controller, socket_t control_socket) { cbuf_init(&controller->queue); if (!(controller->mutex = SDL_CreateMutex())) { @@ -19,7 +19,7 @@ controller_init(struct controller *controller, socket_t video_socket) { return false; } - controller->video_socket = video_socket; + controller->control_socket = control_socket; controller->stopped = false; return true; @@ -57,7 +57,7 @@ process_event(struct controller *controller, if (!length) { return false; } - int w = net_send_all(controller->video_socket, serialized_event, length); + int w = net_send_all(controller->control_socket, serialized_event, length); return w == length; } diff --git a/app/src/controller.h b/app/src/controller.h index 7930bf8a..c006b791 100644 --- a/app/src/controller.h +++ b/app/src/controller.h @@ -12,7 +12,7 @@ struct control_event_queue CBUF(struct control_event, 64); struct controller { - socket_t video_socket; + socket_t control_socket; SDL_Thread *thread; SDL_mutex *mutex; SDL_cond *event_cond; @@ -21,7 +21,7 @@ struct controller { }; bool -controller_init(struct controller *controller, socket_t video_socket); +controller_init(struct controller *controller, socket_t control_socket); void controller_destroy(struct controller *controller); diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 539cd940..08dbacc3 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -300,15 +300,13 @@ scrcpy(const struct scrcpy_options *options) { goto end; } - socket_t device_socket = server.device_socket; - char device_name[DEVICE_NAME_FIELD_LENGTH]; struct size frame_size; // screenrecord does not send frames when the screen content does not // change therefore, we transmit the screen size before the video stream, // to be able to init the window immediately - if (!device_read_info(device_socket, device_name, &frame_size)) { + if (!device_read_info(server.video_socket, device_name, &frame_size)) { goto end; } @@ -344,7 +342,7 @@ scrcpy(const struct scrcpy_options *options) { av_log_set_callback(av_log_callback); - stream_init(&stream, device_socket, dec, rec); + stream_init(&stream, server.video_socket, dec, rec); // now we consumed the header values, the socket receives the video stream // start the stream @@ -355,7 +353,7 @@ scrcpy(const struct scrcpy_options *options) { if (display) { if (control) { - if (!controller_init(&controller, device_socket)) { + if (!controller_init(&controller, server.control_socket)) { goto end; } diff --git a/app/src/server.c b/app/src/server.c index 06405918..3f76e5c7 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -222,8 +222,14 @@ server_start(struct server *server, const char *serial, bool server_connect_to(struct server *server) { if (!server->tunnel_forward) { - server->device_socket = net_accept(server->server_socket); - if (server->device_socket == INVALID_SOCKET) { + server->video_socket = net_accept(server->server_socket); + if (server->video_socket == INVALID_SOCKET) { + return false; + } + + server->control_socket = net_accept(server->server_socket); + if (server->control_socket == INVALID_SOCKET) { + // the video_socket will be clean up on destroy return false; } @@ -232,9 +238,16 @@ server_connect_to(struct server *server) { } else { uint32_t attempts = 100; uint32_t delay = 100; // ms - server->device_socket = connect_to_server(server->local_port, attempts, - delay); - if (server->device_socket == INVALID_SOCKET) { + server->video_socket = + connect_to_server(server->local_port, attempts, delay); + if (server->video_socket == INVALID_SOCKET) { + return false; + } + + // we know that the device is listening, we don't need several attempts + server->control_socket = + net_connect(IPV4_LOCALHOST, server->local_port); + if (server->control_socket == INVALID_SOCKET) { return false; } } @@ -251,8 +264,11 @@ server_stop(struct server *server) { if (server->server_socket != INVALID_SOCKET) { close_socket(&server->server_socket); } - if (server->device_socket != INVALID_SOCKET) { - close_socket(&server->device_socket); + if (server->video_socket != INVALID_SOCKET) { + close_socket(&server->video_socket); + } + if (server->control_socket != INVALID_SOCKET) { + close_socket(&server->control_socket); } SDL_assert(server->process != PROCESS_NONE); diff --git a/app/src/server.h b/app/src/server.h index 4d16fdab..0c8443bb 100644 --- a/app/src/server.h +++ b/app/src/server.h @@ -11,7 +11,8 @@ struct server { char *serial; process_t process; socket_t server_socket; // only used if !tunnel_forward - socket_t device_socket; + socket_t video_socket; + socket_t control_socket; uint16_t local_port; bool tunnel_enabled; bool tunnel_forward; // use "adb forward" instead of "adb reverse" @@ -22,7 +23,8 @@ struct server { .serial = NULL, \ .process = PROCESS_NONE, \ .server_socket = INVALID_SOCKET, \ - .device_socket = INVALID_SOCKET, \ + .video_socket = INVALID_SOCKET, \ + .control_socket = INVALID_SOCKET, \ .local_port = 0, \ .tunnel_enabled = false, \ .tunnel_forward = false, \ diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java index d87a7fd8..3c09e725 100644 --- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java +++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java @@ -16,16 +16,20 @@ public final class DesktopConnection implements Closeable { private static final String SOCKET_NAME = "scrcpy"; - private final LocalSocket socket; - private final InputStream inputStream; - private final FileDescriptor fd; + private final LocalSocket videoSocket; + private final FileDescriptor videoFd; + + private final LocalSocket controlSocket; + private final InputStream controlInputStream; + private final ControlEventReader reader = new ControlEventReader(); - private DesktopConnection(LocalSocket socket) throws IOException { - this.socket = socket; - inputStream = socket.getInputStream(); - fd = socket.getFileDescriptor(); + private DesktopConnection(LocalSocket videoSocket, LocalSocket controlSocket) throws IOException { + this.videoSocket = videoSocket; + this.controlSocket = controlSocket; + controlInputStream = controlSocket.getInputStream(); + videoFd = videoSocket.getFileDescriptor(); } private static LocalSocket connect(String abstractName) throws IOException { @@ -34,35 +38,47 @@ public final class DesktopConnection implements Closeable { return localSocket; } - private static LocalSocket listenAndAccept(String abstractName) throws IOException { - LocalServerSocket localServerSocket = new LocalServerSocket(abstractName); - try { - return localServerSocket.accept(); - } finally { - localServerSocket.close(); - } - } - public static DesktopConnection open(Device device, boolean tunnelForward) throws IOException { - LocalSocket socket; + LocalSocket videoSocket; + LocalSocket controlSocket; if (tunnelForward) { - socket = listenAndAccept(SOCKET_NAME); - // send one byte so the client may read() to detect a connection error - socket.getOutputStream().write(0); + LocalServerSocket localServerSocket = new LocalServerSocket(SOCKET_NAME); + try { + videoSocket = localServerSocket.accept(); + // send one byte so the client may read() to detect a connection error + videoSocket.getOutputStream().write(0); + try { + controlSocket = localServerSocket.accept(); + } catch (IOException | RuntimeException e) { + videoSocket.close(); + throw e; + } + } finally { + localServerSocket.close(); + } } else { - socket = connect(SOCKET_NAME); + videoSocket = connect(SOCKET_NAME); + try { + controlSocket = connect(SOCKET_NAME); + } catch (IOException | RuntimeException e) { + videoSocket.close(); + throw e; + } } - DesktopConnection connection = new DesktopConnection(socket); + DesktopConnection connection = new DesktopConnection(videoSocket, controlSocket); Size videoSize = device.getScreenInfo().getVideoSize(); connection.send(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight()); return connection; } public void close() throws IOException { - socket.shutdownInput(); - socket.shutdownOutput(); - socket.close(); + videoSocket.shutdownInput(); + videoSocket.shutdownOutput(); + videoSocket.close(); + controlSocket.shutdownInput(); + controlSocket.shutdownOutput(); + controlSocket.close(); } @SuppressWarnings("checkstyle:MagicNumber") @@ -78,17 +94,17 @@ public final class DesktopConnection implements Closeable { buffer[DEVICE_NAME_FIELD_LENGTH + 1] = (byte) width; buffer[DEVICE_NAME_FIELD_LENGTH + 2] = (byte) (height >> 8); buffer[DEVICE_NAME_FIELD_LENGTH + 3] = (byte) height; - IO.writeFully(fd, buffer, 0, buffer.length); + IO.writeFully(videoFd, buffer, 0, buffer.length); } - public FileDescriptor getFd() { - return fd; + public FileDescriptor getVideoFd() { + return videoFd; } public ControlEvent receiveControlEvent() throws IOException { ControlEvent event = reader.next(); while (event == null) { - reader.readFrom(inputStream); + reader.readFrom(controlInputStream); event = reader.next(); } return event; diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java index b782101c..db192dd1 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Server.java +++ b/server/src/main/java/com/genymobile/scrcpy/Server.java @@ -24,7 +24,7 @@ public final class Server { try { // synchronous - screenEncoder.streamScreen(device, connection.getFd()); + screenEncoder.streamScreen(device, connection.getVideoFd()); } catch (IOException e) { // this is expected on close Ln.d("Screen streaming stopped");