diff --git a/app/src/common.h b/app/src/common.h new file mode 100644 index 00000000..02d7a977 --- /dev/null +++ b/app/src/common.h @@ -0,0 +1,26 @@ +#ifndef COMMON_H +#define COMMON_H + +#include + +#define MIN(X,Y) (X) < (Y) ? (X) : (Y) +#define MAX(X,Y) (X) > (Y) ? (X) : (Y) + +struct size { + Uint16 width; + Uint16 height; +}; + +struct position { + Uint16 x; + Uint16 y; +}; + +struct point { + // The video screen size may be different from the real device screen size, + // so store to which size the absolute position apply, to scale it accordingly. + struct size screen_size; + struct position position; +}; + +#endif diff --git a/app/src/controlevent.c b/app/src/controlevent.c index 9436139a..46062dcf 100644 --- a/app/src/controlevent.c +++ b/app/src/controlevent.c @@ -17,6 +17,13 @@ static inline void write32(Uint8 *buf, Uint32 value) { buf[3] = value; } +static void write_point(Uint8 *buf, const struct point *point) { + write16(&buf[0], point->position.x); + write16(&buf[2], point->position.y); + write16(&buf[4], point->screen_size.width); + write16(&buf[6], point->screen_size.height); +} + int control_event_serialize(const struct control_event *event, unsigned char *buf) { buf[0] = event->type; switch (event->type) { @@ -38,12 +45,10 @@ int control_event_serialize(const struct control_event *event, unsigned char *bu case CONTROL_EVENT_TYPE_MOUSE: buf[1] = event->mouse_event.action; write32(&buf[2], event->mouse_event.buttons); - write32(&buf[6], (Uint32) event->mouse_event.x); - write32(&buf[10], (Uint32) event->mouse_event.y); + write_point(&buf[6], &event->mouse_event.point); return 14; case CONTROL_EVENT_TYPE_SCROLL: - write32(&buf[1], (Uint32) event->scroll_event.x); - write32(&buf[5], (Uint32) event->scroll_event.y); + write_point(&buf[1], &event->scroll_event.point); write32(&buf[9], (Uint32) event->scroll_event.hscroll); write32(&buf[13], (Uint32) event->scroll_event.vscroll); return 17; diff --git a/app/src/controlevent.h b/app/src/controlevent.h index d0534ed2..1f358cfc 100644 --- a/app/src/controlevent.h +++ b/app/src/controlevent.h @@ -6,6 +6,7 @@ #include "android/input.h" #include "android/keycodes.h" +#include "common.h" #define CONTROL_EVENT_QUEUE_SIZE 64 #define SERIALIZED_EVENT_MAX_SIZE 33 @@ -32,12 +33,10 @@ struct control_event { struct { enum android_motionevent_action action; enum android_motionevent_buttons buttons; - Sint32 x; - Sint32 y; + struct point point; } mouse_event; struct { - Sint32 x; - Sint32 y; + struct point point; Sint32 hscroll; Sint32 vscroll; } scroll_event; diff --git a/app/src/convert.c b/app/src/convert.c index c063a078..0cae970d 100644 --- a/app/src/convert.c +++ b/app/src/convert.c @@ -117,7 +117,8 @@ static enum android_motionevent_buttons convert_mouse_buttons(Uint32 state) { return buttons; } -SDL_bool input_key_from_sdl_to_android(const SDL_KeyboardEvent *from, struct control_event *to) { +SDL_bool input_key_from_sdl_to_android(const SDL_KeyboardEvent *from, + struct control_event *to) { to->type = CONTROL_EVENT_TYPE_KEYCODE; if (!convert_keycode_action(from->type, &to->keycode_event.action)) { @@ -133,7 +134,9 @@ SDL_bool input_key_from_sdl_to_android(const SDL_KeyboardEvent *from, struct con return SDL_TRUE; } -SDL_bool mouse_button_from_sdl_to_android(const SDL_MouseButtonEvent *from, struct control_event *to) { +SDL_bool mouse_button_from_sdl_to_android(const SDL_MouseButtonEvent *from, + struct size screen_size, + struct control_event *to) { to->type = CONTROL_EVENT_TYPE_MOUSE; if (!convert_mouse_action(from->type, &to->mouse_event.action)) { @@ -141,32 +144,36 @@ SDL_bool mouse_button_from_sdl_to_android(const SDL_MouseButtonEvent *from, stru } to->mouse_event.buttons = convert_mouse_buttons(SDL_BUTTON(from->button)); - to->mouse_event.x = from->x; - to->mouse_event.y = from->y; + to->mouse_event.point.screen_size = screen_size; + to->mouse_event.point.position.x = (Uint16) from->x; + to->mouse_event.point.position.y = (Uint16) from->y; return SDL_TRUE; } -SDL_bool mouse_motion_from_sdl_to_android(const SDL_MouseMotionEvent *from, struct control_event *to) { +SDL_bool mouse_motion_from_sdl_to_android(const SDL_MouseMotionEvent *from, + const struct size screen_size, + struct control_event *to) { to->type = CONTROL_EVENT_TYPE_MOUSE; to->mouse_event.action = AMOTION_EVENT_ACTION_MOVE; to->mouse_event.buttons = convert_mouse_buttons(from->state); - to->mouse_event.x = from->x; - to->mouse_event.y = from->y; + to->mouse_event.point.screen_size = screen_size; + to->mouse_event.point.position.x = from->x; + to->mouse_event.point.position.y = from->y; return SDL_TRUE; } -SDL_bool mouse_wheel_from_sdl_to_android(const struct complete_mouse_wheel_event *from, struct control_event *to) { +SDL_bool mouse_wheel_from_sdl_to_android(const SDL_MouseWheelEvent *from, + const struct point point, + struct control_event *to) { to->type = CONTROL_EVENT_TYPE_SCROLL; - to->scroll_event.x = from->x; - to->scroll_event.y = from->y; + to->scroll_event.point = point; - SDL_MouseWheelEvent *wheel = from->mouse_wheel_event; - int mul = wheel->direction == SDL_MOUSEWHEEL_NORMAL ? 1 : -1; - to->scroll_event.hscroll = mul * wheel->x; - to->scroll_event.vscroll = mul * wheel->y; + int mul = from->direction == SDL_MOUSEWHEEL_NORMAL ? 1 : -1; + to->scroll_event.hscroll = mul * from->x; + to->scroll_event.vscroll = mul * from->y; return SDL_TRUE; } diff --git a/app/src/convert.h b/app/src/convert.h index 12a89ad3..a2922a93 100644 --- a/app/src/convert.h +++ b/app/src/convert.h @@ -5,16 +5,31 @@ #include #include "controlevent.h" -// on Android, a scroll event requires the current mouse position -struct complete_mouse_wheel_event { - SDL_MouseWheelEvent *mouse_wheel_event; - Sint32 x; - Sint32 y; +struct complete_mouse_motion_event { + SDL_MouseMotionEvent *mouse_motion_event; + struct size screen_size; }; -SDL_bool input_key_from_sdl_to_android(const SDL_KeyboardEvent *from, struct control_event *to); -SDL_bool mouse_button_from_sdl_to_android(const SDL_MouseButtonEvent *from, struct control_event *to); -SDL_bool mouse_motion_from_sdl_to_android(const SDL_MouseMotionEvent *from, struct control_event *to); -SDL_bool mouse_wheel_from_sdl_to_android(const struct complete_mouse_wheel_event *from, struct control_event *to); +struct complete_mouse_wheel_event { + SDL_MouseWheelEvent *mouse_wheel_event; + struct point position; +}; + +SDL_bool input_key_from_sdl_to_android(const SDL_KeyboardEvent *from, + struct control_event *to); +SDL_bool mouse_button_from_sdl_to_android(const SDL_MouseButtonEvent *from, + struct size screen_size, + struct control_event *to); + +// the video size may be different from the real device size, so we need the size +// to which the absolute position apply, to scale it accordingly +SDL_bool mouse_motion_from_sdl_to_android(const SDL_MouseMotionEvent *from, + struct size screen_size, + struct control_event *to); + +// on Android, a scroll event requires the current mouse position +SDL_bool mouse_wheel_from_sdl_to_android(const SDL_MouseWheelEvent *from, + struct point point, + struct control_event *to); #endif diff --git a/app/src/screen.c b/app/src/screen.c index 9edfae34..8c0116d0 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -9,6 +9,7 @@ #include #include "command.h" +#include "common.h" #include "control.h" #include "convert.h" #include "decoder.h" @@ -20,13 +21,6 @@ #define DEVICE_NAME_FIELD_LENGTH 64 #define DISPLAY_MARGINS 96 -#define MIN(X,Y) (X) < (Y) ? (X) : (Y) -#define MAX(X,Y) (X) > (Y) ? (X) : (Y) - -struct size { - Uint16 width; - Uint16 height; -}; static struct frames frames; static struct decoder decoder; @@ -110,6 +104,17 @@ static inline struct size get_window_size(SDL_Window *window) { return size; } +static inline struct position get_mouse_position() { + int x; + int y; + SDL_GetMouseState(&x, &y); + SDL_assert_release(x >= 0 && x < 0x10000 && y >= 0 && y < 0x10000); + return (struct position) { + .x = (Uint16) x, + .y = (Uint16) y, + }; +} + // return the optimal size of the window, with the following constraints: // - it attempts to keep at least one dimension of the current_size (i.e. it crops the black borders) // - it keeps the aspect ratio @@ -140,7 +145,7 @@ static struct size get_optimal_size(struct size current_size, struct size frame_ } // w and h must fit into 16 bits - SDL_assert_release(!(w & ~0xffff) && !(h & ~0xffff)); + SDL_assert_release(w < 0x10000 && h < 0x10000); return (struct size) {w, h}; } @@ -287,27 +292,27 @@ static void handle_key(const SDL_KeyboardEvent *event) { } } -static void handle_mouse_motion(const SDL_MouseMotionEvent *event) { +static void handle_mouse_motion(const SDL_MouseMotionEvent *event, struct size screen_size) { struct control_event control_event; - if (mouse_motion_from_sdl_to_android(event, &control_event)) { + if (mouse_motion_from_sdl_to_android(event, screen_size, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send mouse motion event"); } } } -static void handle_mouse_button(const SDL_MouseButtonEvent *event) { +static void handle_mouse_button(const SDL_MouseButtonEvent *event, struct size screen_size) { struct control_event control_event; - if (mouse_button_from_sdl_to_android(event, &control_event)) { + if (mouse_button_from_sdl_to_android(event, screen_size, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send mouse button event"); } } } -static void handle_mouse_wheel(const struct complete_mouse_wheel_event *event) { +static void handle_mouse_wheel(const SDL_MouseWheelEvent *event, struct point point) { struct control_event control_event; - if (mouse_wheel_from_sdl_to_android(event, &control_event)) { + if (mouse_wheel_from_sdl_to_android(event, point, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send wheel button event"); } @@ -346,24 +351,22 @@ void event_loop(void) { handle_key(&event.key); break; case SDL_MOUSEMOTION: - handle_mouse_motion(&event.motion); + handle_mouse_motion(&event.motion, frame_size); break; case SDL_MOUSEWHEEL: { - struct complete_mouse_wheel_event complete_event; - complete_event.mouse_wheel_event = &event.wheel; - int x; - int y; - SDL_GetMouseState(&x, &y); - complete_event.x = (Sint32) x; - complete_event.y = (Sint32) y; - handle_mouse_wheel(&complete_event); + struct point point = { + .screen_size = frame_size, + .position = get_mouse_position(), + }; + handle_mouse_wheel(&event.wheel, point); break; } case SDL_MOUSEBUTTONDOWN: - case SDL_MOUSEBUTTONUP: - handle_mouse_button(&event.button); + case SDL_MOUSEBUTTONUP: { + handle_mouse_button(&event.button, frame_size); break; } + } } } diff --git a/app/tests/test_control_event_serialize.c b/app/tests/test_control_event_serialize.c index fd309156..c0fd9f48 100644 --- a/app/tests/test_control_event_serialize.c +++ b/app/tests/test_control_event_serialize.c @@ -51,8 +51,16 @@ static void test_serialize_mouse_event() { .mouse_event = { .action = AMOTION_EVENT_ACTION_DOWN, .buttons = AMOTION_EVENT_BUTTON_PRIMARY, - .x = 260, - .y = 1026, + .point = { + .position = { + .x = 260, + .y = 1026, + }, + .screen_size = { + .width = 1080, + .height = 1920, + }, + }, }, }; @@ -64,8 +72,8 @@ static void test_serialize_mouse_event() { 0x02, // CONTROL_EVENT_TYPE_MOUSE 0x00, // AKEY_EVENT_ACTION_DOWN 0x00, 0x00, 0x00, 0x01, // AMOTION_EVENT_BUTTON_PRIMARY - 0x00, 0x00, 0x01, 0x04, // 260 - 0x00, 0x00, 0x04, 0x02, // 1026 + 0x01, 0x04, 0x04, 0x02, // 260 1026 + 0x04, 0x38, 0x07, 0x80, // 1080 1920 }; assert(!memcmp(buf, expected, sizeof(expected))); } @@ -74,8 +82,16 @@ static void test_serialize_scroll_event() { struct control_event event = { .type = CONTROL_EVENT_TYPE_SCROLL, .scroll_event = { - .x = 260, - .y = 1026, + .point = { + .position = { + .x = 260, + .y = 1026, + }, + .screen_size = { + .width = 1080, + .height = 1920, + }, + }, .hscroll = 1, .vscroll = -1, }, @@ -87,8 +103,8 @@ static void test_serialize_scroll_event() { const unsigned char expected[] = { 0x03, // CONTROL_EVENT_TYPE_SCROLL - 0x00, 0x00, 0x01, 0x04, // 260 - 0x00, 0x00, 0x04, 0x02, // 1026 + 0x01, 0x04, 0x04, 0x02, // 260 1026 + 0x04, 0x38, 0x07, 0x80, // 1080 1920 0x00, 0x00, 0x00, 0x01, // 1 0xFF, 0xFF, 0xFF, 0xFF, // -1 }; diff --git a/server/src/com/genymobile/scrcpy/ControlEvent.java b/server/src/com/genymobile/scrcpy/ControlEvent.java index 567a70f3..76074662 100644 --- a/server/src/com/genymobile/scrcpy/ControlEvent.java +++ b/server/src/com/genymobile/scrcpy/ControlEvent.java @@ -16,8 +16,7 @@ public class ControlEvent { private int action; // KeyEvent.ACTION_* or MotionEvent.ACTION_* private int keycode; // KeyEvent.KEYCODE_* private int buttons; // MotionEvent.BUTTON_* - private int x; - private int y; + private Point point; private int hScroll; private int vScroll; @@ -40,21 +39,19 @@ public class ControlEvent { return event; } - public static ControlEvent createMotionControlEvent(int action, int buttons, int x, int y) { + public static ControlEvent createMotionControlEvent(int action, int buttons, Point point) { ControlEvent event = new ControlEvent(); event.type = TYPE_MOUSE; event.action = action; event.buttons = buttons; - event.x = x; - event.y = y; + event.point = point; return event; } - public static ControlEvent createScrollControlEvent(int x, int y, int hScroll, int vScroll) { + public static ControlEvent createScrollControlEvent(Point point, int hScroll, int vScroll) { ControlEvent event = new ControlEvent(); event.type = TYPE_SCROLL; - event.x = x; - event.y = y; + event.point = point; event.hScroll = hScroll; event.vScroll = vScroll; return event; @@ -84,12 +81,8 @@ public class ControlEvent { return buttons; } - public int getX() { - return x; - } - - public int getY() { - return y; + public Point getPoint() { + return point; } public int getHScroll() { diff --git a/server/src/com/genymobile/scrcpy/ControlEventReader.java b/server/src/com/genymobile/scrcpy/ControlEventReader.java index 722226e8..a37063cb 100644 --- a/server/src/com/genymobile/scrcpy/ControlEventReader.java +++ b/server/src/com/genymobile/scrcpy/ControlEventReader.java @@ -51,7 +51,7 @@ public class ControlEventReader { if (buffer.remaining() < KEYCODE_PAYLOAD_LENGTH) { break; } - int action = buffer.get() & 0xff; // unsigned + int action = toUnsigned(buffer.get()); int keycode = buffer.getInt(); int metaState = buffer.getInt(); return ControlEvent.createKeycodeControlEvent(action, keycode, metaState); @@ -60,7 +60,7 @@ public class ControlEventReader { if (buffer.remaining() < 1) { break; } - int len = buffer.get() & 0xff; // unsigned + int len = toUnsigned(buffer.get()); if (buffer.remaining() < len) { break; } @@ -72,21 +72,19 @@ public class ControlEventReader { if (buffer.remaining() < MOUSE_PAYLOAD_LENGTH) { break; } - int action = buffer.get() & 0xff; // unsigned + int action = toUnsigned(buffer.get()); int buttons = buffer.getInt(); - int x = buffer.getInt(); - int y = buffer.getInt(); - return ControlEvent.createMotionControlEvent(action, buttons, x, y); + Point point = readPoint(buffer); + return ControlEvent.createMotionControlEvent(action, buttons, point); } case ControlEvent.TYPE_SCROLL: { if (buffer.remaining() < SCROLL_PAYLOAD_LENGTH) { break; } - int x = buffer.getInt(); - int y = buffer.getInt(); + Point point = readPoint(buffer); int hscroll = buffer.getInt(); int vscroll = buffer.getInt(); - return ControlEvent.createScrollControlEvent(x, y, hscroll, vscroll); + return ControlEvent.createScrollControlEvent(point, hscroll, vscroll); } default: Ln.w("Unknown event type: " + type); @@ -96,4 +94,20 @@ public class ControlEventReader { buffer.position(savedPosition); return null; } + + private static Point readPoint(ByteBuffer buffer) { + int x = toUnsigned(buffer.getShort()); + int y = toUnsigned(buffer.getShort()); + int screenWidth = toUnsigned(buffer.getShort()); + int screenHeight = toUnsigned(buffer.getShort()); + return new Point(x, y, screenWidth, screenHeight); + } + + private static int toUnsigned(short value) { + return value & 0xffff; + } + + private static int toUnsigned(byte value) { + return value & 0xff; + } } diff --git a/server/src/com/genymobile/scrcpy/Device.java b/server/src/com/genymobile/scrcpy/Device.java index 88d1f30d..f157f8c1 100644 --- a/server/src/com/genymobile/scrcpy/Device.java +++ b/server/src/com/genymobile/scrcpy/Device.java @@ -48,6 +48,15 @@ public class Device { return screenInfo; } + public RawPoint getPhysicalPoint(Point point) { + ScreenInfo screenInfo = getScreenInfo(); + int deviceWidth = screenInfo.getLogicalWidth(); + int deviceHeight = screenInfo.getLogicalHeight(); + int scaledX = point.getX() * deviceWidth / point.getScreenWidth(); + int scaledY = point.getY() * deviceHeight / point.getScreenHeight(); + return new RawPoint(scaledX, scaledY); + } + private ScreenInfo readScreenInfo() { return serviceManager.getDisplayManager().getScreenInfo(); } diff --git a/server/src/com/genymobile/scrcpy/EventController.java b/server/src/com/genymobile/scrcpy/EventController.java index 3e4bbe76..01513ad3 100644 --- a/server/src/com/genymobile/scrcpy/EventController.java +++ b/server/src/com/genymobile/scrcpy/EventController.java @@ -43,10 +43,10 @@ public class EventController { coords.touchMinor = 1; } - private void setPointerCoords(int x, int y) { + private void setPointerCoords(RawPoint rawPoint) { MotionEvent.PointerCoords coords = pointerCoords[0]; - coords.x = x; - coords.y = y; + coords.x = rawPoint.getX(); + coords.y = rawPoint.getY(); } private void setScroll(int hScroll, int vScroll) { @@ -72,10 +72,10 @@ public class EventController { injectText(controlEvent.getText()); break; case ControlEvent.TYPE_MOUSE: - injectMouse(controlEvent.getAction(), controlEvent.getButtons(), controlEvent.getX(), controlEvent.getY()); + injectMouse(controlEvent.getAction(), controlEvent.getButtons(), controlEvent.getPoint()); break; case ControlEvent.TYPE_SCROLL: - injectScroll(controlEvent.getButtons(), controlEvent.getX(), controlEvent.getY(), controlEvent.getHScroll(), controlEvent.getVScroll()); + injectScroll(controlEvent.getPoint(), controlEvent.getHScroll(), controlEvent.getVScroll()); } return true; } @@ -97,19 +97,29 @@ public class EventController { return true; } - private boolean injectMouse(int action, int buttons, int x, int y) { + private boolean injectMouse(int action, int buttons, Point point) { long now = SystemClock.uptimeMillis(); if (action == MotionEvent.ACTION_DOWN) { lastMouseDown = now; } - setPointerCoords(x, y); + RawPoint rawPoint = Device.getInstance().getPhysicalPoint(point); + if (rawPoint == null) { + // ignore event + return false; + } + setPointerCoords(rawPoint); MotionEvent event = MotionEvent.obtain(lastMouseDown, now, action, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, 0, 0, InputDevice.SOURCE_MOUSE, 0); return injectEvent(event); } - private boolean injectScroll(int buttons, int x, int y, int hScroll, int vScroll) { + private boolean injectScroll(Point point, int hScroll, int vScroll) { long now = SystemClock.uptimeMillis(); - setPointerCoords(x, y); + RawPoint rawPoint = Device.getInstance().getPhysicalPoint(point); + if (rawPoint == null) { + // ignore event + return false; + } + setPointerCoords(rawPoint); setScroll(hScroll, vScroll); MotionEvent event = MotionEvent.obtain(lastMouseDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, 0, 0, InputDevice.SOURCE_MOUSE, 0); return injectEvent(event); diff --git a/server/src/com/genymobile/scrcpy/Point.java b/server/src/com/genymobile/scrcpy/Point.java new file mode 100644 index 00000000..6e537108 --- /dev/null +++ b/server/src/com/genymobile/scrcpy/Point.java @@ -0,0 +1,33 @@ +package com.genymobile.scrcpy; + +public class Point { + + private int x; + private int y; + private int screenWidth; + private int screenHeight; + + public Point(int x, int y, int screenWidth, int screenHeight) { + this.x = x; + this.y = y; + this.screenWidth = screenWidth; + this.screenHeight = screenHeight; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + public int getScreenWidth() { + return screenWidth; + } + + public int getScreenHeight() { + return screenHeight; + } + +} diff --git a/server/src/com/genymobile/scrcpy/RawPoint.java b/server/src/com/genymobile/scrcpy/RawPoint.java new file mode 100644 index 00000000..e1b8d921 --- /dev/null +++ b/server/src/com/genymobile/scrcpy/RawPoint.java @@ -0,0 +1,20 @@ +package com.genymobile.scrcpy; + +public class RawPoint { + private int x; + private int y; + + public RawPoint(int x, int y) { + this.x = x; + this.y = y; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + +}