Implement keyboard/mouse control

To control the device from the computer:
 - retrieve mouse and keyboard SDL events;
 - convert them to Android events;
 - serialize them;
 - send them on the same socket used by the video stream (but in the
 opposite direction);
 - deserialize the events on the Android side;
 - inject them using the InputManager.

server/src/com/genymobile/scrcpy/ControlEvent.java

@@ -0,0 +1,102 @@
+package com.genymobile.scrcpy;
+
+/**
+ * Union of all supported event types, identified by their {@code type}.
+ */
+public class ControlEvent {
+
+    public static final int TYPE_KEYCODE = 0;
+    public static final int TYPE_TEXT = 1;
+    public static final int TYPE_MOUSE = 2;
+    public static final int TYPE_SCROLL = 3;
+
+    private int type;
+    private String text;
+    private int metaState; // KeyEvent.META_*
+    private int action; // KeyEvent.ACTION_* or MotionEvent.ACTION_*
+    private int keycode; // KeyEvent.KEYCODE_*
+    private int buttons; // MotionEvent.BUTTON_*
+    private int x;
+    private int y;
+    private int hScroll;
+    private int vScroll;
+
+    private ControlEvent() {
+    }
+
+    public static ControlEvent createKeycodeControlEvent(int action, int keycode, int metaState) {
+        ControlEvent event = new ControlEvent();
+        event.type = TYPE_KEYCODE;
+        event.action = action;
+        event.keycode = keycode;
+        event.metaState = metaState;
+        return event;
+    }
+
+    public static ControlEvent createTextControlEvent(String text) {
+        ControlEvent event = new ControlEvent();
+        event.type = TYPE_TEXT;
+        event.text = text;
+        return event;
+    }
+
+    public static ControlEvent createMotionControlEvent(int action, int buttons, int x, int y) {
+        ControlEvent event = new ControlEvent();
+        event.type = TYPE_MOUSE;
+        event.action = action;
+        event.buttons = buttons;
+        event.x = x;
+        event.y = y;
+        return event;
+    }
+
+    public static ControlEvent createScrollControlEvent(int x, int y, int hScroll, int vScroll) {
+        ControlEvent event = new ControlEvent();
+        event.type = TYPE_SCROLL;
+        event.x = x;
+        event.y = y;
+        event.hScroll = hScroll;
+        event.vScroll = vScroll;
+        return event;
+    }
+
+    public int getType() {
+        return type;
+    }
+
+    public String getText() {
+        return text;
+    }
+
+    public int getMetaState() {
+        return metaState;
+    }
+
+    public int getAction() {
+        return action;
+    }
+
+    public int getKeycode() {
+        return keycode;
+    }
+
+    public int getButtons() {
+        return buttons;
+    }
+
+    public int getX() {
+        return x;
+    }
+
+    public int getY() {
+        return y;
+    }
+
+    public int getHScroll() {
+        return hScroll;
+    }
+
+    public int getVScroll() {
+        return vScroll;
+    }
+}

server/src/com/genymobile/scrcpy/ControlEventReader.java

@@ -0,0 +1,99 @@
+package com.genymobile.scrcpy;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+public class ControlEventReader {
+
+    private static final int KEYCODE_PAYLOAD_LENGTH = 9;
+    private static final int MOUSE_PAYLOAD_LENGTH = 13;
+    private static final int SCROLL_PAYLOAD_LENGTH = 16;
+
+    private final byte[] rawBuffer = new byte[128];
+    private final ByteBuffer buffer = ByteBuffer.wrap(rawBuffer);
+    private final byte[] textBuffer = new byte[32];
+
+    public ControlEventReader() {
+        // invariant: the buffer is always in "get" mode
+        buffer.limit(0);
+    }
+
+    public boolean isFull() {
+        return buffer.remaining() == rawBuffer.length;
+    }
+
+    public boolean readFrom(InputStream input) throws IOException {
+        if (isFull()) {
+            throw new IllegalStateException("Buffer full, call next() to consume");
+        }
+        buffer.compact();
+        int head = buffer.position();
+        int r = input.read(rawBuffer, head, rawBuffer.length - head);
+        if (r == -1) {
+            return false;
+        }
+        buffer.position(head + r);
+        buffer.flip();
+        return true;
+    }
+
+    public ControlEvent next() {
+        if (!buffer.hasRemaining()) {
+            return null;
+        }
+        int savedPosition = buffer.position();
+
+        int type = buffer.get();
+        switch (type) {
+            case ControlEvent.TYPE_KEYCODE: {
+                if (buffer.remaining() < KEYCODE_PAYLOAD_LENGTH) {
+                    break;
+                }
+                int action = buffer.get() & 0xff; // unsigned
+                int keycode = buffer.getInt();
+                int metaState = buffer.getInt();
+                return ControlEvent.createKeycodeControlEvent(action, keycode, metaState);
+            }
+            case ControlEvent.TYPE_TEXT: {
+                if (buffer.remaining() < 1) {
+                    break;
+                }
+                int len = buffer.get() & 0xff; // unsigned
+                if (buffer.remaining() < len) {
+                    break;
+                }
+                buffer.get(textBuffer, 0, len);
+                String text = new String(textBuffer, 0, len, StandardCharsets.UTF_8);
+                return ControlEvent.createTextControlEvent(text);
+            }
+            case ControlEvent.TYPE_MOUSE: {
+                if (buffer.remaining() < MOUSE_PAYLOAD_LENGTH) {
+                    break;
+                }
+                int action = buffer.get() & 0xff; // unsigned
+                int buttons = buffer.getInt();
+                int x = buffer.getInt();
+                int y = buffer.getInt();
+                return ControlEvent.createMotionControlEvent(action, buttons, x, y);
+            }
+            case ControlEvent.TYPE_SCROLL: {
+                if (buffer.remaining() < SCROLL_PAYLOAD_LENGTH) {
+                    break;
+                }
+                int x = buffer.getInt();
+                int y = buffer.getInt();
+                int hscroll = buffer.getInt();
+                int vscroll = buffer.getInt();
+                return ControlEvent.createScrollControlEvent(x, y, hscroll, vscroll);
+            }
+            default:
+                Ln.w("Unknown event type: " + type);
+        }
+
+        // failure, reset savedPosition
+        buffer.position(savedPosition);
+        return null;
+    }
+}

server/src/com/genymobile/scrcpy/DesktopConnection.java

@@ -5,6 +5,8 @@ import android.net.LocalSocketAddress;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 
 public class DesktopConnection implements Closeable {
@@ -14,9 +16,15 @@ public class DesktopConnection implements Closeable {
     private static final String SOCKET_NAME = "scrcpy";
 
     private final LocalSocket socket;
+    private final InputStream inputStream;
+    private final OutputStream outputStream;
+
+    private final ControlEventReader reader = new ControlEventReader();
 
     private DesktopConnection(LocalSocket socket) throws IOException {
         this.socket = socket;
+        inputStream = socket.getInputStream();
+        outputStream = socket.getOutputStream();
     }
 
     private static LocalSocket connect(String abstractName) throws IOException {
@@ -27,8 +35,9 @@ public class DesktopConnection implements Closeable {
 
     public static DesktopConnection open(String deviceName, int width, int height) throws IOException {
         LocalSocket socket = connect(SOCKET_NAME);
-        send(socket, deviceName, width, height);
-        return new DesktopConnection(socket);
+        DesktopConnection connection = new DesktopConnection(socket);
+        connection.send(deviceName, width, height);
+        return connection;
     }
 
     public void close() throws IOException {
@@ -37,7 +46,7 @@ public class DesktopConnection implements Closeable {
         socket.close();
     }
 
-    private static void send(LocalSocket socket, String deviceName, int width, int height) throws IOException {
+    private void send(String deviceName, int width, int height) throws IOException {
         assert width < 0x10000 : "width may not be stored on 16 bits";
         assert height < 0x10000 : "height may not be stored on 16 bits";
         byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH + 4];
@@ -51,11 +60,20 @@ public class DesktopConnection implements Closeable {
         buffer[DEVICE_NAME_FIELD_LENGTH + 1] = (byte) width;
         buffer[DEVICE_NAME_FIELD_LENGTH + 2] = (byte) (height >> 8);
         buffer[DEVICE_NAME_FIELD_LENGTH + 3] = (byte) height;
-        socket.getOutputStream().write(buffer, 0, buffer.length);
+        outputStream.write(buffer, 0, buffer.length);
     }
 
     public void sendVideoStream(byte[] videoStreamBuffer, int len) throws IOException {
-        socket.getOutputStream().write(videoStreamBuffer, 0, len);
+        outputStream.write(videoStreamBuffer, 0, len);
+    }
+
+    public ControlEvent receiveControlEvent() throws IOException {
+        ControlEvent event = reader.next();
+        while (event == null) {
+            reader.readFrom(inputStream);
+            event = reader.next();
+        }
+        return event;
     }
 }
 

server/src/com/genymobile/scrcpy/DeviceUtil.java

@@ -3,6 +3,7 @@ package com.genymobile.scrcpy;
 import android.os.Build;
 import android.view.IRotationWatcher;
 
+import com.genymobile.scrcpy.wrappers.InputManager;
 import com.genymobile.scrcpy.wrappers.ServiceManager;
 
 public class DeviceUtil {
@@ -20,4 +21,8 @@ public class DeviceUtil {
     public static String getDeviceName() {
         return Build.MODEL;
     }
+
+    public static InputManager getInputManager() {
+        return serviceManager.getInputManager();
+    }
 }

server/src/com/genymobile/scrcpy/EventController.java

@@ -0,0 +1,127 @@
+package com.genymobile.scrcpy;
+
+import android.os.SystemClock;
+import android.view.InputDevice;
+import android.view.InputEvent;
+import android.view.KeyCharacterMap;
+import android.view.KeyEvent;
+import android.view.MotionEvent;
+
+import com.genymobile.scrcpy.wrappers.InputManager;
+
+import java.io.IOException;
+
+public class EventController {
+
+    private final InputManager inputManager;
+    private final DesktopConnection connection;
+
+    private final KeyCharacterMap charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD);
+
+    private long lastMouseDown;
+    private final MotionEvent.PointerProperties[] pointerProperties = { new MotionEvent.PointerProperties() };
+    private final MotionEvent.PointerCoords[] pointerCoords = { new MotionEvent.PointerCoords() };
+
+    public EventController(DesktopConnection connection) {
+        this.connection = connection;
+        inputManager = DeviceUtil.getInputManager();
+        initPointer();
+    }
+
+    private void initPointer() {
+        MotionEvent.PointerProperties props = pointerProperties[0];
+        props.id = 0;
+        props.toolType = MotionEvent.TOOL_TYPE_MOUSE;
+
+        MotionEvent.PointerCoords coords = pointerCoords[0];
+        coords.orientation = 0;
+        coords.pressure = 1;
+        coords.size = 1;
+        coords.toolMajor = 1;
+        coords.toolMinor = 1;
+        coords.touchMajor = 1;
+        coords.touchMinor = 1;
+    }
+
+    private void setPointerCoords(int x, int y) {
+        MotionEvent.PointerCoords coords = pointerCoords[0];
+        coords.x = x;
+        coords.y = y;
+    }
+
+    private void setScroll(int hScroll, int vScroll) {
+        MotionEvent.PointerCoords coords = pointerCoords[0];
+        coords.setAxisValue(MotionEvent.AXIS_SCROLL, hScroll);
+        coords.setAxisValue(MotionEvent.AXIS_VSCROLL, vScroll);
+    }
+
+    public void control() throws IOException {
+        while (handleEvent());
+    }
+
+    private boolean handleEvent() throws IOException {
+        ControlEvent controlEvent = connection.receiveControlEvent();
+        if (controlEvent == null) {
+            return false;
+        }
+        switch (controlEvent.getType()) {
+            case ControlEvent.TYPE_KEYCODE:
+                injectKeycode(controlEvent.getAction(), controlEvent.getKeycode(), controlEvent.getMetaState());
+                break;
+            case ControlEvent.TYPE_TEXT:
+                injectText(controlEvent.getText());
+                break;
+            case ControlEvent.TYPE_MOUSE:
+                injectMouse(controlEvent.getAction(), controlEvent.getButtons(), controlEvent.getX(), controlEvent.getY());
+                break;
+            case ControlEvent.TYPE_SCROLL:
+                injectScroll(controlEvent.getButtons(), controlEvent.getX(), controlEvent.getY(), controlEvent.getHScroll(), controlEvent.getVScroll());
+        }
+        return true;
+    }
+
+    private boolean injectKeycode(int action, int keycode, int metaState) {
+        return injectKeyEvent(action, keycode, 0, metaState);
+    }
+
+    private boolean injectText(String text) {
+        KeyEvent[] events = charMap.getEvents(text.toCharArray());
+        if (events == null) {
+            return false;
+        }
+        for (KeyEvent event : events) {
+            if (!injectEvent(event)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean injectMouse(int action, int buttons, int x, int y) {
+        long now = SystemClock.uptimeMillis();
+        if (action == MotionEvent.ACTION_DOWN) {
+            lastMouseDown = now;
+        }
+        setPointerCoords(x, y);
+        MotionEvent event = MotionEvent.obtain(lastMouseDown, now, action, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, 0, 0, InputDevice.SOURCE_MOUSE, 0);
+        return injectEvent(event);
+    }
+
+    private boolean injectScroll(int buttons, int x, int y, int hScroll, int vScroll) {
+        long now = SystemClock.uptimeMillis();
+        setPointerCoords(x, y);
+        setScroll(hScroll, vScroll);
+        MotionEvent event = MotionEvent.obtain(lastMouseDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, 0, 0, InputDevice.SOURCE_MOUSE, 0);
+        return injectEvent(event);
+    }
+
+    private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState) {
+        long now = SystemClock.uptimeMillis();
+        KeyEvent event = new KeyEvent(now, now, action, keyCode, repeat, metaState, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, 0, InputDevice.SOURCE_KEYBOARD);
+        return injectEvent(event);
+    }
+
+    private boolean injectEvent(InputEvent event) {
+        return inputManager.injectInputEvent(event, InputManager.INJECT_INPUT_EVENT_MODE_ASYNC);
+    }
+}

server/src/com/genymobile/scrcpy/ScrCpyServer.java

@@ -6,13 +6,16 @@ public class ScrCpyServer {
 
     private static final String TAG = "scrcpy";
 
-    public static void scrcpy() throws IOException {
+    private static void scrcpy() throws IOException {
         String deviceName = DeviceUtil.getDeviceName();
         ScreenInfo initialScreenInfo = DeviceUtil.getScreenInfo();
         int width = initialScreenInfo.getLogicalWidth();
         int height = initialScreenInfo.getLogicalHeight();
         try (DesktopConnection connection = DesktopConnection.open(deviceName, width, height)) {
             try {
+                // asynchronous
+                startEventController(connection);
+                // synchronous
                 new ScreenStreamer(connection).streamScreen();
             } catch (IOException e) {
                 Ln.e("Screen streaming interrupted", e);
@@ -20,6 +23,19 @@ public class ScrCpyServer {
         }
     }
 
+    private static void startEventController(final DesktopConnection connection) {
+        new Thread(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    new EventController(connection).control();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }).start();
+    }
+
     public static void main(String... args) throws Exception {
         try {
             scrcpy();

server/src/com/genymobile/scrcpy/wrappers/InputManager.java

@@ -0,0 +1,34 @@
+package com.genymobile.scrcpy.wrappers;
+
+import android.os.IInterface;
+import android.view.InputEvent;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+public class InputManager {
+
+    public static final int INJECT_INPUT_EVENT_MODE_ASYNC = 0;
+    public static final int INJECT_INPUT_EVENT_MODE_WAIT_FOR_RESULT = 1;
+    public static final int INJECT_INPUT_EVENT_MODE_WAIT_FOR_FINISH = 2;
+
+    private final IInterface manager;
+    private final Method injectInputEventMethod;
+
+    public InputManager(IInterface manager) {
+        this.manager = manager;
+        try {
+            injectInputEventMethod = manager.getClass().getMethod("injectInputEvent", InputEvent.class, int.class);
+        } catch (NoSuchMethodException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public boolean injectInputEvent(InputEvent inputEvent, int mode) {
+        try {
+            return (Boolean) injectInputEventMethod.invoke(manager, inputEvent, mode);
+        } catch (InvocationTargetException | IllegalAccessException e) {
+            throw new AssertionError(e);
+        }
+    }
+}

server/src/com/genymobile/scrcpy/wrappers/ServiceManager.java

@@ -33,4 +33,8 @@ public class ServiceManager {
     public DisplayManager getDisplayManager() {
         return new DisplayManager(getService("display", "android.hardware.display.IDisplayManager"));
     }
+
+    public InputManager getInputManager() {
+        return new InputManager(getService("input", "android.hardware.input.IInputManager"));
+    }
 }