Display and control your Android device
Inject display-related events to virtual display
Romain Vimontd19396718ee0
diff --git a/server/src/main/java/com/genymobile/scrcpy/control/Controller.java b/server/src/main/java/com/genymobile/scrcpy/control/Controller.javaindex ac870f27..5175ed5e 100644--- a/server/src/main/java/com/genymobile/scrcpy/control/Controller.java+++ b/server/src/main/java/com/genymobile/scrcpy/control/Controller.java@@ -17,7 +17,6 @@ import android.content.Intent;import android.os.Build;import android.os.SystemClock;import android.view.InputDevice;-import android.view.InputEvent;import android.view.KeyCharacterMap;import android.view.KeyEvent;import android.view.MotionEvent;@@ -31,6 +30,28 @@ import java.util.concurrent.atomic.AtomicReference;public class Controller implements AsyncProcessor, VirtualDisplayListener {+ /*+ * For event injection, there are two display ids:+ * - the displayId passed to the constructor (which comes from --display-id passed by the client, 0 for the main display);+ * - the virtualDisplayId used for mirroring, notified by the capture instance via the VirtualDisplayListener interface.+ *+ * (In case the ScreenCapture uses the "SurfaceControl API", then both ids are equals, but this is an implementation detail.)+ *+ * In order to make events work correctly in all cases:+ * - virtualDisplayId must be used for events relative to the display (mouse and touch events with coordinates);+ * - displayId must be used for other events (like key events).+ */++ private static final class DisplayData {+ private final int virtualDisplayId;+ private final PositionMapper positionMapper;++ private DisplayData(int virtualDisplayId, PositionMapper positionMapper) {+ this.virtualDisplayId = virtualDisplayId;+ this.positionMapper = positionMapper;+ }+ }+private static final int DEFAULT_DEVICE_ID = 0;// control_msg.h values of the pointerId field in inject_touch_event message@@ -54,7 +75,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {private final AtomicBoolean isSettingClipboard = new AtomicBoolean();- private final AtomicReference<PositionMapper> positionMapper = new AtomicReference<>();+ private final AtomicReference<DisplayData> displayData = new AtomicReference<>();private long lastTouchDown;private final PointersState pointersState = new PointersState();@@ -102,8 +123,9 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {}@Override- public void onNewVirtualDisplay(PositionMapper positionMapper) {- this.positionMapper.set(positionMapper);+ public void onNewVirtualDisplay(int virtualDisplayId, PositionMapper positionMapper) {+ DisplayData data = new DisplayData(virtualDisplayId, positionMapper);+ this.displayData.set(data);}private UhidManager getUhidManager() {@@ -284,7 +306,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {return false;}for (KeyEvent event : events) {- if (!injectEvent(event, Device.INJECT_MODE_ASYNC)) {+ if (!Device.injectEvent(event, displayId, Device.INJECT_MODE_ASYNC)) {return false;}}@@ -306,7 +328,12 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) {long now = SystemClock.uptimeMillis();- Point point = getPhysicalPoint(position);+ // it hides the field on purpose, to read it with atomic access+ @SuppressWarnings("checkstyle:HiddenField")+ DisplayData displayData = this.displayData.get();+ assert displayData != null : "Cannot receive a touch event without a display";++ Point point = displayData.positionMapper.map(position);if (point == null) {Ln.w("Ignore touch event, it was generated for a different device size");return false;@@ -365,7 +392,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {// First button pressed: ACTION_DOWNMotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties,pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);- if (!injectEvent(downEvent, Device.INJECT_MODE_ASYNC)) {+ if (!Device.injectEvent(downEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {return false;}}@@ -376,7 +403,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {if (!InputManager.setActionButton(pressEvent, actionButton)) {return false;}- if (!injectEvent(pressEvent, Device.INJECT_MODE_ASYNC)) {+ if (!Device.injectEvent(pressEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {return false;}@@ -390,7 +417,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {if (!InputManager.setActionButton(releaseEvent, actionButton)) {return false;}- if (!injectEvent(releaseEvent, Device.INJECT_MODE_ASYNC)) {+ if (!Device.injectEvent(releaseEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {return false;}@@ -398,7 +425,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {// Last button released: ACTION_UPMotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties,pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);- if (!injectEvent(upEvent, Device.INJECT_MODE_ASYNC)) {+ if (!Device.injectEvent(upEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {return false;}}@@ -409,12 +436,18 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f,DEFAULT_DEVICE_ID, 0, source, 0);- return injectEvent(event, Device.INJECT_MODE_ASYNC);+ return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC);}private boolean injectScroll(Position position, float hScroll, float vScroll, int buttons) {long now = SystemClock.uptimeMillis();- Point point = getPhysicalPoint(position);++ // it hides the field on purpose, to read it with atomic access+ @SuppressWarnings("checkstyle:HiddenField")+ DisplayData displayData = this.displayData.get();+ assert displayData != null : "Cannot receive a scroll event without a display";++ Point point = displayData.positionMapper.map(position);if (point == null) {Ln.w("Ignore scroll event, it was generated for a different device size");return false;@@ -431,18 +464,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {MotionEvent event = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f,DEFAULT_DEVICE_ID, 0, InputDevice.SOURCE_MOUSE, 0);- return injectEvent(event, Device.INJECT_MODE_ASYNC);- }-- private Point getPhysicalPoint(Position position) {- // it hides the field on purpose, to read it with atomic access- @SuppressWarnings("checkstyle:HiddenField")- PositionMapper positionMapper = this.positionMapper.get();- if (positionMapper == null) {- return null;- }-- return positionMapper.map(position);+ return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC);}/**@@ -520,10 +542,6 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {ServiceManager.getActivityManager().startActivity(intent);}- private boolean injectEvent(InputEvent event, int injectMode) {- return Device.injectEvent(event, displayId, injectMode);- }-private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState, int injectMode) {return Device.injectKeyEvent(action, keyCode, repeat, metaState, displayId, injectMode);}diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.javaindex 95faaf39..7e516909 100644--- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java+++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java@@ -106,10 +106,16 @@ public class ScreenCapture extends SurfaceCapture {virtualDisplay = null;}+ int virtualDisplayId;+ PositionMapper positionMapper;try {Size videoSize = screenInfo.getVideoSize();virtualDisplay = ServiceManager.getDisplayManager().createVirtualDisplay("scrcpy", videoSize.getWidth(), videoSize.getHeight(), displayId, surface);+ virtualDisplayId = virtualDisplay.getDisplay().getDisplayId();+ Rect contentRect = new Rect(0, 0, videoSize.getWidth(), videoSize.getHeight());+ // The position are relative to the virtual display, not the original display+ positionMapper = new PositionMapper(videoSize, contentRect, 0);Ln.d("Display: using DisplayManager API");} catch (Exception displayManagerException) {try {@@ -123,6 +129,8 @@ public class ScreenCapture extends SurfaceCapture {int layerStack = displayInfo.getLayerStack();setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);+ virtualDisplayId = displayId;+ positionMapper = PositionMapper.from(screenInfo);Ln.d("Display: using SurfaceControl API");} catch (Exception surfaceControlException) {Ln.e("Could not create display using DisplayManager", displayManagerException);@@ -132,8 +140,7 @@ public class ScreenCapture extends SurfaceCapture {}if (vdListener != null) {- PositionMapper positionMapper = PositionMapper.from(screenInfo);- vdListener.onNewVirtualDisplay(positionMapper);+ vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper);}}diff --git a/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java b/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.javaindex d978361e..c079265e 100644--- a/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java+++ b/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java@@ -3,5 +3,5 @@ package com.genymobile.scrcpy.video;import com.genymobile.scrcpy.control.PositionMapper;public interface VirtualDisplayListener {- void onNewVirtualDisplay(PositionMapper positionMapper);+ void onNewVirtualDisplay(int displayId, PositionMapper positionMapper);}