Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(computer-server): add multitouch_gesture action for Android (#1197)

ddupont5b6a10eadb65
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.py
index 955e316736..5d44a862c1 100644
--- a/libs/python/computer-server/computer_server/handlers/android.py
+++ b/libs/python/computer-server/computer_server/handlers/android.py
@@ -1,8 +1,11 @@
import asyncio
import base64
+import logging
from typing import Any, Dict, List, Optional, Tuple
from ..utils.helpers import CommandExecutor
+
+logger = logging.getLogger(__name__)
from .base import (
BaseAccessibilityHandler,
BaseAutomationHandler,
@@ -500,6 +503,129 @@ class AndroidAutomationHandler(BaseAutomationHandler):
return {"success": False, "error": f"Set clipboard error: {str(e)}"}
# Other
+ async def multitouch_gesture(
+ self,
+ fingers: List[Dict[str, Any]],
+ screen_w: int,
+ screen_h: int,
+ duration_ms: int = 400,
+ steps: int = 0,
+ ) -> Dict[str, Any]:
+ """Inject a multi-touch gesture via MT Protocol B sendevent.
+
+ Uses ``adb root`` to restart adbd as root so that ``sendevent`` can
+ write to ``/dev/input/event*`` without requiring ``su`` inside the
+ Android shell.
+
+ Args:
+ fingers: List of ``{"start": [x, y], "end": [x, y]}`` dicts,
+ one per finger (minimum 2).
+ screen_w: Screen width in pixels (used for coordinate scaling).
+ screen_h: Screen height in pixels.
+ duration_ms: Total gesture duration in milliseconds.
+ steps: Interpolation steps (0 = auto).
+
+ Raises:
+ ValueError: If fewer than 2 fingers are provided or a finger dict
+ is missing ``start``/``end`` keys.
+ RuntimeError: If sendevent injection fails.
+ """
+ # Validate fingers
+ if len(fingers) < 2:
+ raise ValueError(f"multitouch_gesture requires at least 2 fingers, got {len(fingers)}")
+ for i, finger in enumerate(fingers):
+ if "start" not in finger or "end" not in finger:
+ raise ValueError(
+ f"fingers[{i}] must have 'start' and 'end' keys, got: {list(finger.keys())}"
+ )
+ if len(finger["start"]) != 2 or len(finger["end"]) != 2:
+ raise ValueError(f"fingers[{i}] 'start' and 'end' must be [x, y] pairs")
+
+ # Restart adbd as root so sendevent works without su
+ ok_root, root_out = await adb_exec.run("root", decode=True)
+ if not ok_root:
+ logger.warning("adb root failed (output: %r); sendevent may lack permissions", root_out)
+ await asyncio.sleep(1.0) # wait for adbd to restart
+
+ # Find the touchscreen event device
+ ok, dev_out = await adb_exec.run(
+ "shell",
+ "for f in /dev/input/event*; do "
+ " getevent -p \"$f\" 2>/dev/null | grep -q '0035' "
+ ' && echo "$f" && break; '
+ "done",
+ decode=True,
+ )
+ if ok and dev_out.strip():
+ dev = dev_out.strip()
+ else:
+ dev = "/dev/input/event1"
+ logger.warning("touch device detection failed; falling back to %s", dev)
+
+ # Detect axis max from getevent -p (e.g. "max 32767")
+ axis_max = 32767
+ ok2, gp_out = await adb_exec.run(
+ "shell", f"getevent -p {dev} 2>/dev/null | grep '0035'", decode=True
+ )
+ if ok2 and "max" in gp_out:
+ import re as _re
+
+ m = _re.search(r"max\s+(\d+)", gp_out)
+ if m:
+ axis_max = int(m.group(1))
+
+ n_steps = steps if steps > 0 else max(5, duration_ms // 20)
+ step_delay = (duration_ms / 1000) / n_steps
+
+ EV_SYN, EV_KEY, EV_ABS = 0, 1, 3
+ SYN_REPORT, BTN_TOUCH = 0, 330
+ SLOT, TID, MTX, MTY, PRESS = 47, 57, 53, 54, 58
+ TID_NONE = 4294967295
+
+ def px_to_raw(px: int, dim: int) -> int:
+ return max(0, min(axis_max, int(px * axis_max / dim)))
+
+ def se(t: int, c: int, v: int) -> str:
+ return f"sendevent {dev} {t} {c} {v}"
+
+ cmds: List[str] = []
+
+ # Touch down — all fingers in one SYN_REPORT frame
+ for idx, finger in enumerate(fingers):
+ x1, y1 = finger["start"]
+ cmds += [
+ se(EV_ABS, SLOT, idx),
+ se(EV_ABS, TID, idx),
+ se(EV_ABS, MTX, px_to_raw(x1, screen_w)),
+ se(EV_ABS, MTY, px_to_raw(y1, screen_h)),
+ se(EV_ABS, PRESS, 64),
+ ]
+ cmds += [se(EV_KEY, BTN_TOUCH, 1), se(EV_SYN, SYN_REPORT, 0)]
+
+ # Interpolated movement
+ for i in range(1, n_steps + 1):
+ t = i / n_steps
+ for idx, finger in enumerate(fingers):
+ x1, y1 = finger["start"]
+ x2, y2 = finger["end"]
+ cmds += [
+ se(EV_ABS, SLOT, idx),
+ se(EV_ABS, MTX, px_to_raw(int(x1 + (x2 - x1) * t), screen_w)),
+ se(EV_ABS, MTY, px_to_raw(int(y1 + (y2 - y1) * t), screen_h)),
+ ]
+ cmds += [se(EV_SYN, SYN_REPORT, 0), f"sleep {step_delay:.3f}"]
+
+ # Touch up — all fingers in one SYN_REPORT frame
+ for idx in range(len(fingers)):
+ cmds += [se(EV_ABS, SLOT, idx), se(EV_ABS, TID, TID_NONE)]
+ cmds += [se(EV_KEY, BTN_TOUCH, 0), se(EV_SYN, SYN_REPORT, 0)]
+
+ script = " && ".join(cmds)
+ success, output = await adb_exec.run("shell", script, decode=True)
+ if not success:
+ raise RuntimeError(f"multitouch_gesture sendevent failed: {output}")
+ return {}
+
async def run_command(self, command: str) -> Dict[str, Any]:
"""Run a shell command inside the Android emulator via adb shell."""
process = await asyncio.create_subprocess_exec(
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py
index 0d68d2dbad..ac8b1c59fd 100644
--- a/libs/python/computer-server/computer_server/main.py
+++ b/libs/python/computer-server/computer_server/main.py
@@ -219,6 +219,11 @@ handlers = {
"set_clipboard": automation_handler.set_clipboard,
}
+# Android-only commands — registered only when the Android handler is active
+# so non-Android server instances don't fail at startup with AttributeError.
+if hasattr(automation_handler, "multitouch_gesture"):
+ handlers["multitouch_gesture"] = automation_handler.multitouch_gesture
+
class AuthenticationManager:
def __init__(self):