Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(computer-server): fix multitouch gesture on VMs with multiple input devices (#1199)
ddupont2a189d4412d3
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.pyindex 4d18c1c526..bc25f5b13d 100644--- a/libs/python/computer-server/computer_server/handlers/android.py+++ b/libs/python/computer-server/computer_server/handlers/android.py@@ -510,7 +510,15 @@ class AndroidAutomationHandler(BaseAutomationHandler):duration_ms: int = 400,steps: int = 0,) -> Dict[str, Any]:- """Inject a multi-touch gesture via MT Protocol B sendevent.+ """Inject a multi-touch gesture via sendevent.++ Supports two hardware topologies:++ * **virtio-input** (cloud VMs): 11 separate ``virtio_input_multi_touch_N``+ devices, one per finger (event2–event12). Each finger is written to its+ own device with ``ABS_MT_SLOT=0``; no cross-device slot sharing.+ * **Single MT device** (local AVDs): classic MT Protocol B — all fingers+ on one device via ``ABS_MT_SLOT`` 0, 1, …Uses ``adb root`` to restart adbd as root so that ``sendevent`` canwrite to ``/dev/input/event*`` without requiring ``su`` inside the@@ -523,48 +531,41 @@ class AndroidAutomationHandler(BaseAutomationHandler):screen_h: Screen height in pixels.duration_ms: Total gesture duration in milliseconds.steps: Interpolation steps (0 = auto).-- Raises:- ValueError: If fewer than 2 fingers are provided or a finger dict- is missing ``start``/``end`` keys.- RuntimeError: If sendevent injection fails."""- # Validate fingers- if len(fingers) < 2:- raise ValueError(f"multitouch_gesture requires at least 2 fingers, got {len(fingers)}")- for i, finger in enumerate(fingers):- if "start" not in finger or "end" not in finger:- raise ValueError(- f"fingers[{i}] must have 'start' and 'end' keys, got: {list(finger.keys())}"- )- if len(finger["start"]) != 2 or len(finger["end"]) != 2:- raise ValueError(f"fingers[{i}] 'start' and 'end' must be [x, y] pairs")-# Restart adbd as root so sendevent works without su- ok_root, root_out = await adb_exec.run("root", decode=True)- if not ok_root:- logger.warning("adb root failed (output: %r); sendevent may lack permissions", root_out)+ await adb_exec.run("root", decode=True)await asyncio.sleep(1.0) # wait for adbd to restart- # Find the touchscreen event device.+ # Enumerate all touch devices and detect topology.# Use `getevent -p` (no file arg) to list all devices at once and search# for "0035 :" which is the ABS axis definition syntax for ABS_MT_POSITION_X.# This avoids false matches from keyboard devices whose KEY listing contains- # 0035 as a scancode (e.g. "AT Translated Set 2 keyboard").+ # 0035 as a scancode (e.g. "AT Translated Set 2 keyboard" on cloud VMs).ok, dev_out = await adb_exec.run("shell",- "getevent -p 2>/dev/null | awk '/add device/{dev=$NF} /0035 :/{print dev; exit}'",+ "getevent -p 2>/dev/null | awk '/add device/{dev=$NF} /0035 :/{print dev}'",decode=True,)+ touch_devs: List[str] = []if ok and dev_out.strip():- dev = dev_out.strip()- else:- dev = "/dev/input/event2"- logger.warning("touch device detection failed; falling back to %s", dev)-- # Detect axis max from getevent -p for the detected device+ touch_devs = [d for d in dev_out.strip().splitlines() if d.strip()]++ if not touch_devs:+ touch_devs = ["/dev/input/event2"]+ logger.warning("touch device detection failed; falling back to %s", touch_devs[0])++ # On virtio VMs, getevent -p lists 11 separate virtio_input_multi_touch_N+ # devices (event2..event12). Android only reads the lowest-numbered one+ # (event2 = virtio_input_multi_touch_1) and supports multiple slots on it.+ # Sort numerically and always use the lowest-numbered device for all fingers+ # via standard MT Protocol B slots — writing to separate devices does NOT+ # produce multi-finger events from Android's perspective.+ touch_devs.sort(key=lambda d: int(d.replace("/dev/input/event", "") or "0"))+ dev = touch_devs[0]++ # Detect axis max from the first touch deviceaxis_max = 32767- dev_name = dev.split("/")[-1] # e.g. "event2"+ dev_name = dev.split("/")[-1]ok2, gp_out = await adb_exec.run("shell","getevent -p 2>/dev/null | awk "@@ -589,22 +590,26 @@ class AndroidAutomationHandler(BaseAutomationHandler):def px_to_raw(px: int, dim: int) -> int:return max(0, min(axis_max, int(px * axis_max / dim)))- def se(t: int, c: int, v: int) -> str:+ def se(dev: str, t: int, c: int, v: int) -> str:return f"sendevent {dev} {t} {c} {v}"cmds: List[str] = []+ # MT Protocol B on the primary touch device: all fingers via slots.+ # This works for both local AVDs (single MT device) and virtio VMs+ # (where event2 = virtio_input_multi_touch_1 supports up to 11 slots).+# Touch down — all fingers in one SYN_REPORT framefor idx, finger in enumerate(fingers):x1, y1 = finger["start"]cmds += [- se(EV_ABS, SLOT, idx),- se(EV_ABS, TID, idx),- se(EV_ABS, MTX, px_to_raw(x1, screen_w)),- se(EV_ABS, MTY, px_to_raw(y1, screen_h)),- se(EV_ABS, PRESS, 64),+ se(dev, EV_ABS, SLOT, idx),+ se(dev, EV_ABS, TID, idx),+ se(dev, EV_ABS, MTX, px_to_raw(x1, screen_w)),+ se(dev, EV_ABS, MTY, px_to_raw(y1, screen_h)),+ se(dev, EV_ABS, PRESS, 64),]- cmds += [se(EV_KEY, BTN_TOUCH, 1), se(EV_SYN, SYN_REPORT, 0)]+ cmds += [se(dev, EV_KEY, BTN_TOUCH, 1), se(dev, EV_SYN, SYN_REPORT, 0)]# Interpolated movementfor i in range(1, n_steps + 1):@@ -613,22 +618,20 @@ class AndroidAutomationHandler(BaseAutomationHandler):x1, y1 = finger["start"]x2, y2 = finger["end"]cmds += [- se(EV_ABS, SLOT, idx),- se(EV_ABS, MTX, px_to_raw(int(x1 + (x2 - x1) * t), screen_w)),- se(EV_ABS, MTY, px_to_raw(int(y1 + (y2 - y1) * t), screen_h)),+ se(dev, EV_ABS, SLOT, idx),+ se(dev, EV_ABS, MTX, px_to_raw(int(x1 + (x2 - x1) * t), screen_w)),+ se(dev, EV_ABS, MTY, px_to_raw(int(y1 + (y2 - y1) * t), screen_h)),]- cmds += [se(EV_SYN, SYN_REPORT, 0), f"sleep {step_delay:.3f}"]+ cmds += [se(dev, EV_SYN, SYN_REPORT, 0), f"sleep {step_delay:.3f}"]# Touch up — all fingers in one SYN_REPORT framefor idx in range(len(fingers)):- cmds += [se(EV_ABS, SLOT, idx), se(EV_ABS, TID, TID_NONE)]- cmds += [se(EV_KEY, BTN_TOUCH, 0), se(EV_SYN, SYN_REPORT, 0)]+ cmds += [se(dev, EV_ABS, SLOT, idx), se(dev, EV_ABS, TID, TID_NONE)]+ cmds += [se(dev, EV_KEY, BTN_TOUCH, 0), se(dev, EV_SYN, SYN_REPORT, 0)]script = " && ".join(cmds)success, output = await adb_exec.run("shell", script, decode=True)- if not success:- raise RuntimeError(f"multitouch_gesture sendevent failed: {output}")- return {}+ return {"success": success, "output": output}async def run_command(self, command: str) -> Dict[str, Any]:"""Run a shell command inside the Android emulator via adb shell."""