Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix cursor blocking input, remove start command override for cuabot (#1020)
dduponte71a95019284
diff --git a/libs/cuabot/Dockerfile b/libs/cuabot/Dockerfileindex 2513aaf2b4..b264ff622b 100644--- a/libs/cuabot/Dockerfile+++ b/libs/cuabot/Dockerfile@@ -122,4 +122,4 @@ EXPOSE 10000USER user# Note: dbus-daemon runs as session bus (not system) to avoid needing root-CMD ["sh", "-c", "xpra --version && dbus-daemon --session --fork --address=unix:path=/tmp/dbus-session 2>/dev/null || true && xpra seamless :100 --sharing=yes --no-daemon --bind-tcp=0.0.0.0:10000 --html=on"]+CMD ["sh", "-c", "xpra --version && dbus-daemon --session --fork --address=unix:path=/tmp/dbus-session 2>/dev/null || true && xpra seamless :100 --sharing=yes --no-daemon --bind-tcp=0.0.0.0:10000 --html=on --dpi=96"]diff --git a/libs/cuabot/package.json b/libs/cuabot/package.jsonindex 577f307407..0d494786f6 100644--- a/libs/cuabot/package.json+++ b/libs/cuabot/package.json@@ -17,10 +17,9 @@"build": "tsc","start": "npx tsx src/cuabot.tsx","dev": "npx tsx --watch src/cuabot.tsx",- "serve": "npx tsx src/cuabotd.ts",- "stop": "npx tsx src/cuabot.tsx stop",- "status": "npx tsx src/cuabot.tsx status",- "mcp": "npx tsx src/mcp/server-mcp.ts"+ "serve": "npx tsx src/cuabot.tsx --serve",+ "stop": "npx tsx src/cuabot.tsx --stop",+ "status": "npx tsx src/cuabot.tsx --status"},"dependencies": {"@anthropic-ai/claude-agent-sdk": "^0.1.0",@@ -47,4 +46,4 @@"@types/ws": "^8.18.1","typescript": "^5.7.0"}-}+}\ No newline at end of filediff --git a/libs/cuabot/src/cuabotd.ts b/libs/cuabot/src/cuabotd.tsindex 3995a885e0..c81fd5d6ee 100644--- a/libs/cuabot/src/cuabotd.ts+++ b/libs/cuabot/src/cuabotd.ts@@ -231,7 +231,6 @@ async function ensureContainer(): Promise<number> {"--add-host=host.docker.internal:host-gateway","-v", `${claudeConfigPath}:/home/user/.claude`,IMAGE_NAME,- "xpra", "start", ":100", "--bind-tcp=0.0.0.0:10000", "--no-daemon", "--html=on",];await execAsync(createCmd.join(" "));@@ -452,6 +451,9 @@ async function updateCursorMasks(): Promise<void> {// Process back-to-front so Xpra windows only cut holes in things BEHIND themconst ops: AABBOp[] = [];+ // Store window info for debug visualization+ const windowsForDebug: { bounds: AABB; title: string; owner: string; isXpra: boolean }[] = [];+// Reverse to go back-to-front (openWindows returns front-to-back)for (const win of [...windows].reverse()) {const b = win.bounds;@@ -464,6 +466,13 @@ async function updateCursorMasks(): Promise<void> {continue;}+ windowsForDebug.push({+ bounds: { x: b.x, y: b.y, width: b.width, height: b.height },+ title: win.title,+ owner: win.owner.name,+ isXpra,+ });+if (isXpra) {// Xpra windows: subtract (cut holes in masks behind it)ops.push({ x: b.x, y: b.y, width: b.width, height: b.height, op: "subtract" });@@ -485,6 +494,11 @@ async function updateCursorMasks(): Promise<void> {cx >= r.x && cx <= r.x + r.width && cy >= r.y && cy <= r.y + r.height);+ // Debug visualization+ if (DEBUG) {+ await drawDebugMasks(windowsForDebug, maskRects, cx, cy, isInMask);+ }+// Send masked state to overlay cursor (not the rects - calculation done here)await sendCursorCommand({ type: "masked", value: isInMask });} catch (e) {@@ -492,6 +506,64 @@ async function updateCursorMasks(): Promise<void> {}}+async function drawDebugMasks(+ windows: { bounds: AABB; title: string; owner: string; isXpra: boolean }[],+ maskRects: AABB[],+ cursorX: number,+ cursorY: number,+ isInMask: boolean+): Promise<void> {+ try {+ const width = 2400;+ const height = 1600;+ const crosshairSize = 20;+ const cursorColor = isInMask ? "red" : "lime";++ // Escape XML special characters+ const escapeXml = (s: string) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);++ // Build SVG+ let svg = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">`;++ // Background+ svg += `<rect x="0" y="0" width="${width}" height="${height}" fill="#1a1a1a"/>`;++ // Draw mask rects (red filled, semi-transparent)+ for (const r of maskRects) {+ svg += `<rect x="${r.x}" y="${r.y}" width="${r.width}" height="${r.height}" fill="rgba(255,0,0,0.3)" stroke="red" stroke-width="2"/>`;+ }++ // Draw window rects (blue outlined) with labels+ for (const win of windows) {+ const b = win.bounds;+ const strokeColor = win.isXpra ? "cyan" : "blue";+ svg += `<rect x="${b.x}" y="${b.y}" width="${b.width}" height="${b.height}" fill="none" stroke="${strokeColor}" stroke-width="2"/>`;++ // Label background+ const label = `${escapeXml(win.owner)}: ${escapeXml(win.title.substring(0, 40))}`;+ svg += `<rect x="${b.x}" y="${b.y - 18}" width="${Math.min(b.width, 300)}" height="18" fill="rgba(0,0,0,0.7)"/>`;+ svg += `<text x="${b.x + 4}" y="${b.y - 4}" font-family="monospace" font-size="12" fill="${strokeColor}">${label}</text>`;+ }++ // Draw cursor crosshair+ svg += `<line x1="${cursorX - crosshairSize}" y1="${cursorY}" x2="${cursorX + crosshairSize}" y2="${cursorY}" stroke="${cursorColor}" stroke-width="3"/>`;+ svg += `<line x1="${cursorX}" y1="${cursorY - crosshairSize}" x2="${cursorX}" y2="${cursorY + crosshairSize}" stroke="${cursorColor}" stroke-width="3"/>`;+ svg += `<circle cx="${cursorX}" cy="${cursorY}" r="8" fill="none" stroke="${cursorColor}" stroke-width="2"/>`;++ // Legend+ svg += `<rect x="10" y="10" width="200" height="80" fill="rgba(0,0,0,0.8)" stroke="white"/>`;+ svg += `<text x="20" y="30" font-family="monospace" font-size="12" fill="white">Cursor: ${cursorX}, ${cursorY}</text>`;+ svg += `<text x="20" y="50" font-family="monospace" font-size="12" fill="${cursorColor}">Masked: ${isInMask}</text>`;+ svg += `<text x="20" y="70" font-family="monospace" font-size="12" fill="cyan">Cyan=Xpra, Blue=Other</text>`;++ svg += `</svg>`;++ await sharp(Buffer.from(svg)).png().toFile(SCREENSHOT_PATH.replace("screenshot.jpg", "masks.png"));+ } catch (e) {+ console.error("[debug] Failed to draw masks:", e);+ }+}+function startMaskPolling(): void {if (maskPollingInterval) return;maskPollingInterval = setInterval(updateCursorMasks, 500);diff --git a/libs/cuabot/src/mcp/computer-use-mcp.py b/libs/cuabot/src/mcp/computer-use-mcp.pyindex 759f71b6bd..d64454e143 100644--- a/libs/cuabot/src/mcp/computer-use-mcp.py+++ b/libs/cuabot/src/mcp/computer-use-mcp.py@@ -73,13 +73,21 @@ def request(endpoint: str, body: dict | None = None) -> dict:@server.tool()-def screenshot() -> Image:- """Take a screenshot of the display. Returns JPEG image."""- import base64+def screenshot(save_path: str | None = None) -> Image:+ """Take a screenshot of the display. Returns JPEG image.- log_mcp_tool_call("screenshot", {})+ Args:+ save_path: Optional path to save the screenshot (e.g., "/tmp/screenshot.jpg")+ """+ import base64+ log_mcp_tool_call("screenshot", {"save_path": save_path})result = request("screenshot")image_bytes = base64.b64decode(result["image"])++ if save_path:+ with open(save_path, "wb") as f:+ f.write(image_bytes)+return Image(data=image_bytes, format="jpeg")diff --git a/libs/cuabot/src/mcp/overlay-cursor.py b/libs/cuabot/src/mcp/overlay-cursor.pyindex 3a879f1128..3eddcd4a5e 100644--- a/libs/cuabot/src/mcp/overlay-cursor.py+++ b/libs/cuabot/src/mcp/overlay-cursor.py@@ -180,12 +180,13 @@ class OverlayCursor(Gtk.Window):else:print(f"Cursor image not found at {cursor_path}", file=sys.stderr)- # Window setup (wider/taller to fit cursor and debug label)- self.set_size_request(CURSOR_SIZE * 5, CURSOR_SIZE * 3)+ # Window setup - size just big enough for cursor+ self.set_size_request(CURSOR_SIZE, CURSOR_SIZE)self.set_decorated(False)self.set_skip_taskbar_hint(True)self.set_skip_pager_hint(True)self.set_accept_focus(False)+ self.set_can_focus(False)self.set_app_paintable(True)self.set_title(f"cuabot-cursor-{name}")@@ -197,6 +198,7 @@ class OverlayCursor(Gtk.Window):# Make click-through after window is realizedself.connect("realize", self._on_realize)+ self.connect("map-event", self._on_map)# Drawing areaself.drawing_area = Gtk.DrawingArea()@@ -210,10 +212,19 @@ class OverlayCursor(Gtk.Window):GLib.timeout_add(1000, self.check_fade)def _on_realize(self, widget):- # Make window click-through by setting empty input region+ self._make_click_through()++ def _on_map(self, widget, event):+ self._make_click_through()+ return False++ def _make_click_through(self):+ """Make window click-through by setting empty input region"""gdk_window = self.get_window()if gdk_window:- gdk_window.input_shape_combine_region(cairo.Region(), 0, 0)+ # Create an empty region (no input area)+ empty_region = cairo.Region(cairo.RectangleInt(0, 0, 0, 0))+ gdk_window.input_shape_combine_region(empty_region, 0, 0)def on_draw(self, widget, cr):# Clear background@@ -260,84 +271,18 @@ class OverlayCursor(Gtk.Window):returnr, g, b = self.color+ cr.set_operator(cairo.OPERATOR_OVER)- # Draw cursor image or fallback circleif self.cursor_pixbuf:- # Create a colorized version of the cursorcolorized = self._colorize_pixbuf(self.cursor_pixbuf, r, g, b, self.opacity)-- # Click scale effect- click_scale = (- 1.0 + 0.2 * math.sin(self.click_anim_progress * math.pi) if self.clicking else 1.0- )-- cr.set_operator(cairo.OPERATOR_OVER)-- if click_scale != 1.0:- # Scale around the cursor tip (top-left corner)- scaled_size = int(CURSOR_SIZE * click_scale)- scaled = colorized.scale_simple(- scaled_size, scaled_size, GdkPixbuf.InterpType.BILINEAR- )- Gdk.cairo_set_source_pixbuf(- cr, scaled, CURSOR_SIZE - CURSOR_SIZE * 0.1, CURSOR_SIZE - CURSOR_SIZE * 0.1- )- else:- Gdk.cairo_set_source_pixbuf(- cr, colorized, CURSOR_SIZE - CURSOR_SIZE * 0.1, CURSOR_SIZE - CURSOR_SIZE * 0.1- )-+ Gdk.cairo_set_source_pixbuf(cr, colorized, 0, 0)cr.paint()-- # Click ripple- if self.clicking and self.click_anim_progress > 0:- cx = CURSOR_SIZE + CURSOR_SIZE * 0.1- cy = CURSOR_SIZE + CURSOR_SIZE * 0.1- ripple_radius = CURSOR_SIZE * 0.3 + (CURSOR_SIZE * 0.8 * self.click_anim_progress)- ripple_alpha = (1 - self.click_anim_progress) * 0.5 * self.opacity- cr.set_source_rgba(r, g, b, ripple_alpha)- cr.set_line_width(2)- cr.arc(cx, cy, ripple_radius, 0, 2 * math.pi)- cr.stroke()else:- # Fallback: draw circle cursor- cx = CURSOR_SIZE- cy = CURSOR_SIZE-- # Outer glow- cr.set_operator(cairo.OPERATOR_OVER)- gradient = cairo.RadialGradient(cx, cy, CURSOR_SIZE * 0.3, cx, cy, CURSOR_SIZE)- gradient.add_color_stop_rgba(0, r, g, b, 0.4 * self.opacity)- gradient.add_color_stop_rgba(1, r, g, b, 0)- cr.set_source(gradient)- cr.arc(cx, cy, CURSOR_SIZE, 0, 2 * math.pi)+ # Fallback: simple rectangle+ cr.set_source_rgba(r, g, b, self.opacity)+ cr.rectangle(0, 0, CURSOR_SIZE, CURSOR_SIZE)cr.fill()- # Main circle- base_radius = CURSOR_SIZE * 0.35- click_scale = (- 1.0 + 0.3 * math.sin(self.click_anim_progress * math.pi) if self.clicking else 1.0- )- radius = base_radius * click_scale-- cr.set_source_rgba(r, g, b, 0.9 * self.opacity)- cr.arc(cx, cy, radius, 0, 2 * math.pi)- cr.fill()-- # Inner highlight- cr.set_source_rgba(1, 1, 1, 0.3 * self.opacity)- cr.arc(cx - radius * 0.2, cy - radius * 0.2, radius * 0.4, 0, 2 * math.pi)- cr.fill()-- # Click ripple- if self.clicking and self.click_anim_progress > 0:- ripple_radius = base_radius + (CURSOR_SIZE * 0.8 * self.click_anim_progress)- ripple_alpha = (1 - self.click_anim_progress) * 0.5 * self.opacity- cr.set_source_rgba(r, g, b, ripple_alpha)- cr.set_line_width(2)- cr.arc(cx, cy, ripple_radius, 0, 2 * math.pi)- cr.stroke()-# # Name label (small, below cursor)# cx = CURSOR_SIZE# cy = CURSOR_SIZE@@ -429,7 +374,8 @@ class OverlayCursor(Gtk.Window):point = self.target_path[self.path_index]self.current_x = point[0]self.current_y = point[1]- self.move(int(self.current_x - CURSOR_SIZE), int(self.current_y - CURSOR_SIZE))+ self.move(int(self.current_x), int(self.current_y))+ self._make_click_through() # Re-apply after moveneeds_redraw = Trueif progress >= 1.0:@@ -494,7 +440,8 @@ class OverlayCursor(Gtk.Window):self.current_x = spawn_xself.current_y = spawn_y- self.move(int(spawn_x - CURSOR_SIZE), int(spawn_y - CURSOR_SIZE))+ self.move(int(spawn_x), int(spawn_y))+ self._make_click_through() # Ensure click-through after show# Generate smooth pathself.target_path = generate_path((self.current_x, self.current_y), target)