Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix cursor blocking input, remove start command override for cuabot (#1020)

dduponte71a95019284
diff --git a/libs/cuabot/Dockerfile b/libs/cuabot/Dockerfile
index 2513aaf2b4..b264ff622b 100644
--- a/libs/cuabot/Dockerfile
+++ b/libs/cuabot/Dockerfile
@@ -122,4 +122,4 @@ EXPOSE 10000
USER user
# Note: dbus-daemon runs as session bus (not system) to avoid needing root
-CMD ["sh", "-c", "xpra --version && dbus-daemon --session --fork --address=unix:path=/tmp/dbus-session 2>/dev/null || true && xpra seamless :100 --sharing=yes --no-daemon --bind-tcp=0.0.0.0:10000 --html=on"]
+CMD ["sh", "-c", "xpra --version && dbus-daemon --session --fork --address=unix:path=/tmp/dbus-session 2>/dev/null || true && xpra seamless :100 --sharing=yes --no-daemon --bind-tcp=0.0.0.0:10000 --html=on --dpi=96"]
diff --git a/libs/cuabot/package.json b/libs/cuabot/package.json
index 577f307407..0d494786f6 100644
--- a/libs/cuabot/package.json
+++ b/libs/cuabot/package.json
@@ -17,10 +17,9 @@
"build": "tsc",
"start": "npx tsx src/cuabot.tsx",
"dev": "npx tsx --watch src/cuabot.tsx",
- "serve": "npx tsx src/cuabotd.ts",
- "stop": "npx tsx src/cuabot.tsx stop",
- "status": "npx tsx src/cuabot.tsx status",
- "mcp": "npx tsx src/mcp/server-mcp.ts"
+ "serve": "npx tsx src/cuabot.tsx --serve",
+ "stop": "npx tsx src/cuabot.tsx --stop",
+ "status": "npx tsx src/cuabot.tsx --status"
},
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.1.0",
@@ -47,4 +46,4 @@
"@types/ws": "^8.18.1",
"typescript": "^5.7.0"
}
-}
+}
\ No newline at end of file
diff --git a/libs/cuabot/src/cuabotd.ts b/libs/cuabot/src/cuabotd.ts
index 3995a885e0..c81fd5d6ee 100644
--- a/libs/cuabot/src/cuabotd.ts
+++ b/libs/cuabot/src/cuabotd.ts
@@ -231,7 +231,6 @@ async function ensureContainer(): Promise<number> {
"--add-host=host.docker.internal:host-gateway",
"-v", `${claudeConfigPath}:/home/user/.claude`,
IMAGE_NAME,
- "xpra", "start", ":100", "--bind-tcp=0.0.0.0:10000", "--no-daemon", "--html=on",
];
await execAsync(createCmd.join(" "));
@@ -452,6 +451,9 @@ async function updateCursorMasks(): Promise<void> {
// Process back-to-front so Xpra windows only cut holes in things BEHIND them
const ops: AABBOp[] = [];
+ // Store window info for debug visualization
+ const windowsForDebug: { bounds: AABB; title: string; owner: string; isXpra: boolean }[] = [];
+
// Reverse to go back-to-front (openWindows returns front-to-back)
for (const win of [...windows].reverse()) {
const b = win.bounds;
@@ -464,6 +466,13 @@ async function updateCursorMasks(): Promise<void> {
continue;
}
+ windowsForDebug.push({
+ bounds: { x: b.x, y: b.y, width: b.width, height: b.height },
+ title: win.title,
+ owner: win.owner.name,
+ isXpra,
+ });
+
if (isXpra) {
// Xpra windows: subtract (cut holes in masks behind it)
ops.push({ x: b.x, y: b.y, width: b.width, height: b.height, op: "subtract" });
@@ -485,6 +494,11 @@ async function updateCursorMasks(): Promise<void> {
cx >= r.x && cx <= r.x + r.width && cy >= r.y && cy <= r.y + r.height
);
+ // Debug visualization
+ if (DEBUG) {
+ await drawDebugMasks(windowsForDebug, maskRects, cx, cy, isInMask);
+ }
+
// Send masked state to overlay cursor (not the rects - calculation done here)
await sendCursorCommand({ type: "masked", value: isInMask });
} catch (e) {
@@ -492,6 +506,64 @@ async function updateCursorMasks(): Promise<void> {
}
}
+async function drawDebugMasks(
+ windows: { bounds: AABB; title: string; owner: string; isXpra: boolean }[],
+ maskRects: AABB[],
+ cursorX: number,
+ cursorY: number,
+ isInMask: boolean
+): Promise<void> {
+ try {
+ const width = 2400;
+ const height = 1600;
+ const crosshairSize = 20;
+ const cursorColor = isInMask ? "red" : "lime";
+
+ // Escape XML special characters
+ const escapeXml = (s: string) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+
+ // Build SVG
+ let svg = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">`;
+
+ // Background
+ svg += `<rect x="0" y="0" width="${width}" height="${height}" fill="#1a1a1a"/>`;
+
+ // Draw mask rects (red filled, semi-transparent)
+ for (const r of maskRects) {
+ svg += `<rect x="${r.x}" y="${r.y}" width="${r.width}" height="${r.height}" fill="rgba(255,0,0,0.3)" stroke="red" stroke-width="2"/>`;
+ }
+
+ // Draw window rects (blue outlined) with labels
+ for (const win of windows) {
+ const b = win.bounds;
+ const strokeColor = win.isXpra ? "cyan" : "blue";
+ svg += `<rect x="${b.x}" y="${b.y}" width="${b.width}" height="${b.height}" fill="none" stroke="${strokeColor}" stroke-width="2"/>`;
+
+ // Label background
+ const label = `${escapeXml(win.owner)}: ${escapeXml(win.title.substring(0, 40))}`;
+ svg += `<rect x="${b.x}" y="${b.y - 18}" width="${Math.min(b.width, 300)}" height="18" fill="rgba(0,0,0,0.7)"/>`;
+ svg += `<text x="${b.x + 4}" y="${b.y - 4}" font-family="monospace" font-size="12" fill="${strokeColor}">${label}</text>`;
+ }
+
+ // Draw cursor crosshair
+ svg += `<line x1="${cursorX - crosshairSize}" y1="${cursorY}" x2="${cursorX + crosshairSize}" y2="${cursorY}" stroke="${cursorColor}" stroke-width="3"/>`;
+ svg += `<line x1="${cursorX}" y1="${cursorY - crosshairSize}" x2="${cursorX}" y2="${cursorY + crosshairSize}" stroke="${cursorColor}" stroke-width="3"/>`;
+ svg += `<circle cx="${cursorX}" cy="${cursorY}" r="8" fill="none" stroke="${cursorColor}" stroke-width="2"/>`;
+
+ // Legend
+ svg += `<rect x="10" y="10" width="200" height="80" fill="rgba(0,0,0,0.8)" stroke="white"/>`;
+ svg += `<text x="20" y="30" font-family="monospace" font-size="12" fill="white">Cursor: ${cursorX}, ${cursorY}</text>`;
+ svg += `<text x="20" y="50" font-family="monospace" font-size="12" fill="${cursorColor}">Masked: ${isInMask}</text>`;
+ svg += `<text x="20" y="70" font-family="monospace" font-size="12" fill="cyan">Cyan=Xpra, Blue=Other</text>`;
+
+ svg += `</svg>`;
+
+ await sharp(Buffer.from(svg)).png().toFile(SCREENSHOT_PATH.replace("screenshot.jpg", "masks.png"));
+ } catch (e) {
+ console.error("[debug] Failed to draw masks:", e);
+ }
+}
+
function startMaskPolling(): void {
if (maskPollingInterval) return;
maskPollingInterval = setInterval(updateCursorMasks, 500);
diff --git a/libs/cuabot/src/mcp/computer-use-mcp.py b/libs/cuabot/src/mcp/computer-use-mcp.py
index 759f71b6bd..d64454e143 100644
--- a/libs/cuabot/src/mcp/computer-use-mcp.py
+++ b/libs/cuabot/src/mcp/computer-use-mcp.py
@@ -73,13 +73,21 @@ def request(endpoint: str, body: dict | None = None) -> dict:
@server.tool()
-def screenshot() -> Image:
- """Take a screenshot of the display. Returns JPEG image."""
- import base64
+def screenshot(save_path: str | None = None) -> Image:
+ """Take a screenshot of the display. Returns JPEG image.
- log_mcp_tool_call("screenshot", {})
+ Args:
+ save_path: Optional path to save the screenshot (e.g., "/tmp/screenshot.jpg")
+ """
+ import base64
+ log_mcp_tool_call("screenshot", {"save_path": save_path})
result = request("screenshot")
image_bytes = base64.b64decode(result["image"])
+
+ if save_path:
+ with open(save_path, "wb") as f:
+ f.write(image_bytes)
+
return Image(data=image_bytes, format="jpeg")
diff --git a/libs/cuabot/src/mcp/overlay-cursor.py b/libs/cuabot/src/mcp/overlay-cursor.py
index 3a879f1128..3eddcd4a5e 100644
--- a/libs/cuabot/src/mcp/overlay-cursor.py
+++ b/libs/cuabot/src/mcp/overlay-cursor.py
@@ -180,12 +180,13 @@ class OverlayCursor(Gtk.Window):
else:
print(f"Cursor image not found at {cursor_path}", file=sys.stderr)
- # Window setup (wider/taller to fit cursor and debug label)
- self.set_size_request(CURSOR_SIZE * 5, CURSOR_SIZE * 3)
+ # Window setup - size just big enough for cursor
+ self.set_size_request(CURSOR_SIZE, CURSOR_SIZE)
self.set_decorated(False)
self.set_skip_taskbar_hint(True)
self.set_skip_pager_hint(True)
self.set_accept_focus(False)
+ self.set_can_focus(False)
self.set_app_paintable(True)
self.set_title(f"cuabot-cursor-{name}")
@@ -197,6 +198,7 @@ class OverlayCursor(Gtk.Window):
# Make click-through after window is realized
self.connect("realize", self._on_realize)
+ self.connect("map-event", self._on_map)
# Drawing area
self.drawing_area = Gtk.DrawingArea()
@@ -210,10 +212,19 @@ class OverlayCursor(Gtk.Window):
GLib.timeout_add(1000, self.check_fade)
def _on_realize(self, widget):
- # Make window click-through by setting empty input region
+ self._make_click_through()
+
+ def _on_map(self, widget, event):
+ self._make_click_through()
+ return False
+
+ def _make_click_through(self):
+ """Make window click-through by setting empty input region"""
gdk_window = self.get_window()
if gdk_window:
- gdk_window.input_shape_combine_region(cairo.Region(), 0, 0)
+ # Create an empty region (no input area)
+ empty_region = cairo.Region(cairo.RectangleInt(0, 0, 0, 0))
+ gdk_window.input_shape_combine_region(empty_region, 0, 0)
def on_draw(self, widget, cr):
# Clear background
@@ -260,84 +271,18 @@ class OverlayCursor(Gtk.Window):
return
r, g, b = self.color
+ cr.set_operator(cairo.OPERATOR_OVER)
- # Draw cursor image or fallback circle
if self.cursor_pixbuf:
- # Create a colorized version of the cursor
colorized = self._colorize_pixbuf(self.cursor_pixbuf, r, g, b, self.opacity)
-
- # Click scale effect
- click_scale = (
- 1.0 + 0.2 * math.sin(self.click_anim_progress * math.pi) if self.clicking else 1.0
- )
-
- cr.set_operator(cairo.OPERATOR_OVER)
-
- if click_scale != 1.0:
- # Scale around the cursor tip (top-left corner)
- scaled_size = int(CURSOR_SIZE * click_scale)
- scaled = colorized.scale_simple(
- scaled_size, scaled_size, GdkPixbuf.InterpType.BILINEAR
- )
- Gdk.cairo_set_source_pixbuf(
- cr, scaled, CURSOR_SIZE - CURSOR_SIZE * 0.1, CURSOR_SIZE - CURSOR_SIZE * 0.1
- )
- else:
- Gdk.cairo_set_source_pixbuf(
- cr, colorized, CURSOR_SIZE - CURSOR_SIZE * 0.1, CURSOR_SIZE - CURSOR_SIZE * 0.1
- )
-
+ Gdk.cairo_set_source_pixbuf(cr, colorized, 0, 0)
cr.paint()
-
- # Click ripple
- if self.clicking and self.click_anim_progress > 0:
- cx = CURSOR_SIZE + CURSOR_SIZE * 0.1
- cy = CURSOR_SIZE + CURSOR_SIZE * 0.1
- ripple_radius = CURSOR_SIZE * 0.3 + (CURSOR_SIZE * 0.8 * self.click_anim_progress)
- ripple_alpha = (1 - self.click_anim_progress) * 0.5 * self.opacity
- cr.set_source_rgba(r, g, b, ripple_alpha)
- cr.set_line_width(2)
- cr.arc(cx, cy, ripple_radius, 0, 2 * math.pi)
- cr.stroke()
else:
- # Fallback: draw circle cursor
- cx = CURSOR_SIZE
- cy = CURSOR_SIZE
-
- # Outer glow
- cr.set_operator(cairo.OPERATOR_OVER)
- gradient = cairo.RadialGradient(cx, cy, CURSOR_SIZE * 0.3, cx, cy, CURSOR_SIZE)
- gradient.add_color_stop_rgba(0, r, g, b, 0.4 * self.opacity)
- gradient.add_color_stop_rgba(1, r, g, b, 0)
- cr.set_source(gradient)
- cr.arc(cx, cy, CURSOR_SIZE, 0, 2 * math.pi)
+ # Fallback: simple rectangle
+ cr.set_source_rgba(r, g, b, self.opacity)
+ cr.rectangle(0, 0, CURSOR_SIZE, CURSOR_SIZE)
cr.fill()
- # Main circle
- base_radius = CURSOR_SIZE * 0.35
- click_scale = (
- 1.0 + 0.3 * math.sin(self.click_anim_progress * math.pi) if self.clicking else 1.0
- )
- radius = base_radius * click_scale
-
- cr.set_source_rgba(r, g, b, 0.9 * self.opacity)
- cr.arc(cx, cy, radius, 0, 2 * math.pi)
- cr.fill()
-
- # Inner highlight
- cr.set_source_rgba(1, 1, 1, 0.3 * self.opacity)
- cr.arc(cx - radius * 0.2, cy - radius * 0.2, radius * 0.4, 0, 2 * math.pi)
- cr.fill()
-
- # Click ripple
- if self.clicking and self.click_anim_progress > 0:
- ripple_radius = base_radius + (CURSOR_SIZE * 0.8 * self.click_anim_progress)
- ripple_alpha = (1 - self.click_anim_progress) * 0.5 * self.opacity
- cr.set_source_rgba(r, g, b, ripple_alpha)
- cr.set_line_width(2)
- cr.arc(cx, cy, ripple_radius, 0, 2 * math.pi)
- cr.stroke()
-
# # Name label (small, below cursor)
# cx = CURSOR_SIZE
# cy = CURSOR_SIZE
@@ -429,7 +374,8 @@ class OverlayCursor(Gtk.Window):
point = self.target_path[self.path_index]
self.current_x = point[0]
self.current_y = point[1]
- self.move(int(self.current_x - CURSOR_SIZE), int(self.current_y - CURSOR_SIZE))
+ self.move(int(self.current_x), int(self.current_y))
+ self._make_click_through() # Re-apply after move
needs_redraw = True
if progress >= 1.0:
@@ -494,7 +440,8 @@ class OverlayCursor(Gtk.Window):
self.current_x = spawn_x
self.current_y = spawn_y
- self.move(int(spawn_x - CURSOR_SIZE), int(spawn_y - CURSOR_SIZE))
+ self.move(int(spawn_x), int(spawn_y))
+ self._make_click_through() # Ensure click-through after show
# Generate smooth path
self.target_path = generate_path((self.current_x, self.current_y), target)