Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
#!/usr/bin/env python3
"""
Overlay Agent Cursor - A transparent floating cursor indicator
Runs inside the container and shows where the agent is clicking
Usage:
overlay-cursor.py [--name=NAME] [--color=HEX]
Environment variables:
CUABOT_NAME - Session name (default: cuabot)
CUABOT_COLOR - Hex color without # (default: derived from name)
"""
import json
import math
import os
import random
import socket
import sys
import threading
import time
import gi
gi.require_version("Gtk", "3.0")
gi.require_version("Gdk", "3.0")
gi.require_version("GdkPixbuf", "2.0")
import cairo
from gi.repository import Gdk, GdkPixbuf, GLib, Gtk
# Configuration
SOCKET_PATH = "/tmp/cuabot-overlay-cursor.sock"
CURSOR_SIZE = 32
FADE_DELAY = 15.0 # seconds before fade starts
FADE_DURATION = 1.0 # seconds to fade out
MOVE_DURATION = 0.1 # seconds to move
SPAWN_DISTANCE = 128 # pixels from initial point
# Debug: show masked value and frame counter as a label
SHOW_MASKED_DEBUG = False
def name_to_color(name: str) -> str:
"""Generate a consistent hex color from a name (matches TypeScript version)"""
hash_val = 0
for char in name:
hash_val = ((hash_val << 5) - hash_val) + ord(char)
hash_val = hash_val & 0xFFFFFFFF # Convert to 32-bit integer
# Use golden ratio for nice hue distribution
hue = (abs(hash_val) * 0.618033988749895) % 1
# Convert HSL to RGB (saturation=0.7, lightness=0.5)
s = 0.7
lightness = 0.5
c = (1 - abs(2 * lightness - 1)) * s
x = c * (1 - abs((hue * 6) % 2 - 1))
m = lightness - c / 2
h = hue * 6
if h < 1:
r, g, b = c, x, 0
elif h < 2:
r, g, b = x, c, 0
elif h < 3:
r, g, b = 0, c, x
elif h < 4:
r, g, b = 0, x, c
elif h < 5:
r, g, b = x, 0, c
else:
r, g, b = c, 0, x
def to_hex(n):
return format(round((n + m) * 255), "02x")
return f"{to_hex(r)}{to_hex(g)}{to_hex(b)}"
def hex_to_rgb(hex_color: str) -> tuple:
"""Convert hex color to RGB tuple (0-1 range)"""
hex_color = hex_color.lstrip("#")
return tuple(int(hex_color[i : i + 2], 16) / 255.0 for i in (0, 2, 4))
def generate_path(from_pos: tuple, to_pos: tuple, target_steps: int = 20) -> list:
"""Generate a human-like mouse path with gravity and wind forces."""
s3 = math.sqrt(3)
s5 = math.sqrt(5)
gravity = 9.0
wind = 3.0
max_v = 15.0
damp_dist = 12.0
x, y = float(from_pos[0]), float(from_pos[1])
dx, dy = float(to_pos[0]), float(to_pos[1])
vx = vy = 0.0
wx = wy = 0.0
mv = max_v
path = [(x, y)]
for _ in range(500):
dist = math.hypot(dx - x, dy - y)
if dist < 1:
break
wm = min(wind, dist)
if dist >= damp_dist:
wx = wx / s3 + (2 * random.random() - 1) * wm / s5
wy = wy / s3 + (2 * random.random() - 1) * wm / s5
else:
wx /= s3
wy /= s3
if mv < 3:
mv = random.random() * 3 + 3
else:
mv /= s5
vx += wx + gravity * (dx - x) / dist
vy += wy + gravity * (dy - y) / dist
vm = math.hypot(vx, vy)
if vm > mv:
vc = mv / 2 + random.random() * mv / 2
vx = (vx / vm) * vc
vy = (vy / vm) * vc
x += vx
y += vy
path.append((x, y))
path.append((dx, dy))
if len(path) > target_steps:
idx = [int(i * (len(path) - 1) / target_steps) for i in range(target_steps + 1)]
path = [path[i] for i in idx]
return path
class OverlayCursor(Gtk.Window):
def __init__(self, name: str, color: str):
super().__init__(type=Gtk.WindowType.POPUP)
self.name = name
self.color = hex_to_rgb(color)
self.opacity = 0.0
self.visible = False
self.clicking = False
self.click_anim_progress = 0.0
self.current_x = 0.0
self.current_y = 0.0
self.target_path = []
self.path_index = 0
self.move_start_time = 0
self.last_activity = time.time()
self.fading = False
self.masked = False # True when cursor should be hidden (inside a mask region)
self.frame_counter = 0 # For debug display
# Load cursor image
self.cursor_pixbuf = None
script_dir = os.path.dirname(os.path.abspath(__file__))
cursor_path = os.path.join(script_dir, "cursor.png")
if os.path.exists(cursor_path):
try:
self.cursor_pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(
cursor_path, CURSOR_SIZE, CURSOR_SIZE, True
)
print(f"Loaded cursor image from {cursor_path}", file=sys.stderr)
except Exception as e:
print(f"Failed to load cursor image: {e}", file=sys.stderr)
else:
print(f"Cursor image not found at {cursor_path}", file=sys.stderr)
# Window setup - size just big enough for cursor
self.set_size_request(CURSOR_SIZE, CURSOR_SIZE)
self.set_decorated(False)
self.set_skip_taskbar_hint(True)
self.set_skip_pager_hint(True)
self.set_accept_focus(False)
self.set_can_focus(False)
self.set_app_paintable(True)
self.set_title(f"cuabot-cursor-{name}")
# Enable transparency
screen = self.get_screen()
visual = screen.get_rgba_visual()
if visual:
self.set_visual(visual)
# Make click-through after window is realized
self.connect("realize", self._on_realize)
self.connect("map-event", self._on_map)
# Drawing area
self.drawing_area = Gtk.DrawingArea()
self.drawing_area.connect("draw", self.on_draw)
self.add(self.drawing_area)
# Start animation loop
GLib.timeout_add(16, self.animation_tick) # ~60fps
# Start fade check
GLib.timeout_add(1000, self.check_fade)
def _on_realize(self, widget):
self._make_click_through()
def _on_map(self, widget, event):
self._make_click_through()
return False
def _make_click_through(self):
"""Make window click-through by setting empty input region"""
gdk_window = self.get_window()
if gdk_window:
# Create an empty region (no input area)
empty_region = cairo.Region(cairo.RectangleInt(0, 0, 0, 0))
gdk_window.input_shape_combine_region(empty_region, 0, 0)
def on_draw(self, widget, cr):
# Clear background
cr.set_operator(cairo.OPERATOR_SOURCE)
cr.set_source_rgba(0, 0, 0, 0)
cr.paint()
# Debug label - always show when enabled so we can see updates
if SHOW_MASKED_DEBUG and self.visible:
cr.set_operator(cairo.OPERATOR_OVER)
cr.select_font_face("Sans", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
cr.set_font_size(14)
debug_text = f"masked:{int(self.masked)} frame:{self.frame_counter % 10}"
extents = cr.text_extents(debug_text)
label_x = CURSOR_SIZE * 0.3
label_y = CURSOR_SIZE * 2.5
padding = 6
# Background
cr.set_source_rgba(0, 0, 0, 0.9)
cr.rectangle(
label_x - padding,
label_y - extents.height - padding,
extents.width + padding * 2,
extents.height + padding * 2,
)
cr.fill()
# Text color: red when masked, green when not
if self.masked:
cr.set_source_rgba(1, 0.3, 0.3, 1) # Red
else:
cr.set_source_rgba(0.3, 1, 0.3, 1) # Green
cr.move_to(label_x, label_y)
cr.show_text(debug_text)
if self.opacity <= 0:
return
# Skip rendering cursor if masked (calculation done in cuabotd.ts)
if self.masked:
return
r, g, b = self.color
cr.set_operator(cairo.OPERATOR_OVER)
if self.cursor_pixbuf:
colorized = self._colorize_pixbuf(self.cursor_pixbuf, r, g, b, self.opacity)
Gdk.cairo_set_source_pixbuf(cr, colorized, 0, 0)
cr.paint()
else:
# Fallback: simple rectangle
cr.set_source_rgba(r, g, b, self.opacity)
cr.rectangle(0, 0, CURSOR_SIZE, CURSOR_SIZE)
cr.fill()
# # Name label (small, below cursor)
# cx = CURSOR_SIZE
# cy = CURSOR_SIZE
# if self.name and self.opacity > 0.5:
# cr.select_font_face("Sans", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
# cr.set_font_size(10)
# extents = cr.text_extents(self.name)
# padding = 3
# color_bar_w = 4
# label_x = cx + CURSOR_SIZE * 0.3
# label_y = cy + CURSOR_SIZE * 0.5
# label_w = color_bar_w + extents.width + padding * 3
# label_h = extents.height + padding * 2
# # Black background
# cr.set_source_rgba(0, 0, 0, 0.7 * self.opacity)
# cr.rectangle(label_x, label_y, label_w, label_h)
# cr.fill()
# # Colored accent bar
# cr.set_source_rgba(r, g, b, self.opacity)
# cr.rectangle(label_x, label_y, color_bar_w, label_h)
# cr.fill()
# # Text
# cr.set_source_rgba(1, 1, 1, self.opacity)
# cr.move_to(label_x + color_bar_w + padding, label_y + extents.height + padding - 1)
# cr.show_text(self.name)
def _colorize_pixbuf(self, pixbuf, r, g, b, opacity):
"""Colorize a greyscale RGBA pixbuf by multiplying with the given color"""
# Create a copy to modify
colorized = pixbuf.copy()
width = colorized.get_width()
height = colorized.get_height()
rowstride = colorized.get_rowstride()
n_channels = colorized.get_n_channels()
pixels = colorized.get_pixels()
# Create a new pixel array
new_pixels = bytearray(pixels)
for y in range(height):
for x in range(width):
idx = y * rowstride + x * n_channels
# Get greyscale value (use as intensity)
grey = new_pixels[idx] / 255.0
# Multiply by color
new_pixels[idx] = int(grey * r * 255) # R
new_pixels[idx + 1] = int(grey * g * 255) # G
new_pixels[idx + 2] = int(grey * b * 255) # B
# Apply opacity to alpha channel
if n_channels == 4:
new_pixels[idx + 3] = int(new_pixels[idx + 3] * opacity)
# Create new pixbuf from modified pixels
return GdkPixbuf.Pixbuf.new_from_data(
bytes(new_pixels),
GdkPixbuf.Colorspace.RGB,
n_channels == 4,
8,
width,
height,
rowstride,
)
def animation_tick(self):
needs_redraw = False
self.frame_counter += 1
# Always redraw when debugging to show frame updates
if SHOW_MASKED_DEBUG:
needs_redraw = True
# Handle movement animation
if self.target_path and self.path_index < len(self.target_path):
elapsed = time.time() - self.move_start_time
progress = min(1.0, elapsed / MOVE_DURATION)
# Map progress to path index
target_index = int(progress * (len(self.target_path) - 1))
if target_index > self.path_index:
self.path_index = target_index
point = self.target_path[self.path_index]
self.current_x = point[0]
self.current_y = point[1]
self.move(int(self.current_x), int(self.current_y))
self._make_click_through() # Re-apply after move
needs_redraw = True
if progress >= 1.0:
# Movement complete
self.target_path = []
self.path_index = 0
# Handle click animation
if self.clicking:
self.click_anim_progress += 0.1
if self.click_anim_progress >= 1.0:
self.clicking = False
self.click_anim_progress = 0.0
needs_redraw = True
# Handle fade in
if self.visible and self.opacity < 1.0 and not self.fading:
self.opacity = min(1.0, self.opacity + 0.1)
needs_redraw = True
# Handle fade out
if self.fading:
self.opacity = max(0.0, self.opacity - 0.05)
needs_redraw = True
if self.opacity <= 0:
# Reset state so cursor can reappear on next move
self.fading = False
self.visible = False
self.hide()
if needs_redraw:
self.drawing_area.queue_draw()
return True # Continue timer
def check_fade(self):
if not self.fading and self.visible:
idle_time = time.time() - self.last_activity
if idle_time >= FADE_DELAY:
self.fading = True
return True
def move_to(self, x: int, y: int, click: bool = False):
"""Move cursor to position with smooth animation"""
self.last_activity = time.time()
self.fading = False
if click:
self.clicking = True
self.click_anim_progress = 0.0
target = (float(x), float(y))
if not self.visible:
# First move - spawn from random direction
self.visible = True
self.show_all()
angle = random.random() * 2 * math.pi
spawn_x = x + math.cos(angle) * SPAWN_DISTANCE
spawn_y = y + math.sin(angle) * SPAWN_DISTANCE
self.current_x = spawn_x
self.current_y = spawn_y
self.move(int(spawn_x), int(spawn_y))
self._make_click_through() # Ensure click-through after show
# Generate smooth path
self.target_path = generate_path((self.current_x, self.current_y), target)
self.path_index = 0
self.move_start_time = time.time()
def play_click(self):
"""Play click animation at current position"""
self.last_activity = time.time()
self.fading = False
self.clicking = True
self.click_anim_progress = 0.0
def hide_cursor(self):
"""Immediately hide the cursor"""
self.opacity = 0.0
self.visible = False
self.fading = False
self.hide()
def set_masked(self, masked: bool):
"""Set whether cursor is masked (should be hidden)."""
changed = self.masked != masked
self.masked = masked
# Always queue redraw to update debug label and cursor visibility
# Don't call hide()/show_all() - let on_draw handle the rendering
self.drawing_area.queue_draw()
if changed:
print(f"[overlay] masked={masked}", file=sys.stderr)
class SocketServer(threading.Thread):
def __init__(self, cursor: OverlayCursor):
super().__init__(daemon=True)
self.cursor = cursor
self.running = True
def run(self):
# Remove existing socket
try:
os.unlink(SOCKET_PATH)
except OSError:
pass
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(SOCKET_PATH)
server.listen(5)
os.chmod(SOCKET_PATH, 0o777)
while self.running:
try:
server.settimeout(1.0)
conn, _ = server.accept()
data = conn.recv(4096).decode("utf-8")
conn.close()
for line in data.strip().split("\n"):
if not line:
continue
try:
cmd = json.loads(line)
self.handle_command(cmd)
except json.JSONDecodeError:
pass
except socket.timeout:
continue
except Exception as e:
print(f"Socket error: {e}", file=sys.stderr)
def handle_command(self, cmd: dict):
cmd_type = cmd.get("type")
if cmd_type == "move":
x = cmd.get("x", 0)
y = cmd.get("y", 0)
click = cmd.get("click", False)
GLib.idle_add(self.cursor.move_to, x, y, click)
elif cmd_type == "click":
GLib.idle_add(self.cursor.play_click)
elif cmd_type == "hide":
GLib.idle_add(self.cursor.hide_cursor)
elif cmd_type == "masked":
value = cmd.get("value", False)
GLib.idle_add(self.cursor.set_masked, value)
elif cmd_type == "quit":
GLib.idle_add(Gtk.main_quit)
def send_command(cmd: dict) -> bool:
"""Send command to running overlay cursor"""
try:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(SOCKET_PATH)
sock.send((json.dumps(cmd) + "\n").encode("utf-8"))
sock.close()
return True
except:
return False
def main():
# Parse arguments
name = os.environ.get("CUABOT_NAME", "cuabot")
color = os.environ.get("CUABOT_COLOR", "")
for arg in sys.argv[1:]:
if arg.startswith("--name="):
name = arg.split("=", 1)[1]
elif arg.startswith("--color="):
color = arg.split("=", 1)[1]
# Generate color from name if not provided
if not color:
color = name_to_color(name)
color = color.lstrip("#")
# Check if already running - send move command instead
if os.path.exists(SOCKET_PATH):
# Try to connect
if send_command({"type": "ping"}):
print("Overlay cursor already running", file=sys.stderr)
sys.exit(0)
else:
# Stale socket, remove it
try:
os.unlink(SOCKET_PATH)
except:
pass
print(f"Starting overlay cursor: name={name}, color=#{color}", file=sys.stderr)
# Initialize GTK
Gtk.init(sys.argv)
# Create cursor window
cursor = OverlayCursor(name, color)
# Start socket server
server = SocketServer(cursor)
server.start()
# Run GTK main loop
try:
Gtk.main()
except KeyboardInterrupt:
pass
finally:
server.running = False
try:
os.unlink(SOCKET_PATH)
except:
pass
if __name__ == "__main__":
main()