Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Draw an on-screen Bot cursor where each computer action points

feiz9d10dc9b5daa
diff --git a/app/src/computer-cursor.mjs b/app/src/computer-cursor.mjs
new file mode 100644
index 00000000..13e303ef
--- /dev/null
+++ b/app/src/computer-cursor.mjs
@@ -0,0 +1,225 @@
+// The on-screen Bot cursor: one small click-through overlay per active Bot that
+// draws the Bot's arrow + avatar where its last computer action pointed, with a
+// ring ripple on clicks. It fades ~2.5 s after the last event and is destroyed.
+// It draws only when the point is genuinely visible: the target window has to
+// own the top-most layer-0 window at that point on the current Space — the
+// "window at point" check runs in a long-lived helper so reading it stays off
+// the main thread and a failure simply means "don't draw".
+
+import { spawn as realSpawn } from 'node:child_process'
+
+const SIZE = 64
+const TIP = { x: 7, y: 3 } // where the arrow's tip sits inside the page
+const FADE_MS = 2500 // gone this long after the last event
+const FADE_AT = FADE_MS - 450 // the CSS fade starts here
+const CHECK_MS = 300 // a slow visibility answer means "don't draw"
+
+/**
+ * @param {{
+ * BrowserWindow: new (options: object) => object,
+ * screen: object,
+ * platform: string,
+ * ownPid?: number,
+ * spawn?: typeof realSpawn,
+ * log?: Pick<Console, 'warn'>,
+ * }} options
+ */
+export function createComputerCursor({ BrowserWindow, screen, platform, ownPid = process.pid, spawn = realSpawn, log = console }) {
+ const windows = new Map()
+ const checker = createPointChecker({ platform, ownPid, spawn, log })
+ const destroy = (botId) => {
+ const held = windows.get(botId)
+ if (held === undefined) return
+ windows.delete(botId)
+ clearTimeout(held.fadeAt)
+ clearTimeout(held.goneAt)
+ try { held.win.close() } catch {}
+ }
+ const overlay = (botId) => {
+ const held = windows.get(botId)
+ if (held !== undefined) return held
+ const win = new BrowserWindow({
+ width: SIZE, height: SIZE, x: -1000, y: -1000,
+ frame: false, transparent: true, alwaysOnTop: true, focusable: false,
+ resizable: false, movable: false, minimizable: false, maximizable: false,
+ skipTaskbar: true, hasShadow: false, show: false,
+ webPreferences: { sandbox: true },
+ })
+ // Floats over full-screen apps too, not just the regular window level.
+ try { win.setAlwaysOnTop(true, 'screen-saver') } catch {}
+ try { win.setIgnoreMouseEvents(true, { forward: true }) } catch {}
+ try { win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }) } catch {}
+ // macOS cua captures a single window, so the Bot never screenshots its own
+ // cursor and the user's recordings may show it — no protection here. The
+ // driver may capture the whole screen on Windows, so it is hidden there.
+ if (platform === 'win32') { try { win.setContentProtection(true) } catch {} }
+ win.on('closed', () => { if (windows.get(botId)?.win === win) windows.delete(botId) })
+ const held2 = { win, fadeAt: undefined, goneAt: undefined }
+ windows.set(botId, held2)
+ return held2
+ }
+ return {
+ /** Move the Bot's cursor to a fresh point; hides it when the point is covered. */
+ async update(event) {
+ if (event === null || typeof event !== 'object' || !Number.isFinite(event.x) || !Number.isFinite(event.y)) return
+ const botId = String(event.botId ?? '')
+ if (botId === '') return
+ const seen = await checker.check(event.x, event.y, event.pid)
+ if (!seen) { destroy(botId); return }
+ const held = overlay(botId)
+ const page = cursorHtml({ color: event.color, avatar: event.avatar, ripple: event.ripple === true })
+ try { held.win.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(page)}`) } catch (error) { log.warn(`DS Bot: the cursor did not load: ${error instanceof Error ? error.message : String(error)}`) }
+ try { held.win.setBounds({ x: Math.round(event.x - TIP.x), y: Math.round(event.y - TIP.y), width: SIZE, height: SIZE }) } catch {}
+ try { if (!held.win.isVisible?.()) held.win.showInactive?.() } catch {}
+ clearTimeout(held.fadeAt)
+ clearTimeout(held.goneAt)
+ held.fadeAt = setTimeout(() => { try { held.win.webContents?.executeJavaScript("document.body.classList.add('fade')") } catch {} }, FADE_AT)
+ held.goneAt = setTimeout(() => destroy(botId), FADE_MS)
+ },
+ dispose() {
+ checker.dispose?.()
+ for (const botId of [...windows.keys()]) destroy(botId)
+ },
+ }
+}
+
+// The arrow and avatar page. The arrow's tip lands on the reported point; the
+// ring ripple replays on every click-type event; .fade dims the whole page out.
+export function cursorHtml({ color = '#4d6bfe', avatar, ripple = false } = {}) {
+ const safeColor = /^#[0-9a-fA-F]{3,8}$/.test(String(color)) ? color : '#4d6bfe'
+ const safeAvatar = typeof avatar === 'string' && /^data:image\/(png|jpeg|webp|svg\+xml);base64,[a-zA-Z0-9+/=]+$/.test(avatar) && avatar.length <= 12000 ? avatar : undefined
+ const face = safeAvatar !== undefined
+ ? `<img class="ava" src="${safeAvatar}" alt="">`
+ : `<span class="ava" style="background:${safeColor}"></span>`
+ return `<!doctype html><html><head><meta charset="utf-8"><style>
+html,body{margin:0;width:100%;height:100%;background:transparent;overflow:hidden}
+body{transition:opacity .4s ease}
+body.fade{opacity:0}
+svg{position:absolute;left:0;top:0;filter:drop-shadow(0 1px 2px rgba(0,0,0,.35))}
+.ava{position:absolute;left:26px;top:26px;width:18px;height:18px;border-radius:50%;border:2px solid #fff;object-fit:cover;box-shadow:0 1px 2px rgba(0,0,0,.3)}
+.ring{position:absolute;left:1px;top:-1px;width:14px;height:14px;border-radius:50%;border:2px solid ${safeColor};opacity:.85;animation:ripple .5s ease-out forwards}
+@keyframes ripple{from{transform:scale(.6);opacity:.85}to{transform:scale(2.6);opacity:0}}
+</style></head><body>
+<svg width="${SIZE}" height="${SIZE}" viewBox="0 0 ${SIZE} ${SIZE}">
+<path d="M7 3 L7 25 L12 20.5 L15.5 27.5 L18.5 26.2 L15 19.5 L22 19.5 Z" fill="${safeColor}" stroke="#fff" stroke-width="1.6" stroke-linejoin="round"/>
+</svg>
+${face}
+${ripple ? '<span class="ring"></span>' : ''}
+</body></html>`
+}
+
+// The "is this point really on the target window" check. A long-lived helper
+// reads `x y pid` lines and answers y/n; anything over CHECK_MS counts as no.
+export function createPointChecker({ platform, ownPid = process.pid, spawn = realSpawn, log = console }) {
+ if (platform !== 'darwin' && platform !== 'win32') return { check: async () => false, dispose() {} } // Linux: no on-screen cursor
+ let child = null
+ let pending = null
+ let queue = Promise.resolve()
+ const ensure = () => {
+ if (child !== null) return child
+ const script = platform === 'darwin' ? JXA_CHECKER : PS_CHECKER
+ // The script travels on the command line so stdin stays free for the `x y pid`
+ // line protocol: osascript takes -e, PowerShell a base64 -EncodedCommand.
+ const file = platform === 'darwin' ? 'osascript' : 'powershell'
+ const args = platform === 'darwin'
+ ? ['-l', 'JavaScript', '-e', script]
+ : ['-NoProfile', '-NonInteractive', '-EncodedCommand', Buffer.from(script, 'utf16le').toString('base64')]
+ try {
+ child = spawn(file, args, { env: { ...process.env, DSBOT_OWN_PID: String(ownPid) }, stdio: ['pipe', 'pipe', 'ignore'] })
+ let buffer = ''
+ child.stdout.on('data', (chunk) => {
+ buffer += chunk
+ let nl
+ while ((nl = buffer.indexOf('\n')) >= 0) {
+ const line = buffer.slice(0, nl).trim()
+ buffer = buffer.slice(nl + 1)
+ if (line === 'y' || line === 'n') { pending?.resolve(line === 'y'); pending = null }
+ }
+ })
+ child.on('error', (error) => { log.warn(`DS Bot: the point checker failed: ${error instanceof Error ? error.message : String(error)}`); pending?.resolve(false); pending = null })
+ child.on('exit', () => { child = null; pending?.resolve(false); pending = null })
+ } catch (error) { log.warn(`DS Bot: the point checker did not start: ${error instanceof Error ? error.message : String(error)}`); child = null }
+ return child
+ }
+ const check = (x, y, pid) => {
+ const running = ensure()
+ // One check in flight at a time keeps the line protocol in order.
+ queue = queue.then(() => new Promise((resolve) => {
+ if (running === null || !Number.isInteger(pid)) { resolve(false); return }
+ const timer = setTimeout(() => { pending = null; resolve(false) }, CHECK_MS)
+ pending = { resolve: (value) => { clearTimeout(timer); resolve(value) } }
+ try { running.stdin.write(`${Math.round(x)} ${Math.round(y)} ${pid}\n`) } catch { clearTimeout(timer); pending = null; resolve(false) }
+ }))
+ return queue
+ }
+ return { check, dispose() { try { child?.kill() } catch {} child = null } }
+}
+
+// macOS: walk the on-screen window list front-to-back; the point is visible
+// when the first layer-0 window containing it belongs to the target pid. Our
+// own overlay windows never block the answer — they are skipped by owner pid.
+const JXA_CHECKER = `
+ObjC.import('Foundation')
+ObjC.import('CoreGraphics')
+var own = Number($.NSProcessInfo.processInfo.environment.objectForKey('DSBOT_OWN_PID').js || 0)
+var input = $.NSFileHandle.fileHandleWithStandardInput
+var output = $.NSFileHandle.fileHandleWithStandardOutput
+var buf = ''
+function answer(s) {
+ var d = $.NSString.alloc.initWithUTF8String(s).dataUsingEncoding($.NSUTF8StringEncoding)
+ output.writeData(d)
+}
+while (true) {
+ var data = input.availableData
+ if (!data || data.length === 0) break
+ buf += $.NSString.alloc.initWithDataEncoding(data, $.NSUTF8StringEncoding).js
+ var nl
+ while ((nl = buf.indexOf('\\n')) >= 0) {
+ var line = buf.slice(0, nl); buf = buf.slice(nl + 1)
+ var parts = line.trim().split(/\\s+/)
+ var x = Number(parts[0]), y = Number(parts[1]), pid = Number(parts[2])
+ var yes = false
+ try {
+ var list = $.CGWindowListCopyWindowInfo($.kCGWindowListOptionOnScreenOnly, 0)
+ var count = list ? list.count : 0
+ for (var i = 0; i < count && !yes; i++) {
+ var win = list.objectAtIndex(i)
+ var owner = Number(win.objectForKey('kCGWindowOwnerPID') || 0)
+ if (owner === own) continue
+ if (Number(win.objectForKey('kCGWindowLayer') || 0) !== 0) continue
+ var b = win.objectForKey('kCGWindowBounds')
+ if (!b) continue
+ var bx = Number(b.objectForKey('X') || 0), by = Number(b.objectForKey('Y') || 0)
+ var bw = Number(b.objectForKey('Width') || 0), bh = Number(b.objectForKey('Height') || 0)
+ if (x >= bx && x < bx + bw && y >= by && y < by + bh) { yes = (owner === pid); break }
+ }
+ } catch (e) {}
+ answer(yes ? 'y\\n' : 'n\\n')
+ }
+}
+`
+
+// Windows: the top-most root window at the point decides visibility; the
+// click-through overlays are skipped by their own process id.
+const PS_CHECKER = String.raw`
+Add-Type @"
+using System;
+using System.Runtime.InteropServices;
+public class PointCheck {
+ [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(System.Drawing.Point p);
+ [DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr hwnd, uint flags);
+ [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid);
+}
+"@
+$own = [int]$env:DSBOT_OWN_PID
+while (($line = [Console]::In.ReadLine()) -ne $null) {
+ $parts = $line.Trim() -split '\s+'
+ if ($parts.Count -lt 3) { continue }
+ $p = New-Object System.Drawing.Point([int]$parts[0], [int]$parts[1])
+ $h = [PointCheck]::WindowFromPoint($p)
+ $root = [PointCheck]::GetAncestor($h, 2)
+ [uint32]$pid2 = 0
+ [void][PointCheck]::GetWindowThreadProcessId($root, [ref]$pid2)
+ if ($pid2 -eq $own) { [Console]::Out.WriteLine('y') } elseif ($pid2 -eq [uint32]$parts[2]) { [Console]::Out.WriteLine('y') } else { [Console]::Out.WriteLine('n') }
+}
+`
diff --git a/app/src/main.mjs b/app/src/main.mjs
index c8c98b94..db01f96f 100644
--- a/app/src/main.mjs
+++ b/app/src/main.mjs
@@ -7,6 +7,7 @@
// window is up (docs/plans/2026-10-26-start-window.md §11).
import { app, BrowserWindow, clipboard, ipcMain, Menu, nativeImage, nativeTheme, powerMonitor, screen, session, shell as electronShell, systemPreferences, Tray, WebContentsView } from 'electron'
import { createComputerIndicator, indicatorHtml } from './computer-indicator.mjs'
+import { createComputerCursor } from './computer-cursor.mjs'
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
import { readFile, rm } from 'node:fs/promises'
import { release } from 'node:os'
@@ -43,7 +44,7 @@ const IPC = {
appUpdateState: 'ds-bot:app-update-state', appUpdateCheck: 'ds-bot:app-update-check',
appUpdateInstall: 'ds-bot:app-update-install', appUpdateOpen: 'ds-bot:app-update-open',
appUpdateChanged: 'ds-bot:app-update-changed',
- idleSeconds: 'ds-bot:idle-seconds', computer: 'ds-bot:computer', computerAction: 'ds-bot:computer-action',
+ idleSeconds: 'ds-bot:idle-seconds', computer: 'ds-bot:computer', computerAction: 'ds-bot:computer-action', computerCursor: 'ds-bot:computer-cursor',
}
// A cookie export larger than this is not a cookie export.
const IMPORT_FILE_MAX = 32 * 1024 * 1024
@@ -739,7 +740,12 @@ function handleIpc() {
shell.setComputer(list)
})
ipcMain.handle(IPC.idleSeconds, (event) => (fromWindow(event) ? powerMonitor.getSystemIdleTime() : 0))
- app.on('before-quit', () => { computerIndicator.dispose() })
+ const computerCursor = createComputerCursor({ BrowserWindow, screen, platform: process.platform, log: logs })
+ ipcMain.on(IPC.computerCursor, (event, entry) => {
+ if (!fromWindow(event)) return
+ void computerCursor.update(entry)
+ })
+ app.on('before-quit', () => { computerIndicator.dispose(); computerCursor.dispose() })
ipcMain.on(IPC.ready, (event, report) => { if (fromWindow(event)) startWindow.ready(report) })
// It has drawn the whale the start window hands over: the windows cross-fade.
ipcMain.on(IPC.handoffReady, (event) => { if (fromWindow(event)) startWindow.teamReady() })
diff --git a/app/src/preload.cjs b/app/src/preload.cjs
index 8e4dec29..21b27bab 100644
--- a/app/src/preload.cjs
+++ b/app/src/preload.cjs
@@ -50,6 +50,7 @@ if (startPage !== undefined) {
// indicator's entries; the window answers its buttons.
idleSeconds: () => ipcRenderer.invoke('ds-bot:idle-seconds'),
computer: (entries) => { ipcRenderer.send('ds-bot:computer', computerEntriesOf(entries)) },
+ computerCursor: (event) => { const clean = cursorEventOf(event); if (clean !== undefined) ipcRenderer.send('ds-bot:computer-cursor', clean) },
onComputerAction: (run) => {
if (typeof run !== 'function') return
ipcRenderer.removeAllListeners('ds-bot:computer-action')
@@ -70,6 +71,28 @@ if (startPage !== undefined) {
}
return place
}
+ // Cursor events carry the Bot's last point: finite numbers, an enum, booleans,
+ // bounded strings, a hex color and an optional bounded avatar data URL.
+ const cursorEventOf = (value) => {
+ if (value === null || typeof value !== 'object') return undefined
+ const clean = {}
+ for (const key of ['x', 'y']) {
+ const number = Number(value[key])
+ if (!Number.isFinite(number)) return undefined
+ clean[key] = number
+ }
+ clean.mode = value.mode === 'point' ? 'point' : 'window'
+ clean.ripple = value.ripple === true
+ clean.botId = String(value.botId ?? '').slice(0, 80)
+ if (clean.botId === '') return undefined
+ if (typeof value.windowId === 'number' && Number.isFinite(value.windowId)) clean.windowId = value.windowId
+ if (Number.isInteger(value.pid)) clean.pid = value.pid
+ if (Number.isFinite(value.seq)) clean.seq = value.seq
+ if (Number.isFinite(value.t)) clean.t = value.t
+ if (typeof value.color === 'string' && /^#[0-9a-fA-F]{3,8}$/.test(value.color)) clean.color = value.color
+ if (typeof value.avatar === 'string' && /^data:image\/(png|jpeg|webp|svg\+xml);base64,[a-zA-Z0-9+/=]+$/.test(value.avatar) && value.avatar.length <= 12000) clean.avatar = value.avatar
+ return clean
+ }
const computerEntriesOf = (value) => {
const rows = Array.isArray(value?.rows) ? value.rows : []
const labels = value?.labels ?? {}
diff --git a/app/test/computer-cursor.test.mjs b/app/test/computer-cursor.test.mjs
new file mode 100644
index 00000000..37666d59
--- /dev/null
+++ b/app/test/computer-cursor.test.mjs
@@ -0,0 +1,129 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { createComputerCursor, cursorHtml, createPointChecker } from '../src/computer-cursor.mjs'
+import { EventEmitter } from 'node:events'
+
+function fakeWindow() {
+ const win = new EventEmitter()
+ win.options = null
+ win.bounds = null
+ win.url = null
+ win.js = []
+ win.closed = false
+ win.shown = false
+ win.alwaysOnTopLevel = null
+ win.ignored = null
+ win.protected = undefined
+ win.loadURL = (url) => { win.url = url; return Promise.resolve() }
+ win.setBounds = (b) => { win.bounds = b }
+ win.showInactive = () => { win.shown = true }
+ win.isVisible = () => win.shown
+ win.setAlwaysOnTop = (on, level) => { win.alwaysOnTopLevel = level }
+ win.setIgnoreMouseEvents = (on, opts) => { win.ignored = on }
+ win.setContentProtection = (on) => { win.protected = on }
+ win.setVisibleOnAllWorkspaces = () => {}
+ win.close = () => { win.closed = true; win.emit('closed') }
+ win.webContents = { executeJavaScript: async code => win.js.push(code) }
+ return win
+}
+function fakeElectron() {
+ const windows = []
+ class BrowserWindow { constructor(options) { const win = fakeWindow(); win.options = options; windows.push(win); return win } }
+ return { windows, BrowserWindow }
+}
+// A fake point-checker child that answers from a list.
+function fakeSpawn(answers = ['y']) {
+ return () => {
+ const child = new EventEmitter()
+ child.stdin = { write: (line) => { const a = answers.length > 1 ? answers.shift() : answers[0]; queueMicrotask(() => child.stdout.emit('data', a + '\n')) } }
+ child.stdout = new EventEmitter()
+ child.kill = () => { child.killed = true; child.emit('exit') }
+ return child
+ }
+}
+const screen = { getPrimaryDisplay: () => ({ workAreaSize: { width: 1440, height: 900 }, workArea: { x: 0, y: 0 } }) }
+const update = (cursor, ev) => cursor.update({ botId: 'w1', x: 100, y: 200, mode: 'point', ripple: true, pid: 9, color: '#22c55e', ...ev })
+
+test('the cursor window is transparent, click-through and floats above apps', async () => {
+ const { windows, BrowserWindow } = fakeElectron()
+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(), log: { warn: () => {} } })
+ await update(cursor)
+ const win = windows[0]
+ assert.equal(win.options.transparent, true)
+ assert.equal(win.options.frame, false)
+ assert.equal(win.options.focusable, false)
+ assert.equal(win.options.skipTaskbar, true)
+ assert.equal(win.alwaysOnTopLevel, 'screen-saver')
+ assert.equal(win.ignored, true)
+ assert.equal(win.protected, undefined)
+ cursor.dispose()
+})
+
+test('the window sits so the arrow tip lands on the point', async () => {
+ const { windows, BrowserWindow } = fakeElectron()
+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(), log: { warn: () => {} } })
+ await update(cursor, { x: 100, y: 200 })
+ assert.equal(windows[0].bounds.x, 100 - 7)
+ assert.equal(windows[0].bounds.y, 200 - 3)
+ cursor.dispose()
+})
+
+test('a covered point draws nothing and hides a live cursor', async () => {
+ const { windows, BrowserWindow } = fakeElectron()
+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(['n']), log: { warn: () => {} } })
+ await update(cursor)
+ assert.equal(windows.length, 0)
+ // then visible once, then covered again → the window is destroyed
+ const cursor2 = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(['y', 'n']), log: { warn: () => {} } })
+ await update(cursor2)
+ assert.equal(windows.length, 1)
+ await update(cursor2, { pid: 5 })
+ assert.equal(windows[0].closed, true)
+ cursor2.dispose()
+})
+
+test('darwin draws unprotected; win32 hides the cursor from captures', async () => {
+ const { windows, BrowserWindow } = fakeElectron()
+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'win32', spawn: fakeSpawn(), log: { warn: () => {} } })
+ await update(cursor)
+ assert.equal(windows[0].protected, true)
+ cursor.dispose()
+})
+
+test('linux has no on-screen cursor', async () => {
+ const { BrowserWindow } = fakeElectron()
+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'linux', log: { warn: () => {} } })
+ await update(cursor)
+ const checker = createPointChecker({ platform: 'linux' })
+ assert.equal(await checker.check(10, 10, 1), false)
+ cursor.dispose()
+})
+
+test('a slow checker answer means do not draw', async () => {
+ const { windows, BrowserWindow } = fakeElectron()
+ const slow = () => { const child = new EventEmitter(); child.stdin = { write: () => {} }; child.stdout = new EventEmitter(); child.kill = () => {}; return child }
+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: slow, log: { warn: () => {} } })
+ await update(cursor)
+ assert.equal(windows.length, 0)
+ cursor.dispose()
+})
+
+test('the checker kills its helper on dispose', () => {
+ let killed = false
+ const spawn = () => { const child = new EventEmitter(); child.stdin = { write: () => {} }; child.stdout = new EventEmitter(); child.kill = () => { killed = true }; return child }
+ const checker = createPointChecker({ platform: 'darwin', spawn, log: { warn: () => {} } })
+ void checker.check(1, 2, 3)
+ checker.dispose()
+ assert.equal(killed, true)
+})
+
+test('cursorHtml paints the arrow in the Bot color with the avatar and a ripple', () => {
+ const page = cursorHtml({ color: '#22c55e', avatar: 'data:image/png;base64,aGk=', ripple: true })
+ assert.match(page, /fill="#22c55e"/)
+ assert.match(page, /class="ava"/)
+ assert.match(page, /class="ring"/)
+ const plain = cursorHtml({ color: 'red', ripple: false })
+ assert.match(plain, /fill="#4d6bfe"/)
+ assert.doesNotMatch(plain, /class="ring"/)
+ assert.match(plain, /background:transparent/)
+})
diff --git a/ds-bot/client.js b/ds-bot/client.js
index 0271ee4c..ac0d218e 100644
--- a/ds-bot/client.js
+++ b/ds-bot/client.js
@@ -25320,6 +25320,7 @@ function apply(ctx) {
if (typeof appComputer?.computer === "function") {
ctx.effect(() => {
const nameOf = (id) => roster.getSnapshot().byId[id]?.name ?? "a Bot";
+ const seenCursors = /* @__PURE__ */ new Map();
const sync = () => {
const live2 = computer.getSnapshot();
const name = (id) => nameOf(id);
@@ -25339,6 +25340,13 @@ function apply(ctx) {
rows: rows2,
labels: { stop: t("Stop"), stopAll: t("Stop all"), letGo: t("Let it continue") }
});
+ for (const event of live2.cursors ?? []) {
+ if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue;
+ seenCursors.set(event.botId, event.seq);
+ if (typeof appComputer.computerCursor === "function") {
+ appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image });
+ }
+ }
};
sync();
return computer.subscribe(sync);
diff --git a/ds-bot/src/client/computer.js b/ds-bot/src/client/computer.js
index f2f0c449..e49c9fda 100644
--- a/ds-bot/src/client/computer.js
+++ b/ds-bot/src/client/computer.js
@@ -1,5 +1,5 @@
import { createElement as h, useEffect, useId, useRef, useState } from 'react'
-import { CheckIcon, MonitorIcon } from './icons.js'
+import { CheckIcon } from './icons.js'
import { useDockHold } from './card-hold.js'
import { heldCardKey } from './sources.js'
import { CellEmpty } from './cell-empty.js'
diff --git a/ds-bot/src/client/index.js b/ds-bot/src/client/index.js
index 0972cd87..72642a6a 100644
--- a/ds-bot/src/client/index.js
+++ b/ds-bot/src/client/index.js
@@ -745,6 +745,7 @@ export function apply(ctx) {
if (typeof appComputer?.computer === 'function') {
ctx.effect(() => {
const nameOf = id => roster.getSnapshot().byId[id]?.name ?? 'a Bot'
+ const seenCursors = new Map()
const sync = () => {
const live = computer.getSnapshot()
const name = id => nameOf(id)
@@ -764,6 +765,14 @@ export function apply(ctx) {
rows,
labels: { stop: t('Stop'), stopAll: t('Stop all'), letGo: t('Let it continue') },
})
+ // Fresh cursor events ride the same sync; seq keeps each published once.
+ for (const event of live.cursors ?? []) {
+ if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue
+ seenCursors.set(event.botId, event.seq)
+ if (typeof appComputer.computerCursor === 'function') {
+ appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image })
+ }
+ }
}
sync()
return computer.subscribe(sync)
diff --git a/ds-bot/src/host/computer/catalog.js b/ds-bot/src/host/computer/catalog.js
index 5e5d9b29..727e1049 100644
--- a/ds-bot/src/host/computer/catalog.js
+++ b/ds-bot/src/host/computer/catalog.js
@@ -79,6 +79,7 @@ export const DRIVER_GLOBAL = new Set([
// lifecycle and the list/permission/health tools stay.
export const DRIVER_REFUSED = new Set([
'set_config', 'start_recording', 'stop_recording', 'replay_trajectory', 'install_ffmpeg',
+ 'set_agent_cursor_enabled', 'set_agent_cursor_motion', 'set_agent_cursor_theme',
])
// The kind a level admits. 'off' sees nothing.
diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.js
index c6333c1f..84efba95 100644
--- a/ds-bot/src/host/computer/index.js
+++ b/ds-bot/src/host/computer/index.js
@@ -418,6 +418,118 @@ export function install(rt) {
if (verdict !== 'allow') return refuse(verdict)
return undefined
}
+ // ---- On-screen Bot cursor: where a just-finished action pointed ----
+ // cua reports window bounds in global points and screenshot pixel sizes per
+ // window — enough to place a cursor at the point a call used. The driver owns
+ // a real cursor inside a per-Bot session; its post-call position is the point.
+ const winGeo = new Map()
+ const geoMapOf = (sessionId) => {
+ let map = winGeo.get(sessionId)
+ if (map === undefined) winGeo.set(sessionId, map = new Map())
+ return map
+ }
+ const cursors = new Map()
+ let cursorSeq = 0
+ const publishCursor = (botId, event) => {
+ cursors.set(botId, { botId, seq: ++cursorSeq, t: Date.now(), ...event })
+ }
+ // Click-type calls ripple; index and keyboard calls only name their window.
+ const RIPPLE_METHODS = new Set(['click', 'double_click', 'performSecondaryAction', 'drag'])
+ const recordGeo = (sessionId, result) => {
+ if (sessionId === undefined || result === null || typeof result !== 'object') return
+ const windows = Array.isArray(result.windows) ? result.windows : []
+ for (const win of windows) {
+ const windowId = win?.windowId ?? win?.id
+ const bounds = win?.bounds
+ if (windowId === undefined || bounds === null || typeof bounds !== 'object') continue
+ if (!Number.isFinite(bounds.x) || !Number.isFinite(bounds.y)) continue
+ const geo = geoMapOf(sessionId).get(windowId) ?? {}
+ geo.x = bounds.x
+ geo.y = bounds.y
+ if (Number.isFinite(bounds.width)) geo.w = bounds.width
+ if (Number.isFinite(bounds.height)) geo.h = bounds.height
+ if (Number.isInteger(win.pid)) geo.pid = win.pid
+ geoMapOf(sessionId).set(windowId, geo)
+ }
+ if (Number.isFinite(result.pixelWidth) && Number.isFinite(result.pixelHeight)) {
+ const windowId = result.windowId ?? result.window_id
+ if (windowId !== undefined) {
+ const geo = geoMapOf(sessionId).get(windowId) ?? {}
+ geo.pw = result.pixelWidth
+ geo.ph = result.pixelHeight
+ geoMapOf(sessionId).set(windowId, geo)
+ }
+ }
+ }
+ const emitCursor = (exec, call, result) => {
+ if (call.kind !== 'background' && call.kind !== 'foreground') return
+ const sessionId = exec.agent?.id
+ const botId = sessionId === undefined ? undefined : rt.selfOf(sessionId)
+ if (botId === undefined || !rt.isBotSession(sessionId)) return
+ if (call.provider === 'cua') {
+ const params = exec.arguments?.params ?? {}
+ const windowId = params.windowId ?? result?.focusedWindowId ?? result?.windowId
+ const win = windowId === undefined ? undefined : geoMapOf(sessionId).get(windowId)
+ if (win === undefined || !Number.isFinite(win.x) || !Number.isFinite(win.y) || !Number.isFinite(win.w) || !Number.isFinite(win.h)) return
+ const common = { pid: win.pid, windowId }
+ const hasPoint = Number.isFinite(params.x) && Number.isFinite(params.y)
+ if (hasPoint && params.relative === true) {
+ // Thousandths of the window — verified against the real provider.
+ publishCursor(botId, { ...common, x: win.x + params.x * win.w / 999, y: win.y + params.y * win.h / 999, mode: 'point', ripple: RIPPLE_METHODS.has(call.method) })
+ return
+ }
+ if (hasPoint && Number.isFinite(win.pw) && win.pw > 0 && Number.isFinite(win.ph) && win.ph > 0) {
+ publishCursor(botId, { ...common, x: win.x + params.x * win.w / win.pw, y: win.y + params.y * win.h / win.ph, mode: 'point', ripple: RIPPLE_METHODS.has(call.method) })
+ return
+ }
+ // Index and keyboard calls name no point: sit at the window's title bar so
+ // the cursor never pretends to point at a control.
+ publishCursor(botId, { ...common, x: win.x + win.w / 2, y: win.y + 14, mode: 'window', ripple: false })
+ return
+ }
+ // The driver's own cursor already moved inside the Bot's session; read where
+ // it ended up after the action, without holding the call's result.
+ if (call.provider === 'cua-driver') {
+ void driverCursorPoint(exec, botId).then(point => {
+ if (point !== undefined) publishCursor(botId, { ...point, mode: 'point', ripple: RIPPLE_METHODS.has(call.method) })
+ })
+ }
+ }
+ // A stable driver session per Bot, so the driver can keep its cursor hidden
+ // for the app (which draws its own) and report where its actions pointed.
+ const driverSessionOf = (botId) => {
+ let hash = 0
+ for (const ch of String(botId)) hash = ((hash << 5) - hash + ch.charCodeAt(0)) | 0
+ return `dsbot-${(hash >>> 0).toString(16)}`
+ }
+ // The registered tool's schema decides whether a session argument is legal —
+ // some driver inputs deny unknown fields, so nothing is injected without it.
+ const injectDriverSession = (exec, call, botId) => {
+ if (call.provider !== 'cua-driver' || DRIVER_REFUSED.has(call.method)) return
+ let def
+ try { def = ctx.tools?.get?.(exec.name, exec.agent) } catch { return }
+ const props = def?.inputSchema?.properties ?? def?.parameters?.properties ?? def?.schema?.properties
+ if (props === null || typeof props !== 'object' || !('session' in props)) return
+ exec.arguments = { ...(exec.arguments ?? {}), session: driverSessionOf(botId) }
+ }
+ const driverCursorOff = new Set()
+ const ensureDriverCursorOff = async (exec, botId) => {
+ const session = driverSessionOf(botId)
+ if (driverCursorOff.has(session)) return
+ driverCursorOff.add(session)
+ try {
+ await ctx.tools?.get?.('cua_driver_native__set_agent_cursor_enabled', exec.agent)?.execute({ session, enabled: false }, exec)
+ } catch {}
+ }
+ const driverCursorPoint = async (exec, botId) => {
+ try {
+ const out = await ctx.tools?.get?.('cua_driver_native__get_agent_cursor_state', exec.agent)?.execute({ session: driverSessionOf(botId) }, exec)
+ const value = out !== null && typeof out === 'object' && 'value' in out ? out.value : out
+ const position = value?.position
+ return position !== null && typeof position === 'object' && Number.isFinite(position.x) && Number.isFinite(position.y) ? { x: position.x, y: position.y } : undefined
+ } catch { return undefined }
+ }
+
// The display name the lease and the card show for an appKey, from any session's
// records or the key itself.
const appNameOf = (key) => {
@@ -573,6 +685,8 @@ export function install(rt) {
if (DRIVER_REFUSED.has(call.method)) {
return { kind: 'deny', reason: `Bots may not use ${call.method}.` }
}
+ injectDriverSession(exec, call, bot.id)
+ await ensureDriverCursorOff(exec, bot.id)
if (app === undefined) {
if (!DRIVER_GLOBAL.has(call.method)) {
return { kind: 'deny', reason: 'Name the window from list_windows first.' }
@@ -653,6 +767,8 @@ export function install(rt) {
return consent.envelope
}
}
+ recordGeo(sessionId, payload)
+ emitCursor(exec, call, payload)
const shot = payload !== null && typeof payload === 'object' ? payload.attachment : undefined
if (shot !== undefined) {
controlOf(botId).lastShot = shot
@@ -1012,6 +1128,7 @@ export function install(rt) {
steps: entry.steps,
}])),
leases: leases.entries().map(entry => ({ ...entry, appName: appNameOf(entry.appKey) })),
+ cursors: [...cursors.values()],
asks: Object.fromEntries([...pendingAsks.values()].map(ask => [ask.id, {
id: ask.id,
botId: ask.botId, sessionId: ask.sessionId, appName: ask.appName,
diff --git a/ds-bot/test/computer.test.js b/ds-bot/test/computer.test.js
index 70ee7897..107ce81d 100644
--- a/ds-bot/test/computer.test.js
+++ b/ds-bot/test/computer.test.js
@@ -919,3 +919,83 @@ test('launch_app resolves by its own name or path: consent, protected check, rec
async () => ({ pid: 50, name: 'Notes', path: '/Applications/Notes.app', window_id: 9 }))
assert.equal(await gate({ name: 'cua_driver_native__click', arguments: { window_id: 9 }, agent: { id: 'w1' }, callId: 'c9' }, allow), 'allowed')
})
+
+test('the Bot cursor: relative, pixel and index calls publish a point', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ const allow = async () => 'allowed'
+ rt.botOf('w1').computer = { apps: { 'com.apple.Notes': { name: 'Notes', id: 'com.apple.Notes', grantedAt: 0 } } }
+ const cua = (method, params, callId = 'c') => ({ name: 'cua', arguments: { method, params }, agent: { id: 'w1' }, callId })
+ // getState records window bounds; a screenshot records that window's pixels
+ await track(cua('getState', { appHandle: 'h1' }, 'g1'), async () => ({ isError: false, value: { appHandle: 'h1', focusedWindowId: 7, windows: [{ windowId: 7, pid: 42, bounds: { x: 100, y: 200, width: 300, height: 400 } }] } }))
+ await track(cua('screenshot', { appHandle: 'h1', windowId: 7 }, 'g2'), async () => ({ isError: false, value: { windowId: 7, pixelWidth: 600, pixelHeight: 800 } }))
+ // relative click: thousandths of the window
+ await gate(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)
+ await track(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)
+ let cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
+ assert.equal(cursor.mode, 'point')
+ assert.equal(cursor.ripple, true)
+ assert.ok(Math.abs(cursor.x - (100 + 500 * 300 / 999)) < 0.5)
+ assert.ok(Math.abs(cursor.y - (200 + 250 * 400 / 999)) < 0.5)
+ assert.equal(cursor.pid, 42)
+ // pixel coordinates scale by that window's last screenshot size
+ await track(cua('click', { appHandle: 'h1', windowId: 7, x: 300, y: 400 }, 'c2'), allow)
+ cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
+ assert.ok(Math.abs(cursor.x - 250) < 0.5)
+ assert.ok(Math.abs(cursor.y - 400) < 0.5)
+ // an element-index click sits at the title bar with no ripple
+ await track(cua('click', { appHandle: 'h1', windowId: 7, index: 6 }, 'c3'), allow)
+ cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
+ assert.equal(cursor.mode, 'window')
+ assert.equal(cursor.ripple, false)
+ assert.ok(Math.abs(cursor.x - 250) < 0.5)
+ assert.ok(Math.abs(cursor.y - 214) < 0.5)
+ // a call on an unknown window publishes nothing
+ await track(cua('click', { appHandle: 'h1', windowId: 99, index: 1 }, 'c4'), allow)
+ assert.equal(rt.computerLive().cursors.find(event => event.botId === 'w1').mode, 'window')
+})
+
+test('driver cursors: session injection, refused cursor tools, one disable per session', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ const allow = async () => 'allowed'
+ const executions = []
+ ctx.tools = {
+ get: (name) => {
+ if (name === 'cua_driver_native__set_agent_cursor_enabled') return { execute: async (args) => { executions.push(['disable', args]); return { value: {} } } }
+ if (name === 'cua_driver_native__get_agent_cursor_state') return { execute: async (args) => { executions.push(['state', args]); return { isError: false, value: { position: { x: 311, y: 222 } } } } }
+ return { inputSchema: { properties: { session: {}, pid: {} } } }
+ },
+ }
+ rt.botOf('w1').computer = { apps: { 'com.apple.Notes': { name: 'Notes', id: 'com.apple.Notes', grantedAt: 0 } } }
+ await track({ name: 'cua_driver_native__list_windows', arguments: {}, agent: { id: 'w1' }, callId: 'list-1' }, async () => ({ windows: [{ pid: 42, name: 'Notes', path: '/Applications/Notes.app', bundleId: 'com.apple.Notes' }] }))
+ // a call whose schema carries session gets a stable per-Bot session injected
+ const exec = { name: 'cua_driver_native__click', arguments: { pid: 42 }, agent: { id: 'w1' }, callId: 'c1' }
+ await gate(exec, allow)
+ assert.match(exec.arguments.session, /^dsbot-[0-9a-f]+$/)
+ await track(exec, allow)
+ // disable ran once for the session; the state query published the point
+ assert.equal(executions.filter(e => e[0] === 'disable').length, 1)
+ await new Promise(resolve => setImmediate(resolve))
+ const cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
+ assert.equal(cursor.x, 311)
+ assert.equal(cursor.y, 222)
+ assert.equal(cursor.ripple, true)
+ // the cursor tools themselves are refused to Bots
+ assert.equal((await gate({ name: 'cua_driver_native__set_agent_cursor_enabled', arguments: { session: 'x', enabled: true }, agent: { id: 'w1' }, callId: 'c2' }, allow)).kind, 'deny')
+ // a tool whose schema has no session property gets nothing injected
+ ctx.tools.get = () => ({ inputSchema: { properties: { pid: {} } } })
+ const bare = { name: 'cua_driver_native__click', arguments: { pid: 42 }, agent: { id: 'w1' }, callId: 'c3' }
+ await gate(bare, allow)
+ assert.equal(bare.arguments.session, undefined)
+})