A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Draw an on-screen Bot cursor where each computer action points
feiz9d10dc9b5daa
diff --git a/app/src/computer-cursor.mjs b/app/src/computer-cursor.mjsnew file mode 100644index 00000000..13e303ef--- /dev/null+++ b/app/src/computer-cursor.mjs@@ -0,0 +1,225 @@+// The on-screen Bot cursor: one small click-through overlay per active Bot that+// draws the Bot's arrow + avatar where its last computer action pointed, with a+// ring ripple on clicks. It fades ~2.5 s after the last event and is destroyed.+// It draws only when the point is genuinely visible: the target window has to+// own the top-most layer-0 window at that point on the current Space — the+// "window at point" check runs in a long-lived helper so reading it stays off+// the main thread and a failure simply means "don't draw".++import { spawn as realSpawn } from 'node:child_process'++const SIZE = 64+const TIP = { x: 7, y: 3 } // where the arrow's tip sits inside the page+const FADE_MS = 2500 // gone this long after the last event+const FADE_AT = FADE_MS - 450 // the CSS fade starts here+const CHECK_MS = 300 // a slow visibility answer means "don't draw"++/**+ * @param {{+ * BrowserWindow: new (options: object) => object,+ * screen: object,+ * platform: string,+ * ownPid?: number,+ * spawn?: typeof realSpawn,+ * log?: Pick<Console, 'warn'>,+ * }} options+ */+export function createComputerCursor({ BrowserWindow, screen, platform, ownPid = process.pid, spawn = realSpawn, log = console }) {+ const windows = new Map()+ const checker = createPointChecker({ platform, ownPid, spawn, log })+ const destroy = (botId) => {+ const held = windows.get(botId)+ if (held === undefined) return+ windows.delete(botId)+ clearTimeout(held.fadeAt)+ clearTimeout(held.goneAt)+ try { held.win.close() } catch {}+ }+ const overlay = (botId) => {+ const held = windows.get(botId)+ if (held !== undefined) return held+ const win = new BrowserWindow({+ width: SIZE, height: SIZE, x: -1000, y: -1000,+ frame: false, transparent: true, alwaysOnTop: true, focusable: false,+ resizable: false, movable: false, minimizable: false, maximizable: false,+ skipTaskbar: true, hasShadow: false, show: false,+ webPreferences: { sandbox: true },+ })+ // Floats over full-screen apps too, not just the regular window level.+ try { win.setAlwaysOnTop(true, 'screen-saver') } catch {}+ try { win.setIgnoreMouseEvents(true, { forward: true }) } catch {}+ try { win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }) } catch {}+ // macOS cua captures a single window, so the Bot never screenshots its own+ // cursor and the user's recordings may show it — no protection here. The+ // driver may capture the whole screen on Windows, so it is hidden there.+ if (platform === 'win32') { try { win.setContentProtection(true) } catch {} }+ win.on('closed', () => { if (windows.get(botId)?.win === win) windows.delete(botId) })+ const held2 = { win, fadeAt: undefined, goneAt: undefined }+ windows.set(botId, held2)+ return held2+ }+ return {+ /** Move the Bot's cursor to a fresh point; hides it when the point is covered. */+ async update(event) {+ if (event === null || typeof event !== 'object' || !Number.isFinite(event.x) || !Number.isFinite(event.y)) return+ const botId = String(event.botId ?? '')+ if (botId === '') return+ const seen = await checker.check(event.x, event.y, event.pid)+ if (!seen) { destroy(botId); return }+ const held = overlay(botId)+ const page = cursorHtml({ color: event.color, avatar: event.avatar, ripple: event.ripple === true })+ try { held.win.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(page)}`) } catch (error) { log.warn(`DS Bot: the cursor did not load: ${error instanceof Error ? error.message : String(error)}`) }+ try { held.win.setBounds({ x: Math.round(event.x - TIP.x), y: Math.round(event.y - TIP.y), width: SIZE, height: SIZE }) } catch {}+ try { if (!held.win.isVisible?.()) held.win.showInactive?.() } catch {}+ clearTimeout(held.fadeAt)+ clearTimeout(held.goneAt)+ held.fadeAt = setTimeout(() => { try { held.win.webContents?.executeJavaScript("document.body.classList.add('fade')") } catch {} }, FADE_AT)+ held.goneAt = setTimeout(() => destroy(botId), FADE_MS)+ },+ dispose() {+ checker.dispose?.()+ for (const botId of [...windows.keys()]) destroy(botId)+ },+ }+}++// The arrow and avatar page. The arrow's tip lands on the reported point; the+// ring ripple replays on every click-type event; .fade dims the whole page out.+export function cursorHtml({ color = '#4d6bfe', avatar, ripple = false } = {}) {+ const safeColor = /^#[0-9a-fA-F]{3,8}$/.test(String(color)) ? color : '#4d6bfe'+ const safeAvatar = typeof avatar === 'string' && /^data:image\/(png|jpeg|webp|svg\+xml);base64,[a-zA-Z0-9+/=]+$/.test(avatar) && avatar.length <= 12000 ? avatar : undefined+ const face = safeAvatar !== undefined+ ? `<img class="ava" src="${safeAvatar}" alt="">`+ : `<span class="ava" style="background:${safeColor}"></span>`+ return `<!doctype html><html><head><meta charset="utf-8"><style>+html,body{margin:0;width:100%;height:100%;background:transparent;overflow:hidden}+body{transition:opacity .4s ease}+body.fade{opacity:0}+svg{position:absolute;left:0;top:0;filter:drop-shadow(0 1px 2px rgba(0,0,0,.35))}+.ava{position:absolute;left:26px;top:26px;width:18px;height:18px;border-radius:50%;border:2px solid #fff;object-fit:cover;box-shadow:0 1px 2px rgba(0,0,0,.3)}+.ring{position:absolute;left:1px;top:-1px;width:14px;height:14px;border-radius:50%;border:2px solid ${safeColor};opacity:.85;animation:ripple .5s ease-out forwards}+@keyframes ripple{from{transform:scale(.6);opacity:.85}to{transform:scale(2.6);opacity:0}}+</style></head><body>+<svg width="${SIZE}" height="${SIZE}" viewBox="0 0 ${SIZE} ${SIZE}">+<path d="M7 3 L7 25 L12 20.5 L15.5 27.5 L18.5 26.2 L15 19.5 L22 19.5 Z" fill="${safeColor}" stroke="#fff" stroke-width="1.6" stroke-linejoin="round"/>+</svg>+${face}+${ripple ? '<span class="ring"></span>' : ''}+</body></html>`+}++// The "is this point really on the target window" check. A long-lived helper+// reads `x y pid` lines and answers y/n; anything over CHECK_MS counts as no.+export function createPointChecker({ platform, ownPid = process.pid, spawn = realSpawn, log = console }) {+ if (platform !== 'darwin' && platform !== 'win32') return { check: async () => false, dispose() {} } // Linux: no on-screen cursor+ let child = null+ let pending = null+ let queue = Promise.resolve()+ const ensure = () => {+ if (child !== null) return child+ const script = platform === 'darwin' ? JXA_CHECKER : PS_CHECKER+ // The script travels on the command line so stdin stays free for the `x y pid`+ // line protocol: osascript takes -e, PowerShell a base64 -EncodedCommand.+ const file = platform === 'darwin' ? 'osascript' : 'powershell'+ const args = platform === 'darwin'+ ? ['-l', 'JavaScript', '-e', script]+ : ['-NoProfile', '-NonInteractive', '-EncodedCommand', Buffer.from(script, 'utf16le').toString('base64')]+ try {+ child = spawn(file, args, { env: { ...process.env, DSBOT_OWN_PID: String(ownPid) }, stdio: ['pipe', 'pipe', 'ignore'] })+ let buffer = ''+ child.stdout.on('data', (chunk) => {+ buffer += chunk+ let nl+ while ((nl = buffer.indexOf('\n')) >= 0) {+ const line = buffer.slice(0, nl).trim()+ buffer = buffer.slice(nl + 1)+ if (line === 'y' || line === 'n') { pending?.resolve(line === 'y'); pending = null }+ }+ })+ child.on('error', (error) => { log.warn(`DS Bot: the point checker failed: ${error instanceof Error ? error.message : String(error)}`); pending?.resolve(false); pending = null })+ child.on('exit', () => { child = null; pending?.resolve(false); pending = null })+ } catch (error) { log.warn(`DS Bot: the point checker did not start: ${error instanceof Error ? error.message : String(error)}`); child = null }+ return child+ }+ const check = (x, y, pid) => {+ const running = ensure()+ // One check in flight at a time keeps the line protocol in order.+ queue = queue.then(() => new Promise((resolve) => {+ if (running === null || !Number.isInteger(pid)) { resolve(false); return }+ const timer = setTimeout(() => { pending = null; resolve(false) }, CHECK_MS)+ pending = { resolve: (value) => { clearTimeout(timer); resolve(value) } }+ try { running.stdin.write(`${Math.round(x)} ${Math.round(y)} ${pid}\n`) } catch { clearTimeout(timer); pending = null; resolve(false) }+ }))+ return queue+ }+ return { check, dispose() { try { child?.kill() } catch {} child = null } }+}++// macOS: walk the on-screen window list front-to-back; the point is visible+// when the first layer-0 window containing it belongs to the target pid. Our+// own overlay windows never block the answer — they are skipped by owner pid.+const JXA_CHECKER = `+ObjC.import('Foundation')+ObjC.import('CoreGraphics')+var own = Number($.NSProcessInfo.processInfo.environment.objectForKey('DSBOT_OWN_PID').js || 0)+var input = $.NSFileHandle.fileHandleWithStandardInput+var output = $.NSFileHandle.fileHandleWithStandardOutput+var buf = ''+function answer(s) {+ var d = $.NSString.alloc.initWithUTF8String(s).dataUsingEncoding($.NSUTF8StringEncoding)+ output.writeData(d)+}+while (true) {+ var data = input.availableData+ if (!data || data.length === 0) break+ buf += $.NSString.alloc.initWithDataEncoding(data, $.NSUTF8StringEncoding).js+ var nl+ while ((nl = buf.indexOf('\\n')) >= 0) {+ var line = buf.slice(0, nl); buf = buf.slice(nl + 1)+ var parts = line.trim().split(/\\s+/)+ var x = Number(parts[0]), y = Number(parts[1]), pid = Number(parts[2])+ var yes = false+ try {+ var list = $.CGWindowListCopyWindowInfo($.kCGWindowListOptionOnScreenOnly, 0)+ var count = list ? list.count : 0+ for (var i = 0; i < count && !yes; i++) {+ var win = list.objectAtIndex(i)+ var owner = Number(win.objectForKey('kCGWindowOwnerPID') || 0)+ if (owner === own) continue+ if (Number(win.objectForKey('kCGWindowLayer') || 0) !== 0) continue+ var b = win.objectForKey('kCGWindowBounds')+ if (!b) continue+ var bx = Number(b.objectForKey('X') || 0), by = Number(b.objectForKey('Y') || 0)+ var bw = Number(b.objectForKey('Width') || 0), bh = Number(b.objectForKey('Height') || 0)+ if (x >= bx && x < bx + bw && y >= by && y < by + bh) { yes = (owner === pid); break }+ }+ } catch (e) {}+ answer(yes ? 'y\\n' : 'n\\n')+ }+}+`++// Windows: the top-most root window at the point decides visibility; the+// click-through overlays are skipped by their own process id.+const PS_CHECKER = String.raw`+Add-Type @"+using System;+using System.Runtime.InteropServices;+public class PointCheck {+ [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(System.Drawing.Point p);+ [DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr hwnd, uint flags);+ [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid);+}+"@+$own = [int]$env:DSBOT_OWN_PID+while (($line = [Console]::In.ReadLine()) -ne $null) {+ $parts = $line.Trim() -split '\s+'+ if ($parts.Count -lt 3) { continue }+ $p = New-Object System.Drawing.Point([int]$parts[0], [int]$parts[1])+ $h = [PointCheck]::WindowFromPoint($p)+ $root = [PointCheck]::GetAncestor($h, 2)+ [uint32]$pid2 = 0+ [void][PointCheck]::GetWindowThreadProcessId($root, [ref]$pid2)+ if ($pid2 -eq $own) { [Console]::Out.WriteLine('y') } elseif ($pid2 -eq [uint32]$parts[2]) { [Console]::Out.WriteLine('y') } else { [Console]::Out.WriteLine('n') }+}+`diff --git a/app/src/main.mjs b/app/src/main.mjsindex c8c98b94..db01f96f 100644--- a/app/src/main.mjs+++ b/app/src/main.mjs@@ -7,6 +7,7 @@// window is up (docs/plans/2026-10-26-start-window.md §11).import { app, BrowserWindow, clipboard, ipcMain, Menu, nativeImage, nativeTheme, powerMonitor, screen, session, shell as electronShell, systemPreferences, Tray, WebContentsView } from 'electron'import { createComputerIndicator, indicatorHtml } from './computer-indicator.mjs'+import { createComputerCursor } from './computer-cursor.mjs'import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'import { readFile, rm } from 'node:fs/promises'import { release } from 'node:os'@@ -43,7 +44,7 @@ const IPC = {appUpdateState: 'ds-bot:app-update-state', appUpdateCheck: 'ds-bot:app-update-check',appUpdateInstall: 'ds-bot:app-update-install', appUpdateOpen: 'ds-bot:app-update-open',appUpdateChanged: 'ds-bot:app-update-changed',- idleSeconds: 'ds-bot:idle-seconds', computer: 'ds-bot:computer', computerAction: 'ds-bot:computer-action',+ idleSeconds: 'ds-bot:idle-seconds', computer: 'ds-bot:computer', computerAction: 'ds-bot:computer-action', computerCursor: 'ds-bot:computer-cursor',}// A cookie export larger than this is not a cookie export.const IMPORT_FILE_MAX = 32 * 1024 * 1024@@ -739,7 +740,12 @@ function handleIpc() {shell.setComputer(list)})ipcMain.handle(IPC.idleSeconds, (event) => (fromWindow(event) ? powerMonitor.getSystemIdleTime() : 0))- app.on('before-quit', () => { computerIndicator.dispose() })+ const computerCursor = createComputerCursor({ BrowserWindow, screen, platform: process.platform, log: logs })+ ipcMain.on(IPC.computerCursor, (event, entry) => {+ if (!fromWindow(event)) return+ void computerCursor.update(entry)+ })+ app.on('before-quit', () => { computerIndicator.dispose(); computerCursor.dispose() })ipcMain.on(IPC.ready, (event, report) => { if (fromWindow(event)) startWindow.ready(report) })// It has drawn the whale the start window hands over: the windows cross-fade.ipcMain.on(IPC.handoffReady, (event) => { if (fromWindow(event)) startWindow.teamReady() })diff --git a/app/src/preload.cjs b/app/src/preload.cjsindex 8e4dec29..21b27bab 100644--- a/app/src/preload.cjs+++ b/app/src/preload.cjs@@ -50,6 +50,7 @@ if (startPage !== undefined) {// indicator's entries; the window answers its buttons.idleSeconds: () => ipcRenderer.invoke('ds-bot:idle-seconds'),computer: (entries) => { ipcRenderer.send('ds-bot:computer', computerEntriesOf(entries)) },+ computerCursor: (event) => { const clean = cursorEventOf(event); if (clean !== undefined) ipcRenderer.send('ds-bot:computer-cursor', clean) },onComputerAction: (run) => {if (typeof run !== 'function') returnipcRenderer.removeAllListeners('ds-bot:computer-action')@@ -70,6 +71,28 @@ if (startPage !== undefined) {}return place}+ // Cursor events carry the Bot's last point: finite numbers, an enum, booleans,+ // bounded strings, a hex color and an optional bounded avatar data URL.+ const cursorEventOf = (value) => {+ if (value === null || typeof value !== 'object') return undefined+ const clean = {}+ for (const key of ['x', 'y']) {+ const number = Number(value[key])+ if (!Number.isFinite(number)) return undefined+ clean[key] = number+ }+ clean.mode = value.mode === 'point' ? 'point' : 'window'+ clean.ripple = value.ripple === true+ clean.botId = String(value.botId ?? '').slice(0, 80)+ if (clean.botId === '') return undefined+ if (typeof value.windowId === 'number' && Number.isFinite(value.windowId)) clean.windowId = value.windowId+ if (Number.isInteger(value.pid)) clean.pid = value.pid+ if (Number.isFinite(value.seq)) clean.seq = value.seq+ if (Number.isFinite(value.t)) clean.t = value.t+ if (typeof value.color === 'string' && /^#[0-9a-fA-F]{3,8}$/.test(value.color)) clean.color = value.color+ if (typeof value.avatar === 'string' && /^data:image\/(png|jpeg|webp|svg\+xml);base64,[a-zA-Z0-9+/=]+$/.test(value.avatar) && value.avatar.length <= 12000) clean.avatar = value.avatar+ return clean+ }const computerEntriesOf = (value) => {const rows = Array.isArray(value?.rows) ? value.rows : []const labels = value?.labels ?? {}diff --git a/app/test/computer-cursor.test.mjs b/app/test/computer-cursor.test.mjsnew file mode 100644index 00000000..37666d59--- /dev/null+++ b/app/test/computer-cursor.test.mjs@@ -0,0 +1,129 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import { createComputerCursor, cursorHtml, createPointChecker } from '../src/computer-cursor.mjs'+import { EventEmitter } from 'node:events'++function fakeWindow() {+ const win = new EventEmitter()+ win.options = null+ win.bounds = null+ win.url = null+ win.js = []+ win.closed = false+ win.shown = false+ win.alwaysOnTopLevel = null+ win.ignored = null+ win.protected = undefined+ win.loadURL = (url) => { win.url = url; return Promise.resolve() }+ win.setBounds = (b) => { win.bounds = b }+ win.showInactive = () => { win.shown = true }+ win.isVisible = () => win.shown+ win.setAlwaysOnTop = (on, level) => { win.alwaysOnTopLevel = level }+ win.setIgnoreMouseEvents = (on, opts) => { win.ignored = on }+ win.setContentProtection = (on) => { win.protected = on }+ win.setVisibleOnAllWorkspaces = () => {}+ win.close = () => { win.closed = true; win.emit('closed') }+ win.webContents = { executeJavaScript: async code => win.js.push(code) }+ return win+}+function fakeElectron() {+ const windows = []+ class BrowserWindow { constructor(options) { const win = fakeWindow(); win.options = options; windows.push(win); return win } }+ return { windows, BrowserWindow }+}+// A fake point-checker child that answers from a list.+function fakeSpawn(answers = ['y']) {+ return () => {+ const child = new EventEmitter()+ child.stdin = { write: (line) => { const a = answers.length > 1 ? answers.shift() : answers[0]; queueMicrotask(() => child.stdout.emit('data', a + '\n')) } }+ child.stdout = new EventEmitter()+ child.kill = () => { child.killed = true; child.emit('exit') }+ return child+ }+}+const screen = { getPrimaryDisplay: () => ({ workAreaSize: { width: 1440, height: 900 }, workArea: { x: 0, y: 0 } }) }+const update = (cursor, ev) => cursor.update({ botId: 'w1', x: 100, y: 200, mode: 'point', ripple: true, pid: 9, color: '#22c55e', ...ev })++test('the cursor window is transparent, click-through and floats above apps', async () => {+ const { windows, BrowserWindow } = fakeElectron()+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(), log: { warn: () => {} } })+ await update(cursor)+ const win = windows[0]+ assert.equal(win.options.transparent, true)+ assert.equal(win.options.frame, false)+ assert.equal(win.options.focusable, false)+ assert.equal(win.options.skipTaskbar, true)+ assert.equal(win.alwaysOnTopLevel, 'screen-saver')+ assert.equal(win.ignored, true)+ assert.equal(win.protected, undefined)+ cursor.dispose()+})++test('the window sits so the arrow tip lands on the point', async () => {+ const { windows, BrowserWindow } = fakeElectron()+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(), log: { warn: () => {} } })+ await update(cursor, { x: 100, y: 200 })+ assert.equal(windows[0].bounds.x, 100 - 7)+ assert.equal(windows[0].bounds.y, 200 - 3)+ cursor.dispose()+})++test('a covered point draws nothing and hides a live cursor', async () => {+ const { windows, BrowserWindow } = fakeElectron()+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(['n']), log: { warn: () => {} } })+ await update(cursor)+ assert.equal(windows.length, 0)+ // then visible once, then covered again → the window is destroyed+ const cursor2 = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: fakeSpawn(['y', 'n']), log: { warn: () => {} } })+ await update(cursor2)+ assert.equal(windows.length, 1)+ await update(cursor2, { pid: 5 })+ assert.equal(windows[0].closed, true)+ cursor2.dispose()+})++test('darwin draws unprotected; win32 hides the cursor from captures', async () => {+ const { windows, BrowserWindow } = fakeElectron()+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'win32', spawn: fakeSpawn(), log: { warn: () => {} } })+ await update(cursor)+ assert.equal(windows[0].protected, true)+ cursor.dispose()+})++test('linux has no on-screen cursor', async () => {+ const { BrowserWindow } = fakeElectron()+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'linux', log: { warn: () => {} } })+ await update(cursor)+ const checker = createPointChecker({ platform: 'linux' })+ assert.equal(await checker.check(10, 10, 1), false)+ cursor.dispose()+})++test('a slow checker answer means do not draw', async () => {+ const { windows, BrowserWindow } = fakeElectron()+ const slow = () => { const child = new EventEmitter(); child.stdin = { write: () => {} }; child.stdout = new EventEmitter(); child.kill = () => {}; return child }+ const cursor = createComputerCursor({ BrowserWindow, screen, platform: 'darwin', spawn: slow, log: { warn: () => {} } })+ await update(cursor)+ assert.equal(windows.length, 0)+ cursor.dispose()+})++test('the checker kills its helper on dispose', () => {+ let killed = false+ const spawn = () => { const child = new EventEmitter(); child.stdin = { write: () => {} }; child.stdout = new EventEmitter(); child.kill = () => { killed = true }; return child }+ const checker = createPointChecker({ platform: 'darwin', spawn, log: { warn: () => {} } })+ void checker.check(1, 2, 3)+ checker.dispose()+ assert.equal(killed, true)+})++test('cursorHtml paints the arrow in the Bot color with the avatar and a ripple', () => {+ const page = cursorHtml({ color: '#22c55e', avatar: 'data:image/png;base64,aGk=', ripple: true })+ assert.match(page, /fill="#22c55e"/)+ assert.match(page, /class="ava"/)+ assert.match(page, /class="ring"/)+ const plain = cursorHtml({ color: 'red', ripple: false })+ assert.match(plain, /fill="#4d6bfe"/)+ assert.doesNotMatch(plain, /class="ring"/)+ assert.match(plain, /background:transparent/)+})diff --git a/ds-bot/client.js b/ds-bot/client.jsindex 0271ee4c..ac0d218e 100644--- a/ds-bot/client.js+++ b/ds-bot/client.js@@ -25320,6 +25320,7 @@ function apply(ctx) {if (typeof appComputer?.computer === "function") {ctx.effect(() => {const nameOf = (id) => roster.getSnapshot().byId[id]?.name ?? "a Bot";+ const seenCursors = /* @__PURE__ */ new Map();const sync = () => {const live2 = computer.getSnapshot();const name = (id) => nameOf(id);@@ -25339,6 +25340,13 @@ function apply(ctx) {rows: rows2,labels: { stop: t("Stop"), stopAll: t("Stop all"), letGo: t("Let it continue") }});+ for (const event of live2.cursors ?? []) {+ if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue;+ seenCursors.set(event.botId, event.seq);+ if (typeof appComputer.computerCursor === "function") {+ appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image });+ }+ }};sync();return computer.subscribe(sync);diff --git a/ds-bot/src/client/computer.js b/ds-bot/src/client/computer.jsindex f2f0c449..e49c9fda 100644--- a/ds-bot/src/client/computer.js+++ b/ds-bot/src/client/computer.js@@ -1,5 +1,5 @@import { createElement as h, useEffect, useId, useRef, useState } from 'react'-import { CheckIcon, MonitorIcon } from './icons.js'+import { CheckIcon } from './icons.js'import { useDockHold } from './card-hold.js'import { heldCardKey } from './sources.js'import { CellEmpty } from './cell-empty.js'diff --git a/ds-bot/src/client/index.js b/ds-bot/src/client/index.jsindex 0972cd87..72642a6a 100644--- a/ds-bot/src/client/index.js+++ b/ds-bot/src/client/index.js@@ -745,6 +745,7 @@ export function apply(ctx) {if (typeof appComputer?.computer === 'function') {ctx.effect(() => {const nameOf = id => roster.getSnapshot().byId[id]?.name ?? 'a Bot'+ const seenCursors = new Map()const sync = () => {const live = computer.getSnapshot()const name = id => nameOf(id)@@ -764,6 +765,14 @@ export function apply(ctx) {rows,labels: { stop: t('Stop'), stopAll: t('Stop all'), letGo: t('Let it continue') },})+ // Fresh cursor events ride the same sync; seq keeps each published once.+ for (const event of live.cursors ?? []) {+ if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue+ seenCursors.set(event.botId, event.seq)+ if (typeof appComputer.computerCursor === 'function') {+ appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image })+ }+ }}sync()return computer.subscribe(sync)diff --git a/ds-bot/src/host/computer/catalog.js b/ds-bot/src/host/computer/catalog.jsindex 5e5d9b29..727e1049 100644--- a/ds-bot/src/host/computer/catalog.js+++ b/ds-bot/src/host/computer/catalog.js@@ -79,6 +79,7 @@ export const DRIVER_GLOBAL = new Set([// lifecycle and the list/permission/health tools stay.export const DRIVER_REFUSED = new Set(['set_config', 'start_recording', 'stop_recording', 'replay_trajectory', 'install_ffmpeg',+ 'set_agent_cursor_enabled', 'set_agent_cursor_motion', 'set_agent_cursor_theme',])// The kind a level admits. 'off' sees nothing.diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.jsindex c6333c1f..84efba95 100644--- a/ds-bot/src/host/computer/index.js+++ b/ds-bot/src/host/computer/index.js@@ -418,6 +418,118 @@ export function install(rt) {if (verdict !== 'allow') return refuse(verdict)return undefined}+ // ---- On-screen Bot cursor: where a just-finished action pointed ----+ // cua reports window bounds in global points and screenshot pixel sizes per+ // window — enough to place a cursor at the point a call used. The driver owns+ // a real cursor inside a per-Bot session; its post-call position is the point.+ const winGeo = new Map()+ const geoMapOf = (sessionId) => {+ let map = winGeo.get(sessionId)+ if (map === undefined) winGeo.set(sessionId, map = new Map())+ return map+ }+ const cursors = new Map()+ let cursorSeq = 0+ const publishCursor = (botId, event) => {+ cursors.set(botId, { botId, seq: ++cursorSeq, t: Date.now(), ...event })+ }+ // Click-type calls ripple; index and keyboard calls only name their window.+ const RIPPLE_METHODS = new Set(['click', 'double_click', 'performSecondaryAction', 'drag'])+ const recordGeo = (sessionId, result) => {+ if (sessionId === undefined || result === null || typeof result !== 'object') return+ const windows = Array.isArray(result.windows) ? result.windows : []+ for (const win of windows) {+ const windowId = win?.windowId ?? win?.id+ const bounds = win?.bounds+ if (windowId === undefined || bounds === null || typeof bounds !== 'object') continue+ if (!Number.isFinite(bounds.x) || !Number.isFinite(bounds.y)) continue+ const geo = geoMapOf(sessionId).get(windowId) ?? {}+ geo.x = bounds.x+ geo.y = bounds.y+ if (Number.isFinite(bounds.width)) geo.w = bounds.width+ if (Number.isFinite(bounds.height)) geo.h = bounds.height+ if (Number.isInteger(win.pid)) geo.pid = win.pid+ geoMapOf(sessionId).set(windowId, geo)+ }+ if (Number.isFinite(result.pixelWidth) && Number.isFinite(result.pixelHeight)) {+ const windowId = result.windowId ?? result.window_id+ if (windowId !== undefined) {+ const geo = geoMapOf(sessionId).get(windowId) ?? {}+ geo.pw = result.pixelWidth+ geo.ph = result.pixelHeight+ geoMapOf(sessionId).set(windowId, geo)+ }+ }+ }+ const emitCursor = (exec, call, result) => {+ if (call.kind !== 'background' && call.kind !== 'foreground') return+ const sessionId = exec.agent?.id+ const botId = sessionId === undefined ? undefined : rt.selfOf(sessionId)+ if (botId === undefined || !rt.isBotSession(sessionId)) return+ if (call.provider === 'cua') {+ const params = exec.arguments?.params ?? {}+ const windowId = params.windowId ?? result?.focusedWindowId ?? result?.windowId+ const win = windowId === undefined ? undefined : geoMapOf(sessionId).get(windowId)+ if (win === undefined || !Number.isFinite(win.x) || !Number.isFinite(win.y) || !Number.isFinite(win.w) || !Number.isFinite(win.h)) return+ const common = { pid: win.pid, windowId }+ const hasPoint = Number.isFinite(params.x) && Number.isFinite(params.y)+ if (hasPoint && params.relative === true) {+ // Thousandths of the window — verified against the real provider.+ publishCursor(botId, { ...common, x: win.x + params.x * win.w / 999, y: win.y + params.y * win.h / 999, mode: 'point', ripple: RIPPLE_METHODS.has(call.method) })+ return+ }+ if (hasPoint && Number.isFinite(win.pw) && win.pw > 0 && Number.isFinite(win.ph) && win.ph > 0) {+ publishCursor(botId, { ...common, x: win.x + params.x * win.w / win.pw, y: win.y + params.y * win.h / win.ph, mode: 'point', ripple: RIPPLE_METHODS.has(call.method) })+ return+ }+ // Index and keyboard calls name no point: sit at the window's title bar so+ // the cursor never pretends to point at a control.+ publishCursor(botId, { ...common, x: win.x + win.w / 2, y: win.y + 14, mode: 'window', ripple: false })+ return+ }+ // The driver's own cursor already moved inside the Bot's session; read where+ // it ended up after the action, without holding the call's result.+ if (call.provider === 'cua-driver') {+ void driverCursorPoint(exec, botId).then(point => {+ if (point !== undefined) publishCursor(botId, { ...point, mode: 'point', ripple: RIPPLE_METHODS.has(call.method) })+ })+ }+ }+ // A stable driver session per Bot, so the driver can keep its cursor hidden+ // for the app (which draws its own) and report where its actions pointed.+ const driverSessionOf = (botId) => {+ let hash = 0+ for (const ch of String(botId)) hash = ((hash << 5) - hash + ch.charCodeAt(0)) | 0+ return `dsbot-${(hash >>> 0).toString(16)}`+ }+ // The registered tool's schema decides whether a session argument is legal —+ // some driver inputs deny unknown fields, so nothing is injected without it.+ const injectDriverSession = (exec, call, botId) => {+ if (call.provider !== 'cua-driver' || DRIVER_REFUSED.has(call.method)) return+ let def+ try { def = ctx.tools?.get?.(exec.name, exec.agent) } catch { return }+ const props = def?.inputSchema?.properties ?? def?.parameters?.properties ?? def?.schema?.properties+ if (props === null || typeof props !== 'object' || !('session' in props)) return+ exec.arguments = { ...(exec.arguments ?? {}), session: driverSessionOf(botId) }+ }+ const driverCursorOff = new Set()+ const ensureDriverCursorOff = async (exec, botId) => {+ const session = driverSessionOf(botId)+ if (driverCursorOff.has(session)) return+ driverCursorOff.add(session)+ try {+ await ctx.tools?.get?.('cua_driver_native__set_agent_cursor_enabled', exec.agent)?.execute({ session, enabled: false }, exec)+ } catch {}+ }+ const driverCursorPoint = async (exec, botId) => {+ try {+ const out = await ctx.tools?.get?.('cua_driver_native__get_agent_cursor_state', exec.agent)?.execute({ session: driverSessionOf(botId) }, exec)+ const value = out !== null && typeof out === 'object' && 'value' in out ? out.value : out+ const position = value?.position+ return position !== null && typeof position === 'object' && Number.isFinite(position.x) && Number.isFinite(position.y) ? { x: position.x, y: position.y } : undefined+ } catch { return undefined }+ }+// The display name the lease and the card show for an appKey, from any session's// records or the key itself.const appNameOf = (key) => {@@ -573,6 +685,8 @@ export function install(rt) {if (DRIVER_REFUSED.has(call.method)) {return { kind: 'deny', reason: `Bots may not use ${call.method}.` }}+ injectDriverSession(exec, call, bot.id)+ await ensureDriverCursorOff(exec, bot.id)if (app === undefined) {if (!DRIVER_GLOBAL.has(call.method)) {return { kind: 'deny', reason: 'Name the window from list_windows first.' }@@ -653,6 +767,8 @@ export function install(rt) {return consent.envelope}}+ recordGeo(sessionId, payload)+ emitCursor(exec, call, payload)const shot = payload !== null && typeof payload === 'object' ? payload.attachment : undefinedif (shot !== undefined) {controlOf(botId).lastShot = shot@@ -1012,6 +1128,7 @@ export function install(rt) {steps: entry.steps,}])),leases: leases.entries().map(entry => ({ ...entry, appName: appNameOf(entry.appKey) })),+ cursors: [...cursors.values()],asks: Object.fromEntries([...pendingAsks.values()].map(ask => [ask.id, {id: ask.id,botId: ask.botId, sessionId: ask.sessionId, appName: ask.appName,diff --git a/ds-bot/test/computer.test.js b/ds-bot/test/computer.test.jsindex 70ee7897..107ce81d 100644--- a/ds-bot/test/computer.test.js+++ b/ds-bot/test/computer.test.js@@ -919,3 +919,83 @@ test('launch_app resolves by its own name or path: consent, protected check, recasync () => ({ pid: 50, name: 'Notes', path: '/Applications/Notes.app', window_id: 9 }))assert.equal(await gate({ name: 'cua_driver_native__click', arguments: { window_id: 9 }, agent: { id: 'w1' }, callId: 'c9' }, allow), 'allowed')})++test('the Bot cursor: relative, pixel and index calls publish a point', async (t) => {+ const { ctx, home, rt } = setup()+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })+ await rt.load()+ await rt.computerEndpoint('set-enabled', { enabled: true })+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')+ const allow = async () => 'allowed'+ rt.botOf('w1').computer = { apps: { 'com.apple.Notes': { name: 'Notes', id: 'com.apple.Notes', grantedAt: 0 } } }+ const cua = (method, params, callId = 'c') => ({ name: 'cua', arguments: { method, params }, agent: { id: 'w1' }, callId })+ // getState records window bounds; a screenshot records that window's pixels+ await track(cua('getState', { appHandle: 'h1' }, 'g1'), async () => ({ isError: false, value: { appHandle: 'h1', focusedWindowId: 7, windows: [{ windowId: 7, pid: 42, bounds: { x: 100, y: 200, width: 300, height: 400 } }] } }))+ await track(cua('screenshot', { appHandle: 'h1', windowId: 7 }, 'g2'), async () => ({ isError: false, value: { windowId: 7, pixelWidth: 600, pixelHeight: 800 } }))+ // relative click: thousandths of the window+ await gate(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)+ await track(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)+ let cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')+ assert.equal(cursor.mode, 'point')+ assert.equal(cursor.ripple, true)+ assert.ok(Math.abs(cursor.x - (100 + 500 * 300 / 999)) < 0.5)+ assert.ok(Math.abs(cursor.y - (200 + 250 * 400 / 999)) < 0.5)+ assert.equal(cursor.pid, 42)+ // pixel coordinates scale by that window's last screenshot size+ await track(cua('click', { appHandle: 'h1', windowId: 7, x: 300, y: 400 }, 'c2'), allow)+ cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')+ assert.ok(Math.abs(cursor.x - 250) < 0.5)+ assert.ok(Math.abs(cursor.y - 400) < 0.5)+ // an element-index click sits at the title bar with no ripple+ await track(cua('click', { appHandle: 'h1', windowId: 7, index: 6 }, 'c3'), allow)+ cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')+ assert.equal(cursor.mode, 'window')+ assert.equal(cursor.ripple, false)+ assert.ok(Math.abs(cursor.x - 250) < 0.5)+ assert.ok(Math.abs(cursor.y - 214) < 0.5)+ // a call on an unknown window publishes nothing+ await track(cua('click', { appHandle: 'h1', windowId: 99, index: 1 }, 'c4'), allow)+ assert.equal(rt.computerLive().cursors.find(event => event.botId === 'w1').mode, 'window')+})++test('driver cursors: session injection, refused cursor tools, one disable per session', async (t) => {+ const { ctx, home, rt } = setup()+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })+ await rt.load()+ await rt.computerEndpoint('set-enabled', { enabled: true })+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')+ const allow = async () => 'allowed'+ const executions = []+ ctx.tools = {+ get: (name) => {+ if (name === 'cua_driver_native__set_agent_cursor_enabled') return { execute: async (args) => { executions.push(['disable', args]); return { value: {} } } }+ if (name === 'cua_driver_native__get_agent_cursor_state') return { execute: async (args) => { executions.push(['state', args]); return { isError: false, value: { position: { x: 311, y: 222 } } } } }+ return { inputSchema: { properties: { session: {}, pid: {} } } }+ },+ }+ rt.botOf('w1').computer = { apps: { 'com.apple.Notes': { name: 'Notes', id: 'com.apple.Notes', grantedAt: 0 } } }+ await track({ name: 'cua_driver_native__list_windows', arguments: {}, agent: { id: 'w1' }, callId: 'list-1' }, async () => ({ windows: [{ pid: 42, name: 'Notes', path: '/Applications/Notes.app', bundleId: 'com.apple.Notes' }] }))+ // a call whose schema carries session gets a stable per-Bot session injected+ const exec = { name: 'cua_driver_native__click', arguments: { pid: 42 }, agent: { id: 'w1' }, callId: 'c1' }+ await gate(exec, allow)+ assert.match(exec.arguments.session, /^dsbot-[0-9a-f]+$/)+ await track(exec, allow)+ // disable ran once for the session; the state query published the point+ assert.equal(executions.filter(e => e[0] === 'disable').length, 1)+ await new Promise(resolve => setImmediate(resolve))+ const cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')+ assert.equal(cursor.x, 311)+ assert.equal(cursor.y, 222)+ assert.equal(cursor.ripple, true)+ // the cursor tools themselves are refused to Bots+ assert.equal((await gate({ name: 'cua_driver_native__set_agent_cursor_enabled', arguments: { session: 'x', enabled: true }, agent: { id: 'w1' }, callId: 'c2' }, allow)).kind, 'deny')+ // a tool whose schema has no session property gets nothing injected+ ctx.tools.get = () => ({ inputSchema: { properties: { pid: {} } } })+ const bare = { name: 'cua_driver_native__click', arguments: { pid: 42 }, agent: { id: 'w1' }, callId: 'c3' }+ await gate(bare, allow)+ assert.equal(bare.arguments.session, undefined)+})