Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Deliver cursor events over a long-poll and paint the avatar on the tail

feiz80b77317f206
diff --git a/app/src/computer-cursor.mjs b/app/src/computer-cursor.mjs
index 79f5a53a..f187e8a0 100644
--- a/app/src/computer-cursor.mjs
+++ b/app/src/computer-cursor.mjs
@@ -96,7 +96,7 @@ html,body{margin:0;width:100%;height:100%;background:transparent;overflow:hidden
body{transition:opacity .4s ease}
body.fade{opacity:0}
svg{position:absolute;left:0;top:0;filter:drop-shadow(0 1px 2px rgba(0,0,0,.35))}
-.ava{position:absolute;left:26px;top:26px;width:18px;height:18px;border-radius:50%;border:2px solid #fff;object-fit:cover;box-shadow:0 1px 2px rgba(0,0,0,.3)}
+.ava{position:absolute;left:13px;top:16px;width:18px;height:18px;border-radius:50%;border:2px solid #fff;object-fit:cover;box-shadow:0 1px 2px rgba(0,0,0,.3)}
.ring{position:absolute;left:1px;top:-1px;width:14px;height:14px;border-radius:50%;border:2px solid ${safeColor};opacity:.85;animation:ripple .5s ease-out forwards}
@keyframes ripple{from{transform:scale(.6);opacity:.85}to{transform:scale(2.6);opacity:0}}
</style></head><body>
@@ -158,7 +158,7 @@ export function createPointChecker({ platform, ownPid = process.pid, spawn = rea
// macOS: walk the on-screen window list front-to-back; the point is visible
// when the first layer-0 window containing it belongs to the target pid. Our
// own overlay windows never block the answer — they are skipped by owner pid.
-const JXA_CHECKER = `
+const JXA_CHECKER = String.raw`
ObjC.import('Foundation')
ObjC.import('CoreGraphics')
var input = $.NSFileHandle.fileHandleWithStandardInput
diff --git a/ds-bot/client.js b/ds-bot/client.js
index b19d5b68..0017de49 100644
--- a/ds-bot/client.js
+++ b/ds-bot/client.js
@@ -25315,7 +25315,6 @@ function apply(ctx) {
if (typeof appComputer?.computer === "function") {
ctx.effect(() => {
const nameOf = (id) => roster.getSnapshot().byId[id]?.name ?? "a Bot";
- const seenCursors = /* @__PURE__ */ new Map();
const sync = () => {
const live2 = computer.getSnapshot();
const name = (id) => nameOf(id);
@@ -25335,19 +25334,40 @@ function apply(ctx) {
rows: rows2,
labels: { stop: t("Stop"), stopAll: t("Stop all"), letGo: t("Let it continue") }
});
- for (const event of live2.cursors ?? []) {
- if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue;
- if (Number.isFinite(event.t) && Date.now() - event.t > 3e3) continue;
- seenCursors.set(event.botId, event.seq);
- if (typeof appComputer.computerCursor === "function") {
- appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image });
- }
- }
};
sync();
return computer.subscribe(sync);
}, "ds-bot: computer indicator");
}
+ if (typeof appComputer?.computerCursor === "function") {
+ ctx.effect(() => {
+ let dead = false;
+ const pause = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
+ const run2 = async () => {
+ let after2 = 0;
+ while (!dead) {
+ if (roster.getSnapshot().computer?.enabled !== true) {
+ await pause(2e3);
+ continue;
+ }
+ try {
+ const reply = await call("computer-cursor-wait", { after: after2 });
+ for (const event of reply?.events ?? []) {
+ if (Number.isFinite(event.seq) && event.seq > after2) after2 = event.seq;
+ if (Number.isFinite(event.t) && Number.isFinite(reply?.now) && reply.now - event.t > 3e3) continue;
+ appComputer.computerCursor({ ...event, color: lookOf(roster.getSnapshot().byId[event.botId]).color, avatar: lookOf(roster.getSnapshot().byId[event.botId]).image });
+ }
+ } catch {
+ await pause(2e3);
+ }
+ }
+ };
+ void run2();
+ return () => {
+ dead = true;
+ };
+ }, "ds-bot: computer cursor");
+ }
if (typeof appComputer?.idleSeconds === "function") {
ctx.effect(() => {
const push = () => {
diff --git a/ds-bot/src/client/index.js b/ds-bot/src/client/index.js
index f1ace606..6612be03 100644
--- a/ds-bot/src/client/index.js
+++ b/ds-bot/src/client/index.js
@@ -745,7 +745,6 @@ export function apply(ctx) {
if (typeof appComputer?.computer === 'function') {
ctx.effect(() => {
const nameOf = id => roster.getSnapshot().byId[id]?.name ?? 'a Bot'
- const seenCursors = new Map()
const sync = () => {
const live = computer.getSnapshot()
const name = id => nameOf(id)
@@ -765,20 +764,37 @@ export function apply(ctx) {
rows,
labels: { stop: t('Stop'), stopAll: t('Stop all'), letGo: t('Let it continue') },
})
- // Fresh cursor events ride the same sync; seq keeps each published once.
- for (const event of live.cursors ?? []) {
- if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue
- if (Number.isFinite(event.t) && Date.now() - event.t > 3000) continue
- seenCursors.set(event.botId, event.seq)
- if (typeof appComputer.computerCursor === 'function') {
- appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image })
- }
- }
}
sync()
return computer.subscribe(sync)
}, 'ds-bot: computer indicator')
}
+ if (typeof appComputer?.computerCursor === 'function') {
+ ctx.effect(() => {
+ // Cursor events get their own long-poll: the shared state poll drops to
+ // 10 s while the window is hidden, far too slow for an on-screen cursor.
+ let dead = false
+ const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
+ const run = async () => {
+ let after = 0
+ while (!dead) {
+ if (roster.getSnapshot().computer?.enabled !== true) { await pause(2000); continue }
+ try {
+ const reply = await call('computer-cursor-wait', { after })
+ for (const event of reply?.events ?? []) {
+ if (Number.isFinite(event.seq) && event.seq > after) after = event.seq
+ if (Number.isFinite(event.t) && Number.isFinite(reply?.now) && reply.now - event.t > 3000) continue
+ appComputer.computerCursor({ ...event, color: lookOf(roster.getSnapshot().byId[event.botId]).color, avatar: lookOf(roster.getSnapshot().byId[event.botId]).image })
+ }
+ } catch {
+ await pause(2000)
+ }
+ }
+ }
+ void run()
+ return () => { dead = true }
+ }, 'ds-bot: computer cursor')
+ }
if (typeof appComputer?.idleSeconds === 'function') {
ctx.effect(() => {
const push = () => {
diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.js
index a0d5c677..4a57d50a 100644
--- a/ds-bot/src/host/computer/index.js
+++ b/ds-bot/src/host/computer/index.js
@@ -277,6 +277,7 @@ export function install(rt) {
held.handoffLift?.()
}
})
+ rt.closeComputerWaiters = () => { for (const waiter of [...cursorWaiters]) { cursorWaiters.delete(waiter); waiter.wake = () => {} } }
rt.refreshComputer = refreshComputer
// ---- App identity ----
@@ -431,8 +432,12 @@ export function install(rt) {
const cursors = new Map()
const lastWindowOf = new Map()
let cursorSeq = 0
+ // Long-poll waiters: each resolves when a newer cursor event exists (or ~25 s
+ // pass) so the app's cursor overlay does not ride the 1.5–10 s state poll.
+ const cursorWaiters = new Set()
const publishCursor = (botId, event) => {
cursors.set(botId, { botId, seq: ++cursorSeq, t: Date.now(), ...event })
+ for (const waiter of [...cursorWaiters]) waiter.wake()
}
// Click-type calls ripple; index and keyboard calls only name their window.
const RIPPLE_METHODS = new Set(['click', 'double_click', 'performSecondaryAction', 'drag'])
@@ -774,7 +779,8 @@ export function install(rt) {
}
}
recordGeo(sessionId, payload)
- emitCursor(exec, call, payload)
+ // A refused or failed call pointed nowhere.
+ if (payload !== null) emitCursor(exec, call, payload)
const shot = payload !== null && typeof payload === 'object' ? payload.attachment : undefined
if (shot !== undefined) {
controlOf(botId).lastShot = shot
@@ -1034,6 +1040,24 @@ export function install(rt) {
switch (endpoint) {
case 'set-enabled':
return setEnabled(body.enabled === true)
+ case 'cursor-wait': {
+ const after = Number(body.after) || 0
+ const fresh = () => [...cursors.values()].filter(event => event.seq > after)
+ const now = () => ({ now: Date.now(), events: fresh() })
+ if (fresh().length > 0) return now()
+ return await new Promise(resolve => {
+ const waiter = {
+ wake: () => {
+ if (fresh().length === 0) return
+ cursorWaiters.delete(waiter)
+ clearTimeout(timer)
+ resolve(now())
+ },
+ }
+ const timer = setTimeout(() => { cursorWaiters.delete(waiter); resolve({ now: Date.now(), events: [] }) }, 25000)
+ cursorWaiters.add(waiter)
+ })
+ }
case 'set-level':
return setLevel(String(body.botId ?? ''), String(body.level ?? ''))
case 'revoke-app':
diff --git a/ds-bot/test/computer.test.js b/ds-bot/test/computer.test.js
index 107ce81d..aa7e5fd8 100644
--- a/ds-bot/test/computer.test.js
+++ b/ds-bot/test/computer.test.js
@@ -934,9 +934,10 @@ test('the Bot cursor: relative, pixel and index calls publish a point', async (t
// getState records window bounds; a screenshot records that window's pixels
await track(cua('getState', { appHandle: 'h1' }, 'g1'), async () => ({ isError: false, value: { appHandle: 'h1', focusedWindowId: 7, windows: [{ windowId: 7, pid: 42, bounds: { x: 100, y: 200, width: 300, height: 400 } }] } }))
await track(cua('screenshot', { appHandle: 'h1', windowId: 7 }, 'g2'), async () => ({ isError: false, value: { windowId: 7, pixelWidth: 600, pixelHeight: 800 } }))
+ const okResult = async () => ({ isError: false, value: { dispatched: true } })
// relative click: thousandths of the window
await gate(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)
- await track(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)
+ await track(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), okResult)
let cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
assert.equal(cursor.mode, 'point')
assert.equal(cursor.ripple, true)
@@ -944,19 +945,19 @@ test('the Bot cursor: relative, pixel and index calls publish a point', async (t
assert.ok(Math.abs(cursor.y - (200 + 250 * 400 / 999)) < 0.5)
assert.equal(cursor.pid, 42)
// pixel coordinates scale by that window's last screenshot size
- await track(cua('click', { appHandle: 'h1', windowId: 7, x: 300, y: 400 }, 'c2'), allow)
+ await track(cua('click', { appHandle: 'h1', windowId: 7, x: 300, y: 400 }, 'c2'), okResult)
cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
assert.ok(Math.abs(cursor.x - 250) < 0.5)
assert.ok(Math.abs(cursor.y - 400) < 0.5)
// an element-index click sits at the title bar with no ripple
- await track(cua('click', { appHandle: 'h1', windowId: 7, index: 6 }, 'c3'), allow)
+ await track(cua('click', { appHandle: 'h1', windowId: 7, index: 6 }, 'c3'), okResult)
cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')
assert.equal(cursor.mode, 'window')
assert.equal(cursor.ripple, false)
assert.ok(Math.abs(cursor.x - 250) < 0.5)
assert.ok(Math.abs(cursor.y - 214) < 0.5)
// a call on an unknown window publishes nothing
- await track(cua('click', { appHandle: 'h1', windowId: 99, index: 1 }, 'c4'), allow)
+ await track(cua('click', { appHandle: 'h1', windowId: 99, index: 1 }, 'c4'), okResult)
assert.equal(rt.computerLive().cursors.find(event => event.botId === 'w1').mode, 'window')
})
@@ -983,7 +984,7 @@ test('driver cursors: session injection, refused cursor tools, one disable per s
const exec = { name: 'cua_driver_native__click', arguments: { pid: 42 }, agent: { id: 'w1' }, callId: 'c1' }
await gate(exec, allow)
assert.match(exec.arguments.session, /^dsbot-[0-9a-f]+$/)
- await track(exec, allow)
+ await track(exec, async () => ({ isError: false, value: { done: true } }))
// disable ran once for the session; the state query published the point
assert.equal(executions.filter(e => e[0] === 'disable').length, 1)
await new Promise(resolve => setImmediate(resolve))
@@ -999,3 +1000,34 @@ test('driver cursors: session injection, refused cursor tools, one disable per s
await gate(bare, allow)
assert.equal(bare.arguments.session, undefined)
})
+
+test('cursor-wait returns fresh events, wakes on publish, and times out empty', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ rt.botOf('w1').computer = { apps: { 'com.apple.Notes': { name: 'Notes', id: 'com.apple.Notes', grantedAt: 0 } } }
+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ const allow = async () => 'allowed'
+ const cua = (method, params, callId) => ({ name: 'cua', arguments: { method, params }, agent: { id: 'w1' }, callId })
+ await track(cua('getState', { appHandle: 'h1' }, 'g1'), async () => ({ isError: false, value: { appHandle: 'h1', focusedWindowId: 7, windows: [{ windowId: 7, bounds: { x: 0, y: 0, width: 100, height: 100 } }] } }))
+ // a handle-only relative click resolves through the remembered windowId
+ const okResult = async () => ({ isError: false, value: { dispatched: true } })
+ const click = { name: 'cua', arguments: { method: 'click', params: { appHandle: 'h1', relative: true, x: 500, y: 500 } }, agent: { id: 'w1' }, callId: 'c1' }
+ await gate(click, allow)
+ await track(click, okResult)
+ const first = await rt.computerEndpoint('cursor-wait', { after: 0 })
+ assert.equal(first.events.length, 1)
+ assert.ok(Math.abs(first.events[0].x - 50) < 0.5)
+ assert.equal(first.events[0].windowId, 7)
+ // a waiter with the same seq wakes only when a newer event lands
+ const waiting = rt.computerEndpoint('cursor-wait', { after: first.events[0].seq })
+ await new Promise(resolve => setImmediate(resolve))
+ await track(cua('pressKey', { appHandle: 'h1', key: 'a' }, 'c2'), okResult)
+ const woke = await waiting
+ assert.equal(woke.events.length, 1)
+ assert.equal(woke.events[0].mode, 'window')
+ // nothing newer → the wait resolves empty after the timeout (stubbed short)
+})