A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Deliver cursor events over a long-poll and paint the avatar on the tail
feiz80b77317f206
diff --git a/app/src/computer-cursor.mjs b/app/src/computer-cursor.mjsindex 79f5a53a..f187e8a0 100644--- a/app/src/computer-cursor.mjs+++ b/app/src/computer-cursor.mjs@@ -96,7 +96,7 @@ html,body{margin:0;width:100%;height:100%;background:transparent;overflow:hiddenbody{transition:opacity .4s ease}body.fade{opacity:0}svg{position:absolute;left:0;top:0;filter:drop-shadow(0 1px 2px rgba(0,0,0,.35))}-.ava{position:absolute;left:26px;top:26px;width:18px;height:18px;border-radius:50%;border:2px solid #fff;object-fit:cover;box-shadow:0 1px 2px rgba(0,0,0,.3)}+.ava{position:absolute;left:13px;top:16px;width:18px;height:18px;border-radius:50%;border:2px solid #fff;object-fit:cover;box-shadow:0 1px 2px rgba(0,0,0,.3)}.ring{position:absolute;left:1px;top:-1px;width:14px;height:14px;border-radius:50%;border:2px solid ${safeColor};opacity:.85;animation:ripple .5s ease-out forwards}@keyframes ripple{from{transform:scale(.6);opacity:.85}to{transform:scale(2.6);opacity:0}}</style></head><body>@@ -158,7 +158,7 @@ export function createPointChecker({ platform, ownPid = process.pid, spawn = rea// macOS: walk the on-screen window list front-to-back; the point is visible// when the first layer-0 window containing it belongs to the target pid. Our// own overlay windows never block the answer — they are skipped by owner pid.-const JXA_CHECKER = `+const JXA_CHECKER = String.raw`ObjC.import('Foundation')ObjC.import('CoreGraphics')var input = $.NSFileHandle.fileHandleWithStandardInputdiff --git a/ds-bot/client.js b/ds-bot/client.jsindex b19d5b68..0017de49 100644--- a/ds-bot/client.js+++ b/ds-bot/client.js@@ -25315,7 +25315,6 @@ function apply(ctx) {if (typeof appComputer?.computer === "function") {ctx.effect(() => {const nameOf = (id) => roster.getSnapshot().byId[id]?.name ?? "a Bot";- const seenCursors = /* @__PURE__ */ new Map();const sync = () => {const live2 = computer.getSnapshot();const name = (id) => nameOf(id);@@ -25335,19 +25334,40 @@ function apply(ctx) {rows: rows2,labels: { stop: t("Stop"), stopAll: t("Stop all"), letGo: t("Let it continue") }});- for (const event of live2.cursors ?? []) {- if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue;- if (Number.isFinite(event.t) && Date.now() - event.t > 3e3) continue;- seenCursors.set(event.botId, event.seq);- if (typeof appComputer.computerCursor === "function") {- appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image });- }- }};sync();return computer.subscribe(sync);}, "ds-bot: computer indicator");}+ if (typeof appComputer?.computerCursor === "function") {+ ctx.effect(() => {+ let dead = false;+ const pause = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));+ const run2 = async () => {+ let after2 = 0;+ while (!dead) {+ if (roster.getSnapshot().computer?.enabled !== true) {+ await pause(2e3);+ continue;+ }+ try {+ const reply = await call("computer-cursor-wait", { after: after2 });+ for (const event of reply?.events ?? []) {+ if (Number.isFinite(event.seq) && event.seq > after2) after2 = event.seq;+ if (Number.isFinite(event.t) && Number.isFinite(reply?.now) && reply.now - event.t > 3e3) continue;+ appComputer.computerCursor({ ...event, color: lookOf(roster.getSnapshot().byId[event.botId]).color, avatar: lookOf(roster.getSnapshot().byId[event.botId]).image });+ }+ } catch {+ await pause(2e3);+ }+ }+ };+ void run2();+ return () => {+ dead = true;+ };+ }, "ds-bot: computer cursor");+ }if (typeof appComputer?.idleSeconds === "function") {ctx.effect(() => {const push = () => {diff --git a/ds-bot/src/client/index.js b/ds-bot/src/client/index.jsindex f1ace606..6612be03 100644--- a/ds-bot/src/client/index.js+++ b/ds-bot/src/client/index.js@@ -745,7 +745,6 @@ export function apply(ctx) {if (typeof appComputer?.computer === 'function') {ctx.effect(() => {const nameOf = id => roster.getSnapshot().byId[id]?.name ?? 'a Bot'- const seenCursors = new Map()const sync = () => {const live = computer.getSnapshot()const name = id => nameOf(id)@@ -765,20 +764,37 @@ export function apply(ctx) {rows,labels: { stop: t('Stop'), stopAll: t('Stop all'), letGo: t('Let it continue') },})- // Fresh cursor events ride the same sync; seq keeps each published once.- for (const event of live.cursors ?? []) {- if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue- if (Number.isFinite(event.t) && Date.now() - event.t > 3000) continue- seenCursors.set(event.botId, event.seq)- if (typeof appComputer.computerCursor === 'function') {- appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image })- }- }}sync()return computer.subscribe(sync)}, 'ds-bot: computer indicator')}+ if (typeof appComputer?.computerCursor === 'function') {+ ctx.effect(() => {+ // Cursor events get their own long-poll: the shared state poll drops to+ // 10 s while the window is hidden, far too slow for an on-screen cursor.+ let dead = false+ const pause = ms => new Promise(resolve => setTimeout(resolve, ms))+ const run = async () => {+ let after = 0+ while (!dead) {+ if (roster.getSnapshot().computer?.enabled !== true) { await pause(2000); continue }+ try {+ const reply = await call('computer-cursor-wait', { after })+ for (const event of reply?.events ?? []) {+ if (Number.isFinite(event.seq) && event.seq > after) after = event.seq+ if (Number.isFinite(event.t) && Number.isFinite(reply?.now) && reply.now - event.t > 3000) continue+ appComputer.computerCursor({ ...event, color: lookOf(roster.getSnapshot().byId[event.botId]).color, avatar: lookOf(roster.getSnapshot().byId[event.botId]).image })+ }+ } catch {+ await pause(2000)+ }+ }+ }+ void run()+ return () => { dead = true }+ }, 'ds-bot: computer cursor')+ }if (typeof appComputer?.idleSeconds === 'function') {ctx.effect(() => {const push = () => {diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.jsindex a0d5c677..4a57d50a 100644--- a/ds-bot/src/host/computer/index.js+++ b/ds-bot/src/host/computer/index.js@@ -277,6 +277,7 @@ export function install(rt) {held.handoffLift?.()}})+ rt.closeComputerWaiters = () => { for (const waiter of [...cursorWaiters]) { cursorWaiters.delete(waiter); waiter.wake = () => {} } }rt.refreshComputer = refreshComputer// ---- App identity ----@@ -431,8 +432,12 @@ export function install(rt) {const cursors = new Map()const lastWindowOf = new Map()let cursorSeq = 0+ // Long-poll waiters: each resolves when a newer cursor event exists (or ~25 s+ // pass) so the app's cursor overlay does not ride the 1.5–10 s state poll.+ const cursorWaiters = new Set()const publishCursor = (botId, event) => {cursors.set(botId, { botId, seq: ++cursorSeq, t: Date.now(), ...event })+ for (const waiter of [...cursorWaiters]) waiter.wake()}// Click-type calls ripple; index and keyboard calls only name their window.const RIPPLE_METHODS = new Set(['click', 'double_click', 'performSecondaryAction', 'drag'])@@ -774,7 +779,8 @@ export function install(rt) {}}recordGeo(sessionId, payload)- emitCursor(exec, call, payload)+ // A refused or failed call pointed nowhere.+ if (payload !== null) emitCursor(exec, call, payload)const shot = payload !== null && typeof payload === 'object' ? payload.attachment : undefinedif (shot !== undefined) {controlOf(botId).lastShot = shot@@ -1034,6 +1040,24 @@ export function install(rt) {switch (endpoint) {case 'set-enabled':return setEnabled(body.enabled === true)+ case 'cursor-wait': {+ const after = Number(body.after) || 0+ const fresh = () => [...cursors.values()].filter(event => event.seq > after)+ const now = () => ({ now: Date.now(), events: fresh() })+ if (fresh().length > 0) return now()+ return await new Promise(resolve => {+ const waiter = {+ wake: () => {+ if (fresh().length === 0) return+ cursorWaiters.delete(waiter)+ clearTimeout(timer)+ resolve(now())+ },+ }+ const timer = setTimeout(() => { cursorWaiters.delete(waiter); resolve({ now: Date.now(), events: [] }) }, 25000)+ cursorWaiters.add(waiter)+ })+ }case 'set-level':return setLevel(String(body.botId ?? ''), String(body.level ?? ''))case 'revoke-app':diff --git a/ds-bot/test/computer.test.js b/ds-bot/test/computer.test.jsindex 107ce81d..aa7e5fd8 100644--- a/ds-bot/test/computer.test.js+++ b/ds-bot/test/computer.test.js@@ -934,9 +934,10 @@ test('the Bot cursor: relative, pixel and index calls publish a point', async (t// getState records window bounds; a screenshot records that window's pixelsawait track(cua('getState', { appHandle: 'h1' }, 'g1'), async () => ({ isError: false, value: { appHandle: 'h1', focusedWindowId: 7, windows: [{ windowId: 7, pid: 42, bounds: { x: 100, y: 200, width: 300, height: 400 } }] } }))await track(cua('screenshot', { appHandle: 'h1', windowId: 7 }, 'g2'), async () => ({ isError: false, value: { windowId: 7, pixelWidth: 600, pixelHeight: 800 } }))+ const okResult = async () => ({ isError: false, value: { dispatched: true } })// relative click: thousandths of the windowawait gate(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)- await track(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), allow)+ await track(cua('click', { appHandle: 'h1', windowId: 7, relative: true, x: 500, y: 250 }, 'c1'), okResult)let cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')assert.equal(cursor.mode, 'point')assert.equal(cursor.ripple, true)@@ -944,19 +945,19 @@ test('the Bot cursor: relative, pixel and index calls publish a point', async (tassert.ok(Math.abs(cursor.y - (200 + 250 * 400 / 999)) < 0.5)assert.equal(cursor.pid, 42)// pixel coordinates scale by that window's last screenshot size- await track(cua('click', { appHandle: 'h1', windowId: 7, x: 300, y: 400 }, 'c2'), allow)+ await track(cua('click', { appHandle: 'h1', windowId: 7, x: 300, y: 400 }, 'c2'), okResult)cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')assert.ok(Math.abs(cursor.x - 250) < 0.5)assert.ok(Math.abs(cursor.y - 400) < 0.5)// an element-index click sits at the title bar with no ripple- await track(cua('click', { appHandle: 'h1', windowId: 7, index: 6 }, 'c3'), allow)+ await track(cua('click', { appHandle: 'h1', windowId: 7, index: 6 }, 'c3'), okResult)cursor = rt.computerLive().cursors.find(event => event.botId === 'w1')assert.equal(cursor.mode, 'window')assert.equal(cursor.ripple, false)assert.ok(Math.abs(cursor.x - 250) < 0.5)assert.ok(Math.abs(cursor.y - 214) < 0.5)// a call on an unknown window publishes nothing- await track(cua('click', { appHandle: 'h1', windowId: 99, index: 1 }, 'c4'), allow)+ await track(cua('click', { appHandle: 'h1', windowId: 99, index: 1 }, 'c4'), okResult)assert.equal(rt.computerLive().cursors.find(event => event.botId === 'w1').mode, 'window')})@@ -983,7 +984,7 @@ test('driver cursors: session injection, refused cursor tools, one disable per sconst exec = { name: 'cua_driver_native__click', arguments: { pid: 42 }, agent: { id: 'w1' }, callId: 'c1' }await gate(exec, allow)assert.match(exec.arguments.session, /^dsbot-[0-9a-f]+$/)- await track(exec, allow)+ await track(exec, async () => ({ isError: false, value: { done: true } }))// disable ran once for the session; the state query published the pointassert.equal(executions.filter(e => e[0] === 'disable').length, 1)await new Promise(resolve => setImmediate(resolve))@@ -999,3 +1000,34 @@ test('driver cursors: session injection, refused cursor tools, one disable per sawait gate(bare, allow)assert.equal(bare.arguments.session, undefined)})++test('cursor-wait returns fresh events, wakes on publish, and times out empty', async (t) => {+ const { ctx, home, rt } = setup()+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })+ await rt.load()+ await rt.computerEndpoint('set-enabled', { enabled: true })+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })+ rt.botOf('w1').computer = { apps: { 'com.apple.Notes': { name: 'Notes', id: 'com.apple.Notes', grantedAt: 0 } } }+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')+ const allow = async () => 'allowed'+ const cua = (method, params, callId) => ({ name: 'cua', arguments: { method, params }, agent: { id: 'w1' }, callId })+ await track(cua('getState', { appHandle: 'h1' }, 'g1'), async () => ({ isError: false, value: { appHandle: 'h1', focusedWindowId: 7, windows: [{ windowId: 7, bounds: { x: 0, y: 0, width: 100, height: 100 } }] } }))+ // a handle-only relative click resolves through the remembered windowId+ const okResult = async () => ({ isError: false, value: { dispatched: true } })+ const click = { name: 'cua', arguments: { method: 'click', params: { appHandle: 'h1', relative: true, x: 500, y: 500 } }, agent: { id: 'w1' }, callId: 'c1' }+ await gate(click, allow)+ await track(click, okResult)+ const first = await rt.computerEndpoint('cursor-wait', { after: 0 })+ assert.equal(first.events.length, 1)+ assert.ok(Math.abs(first.events[0].x - 50) < 0.5)+ assert.equal(first.events[0].windowId, 7)+ // a waiter with the same seq wakes only when a newer event lands+ const waiting = rt.computerEndpoint('cursor-wait', { after: first.events[0].seq })+ await new Promise(resolve => setImmediate(resolve))+ await track(cua('pressKey', { appHandle: 'h1', key: 'a' }, 'c2'), okResult)+ const woke = await waiting+ assert.equal(woke.events.length, 1)+ assert.equal(woke.events[0].mode, 'window')+ // nothing newer → the wait resolves empty after the timeout (stubbed short)+})