A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Fix three flaky unit tests at their causes
Fei2411dbdd76515576
diff --git a/ds-bot/test/follow-through.test.js b/ds-bot/test/follow-through.test.jsindex e95157c5..5341c4e6 100644--- a/ds-bot/test/follow-through.test.js+++ b/ds-bot/test/follow-through.test.js@@ -95,9 +95,14 @@ const settleState = async (t, home, check) => {}assert.ok(check(), 'condition not reached')}-const settle = async (check) => {+// Some waits in the product run on the clock the tests mock: driving it here, as+// settleState does, keeps a busy wait from starving the work it waits for.+const settle = async (t, check) => {const until = Date.now() + 3000- while (!check() && Date.now() < until) await new Promise(resolve => setImmediate(resolve))+ while (!check() && Date.now() < until) {+ try { t.mock.timers.tick(100) } catch { /* the clock is not mocked in this test */ }+ await new Promise(resolve => setImmediate(resolve))+ }assert.ok(check(), 'condition not reached')}const text = message => message.content.map(block => block.text ?? '').join('\n')@@ -117,7 +122,7 @@ async function openRoomTurn(t, host) {const stop = new AbortController()t.after(() => stop.abort())void host.tool('group_relay', {}, 'r1', { callId: 'c1', signal: stop.signal }).then((value) => { result = JSON.parse(value) })- await settle(() => host.agentOf('gb').calls.length === 1)+ await settle(t, () => host.agentOf('gb').calls.length === 1)run(host, 'gb', 1, host.agentOf('gb').calls[0])return () => result}@@ -128,20 +133,20 @@ test('a group turn cut off by a restart is picked up once, and the group is toldconst first = await start(home)await openRoomTurn(t, first)void first.askApproval('gb')- await settle(() => inflight(home).gb?.approval !== undefined)+ await settle(t, () => inflight(home).gb?.approval !== undefined)assert.deepEqual([inflight(home).gb.botId, inflight(home).gb.roomId, inflight(home).gb.trigger.role], ['b', 'r1', 'room'])await first.stop()const second = await start(home)t.mock.timers.tick(3000)- await settle(() => second.agentOf('gb').calls.length === 1)+ await settle(t, () => second.agentOf('gb').calls.length === 1)const post = second.agentOf('r1').calls[0]assert.deepEqual([post.source.role, post.source.resumed, post.source.quiet, post.source.senderSessionId], ['post', true, true, 'b'])const resumed = second.agentOf('gb').calls[0]assert.deepEqual([resumed.source.role, resumed.source.resumed, resumed.source.roomId], ['room', true, 'r1'])assert.match(text(resumed), /^\[Group chat "Ops" · picking up after a restart\]/)assert.match(text(resumed), /approval to run pwsh \(escalate sandbox\)/)- await settle(() => Object.keys(inflight(home)).length === 0)+ await settle(t, () => Object.keys(inflight(home)).length === 0)// The group shows the event, and the admin reads it on its next turn.second.emit('r1', 'user/message', { content: post.content, source: post.source })@@ -152,11 +157,11 @@ test('a group turn cut off by a restart is picked up once, and the group is told// Cut off again while picking it up: the group hears it, and the turn stays stopped.run(second, 'gb', 2, resumed)- await settle(() => inflight(home).gb?.resumed === true)+ await settle(t, () => inflight(home).gb?.resumed === true)await second.stop()const third = await start(home)t.mock.timers.tick(3000)- await settle(() => third.agentOf('r1').calls.length === 1)+ await settle(t, () => third.agentOf('r1').calls.length === 1)assert.equal(third.agentOf('r1').calls[0].source.again, true)assert.equal(third.agentOf('gb').calls.length, 0)})@@ -167,12 +172,12 @@ test('a request cut off by a restart goes on under the same exchange; own-chat wconst first = await start(home)run(first, 'b', 1, { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } })run(first, 'a', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })- await settle(() => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)+ await settle(t, () => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)await first.stop()const second = await start(home)t.mock.timers.tick(3000)- await settle(() => second.agentOf('b').calls.length === 1 && second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('b').calls.length === 1 && second.agentOf('a').calls.length === 1)const request = second.agentOf('b').calls[0]assert.deepEqual([request.source.role, request.source.exchangeId, request.source.senderSessionId, request.source.resumed], ['request', 'x1', 'a', true])assert.match(text(request), /^\[Message from Chief · DeepSeek Harness restarted while you were working on it\]/)@@ -189,7 +194,7 @@ test('when only DS Bot restarts, a turn still running or finished meanwhile is nconst first = await start(home)run(first, 'b', 1, { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } })run(first, 'a', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })- await settle(() => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)+ await settle(t, () => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)await first.stop()// DSH went on: Writer is still at work, and Chief's turn ends before the pick-up.@@ -198,7 +203,7 @@ test('when only DS Bot restarts, a turn still running or finished meanwhile is nsecond.emit('a', 'turn/end', { turn: 1, reason: { kind: 'completed' } })t.mock.timers.tick(3000)// One read per check: two reads can see the old notes and the cleared ones.- await settle(() => {+ await settle(t, () => {const noted = inflight(home)return noted.b?.trigger?.exchangeId === 'x1' && noted.a === undefined})@@ -210,16 +215,16 @@ test('when only DS Bot restarts, a turn still running or finished meanwhile is nawait second.stop()const third = await start(home)t.mock.timers.tick(3000)- await settle(() => third.agentOf('b').calls.length === 1)+ await settle(t, () => third.agentOf('b').calls.length === 1)assert.deepEqual([third.agentOf('b').calls[0].source.exchangeId, third.agentOf('a').calls.length], ['x1', 0])})// Chief asks Writer for the plan; Writer's turn takes it up and the work waits.-const askWriter = async (host, home) => {+const askWriter = async (t, host, home) => {await host.tool('message_bot', { to: 'Writer', message: 'Draft the plan' }, 'a')const request = host.agentOf('b').calls[0]run(host, 'b', 1, request)- await settle(() => inflight(home).b?.trigger?.exchangeId === request.source.exchangeId)+ await settle(t, () => inflight(home).b?.trigger?.exchangeId === request.source.exchangeId)return request}@@ -242,13 +247,13 @@ for (const role of ['reply', 'report', 'resume']) {const first = await start(home, logs)const request = { content: [{ type: 'text', text: 'Go on with the plan.' }], source: { kind: 'bot', form: 'relay', role, exchangeId: 'continuation-1', work: 'x1', ...(role === 'reply' ? { senderSessionId: 'c', replyTo: 'y1' } : {}) } }run(first, 'b', 1, request)- await settle(() => inflight(home).b?.trigger?.work === 'x1')+ await settle(t, () => inflight(home).b?.trigger?.work === 'x1')logs.set('b', writerLog(request, 'The final plan is ready.'))await first.stop()const second = await start(home, logs)second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })t.mock.timers.tick(3000)- await settle(() => second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('a').calls.length === 1)const reply = second.agentOf('a').calls[0]assert.equal(reply.source.replyTo, 'x1')assert.match(text(reply), /The final plan is ready/)@@ -263,7 +268,7 @@ test('an answer a turn produced while DS Bot was not watching goes to the Bot thconst home = freshHome(t)const logs = new Map()const first = await start(home, logs)- const request = await askWriter(first, home)+ const request = await askWriter(t, first, home)const exchangeId = request.source.exchangeId// Writer answers, and DS Bot restarts before it sees the end of that turn.logs.set('b', writerLog(request, 'The plan is ready.'))@@ -272,7 +277,7 @@ test('an answer a turn produced while DS Bot was not watching goes to the Bot thconst second = await start(home, logs)second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })t.mock.timers.tick(3000)- await settle(() => second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('a').calls.length === 1)const reply = second.agentOf('a').calls[0]assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])assert.match(text(reply), /The plan is ready\./)@@ -295,7 +300,7 @@ test('a turn whose record cannot be read says the result is unknown, and is notconst home = freshHome(t)const logs = new Map()const first = await start(home, logs)- const request = await askWriter(first, home)+ const request = await askWriter(t, first, home)const exchangeId = request.source.exchangeIdlogs.set('b', writerLog(request, 'The plan is ready.'))await first.stop()@@ -306,7 +311,7 @@ test('a turn whose record cannot be read says the result is unknown, and is notconst second = await start(home, logs, unreadable)second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })t.mock.timers.tick(3000)- await settle(() => second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('a').calls.length === 1)const reply = second.agentOf('a').calls[0]assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])assert.match(text(reply), /what it did is unknown/)@@ -329,7 +334,7 @@ test('a turn that ended without saying anything says it finished', async (t) =>const home = freshHome(t)const logs = new Map()const first = await start(home, logs)- const request = await askWriter(first, home)+ const request = await askWriter(t, first, home)const exchangeId = request.source.exchangeIdlogs.set('b', writerLog(request, undefined))await first.stop()@@ -337,7 +342,7 @@ test('a turn that ended without saying anything says it finished', async (t) =>const second = await start(home, logs)second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })t.mock.timers.tick(3000)- await settle(() => second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('a').calls.length === 1)const reply = second.agentOf('a').calls[0]assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])assert.match(text(reply), /Writer finished without a reply\./)@@ -351,7 +356,7 @@ test('a turn that ended on the token limit gives what it said', async (t) => {const home = freshHome(t)const logs = new Map()const first = await start(home, logs)- const request = await askWriter(first, home)+ const request = await askWriter(t, first, home)const exchangeId = request.source.exchangeIdlogs.set('b', writerLog(request, 'The plan is ready, up to the limit.', { kind: 'max-tokens' }))await first.stop()@@ -359,7 +364,7 @@ test('a turn that ended on the token limit gives what it said', async (t) => {const second = await start(home, logs)second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'max-tokens' } })t.mock.timers.tick(3000)- await settle(() => second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('a').calls.length === 1)const reply = second.agentOf('a').calls[0]assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])assert.match(text(reply), /The plan is ready, up to the limit\./)@@ -378,14 +383,14 @@ test('the answer that goes up keeps what came back for the work', async (t) => {const first = await start(home, logs)const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }run(first, 'b', 1, request)- await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')+ await settle(t, () => inflight(home).b?.trigger?.exchangeId === 'x1')logs.set('b', writerLog(request, 'The plan is ready.'))await first.stop()const second = await start(home, logs)second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })t.mock.timers.tick(3000)- await settle(() => second.agentOf('a').calls.length === 1)+ await settle(t, () => second.agentOf('a').calls.length === 1)const reply = text(second.agentOf('a').calls[0])assert.match(reply, /The plan is ready\./)assert.match(reply, /Reviewer:\nThe numbers check out\./)@@ -399,7 +404,7 @@ test('an answer that already went up is not sent again', async (t) => {const home = freshHome(t)const logs = new Map()const first = await start(home, logs)- const request = await askWriter(first, home)+ const request = await askWriter(t, first, home)const exchangeId = request.source.exchangeIdlogs.set('b', writerLog(request, 'The plan is ready.'))// Chief already has the answer: another run delivered it.@@ -422,7 +427,7 @@ test('a work that still waits for what it handed on holds its text, and is not sconst first = await start(home, logs)const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }run(first, 'b', 1, request)- await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')+ await settle(t, () => inflight(home).b?.trigger?.exchangeId === 'x1')logs.set('b', writerLog(request, 'The plan is ready.'))await first.stop()@@ -444,7 +449,7 @@ test('a finished turn leaves nothing to pick up', async (t) => {const first = await start(home)run(first, 'a', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })first.emit('a', 'turn/end', { turn: 1, reason: { kind: 'completed' } })- await settle(() => Object.keys(inflight(home)).length === 0 && existsSync(join(home, 'inflight.json')))+ await settle(t, () => Object.keys(inflight(home)).length === 0 && existsSync(join(home, 'inflight.json')))await first.stop()const second = await start(home)t.mock.timers.tick(3000)@@ -459,7 +464,7 @@ test('a turn DSH stops as it shuts down stays noted; one the user stops does notrun(host, 'b', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })host.emit('a', 'turn/end', { turn: 1, reason: { kind: 'aborted', reason: { kind: 'disposed' } } })host.emit('b', 'turn/end', { turn: 1, reason: { kind: 'aborted', reason: { kind: 'user' } } })- await settle(() => inflight(home).a !== undefined && inflight(home).b === undefined)+ await settle(t, () => inflight(home).a !== undefined && inflight(home).b === undefined)})test('a message a shutdown cancelled is sent again once; one still waiting, taken up or withdrawn is not', async (t) => {@@ -474,7 +479,7 @@ test('a message a shutdown cancelled is sent again once; one still waiting, take// Taken up by a turn, or withdrawn while DS Bot runs (the user stopped the Bot).first.emit('b', 'user/message', { content: notesMessage.content, source: notesMessage.source })for (const listener of first.ctx.listeners.get('agent/inbox/discarded')) listener({ message: withdrawn })- await settle(() => (notes(home).queued.b ?? []).length === 2)+ await settle(t, () => (notes(home).queued.b ?? []).length === 2)await first.stop()t.mock.timers.enable({ apis: ['setTimeout'] })@@ -482,7 +487,7 @@ test('a message a shutdown cancelled is sent again once; one still waiting, take// A hard stop keeps the inbox: that message runs when the Session wakes.second.agentOf('b').inbox.nextTurn.push(numbers)t.mock.timers.tick(3000)- await settle(() => second.agentOf('b').calls.length === 1)+ await settle(t, () => second.agentOf('b').calls.length === 1)const [again] = second.agentOf('b').callsassert.deepEqual([again.source.exchangeId, again.source.role, text(again)], [plan.source.exchangeId, 'request', text(plan)])// The resend is the same exchange, not a new one.@@ -497,7 +502,7 @@ test('a tool call written as text gets one more turn on the same round slot', asconst result = await openRoomTurn(t, host)host.emit('gb', 'assistant/message', { message: { content: [{ type: 'text', text: WRITTEN }] } })host.emit('gb', 'turn/end', { turn: 1, reason: { kind: 'completed' } })- await settle(() => host.agentOf('gb').calls.length === 2)+ await settle(t, () => host.agentOf('gb').calls.length === 2)const retry = host.agentOf('gb').calls[1]assert.deepEqual([retry.source.role, retry.source.retried, retry.source.exchangeId], ['room', true, host.agentOf('gb').calls[0].source.exchangeId])assert.match(text(retry), /^\[Group chat "Ops" · your tool call did not run\]/)@@ -507,7 +512,7 @@ test('a tool call written as text gets one more turn on the same round slot', asrun(host, 'gb', 2, retry)host.emit('gb', 'assistant/message', { message: { content: [{ type: 'text', text: WRITTEN }] } })host.emit('gb', 'turn/end', { turn: 2, reason: { kind: 'completed' } })- await settle(() => result() !== undefined)+ await settle(t, () => result() !== undefined)assert.deepEqual(result().replies.map(reply => reply.text), ['Checking now.'])assert.equal(host.agentOf('gb').calls.length, 2)})@@ -520,7 +525,7 @@ test('in its own chat, a Bot that wrote a tool call as text is asked to make it'host.emit('b', 'assistant/message', { message: { content: [{ type: 'text', text: 'On it.' }, { type: 'tool-call', id: 'k1', name: 'pwsh', arguments: '{}' }] } })host.emit('b', 'assistant/message', { message: { content: [{ type: 'text', text: WRITTEN }] } })host.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })- await settle(() => host.agentOf('b').calls.length === 1)+ await settle(t, () => host.agentOf('b').calls.length === 1)const retry = host.agentOf('b').calls[0]assert.deepEqual([retry.source.role, retry.source.retried], ['retry', true])assert.match(text(retry), /^\[Your last reply wrote a tool call as text, so nothing ran\]/)diff --git a/ds-bot/test/learning.test.js b/ds-bot/test/learning.test.jsindex 91f1e564..10b868b0 100644--- a/ds-bot/test/learning.test.js+++ b/ds-bot/test/learning.test.js@@ -567,12 +567,21 @@ test('long turns ending during review remain pending for the next pass', async tconfig: { learningTimes: { tickMs: 100_000 } },})t.after(() => runtime.stop())+ // chatOwners is a derived index: load() and save() rebuild it from the saved+ // team, so the extra Session standing in for a second chat of b2 goes back on+ // after each rebuild.const load = runtime.rt.loadruntime.rt.load = async (...args) => {const result = await load(...args)runtime.rt.chatOwners.set('b3', 'b2')return result}+ const save = runtime.rt.save+ runtime.rt.save = (...args) => {+ const result = save(...args)+ runtime.rt.chatOwners.set('b3', 'b2')+ return result+ }runtime.rt.chatOwners.set('b3', 'b2')const b3Statuses = []runtime.ctx.on('agent/status', ({ agent, status }) => {diff --git a/ds-bot/test/privacy.test.js b/ds-bot/test/privacy.test.jsindex 3107059f..0115ea28 100644--- a/ds-bot/test/privacy.test.js+++ b/ds-bot/test/privacy.test.js@@ -65,6 +65,7 @@ async function start({ services = {}, privateBot = true, archives } = {}) {return agents.get(sessionId)}const registry = new Map([['w1', { id: 'w1', path: team, title: 'DS Bot' }]])+ let created = 0Object.assign(ctx, {llm: { listProviders: () => [], listModels: async () => [], resolveModelInfo: async () => ({}) },workspaceRegistry: {@@ -79,7 +80,8 @@ async function start({ services = {}, privateBot = true, archives } = {}) {archiveSession: async () => {},},sessionController: {- create: async () => ({ sessionId: 's1' }),+ // Ids shaped like real Session ids: a short one would match any path it lands in.+ create: async () => ({ sessionId: `0000000${++created}-1111-4222-8333-444444444444` }),resolveAgent: async sessionId => ({ agent: agentOf(sessionId) }),selectModel: async () => {},page: async () => ({ records: [], hasMore: false }),