Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Fix three flaky unit tests at their causes

Fei2411dbdd76515576
diff --git a/ds-bot/test/follow-through.test.js b/ds-bot/test/follow-through.test.js
index e95157c5..5341c4e6 100644
--- a/ds-bot/test/follow-through.test.js
+++ b/ds-bot/test/follow-through.test.js
@@ -95,9 +95,14 @@ const settleState = async (t, home, check) => {
}
assert.ok(check(), 'condition not reached')
}
-const settle = async (check) => {
+// Some waits in the product run on the clock the tests mock: driving it here, as
+// settleState does, keeps a busy wait from starving the work it waits for.
+const settle = async (t, check) => {
const until = Date.now() + 3000
- while (!check() && Date.now() < until) await new Promise(resolve => setImmediate(resolve))
+ while (!check() && Date.now() < until) {
+ try { t.mock.timers.tick(100) } catch { /* the clock is not mocked in this test */ }
+ await new Promise(resolve => setImmediate(resolve))
+ }
assert.ok(check(), 'condition not reached')
}
const text = message => message.content.map(block => block.text ?? '').join('\n')
@@ -117,7 +122,7 @@ async function openRoomTurn(t, host) {
const stop = new AbortController()
t.after(() => stop.abort())
void host.tool('group_relay', {}, 'r1', { callId: 'c1', signal: stop.signal }).then((value) => { result = JSON.parse(value) })
- await settle(() => host.agentOf('gb').calls.length === 1)
+ await settle(t, () => host.agentOf('gb').calls.length === 1)
run(host, 'gb', 1, host.agentOf('gb').calls[0])
return () => result
}
@@ -128,20 +133,20 @@ test('a group turn cut off by a restart is picked up once, and the group is told
const first = await start(home)
await openRoomTurn(t, first)
void first.askApproval('gb')
- await settle(() => inflight(home).gb?.approval !== undefined)
+ await settle(t, () => inflight(home).gb?.approval !== undefined)
assert.deepEqual([inflight(home).gb.botId, inflight(home).gb.roomId, inflight(home).gb.trigger.role], ['b', 'r1', 'room'])
await first.stop()
const second = await start(home)
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('gb').calls.length === 1)
+ await settle(t, () => second.agentOf('gb').calls.length === 1)
const post = second.agentOf('r1').calls[0]
assert.deepEqual([post.source.role, post.source.resumed, post.source.quiet, post.source.senderSessionId], ['post', true, true, 'b'])
const resumed = second.agentOf('gb').calls[0]
assert.deepEqual([resumed.source.role, resumed.source.resumed, resumed.source.roomId], ['room', true, 'r1'])
assert.match(text(resumed), /^\[Group chat "Ops" · picking up after a restart\]/)
assert.match(text(resumed), /approval to run pwsh \(escalate sandbox\)/)
- await settle(() => Object.keys(inflight(home)).length === 0)
+ await settle(t, () => Object.keys(inflight(home)).length === 0)
// The group shows the event, and the admin reads it on its next turn.
second.emit('r1', 'user/message', { content: post.content, source: post.source })
@@ -152,11 +157,11 @@ test('a group turn cut off by a restart is picked up once, and the group is told
// Cut off again while picking it up: the group hears it, and the turn stays stopped.
run(second, 'gb', 2, resumed)
- await settle(() => inflight(home).gb?.resumed === true)
+ await settle(t, () => inflight(home).gb?.resumed === true)
await second.stop()
const third = await start(home)
t.mock.timers.tick(3000)
- await settle(() => third.agentOf('r1').calls.length === 1)
+ await settle(t, () => third.agentOf('r1').calls.length === 1)
assert.equal(third.agentOf('r1').calls[0].source.again, true)
assert.equal(third.agentOf('gb').calls.length, 0)
})
@@ -167,12 +172,12 @@ test('a request cut off by a restart goes on under the same exchange; own-chat w
const first = await start(home)
run(first, 'b', 1, { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } })
run(first, 'a', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })
- await settle(() => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)
+ await settle(t, () => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)
await first.stop()
const second = await start(home)
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('b').calls.length === 1 && second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('b').calls.length === 1 && second.agentOf('a').calls.length === 1)
const request = second.agentOf('b').calls[0]
assert.deepEqual([request.source.role, request.source.exchangeId, request.source.senderSessionId, request.source.resumed], ['request', 'x1', 'a', true])
assert.match(text(request), /^\[Message from Chief · DeepSeek Harness restarted while you were working on it\]/)
@@ -189,7 +194,7 @@ test('when only DS Bot restarts, a turn still running or finished meanwhile is n
const first = await start(home)
run(first, 'b', 1, { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } })
run(first, 'a', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })
- await settle(() => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)
+ await settle(t, () => inflight(home).b?.trigger !== undefined && inflight(home).a !== undefined)
await first.stop()
// DSH went on: Writer is still at work, and Chief's turn ends before the pick-up.
@@ -198,7 +203,7 @@ test('when only DS Bot restarts, a turn still running or finished meanwhile is n
second.emit('a', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
t.mock.timers.tick(3000)
// One read per check: two reads can see the old notes and the cleared ones.
- await settle(() => {
+ await settle(t, () => {
const noted = inflight(home)
return noted.b?.trigger?.exchangeId === 'x1' && noted.a === undefined
})
@@ -210,16 +215,16 @@ test('when only DS Bot restarts, a turn still running or finished meanwhile is n
await second.stop()
const third = await start(home)
t.mock.timers.tick(3000)
- await settle(() => third.agentOf('b').calls.length === 1)
+ await settle(t, () => third.agentOf('b').calls.length === 1)
assert.deepEqual([third.agentOf('b').calls[0].source.exchangeId, third.agentOf('a').calls.length], ['x1', 0])
})
// Chief asks Writer for the plan; Writer's turn takes it up and the work waits.
-const askWriter = async (host, home) => {
+const askWriter = async (t, host, home) => {
await host.tool('message_bot', { to: 'Writer', message: 'Draft the plan' }, 'a')
const request = host.agentOf('b').calls[0]
run(host, 'b', 1, request)
- await settle(() => inflight(home).b?.trigger?.exchangeId === request.source.exchangeId)
+ await settle(t, () => inflight(home).b?.trigger?.exchangeId === request.source.exchangeId)
return request
}
@@ -242,13 +247,13 @@ for (const role of ['reply', 'report', 'resume']) {
const first = await start(home, logs)
const request = { content: [{ type: 'text', text: 'Go on with the plan.' }], source: { kind: 'bot', form: 'relay', role, exchangeId: 'continuation-1', work: 'x1', ...(role === 'reply' ? { senderSessionId: 'c', replyTo: 'y1' } : {}) } }
run(first, 'b', 1, request)
- await settle(() => inflight(home).b?.trigger?.work === 'x1')
+ await settle(t, () => inflight(home).b?.trigger?.work === 'x1')
logs.set('b', writerLog(request, 'The final plan is ready.'))
await first.stop()
const second = await start(home, logs)
second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('a').calls.length === 1)
const reply = second.agentOf('a').calls[0]
assert.equal(reply.source.replyTo, 'x1')
assert.match(text(reply), /The final plan is ready/)
@@ -263,7 +268,7 @@ test('an answer a turn produced while DS Bot was not watching goes to the Bot th
const home = freshHome(t)
const logs = new Map()
const first = await start(home, logs)
- const request = await askWriter(first, home)
+ const request = await askWriter(t, first, home)
const exchangeId = request.source.exchangeId
// Writer answers, and DS Bot restarts before it sees the end of that turn.
logs.set('b', writerLog(request, 'The plan is ready.'))
@@ -272,7 +277,7 @@ test('an answer a turn produced while DS Bot was not watching goes to the Bot th
const second = await start(home, logs)
second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('a').calls.length === 1)
const reply = second.agentOf('a').calls[0]
assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
assert.match(text(reply), /The plan is ready\./)
@@ -295,7 +300,7 @@ test('a turn whose record cannot be read says the result is unknown, and is not
const home = freshHome(t)
const logs = new Map()
const first = await start(home, logs)
- const request = await askWriter(first, home)
+ const request = await askWriter(t, first, home)
const exchangeId = request.source.exchangeId
logs.set('b', writerLog(request, 'The plan is ready.'))
await first.stop()
@@ -306,7 +311,7 @@ test('a turn whose record cannot be read says the result is unknown, and is not
const second = await start(home, logs, unreadable)
second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('a').calls.length === 1)
const reply = second.agentOf('a').calls[0]
assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
assert.match(text(reply), /what it did is unknown/)
@@ -329,7 +334,7 @@ test('a turn that ended without saying anything says it finished', async (t) =>
const home = freshHome(t)
const logs = new Map()
const first = await start(home, logs)
- const request = await askWriter(first, home)
+ const request = await askWriter(t, first, home)
const exchangeId = request.source.exchangeId
logs.set('b', writerLog(request, undefined))
await first.stop()
@@ -337,7 +342,7 @@ test('a turn that ended without saying anything says it finished', async (t) =>
const second = await start(home, logs)
second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('a').calls.length === 1)
const reply = second.agentOf('a').calls[0]
assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
assert.match(text(reply), /Writer finished without a reply\./)
@@ -351,7 +356,7 @@ test('a turn that ended on the token limit gives what it said', async (t) => {
const home = freshHome(t)
const logs = new Map()
const first = await start(home, logs)
- const request = await askWriter(first, home)
+ const request = await askWriter(t, first, home)
const exchangeId = request.source.exchangeId
logs.set('b', writerLog(request, 'The plan is ready, up to the limit.', { kind: 'max-tokens' }))
await first.stop()
@@ -359,7 +364,7 @@ test('a turn that ended on the token limit gives what it said', async (t) => {
const second = await start(home, logs)
second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'max-tokens' } })
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('a').calls.length === 1)
const reply = second.agentOf('a').calls[0]
assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
assert.match(text(reply), /The plan is ready, up to the limit\./)
@@ -378,14 +383,14 @@ test('the answer that goes up keeps what came back for the work', async (t) => {
const first = await start(home, logs)
const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }
run(first, 'b', 1, request)
- await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')
+ await settle(t, () => inflight(home).b?.trigger?.exchangeId === 'x1')
logs.set('b', writerLog(request, 'The plan is ready.'))
await first.stop()
const second = await start(home, logs)
second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('a').calls.length === 1)
+ await settle(t, () => second.agentOf('a').calls.length === 1)
const reply = text(second.agentOf('a').calls[0])
assert.match(reply, /The plan is ready\./)
assert.match(reply, /Reviewer:\nThe numbers check out\./)
@@ -399,7 +404,7 @@ test('an answer that already went up is not sent again', async (t) => {
const home = freshHome(t)
const logs = new Map()
const first = await start(home, logs)
- const request = await askWriter(first, home)
+ const request = await askWriter(t, first, home)
const exchangeId = request.source.exchangeId
logs.set('b', writerLog(request, 'The plan is ready.'))
// Chief already has the answer: another run delivered it.
@@ -422,7 +427,7 @@ test('a work that still waits for what it handed on holds its text, and is not s
const first = await start(home, logs)
const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }
run(first, 'b', 1, request)
- await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')
+ await settle(t, () => inflight(home).b?.trigger?.exchangeId === 'x1')
logs.set('b', writerLog(request, 'The plan is ready.'))
await first.stop()
@@ -444,7 +449,7 @@ test('a finished turn leaves nothing to pick up', async (t) => {
const first = await start(home)
run(first, 'a', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })
first.emit('a', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
- await settle(() => Object.keys(inflight(home)).length === 0 && existsSync(join(home, 'inflight.json')))
+ await settle(t, () => Object.keys(inflight(home)).length === 0 && existsSync(join(home, 'inflight.json')))
await first.stop()
const second = await start(home)
t.mock.timers.tick(3000)
@@ -459,7 +464,7 @@ test('a turn DSH stops as it shuts down stays noted; one the user stops does not
run(host, 'b', 1, { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } })
host.emit('a', 'turn/end', { turn: 1, reason: { kind: 'aborted', reason: { kind: 'disposed' } } })
host.emit('b', 'turn/end', { turn: 1, reason: { kind: 'aborted', reason: { kind: 'user' } } })
- await settle(() => inflight(home).a !== undefined && inflight(home).b === undefined)
+ await settle(t, () => inflight(home).a !== undefined && inflight(home).b === undefined)
})
test('a message a shutdown cancelled is sent again once; one still waiting, taken up or withdrawn is not', async (t) => {
@@ -474,7 +479,7 @@ test('a message a shutdown cancelled is sent again once; one still waiting, take
// Taken up by a turn, or withdrawn while DS Bot runs (the user stopped the Bot).
first.emit('b', 'user/message', { content: notesMessage.content, source: notesMessage.source })
for (const listener of first.ctx.listeners.get('agent/inbox/discarded')) listener({ message: withdrawn })
- await settle(() => (notes(home).queued.b ?? []).length === 2)
+ await settle(t, () => (notes(home).queued.b ?? []).length === 2)
await first.stop()
t.mock.timers.enable({ apis: ['setTimeout'] })
@@ -482,7 +487,7 @@ test('a message a shutdown cancelled is sent again once; one still waiting, take
// A hard stop keeps the inbox: that message runs when the Session wakes.
second.agentOf('b').inbox.nextTurn.push(numbers)
t.mock.timers.tick(3000)
- await settle(() => second.agentOf('b').calls.length === 1)
+ await settle(t, () => second.agentOf('b').calls.length === 1)
const [again] = second.agentOf('b').calls
assert.deepEqual([again.source.exchangeId, again.source.role, text(again)], [plan.source.exchangeId, 'request', text(plan)])
// The resend is the same exchange, not a new one.
@@ -497,7 +502,7 @@ test('a tool call written as text gets one more turn on the same round slot', as
const result = await openRoomTurn(t, host)
host.emit('gb', 'assistant/message', { message: { content: [{ type: 'text', text: WRITTEN }] } })
host.emit('gb', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
- await settle(() => host.agentOf('gb').calls.length === 2)
+ await settle(t, () => host.agentOf('gb').calls.length === 2)
const retry = host.agentOf('gb').calls[1]
assert.deepEqual([retry.source.role, retry.source.retried, retry.source.exchangeId], ['room', true, host.agentOf('gb').calls[0].source.exchangeId])
assert.match(text(retry), /^\[Group chat "Ops" · your tool call did not run\]/)
@@ -507,7 +512,7 @@ test('a tool call written as text gets one more turn on the same round slot', as
run(host, 'gb', 2, retry)
host.emit('gb', 'assistant/message', { message: { content: [{ type: 'text', text: WRITTEN }] } })
host.emit('gb', 'turn/end', { turn: 2, reason: { kind: 'completed' } })
- await settle(() => result() !== undefined)
+ await settle(t, () => result() !== undefined)
assert.deepEqual(result().replies.map(reply => reply.text), ['Checking now.'])
assert.equal(host.agentOf('gb').calls.length, 2)
})
@@ -520,7 +525,7 @@ test('in its own chat, a Bot that wrote a tool call as text is asked to make it'
host.emit('b', 'assistant/message', { message: { content: [{ type: 'text', text: 'On it.' }, { type: 'tool-call', id: 'k1', name: 'pwsh', arguments: '{}' }] } })
host.emit('b', 'assistant/message', { message: { content: [{ type: 'text', text: WRITTEN }] } })
host.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
- await settle(() => host.agentOf('b').calls.length === 1)
+ await settle(t, () => host.agentOf('b').calls.length === 1)
const retry = host.agentOf('b').calls[0]
assert.deepEqual([retry.source.role, retry.source.retried], ['retry', true])
assert.match(text(retry), /^\[Your last reply wrote a tool call as text, so nothing ran\]/)
diff --git a/ds-bot/test/learning.test.js b/ds-bot/test/learning.test.js
index 91f1e564..10b868b0 100644
--- a/ds-bot/test/learning.test.js
+++ b/ds-bot/test/learning.test.js
@@ -567,12 +567,21 @@ test('long turns ending during review remain pending for the next pass', async t
config: { learningTimes: { tickMs: 100_000 } },
})
t.after(() => runtime.stop())
+ // chatOwners is a derived index: load() and save() rebuild it from the saved
+ // team, so the extra Session standing in for a second chat of b2 goes back on
+ // after each rebuild.
const load = runtime.rt.load
runtime.rt.load = async (...args) => {
const result = await load(...args)
runtime.rt.chatOwners.set('b3', 'b2')
return result
}
+ const save = runtime.rt.save
+ runtime.rt.save = (...args) => {
+ const result = save(...args)
+ runtime.rt.chatOwners.set('b3', 'b2')
+ return result
+ }
runtime.rt.chatOwners.set('b3', 'b2')
const b3Statuses = []
runtime.ctx.on('agent/status', ({ agent, status }) => {
diff --git a/ds-bot/test/privacy.test.js b/ds-bot/test/privacy.test.js
index 3107059f..0115ea28 100644
--- a/ds-bot/test/privacy.test.js
+++ b/ds-bot/test/privacy.test.js
@@ -65,6 +65,7 @@ async function start({ services = {}, privateBot = true, archives } = {}) {
return agents.get(sessionId)
}
const registry = new Map([['w1', { id: 'w1', path: team, title: 'DS Bot' }]])
+ let created = 0
Object.assign(ctx, {
llm: { listProviders: () => [], listModels: async () => [], resolveModelInfo: async () => ({}) },
workspaceRegistry: {
@@ -79,7 +80,8 @@ async function start({ services = {}, privateBot = true, archives } = {}) {
archiveSession: async () => {},
},
sessionController: {
- create: async () => ({ sessionId: 's1' }),
+ // Ids shaped like real Session ids: a short one would match any path it lands in.
+ create: async () => ({ sessionId: `0000000${++created}-1111-4222-8333-444444444444` }),
resolveAgent: async sessionId => ({ agent: agentOf(sessionId) }),
selectModel: async () => {},
page: async () => ({ records: [], hasMore: false }),