Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Reconcile continued handoff work after plugin restarts

Fei2411c2b9a1b4fece
diff --cc ds-bot/src/host/resume.js
index 01ede55e,27e52740..e8b44c65
--- a/ds-bot/src/host/resume.js
+++ b/ds-bot/src/host/resume.js
@@@ -261,99 -211,6 +261,96 @@@ export function install(rt)
}
}
+ // A Session's durable log, as DSH keeps it: what happened there, and what it said.
+ // Undefined when this DSH cannot read it, or when the Session is gone.
+ async function logOf(sessionId) {
+ const inspect = ctx.sessionController?.inspect
+ if (typeof inspect !== 'function') return undefined
+ const seen = await inspect(sessionId, AbortSignal.timeout(HISTORY_TIMEOUT_MS)).catch(() => undefined)
+ return Array.isArray(seen?.events) ? seen.events : undefined
+ }
+
+ // A turn that ended while DS Bot was not watching it: its answer may exist and never
+ // have gone to the Bot that waits for it. The Session's durable log says what it did;
+ // the work it was for is settled once, the way a finished turn settles it.
+ async function reconcileEnded(sessionId, record, ended) {
+ const trigger = record.trigger
+ if (trigger === undefined) return
- if (trigger.role !== 'request' || trigger.audience === 'user') return
++ const workId = rt.workOf(trigger)
++ if (!rt.ownsWork(record.botId, workId)) return
+ const exchangeId = trigger.exchangeId
- const sender = trigger.senderSessionId
++ const asker = rt.workAsker(workId)
++ const sender = asker?.senderSessionId
+ if (exchangeId === undefined || sender === undefined || rt.botOf(sender) === undefined) return
- // The work it was for: gone when the answer already went up.
- const work = rt.state.works?.[exchangeId]
- if (work === undefined) return
+ const name = rt.botOf(rt.selfOf(sessionId))?.name ?? 'The other Bot'
- const asker = rt.workAsker(exchangeId)
- if (asker === undefined) return
+ // DSH stopped that turn as it shut down: it is picked up at the next start.
+ if (ended.reason?.kind === 'aborted' && ended.reason.reason?.kind === 'disposed') {
+ update(sessionId, () => record)
+ return
+ }
+ const events = await logOf(sessionId)
+ if (events === undefined) {
+ // The record of that work cannot be read: whether it answered cannot be
+ // established. The sender is told that, and the work is not run again.
+ const why = 'the record of that work could not be read'
+ const text = `${name} could not answer your message: ${why}, so what it did is unknown (its session is ${sessionId}). It may have done part of the work. Tell the user if it matters; do not run the work again without asking.`
+ const room = rt.roomOf(rt.groupOwners.get(sessionId)?.roomId)
+ rt.noteFailure?.({
+ level: 'warn',
+ source: 'turn',
+ text: `the result of the work ${name} was given is unknown`,
+ bot: name,
+ where: room === undefined ? 'own chat' : `group "${room.name}"`,
+ sessionId,
+ })
+ answerSender(rt, sessionId, asker, text)
+ return
+ }
+ const turn = turnOf(events, exchangeId)
+ // The request was never taken up: it is still waiting, and the delivery that is
+ // waiting for it keeps what it had.
+ if (turn === undefined) return
+ const said = turn.texts.join('\n\n')
+ // The answer went up in another run: the work closes with what it said, and
+ // nothing is sent twice.
- const back = rt.roomOf(trigger.senderRoomId)?.sessions?.[sender]
++ const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender]
+ const sent = await logOf(back ?? rt.chatOf(sender))
- if (sent !== undefined && answeredIn(sent, exchangeId)) {
- rt.closeWork?.(exchangeId, name, rt.answerText(exchangeId, said, name))
++ if (sent !== undefined && answeredIn(sent, workId)) {
++ rt.closeWork?.(workId, name, rt.answerText(workId, said, name))
+ return
+ }
+ const reason = turn.reason
+ // A turn that went on to a fresh part goes on there; it is not settled here.
+ if (reason?.kind === 'blocked') return
+ // The turn was cut off before it finished: the work is picked up again, as after
+ // a restart that cut it off.
+ if (reason === undefined || reason.kind === 'interrupted') {
+ await resumeTurn(sessionId, record)
+ return
+ }
+ if (reason.kind === 'error') {
+ const failure = turnFailure(reason.error)
+ answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`)
+ return
+ }
+ if (reason.kind === 'aborted') {
+ // The turn was stopped before it could answer: the sender is told, as it is told
+ // when DS Bot watches the turn end.
+ answerSender(rt, sessionId, asker, `${name} was stopped before it could answer your message.`)
+ return
+ }
+ // The turn ended with what it said: the answer goes up the way a finished turn's
+ // does, with the results that came back for it and the text it held. A work that
+ // still waits for what it handed on keeps its part until that comes back.
- if (rt.waitsWork(exchangeId)) {
- rt.holdWork(exchangeId, said)
++ if (rt.waitsWork(workId)) {
++ rt.holdWork(workId, said)
+ return
+ }
- const text = rt.answerText(exchangeId, said, name)
++ const text = rt.answerText(workId, said, name)
+ const long = text.length > MESSAGE_INLINE_MAX ? await rt.attachLong(text, `Answer from ${name}.txt`) : undefined
+ answerSender(rt, sessionId, asker, text, long)
+ }
+
async function resumeAll() {
await rt.load()
if (rt.readOnly() !== undefined) return
diff --cc ds-bot/test/follow-through.test.js
index b6aaffe0,8704795e..e95157c5
--- a/ds-bot/test/follow-through.test.js
+++ b/ds-bot/test/follow-through.test.js
@@@ -214,203 -195,6 +214,230 @@@ test('when only DS Bot restarts, a tur
assert.deepEqual([third.agentOf('b').calls[0].source.exchangeId, third.agentOf('a').calls.length], ['x1', 0])
})
+// Chief asks Writer for the plan; Writer's turn takes it up and the work waits.
+const askWriter = async (host, home) => {
+ await host.tool('message_bot', { to: 'Writer', message: 'Draft the plan' }, 'a')
+ const request = host.agentOf('b').calls[0]
+ run(host, 'b', 1, request)
+ await settle(() => inflight(home).b?.trigger?.exchangeId === request.source.exchangeId)
+ return request
+}
+
+// The durable log of Writer's Session: the turn that took the request up, and what it said.
+const writerLog = (request, said, reason = { kind: 'completed' }) => [
+ { type: 'turn/start', seq: 1, data: { turn: 1 } },
+ { type: 'user/message', seq: 2, data: { source: request.source, content: request.content } },
+ ...(said === undefined ? [] : [{ type: 'assistant/message', seq: 3, data: { turn: 1, message: { content: [{ type: 'text', text: said }] } } }]),
+ { type: 'turn/end', seq: 4, data: { turn: 1, reason } },
+]
+
++for (const role of ['reply', 'report', 'resume']) {
++ test(`a completed ${role} turn hands its continued work back after a plugin restart`, async (t) => {
++ t.mock.timers.enable({ apis: ['setTimeout'] })
++ const state = seed()
++ state.bots.c = { id: 'c', name: 'Reviewer', sessionId: 'c', createdAt: 3 }
++ state.works = { x1: { from: 'a', to: 'b', open: [], results: [{ id: 'y1', name: 'Reviewer', text: 'The numbers check out.' }], at: 1 } }
++ const home = freshHome(t, state)
++ const logs = new Map()
++ const first = await start(home, logs)
++ const request = { content: [{ type: 'text', text: 'Go on with the plan.' }], source: { kind: 'bot', form: 'relay', role, exchangeId: 'continuation-1', work: 'x1', ...(role === 'reply' ? { senderSessionId: 'c', replyTo: 'y1' } : {}) } }
++ run(first, 'b', 1, request)
++ await settle(() => inflight(home).b?.trigger?.work === 'x1')
++ logs.set('b', writerLog(request, 'The final plan is ready.'))
++ await first.stop()
++ const second = await start(home, logs)
++ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
++ t.mock.timers.tick(3000)
++ await settle(() => second.agentOf('a').calls.length === 1)
++ const reply = second.agentOf('a').calls[0]
++ assert.equal(reply.source.replyTo, 'x1')
++ assert.match(text(reply), /The final plan is ready/)
++ assert.match(text(reply), /The numbers check out/)
++ assert.equal(second.agentOf('c').calls.length, 0)
++ await settleState(t, home, () => stateOf(home).works?.x1 === undefined)
++ })
++}
++
+test('an answer a turn produced while DS Bot was not watching goes to the Bot that waits for it, once', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const home = freshHome(t)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = await askWriter(first, home)
+ const exchangeId = request.source.exchangeId
+ // Writer answers, and DS Bot restarts before it sees the end of that turn.
+ logs.set('b', writerLog(request, 'The plan is ready.'))
+ await first.stop()
+
+ const second = await start(home, logs)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
+ t.mock.timers.tick(3000)
+ await settle(() => second.agentOf('a').calls.length === 1)
+ const reply = second.agentOf('a').calls[0]
+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
+ assert.match(text(reply), /The plan is ready\./)
+ // The work is settled, and the request is not run again.
+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)
+ assert.equal(second.agentOf('b').calls.length, 0)
+
+ // Chief reads the answer; another restart finds nothing left to settle.
+ second.emit('a', 'user/message', { content: reply.content, source: reply.source })
+ await settleState(t, home, () => Object.keys(notes(home).queued ?? {}).length === 0)
+ await second.stop()
+ const third = await start(home, logs)
+ t.mock.timers.tick(3000)
+ await new Promise(resolve => setImmediate(resolve))
+ assert.deepEqual([third.agentOf('a').calls.length, third.agentOf('b').calls.length], [0, 0])
+})
+
+test('a turn whose record cannot be read says the result is unknown, and is not run again', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const home = freshHome(t)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = await askWriter(first, home)
+ const exchangeId = request.source.exchangeId
+ logs.set('b', writerLog(request, 'The plan is ready.'))
+ await first.stop()
+
+ // DSH cannot read that Session's record: whether the turn answered cannot be
+ // established, so the sender is told that, and the work is not run again.
+ const unreadable = async () => { throw new Error('session not found') }
+ const second = await start(home, logs, unreadable)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
+ t.mock.timers.tick(3000)
+ await settle(() => second.agentOf('a').calls.length === 1)
+ const reply = second.agentOf('a').calls[0]
+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
+ assert.match(text(reply), /what it did is unknown/)
+ // The work is settled, and the request is not run again.
+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)
+ assert.equal(second.agentOf('b').calls.length, 0)
+
+ // Another restart finds nothing left to settle: nothing is said or run twice.
+ second.emit('a', 'user/message', { content: reply.content, source: reply.source })
+ await settleState(t, home, () => Object.keys(notes(home).queued ?? {}).length === 0)
+ await second.stop()
+ const third = await start(home, logs)
+ t.mock.timers.tick(3000)
+ await new Promise(resolve => setImmediate(resolve))
+ assert.deepEqual([third.agentOf('a').calls.length, third.agentOf('b').calls.length], [0, 0])
+})
+
+test('a turn that ended without saying anything says it finished', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const home = freshHome(t)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = await askWriter(first, home)
+ const exchangeId = request.source.exchangeId
+ logs.set('b', writerLog(request, undefined))
+ await first.stop()
+
+ const second = await start(home, logs)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
+ t.mock.timers.tick(3000)
+ await settle(() => second.agentOf('a').calls.length === 1)
+ const reply = second.agentOf('a').calls[0]
+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
+ assert.match(text(reply), /Writer finished without a reply\./)
+ // The work is settled, and the request is not run again.
+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)
+ assert.equal(second.agentOf('b').calls.length, 0)
+})
+
+test('a turn that ended on the token limit gives what it said', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const home = freshHome(t)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = await askWriter(first, home)
+ const exchangeId = request.source.exchangeId
+ logs.set('b', writerLog(request, 'The plan is ready, up to the limit.', { kind: 'max-tokens' }))
+ await first.stop()
+
+ const second = await start(home, logs)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'max-tokens' } })
+ t.mock.timers.tick(3000)
+ await settle(() => second.agentOf('a').calls.length === 1)
+ const reply = second.agentOf('a').calls[0]
+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])
+ assert.match(text(reply), /The plan is ready, up to the limit\./)
+ // The work is settled, and the request is not run again.
+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)
+ assert.equal(second.agentOf('b').calls.length, 0)
+})
+
+test('the answer that goes up keeps what came back for the work', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const state = seed()
+ // Writer handed the plan on; what came back for it is kept with the work.
+ state.works = { x1: { from: 'a', to: 'b', open: [], results: [{ id: 'y1', name: 'Reviewer', text: 'The numbers check out.' }], at: 1 } }
+ const home = freshHome(t, state)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }
+ run(first, 'b', 1, request)
+ await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')
+ logs.set('b', writerLog(request, 'The plan is ready.'))
+ await first.stop()
+
+ const second = await start(home, logs)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
+ t.mock.timers.tick(3000)
+ await settle(() => second.agentOf('a').calls.length === 1)
+ const reply = text(second.agentOf('a').calls[0])
+ assert.match(reply, /The plan is ready\./)
+ assert.match(reply, /Reviewer:\nThe numbers check out\./)
+ // The work is settled, and the request is not run again.
+ await settleState(t, home, () => stateOf(home).works?.x1 === undefined)
+ assert.equal(second.agentOf('b').calls.length, 0)
+})
+
+test('an answer that already went up is not sent again', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const home = freshHome(t)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = await askWriter(first, home)
+ const exchangeId = request.source.exchangeId
+ logs.set('b', writerLog(request, 'The plan is ready.'))
+ // Chief already has the answer: another run delivered it.
+ logs.set('a', [{ type: 'user/message', seq: 1, data: { source: { kind: 'bot', role: 'reply', replyTo: exchangeId, senderSessionId: 'b' }, content: [{ type: 'text', text: '[Reply from Writer]\nThe plan is ready.' }] } }])
+ await first.stop()
+
+ const second = await start(home, logs)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
+ t.mock.timers.tick(3000)
+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)
+ assert.deepEqual([second.agentOf('a').calls.length, second.agentOf('b').calls.length], [0, 0])
+})
+
+test('a work that still waits for what it handed on holds its text, and is not settled', async (t) => {
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ const state = seed()
+ state.works = { x1: { from: 'a', to: 'b', open: ['y1'], results: [], at: 1 } }
+ const home = freshHome(t, state)
+ const logs = new Map()
+ const first = await start(home, logs)
+ const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }
+ run(first, 'b', 1, request)
+ await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')
+ logs.set('b', writerLog(request, 'The plan is ready.'))
+ await first.stop()
+
+ const second = await start(home, logs)
+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })
+ t.mock.timers.tick(3000)
+ await new Promise(resolve => setImmediate(resolve))
+ // The work waits for what Writer handed on: nothing is sent, and it stays open.
+ assert.deepEqual([second.agentOf('a').calls.length, second.agentOf('b').calls.length], [0, 0])
+ // What the turn said is held for it, until what it handed on comes back.
+ await settleState(t, home, () => stateOf(home).works.x1.held !== undefined)
+ assert.equal(stateOf(home).works.x1.held, 'The plan is ready.')
+ assert.deepEqual(stateOf(home).works.x1.open, ['y1'])
+})
+
test('a finished turn leaves nothing to pick up', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] })
const home = freshHome(t)