A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Reconcile continued handoff work after plugin restarts
Fei2411c2b9a1b4fece
diff --cc ds-bot/src/host/resume.jsindex 01ede55e,27e52740..e8b44c65--- a/ds-bot/src/host/resume.js+++ b/ds-bot/src/host/resume.js@@@ -261,99 -211,6 +261,96 @@@ export function install(rt)}}+ // A Session's durable log, as DSH keeps it: what happened there, and what it said.+ // Undefined when this DSH cannot read it, or when the Session is gone.+ async function logOf(sessionId) {+ const inspect = ctx.sessionController?.inspect+ if (typeof inspect !== 'function') return undefined+ const seen = await inspect(sessionId, AbortSignal.timeout(HISTORY_TIMEOUT_MS)).catch(() => undefined)+ return Array.isArray(seen?.events) ? seen.events : undefined+ }++ // A turn that ended while DS Bot was not watching it: its answer may exist and never+ // have gone to the Bot that waits for it. The Session's durable log says what it did;+ // the work it was for is settled once, the way a finished turn settles it.+ async function reconcileEnded(sessionId, record, ended) {+ const trigger = record.trigger+ if (trigger === undefined) return- if (trigger.role !== 'request' || trigger.audience === 'user') return++ const workId = rt.workOf(trigger)++ if (!rt.ownsWork(record.botId, workId)) return+ const exchangeId = trigger.exchangeId- const sender = trigger.senderSessionId++ const asker = rt.workAsker(workId)++ const sender = asker?.senderSessionId+ if (exchangeId === undefined || sender === undefined || rt.botOf(sender) === undefined) return- // The work it was for: gone when the answer already went up.- const work = rt.state.works?.[exchangeId]- if (work === undefined) return+ const name = rt.botOf(rt.selfOf(sessionId))?.name ?? 'The other Bot'- const asker = rt.workAsker(exchangeId)- if (asker === undefined) return+ // DSH stopped that turn as it shut down: it is picked up at the next start.+ if (ended.reason?.kind === 'aborted' && ended.reason.reason?.kind === 'disposed') {+ update(sessionId, () => record)+ return+ }+ const events = await logOf(sessionId)+ if (events === undefined) {+ // The record of that work cannot be read: whether it answered cannot be+ // established. The sender is told that, and the work is not run again.+ const why = 'the record of that work could not be read'+ const text = `${name} could not answer your message: ${why}, so what it did is unknown (its session is ${sessionId}). It may have done part of the work. Tell the user if it matters; do not run the work again without asking.`+ const room = rt.roomOf(rt.groupOwners.get(sessionId)?.roomId)+ rt.noteFailure?.({+ level: 'warn',+ source: 'turn',+ text: `the result of the work ${name} was given is unknown`,+ bot: name,+ where: room === undefined ? 'own chat' : `group "${room.name}"`,+ sessionId,+ })+ answerSender(rt, sessionId, asker, text)+ return+ }+ const turn = turnOf(events, exchangeId)+ // The request was never taken up: it is still waiting, and the delivery that is+ // waiting for it keeps what it had.+ if (turn === undefined) return+ const said = turn.texts.join('\n\n')+ // The answer went up in another run: the work closes with what it said, and+ // nothing is sent twice.- const back = rt.roomOf(trigger.senderRoomId)?.sessions?.[sender]++ const back = rt.roomOf(asker.senderRoomId)?.sessions?.[sender]+ const sent = await logOf(back ?? rt.chatOf(sender))- if (sent !== undefined && answeredIn(sent, exchangeId)) {- rt.closeWork?.(exchangeId, name, rt.answerText(exchangeId, said, name))++ if (sent !== undefined && answeredIn(sent, workId)) {++ rt.closeWork?.(workId, name, rt.answerText(workId, said, name))+ return+ }+ const reason = turn.reason+ // A turn that went on to a fresh part goes on there; it is not settled here.+ if (reason?.kind === 'blocked') return+ // The turn was cut off before it finished: the work is picked up again, as after+ // a restart that cut it off.+ if (reason === undefined || reason.kind === 'interrupted') {+ await resumeTurn(sessionId, record)+ return+ }+ if (reason.kind === 'error') {+ const failure = turnFailure(reason.error)+ answerSender(rt, sessionId, asker, `${name} could not answer your message: its model call failed (${failureLine(failure)}). Tell the user if it matters; do not retry right away.`)+ return+ }+ if (reason.kind === 'aborted') {+ // The turn was stopped before it could answer: the sender is told, as it is told+ // when DS Bot watches the turn end.+ answerSender(rt, sessionId, asker, `${name} was stopped before it could answer your message.`)+ return+ }+ // The turn ended with what it said: the answer goes up the way a finished turn's+ // does, with the results that came back for it and the text it held. A work that+ // still waits for what it handed on keeps its part until that comes back.- if (rt.waitsWork(exchangeId)) {- rt.holdWork(exchangeId, said)++ if (rt.waitsWork(workId)) {++ rt.holdWork(workId, said)+ return+ }- const text = rt.answerText(exchangeId, said, name)++ const text = rt.answerText(workId, said, name)+ const long = text.length > MESSAGE_INLINE_MAX ? await rt.attachLong(text, `Answer from ${name}.txt`) : undefined+ answerSender(rt, sessionId, asker, text, long)+ }+async function resumeAll() {await rt.load()if (rt.readOnly() !== undefined) returndiff --cc ds-bot/test/follow-through.test.jsindex b6aaffe0,8704795e..e95157c5--- a/ds-bot/test/follow-through.test.js+++ b/ds-bot/test/follow-through.test.js@@@ -214,203 -195,6 +214,230 @@@ test('when only DS Bot restarts, a turassert.deepEqual([third.agentOf('b').calls[0].source.exchangeId, third.agentOf('a').calls.length], ['x1', 0])})+// Chief asks Writer for the plan; Writer's turn takes it up and the work waits.+const askWriter = async (host, home) => {+ await host.tool('message_bot', { to: 'Writer', message: 'Draft the plan' }, 'a')+ const request = host.agentOf('b').calls[0]+ run(host, 'b', 1, request)+ await settle(() => inflight(home).b?.trigger?.exchangeId === request.source.exchangeId)+ return request+}++// The durable log of Writer's Session: the turn that took the request up, and what it said.+const writerLog = (request, said, reason = { kind: 'completed' }) => [+ { type: 'turn/start', seq: 1, data: { turn: 1 } },+ { type: 'user/message', seq: 2, data: { source: request.source, content: request.content } },+ ...(said === undefined ? [] : [{ type: 'assistant/message', seq: 3, data: { turn: 1, message: { content: [{ type: 'text', text: said }] } } }]),+ { type: 'turn/end', seq: 4, data: { turn: 1, reason } },+]+++for (const role of ['reply', 'report', 'resume']) {++ test(`a completed ${role} turn hands its continued work back after a plugin restart`, async (t) => {++ t.mock.timers.enable({ apis: ['setTimeout'] })++ const state = seed()++ state.bots.c = { id: 'c', name: 'Reviewer', sessionId: 'c', createdAt: 3 }++ state.works = { x1: { from: 'a', to: 'b', open: [], results: [{ id: 'y1', name: 'Reviewer', text: 'The numbers check out.' }], at: 1 } }++ const home = freshHome(t, state)++ const logs = new Map()++ const first = await start(home, logs)++ const request = { content: [{ type: 'text', text: 'Go on with the plan.' }], source: { kind: 'bot', form: 'relay', role, exchangeId: 'continuation-1', work: 'x1', ...(role === 'reply' ? { senderSessionId: 'c', replyTo: 'y1' } : {}) } }++ run(first, 'b', 1, request)++ await settle(() => inflight(home).b?.trigger?.work === 'x1')++ logs.set('b', writerLog(request, 'The final plan is ready.'))++ await first.stop()++ const second = await start(home, logs)++ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })++ t.mock.timers.tick(3000)++ await settle(() => second.agentOf('a').calls.length === 1)++ const reply = second.agentOf('a').calls[0]++ assert.equal(reply.source.replyTo, 'x1')++ assert.match(text(reply), /The final plan is ready/)++ assert.match(text(reply), /The numbers check out/)++ assert.equal(second.agentOf('c').calls.length, 0)++ await settleState(t, home, () => stateOf(home).works?.x1 === undefined)++ })++}+++test('an answer a turn produced while DS Bot was not watching goes to the Bot that waits for it, once', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const home = freshHome(t)+ const logs = new Map()+ const first = await start(home, logs)+ const request = await askWriter(first, home)+ const exchangeId = request.source.exchangeId+ // Writer answers, and DS Bot restarts before it sees the end of that turn.+ logs.set('b', writerLog(request, 'The plan is ready.'))+ await first.stop()++ const second = await start(home, logs)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })+ t.mock.timers.tick(3000)+ await settle(() => second.agentOf('a').calls.length === 1)+ const reply = second.agentOf('a').calls[0]+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])+ assert.match(text(reply), /The plan is ready\./)+ // The work is settled, and the request is not run again.+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)+ assert.equal(second.agentOf('b').calls.length, 0)++ // Chief reads the answer; another restart finds nothing left to settle.+ second.emit('a', 'user/message', { content: reply.content, source: reply.source })+ await settleState(t, home, () => Object.keys(notes(home).queued ?? {}).length === 0)+ await second.stop()+ const third = await start(home, logs)+ t.mock.timers.tick(3000)+ await new Promise(resolve => setImmediate(resolve))+ assert.deepEqual([third.agentOf('a').calls.length, third.agentOf('b').calls.length], [0, 0])+})++test('a turn whose record cannot be read says the result is unknown, and is not run again', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const home = freshHome(t)+ const logs = new Map()+ const first = await start(home, logs)+ const request = await askWriter(first, home)+ const exchangeId = request.source.exchangeId+ logs.set('b', writerLog(request, 'The plan is ready.'))+ await first.stop()++ // DSH cannot read that Session's record: whether the turn answered cannot be+ // established, so the sender is told that, and the work is not run again.+ const unreadable = async () => { throw new Error('session not found') }+ const second = await start(home, logs, unreadable)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })+ t.mock.timers.tick(3000)+ await settle(() => second.agentOf('a').calls.length === 1)+ const reply = second.agentOf('a').calls[0]+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])+ assert.match(text(reply), /what it did is unknown/)+ // The work is settled, and the request is not run again.+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)+ assert.equal(second.agentOf('b').calls.length, 0)++ // Another restart finds nothing left to settle: nothing is said or run twice.+ second.emit('a', 'user/message', { content: reply.content, source: reply.source })+ await settleState(t, home, () => Object.keys(notes(home).queued ?? {}).length === 0)+ await second.stop()+ const third = await start(home, logs)+ t.mock.timers.tick(3000)+ await new Promise(resolve => setImmediate(resolve))+ assert.deepEqual([third.agentOf('a').calls.length, third.agentOf('b').calls.length], [0, 0])+})++test('a turn that ended without saying anything says it finished', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const home = freshHome(t)+ const logs = new Map()+ const first = await start(home, logs)+ const request = await askWriter(first, home)+ const exchangeId = request.source.exchangeId+ logs.set('b', writerLog(request, undefined))+ await first.stop()++ const second = await start(home, logs)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })+ t.mock.timers.tick(3000)+ await settle(() => second.agentOf('a').calls.length === 1)+ const reply = second.agentOf('a').calls[0]+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])+ assert.match(text(reply), /Writer finished without a reply\./)+ // The work is settled, and the request is not run again.+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)+ assert.equal(second.agentOf('b').calls.length, 0)+})++test('a turn that ended on the token limit gives what it said', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const home = freshHome(t)+ const logs = new Map()+ const first = await start(home, logs)+ const request = await askWriter(first, home)+ const exchangeId = request.source.exchangeId+ logs.set('b', writerLog(request, 'The plan is ready, up to the limit.', { kind: 'max-tokens' }))+ await first.stop()++ const second = await start(home, logs)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'max-tokens' } })+ t.mock.timers.tick(3000)+ await settle(() => second.agentOf('a').calls.length === 1)+ const reply = second.agentOf('a').calls[0]+ assert.deepEqual([reply.source.role, reply.source.replyTo], ['reply', exchangeId])+ assert.match(text(reply), /The plan is ready, up to the limit\./)+ // The work is settled, and the request is not run again.+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)+ assert.equal(second.agentOf('b').calls.length, 0)+})++test('the answer that goes up keeps what came back for the work', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const state = seed()+ // Writer handed the plan on; what came back for it is kept with the work.+ state.works = { x1: { from: 'a', to: 'b', open: [], results: [{ id: 'y1', name: 'Reviewer', text: 'The numbers check out.' }], at: 1 } }+ const home = freshHome(t, state)+ const logs = new Map()+ const first = await start(home, logs)+ const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }+ run(first, 'b', 1, request)+ await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')+ logs.set('b', writerLog(request, 'The plan is ready.'))+ await first.stop()++ const second = await start(home, logs)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })+ t.mock.timers.tick(3000)+ await settle(() => second.agentOf('a').calls.length === 1)+ const reply = text(second.agentOf('a').calls[0])+ assert.match(reply, /The plan is ready\./)+ assert.match(reply, /Reviewer:\nThe numbers check out\./)+ // The work is settled, and the request is not run again.+ await settleState(t, home, () => stateOf(home).works?.x1 === undefined)+ assert.equal(second.agentOf('b').calls.length, 0)+})++test('an answer that already went up is not sent again', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const home = freshHome(t)+ const logs = new Map()+ const first = await start(home, logs)+ const request = await askWriter(first, home)+ const exchangeId = request.source.exchangeId+ logs.set('b', writerLog(request, 'The plan is ready.'))+ // Chief already has the answer: another run delivered it.+ logs.set('a', [{ type: 'user/message', seq: 1, data: { source: { kind: 'bot', role: 'reply', replyTo: exchangeId, senderSessionId: 'b' }, content: [{ type: 'text', text: '[Reply from Writer]\nThe plan is ready.' }] } }])+ await first.stop()++ const second = await start(home, logs)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })+ t.mock.timers.tick(3000)+ await settleState(t, home, () => stateOf(home).works?.[exchangeId] === undefined)+ assert.deepEqual([second.agentOf('a').calls.length, second.agentOf('b').calls.length], [0, 0])+})++test('a work that still waits for what it handed on holds its text, and is not settled', async (t) => {+ t.mock.timers.enable({ apis: ['setTimeout'] })+ const state = seed()+ state.works = { x1: { from: 'a', to: 'b', open: ['y1'], results: [], at: 1 } }+ const home = freshHome(t, state)+ const logs = new Map()+ const first = await start(home, logs)+ const request = { content: [{ type: 'text', text: '[Message from Chief]\nDraft the plan' }], source: { kind: 'bot', form: 'relay', role: 'request', exchangeId: 'x1', senderName: 'Chief', senderSessionId: 'a', hop: 0 } }+ run(first, 'b', 1, request)+ await settle(() => inflight(home).b?.trigger?.exchangeId === 'x1')+ logs.set('b', writerLog(request, 'The plan is ready.'))+ await first.stop()++ const second = await start(home, logs)+ second.emit('b', 'turn/end', { turn: 1, reason: { kind: 'completed' } })+ t.mock.timers.tick(3000)+ await new Promise(resolve => setImmediate(resolve))+ // The work waits for what Writer handed on: nothing is sent, and it stays open.+ assert.deepEqual([second.agentOf('a').calls.length, second.agentOf('b').calls.length], [0, 0])+ // What the turn said is held for it, until what it handed on comes back.+ await settleState(t, home, () => stateOf(home).works.x1.held !== undefined)+ assert.equal(stateOf(home).works.x1.held, 'The plan is ready.')+ assert.deepEqual(stateOf(home).works.x1.open, ['y1'])+})+test('a finished turn leaves nothing to pick up', async (t) => {t.mock.timers.enable({ apis: ['setTimeout'] })const home = freshHome(t)