Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Judge part recovery by the replacement context

Fei2411c9ec1c32af40
diff --git a/ds-bot/src/host/parts.js b/ds-bot/src/host/parts.js
index 02387126..7a109ba3 100644
--- a/ds-bot/src/host/parts.js
+++ b/ds-bot/src/host/parts.js
@@ -208,12 +208,34 @@ export function install(rt) {
}
}
const clock = createClock(config.timeZone)
- // A fresh part has only its handoff and perhaps our synthetic acknowledgement.
- // Replacing it again cannot make room until the model has added real work.
- const canMakeRoom = (session) => {
+ // Replacing a handoff must shrink the request, not just create another Session.
+ async function canMakeRoom(agent, carry = []) {
+ const session = agent.session
const messages = session.deriveMessages()
- return !messages.some(message => message.source?.handoff === true)
- || messages.some(message => message.role === 'assistant' && textOf(message.content) !== HANDOFF_ACK)
+ if (!messages.some(message => message.source?.handoff === true)) return true
+ const place = placeOf(agent.id)
+ const room = rt.roomOf(place.roomId)
+ const parts = partsOf(place.botId, place.roomId)
+ const previous = parts.at(-2)
+ const lines = await contextLines(session).catch(() => undefined)
+ const budget = Math.min(RECENT_TOKENS, Math.floor((lines?.check ?? 4 * RECENT_TOKENS) / 4))
+ const text = handoffPackage({
+ room,
+ segment: { fallback: 'overflow', ...(previous?.note ? { note: previous.note, noteFrom: previous.noteFrom ?? previous.number } : {}) },
+ lines: await handoffLines(agent.id, { budget, group: room !== undefined, number: parts.length }),
+ number: parts.length + 1,
+ resume: true,
+ overflow: true,
+ })
+ const meter = ctx.get('tokenMeter')
+ const size = message => meter?.estimateMessage(message) ?? estimateTokens(textOf(message.content))
+ const history = messages.filter(message => message.role !== 'system').reduce((total, message) => total + size(message), 0)
+ const pending = carry.reduce((total, message) => total + size(message), 0)
+ const replacement = size({ role: 'user', content: [{ type: 'text', text }] }) + pending
+ const tokens = tokensOf(session)
+ // The system prompt and tool definitions stay; pending input is forwarded whole.
+ return replacement < history + pending
+ && (lines === undefined || tokens === undefined || tokens - history + replacement <= lines.usable)
}
// Lines for the model the Session last routed to; undefined while it has not
@@ -1039,8 +1061,9 @@ export function install(rt) {
if (!readOnly() && await folderMove(agent, { midTurn: payload.step > 1 }) !== undefined) return switchBeforeStep(agent, payload, { folder: true })
const lines = await contextLines(agent.session).catch(() => undefined)
const tokens = tokensOf(agent.session)
- if (lines === undefined || tokens === undefined || tokens < lines.hard || !grown(agent.id, tokens, lines) || readOnly() || !canMakeRoom(agent.session)) return next()
+ if (lines === undefined || tokens === undefined || tokens < lines.hard || !grown(agent.id, tokens, lines) || readOnly()) return next()
if (await decide(agent, { midTurn: payload.step > 1 }) === 'compact' && await condense(agent, lines, { inTurn: true, signal: payload.signal })) return next()
+ if (!await canMakeRoom(agent, payload.messages)) return next()
return switchBeforeStep(agent, payload, { hard: true })
}, { prepend: true })
@@ -1135,7 +1158,7 @@ export function install(rt) {
if (payload.failure?.code !== OVERFLOW || payload.signal?.aborted) return next()
await rt.load()
const { agent } = payload
- if (!rt.isBotSession(agent.id) || switches.has(agent.id) || readOnly() || !canMakeRoom(agent.session)) return next()
+ if (!rt.isBotSession(agent.id) || switches.has(agent.id) || readOnly() || !await canMakeRoom(agent)) return next()
const entry = rt.liveOf(agent.id)
entry.switched = true
startSwitch(agent, { resume: true, info: entry.info, joined: [...(entry.joined ?? [])], overflow: true })
@@ -1150,7 +1173,7 @@ export function install(rt) {
if (lastTurnEnd(agent)?.data?.reason?.kind === 'error') return
const lines = await contextLines(agent.session)
const tokens = tokensOf(agent.session)
- if (lines === undefined || tokens === undefined || tokens < lines.check || !grown(agent.id, tokens, lines) || !canMakeRoom(agent.session)) return
+ if (lines === undefined || tokens === undefined || tokens < lines.check || !grown(agent.id, tokens, lines)) return
const hard = tokens >= lines.hard
if (!hard && Date.now() < (noteFailures.get(agent.id)?.until ?? 0)) return
checking.add(agent.id)
@@ -1160,6 +1183,7 @@ export function install(rt) {
if (condensed !== false) return
}
if (switches.has(agent.id)) return
+ if (!await canMakeRoom(agent)) return
startSwitch(agent, { hard: (tokensOf(agent.session) ?? tokens) >= lines.hard })
} finally {
checking.delete(agent.id)
diff --git a/ds-bot/test/bot-messages.test.js b/ds-bot/test/bot-messages.test.js
index 9d5589d0..d57fc5d3 100644
--- a/ds-bot/test/bot-messages.test.js
+++ b/ds-bot/test/bot-messages.test.js
@@ -9,6 +9,7 @@ import { fakeAttachments, fakeContext } from './fake-ctx.js'
import { install as installTurns } from '../src/host/turns.js'
import { install as installParts } from '../src/host/parts.js'
import { HANDOFF_ACK, SOURCE_KIND } from '../src/host/constants.js'
+import { estimateTokens } from '../src/host/text.js'
// A two-Bot team, optionally with b in a group Session 'gb' of room 'Ops'.
function team({ group = false } = {}) {
@@ -953,14 +954,17 @@ test('a failed turn past the hard line keeps the real error instead of switching
})
for (const id of ['b', 'gb']) {
- test(`a fresh part with oversized tools surfaces overflow, and can switch again after real work (${id})`, async (t) => {
+ test(`a fresh part with oversized tools stops until the replacement context can fit (${id})`, async (t) => {
const { ctx, agentOf, emit, fire, stop } = await start(team({ group: true }))
t.after(stop)
let made = 0
let tokens = 272_000
ctx.sessionController.create = async () => ({ sessionId: `s${++made}` })
const get = ctx.get
- ctx.get = name => name === 'tokenMeter' ? { measure: () => ({ totalTokens: tokens }) } : get(name)
+ ctx.get = name => name === 'tokenMeter' ? {
+ measure: () => ({ totalTokens: tokens }),
+ estimateMessage: message => estimateTokens(textOf(message.content)),
+ } : get(name)
ctx.llm.resolveModelInfo = async () => ({ context: { contextWindow: 192_000 } })
const preStep = async (agent) => {
agent.session.requestHeader = () => ({ config: { provider: 'custom', model: 'model' } })
@@ -992,12 +996,73 @@ for (const id of ['b', 'gb']) {
await delay(30)
assert.equal(made, 1)
- // Once the user fixes the cause and the model works, later overflow can recover.
- emit('s1', 'assistant/message', assistantText('Work after reducing the tool set'))
+ // A model reply alone does not fix a tool set that still exceeds the window.
+ emit('s1', 'assistant/message', assistantText('A short reply'))
+ await overflow(agentOf('s1'))
+ await delay(30)
+ assert.equal(made, 1)
+
+ // Once the tool set fits, shortening the conversation can help again.
+ tokens = 1_000
await overflow(agentOf('s1'))
await waitForHandoff('s2')
assert.equal(made, 2)
})
+
+ for (const pending of [false, true]) {
+ test(`a long ${pending ? 'pending' : 'admitted'} request after a handoff can recover without a model reply (${id})`, async (t) => {
+ const seed = team({ group: true })
+ const segment = { sessionId: 'old', note: 'Keep working on the request.', endedAt: 1 }
+ if (id === 'gb') seed.rooms.r1.segments = { b: [segment] }
+ else seed.bots.b.segments = [segment]
+ const { ctx, agentOf, emit, fire, stop } = await start(seed)
+ t.after(stop)
+ let made = 0
+ ctx.sessionController.create = async () => ({ sessionId: `s${++made}` })
+ const get = ctx.get
+ ctx.get = name => name === 'tokenMeter' ? {
+ measure: session => ({ totalTokens: 9_000 + estimateTokens(session.deriveMessages().map(message => textOf(message.content)).join('\n')) }),
+ estimateMessage: message => estimateTokens(textOf(message.content)),
+ } : get(name)
+ ctx.llm.resolveModelInfo = async () => ({ context: { contextWindow: 50_000 } })
+ ctx.llm.stream = async function* () {
+ yield { type: 'finish', reason: { kind: 'error', failure: { code: 'CONTEXT_WINDOW_EXCEEDED', message: 'Context window exceeded' } } }
+ }
+ const agent = agentOf(id)
+ agent.session.requestHeader = () => ({ config: { provider: 'custom', model: 'model' } })
+ agent.session.toolHistory = () => []
+ const events = []
+ const record = (type, data) => {
+ events.push({ type, data, seq: events.length, time: 1 })
+ agent.session.seq = events.length
+ emit(id, type, data)
+ }
+ ctx.sessionController.page = async ({ address, beforeSeq }) => ({
+ records: address.sessionId === id ? events.filter(event => beforeSeq === undefined || event.seq < beforeSeq).map(event => ({ event })) : [],
+ hasMore: false,
+ })
+ record('user/message', { source: { kind: SOURCE_KIND, handoff: true }, content: textBlock('Earlier context. '.padEnd(120_000, 'x')) })
+ record('assistant/message', assistantText(HANDOFF_ACK))
+ const input = { role: 'user', source: { kind: 'user' }, content: textBlock('New request. '.padEnd(20_000, 'y')) }
+ assert.ok(estimateTokens(textOf(input.content)) < 50_000, 'the new request fits on its own')
+ assert.ok(ctx.get('tokenMeter').measure(agent.session).totalTokens + estimateTokens(textOf(input.content)) > 50_000, 'it overflows together with the handoff')
+
+ if (pending) {
+ const decision = await ctx.listeners.get('agent/pre-step').at(-1)({ agent, messages: [input], step: 1, signal: AbortSignal.timeout(60_000) }, async () => 'next')
+ assert.deepEqual(decision, { kind: 'reject' })
+ } else {
+ record('user/message', input)
+ await fire('agent/request-error', { agent, failure: { code: 'CONTEXT_WINDOW_EXCEEDED' }, signal: AbortSignal.timeout(60_000) })
+ }
+ const ready = () => agentOf('s1').calls.some(([, message]) => message.source?.handoff)
+ && (!pending || agentOf('s1').calls.some(([, message]) => textOf(message.content) === textOf(input.content)))
+ for (let count = 0; count < 200 && !ready(); count += 1) await delay(5)
+ assert.ok(ready(), 'the shorter part gets the handoff and any still-pending input')
+ assert.equal(made, 1)
+ const handoff = agentOf('s1').calls.find(([, message]) => message.source?.handoff)[1]
+ assert.ok(textOf(handoff.content).length < 10_000, 'the handoff replaces the 140k-character context')
+ })
+ }
}
// #206: a message between Bots used to go whole into state.json, which is written whole