A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
import assert from 'node:assert/strict'
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { test } from 'node:test'
import { apply } from '../index.js'
import { foldCalls } from '../src/host/usage.js'
import { fakeContext } from './fake-ctx.js'
const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } })
const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } })
const streamed = (type, turn, step, usage, time = 0) => ({ type, time, data: { turn, step, stream: [{ type: 'chunk', chunk: { type: 'usage', usage } }] } })
const usage = (inputTokens, outputTokens, cacheReadTokens = 0) => ({ inputTokens, outputTokens, cacheReadTokens })
const tokens = row => [row.model, row.input, row.cacheRead, row.output]
test('each billed attempt is one row, on the model its request named', () => {
const { rows, state } = foldCalls([
header('deepseek', 'v4-flash'),
// A failed attempt, then its retry: two billed attempts in one step.
streamed('assistant/attempt', 1, 0, usage(10, 0)),
{ type: 'llm/retry-started', data: { turn: 1, step: 0 } },
message(1, 0, usage(20, 3, 5)),
// A later sample of the same attempt replaces the earlier one.
streamed('assistant/message', 1, 1, usage(30, 1)),
message(1, 1, usage(30, 4, 2)),
header('deepseek', 'v4-pro'),
streamed('assistant/message', 2, 0, usage(7, 7)),
// No usage reported: nothing billed to count.
message(2, 1, undefined),
])
assert.deepEqual(rows.map(tokens), [['v4-flash', 10, 0, 0], ['v4-flash', 20, 5, 3], ['v4-flash', 30, 2, 4], ['v4-pro', 7, 0, 7]])
assert.deepEqual(state, { provider: 'deepseek', model: 'v4-pro', retry: { turn: 1, step: 0, count: 1 } })
})
test('a fold goes on from where the last read stopped', () => {
const first = foldCalls([header('p', 'm'), streamed('assistant/attempt', 3, 2, usage(1, 0)), { type: 'llm/retry-started', data: { turn: 3, step: 2 } }])
const { rows } = foldCalls([message(3, 2, usage(2, 2))], first.state)
assert.deepEqual(rows.map(row => [row.key, ...tokens(row)]), [['3:2:1', 'm', 2, 0, 2]])
})
test('a fork does not count the calls it inherited from its parent', () => {
const { rows, reset } = foldCalls([header('p', 'm'), message(1, 0, usage(100, 100)), { type: 'session/end-seed', data: { inherited: true } }, message(2, 0, usage(1, 1))])
assert.equal(reset, true)
assert.deepEqual(rows.map(tokens), [['m', 1, 0, 1]])
})
const HOUR = 3_600_000
const LONG_AGO = 1_000
function start(home, { listedExtra, logsExtra, subagentsExtra } = {}) {
const fresh = home === undefined
home ??= mkdtempSync(join(tmpdir(), 'ds-bot-usage-'))
if (fresh) writeFileSync(join(home, 'state.json'), JSON.stringify({
version: 1,
mainBotIds: ['b1'],
revision: 1,
bots: {
b1: { id: 'b1', sessionId: 's-b1', segments: [{ sessionId: 'b1', endedAt: LONG_AGO }], name: 'Chief', instructions: '', createdAt: 1 },
b2: { id: 'b2', sessionId: 'b2', name: 'Writer', instructions: '', createdAt: 2 },
},
rooms: { r1: { id: 'r1', name: '研发群', named: true, members: ['b1', 'b2'], admin: 'b1', mode: 'everyone', notice: '', sessions: { b1: 's-g1' }, log: [], createdAt: 3 } },
agents: { a1: { id: 'a1', name: 'DSH Agent', sessions: ['s-a1'], createdAt: 4 } },
exchanges: [],
exchangeTurns: {},
questions: {},
whaleStored: true,
}))
const logs = {
'b1': [header('deepseek', 'v4-flash'), message(1, 0, usage(100, 10, 50), 0)],
's-b1': [header('deepseek', 'v4-flash'), message(1, 0, usage(200, 20), HOUR)],
's-g1': [header('deepseek', 'v4-pro'), message(1, 0, usage(300, 30), HOUR)],
's-sub': [header('deepseek', 'v4-flash'), message(1, 0, usage(5, 5), HOUR)],
's-x': [header('kimi', 'k3'), message(1, 0, usage(40, 4), 2 * HOUR)],
's-a1': [header('deepseek', 'v4-flash'), message(1, 0, usage(70, 7), HOUR)],
...logsExtra,
}
for (const log of Object.values(logs)) log.forEach((event, seq) => { event.seq = seq })
// Child Session → the parent its header names, as DSH checks a page address.
const subagents = { 's-sub': 's-b1', ...subagentsExtra }
const pages = []
const warnings = []
const ctx = fakeContext()
ctx.logger.warn = (template, ...args) => warnings.push([template, ...args])
ctx.sessionController = {
list: async () => ({ items: [
{ sessionId: 's-x', projections: { kind: 'cached', asOfSeq: 1, values: {} } },
{ sessionId: 's-sub', parentSessionId: 's-b1', origin: 'subagent', projections: { kind: 'cached', asOfSeq: 1, values: {} } },
{ sessionId: 's-empty', blank: true },
...(listedExtra ?? []),
] }),
projections: async ({ sessionId }) => (logs[sessionId] ? { asOfSeq: logs[sessionId].length - 1, values: {} } : null),
page: async ({ address, throughSeq, beforeSeq }) => {
const sessionId = address.kind === 'session' ? address.sessionId : address.childSessionId
if (address.kind === 'session' && sessionId in subagents) throw new Error('subagent Sessions require their durable parent address')
if (address.kind === 'subagent' && subagents[sessionId] !== address.parentSessionId) throw new Error('subagent does not belong to the supplied parent')
if (logs[sessionId] === undefined) throw new Error(`no log for ${sessionId}`)
pages.push(sessionId)
const events = logs[sessionId].filter(event => event.seq <= throughSeq && (beforeSeq === undefined || event.seq < beforeSeq))
return { records: events.map(event => ({ event })), hasMore: false }
},
}
let route
ctx.connection.fetch.register = (options) => { route = options; return () => {} }
apply(ctx, { home })
const call = async (endpoint, payload) => {
const response = await route.fetch(new Request('http://localhost/api/bot', { method: 'POST', body: JSON.stringify({ endpoint, payload }) }))
return response.json()
}
const meter = ctx.listeners.get('llm/stream').at(-1)
const stop = async () => {
await ctx.dispose()
rmSync(home, { recursive: true, force: true, maxRetries: 5 })
}
return { call, ctx, home, meter, pages, stop, warnings }
}
// Total tokens per owner name and use.
const totals = (view) => {
const sums = {}
for (const [, owner, , kind, input, cacheRead, cacheWrite, output] of view.rows) {
const name = view.owners[owner].name ?? view.owners[owner].key
const key = `${name}/${view.kinds[kind]}`
sums[key] = (sums[key] ?? 0) + input + cacheRead + cacheWrite + output
}
return sums
}
// Sessions whose owner the ledger holds; none until the ledger has its table.
async function journaledSessions(home) {
const path = join(home, 'usage.sqlite')
if (!existsSync(path)) return []
const { DatabaseSync } = await import('node:sqlite')
const db = new DatabaseSync(path)
try {
if (db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'owners'").all().length === 0) return []
return db.prepare('SELECT session FROM owners').all().map(row => row.session)
} finally {
db.close()
}
}
async function drain(stream) {
const chunks = []
for await (const chunk of stream) chunks.push(chunk)
return chunks
}
test('the usage view counts every Session under its owner, by hour in the asked time zone', async (t) => {
const { call, meter, pages, stop } = start()
t.after(stop)
const view = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
// Chief: both parts of its own chat and its subagent; its group Session counts as group use.
// The DSH Agent's Sessions count beside the Bots, not with the other sessions.
assert.deepEqual(totals(view), { 'Chief/chat': 160 + 220 + 10, 'Chief/group': 330, 'DSH Agent/chat': 77, 'other/chat': 44 })
assert.deepEqual(view.rows.map(row => row[0]), ['1970-01-01 08', '1970-01-01 09', '1970-01-01 09', '1970-01-01 09', '1970-01-01 10'])
assert.deepEqual(view.models.map(model => model.key).sort(), ['deepseek/v4-flash', 'deepseek/v4-pro', 'kimi/k3'])
assert.equal(view.scanning, false)
assert.ok(!pages.includes('s-empty'))
// A Session's own requests are in its log, so the meter lets them through untouched.
const passed = (async function* () { yield { type: 'usage', usage: usage(1, 1) } })()
assert.equal(meter({ sessionId: 's-b1', messages: [] }, () => passed), passed)
// A checkpoint is outside every log: the meter counts it, once.
const chunk = { type: 'usage', usage: usage(1000, 100) }
const chunks = await drain(meter({ sessionId: 's-b1', purpose: 'compaction', provider: 'deepseek', model: 'v4-flash', messages: [] }, async function* () { yield chunk }))
assert.deepEqual(chunks, [chunk])
await drain(meter({ sessionId: 's-b1', purpose: 'compaction', provider: 'deepseek', model: 'v4-flash', messages: [] }, async function* () { yield chunk }))
const again = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(again)['Chief/compaction'], 1100)
})
test('a call made in a group chat stays group use after the room is gone', async (t) => {
const first = start()
const { home } = first
t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))
const before = (await first.call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(before)['Chief/group'], 330)
await first.ctx.dispose()
const state = JSON.parse(readFileSync(join(home, 'state.json'), 'utf8'))
state.rooms = {}
writeFileSync(join(home, 'state.json'), JSON.stringify(state))
const second = start(home)
const after = (await second.call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(after)['Chief/group'], 330)
await second.ctx.dispose()
})
test('a plain chat journals its owner before the first scan, so a deleted Bot keeps its name', async (t) => {
const first = start()
const { home } = first
t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))
// Writer talks once, Usage is never opened: the meter lets the request
// through, but the Session's owner lands in the ledger on its own.
const passed = (async function* () { yield { type: 'text', text: 'hi' } })()
assert.equal(first.meter({ sessionId: 'b2', messages: [] }, () => passed), passed)
// The owner is written once the team and the ledger have loaded behind the meter:
// wait for its row, not for a fixed time, before the shutdown closes the ledger.
for (let tries = 0; !(await journaledSessions(home)).includes('b2'); tries += 1) {
assert.ok(tries < 200, 'the owner of b2 was never journaled')
await new Promise(resolve => setTimeout(resolve, 20))
}
await first.ctx.dispose()
const state = JSON.parse(readFileSync(join(home, 'state.json'), 'utf8'))
delete state.bots.b2
writeFileSync(join(home, 'state.json'), JSON.stringify(state))
// Writer's old Session is still listed (archived, not deleted), so the scan
// reads its log and the journaled owner attributes it to the deleted Bot.
const second = start(home, {
listedExtra: [{ sessionId: 'b2', projections: { kind: 'cached', asOfSeq: 1, values: {} } }],
logsExtra: { b2: [header('deepseek', 'v4-flash'), message(1, 0, usage(50, 5), HOUR)] },
})
const view = (await second.call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(view)['Writer/chat'], 55)
await second.ctx.dispose()
})
test('a subagent Session is read through its parent, even a parent that is gone; one that cannot be read warns once', async (t) => {
t.mock.timers.enable({ apis: ['Date'], now: 10 * HOUR })
const { call, pages, stop, warnings } = start(undefined, {
listedExtra: [
{ sessionId: 's-orphan', parentSessionId: 's-gone', origin: 'subagent', projections: { kind: 'cached', asOfSeq: 1, values: {} } },
{ sessionId: 's-lost', origin: 'subagent', projections: { kind: 'cached', asOfSeq: 1, values: {} } },
{ sessionId: 's-broken', projections: { kind: 'sequenced', asOfSeq: 1, values: {} } },
{ sessionId: 's-broken2', projections: { kind: 'sequenced', asOfSeq: 1, values: {} } },
],
logsExtra: {
's-orphan': [header('deepseek', 'v4-flash'), message(1, 0, usage(8, 1), HOUR)],
's-lost': [header('deepseek', 'v4-flash'), message(1, 0, usage(9, 9), HOUR)],
},
subagentsExtra: { 's-orphan': 's-gone', 's-lost': 's-gone' },
})
t.after(stop)
const view = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(view)['Chief/chat'], 160 + 220 + 10)
assert.equal(totals(view)['other/chat'], 44 + 9)
// No parent on record: it cannot be addressed, so it is not asked for.
assert.ok(!pages.includes('s-lost'))
// A scan reports one summarized line, not one per Session it could not read.
const unread = () => warnings
.map(([template, ...args]) => template === '%s' ? args[0] : template)
.filter(line => line.includes('usage could not read'))
assert.equal(unread().length, 1)
assert.match(unread()[0], /usage could not read 2 Sessions: s-broken, s-broken2 \(first error: /)
t.mock.timers.tick(60_000)
await call('usage', { timeZone: 'Asia/Shanghai' })
assert.equal(pages.filter(sessionId => sessionId === 's-orphan').length, 1)
assert.equal(unread().length, 1)
})
// #654: a Session with no Agent attached seals once read to its tip, at the checkpoint
// watermark its listing shows. Later scans skip it without a read while that watermark
// stands; a warm scan clears the seal, and a moved watermark means the log grew.
test('a cold Session seals at its checkpoint watermark and is read again once it warms or the watermark moves', async (t) => {
t.mock.timers.enable({ apis: ['Date'], now: 10 * HOUR })
const logX = [header('kimi', 'k3'), message(1, 0, usage(40, 4), 2 * HOUR)]
const { call, ctx, stop } = start(undefined, { logsExtra: { 's-x': logX } })
t.after(stop)
let reads = 0
const projections = ctx.sessionController.projections
ctx.sessionController.projections = async (request) => {
if (request.sessionId === 's-x') reads += 1
return projections(request)
}
let agentAvailable = false
let watermark = 1
ctx.sessionController.list = async () => ({
items: [{ sessionId: 's-x', agentAvailable, running: false, projections: { kind: 'cached', asOfSeq: watermark, values: {} } }],
})
const scan = async () => {
t.mock.timers.tick(60_000)
return (await call('usage', { timeZone: 'Asia/Shanghai' })).value
}
// First scan: cold but never read, so read once and sealed.
await call('usage', { timeZone: 'Asia/Shanghai' })
assert.equal(reads, 1)
// Still cold at the same watermark: the seal skips it without any read.
await scan()
assert.equal(reads, 1)
// Listed warm, a scan reads it and clears the seal, since it may run.
agentAvailable = true
await scan()
assert.equal(reads, 2)
// Cold again: read once more, then sealed at the watermark and skipped.
agentAvailable = false
await scan()
assert.equal(reads, 3)
await scan()
assert.equal(reads, 3)
// The log grew and its checkpoint moved on: read again, and the new calls count.
logX.push({ ...message(2, 0, usage(7, 7)), seq: 2 })
watermark = 2
const view = await scan()
assert.equal(reads, 4)
assert.equal(totals(view)['other/chat'], 44 + 14)
await scan()
assert.equal(reads, 4)
})
// The ledger of the previous version has no seal watermark; it is kept, not rebuilt.
test('a ledger from the previous version keeps its metered calls and gains the seal watermark', async (t) => {
const first = start()
const { home } = first
t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))
const chunk = { type: 'usage', usage: usage(1000, 100) }
await drain(first.meter({ sessionId: 's-b1', purpose: 'compaction', provider: 'deepseek', model: 'v4-flash', messages: [] }, async function* () { yield chunk }))
await first.call('usage', { timeZone: 'Asia/Shanghai' })
await first.ctx.dispose()
const { DatabaseSync } = await import('node:sqlite')
const db = new DatabaseSync(join(home, 'usage.sqlite'))
db.exec('ALTER TABLE marks DROP COLUMN listed; PRAGMA user_version = 2')
db.close()
const second = start(home)
const view = (await second.call('usage', { timeZone: 'Asia/Shanghai' })).value
await second.ctx.dispose()
assert.equal(totals(view)['Chief/compaction'], 1100)
assert.equal(totals(view)['Chief/chat'], 160 + 220 + 10)
assert.deepEqual(second.warnings, [])
})
test('a cached projection watermark beyond the log is not used as a page cursor', async (t) => {
const { call, ctx, stop, warnings } = start(undefined, {
listedExtra: [{ sessionId: 's-x', projections: { kind: 'cached', asOfSeq: 202, values: {} } }],
})
t.after(stop)
const page = ctx.sessionController.page
ctx.sessionController.page = async (request) => {
if (request.address.sessionId === 's-x' && request.throughSeq > 1) throw new Error(`session page through seq ${request.throughSeq} is past cursor 1`)
return page(request)
}
const view = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(view)['other/chat'], 44)
assert.deepEqual(warnings, [])
})
test('a cached projection watermark behind the log does not omit later usage', async (t) => {
const { call, stop, warnings } = start(undefined, {
listedExtra: [{ sessionId: 's-x', projections: { kind: 'cached', asOfSeq: 0, values: {} } }],
})
t.after(stop)
const view = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(view)['other/chat'], 44)
assert.deepEqual(warnings, [])
})
test('a sequenced projection watermark cuts the page without another projection read', async (t) => {
const { call, ctx, stop } = start(undefined, {
listedExtra: [{ sessionId: 's-x', projections: { kind: 'sequenced', asOfSeq: 1, values: {} } }],
})
t.after(stop)
const projections = ctx.sessionController.projections
ctx.sessionController.projections = async (request) => {
assert.notEqual(request.sessionId, 's-x')
return projections(request)
}
const view = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(view)['other/chat'], 44)
})
test('a projection hint without a sequence space is not used as a page cursor', async (t) => {
const { call, stop } = start(undefined, {
listedExtra: [{ sessionId: 's-x', projections: { asOfSeq: 0, values: {} } }],
})
t.after(stop)
const view = (await call('usage', { timeZone: 'Asia/Shanghai' })).value
assert.equal(totals(view)['other/chat'], 44)
})