A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Re-read a sealed cold Session once its checkpoint moves
Fei24117d6593b3ebb6
diff --git a/ds-bot/src/host/usage.js b/ds-bot/src/host/usage.jsindex fd07b9f1..ad6e4000 100644--- a/ds-bot/src/host/usage.js+++ b/ds-bot/src/host/usage.js@@ -14,7 +14,7 @@ import { randomUUID } from 'node:crypto'import { mkdir, unlink } from 'node:fs/promises'import { join } from 'node:path'-const LEDGER_VERSION = 2+const LEDGER_VERSION = 3const PAGE_MESSAGES = 400const PAGE_TIMEOUT_MS = 20_000const SCAN_TIMEOUT_MS = 180_000@@ -105,6 +105,11 @@ function addTo(groups, row) {groups.set(id, group)}+// A Session's seal, as the marks keep it. `seal` is undefined while the Session can+// still grow; `{listed: null}` for an ended part; `{listed}` for a cold Session, at+// the checkpoint watermark its listing showed.+const sealFields = seal => ({ sealed: seal !== undefined, listed: seal?.listed ?? null })+function memoryLedger() {const calls = new Map()const marks = new Map()@@ -138,7 +143,22 @@ function sqliteLedger(DatabaseSync, path) {// briefly, and the timeout must be armed before the WAL pragma can block.db.exec('PRAGMA busy_timeout = 5000')db.exec('PRAGMA journal_mode = WAL')- if (db.prepare('PRAGMA user_version').get().user_version !== LEDGER_VERSION) {+ const version = db.prepare('PRAGMA user_version').get().user_version+ // Version 2 lacks only the seal's watermark, so its rows are kept and gain the column.+ // Desktop and Web may open it at once: the version is read again under the write lock.+ if (version === 2) {+ db.exec('BEGIN IMMEDIATE')+ try {+ if (db.prepare('PRAGMA user_version').get().user_version === 2) {+ db.exec(`ALTER TABLE marks ADD COLUMN listed INTEGER;+ PRAGMA user_version = ${LEDGER_VERSION};`)+ }+ db.exec('COMMIT')+ } catch (error) {+ db.exec('ROLLBACK')+ throw error+ }+ } else if (version !== LEDGER_VERSION) {db.exec(`DROP TABLE IF EXISTS calls;DROP TABLE IF EXISTS marks;@@ -146,15 +166,15 @@ function sqliteLedger(DatabaseSync, path) {DROP TABLE IF EXISTS kinds;CREATE TABLE calls (key TEXT PRIMARY KEY, session TEXT NOT NULL, time INTEGER NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, kind TEXT NOT NULL, live INTEGER NOT NULL, input INTEGER NOT NULL, cache_read INTEGER NOT NULL, cache_write INTEGER NOT NULL, output INTEGER NOT NULL);CREATE INDEX calls_session ON calls (session);- CREATE TABLE marks (session TEXT PRIMARY KEY, through INTEGER NOT NULL, fold TEXT NOT NULL, sealed INTEGER NOT NULL);+ CREATE TABLE marks (session TEXT PRIMARY KEY, through INTEGER NOT NULL, fold TEXT NOT NULL, sealed INTEGER NOT NULL, listed INTEGER);CREATE TABLE owners (session TEXT PRIMARY KEY, owner TEXT NOT NULL, name TEXT NOT NULL);CREATE TABLE kinds (session TEXT PRIMARY KEY, kind TEXT NOT NULL);PRAGMA user_version = ${LEDGER_VERSION};`)}const put = db.prepare('INSERT OR REPLACE INTO calls (key, session, time, provider, model, kind, live, input, cache_read, cache_write, output) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')const dropLogged = db.prepare('DELETE FROM calls WHERE session = ? AND live = 0')- const getMark = db.prepare('SELECT through, fold, sealed FROM marks WHERE session = ?')- const setMark = db.prepare('INSERT OR REPLACE INTO marks (session, through, fold, sealed) VALUES (?, ?, ?, ?)')+ const getMark = db.prepare('SELECT through, fold, sealed, listed FROM marks WHERE session = ?')+ const setMark = db.prepare('INSERT OR REPLACE INTO marks (session, through, fold, sealed, listed) VALUES (?, ?, ?, ?, ?)')const allOwners = db.prepare('SELECT session, owner, name FROM owners')const putOwner = db.prepare('INSERT OR REPLACE INTO owners (session, owner, name) VALUES (?, ?, ?)')const allKinds = db.prepare('SELECT session, kind FROM kinds')@@ -164,14 +184,14 @@ function sqliteLedger(DatabaseSync, path) {return {mark(session) {const row = getMark.get(session)- return row === undefined ? undefined : { through: row.through, fold: JSON.parse(row.fold), sealed: row.sealed === 1 }+ return row === undefined ? undefined : { through: row.through, fold: JSON.parse(row.fold), sealed: row.sealed === 1, listed: row.listed ?? null }},commit(session, { rows, reset, mark }) {db.exec('BEGIN')try {if (reset) dropLogged.run(session)for (const row of rows) insert(row, false)- setMark.run(session, mark.through, JSON.stringify(mark.fold), mark.sealed ? 1 : 0)+ setMark.run(session, mark.through, JSON.stringify(mark.fold), mark.sealed ? 1 : 0, mark.listed ?? null)db.exec('COMMIT')} catch (error) {db.exec('ROLLBACK')@@ -330,12 +350,12 @@ export function install(rt) {// Sessions that already failed to read, so each one warns once, not every scan.const unreadable = new Set()- async function scanSession(ledger, address, asOfSeq, sealable, signal) {+ async function scanSession(ledger, address, asOfSeq, seal, signal) {const sessionId = address.kind === 'session' ? address.sessionId : address.childSessionIdconst mark = ledger.mark(sessionId)- // A sealed Session is skipped only while it still cannot grow: one sealed while- // cold can resume, so a warm listing clears the seal by reading it again.- if (mark?.sealed && sealable) return+ // A seal stands only while the listing shows the watermark it was sealed at, so a+ // cold Session whose log grew since is read again.+ if (mark?.sealed && seal !== undefined && mark.listed === seal.listed) returnlet through = asOfSeqif (through === undefined) {const projections = await ctx.sessionController.projections({ sessionId }, signal)@@ -344,7 +364,8 @@ export function install(rt) {}const from = mark?.through ?? -1if (!Number.isSafeInteger(through) || through <= from) {- if (mark !== undefined && sealable) ledger.commit(sessionId, { rows: [], reset: false, mark: { ...mark, sealed: true } })+ // Nothing new to read: a warm Session's seal is cleared, and a cold one is sealed again at its watermark.+ if (mark !== undefined && (seal !== undefined || mark.sealed)) ledger.commit(sessionId, { rows: [], reset: false, mark: { ...mark, ...sealFields(seal) } })return}const events = []@@ -373,7 +394,7 @@ export function install(rt) {ledger.commit(sessionId, {rows: fold.rows.map(row => ({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })),reset: fold.reset,- mark: { through, fold: fold.state, sealed: sealable },+ mark: { through, fold: fold.state, ...sealFields(seal) },})}@@ -412,40 +433,49 @@ export function install(rt) {// the persisted cache's own watermark, so it must not cut a page. The// Session's exact cursor is read below when no hint is usable.const listedCursor = item => (item.projections?.kind === 'sequenced' ? item.projections.asOfSeq : undefined)- // A Session with no Agent attached is cold: its log cannot grow right now, so- // once it is read to its tip it is as finished as an ended part and seals.- const cold = new Set(listed.filter(item => item.agentAvailable !== true && item.running !== true).map(item => item.sessionId))+ // A Session with no Agent attached is cold. Its listing carries the watermark of its+ // checkpoint, which DSH rewrites at each turn end and when the Session is detached, so+ // the watermark moves when the log does. A cold Session seals at that watermark only:+ // the watermark is compared with an earlier listing, never with a log sequence.+ const coldWatermarks = new Map(listed+ .filter(item => item.agentAvailable !== true && item.running !== true && item.projections?.kind === 'cached' && Number.isSafeInteger(item.projections.asOfSeq))+ .map(item => [item.sessionId, item.projections.asOfSeq]))+ // An ended part is complete, so it seals for good.+ const sealOf = sessionId => (ended.has(sessionId) ? { listed: null } : coldWatermarks.has(sessionId) ? { listed: coldWatermarks.get(sessionId) } : undefined)const targets = new Map()for (const item of listed) if (item.blank !== true) targets.set(item.sessionId, listedCursor(item))for (const sessionId of owners.keys()) if (!targets.has(sessionId)) targets.set(sessionId, undefined)const addresses = new Map(listed.map(item => [item.sessionId, addressOf(item)]))- let reached = 0+ let cut = 0const failed = []await eachLimited([...targets], SCAN_PARALLEL, async ([sessionId, asOfSeq]) => {- if (signal.aborted) return- reached += 1+ if (signal.aborted) {+ cut += 1+ return+ }const address = addresses.has(sessionId) ? addresses.get(sessionId) : { kind: 'session', sessionId }if (address === undefined) returntry {- await scanSession(ledger, address, asOfSeq, ended.has(sessionId) || cold.has(sessionId), signal)+ await scanSession(ledger, address, asOfSeq, sealOf(sessionId), signal)unreadable.delete(sessionId)} catch (error) {- // The scan timeout cancels the reads still open; those Sessions are not- // broken, so they are counted below but not remembered as unreadable.- if (!signal.aborted) {- if (unreadable.has(sessionId)) return- unreadable.add(sessionId)+ // The scan timeout cancels the reads still open. Those Sessions are not broken,+ // so they are counted with the ones it never reached, not remembered as unreadable.+ if (signal.aborted) {+ cut += 1+ return}+ if (unreadable.has(sessionId)) return+ unreadable.add(sessionId)failed.push([sessionId, error])}})// One line for the whole scan, not one per Session it could not read.- const missed = failed.length + (signal.aborted ? targets.size - reached : 0)+ const missed = failed.length + cutif (missed > 0) {- const first = failed[0]rt.warn('ds-bot: usage could not read %d Session%s%s%s', missed, missed === 1 ? '' : 's',- signal.aborted ? ' before the scan ran out of time' : '',- first !== undefined ? ` (first: ${first[0]}: ${first[1]})` : '')+ cut > 0 ? ' before the scan ran out of time' : '',+ failed.length > 0 ? `: ${failed.map(([sessionId]) => sessionId).join(', ')} (first error: ${failed[0][1]})` : '')}})().finally(() => {scannedAt = Date.now()diff --git a/ds-bot/test/usage.test.js b/ds-bot/test/usage.test.jsindex d811fd97..5d87e5e8 100644--- a/ds-bot/test/usage.test.js+++ b/ds-bot/test/usage.test.js@@ -226,7 +226,7 @@ test('a subagent Session is read through its parent, even a parent that is gone;.map(([template, ...args]) => template === '%s' ? args[0] : template).filter(line => line.includes('usage could not read'))assert.equal(unread().length, 1)- assert.match(unread()[0], /usage could not read 2 Sessions \(first: s-broken/)+ assert.match(unread()[0], /usage could not read 2 Sessions: s-broken, s-broken2 \(first error: /)t.mock.timers.tick(60_000)await call('usage', { timeZone: 'Asia/Shanghai' })@@ -234,13 +234,13 @@ test('a subagent Session is read through its parent, even a parent that is gone;assert.equal(unread().length, 1)})-// #654: a Session with no Agent attached cannot grow, so once read to its tip it-// seals and later scans skip it without reading it again — until it comes back.-test('a cold Session seals once read and is read again only once it warms', async (t) => {+// #654: a Session with no Agent attached seals once read to its tip, at the checkpoint+// watermark its listing shows. Later scans skip it without a read while that watermark+// stands; a warm scan clears the seal, and a moved watermark means the log grew.+test('a cold Session seals at its checkpoint watermark and is read again once it warms or the watermark moves', async (t) => {t.mock.timers.enable({ apis: ['Date'], now: 10 * HOUR })- const { call, ctx, stop } = start(undefined, {- listedExtra: [{ sessionId: 's-x', projections: { kind: 'cached', asOfSeq: 1, values: {} } }],- })+ const logX = [header('kimi', 'k3'), message(1, 0, usage(40, 4), 2 * HOUR)]+ const { call, ctx, stop } = start(undefined, { logsExtra: { 's-x': logX } })t.after(stop)let reads = 0const projections = ctx.sessionController.projections@@ -248,28 +248,61 @@ test('a cold Session seals once read and is read again only once it warms', asynif (request.sessionId === 's-x') reads += 1return projections(request)}- const relist = agentAvailable => {- ctx.sessionController.list = async () => ({- items: [{ sessionId: 's-x', agentAvailable, running: false, projections: { kind: 'cached', asOfSeq: 1, values: {} } }],- })+ let agentAvailable = false+ let watermark = 1+ ctx.sessionController.list = async () => ({+ items: [{ sessionId: 's-x', agentAvailable, running: false, projections: { kind: 'cached', asOfSeq: watermark, values: {} } }],+ })+ const scan = async () => {+ t.mock.timers.tick(60_000)+ return (await call('usage', { timeZone: 'Asia/Shanghai' })).value}// First scan: cold but never read, so read once and sealed.await call('usage', { timeZone: 'Asia/Shanghai' })assert.equal(reads, 1)- // Still cold: the sealed mark skips it without any read.- t.mock.timers.tick(60_000)- await call('usage', { timeZone: 'Asia/Shanghai' })+ // Still cold at the same watermark: the seal skips it without any read.+ await scan()assert.equal(reads, 1)- // Listed warm again, the seal no longer skips it.- relist(true)- t.mock.timers.tick(60_000)- await call('usage', { timeZone: 'Asia/Shanghai' })- assert.equal(reads, 2)- // Cold once more, it is sealed at the same tip and skipped again.- relist(false)- t.mock.timers.tick(60_000)- await call('usage', { timeZone: 'Asia/Shanghai' })+ // Listed warm, a scan reads it and clears the seal, since it may run.+ agentAvailable = true+ await scan()assert.equal(reads, 2)+ // Cold again: read once more, then sealed at the watermark and skipped.+ agentAvailable = false+ await scan()+ assert.equal(reads, 3)+ await scan()+ assert.equal(reads, 3)+ // The log grew and its checkpoint moved on: read again, and the new calls count.+ logX.push({ ...message(2, 0, usage(7, 7)), seq: 2 })+ watermark = 2+ const view = await scan()+ assert.equal(reads, 4)+ assert.equal(totals(view)['other/chat'], 44 + 14)+ await scan()+ assert.equal(reads, 4)+})++// The ledger of the previous version has no seal watermark; it is kept, not rebuilt.+test('a ledger from the previous version keeps its metered calls and gains the seal watermark', async (t) => {+ const first = start()+ const { home } = first+ t.after(() => rmSync(home, { recursive: true, force: true, maxRetries: 5 }))+ const chunk = { type: 'usage', usage: usage(1000, 100) }+ await drain(first.meter({ sessionId: 's-b1', purpose: 'compaction', provider: 'deepseek', model: 'v4-flash', messages: [] }, async function* () { yield chunk }))+ await first.call('usage', { timeZone: 'Asia/Shanghai' })+ await first.ctx.dispose()+ const { DatabaseSync } = await import('node:sqlite')+ const db = new DatabaseSync(join(home, 'usage.sqlite'))+ db.exec('ALTER TABLE marks DROP COLUMN listed; PRAGMA user_version = 2')+ db.close()++ const second = start(home)+ t.after(() => second.ctx.dispose())+ const view = (await second.call('usage', { timeZone: 'Asia/Shanghai' })).value+ assert.equal(totals(view)['Chief/compaction'], 1100)+ assert.equal(totals(view)['Chief/chat'], 160 + 220 + 10)+ assert.deepEqual(second.warnings, [])})test('a cached projection watermark beyond the log is not used as a page cursor', async (t) => {