A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Set aside what a memory review cannot keep, and move on
Fei241145917e057be7
diff --git a/ds-bot/src/host/memory.js b/ds-bot/src/host/memory.jsindex ea30cd13..4a4bd58a 100644--- a/ds-bot/src/host/memory.js+++ b/ds-bot/src/host/memory.js@@ -71,8 +71,10 @@ const REVIEW_PASSES_MAX = 4const NO_MATCH = 'memory/no-match'const GONE = 'memory/bot-gone'const DENIED = 'memory/denied'+// A change a file has no room for.+const FULL = 'memory/full'// The codes the memory review routes on. A failure that carries one keeps it.-const ROUTED = new Set([NO_MATCH, GONE, DENIED])+const ROUTED = new Set([NO_MATCH, GONE, DENIED, FULL])// A write the tools refused, and a write that broke. The code says which, so// nothing has to read the message to know what happened.const REFUSED = 'memory/refused'@@ -279,8 +281,8 @@ export function reviewEntries(home, team, budget = REVIEW_MEMORY_MAX, about = ''}// The memory review's answer: {"changes": […]}, perhaps in a code fence, where-// every change is well formed. Anything else is no answer, so the review keeps its-// place and reads the turn again.+// every change is well formed. Anything else is no answer: the review says so and+// moves on — the same lines are not read again.export function parseReview(text) {const start = text.indexOf('{')const end = text.lastIndexOf('}')@@ -737,9 +739,9 @@ export function install(rt) {const limit = limitOf(file)// A file already past its size (edited by hand) may still shrink.if (next.length <= limit || next.length <= file.content.length) return- throw new Error(file.topic === undefined+ throw Object.assign(new Error(file.topic === undefined? `${capital(label)}'s MEMORY.md would have ${next.length} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.`- : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`)+ : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`), { code: FULL })}const capital = text => text[0].toUpperCase() + text.slice(1)const sizeNote = (file, size) => `${file.file}: ${size} of ${limitOf(file)} characters`@@ -1548,21 +1550,34 @@ export function install(rt) {const label = labelOf(scope, keeper.id)const text = cleanEntry(change?.text)const meta = { source, ...(scope.kind === 'team' ? { by: keeper.name } : {}), added: today() }+ // A change memory cannot keep is set aside and said, so what was not saved+ // shows in the diagnostics; the rest of the answer is kept all the same —+ // the same lines read again would meet the same limit. A secret's text is+ // not repeated into the log.+ const badTopic = op === 'add' && change?.topic !== undefined && String(change.topic).trim() !== '' && topic === undefined+ const problem = op === 'add' || op === 'edit' ? entryProblem(text) : undefined+ const leftOut =+ op !== 'add' && target === undefined ? `it names entry ${change?.entry}, which is not listed` :+ scope.kind === 'team' && !team ? 'it is for the team memory, which the review may not write' :+ badTopic ? `it names the topic "${clip(String(change.topic), 60)}", which cannot be a topic name` :+ problem !== undefined ? (secretEntry(text) ? 'the entry looks like a key, token, password, or ID number, which memory never keeps' : `the entry "${clip(text, 160)}": ${problem}`) :+ undefined+ if (leftOut !== undefined) {+ rt.warn('ds-bot: memory review of %s left out a change: %s', name, leftOut)+ continue+ }try {- if (op !== 'add' && target === undefined) continue- if (scope.kind === 'team' && !team) continue- if ((op === 'add' || op === 'edit') && entryProblem(text) !== undefined) continueif (op === 'add') done.push(await addEntry({ scope, topic, text, meta, actor, label }))else if (op === 'edit') done.push(await replaceEntry({ scope, topic: target.file.topic, match: target.text, text, meta, actor, label }))else if (op === 'remove') done.push(await removeEntry({ scope, topic: target.file.topic, match: target.text, actor, label }))} catch (failure) {if (failure?.code === GONE) return 0rt.warn('ds-bot: memory review of %s left out a change: %s', name, failure?.message ?? failure)- if (failure?.code !== NO_MATCH && failure?.code !== DENIED) failed ??= failure+ if (!ROUTED.has(failure?.code)) failed ??= failure}}- // A full file or a failed write is no reason to pass the lines: the next- // review reads them again, and its request shows how full MEMORY.md is.+ // A write that broke keeps the review's place: it may work when the lines are+ // next read. What memory could not keep was set aside above, said already.if (failed !== undefined) throw failedreturn done.filter(result => result.changed).length}@@ -1653,30 +1668,26 @@ export function install(rt) {const earlier = batch.earlier.map(line => reviewLine(line.who, line.text, keeper))const language = writtenIn(read.filter(line => line.who === 'user').map(line => line.text.replace(/^User: /, '')), languageNow())const reply = await botShot(keeper, reviewSystem({ teamWrites: teamWrites(), group }), reviewRequest({ who, tag: tagged, numbered, left, sizes, where: batch.where, earlier, fresh: fresh.texts, more, language }), tag)- const changes = parseReview(reply)- if (changes === undefined) throw new Error(`the model's answer is not a list of changes: ${clip(reply.replace(/\s+/g, ' '), 120)}`)- const unknown = changes.find(change => change.op !== 'add' && numbered[Number(change.entry) - 1] === undefined)- if (unknown !== undefined) throw new Error(`the model's answer names entry ${unknown.entry}, which is not listed`)- // A secret is left out for good; any other entry memory cannot keep (one too- // long, say) fails the answer, so the next review reads the lines again.- const applied = changes.slice(0, REVIEW_CHANGES_MAX)- const unfit = applied.filter(change => change.op !== 'remove').map(change => cleanEntry(change.text)).find(text => !secretEntry(text) && entryProblem(text) !== undefined)- if (unfit !== undefined) throw new Error(`the model's answer holds an entry memory cannot keep: ${entryProblem(unfit)}`)- const badTopic = applied.find(change => change.op === 'add' && typeof change.topic === 'string' && change.topic.trim() !== '' && cleanTopic(change.topic) === undefined)- if (badTopic !== undefined) throw new Error(`the model's answer names the topic "${clip(badTopic.topic, 60)}", which cannot be a topic name`)- if (!alive()) return- const changed = await applyReview({ keeper, name, home, numbered, changes: applied, source: batch.source(read), teamWrites })if (!alive()) return- // An answer at the limit may have left some out, so the same lines are read- // again, a few times at most, while each pass still changes something.- const prior = overflowed.get(key)- const passes = prior !== undefined && prior.since === since ? prior.passes : 0- if (changes.length >= REVIEW_CHANGES_MAX && changed > 0 && passes < REVIEW_PASSES_MAX) {- overflowed.set(key, { since, passes: passes + 1 })- through = since- again = true+ const changes = parseReview(reply)+ // An answer that is no list of changes keeps nothing of the batch; the miss+ // is said, and the place moves on — the same lines are not read again.+ if (changes === undefined) {+ rt.warn('ds-bot: memory review of %s got no list of changes from the model: %s', name, clip(reply.replace(/\s+/g, ' '), 120))} else {- overflowed.delete(key)+ const changed = await applyReview({ keeper, name, home, numbered, changes: changes.slice(0, REVIEW_CHANGES_MAX), source: batch.source(read), teamWrites })+ if (!alive()) return+ // An answer at the limit may have left some out, so the same lines are read+ // again, a few times at most, while each pass still changes something.+ const prior = overflowed.get(key)+ const passes = prior !== undefined && prior.since === since ? prior.passes : 0+ if (changes.length >= REVIEW_CHANGES_MAX && changed > 0 && passes < REVIEW_PASSES_MAX) {+ overflowed.set(key, { since, passes: passes + 1 })+ through = since+ again = true+ } else {+ overflowed.delete(key)+ }}}const now = cursorsNow()[key]diff --git a/ds-bot/test/memory.test.js b/ds-bot/test/memory.test.jsindex ae8a798a..16fc93da 100644--- a/ds-bot/test/memory.test.js+++ b/ds-bot/test/memory.test.js@@ -3,6 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync,import { hostname, tmpdir } from 'node:os'import { join } from 'node:path'import { test } from 'node:test'+import { format } from 'node:util'import { apply } from '../index.js'import {MEMORY_LIMITS, addLink, cleanEntry, cleanTopic, formatEntry, insertEntry, looksLikeKey,@@ -125,6 +126,8 @@ function start({ home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-')), replies =if (!existsSync(join(home, 'state.json'))) writeFileSync(join(home, 'state.json'), JSON.stringify(TEAM))const services = defaultModel === undefined ? {} : { agentDefaultModel: { currentSelection: () => ({ provider: 'deepseek', model: defaultModel }) } }const ctx = fakeContext({ services })+ const warnings = []+ ctx.logger.warn = (...args) => warnings.push(format(...args))const requests = []Object.assign(ctx, {llm: {@@ -172,7 +175,7 @@ function start({ home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-')), replies =await ctx.dispose()if (!keep) rmSync(home, { recursive: true, force: true, maxRetries: 5 })}- return { ctx, home, call, tool, section, read, journal, requests, stop }+ return { ctx, home, call, tool, section, read, journal, requests, warnings, stop }}const settle = () => new Promise(resolve => setTimeout(resolve, 20))@@ -682,6 +685,25 @@ test('a group\'s review keeps what is said there in the group memory, read onceassert.equal((await call('state', {})).value.memoryNews.r1.op, 'add')})+test('a group review that gets no list of changes is not re-armed by every member\'s idle', async (t) => {+ const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))+ twoMembers(home)+ const { ctx, call, read, requests, warnings, stop } = start({ home, replies: ['Let me think about that out loud.'], config: { memoryReviewDelayMs: 0 } })+ t.after(() => stop())+ await call('state', {})+ ctx.emit('agent/status', { agent: { id: 'g3' }, status: 'idle' })+ await reviewed(call)+ assert.equal(requests.length, 1)+ // The batch is marked read even though nothing could be saved: the next member+ // to go idle finds nothing new and starts no model call.+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'group:r1': 3 })+ assert.ok(warnings.some(line => line.includes('no list of changes')), warnings.join('\n'))+ ctx.emit('agent/status', { agent: { id: 'g2' }, status: 'idle' })+ ctx.emit('agent/status', { agent: { id: 'g3' }, status: 'idle' })+ await reviewed(call)+ assert.equal(requests.length, 1)+})+test('a first group review reads back to the user\'s line past a round of replies', async (t) => {const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))const replies = Array.from({ length: 6 }, (_, at) => ({ who: at % 2 ? 'Coder' : 'Writer', botId: at % 2 ? 'b3' : 'b2', text: `Reply ${at + 1}`, seq: at + 2 }))@@ -697,47 +719,54 @@ test('a first group review reads back to the user\'s line past a round of replieassert.deepEqual(JSON.parse(read('reviewed.json')), { 'group:r1': 7 })})-test('an answer that is no list of changes keeps the review\'s place, and the next one reads the turn again', async (t) => {+test('an answer that is no list of changes is said and passed over, never read again', async (t) => {const replies = ['Nothing worth keeping here.', JSON.stringify({ changes: [{ op: 'add', text: 'The user wants this remembered' }] })]- const { ctx, call, read, requests, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })+ const { ctx, call, read, requests, warnings, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })t.after(() => stop())await call('state', {})ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })await reviewed(call)assert.equal(requests.length, 1)- // The first review fixed where it starts, just before the user's line.- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 9 })+ assert.throws(() => read('bots', 'b2', 'MEMORY.md'), { code: 'ENOENT' })+ // The miss is said, and the batch is marked read: a later turn does not read it again.+ assert.ok(warnings.some(line => line.includes('no list of changes')), warnings.join('\n'))+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })await reviewed(call)- assert.equal(requests.length, 2)- assert.equal(requests[1].messages[0].content[0].text, requests[0].messages[0].content[0].text)- assert.match(read('bots', 'b2', 'MEMORY.md'), /- The user wants this remembered \[source: part 1 #10;/)- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })+ assert.equal(requests.length, 1)})-test('an answer with a malformed change, an unlisted entry, an entry too long, or a bad topic changes nothing and keeps the review\'s place', async (t) => {- const replies = [- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add' }] }),- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'remove', entry: 7 }] }),- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add', text: `The user tells ${'a long story '.repeat(30)}` }] }),- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add', text: 'The user writes C++', topic: 'C++' }] }),- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add', text: 'I use hunter2 as my password' }] }),- ]- const { ctx, call, read, requests, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })+test('a review keeps what it can, sets aside what memory cannot keep, and says each', async (t) => {+ const replies = [JSON.stringify({ changes: [+ { op: 'add', text: 'The user likes tea' },+ { op: 'remove', entry: 7 },+ { op: 'edit', entry: 9, text: 'The user likes black tea' },+ { op: 'add', text: `The user tells ${'a long story '.repeat(30)}` },+ { op: 'add', text: 'The user writes C++', topic: 'C++' },+ { op: 'add', text: 'I use hunter2 as my password' },+ ] })]+ const { ctx, call, read, requests, warnings, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })t.after(() => stop())await call('state', {})- for (let tries = 0; tries < 4; tries += 1) {- ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })- await reviewed(call)- assert.throws(() => read('bots', 'b2', 'MEMORY.md'), { code: 'ENOENT' })- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 9 })- }ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })await reviewed(call)- // A password is only left out: the rest is written and the place moves on.- assert.equal(requests.length, 5)+ assert.equal(requests.length, 1)+ // What memory could keep was saved; the rest was set aside and said — except a+ // secret, whose text stays out of the log.assert.deepEqual(parseMemory(read('bots', 'b2', 'MEMORY.md')).entries.map(entry => entry.text), ['The user likes tea'])assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })+ const dropped = warnings.filter(line => line.includes('left out a change'))+ assert.equal(dropped.length, 5, warnings.join('\n'))+ assert.ok(dropped.some(line => line.includes('entry 7, which is not listed')))+ assert.ok(dropped.some(line => line.includes('entry 9, which is not listed')))+ assert.ok(dropped.some(line => line.includes('at most 300 characters') && line.includes('a long story')))+ assert.ok(dropped.some(line => line.includes('the topic "C++", which cannot be a topic name')))+ assert.ok(dropped.some(line => line.includes('looks like a key, token, password, or ID number')))+ assert.ok(!dropped.some(line => line.includes('hunter2')), warnings.join('\n'))+ // The same lines are not read again: nothing was retried.+ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })+ await reviewed(call)+ assert.equal(requests.length, 1)})test('a backlog past the text limit is reviewed in passes, oldest first', async (t) => {@@ -788,25 +817,35 @@ test('an answer that reaches the change limit exactly is followed by another pasassert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })})-test('a full MEMORY.md keeps the review\'s place, and an add with a topic lands there', async (t) => {+test('a full MEMORY.md sets an add aside, says so, and an add with a topic still lands', async (t) => {const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))mkdirSync(join(home, 'memory', 'bots', 'b2'), { recursive: true })const filler = Array.from({ length: 13 }, (_, at) => `- Filler fact number ${at + 1} ${'y'.repeat(280)}`)writeFileSync(join(home, 'memory', 'bots', 'b2', 'MEMORY.md'), `# Memory\n\n${filler.join('\n')}\n\n## Index\n`)- const replies = [JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China' }] }), JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China', topic: '出行' }] })]- const { ctx, call, read, requests, stop } = start({ home, replies, config: { memoryReviewDelayMs: 0 } })+ const history = { b2: [...ownChat] }+ const replies = [+ JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China' }] }),+ JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China', topic: '出行' }] }),+ ]+ const { ctx, call, read, requests, warnings, stop } = start({ home, replies, history, config: { memoryReviewDelayMs: 0 } })t.after(() => stop())await call('state', {})ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })await reviewed(call)assert.match(requests[0].messages[0].content[0].text, /MEMORY\.md has 3\d{3} of 4000 characters \(own\)/)assert.match(requests[0].system, /"topic": "travel"/)- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 9 })+ // The add had no room: it was set aside and said, and the batch is marked read.+ assert.equal(parseMemory(read('bots', 'b2', 'MEMORY.md')).entries.length, 13)+ assert.ok(warnings.some(line => line.includes('left out a change') && line.includes('would have') && line.includes('of its 4000 characters')), warnings.join('\n'))+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })+ // The review request goes on telling how full MEMORY.md is, so the next one+ // can put the detail in a topic instead.+ history.b2.push({ event: { seq: 12, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'I fly again next week' }] } } })ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })await reviewed(call)assert.match(read('bots', 'b2', '出行.md'), /- The user flies with Air China \[/)assert.match(read('bots', 'b2', 'MEMORY.md'), /\[\[出行\]\]/)- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 12 })})test('topic entries that share words with the user\'s new lines come first', () => {