Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Set aside what a memory review cannot keep, and move on

Fei241145917e057be7
diff --git a/ds-bot/src/host/memory.js b/ds-bot/src/host/memory.js
index ea30cd13..4a4bd58a 100644
--- a/ds-bot/src/host/memory.js
+++ b/ds-bot/src/host/memory.js
@@ -71,8 +71,10 @@ const REVIEW_PASSES_MAX = 4
const NO_MATCH = 'memory/no-match'
const GONE = 'memory/bot-gone'
const DENIED = 'memory/denied'
+// A change a file has no room for.
+const FULL = 'memory/full'
// The codes the memory review routes on. A failure that carries one keeps it.
-const ROUTED = new Set([NO_MATCH, GONE, DENIED])
+const ROUTED = new Set([NO_MATCH, GONE, DENIED, FULL])
// A write the tools refused, and a write that broke. The code says which, so
// nothing has to read the message to know what happened.
const REFUSED = 'memory/refused'
@@ -279,8 +281,8 @@ export function reviewEntries(home, team, budget = REVIEW_MEMORY_MAX, about = ''
}
// The memory review's answer: {"changes": […]}, perhaps in a code fence, where
-// every change is well formed. Anything else is no answer, so the review keeps its
-// place and reads the turn again.
+// every change is well formed. Anything else is no answer: the review says so and
+// moves on — the same lines are not read again.
export function parseReview(text) {
const start = text.indexOf('{')
const end = text.lastIndexOf('}')
@@ -737,9 +739,9 @@ export function install(rt) {
const limit = limitOf(file)
// A file already past its size (edited by hand) may still shrink.
if (next.length <= limit || next.length <= file.content.length) return
- throw new Error(file.topic === undefined
+ throw Object.assign(new Error(file.topic === undefined
? `${capital(label)}'s MEMORY.md would have ${next.length} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.`
- : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`)
+ : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`), { code: FULL })
}
const capital = text => text[0].toUpperCase() + text.slice(1)
const sizeNote = (file, size) => `${file.file}: ${size} of ${limitOf(file)} characters`
@@ -1548,21 +1550,34 @@ export function install(rt) {
const label = labelOf(scope, keeper.id)
const text = cleanEntry(change?.text)
const meta = { source, ...(scope.kind === 'team' ? { by: keeper.name } : {}), added: today() }
+ // A change memory cannot keep is set aside and said, so what was not saved
+ // shows in the diagnostics; the rest of the answer is kept all the same —
+ // the same lines read again would meet the same limit. A secret's text is
+ // not repeated into the log.
+ const badTopic = op === 'add' && change?.topic !== undefined && String(change.topic).trim() !== '' && topic === undefined
+ const problem = op === 'add' || op === 'edit' ? entryProblem(text) : undefined
+ const leftOut =
+ op !== 'add' && target === undefined ? `it names entry ${change?.entry}, which is not listed` :
+ scope.kind === 'team' && !team ? 'it is for the team memory, which the review may not write' :
+ badTopic ? `it names the topic "${clip(String(change.topic), 60)}", which cannot be a topic name` :
+ problem !== undefined ? (secretEntry(text) ? 'the entry looks like a key, token, password, or ID number, which memory never keeps' : `the entry "${clip(text, 160)}": ${problem}`) :
+ undefined
+ if (leftOut !== undefined) {
+ rt.warn('ds-bot: memory review of %s left out a change: %s', name, leftOut)
+ continue
+ }
try {
- if (op !== 'add' && target === undefined) continue
- if (scope.kind === 'team' && !team) continue
- if ((op === 'add' || op === 'edit') && entryProblem(text) !== undefined) continue
if (op === 'add') done.push(await addEntry({ scope, topic, text, meta, actor, label }))
else if (op === 'edit') done.push(await replaceEntry({ scope, topic: target.file.topic, match: target.text, text, meta, actor, label }))
else if (op === 'remove') done.push(await removeEntry({ scope, topic: target.file.topic, match: target.text, actor, label }))
} catch (failure) {
if (failure?.code === GONE) return 0
rt.warn('ds-bot: memory review of %s left out a change: %s', name, failure?.message ?? failure)
- if (failure?.code !== NO_MATCH && failure?.code !== DENIED) failed ??= failure
+ if (!ROUTED.has(failure?.code)) failed ??= failure
}
}
- // A full file or a failed write is no reason to pass the lines: the next
- // review reads them again, and its request shows how full MEMORY.md is.
+ // A write that broke keeps the review's place: it may work when the lines are
+ // next read. What memory could not keep was set aside above, said already.
if (failed !== undefined) throw failed
return done.filter(result => result.changed).length
}
@@ -1653,30 +1668,26 @@ export function install(rt) {
const earlier = batch.earlier.map(line => reviewLine(line.who, line.text, keeper))
const language = writtenIn(read.filter(line => line.who === 'user').map(line => line.text.replace(/^User: /, '')), languageNow())
const reply = await botShot(keeper, reviewSystem({ teamWrites: teamWrites(), group }), reviewRequest({ who, tag: tagged, numbered, left, sizes, where: batch.where, earlier, fresh: fresh.texts, more, language }), tag)
- const changes = parseReview(reply)
- if (changes === undefined) throw new Error(`the model's answer is not a list of changes: ${clip(reply.replace(/\s+/g, ' '), 120)}`)
- const unknown = changes.find(change => change.op !== 'add' && numbered[Number(change.entry) - 1] === undefined)
- if (unknown !== undefined) throw new Error(`the model's answer names entry ${unknown.entry}, which is not listed`)
- // A secret is left out for good; any other entry memory cannot keep (one too
- // long, say) fails the answer, so the next review reads the lines again.
- const applied = changes.slice(0, REVIEW_CHANGES_MAX)
- const unfit = applied.filter(change => change.op !== 'remove').map(change => cleanEntry(change.text)).find(text => !secretEntry(text) && entryProblem(text) !== undefined)
- if (unfit !== undefined) throw new Error(`the model's answer holds an entry memory cannot keep: ${entryProblem(unfit)}`)
- const badTopic = applied.find(change => change.op === 'add' && typeof change.topic === 'string' && change.topic.trim() !== '' && cleanTopic(change.topic) === undefined)
- if (badTopic !== undefined) throw new Error(`the model's answer names the topic "${clip(badTopic.topic, 60)}", which cannot be a topic name`)
- if (!alive()) return
- const changed = await applyReview({ keeper, name, home, numbered, changes: applied, source: batch.source(read), teamWrites })
if (!alive()) return
- // An answer at the limit may have left some out, so the same lines are read
- // again, a few times at most, while each pass still changes something.
- const prior = overflowed.get(key)
- const passes = prior !== undefined && prior.since === since ? prior.passes : 0
- if (changes.length >= REVIEW_CHANGES_MAX && changed > 0 && passes < REVIEW_PASSES_MAX) {
- overflowed.set(key, { since, passes: passes + 1 })
- through = since
- again = true
+ const changes = parseReview(reply)
+ // An answer that is no list of changes keeps nothing of the batch; the miss
+ // is said, and the place moves on — the same lines are not read again.
+ if (changes === undefined) {
+ rt.warn('ds-bot: memory review of %s got no list of changes from the model: %s', name, clip(reply.replace(/\s+/g, ' '), 120))
} else {
- overflowed.delete(key)
+ const changed = await applyReview({ keeper, name, home, numbered, changes: changes.slice(0, REVIEW_CHANGES_MAX), source: batch.source(read), teamWrites })
+ if (!alive()) return
+ // An answer at the limit may have left some out, so the same lines are read
+ // again, a few times at most, while each pass still changes something.
+ const prior = overflowed.get(key)
+ const passes = prior !== undefined && prior.since === since ? prior.passes : 0
+ if (changes.length >= REVIEW_CHANGES_MAX && changed > 0 && passes < REVIEW_PASSES_MAX) {
+ overflowed.set(key, { since, passes: passes + 1 })
+ through = since
+ again = true
+ } else {
+ overflowed.delete(key)
+ }
}
}
const now = cursorsNow()[key]
diff --git a/ds-bot/test/memory.test.js b/ds-bot/test/memory.test.js
index ae8a798a..16fc93da 100644
--- a/ds-bot/test/memory.test.js
+++ b/ds-bot/test/memory.test.js
@@ -3,6 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync,
import { hostname, tmpdir } from 'node:os'
import { join } from 'node:path'
import { test } from 'node:test'
+import { format } from 'node:util'
import { apply } from '../index.js'
import {
MEMORY_LIMITS, addLink, cleanEntry, cleanTopic, formatEntry, insertEntry, looksLikeKey,
@@ -125,6 +126,8 @@ function start({ home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-')), replies =
if (!existsSync(join(home, 'state.json'))) writeFileSync(join(home, 'state.json'), JSON.stringify(TEAM))
const services = defaultModel === undefined ? {} : { agentDefaultModel: { currentSelection: () => ({ provider: 'deepseek', model: defaultModel }) } }
const ctx = fakeContext({ services })
+ const warnings = []
+ ctx.logger.warn = (...args) => warnings.push(format(...args))
const requests = []
Object.assign(ctx, {
llm: {
@@ -172,7 +175,7 @@ function start({ home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-')), replies =
await ctx.dispose()
if (!keep) rmSync(home, { recursive: true, force: true, maxRetries: 5 })
}
- return { ctx, home, call, tool, section, read, journal, requests, stop }
+ return { ctx, home, call, tool, section, read, journal, requests, warnings, stop }
}
const settle = () => new Promise(resolve => setTimeout(resolve, 20))
@@ -682,6 +685,25 @@ test('a group\'s review keeps what is said there in the group memory, read once
assert.equal((await call('state', {})).value.memoryNews.r1.op, 'add')
})
+test('a group review that gets no list of changes is not re-armed by every member\'s idle', async (t) => {
+ const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))
+ twoMembers(home)
+ const { ctx, call, read, requests, warnings, stop } = start({ home, replies: ['Let me think about that out loud.'], config: { memoryReviewDelayMs: 0 } })
+ t.after(() => stop())
+ await call('state', {})
+ ctx.emit('agent/status', { agent: { id: 'g3' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
+ // The batch is marked read even though nothing could be saved: the next member
+ // to go idle finds nothing new and starts no model call.
+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'group:r1': 3 })
+ assert.ok(warnings.some(line => line.includes('no list of changes')), warnings.join('\n'))
+ ctx.emit('agent/status', { agent: { id: 'g2' }, status: 'idle' })
+ ctx.emit('agent/status', { agent: { id: 'g3' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
+})
+
test('a first group review reads back to the user\'s line past a round of replies', async (t) => {
const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))
const replies = Array.from({ length: 6 }, (_, at) => ({ who: at % 2 ? 'Coder' : 'Writer', botId: at % 2 ? 'b3' : 'b2', text: `Reply ${at + 1}`, seq: at + 2 }))
@@ -697,47 +719,54 @@ test('a first group review reads back to the user\'s line past a round of replie
assert.deepEqual(JSON.parse(read('reviewed.json')), { 'group:r1': 7 })
})
-test('an answer that is no list of changes keeps the review\'s place, and the next one reads the turn again', async (t) => {
+test('an answer that is no list of changes is said and passed over, never read again', async (t) => {
const replies = ['Nothing worth keeping here.', JSON.stringify({ changes: [{ op: 'add', text: 'The user wants this remembered' }] })]
- const { ctx, call, read, requests, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })
+ const { ctx, call, read, requests, warnings, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })
t.after(() => stop())
await call('state', {})
ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
await reviewed(call)
assert.equal(requests.length, 1)
- // The first review fixed where it starts, just before the user's line.
- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 9 })
+ assert.throws(() => read('bots', 'b2', 'MEMORY.md'), { code: 'ENOENT' })
+ // The miss is said, and the batch is marked read: a later turn does not read it again.
+ assert.ok(warnings.some(line => line.includes('no list of changes')), warnings.join('\n'))
+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
await reviewed(call)
- assert.equal(requests.length, 2)
- assert.equal(requests[1].messages[0].content[0].text, requests[0].messages[0].content[0].text)
- assert.match(read('bots', 'b2', 'MEMORY.md'), /- The user wants this remembered \[source: part 1 #10;/)
- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
+ assert.equal(requests.length, 1)
})
-test('an answer with a malformed change, an unlisted entry, an entry too long, or a bad topic changes nothing and keeps the review\'s place', async (t) => {
- const replies = [
- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add' }] }),
- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'remove', entry: 7 }] }),
- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add', text: `The user tells ${'a long story '.repeat(30)}` }] }),
- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add', text: 'The user writes C++', topic: 'C++' }] }),
- JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add', text: 'I use hunter2 as my password' }] }),
- ]
- const { ctx, call, read, requests, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })
+test('a review keeps what it can, sets aside what memory cannot keep, and says each', async (t) => {
+ const replies = [JSON.stringify({ changes: [
+ { op: 'add', text: 'The user likes tea' },
+ { op: 'remove', entry: 7 },
+ { op: 'edit', entry: 9, text: 'The user likes black tea' },
+ { op: 'add', text: `The user tells ${'a long story '.repeat(30)}` },
+ { op: 'add', text: 'The user writes C++', topic: 'C++' },
+ { op: 'add', text: 'I use hunter2 as my password' },
+ ] })]
+ const { ctx, call, read, requests, warnings, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })
t.after(() => stop())
await call('state', {})
- for (let tries = 0; tries < 4; tries += 1) {
- ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
- await reviewed(call)
- assert.throws(() => read('bots', 'b2', 'MEMORY.md'), { code: 'ENOENT' })
- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 9 })
- }
ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
await reviewed(call)
- // A password is only left out: the rest is written and the place moves on.
- assert.equal(requests.length, 5)
+ assert.equal(requests.length, 1)
+ // What memory could keep was saved; the rest was set aside and said — except a
+ // secret, whose text stays out of the log.
assert.deepEqual(parseMemory(read('bots', 'b2', 'MEMORY.md')).entries.map(entry => entry.text), ['The user likes tea'])
assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
+ const dropped = warnings.filter(line => line.includes('left out a change'))
+ assert.equal(dropped.length, 5, warnings.join('\n'))
+ assert.ok(dropped.some(line => line.includes('entry 7, which is not listed')))
+ assert.ok(dropped.some(line => line.includes('entry 9, which is not listed')))
+ assert.ok(dropped.some(line => line.includes('at most 300 characters') && line.includes('a long story')))
+ assert.ok(dropped.some(line => line.includes('the topic "C++", which cannot be a topic name')))
+ assert.ok(dropped.some(line => line.includes('looks like a key, token, password, or ID number')))
+ assert.ok(!dropped.some(line => line.includes('hunter2')), warnings.join('\n'))
+ // The same lines are not read again: nothing was retried.
+ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
})
test('a backlog past the text limit is reviewed in passes, oldest first', async (t) => {
@@ -788,25 +817,35 @@ test('an answer that reaches the change limit exactly is followed by another pas
assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
})
-test('a full MEMORY.md keeps the review\'s place, and an add with a topic lands there', async (t) => {
+test('a full MEMORY.md sets an add aside, says so, and an add with a topic still lands', async (t) => {
const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))
mkdirSync(join(home, 'memory', 'bots', 'b2'), { recursive: true })
const filler = Array.from({ length: 13 }, (_, at) => `- Filler fact number ${at + 1} ${'y'.repeat(280)}`)
writeFileSync(join(home, 'memory', 'bots', 'b2', 'MEMORY.md'), `# Memory\n\n${filler.join('\n')}\n\n## Index\n`)
- const replies = [JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China' }] }), JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China', topic: '出行' }] })]
- const { ctx, call, read, requests, stop } = start({ home, replies, config: { memoryReviewDelayMs: 0 } })
+ const history = { b2: [...ownChat] }
+ const replies = [
+ JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China' }] }),
+ JSON.stringify({ changes: [{ op: 'add', text: 'The user flies with Air China', topic: '出行' }] }),
+ ]
+ const { ctx, call, read, requests, warnings, stop } = start({ home, replies, history, config: { memoryReviewDelayMs: 0 } })
t.after(() => stop())
await call('state', {})
ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
await reviewed(call)
assert.match(requests[0].messages[0].content[0].text, /MEMORY\.md has 3\d{3} of 4000 characters \(own\)/)
assert.match(requests[0].system, /"topic": "travel"/)
- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 9 })
+ // The add had no room: it was set aside and said, and the batch is marked read.
+ assert.equal(parseMemory(read('bots', 'b2', 'MEMORY.md')).entries.length, 13)
+ assert.ok(warnings.some(line => line.includes('left out a change') && line.includes('would have') && line.includes('of its 4000 characters')), warnings.join('\n'))
+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
+ // The review request goes on telling how full MEMORY.md is, so the next one
+ // can put the detail in a topic instead.
+ history.b2.push({ event: { seq: 12, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'I fly again next week' }] } } })
ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
await reviewed(call)
assert.match(read('bots', 'b2', '出行.md'), /- The user flies with Air China \[/)
assert.match(read('bots', 'b2', 'MEMORY.md'), /\[\[出行\]\]/)
- assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 12 })
})
test('topic entries that share words with the user\'s new lines come first', () => {