Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Name each change a memory review sets aside, and parse one at a time

Fei2411f88af8a60b83
diff --git a/ds-bot/src/host/memory.js b/ds-bot/src/host/memory.js
index 4a4bd58a..6e617440 100644
--- a/ds-bot/src/host/memory.js
+++ b/ds-bot/src/host/memory.js
@@ -71,7 +71,7 @@ const REVIEW_PASSES_MAX = 4
const NO_MATCH = 'memory/no-match'
const GONE = 'memory/bot-gone'
const DENIED = 'memory/denied'
-// A change a file has no room for.
+// A change the memory has no room for: a file at its size, or the last topic.
const FULL = 'memory/full'
// The codes the memory review routes on. A failure that carries one keeps it.
const ROUTED = new Set([NO_MATCH, GONE, DENIED, FULL])
@@ -280,9 +280,11 @@ export function reviewEntries(home, team, budget = REVIEW_MEMORY_MAX, about = ''
return { numbered: [...main, ...topics.filter(entry => shown.has(entry))], left: topics.length - shown.size }
}
-// The memory review's answer: {"changes": […]}, perhaps in a code fence, where
-// every change is well formed. Anything else is no answer: the review says so and
-// moves on — the same lines are not read again.
+// The memory review's answer: {"changes": […]}, perhaps in a code fence. `changes`
+// are the well-formed ones; `leftOut` the rest, so a malformed change does not
+// take the whole answer down — it is set aside and said, and the well-formed ones
+// are saved. Anything else is no answer: the review says so and moves on — the
+// same lines are not read again.
export function parseReview(text) {
const start = text.indexOf('{')
const end = text.lastIndexOf('}')
@@ -294,7 +296,10 @@ export function parseReview(text) {
return undefined
}
if (value === null || typeof value !== 'object' || !Array.isArray(value.changes)) return undefined
- return value.changes.every(wellFormed) ? value.changes : undefined
+ const changes = []
+ const leftOut = []
+ for (const change of value.changes) (wellFormed(change) ? changes : leftOut).push(change)
+ return { changes, leftOut }
}
const wellFormed = (change) => {
if (change === null || typeof change !== 'object') return false
@@ -771,7 +776,7 @@ export function install(rt) {
let target = topic === undefined ? main : files.find(file => file.topic === topic)
const fresh = target === undefined
if (fresh) {
- if (files.length - 1 >= MEMORY_LIMITS.topics) throw new Error(`${capital(label)} already has ${MEMORY_LIMITS.topics} topics, the most it holds. Add to one of them (recall lists them), or forget one.`)
+ if (files.length - 1 >= MEMORY_LIMITS.topics) throw Object.assign(new Error(`${capital(label)} already has ${MEMORY_LIMITS.topics} topics, the most it holds. Add to one of them (recall lists them), or forget one.`), { code: FULL })
target = { topic, file: `${topic}.md`, path: join(dirOf(scope), `${topic}.md`), content: '', ...parseMemory(`# ${topic}\n`) }
}
const base = target.lines.length > 0 ? target.lines : parseMemory(target === main ? EMPTY_MAIN : `# ${topic}\n`).lines
@@ -1572,7 +1577,11 @@ export function install(rt) {
else if (op === 'remove') done.push(await removeEntry({ scope, topic: target.file.topic, match: target.text, actor, label }))
} catch (failure) {
if (failure?.code === GONE) return 0
- rt.warn('ds-bot: memory review of %s left out a change: %s', name, failure?.message ?? failure)
+ // The change is named by its text and where it was headed, so the
+ // diagnostics say what was not kept, not only why.
+ const held = op === 'remove' ? `the removal of "${clip(target.text, 160)}"` : `the entry "${clip(text, 160)}"`
+ const aimed = `${fileName(op === 'add' ? { topic } : target.file)} of ${label}`
+ rt.warn('ds-bot: memory review of %s left out a change: %s for %s: %s', name, held, aimed, failure?.message ?? failure)
if (!ROUTED.has(failure?.code)) failed ??= failure
}
}
@@ -1669,19 +1678,25 @@ export function install(rt) {
const language = writtenIn(read.filter(line => line.who === 'user').map(line => line.text.replace(/^User: /, '')), languageNow())
const reply = await botShot(keeper, reviewSystem({ teamWrites: teamWrites(), group }), reviewRequest({ who, tag: tagged, numbered, left, sizes, where: batch.where, earlier, fresh: fresh.texts, more, language }), tag)
if (!alive()) return
- const changes = parseReview(reply)
+ const parsed = parseReview(reply)
// An answer that is no list of changes keeps nothing of the batch; the miss
- // is said, and the place moves on — the same lines are not read again.
- if (changes === undefined) {
+ // is said, and the place moves on — the same lines are not read again. A
+ // change that is not well formed is set aside and said, one by one.
+ if (parsed === undefined) {
rt.warn('ds-bot: memory review of %s got no list of changes from the model: %s', name, clip(reply.replace(/\s+/g, ' '), 120))
} else {
+ const { changes, leftOut } = parsed
+ for (const change of leftOut) {
+ const shown = clip(JSON.stringify(change), 160)
+ rt.warn('ds-bot: memory review of %s left out a change: %s', name, secretEntry(shown) ? 'it looks like a key, token, password, or ID number, which memory never keeps' : `it is not a well-formed change: ${shown}`)
+ }
const changed = await applyReview({ keeper, name, home, numbered, changes: changes.slice(0, REVIEW_CHANGES_MAX), source: batch.source(read), teamWrites })
if (!alive()) return
// An answer at the limit may have left some out, so the same lines are read
// again, a few times at most, while each pass still changes something.
const prior = overflowed.get(key)
const passes = prior !== undefined && prior.since === since ? prior.passes : 0
- if (changes.length >= REVIEW_CHANGES_MAX && changed > 0 && passes < REVIEW_PASSES_MAX) {
+ if (changes.length + leftOut.length >= REVIEW_CHANGES_MAX && changed > 0 && passes < REVIEW_PASSES_MAX) {
overflowed.set(key, { since, passes: passes + 1 })
through = since
again = true
diff --git a/ds-bot/test/memory.test.js b/ds-bot/test/memory.test.js
index 16fc93da..acec5489 100644
--- a/ds-bot/test/memory.test.js
+++ b/ds-bot/test/memory.test.js
@@ -68,10 +68,16 @@ test('the memory panel reads JSON from the model, in a fence or not', () => {
assert.deepEqual(parseAsk('I could not do that.'), { reply: 'I could not do that.', changes: [] })
assert.deepEqual(parseAsk('{"reply": 3}'), { reply: '3', changes: [] })
// The review takes only a list of changes; prose or a broken object is no answer.
- assert.deepEqual(parseReview('```json\n{"changes": [{"op": "add", "text": "x"}]}\n```'), [{ op: 'add', text: 'x' }])
- assert.deepEqual(parseReview('{"changes": []}'), [])
- for (const bad of ['Nothing to keep.', '{"changes": "none"}', '{"changes": [{"op": "add"', '{"changes": ["remember this"]}', '{"changes": [{"text": "x"}]}', '{"changes": [{"op": "add", "text": " "}]}', '{"changes": [{"op": "edit", "entry": "two", "text": "x"}]}', '{"changes": [{"op": "remove", "entry": 0}]}', '{"changes": [{"op": "add", "text": "x", "team": "true"}]}']) assert.equal(parseReview(bad), undefined, bad)
- assert.deepEqual(parseReview('{"changes": [{"op": "remove", "entry": "2"}]}'), [{ op: 'remove', entry: '2' }])
+ assert.deepEqual(parseReview('```json\n{"changes": [{"op": "add", "text": "x"}]}\n```'), { changes: [{ op: 'add', text: 'x' }], leftOut: [] })
+ assert.deepEqual(parseReview('{"changes": []}'), { changes: [], leftOut: [] })
+ for (const bad of ['Nothing to keep.', '{"changes": "none"}', '{"changes": [{"op": "add"']) assert.equal(parseReview(bad), undefined, bad)
+ // A malformed change is set aside; the well-formed ones still count.
+ for (const bad of ['"remember this"', '{"text": "x"}', '{"op": "add", "text": " "}', '{"op": "edit", "entry": "two", "text": "x"}', '{"op": "remove", "entry": 0}', '{"op": "add", "text": "x", "team": "true"}']) {
+ const parsed = parseReview(`{"changes": [${bad}, {"op": "add", "text": "y"}]}`)
+ assert.deepEqual(parsed.changes, [{ op: 'add', text: 'y' }], bad)
+ assert.deepEqual(parsed.leftOut, [JSON.parse(bad)], bad)
+ }
+ assert.deepEqual(parseReview('{"changes": [{"op": "remove", "entry": "2"}]}'), { changes: [{ op: 'remove', entry: '2' }], leftOut: [] })
})
test('the panel shows a summary as headings over paragraphs', () => {
@@ -769,6 +775,24 @@ test('a review keeps what it can, sets aside what memory cannot keep, and says e
assert.equal(requests.length, 1)
})
+test('a review saves the well-formed changes of a partly-malformed answer, and says the rest', async (t) => {
+ const replies = [JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea' }, { op: 'add' }] })]
+ const { ctx, call, read, requests, warnings, stop } = start({ replies, config: { memoryReviewDelayMs: 0 } })
+ t.after(() => stop())
+ await call('state', {})
+ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
+ // The malformed change did not take the answer down: the well-formed one was
+ // saved, the malformed one was said, and the batch is marked read.
+ assert.deepEqual(parseMemory(read('bots', 'b2', 'MEMORY.md')).entries.map(entry => entry.text), ['The user likes tea'])
+ assert.ok(warnings.some(line => line.includes('left out a change') && line.includes('not a well-formed change') && line.includes('{"op":"add"}')), warnings.join('\n'))
+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
+ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
+})
+
test('a backlog past the text limit is reviewed in passes, oldest first', async (t) => {
const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))
mkdirSync(join(home, 'memory'), { recursive: true })
@@ -834,9 +858,10 @@ test('a full MEMORY.md sets an add aside, says so, and an add with a topic still
await reviewed(call)
assert.match(requests[0].messages[0].content[0].text, /MEMORY\.md has 3\d{3} of 4000 characters \(own\)/)
assert.match(requests[0].system, /"topic": "travel"/)
- // The add had no room: it was set aside and said, and the batch is marked read.
+ // The add had no room: it was set aside and said — named by its text and where
+ // it was headed — and the batch is marked read.
assert.equal(parseMemory(read('bots', 'b2', 'MEMORY.md')).entries.length, 13)
- assert.ok(warnings.some(line => line.includes('left out a change') && line.includes('would have') && line.includes('of its 4000 characters')), warnings.join('\n'))
+ assert.ok(warnings.some(line => line.includes('left out a change') && line.includes('"The user flies with Air China"') && line.includes('for MEMORY.md') && line.includes('of its 4000 characters')), warnings.join('\n'))
assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
// The review request goes on telling how full MEMORY.md is, so the next one
// can put the detail in a topic instead.
@@ -848,6 +873,29 @@ test('a full MEMORY.md sets an add aside, says so, and an add with a topic still
assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 12 })
})
+test('a memory that holds the most topics sets a new topic aside, says it, and moves on', async (t) => {
+ const home = mkdtempSync(join(tmpdir(), 'ds-bot-memory-'))
+ const dir = join(home, 'memory', 'bots', 'b2')
+ mkdirSync(dir, { recursive: true })
+ for (let at = 1; at <= MEMORY_LIMITS.topics; at += 1) writeFileSync(join(dir, `topic-${at}.md`), `# topic-${at}\n\n- fact ${at}\n`)
+ writeFileSync(join(dir, 'MEMORY.md'), '# Memory\n\n## Index\n')
+ const replies = [JSON.stringify({ changes: [{ op: 'add', text: 'The user likes tea', topic: 'tea' }] })]
+ const { ctx, call, read, requests, warnings, stop } = start({ home, replies, config: { memoryReviewDelayMs: 0 } })
+ t.after(() => stop())
+ await call('state', {})
+ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
+ // The refusal names the entry and its topic; the batch is marked read, so the
+ // same lines are not read again on the next idle.
+ assert.equal(existsSync(join(dir, 'tea.md')), false)
+ assert.ok(warnings.some(line => line.includes('left out a change') && line.includes('"The user likes tea"') && line.includes('topic [[tea]]') && line.includes('the most it holds')), warnings.join('\n'))
+ assert.deepEqual(JSON.parse(read('reviewed.json')), { 'chat:b2': 11 })
+ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })
+ await reviewed(call)
+ assert.equal(requests.length, 1)
+})
+
test('topic entries that share words with the user\'s new lines come first', () => {
const own = [{ entries: [] }, { topic: 'work', entries: [
{ text: 'Ships on Fridays', meta: { added: '2026-01-01' } },