A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Refuse to treat an unreadable memory or secrets file as empty
Fei2411e480b2df4c7c
diff --git a/ds-bot/src/host/memory.js b/ds-bot/src/host/memory.jsindex becf5ab7..41092863 100644--- a/ds-bot/src/host/memory.js+++ b/ds-bot/src/host/memory.js@@ -469,19 +469,27 @@ export function install(rt) {// The Bot whose model keeps a group's memory: its admin, else its first member.const keeperOf = room => rt.botOf(room.admin) ?? room.members.map(id => rt.botOf(id)).find(Boolean)+ // An unreadable memory is not an empty one: callers that change it fail instead of+ // writing over what could not be read, the way store.js treats a broken state.json.+ const unreadable = (path, error) => {+ rt.warn('ds-bot: unreadable memory file %s: %s', path, error)+ throw Object.assign(new Error(`Memory file ${path} could not be read (${error?.code ?? error?.message}). DS Bot leaves it as it is; fix or move the file.`), { code: 'memory/unreadable' })+ }const readText = (path) => {try {return readFileSync(path, 'utf8')} catch (error) {- if (error?.code !== 'ENOENT') rt.warn('ds-bot: unreadable memory file %s: %s', path, error)- return ''+ if (error?.code === 'ENOENT') return ''+ return unreadable(path, error)}}const topicNames = (scope) => {+ const dir = dirOf(scope)try {- return readdirSync(dirOf(scope)).filter(name => name.endsWith('.md') && name !== MAIN).map(name => name.slice(0, -3)).sort()- } catch {- return []+ return readdirSync(dir).filter(name => name.endsWith('.md') && name !== MAIN).map(name => name.slice(0, -3)).sort()+ } catch (error) {+ if (error?.code === 'ENOENT') return []+ return unreadable(dir, error)}}// MEMORY.md first, then the topics by name.@@ -726,34 +734,62 @@ export function install(rt) {const names = topicNames(scope)return names.length > 0 ? [`Topics: ${names.map(name => `[[${name}]]`).join(', ')}`] : []}+ // The prompt section reads on its own: it may not throw, or one unreadable memory+ // would fail the whole turn. The unreadable file shows as a line instead of entries.+ const sectionRead = (path) => {+ try {+ return { text: readFileSync(path, 'utf8') }+ } catch (error) {+ if (error?.code === 'ENOENT') return { text: '' }+ rt.warn('ds-bot: unreadable memory file %s: %s', path, error)+ return { text: '', failed: `(this memory could not be read: ${error?.code ?? error?.name ?? 'error'})` }+ }+ }+ const sectionEntries = read => (read.failed === undefined ? shownEntries(parseMemory(read.text)) : [read.failed])+ const sectionTopics = (scope) => {+ try {+ return { lines: topicLine(scope) }+ } catch {+ return { lines: [], failed: true }+ }+ }function memorySection(bot, roomId) {const own = ownScope(bot.id)- const main = parseMemory(readText(join(dirOf(own), MAIN)))- const team = parseMemory(readText(join(dirOf(TEAM), MAIN)))+ const main = sectionRead(join(dirOf(own), MAIN))+ const team = sectionRead(join(dirOf(TEAM), MAIN))+ const ownTopics = sectionTopics(own)+ const teamTopics = sectionTopics(TEAM)const lines = ['# Your memory',memoryRules(reviewOn),teamRule(rt.isMain(bot.id), writesTeam(bot.id)),'','## Your memory (MEMORY.md)',- ...shownEntries(main),- ...topicLine(own),+ ...sectionEntries(main),+ ...ownTopics.lines,'','## Team memory, shared by every Bot (team/MEMORY.md)',- ...shownEntries(team),- ...topicLine(TEAM),+ ...sectionEntries(team),+ ...teamTopics.lines,]- if (roomId === undefined) return lines.join('\n')+ let failed = main.failed !== undefined || team.failed !== undefined || ownTopics.failed === true || teamTopics.failed === true+ if (roomId === undefined) return { text: lines.join('\n'), failed }const group = groupScope(roomId)- return [- ...lines,- '',- GROUP_HEADING,- groupNote(reviewOn),- '',- ...shownEntries(parseMemory(readText(join(dirOf(group), MAIN)))),- ...topicLine(group),- ].join('\n')+ const groupRead = sectionRead(join(dirOf(group), MAIN))+ const groupTopics = sectionTopics(group)+ failed = failed || groupRead.failed !== undefined || groupTopics.failed === true+ return {+ text: [+ ...lines,+ '',+ GROUP_HEADING,+ groupNote(reviewOn),+ '',+ ...sectionEntries(groupRead),+ ...groupTopics.lines,+ ].join('\n'),+ failed,+ }}/** @type {Map<string, string>} */@@ -772,10 +808,17 @@ export function install(rt) {const kept = frozen.get(sessionId)if (kept !== undefined && holds(kept)) return keptconst path = snapshotPath(sessionId)- let text = existsSync(path) ? readText(path) : ''+ const snap = existsSync(path) ? sectionRead(path) : { text: '' }+ let text = snap.text+ // A render that could not read everything is used but not kept: it is written+ // down neither as the snapshot nor as the frozen copy, so the next render tries+ // the files again.+ let failed = snap.failed !== undefinedif (text === '' || !holds(text)) {- text = memorySection(bot, roomId)- if (rt.readOnly() === undefined) {+ const section = memorySection(bot, roomId)+ text = section.text+ failed = failed || section.failed+ if (!failed && rt.readOnly() === undefined) {try {mkdirSync(dirname(path), { recursive: true })writeFileSync(`${path}.tmp`, text, 'utf8')@@ -785,7 +828,7 @@ export function install(rt) {}}}- frozen.set(sessionId, text)+ if (!failed) frozen.set(sessionId, text)return text}// The Session's next render freezes the memory as it is then.@@ -943,7 +986,12 @@ export function install(rt) {const self = caller.bot.idconst { scopes, error } = readScopes(args.scope, self, caller.roomId)if (error) return error- const loaded = scopes.map(scope => ({ scope, tag: tagOf(scope, self), label: labelOf(scope, self), files: loadScope(scope) }))+ let loaded+ try {+ loaded = scopes.map(scope => ({ scope, tag: tagOf(scope, self), label: labelOf(scope, self), files: loadScope(scope) }))+ } catch (failure) {+ return failure.message+ }if (args.topic !== undefined && String(args.topic).trim() !== '') {const topic = cleanTopic(args.topic)for (const { label, files } of loaded) {diff --git a/ds-bot/src/host/secrets.js b/ds-bot/src/host/secrets.jsindex 86c3e3b3..20a86476 100644--- a/ds-bot/src/host/secrets.js+++ b/ds-bot/src/host/secrets.js@@ -3,7 +3,7 @@// their shell, tool results lose it before they are saved, and every model request// has it replaced by [secret:<NAME>] before it leaves.import { randomUUID } from 'node:crypto'-import { chmod, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'+import { chmod, copyFile, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'import { join } from 'node:path'import { defineTool } from '@deepseek-ai/dsh-tools'import { replaceFile } from './files.js'@@ -110,38 +110,74 @@ export function install(rt) {// reads it again each time; a copy read then is also read again after taking over,// before this host saves anything.let loadedReadOnly = false+ // An existing secrets file that cannot be read is not an empty one: writing over it+ // would lose every key in it. While `blocked` holds the reason nothing is written,+ // and each load reads the file again (a busy file may only be briefly unreadable).+ let blockedconst loadSecrets = () => {- if (loaded !== null && (loadedReadOnly || rt.readOnly() !== undefined)) loaded = null+ if (loaded !== null && (loadedReadOnly || rt.readOnly() !== undefined || blocked !== undefined)) loaded = nullreturn loaded ??= readSecrets()}const readSecrets = async () => {loadedReadOnly = rt.readOnly() !== undefined+ let contentstry {- const parsed = JSON.parse(await readFile(path, 'utf8'))- secrets.clear()- for (const entry of Array.isArray(parsed?.secrets) ? parsed.secrets : []) {- if (!SECRET_NAME.test(entry?.name ?? '') || typeof entry.value !== 'string') continue- secrets.set(entry.name, {- name: entry.name,- value: entry.value,- purpose: typeof entry.purpose === 'string' ? entry.purpose : '',- scope: entry.scope === 'all' ? 'all' : Array.isArray(entry.scope) ? entry.scope.filter(id => typeof id === 'string') : 'all',- requestedBy: typeof entry.requestedBy === 'string' ? entry.requestedBy : null,- createdAt: Number(entry.createdAt) || Date.now(),- updatedAt: Number(entry.updatedAt) || Date.now(),- ...(Number.isFinite(entry.lastUsedAt) ? { lastUsedAt: entry.lastUsedAt } : {}),- })- }- if (!loadedReadOnly) await chmod(path, 0o600).catch(() => {})+ contents = await readFile(path, 'utf8')} catch (error) {- if (error?.code === 'ENOENT') secrets.clear()- else rt.warn('ds-bot: unreadable secrets file %s (%s)', path, reason(error))+ if (error?.code === 'ENOENT') {+ blocked = undefined+ secrets.clear()+ } else {+ blocked = reason(error)+ rt.warn('ds-bot: unreadable secrets file %s (%s)', path, blocked)+ }+ changed()+ return+ }+ let parsed+ try {+ parsed = JSON.parse(contents)+ } catch {+ parsed = null+ }+ if (parsed === null || typeof parsed !== 'object' || !Array.isArray(parsed.secrets)) {+ // A file that is not a secrets file is kept aside, like a broken state.json, then+ // the store goes on with an empty one.+ const backup = `${path}.broken-${Date.now()}`+ try {+ await copyFile(path, backup)+ blocked = undefined+ secrets.clear()+ rt.warn('ds-bot: unreadable secrets file %s; backed up as %s', path, backup)+ } catch (error) {+ blocked = reason(error)+ rt.warn('ds-bot: unreadable secrets file %s; backup failed: %s', path, error)+ }+ changed()+ return+ }+ blocked = undefined+ secrets.clear()+ for (const entry of parsed.secrets) {+ if (!SECRET_NAME.test(entry?.name ?? '') || typeof entry.value !== 'string') continue+ secrets.set(entry.name, {+ name: entry.name,+ value: entry.value,+ purpose: typeof entry.purpose === 'string' ? entry.purpose : '',+ scope: entry.scope === 'all' ? 'all' : Array.isArray(entry.scope) ? entry.scope.filter(id => typeof id === 'string') : 'all',+ requestedBy: typeof entry.requestedBy === 'string' ? entry.requestedBy : null,+ createdAt: Number(entry.createdAt) || Date.now(),+ updatedAt: Number(entry.updatedAt) || Date.now(),+ ...(Number.isFinite(entry.lastUsedAt) ? { lastUsedAt: entry.lastUsedAt } : {}),+ })}+ if (!loadedReadOnly) await chmod(path, 0o600).catch(() => {})changed()}// Resolves once secrets.json holds the current map; rejects when the write failed.const persist = () => {+ if (blocked !== undefined) return Promise.reject(new Error(`The secrets file ${path} could not be read (${blocked}). DS Bot does not change it; fix or move the file and reload.`))const readOnly = rt.readOnly()if (readOnly !== undefined) return Promise.reject(new Error(readOnly))const snapshot = JSON.stringify({ version: 1, secrets: [...secrets.values()] }, null, 2)@@ -166,7 +202,7 @@ export function install(rt) {if (usedTimer !== null) returnusedTimer = setTimeout(() => {usedTimer = null- if (loadedReadOnly || rt.readOnly() !== undefined) return+ if (loadedReadOnly || rt.readOnly() !== undefined || blocked !== undefined) returnvoid persist().then(() => rt.save()).catch(() => {})}, USED_SAVE_MS)usedTimer.unref?.()diff --git a/ds-bot/test/memory.test.js b/ds-bot/test/memory.test.jsindex 0a499602..2978bccc 100644--- a/ds-bot/test/memory.test.js+++ b/ds-bot/test/memory.test.js@@ -296,6 +296,23 @@ test('a Session keeps the memory it began with, across restarts, until its snapsassert.match(renewed.section('b2'), /- Likes tea \[.*\]\nTopics: \[\[home\]\]\n/)})+test('a memory file that cannot be read is left alone, and the section says so', async (t) => {+ const { home, tool, section, stop } = start()+ t.after(() => stop())+ // A directory reads as EISDIR on every platform; no chmod games needed.+ mkdirSync(join(home, 'memory', 'bots', 'b2', 'MEMORY.md'), { recursive: true })+ const answer = await tool('remember', { text: 'Likes tea' }, 'b2')+ assert.match(answer, /Memory file .*MEMORY\.md could not be read \(EISDIR\)\. DS Bot leaves it as it is/)+ assert.deepEqual(readdirSync(join(home, 'memory', 'bots', 'b2')), ['MEMORY.md'])+ // The prompt shows the failure instead of looking empty, and nothing is frozen.+ assert.match(section('b2'), /\(this memory could not be read: EISDIR\)/)+ assert.equal(existsSync(join(home, 'memory', 'snapshots', 'b2.md')), false)+ rmSync(join(home, 'memory', 'bots', 'b2', 'MEMORY.md'), { recursive: true })+ const after = section('b2')+ assert.doesNotMatch(after, /could not be read/)+ assert.match(after, /\(nothing saved yet\)/)+})+test('a change of role freezes the section again, so its team rule matches the tools', async (t) => {const { call, tool, section, stop } = start()t.after(() => stop())diff --git a/ds-bot/test/secrets.test.js b/ds-bot/test/secrets.test.jsindex a4e844e0..b9108818 100644--- a/ds-bot/test/secrets.test.js+++ b/ds-bot/test/secrets.test.js@@ -1,6 +1,6 @@import assert from 'node:assert/strict'import { randomBytes } from 'node:crypto'-import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'+import { mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'import { tmpdir } from 'node:os'import { join } from 'node:path'import { test } from 'node:test'@@ -82,8 +82,7 @@ const TEAM = {questions: {},}-function start() {- const home = mkdtempSync(join(tmpdir(), 'ds-bot-secrets-'))+function start({ home = mkdtempSync(join(tmpdir(), 'ds-bot-secrets-')) } = {}) {writeFileSync(join(home, 'state.json'), JSON.stringify(TEAM))const contributors = []const shellEnv = {@@ -312,3 +311,33 @@ test('tool results and model requests lose the values', async (t) => {stream({ ...options, system: 'clean', messages: [] }, next)assert.equal(passed.length, 2)})++test('a secrets file that cannot be read blocks every write instead of being overwritten', async (t) => {+ // A directory reads as EISDIR on every platform; no chmod games needed. It has to+ // exist before the plugin loads, or the first read has already found the file empty.+ const home = mkdtempSync(join(tmpdir(), 'ds-bot-secrets-'))+ mkdirSync(join(home, 'secrets.json'))+ const bench = start({ home })+ t.after(bench.stop)+ const file = join(home, 'secrets.json')+ const denied = await bench.call('secret-set', { name: 'TEST_TOKEN', value: '12345678' })+ assert.equal(denied.ok, false)+ assert.match(denied.error.message, /secrets\.json could not be read \(EISDIR\)\. DS Bot does not change it/)+ assert.ok(statSync(file).isDirectory())+ // While it stays unreadable, nothing is written; once it is gone, saving works.+ rmSync(file, { recursive: true })+ const saved = await bench.call('secret-set', { name: 'TEST_TOKEN', value: '12345678' })+ assert.equal(saved.ok, true, JSON.stringify(saved.error))+})++test('a secrets file that is not JSON is kept aside, and the store goes on', async (t) => {+ const home = mkdtempSync(join(tmpdir(), 'ds-bot-secrets-'))+ writeFileSync(join(home, 'secrets.json'), 'not a secrets file\n')+ const bench = start({ home })+ t.after(bench.stop)+ const saved = await bench.call('secret-set', { name: 'TEST_TOKEN', value: '12345678' })+ assert.equal(saved.ok, true, JSON.stringify(saved.error))+ const broken = readdirSync(home).filter(name => name.startsWith('secrets.json.broken-'))+ assert.deepEqual(broken.length, 1)+ assert.equal(readFileSync(join(home, broken[0]), 'utf8'), 'not a secrets file\n')+})