A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Fix learned skill review edge cases
Fei2411ca5f5ccbf29a
diff --git a/ds-bot/src/host/learning.js b/ds-bot/src/host/learning.jsindex 6e6478e5..c8123a87 100644--- a/ds-bot/src/host/learning.js+++ b/ds-bot/src/host/learning.js@@ -20,6 +20,7 @@ export const LEARNING = {tickMs: 60_000,readMs: 15_000,textMax: 24_000,+ userTextMax: 8_000,userLineMax: 2_000,botLineMax: 1_200,learnedMax: 5,@@ -112,9 +113,10 @@ export function skillProblem(proposal, { skills, learned, limit = LEARNING.learn}const description = typeof proposal.description === 'string' ? proposal.description.trim() : ''const content = typeof proposal.content === 'string' ? proposal.content.trim() : ''+ const why = typeof proposal.why === 'string' ? proposal.why.trim() : ''if (description === '' || description.length > LEARNING.descriptionMax) return `the description must have 1 to ${LEARNING.descriptionMax} characters`if (content === '' || content.length > LEARNING.contentMax) return `the content must have 1 to ${LEARNING.contentMax} characters`- if (holdsSecret(description) || holdsSecret(content)) return 'it looks like it holds a secret'+ if (holdsSecret(description) || holdsSecret(content) || holdsSecret(why)) return 'it looks like it holds a secret'const existing = Object.hasOwn(skills, name) ? skills[name] : undefinedif (action === 'create') {if (existing !== undefined) return `${name} already exists`@@ -142,7 +144,7 @@ export function install(rt) {const path = join(root, 'state.json')// `pending`: Bot id → the chat Sessions with a long turn since the last pass.// `cursors`: Session id → the newest line a pass has read. `proposals`: waiting- // for the user, by id. `declined`: the latest proposals the user turned down.+ // for the user, by id. `learned`: skill names learned by each Bot.let savedconst data = () => {if (saved !== undefined) return saved@@ -156,6 +158,7 @@ export function install(rt) {saved.cursors ??= {}saved.proposals ??= {}saved.declined ??= []+ saved.learned ??= {}return saved}let writing = Promise.resolve()@@ -190,25 +193,35 @@ export function install(rt) {void persist()})- // What was said in a chat since the last pass, newest lines first until the budget.+ // After the text budget, keep scanning to the cursor for user corrections.async function linesSince(sessionId) {const cursor = data().cursors[sessionId]const kept = []+ const userLines = []let used = 0+ let userUsed = 0+ let overBudget = falselet throughfor await (const line of rt.partLines(sessionId, AbortSignal.timeout(settings.readMs), { pages: Infinity })) {through ??= line.seqif (cursor !== undefined && line.seq <= cursor) breakconst text = clip(line.text, line.who === 'user' ? settings.userLineMax : settings.botLineMax)- if (kept.length > 0 && used + text.length > settings.textMax) break- kept.push(text)- used += text.length+ if (!overBudget && used + text.length <= settings.textMax) {+ kept.push(text)+ used += text.length+ continue+ }+ overBudget = true+ if (text.startsWith('User: ') && userUsed + text.length <= settings.userTextMax) {+ userLines.push(text)+ userUsed += text.length+ }}- return { lines: kept.reverse(), through }+ return { lines: [...kept, ...userLines].reverse(), through }}const skillsOf = () => Object.fromEntries(Object.entries(state.capabilities?.items ?? {}).filter(([, item]) => item.kind === 'skill'))- const learnedOf = bot => (bot.capabilities ?? []).filter(name => skillsOf()[name]?.learned === true)+ const learnedOf = bot => (data().learned[bot.id] ?? []).filter(name => skillsOf()[name] !== undefined && (bot.capabilities ?? []).includes(name))const waitingFor = botId => Object.values(data().proposals).filter(entry => entry.botId === botId)async function reviewSkill(bot, lines) {@@ -226,6 +239,13 @@ export function install(rt) {})const reply = await rt.botShot(bot, skillSystem(), text, { sessionId: rt.chatOf(bot.id), kind: 'memory' })const proposal = parseJson(reply)+ if (['description', 'content', 'why'].some(key => {+ const value = proposal?.[key]+ return typeof value === 'string' && rt.redactSecrets(value) !== value+ })) {+ rt.warn('ds-bot: the skill review of %s proposed content containing a saved secret', bot.name)+ return undefined+ }const problem = skillProblem(proposal, { skills, learned, limit: settings.learnedMax })if (problem !== undefined) {rt.warn('ds-bot: the skill review of %s proposed nothing usable: %s', bot.name, problem)@@ -244,7 +264,10 @@ export function install(rt) {description: proposal.description.trim(),content: proposal.content.trim(),why: typeof proposal.why === 'string' ? clip(proposal.why.trim(), 300) : '',- ...(previous ? { previous: { description: previous.description, content: previous.content }, base: hashOf(previous.content) } : {}),+ ...(previous ? {+ previous: { description: previous.description, content: previous.content },+ base: hashOf(`${previous.description}\n${previous.content}`),+ } : {}),}}@@ -264,21 +287,21 @@ export function install(rt) {continue}if (waitingFor(botId).length > 0) continue- const read = []- for (const sessionId of sessions) read.push({ sessionId, ...await linesSince(sessionId) })- const lines = read.flatMap(entry => entry.lines)try {+ const read = []+ for (const sessionId of sessions) read.push({ sessionId, ...await linesSince(sessionId) })+ const lines = read.flatMap(entry => entry.lines)const proposal = lines.length === 0 ? undefined : await reviewSkill(bot, lines)if (proposal !== undefined) {store.proposals[proposal.id] = proposalmade += 1}+ for (const entry of read) if (entry.through !== undefined) store.cursors[entry.sessionId] = entry.through+ delete store.pending[botId]} catch (error) {rt.warn('ds-bot: the skill review of %s failed: %s', bot.name, error?.message ?? error)continue}- for (const entry of read) if (entry.through !== undefined) store.cursors[entry.sessionId] = entry.through- delete store.pending[botId]}await persist()return made@@ -322,12 +345,20 @@ export function install(rt) {const bot = rt.botOf(proposal.botId)if (bot === undefined) throw new Error('The Bot that learned this skill is gone')const current = skillsOf()[proposal.name]- if (proposal.action === 'create' && current !== undefined) throw new Error(`A skill named ${proposal.name} exists now`)- if (proposal.action === 'update' && (current === undefined || hashOf(current.content) !== proposal.base)) throw new Error(`${proposal.name} changed since this was proposed`)- const item = await rt.capabilitySave({ name: proposal.name, kind: 'skill', description: proposal.description, content: proposal.content }, { botId: bot.id, ui: false })- item.learned = true- await rt.save()- if (!(bot.capabilities ?? []).includes(item.name)) await rt.capabilityEndpoint('bot-capability', { botId: bot.id, name: item.name, attached: true })+ const store = data()+ if (proposal.action === 'create' && !(proposal.saved && current !== undefined)) {+ if (current !== undefined) throw new Error(`A skill named ${proposal.name} exists now`)+ const item = await rt.capabilitySave({ name: proposal.name, kind: 'skill', description: proposal.description, content: proposal.content }, { botId: bot.id, ui: false })+ proposal.saved = true+ store.learned[bot.id] = [...new Set([...(store.learned[bot.id] ?? []), item.name])]+ await persist()+ } else if (proposal.action === 'update') {+ if (current === undefined || hashOf(`${current.description}\n${current.content}`) !== proposal.base) throw new Error(`${proposal.name} changed since this was proposed`)+ const item = await rt.capabilitySave({ name: proposal.name, kind: 'skill', description: proposal.description, content: proposal.content }, { botId: bot.id, ui: false })+ store.learned[bot.id] = [...new Set([...(store.learned[bot.id] ?? []), item.name])]+ await persist()+ }+ if (!(bot.capabilities ?? []).includes(proposal.name)) await rt.capabilityEndpoint('bot-capability', { botId: bot.id, name: proposal.name, attached: true })}async function learningEndpoint(endpoint, body) {diff --git a/ds-bot/test/learning.test.js b/ds-bot/test/learning.test.jsindex 27d17171..2326c088 100644--- a/ds-bot/test/learning.test.js+++ b/ds-bot/test/learning.test.js@@ -1,9 +1,9 @@import assert from 'node:assert/strict'-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'import { tmpdir } from 'node:os'import { join } from 'node:path'import { test } from 'node:test'-import { apply } from '../index.js'+import { createRuntime } from '../index.js'import { LEARNING, parseJson, skillProblem } from '../src/host/learning.js'import { fakeContext } from './fake-ctx.js'@@ -37,9 +37,13 @@ const createReply = JSON.stringify({})const noneReply = JSON.stringify({ action: 'none', reason: 'This work taught nothing reusable.' })-function start({ replies = [], config = {}, history = {}, seed = TEAM } = {}) {+function start({ replies = [], config = {}, history = {}, seed = TEAM, learningSeed = {} } = {}) {const home = mkdtempSync(join(tmpdir(), 'ds-bot-learning-'))writeFileSync(join(home, 'state.json'), JSON.stringify(JSON.parse(JSON.stringify(seed))))+ if (Object.keys(learningSeed).length > 0) {+ mkdirSync(join(home, 'learning'), { recursive: true })+ writeFileSync(join(home, 'learning', 'state.json'), JSON.stringify(learningSeed))+ }const ctx = fakeContext()const requests = []Object.assign(ctx, {@@ -88,7 +92,7 @@ function start({ replies = [], config = {}, history = {}, seed = TEAM } = {}) {...config,learningTimes: { minTurnMs: 0, idleMs: 0, intervalMs: 0, tickMs: 10, ...config.learningTimes },}- apply(ctx, { home, ...settings })+ const rt = createRuntime(ctx, { home, ...settings })const call = async (endpoint, payload = {}) => {const response = await route.fetch(new Request('http://localhost/api/bot', {method: 'POST',@@ -103,7 +107,7 @@ function start({ replies = [], config = {}, history = {}, seed = TEAM } = {}) {await ctx.dispose()rmSync(home, { recursive: true, force: true, maxRetries: 5 })}- return { ctx, home, call, requests, history, learningPath, learningState, teamState, stop }+ return { ctx, home, call, requests, history, learningPath, learningState, teamState, rt, stop }}const settle = () => new Promise(resolve => setTimeout(resolve, 20))@@ -164,6 +168,7 @@ test('skillProblem validates actions, learned-skill limits, and secret-looking clearned: ['release-checks'],}), /changes nothing/)assert.match(skillProblem({ ...proposal, content: 'The root password is hunter2' }, { skills: {}, learned: [] }), /secret/)+ assert.match(skillProblem({ ...proposal, why: 'The root password is hunter2' }, { skills: {}, learned: [] }), /secret/)assert.equal(skillProblem({ ...proposal, content: 'Compare `password == input` with a timing-safe check' }, { skills: {}, learned: [] }), undefined)assert.match(skillProblem({ ...proposal, description: 'x'.repeat(LEARNING.descriptionMax + 1) }, { skills: {}, learned: [] }), /description/)})@@ -191,11 +196,23 @@ test('a none reply makes no proposal and advances the chat cursor', async t => {assert.deepEqual((await runtime.call('state')).value.learning ?? {}, {})})-test('a waiting proposal is skipped and approval saves and attaches a learned skill', async t => {+test('approval ownership survives a normal skill edit and remains learned on review', async t => {const history = { b2: [...ownChat] }- const runtime = await startRuntime({ history, replies: [createReply, noneReply] })+ const updateReply = JSON.stringify({+ action: 'update',+ name: 'release-checks',+ description: 'Use when checking any release.',+ content: 'Run the release checks.',+ why: 'The work showed the scope should be broader.',+ })+ const runtime = await startRuntime({+ history,+ replies: [createReply, updateReply],+ config: { learningTimes: { tickMs: 100_000 } },+ })t.after(() => runtime.stop())await finishTurn(runtime.ctx)+ await runtime.rt.learningRun()await until(() => Object.keys(runtime.learningState().proposals ?? {}).length === 1)const proposal = (await runtime.call('state')).value.learning.b2[0]assert.equal(proposal.name, 'release-checks')@@ -205,14 +222,50 @@ test('a waiting proposal is skipped and approval saves and attaches a learned skawait until(() => runtime.learningState().pending?.b2?.includes('b2'))await settle()assert.equal(runtime.requests.length, 1)-const answer = await runtime.call('learning-answer', { id: proposal.id, approve: true })assert.equal(answer.value.applied, true)- await until(() => runtime.learningState().cursors?.b2 === 13 && runtime.learningState().pending?.b2 === undefined)+ await runtime.call('capability-save', {+ name: 'release-checks',+ kind: 'skill',+ description: 'Use when checking a release candidate.',+ content: 'Run the release checks.',+ })+ await runtime.rt.learningRun()+ await until(() => runtime.requests.length === 2)+ await until(() => Object.values(runtime.learningState().proposals ?? {}).some(entry => entry.action === 'update'))const saved = runtime.teamState()- assert.equal(saved.capabilities.items['release-checks'].learned, true)+ assert.equal(saved.capabilities.items['release-checks'].learned, undefined)assert.ok(saved.bots.b2.capabilities.includes('release-checks'))- assert.equal(runtime.requests.length, 2)+ assert.deepEqual(runtime.learningState().learned.b2, ['release-checks'])+ assert.match(runtime.requests[1].messages[0].content[0].text, /This Bot's learned skills[\s\S]*Use when checking a release candidate\./)+})++test('a failed create attachment can be retried without resaving or losing the proposal', async t => {+ const runtime = await startRuntime({ replies: [createReply] })+ t.after(() => runtime.stop())+ await finishTurn(runtime.ctx)+ await until(() => Object.keys(runtime.learningState().proposals ?? {}).length === 1)+ const proposal = Object.values(runtime.learningState().proposals)[0]+ const attach = runtime.rt.capabilityEndpoint+ let fail = true+ runtime.rt.capabilityEndpoint = async (...args) => {+ if (fail) {+ fail = false+ throw new Error('temporary attachment failure')+ }+ return attach(...args)+ }++ const first = await runtime.call('learning-answer', { id: proposal.id, approve: true })+ assert.equal(first.ok, false)+ assert.equal(runtime.learningState().proposals[proposal.id].saved, true)+ assert.ok(runtime.learningState().learned.b2.includes('release-checks'))+ assert.ok(runtime.learningState().proposals[proposal.id])++ const second = await runtime.call('learning-answer', { id: proposal.id, approve: true })+ assert.equal(second.value.applied, true)+ assert.equal(runtime.learningState().proposals[proposal.id], undefined)+ assert.ok(runtime.teamState().bots.b2.capabilities.includes('release-checks'))})test('declined proposals are recorded and included in the next skill request', async t => {@@ -237,7 +290,7 @@ test('declined proposals are recorded and included in the next skill request', aassert.match(runtime.requests[1].messages[0].content[0].text, /Proposals the user declined[\s\S]*- release-checks: Use when checking a release\./)})-test('an update approval fails if the learned skill changed after the proposal', async t => {+test('an update approval fails if the learned skill description alone changed', async t => {const seed = JSON.parse(JSON.stringify(TEAM))seed.bots.b2.capabilities = ['release-checks']seed.capabilities.items['release-checks'] = {@@ -257,7 +310,7 @@ test('an update approval fails if the learned skill changed after the proposal',content: 'Run the new release checks.',why: 'The work showed an additional step.',})- const runtime = await startRuntime({ seed, replies: [updateReply] })+ const runtime = await startRuntime({ seed, replies: [updateReply], learningSeed: { learned: { b2: ['release-checks'] } } })t.after(() => runtime.stop())await finishTurn(runtime.ctx)await until(() => Object.keys(runtime.learningState().proposals ?? {}).length === 1)@@ -266,15 +319,71 @@ test('an update approval fails if the learned skill changed after the proposal',await runtime.call('capability-save', {name: 'release-checks',kind: 'skill',- description: 'Use when checking a release.',- content: 'Changed independently after the proposal.',+ description: 'Use when checking a release candidate.',+ content: 'Run the original release checks.',})const answer = await runtime.call('learning-answer', { id: proposal.id, approve: true })assert.equal(answer.ok, false)assert.match(JSON.stringify(answer), /release-checks changed since this was proposed/)- assert.equal(runtime.teamState().capabilities.items['release-checks'].content, 'Changed independently after the proposal.')+ assert.equal(runtime.teamState().capabilities.items['release-checks'].description, 'Use when checking a release candidate.')assert.equal(Object.keys(runtime.learningState().proposals).length, 1)const declined = await runtime.call('learning-answer', { id: proposal.id, approve: false })assert.equal(declined.ok, true)assert.equal(Object.keys(runtime.learningState().proposals).length, 0)})++test('a saved secret in a proposed skill is not shown as a proposal', async t => {+ const secretValue = 'saved-secret-value-37'+ const runtime = await startRuntime({+ replies: [JSON.stringify({+ action: 'create',+ name: 'release-checks',+ description: 'Use when checking a release.',+ content: `Run with ${secretValue}.`,+ why: 'The work showed it is needed.',+ })],+ })+ t.after(() => runtime.stop())+ await runtime.call('secret-set', { name: 'LEARNING_TEST', value: secretValue })+ await finishTurn(runtime.ctx)+ await until(() => runtime.learningState().cursors?.b2 === 11)+ assert.deepEqual(runtime.learningState().proposals, {})+})++test('user corrections remain in long chat reviews after the text budget', async t => {+ const runtime = await startRuntime({ replies: [noneReply], config: { learningTimes: { tickMs: 100_000 } } })+ t.after(() => runtime.stop())+ const lines = Array.from({ length: 30 }, (_, index) => ({+ seq: 41 - index,+ who: 'assistant',+ text: `Bot: ${'x'.repeat(1_200)}`,+ }))+ lines.push({ seq: 10, who: 'user', text: 'User: Keep the correction even when the Bot writes a long answer.' })+ runtime.rt.partLines = async function* () {+ yield* lines+ }+ await finishTurn(runtime.ctx)+ await runtime.rt.learningRun()+ assert.match(runtime.requests[0].messages[0].content[0].text, /User: Keep the correction even when the Bot writes a long answer\./)+})++test('a failed chat read keeps that Bot pending while other Bots are reviewed', async t => {+ const seed = JSON.parse(JSON.stringify(TEAM))+ seed.bots.b3 = { id: 'b3', sessionId: 'b3', name: 'Coder', role: 'Coding', model: 'deepseek/v4', instructions: '', capabilities: [], createdAt: 3 }+ const runtime = await startRuntime({ seed, replies: [noneReply], config: { learningTimes: { tickMs: 100_000 } } })+ t.after(() => runtime.stop())+ runtime.rt.partLines = async function* (sessionId) {+ if (sessionId === 'b2') throw new Error('session read failed')+ yield { seq: 11, who: 'user', text: 'User: Review this work' }+ }+ await finishTurn(runtime.ctx, { id: 'b2' })+ await finishTurn(runtime.ctx, { id: 'b3' })+ await until(() => runtime.learningState().pending?.b2?.length && runtime.learningState().pending?.b3?.length)++ await runtime.rt.learningRun()++ assert.deepEqual(runtime.learningState().pending.b2, ['b2'])+ assert.equal(runtime.learningState().pending.b3, undefined)+ assert.equal(runtime.learningState().cursors.b3, 11)+ assert.equal(runtime.requests.length, 1)+})