A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Check Bot capabilities in the smoke test with the mock model
Fei241131da23f490f8
diff --git a/ds-bot/dev/mock-llm.mjs b/ds-bot/dev/mock-llm.mjsindex 537c0773..c4bdbd7a 100644--- a/ds-bot/dev/mock-llm.mjs+++ b/ds-bot/dev/mock-llm.mjs@@ -23,6 +23,7 @@// 报错 (fake rejected key, 401; also as the text of 转告, so the receiving Bot fails)// 卡片 K (K×1000 characters, then a question card) · 笔记模式 截断一次|缺节一次|总截断|总缺节|失败|正常// 记住[@话题] 事实 · 团队记住 事实 (remember) · 忘掉 事实 (forget) · 回忆 [词] (recall)+// 查能力 [词] (names the mcp__ tools and catalog skills that contain 词; also in a group)// Memory panel: a summary restating the entries; asks 记住|remember X, 团队记住 X,// 删掉|remove N, 改|edit N: X change entries, anything else gets the entry count.// Commands in a group:@@ -321,6 +322,7 @@ function decide(body) {return [text(earlier.includes(ask[1]) ? `上下文里有「${ask[1]}」。` : `上下文里没有「${ask[1]}」。`)]}// Group Sessions: the frozen brief is in the Session's first message, the lookup a tool.+ if ((ask = /查能力\s*(\S*)/.exec(userLine))) return [text(capabilityReport(me, tools, messages, ask[1]))]if ((ask = /查私聊\s*(\S*)/.exec(userLine))) return [tool('read_own_chat', ask[1] ? { query: ask[1] } : {})]if (/私聊聊了啥/.test(userLine)) {const all = messages.map(m => textOf(m.content)).join('\n')@@ -377,6 +379,7 @@ function decide(body) {if ((order = /^查记录\s*([^\n]+)/.exec(userText))) return [tool('read_own_chat', { query: order[1].trim() })]if ((order = /^查锚点\s*([^\n]+)/.exec(userText))) return [tool('read_own_chat', { around: order[1].trim() })]if ((order = /^查第\s*(\d+)\s*段/.exec(userText))) return [tool('read_own_chat', { part: Number(order[1]) })]+ if ((order = /^查能力\s*(\S*)/.exec(userText))) return [text(capabilityReport(me, tools, messages, order[1]))]if ((order = /^上下文里有\s*(\S+)/.exec(userText))) {const all = messages.slice(0, -1).map(m => textOf(m.content)).join('\n')return [text(all.includes(order[1]) ? `上下文里有「${order[1]}」。` : `上下文里没有「${order[1]}」。`)]@@ -475,6 +478,16 @@ function memoryReview(request) {return { changes }}+// The MCP tools in this request and the entries of the latest skill catalog, both limited+// to names that contain `filter`.+function capabilityReport(me, tools, messages, filter) {+ const catalogs = messages.flatMap(m => textBlocks(m.content)).flatMap(value => [...value.matchAll(/<available_skills>([\s\S]*?)<\/available_skills>/g)])+ const skills = (catalogs.at(-1)?.[1] ?? '').split('\n').map(line => /^- `([^`]+)`: (.*)$/.exec(line))+ .filter(entry => entry && entry[1].includes(filter)).map(entry => `${entry[1]}(${entry[2]})`)+ const mcp = [...tools].filter(name => name.startsWith('mcp__') && name.includes(filter)).sort()+ return `${me} 能力:工具 ${mcp.join(', ') || '无'};技能 ${skills.join(', ') || '无'}`+}+function secretNote(note, tools) {const ready = /The user (?:saved|let you use) ([A-Z][A-Z0-9_]*)/.exec(note)return ready ? [text(`收到 ${ready[1]},我来试一下。`), echoSecret(ready[1], tools)] : [text('好,不用这个密钥了。')]diff --git a/ds-bot/dev/smoke.mjs b/ds-bot/dev/smoke.mjsindex b84606c0..05877dea 100644--- a/ds-bot/dev/smoke.mjs+++ b/ds-bot/dev/smoke.mjs@@ -8,7 +8,7 @@//// `--out <file>` writes a normalized transcript (names instead of ids, sorted where the// order depends on timing), so two runs, before and after a refactor, can be diffed.-import { readFileSync, readdirSync, writeFileSync } from 'node:fs'+import { readFileSync, writeFileSync } from 'node:fs'import { dirname, join } from 'node:path'import { fileURLToPath } from 'node:url'@@ -34,26 +34,6 @@ const mockLog = () => {throw error}}-const tapFiles = bot => {- try {- return readdirSync(join(HOME, 'tap-bodies'))- .filter(file => new RegExp(`^\\d+-main-${bot.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.json$`).test(file))- .sort((a, b) => Number(a.split('-')[0]) - Number(b.split('-')[0]))- } catch (error) {- if (error.code === 'ENOENT') return []- throw error- }-}-const tapBody = bot => {- const file = tapFiles(bot).at(-1)- if (!file) throw new Error(`no llm-tap main request for ${bot}`)- return { path: join(HOME, 'tap-bodies', file), body: readJson(join(HOME, 'tap-bodies', file)) }-}-const bodyText = body => JSON.stringify(body)-const latestReminder = body => {- const reminders = bodyText(body).match(/<system-reminder>[\s\S]*?<\/system-reminder>/g) ?? []- return reminders.at(-1) ?? ''-}async function login() {const log = readFileSync(join(HOME, 'web.log'), 'utf8')@@ -312,49 +292,41 @@ await step('delete the copy', async () => {return copy.id}, id => expect(state().bots[id] === undefined, 'copy still there'))+// The mock answers 查能力 with the mcp__ tools and catalog skills it sees in the request.const echoName = 'smoke-echo'const skillName = 'smoke-guide'const skillDescription = 'Smoke test skill'const echoServer = join(ROOT, 'ds-bot', 'dev', 'mcp-echo-server.mjs')+const sees = `工具 mcp__${echoName}__echo;技能 ${skillName}(${skillDescription})`+const seesNothing = '工具 无;技能 无'+const reportOf = (calls, bot) => calls.flatMap(call => call.out ?? []).find(block => block.type === 'text' && block.text.startsWith(`${bot} 能力:`))?.text+const attach = (bot, name, attached) => api('bot-capability', { botId: bot.id, name, attached })await api('capability-save', { name: echoName, kind: 'mcp', description: 'Smoke test echo server', transport: 'stdio', command: process.execPath, args: [echoServer], env: {} })await api('capability-save', { name: skillName, kind: 'skill', description: skillDescription, content: 'Use this skill when asked about capability smoke checks.' })-await api('bot-capabilities', { botId: chief().id, capabilities: [echoName, skillName] })-for (let attempt = 0; attempt < 60; attempt += 1) {- const live = await api('state')- const statuses = live.capabilityStatuses?.[chief().id] ?? {}- if (statuses[echoName]?.state === 'ready' && statuses[skillName]?.state === 'ready') break- await sleep(250)-}+await attach(chief(), echoName, true)+await attach(chief(), skillName, true)-await step('Chief sees its attached MCP and skill', () => send(chief().id, 'Check your available capabilities.'), () => {- const { body, path } = tapBody('Chief')- const tools = (body.tools ?? []).map(tool => tool.name)- return expect(tools.includes(`mcp__${echoName}__echo`), `MCP tool missing from ${path}`)- ?? expect(latestReminder(body).includes(skillName) && latestReminder(body).includes(skillDescription), `skill catalog entry missing from ${path}`)+await step('Chief sees its attached MCP and skill', () => send(chief().id, '查能力 smoke'), (_value, calls) => {+ const report = reportOf(calls, 'Chief')+ return expect(report?.endsWith(sees), `Chief reported ${report}`)})const capabilityRoom = await api('create-room', { name: 'Capability smoke', members: [chief().id, botNamed('Coder').id], admin: chief().id })-await step('Chief first group turn sees its attached MCP and skill', () => send(capabilityRoom.id, 'Check your available capabilities.'), () => {- const { body, path } = tapBody('Chief')- const tools = (body.tools ?? []).map(tool => tool.name)- return expect(tools.includes(`mcp__${echoName}__echo`), `MCP tool missing from the first group request in ${path}`)- ?? expect(latestReminder(body).includes(skillName) && latestReminder(body).includes(skillDescription), `skill catalog entry missing from the first group request in ${path}`)+await step('Chief first group turn sees its attached MCP and skill', () => send(capabilityRoom.id, '查能力 smoke'), (_value, calls) => {+ const report = reportOf(calls, 'Chief')+ return expect(report?.endsWith(sees), `Chief reported ${report} in the group`)})-await step('Coder does not inherit Chief capabilities', () => send(botNamed('Coder').id, 'Check your available capabilities.'), () => {- const { body, path } = tapBody('Coder')- const tools = (body.tools ?? []).map(tool => tool.name)- return expect(!tools.includes(`mcp__${echoName}__echo`), `MCP tool leaked into ${path}`)- ?? expect(!latestReminder(body).includes(skillName), `skill leaked into ${path}`)+await step('Coder does not inherit Chief capabilities', () => send(botNamed('Coder').id, '查能力 smoke'), (_value, calls) => {+ const report = reportOf(calls, 'Coder')+ return expect(report?.endsWith(seesNothing), `Coder reported ${report}`)})-await api('bot-capabilities', { botId: chief().id, capabilities: [] })-await sleep(500)-await step('detached capabilities disappear from Chief', () => send(chief().id, 'Check your available capabilities.'), () => {- const { body, path } = tapBody('Chief')- const tools = (body.tools ?? []).map(tool => tool.name)- return expect(!tools.includes(`mcp__${echoName}__echo`), `MCP tool remained in ${path}`)- ?? expect(!latestReminder(body).includes(skillName), `latest skill reminder still lists the skill in ${path}`)+await attach(chief(), echoName, false)+await attach(chief(), skillName, false)+await step('detached capabilities disappear from Chief', () => send(chief().id, '查能力 smoke'), (_value, calls) => {+ const report = reportOf(calls, 'Chief')+ return expect(report?.endsWith(seesNothing), `Chief reported ${report}`)})if (outFile) writeFileSync(outFile, `${JSON.stringify(transcript, null, 2)}\n`)