Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Check Bot capabilities in the smoke test with the mock model

Fei241131da23f490f8
diff --git a/ds-bot/dev/mock-llm.mjs b/ds-bot/dev/mock-llm.mjs
index 537c0773..c4bdbd7a 100644
--- a/ds-bot/dev/mock-llm.mjs
+++ b/ds-bot/dev/mock-llm.mjs
@@ -23,6 +23,7 @@
// 报错 (fake rejected key, 401; also as the text of 转告, so the receiving Bot fails)
// 卡片 K (K×1000 characters, then a question card) · 笔记模式 截断一次|缺节一次|总截断|总缺节|失败|正常
// 记住[@话题] 事实 · 团队记住 事实 (remember) · 忘掉 事实 (forget) · 回忆 [词] (recall)
+// 查能力 [词] (names the mcp__ tools and catalog skills that contain 词; also in a group)
// Memory panel: a summary restating the entries; asks 记住|remember X, 团队记住 X,
// 删掉|remove N, 改|edit N: X change entries, anything else gets the entry count.
// Commands in a group:
@@ -321,6 +322,7 @@ function decide(body) {
return [text(earlier.includes(ask[1]) ? `上下文里有「${ask[1]}」。` : `上下文里没有「${ask[1]}」。`)]
}
// Group Sessions: the frozen brief is in the Session's first message, the lookup a tool.
+ if ((ask = /查能力\s*(\S*)/.exec(userLine))) return [text(capabilityReport(me, tools, messages, ask[1]))]
if ((ask = /查私聊\s*(\S*)/.exec(userLine))) return [tool('read_own_chat', ask[1] ? { query: ask[1] } : {})]
if (/私聊聊了啥/.test(userLine)) {
const all = messages.map(m => textOf(m.content)).join('\n')
@@ -377,6 +379,7 @@ function decide(body) {
if ((order = /^查记录\s*([^\n]+)/.exec(userText))) return [tool('read_own_chat', { query: order[1].trim() })]
if ((order = /^查锚点\s*([^\n]+)/.exec(userText))) return [tool('read_own_chat', { around: order[1].trim() })]
if ((order = /^查第\s*(\d+)\s*段/.exec(userText))) return [tool('read_own_chat', { part: Number(order[1]) })]
+ if ((order = /^查能力\s*(\S*)/.exec(userText))) return [text(capabilityReport(me, tools, messages, order[1]))]
if ((order = /^上下文里有\s*(\S+)/.exec(userText))) {
const all = messages.slice(0, -1).map(m => textOf(m.content)).join('\n')
return [text(all.includes(order[1]) ? `上下文里有「${order[1]}」。` : `上下文里没有「${order[1]}」。`)]
@@ -475,6 +478,16 @@ function memoryReview(request) {
return { changes }
}
+// The MCP tools in this request and the entries of the latest skill catalog, both limited
+// to names that contain `filter`.
+function capabilityReport(me, tools, messages, filter) {
+ const catalogs = messages.flatMap(m => textBlocks(m.content)).flatMap(value => [...value.matchAll(/<available_skills>([\s\S]*?)<\/available_skills>/g)])
+ const skills = (catalogs.at(-1)?.[1] ?? '').split('\n').map(line => /^- `([^`]+)`: (.*)$/.exec(line))
+ .filter(entry => entry && entry[1].includes(filter)).map(entry => `${entry[1]}(${entry[2]})`)
+ const mcp = [...tools].filter(name => name.startsWith('mcp__') && name.includes(filter)).sort()
+ return `${me} 能力:工具 ${mcp.join(', ') || '无'};技能 ${skills.join(', ') || '无'}`
+}
+
function secretNote(note, tools) {
const ready = /The user (?:saved|let you use) ([A-Z][A-Z0-9_]*)/.exec(note)
return ready ? [text(`收到 ${ready[1]},我来试一下。`), echoSecret(ready[1], tools)] : [text('好,不用这个密钥了。')]
diff --git a/ds-bot/dev/smoke.mjs b/ds-bot/dev/smoke.mjs
index b84606c0..05877dea 100644
--- a/ds-bot/dev/smoke.mjs
+++ b/ds-bot/dev/smoke.mjs
@@ -8,7 +8,7 @@
//
// `--out <file>` writes a normalized transcript (names instead of ids, sorted where the
// order depends on timing), so two runs, before and after a refactor, can be diffed.
-import { readFileSync, readdirSync, writeFileSync } from 'node:fs'
+import { readFileSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
@@ -34,26 +34,6 @@ const mockLog = () => {
throw error
}
}
-const tapFiles = bot => {
- try {
- return readdirSync(join(HOME, 'tap-bodies'))
- .filter(file => new RegExp(`^\\d+-main-${bot.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.json$`).test(file))
- .sort((a, b) => Number(a.split('-')[0]) - Number(b.split('-')[0]))
- } catch (error) {
- if (error.code === 'ENOENT') return []
- throw error
- }
-}
-const tapBody = bot => {
- const file = tapFiles(bot).at(-1)
- if (!file) throw new Error(`no llm-tap main request for ${bot}`)
- return { path: join(HOME, 'tap-bodies', file), body: readJson(join(HOME, 'tap-bodies', file)) }
-}
-const bodyText = body => JSON.stringify(body)
-const latestReminder = body => {
- const reminders = bodyText(body).match(/<system-reminder>[\s\S]*?<\/system-reminder>/g) ?? []
- return reminders.at(-1) ?? ''
-}
async function login() {
const log = readFileSync(join(HOME, 'web.log'), 'utf8')
@@ -312,49 +292,41 @@ await step('delete the copy', async () => {
return copy.id
}, id => expect(state().bots[id] === undefined, 'copy still there'))
+// The mock answers 查能力 with the mcp__ tools and catalog skills it sees in the request.
const echoName = 'smoke-echo'
const skillName = 'smoke-guide'
const skillDescription = 'Smoke test skill'
const echoServer = join(ROOT, 'ds-bot', 'dev', 'mcp-echo-server.mjs')
+const sees = `工具 mcp__${echoName}__echo;技能 ${skillName}(${skillDescription})`
+const seesNothing = '工具 无;技能 无'
+const reportOf = (calls, bot) => calls.flatMap(call => call.out ?? []).find(block => block.type === 'text' && block.text.startsWith(`${bot} 能力:`))?.text
+const attach = (bot, name, attached) => api('bot-capability', { botId: bot.id, name, attached })
await api('capability-save', { name: echoName, kind: 'mcp', description: 'Smoke test echo server', transport: 'stdio', command: process.execPath, args: [echoServer], env: {} })
await api('capability-save', { name: skillName, kind: 'skill', description: skillDescription, content: 'Use this skill when asked about capability smoke checks.' })
-await api('bot-capabilities', { botId: chief().id, capabilities: [echoName, skillName] })
-for (let attempt = 0; attempt < 60; attempt += 1) {
- const live = await api('state')
- const statuses = live.capabilityStatuses?.[chief().id] ?? {}
- if (statuses[echoName]?.state === 'ready' && statuses[skillName]?.state === 'ready') break
- await sleep(250)
-}
+await attach(chief(), echoName, true)
+await attach(chief(), skillName, true)
-await step('Chief sees its attached MCP and skill', () => send(chief().id, 'Check your available capabilities.'), () => {
- const { body, path } = tapBody('Chief')
- const tools = (body.tools ?? []).map(tool => tool.name)
- return expect(tools.includes(`mcp__${echoName}__echo`), `MCP tool missing from ${path}`)
- ?? expect(latestReminder(body).includes(skillName) && latestReminder(body).includes(skillDescription), `skill catalog entry missing from ${path}`)
+await step('Chief sees its attached MCP and skill', () => send(chief().id, '查能力 smoke'), (_value, calls) => {
+ const report = reportOf(calls, 'Chief')
+ return expect(report?.endsWith(sees), `Chief reported ${report}`)
})
const capabilityRoom = await api('create-room', { name: 'Capability smoke', members: [chief().id, botNamed('Coder').id], admin: chief().id })
-await step('Chief first group turn sees its attached MCP and skill', () => send(capabilityRoom.id, 'Check your available capabilities.'), () => {
- const { body, path } = tapBody('Chief')
- const tools = (body.tools ?? []).map(tool => tool.name)
- return expect(tools.includes(`mcp__${echoName}__echo`), `MCP tool missing from the first group request in ${path}`)
- ?? expect(latestReminder(body).includes(skillName) && latestReminder(body).includes(skillDescription), `skill catalog entry missing from the first group request in ${path}`)
+await step('Chief first group turn sees its attached MCP and skill', () => send(capabilityRoom.id, '查能力 smoke'), (_value, calls) => {
+ const report = reportOf(calls, 'Chief')
+ return expect(report?.endsWith(sees), `Chief reported ${report} in the group`)
})
-await step('Coder does not inherit Chief capabilities', () => send(botNamed('Coder').id, 'Check your available capabilities.'), () => {
- const { body, path } = tapBody('Coder')
- const tools = (body.tools ?? []).map(tool => tool.name)
- return expect(!tools.includes(`mcp__${echoName}__echo`), `MCP tool leaked into ${path}`)
- ?? expect(!latestReminder(body).includes(skillName), `skill leaked into ${path}`)
+await step('Coder does not inherit Chief capabilities', () => send(botNamed('Coder').id, '查能力 smoke'), (_value, calls) => {
+ const report = reportOf(calls, 'Coder')
+ return expect(report?.endsWith(seesNothing), `Coder reported ${report}`)
})
-await api('bot-capabilities', { botId: chief().id, capabilities: [] })
-await sleep(500)
-await step('detached capabilities disappear from Chief', () => send(chief().id, 'Check your available capabilities.'), () => {
- const { body, path } = tapBody('Chief')
- const tools = (body.tools ?? []).map(tool => tool.name)
- return expect(!tools.includes(`mcp__${echoName}__echo`), `MCP tool remained in ${path}`)
- ?? expect(!latestReminder(body).includes(skillName), `latest skill reminder still lists the skill in ${path}`)
+await attach(chief(), echoName, false)
+await attach(chief(), skillName, false)
+await step('detached capabilities disappear from Chief', () => send(chief().id, '查能力 smoke'), (_value, calls) => {
+ const report = reportOf(calls, 'Chief')
+ return expect(report?.endsWith(seesNothing), `Chief reported ${report}`)
})
if (outFile) writeFileSync(outFile, `${JSON.stringify(transcript, null, 2)}\n`)