A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Keep a learned skill proposal when its approval fails; add held-out eval cases
Fei2411d585bfb87eaa
diff --git a/ds-bot/dev/learning-eval/skill-cases-test.json b/ds-bot/dev/learning-eval/skill-cases-test.jsonnew file mode 100644index 00000000..4f6d24fa--- /dev/null+++ b/ds-bot/dev/learning-eval/skill-cases-test.json@@ -0,0 +1,60 @@+{+ "note": "Held-out skill review cases: never used while tuning the prompt. Run once per prompt version.",+ "cases": [+ {+ "id": "t-explain-concept", "expect": "none",+ "bot": { "name": "Tutor", "role": "Teacher" },+ "lines": [+ "User: 什么是闭包?",+ "闭包是函数和它定义时所在作用域的组合……(附一个计数器例子)",+ "User: 懂了,谢谢。"+ ]+ },+ {+ "id": "t-long-but-routine", "expect": "none",+ "bot": { "name": "Coder", "role": "Developer" },+ "lines": [+ "User: Add pagination to the /orders API.",+ "Added `page` and `pageSize` query params with defaults 1 and 20, a total count in the response, and tests for the first page, the last page, and an out-of-range page. All tests pass.",+ "User: Also cap pageSize at 100.",+ "Capped at 100 and added a test.",+ "User: Great, merge it."+ ]+ },+ {+ "id": "t-env-pitfall", "expect": "create",+ "bot": { "name": "Builder", "role": "Build engineer" },+ "lines": [+ "User: Windows CI 构建又挂了,看看。",+ "错误是 Filename too long。我把 node_modules 删掉重装,还是失败。",+ "改用短一点的工作目录 C:\\b,构建通过。但这个仓库的 checkout 路径是 CI 固定的,改不了。",+ "User: 别动目录。在 checkout 之后加 git config --system core.longpaths true,我们其他仓库也这么处理。",+ "已加这一步,构建通过。",+ "User: 以后 Windows 上遇到 Filename too long 都先这样处理,不要去删 node_modules。"+ ],+ "must": ["longpaths", "windows", "filename too long"],+ "mustNot": ["C:\\\\b"],+ "heldOut": [+ { "task": "我们的 GitHub Actions Windows 任务在 npm ci 时报 'Filename too long'。给出你的修复,写出要改的 workflow 步骤。", "expect": ["longpaths"] }+ ],+ "unrelated": { "task": "用一句话解释什么是 HTTP 缓存。", "mustNot": ["longpaths"] }+ },+ {+ "id": "t-delivery-pref", "expect": "create",+ "bot": { "name": "Analyst", "role": "Data analyst" },+ "lines": [+ "User: Pull last month's signups by country.",+ "Here is a 40-row table of signups by country, sorted alphabetically.",+ "User: Sort by count, biggest first, top 10 only, and put the total on top. Always give me the SQL you ran under the table so I can check it.",+ "Done: total 12,430; top 10 countries by count; the SQL is below the table.",+ "User: Do every data pull like this from now on."+ ],+ "must": ["sql", "(top|10)", "total"],+ "mustNot": ["12,?430", "signups by country"],+ "heldOut": [+ { "task": "Pull the number of orders per product category for last week from the `orders` table (columns: id, category, created_at). Show the result as you would deliver it to me, with made-up numbers.", "expect": ["select", "total"] }+ ],+ "unrelated": { "task": "Write a haiku about autumn.", "mustNot": ["select", "sql"] }+ }+ ]+}diff --git a/ds-bot/src/host/learning.js b/ds-bot/src/host/learning.jsindex 923f9424..6e6478e5 100644--- a/ds-bot/src/host/learning.js+++ b/ds-bot/src/host/learning.js@@ -305,17 +305,16 @@ export function install(rt) {const store = data()const proposal = Object.hasOwn(store.proposals, id) ? store.proposals[id] : undefinedif (proposal === undefined) throw Object.assign(new Error('This proposal is no longer open'), { code: 'bot/stale' })- delete store.proposals[id]if (approve !== true) {+ delete store.proposals[id]store.declined = [...store.declined, { name: proposal.name, description: proposal.description, at: Date.now() }].slice(-settings.declinedMax)await persist()return { applied: false }}- try {- await applySkill(proposal)- } finally {- await persist()- }+ // A failed approval keeps the card, so the user can still decline it.+ await applySkill(proposal)+ delete store.proposals[id]+ await persist()return { applied: true }}diff --git a/ds-bot/test/learning.test.js b/ds-bot/test/learning.test.jsindex 9f01e976..27d17171 100644--- a/ds-bot/test/learning.test.js+++ b/ds-bot/test/learning.test.js@@ -273,5 +273,8 @@ test('an update approval fails if the learned skill changed after the proposal',assert.equal(answer.ok, false)assert.match(JSON.stringify(answer), /release-checks changed since this was proposed/)assert.equal(runtime.teamState().capabilities.items['release-checks'].content, 'Changed independently after the proposal.')+ assert.equal(Object.keys(runtime.learningState().proposals).length, 1)+ const declined = await runtime.call('learning-answer', { id: proposal.id, approve: false })+ assert.equal(declined.ok, true)assert.equal(Object.keys(runtime.learningState().proposals).length, 0)})