Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Add learned skill proposals

Fei24110e896ef59711
diff --git a/ds-bot/client.js b/ds-bot/client.js
index 6d5969a5..3883548c 100644
--- a/ds-bot/client.js
+++ b/ds-bot/client.js
@@ -117,6 +117,11 @@ var ZH = {
URL: "URL",
"Headers (JSON)": "请求头(JSON)",
"Skill instructions": "技能说明",
+ "New skill": "新技能",
+ "Skill update": "技能更新",
+ Preview: "预览",
+ Approve: "批准",
+ Decline: "拒绝",
"Save changes": "保存更改",
Edit: "编辑",
"Click again to delete.": "再点一次删除。",
@@ -1440,8 +1445,8 @@ function bodyOf(def, paint, flat = false) {
return "";
}
}
-var eyeMarkup = ([cx, cy, w, h45, tilt]) => `<g transform="translate(${cx} ${cy}) rotate(${tilt})"><rect class="bt-eye" x="${-w / 2}" y="${-h45 / 2}" width="${w}" height="${h45}" rx="${w / 2}"/></g>`;
-var happyEyeMarkup = ([cx, cy, w, h45, tilt]) => `<g transform="translate(${cx} ${cy}) rotate(${tilt})"><path class="bt-eye-happy" opacity="0" d="M${-w * 0.62} ${h45 * 0.14}Q0 ${-h45 * 0.42} ${w * 0.62} ${h45 * 0.14}" fill="none" stroke="#000" stroke-width="${w * 0.5}" stroke-linecap="round"/></g>`;
+var eyeMarkup = ([cx, cy, w, h46, tilt]) => `<g transform="translate(${cx} ${cy}) rotate(${tilt})"><rect class="bt-eye" x="${-w / 2}" y="${-h46 / 2}" width="${w}" height="${h46}" rx="${w / 2}"/></g>`;
+var happyEyeMarkup = ([cx, cy, w, h46, tilt]) => `<g transform="translate(${cx} ${cy}) rotate(${tilt})"><path class="bt-eye-happy" opacity="0" d="M${-w * 0.62} ${h46 * 0.14}Q0 ${-h46 * 0.42} ${w * 0.62} ${h46 * 0.14}" fill="none" stroke="#000" stroke-width="${w * 0.5}" stroke-linecap="round"/></g>`;
var maskBox = (size) => `maskUnits="userSpaceOnUse" x="${-size}" y="${-size}" width="${size * 3}" height="${size * 3}"`;
var holes = (def, eyes) => `<rect x="${-def.size}" y="${-def.size}" width="${def.size * 3}" height="${def.size * 3}" fill="#fff"/>${eyes}`;
var MASK_ID = "%MASK%";
@@ -1496,7 +1501,7 @@ function createSource(initial) {
}
};
}
-var EMPTY_ROSTER = { revision: -1, mainBotId: null, mainBotIds: [], bots: [], rooms: [], agents: [], exchangeTurns: {}, questions: {}, btwCards: {}, sideChats: {}, secrets: [], capabilities: [], capabilityStatuses: {}, secretRequests: {}, models: [], defaultModel: null, modelsKey: null, prefs: {}, byId: {}, roomsById: {}, agentsById: {}, agentOf: {}, ready: false };
+var EMPTY_ROSTER = { revision: -1, mainBotId: null, mainBotIds: [], bots: [], rooms: [], agents: [], exchangeTurns: {}, questions: {}, btwCards: {}, sideChats: {}, learning: {}, secrets: [], capabilities: [], capabilityStatuses: {}, secretRequests: {}, models: [], defaultModel: null, modelsKey: null, prefs: {}, byId: {}, roomsById: {}, agentsById: {}, agentOf: {}, ready: false };
var isMainOf = (roster, id) => id !== void 0 && (roster.mainBotIds ?? []).includes(id);
function indexRoster(value) {
const byId = {};
@@ -2188,12 +2193,12 @@ function liveSprings(apply2) {
const dt = Math.min(0.5, Math.max(1e-3, (now - last) / 1e3));
last = now;
const steps = Math.ceil(dt * 240);
- const h45 = dt / steps;
+ const h46 = dt / steps;
let moving = false;
for (const value of values.values()) {
for (let index = 0; index < steps; index += 1) {
- value.v += (-value.k * (value.x - value.to) - value.c * value.v) * h45;
- value.x += value.v * h45;
+ value.v += (-value.k * (value.x - value.to) - value.c * value.v) * h46;
+ value.x += value.v * h46;
}
if (Math.abs(value.x - value.to) < 0.05 && Math.abs(value.v) < 1) {
value.x = value.to;
@@ -18953,6 +18958,100 @@ function serveBrowserReads({ browsers, post, clientId = globalThis.crypto.random
};
}
+// src/client/learning-card.js
+var import_react46 = require("react");
+function ProposalCard({ proposal, initialError = "", onFailure, onResolved, actions }) {
+ const [busy, setBusy] = (0, import_react46.useState)(false);
+ const [error, setError] = (0, import_react46.useState)(initialError);
+ const busyRef = (0, import_react46.useRef)(false);
+ const titleId = (0, import_react46.useId)();
+ const answer = (approve) => {
+ if (busyRef.current) return;
+ busyRef.current = true;
+ setBusy(true);
+ setError("");
+ void Promise.resolve().then(() => actions.learningAnswer(proposal.id, approve)).then(() => onResolved(proposal.id), (failure) => {
+ const message = failure?.message ?? String(failure);
+ setError(message);
+ onFailure(proposal, message);
+ }).finally(() => {
+ busyRef.current = false;
+ setBusy(false);
+ });
+ };
+ return (0, import_react46.createElement)(
+ "section",
+ { className: "bt-q bt-secret bt-learning-card", role: "group", "aria-labelledby": titleId, "aria-busy": busy },
+ (0, import_react46.createElement)(
+ "div",
+ { className: "bt-q-head" },
+ (0, import_react46.createElement)(
+ "div",
+ { className: "bt-q-text" },
+ (0, import_react46.createElement)("p", { className: "bt-q-title", id: titleId }, t(proposal.action === "update" ? "Skill update" : "New skill")),
+ (0, import_react46.createElement)("div", { className: "bt-q-detail" }, (0, import_react46.createElement)("code", { className: "bt-secret-name" }, proposal.name))
+ )
+ ),
+ (0, import_react46.createElement)(
+ "div",
+ { className: "bt-secret-body" },
+ (0, import_react46.createElement)("p", { className: "bt-learning-description" }, proposal.description),
+ proposal.why ? (0, import_react46.createElement)("p", { className: "bt-learning-why" }, proposal.why) : null,
+ (0, import_react46.createElement)(
+ "details",
+ { className: "bt-learning-preview" },
+ (0, import_react46.createElement)("summary", null, t("Preview")),
+ (0, import_react46.createElement)("pre", null, proposal.content)
+ )
+ ),
+ error ? (0, import_react46.createElement)("div", { className: "bt-secret-error", role: "alert" }, error) : null,
+ (0, import_react46.createElement)(
+ "div",
+ { className: "bt-q-foot" },
+ (0, import_react46.createElement)("button", { type: "button", className: "bt-q-btn", disabled: busy, onClick: () => answer(false) }, t("Decline")),
+ (0, import_react46.createElement)("button", { type: "button", className: "bt-q-submit", disabled: busy, onClick: () => answer(true) }, t("Approve"))
+ )
+ );
+}
+function LearningDock({ sessionId, useRoster, actions }) {
+ const roster = useRoster((value) => value);
+ const [failed, setFailed] = (0, import_react46.useState)({});
+ const bot = roster.byId?.[sessionId];
+ if (!bot || currentPart(bot) !== sessionId) return null;
+ const proposals = roster.learning?.[bot.id] ?? [];
+ const visible = [...proposals];
+ for (const entry of Object.values(failed)) {
+ if (entry.proposal.botId === bot.id && !proposals.some((proposal) => proposal.id === entry.proposal.id)) visible.push(entry.proposal);
+ }
+ if (visible.length === 0) return null;
+ const onFailure = (proposal, error) => setFailed((current) => ({ ...current, [proposal.id]: { proposal, error } }));
+ const onResolved = (id) => setFailed((current) => {
+ if (!(id in current)) return current;
+ const next = { ...current };
+ delete next[id];
+ return next;
+ });
+ return (0, import_react46.createElement)(
+ "div",
+ { className: "bt-q-dock bt-secret-dock bt-learning-dock" },
+ visible.map((proposal) => (0, import_react46.createElement)(ProposalCard, {
+ key: proposal.id,
+ proposal,
+ initialError: failed[proposal.id]?.error,
+ onFailure,
+ onResolved,
+ actions
+ }))
+ );
+}
+var LEARNING_CSS = `
+.bt-learning-description{margin:0;font-size:13px;line-height:18px;color:var(--bt-ink)}
+.bt-learning-why{margin:0;font-size:12px;line-height:16px;color:var(--bt-ink-2)}
+.bt-learning-preview{font-size:12px;line-height:16px;color:var(--bt-ink-2)}
+.bt-learning-preview summary{width:fit-content;cursor:pointer}
+.bt-learning-preview pre{max-height:220px;overflow:auto;margin:6px 0 0;padding:8px;border:1px solid var(--bt-line);border-radius:8px;background:var(--bt-card);color:var(--bt-ink);font-family:ui-monospace,monospace;font-size:12px;line-height:16px;white-space:pre-wrap;overflow-wrap:anywhere}
+`;
+
// src/client/right-panel.js
var RIGHTBAR_ATTRIBUTES = ["data-rightbar-collapsed", "data-rightbar-fullscreen"];
function rightColumnOpen(doc) {
@@ -18983,7 +19082,7 @@ function createSwitchWindow(settle) {
}
// src/client/user-message.js
-var import_react46 = require("react");
+var import_react47 = require("react");
var ui = __toESM(require("@deepseek-ai/dsh-client-ui-primitives"), 1);
var canDrawUserMessages = () => typeof ui.projectUserText === "function";
function contentParts(content) {
@@ -19015,38 +19114,38 @@ function userTextParts(text, bots, referenceLabels = []) {
return parts;
}
function UserText({ text, bots, referenceLabels, skillNames, references, actions }) {
- return userTextParts(text, bots, referenceLabels).map((part, index) => part.bot ? (0, import_react46.createElement)(
+ return userTextParts(text, bots, referenceLabels).map((part, index) => part.bot ? (0, import_react47.createElement)(
"button",
{ key: index, type: "button", className: "bt-umention", title: `@${part.bot.name}`, onClick: () => actions.openSession(part.bot.id) },
- (0, import_react46.createElement)(BotMark, { bot: part.bot, size: 16 }),
+ (0, import_react47.createElement)(BotMark, { bot: part.bot, size: 16 }),
part.label
- ) : (0, import_react46.createElement)(import_react46.Fragment, { key: index }, ui.projectUserText(part.text, referenceLabels, skillNames, "skill", references)));
+ ) : (0, import_react47.createElement)(import_react47.Fragment, { key: index }, ui.projectUserText(part.text, referenceLabels, skillNames, "skill", references)));
}
function Attachments({ attachments, renderMessageImages }) {
const compact2 = attachments.length > 1;
- return (0, import_react46.createElement)("div", { className: "bt-uattach" }, attachments.map((block, index) => {
+ return (0, import_react47.createElement)("div", { className: "bt-uattach" }, attachments.map((block, index) => {
if (block.type === "image") {
- return renderMessageImages ? (0, import_react46.createElement)(import_react46.Fragment, { key: index }, renderMessageImages({ images: [{ attachment: block.attachment }], align: "end", compact: compact2 })) : null;
+ return renderMessageImages ? (0, import_react47.createElement)(import_react47.Fragment, { key: index }, renderMessageImages({ images: [{ attachment: block.attachment }], align: "end", compact: compact2 })) : null;
}
const file = block.attachment;
const meta = [ui.fileExtension?.(file.name)?.toUpperCase().slice(0, 8), ui.fileSizeText?.(file.bytes)].filter(Boolean).join(" · ");
- return (0, import_react46.createElement)(
+ return (0, import_react47.createElement)(
"span",
{ key: index, className: "bt-ufile", title: file.name },
- ui.FileTypeIcon ? (0, import_react46.createElement)(ui.FileTypeIcon, { path: file.name, className: "bt-ufile-icon" }) : null,
- (0, import_react46.createElement)("span", { className: "bt-ufile-text" }, (0, import_react46.createElement)("span", { className: "bt-ufile-name" }, file.name), meta ? (0, import_react46.createElement)("span", { className: "bt-ufile-meta" }, meta) : null)
+ ui.FileTypeIcon ? (0, import_react47.createElement)(ui.FileTypeIcon, { path: file.name, className: "bt-ufile-icon" }) : null,
+ (0, import_react47.createElement)("span", { className: "bt-ufile-text" }, (0, import_react47.createElement)("span", { className: "bt-ufile-name" }, file.name), meta ? (0, import_react47.createElement)("span", { className: "bt-ufile-meta" }, meta) : null)
);
}));
}
function CopyButton({ text }) {
- const [copied, setCopied] = (0, import_react46.useState)(false);
+ const [copied, setCopied] = (0, import_react47.useState)(false);
const copy2 = () => {
void navigator.clipboard?.writeText(text).then(() => {
setCopied(true);
setTimeout(() => setCopied(false), 1e3);
});
};
- return (0, import_react46.createElement)("button", { type: "button", className: "bt-tool", "aria-label": copied ? t("Copied") : t("Copy"), title: copied ? t("Copied") : t("Copy"), onClick: copy2 }, copied ? (0, import_react46.createElement)(CheckIcon) : (0, import_react46.createElement)(CopyIcon));
+ return (0, import_react47.createElement)("button", { type: "button", className: "bt-tool", "aria-label": copied ? t("Copied") : t("Copy"), title: copied ? t("Copied") : t("Copy"), onClick: copy2 }, copied ? (0, import_react47.createElement)(CheckIcon) : (0, import_react47.createElement)(CopyIcon));
}
var clock3 = (time) => typeof time === "number" ? new Date(time).toLocaleTimeString(dateLocale(), { hour: "numeric", minute: "2-digit" }) : void 0;
function UserMessageCell({ node, sessionId, useRoster, actions, renderMessageImages, openFile, openSkill }) {
@@ -19058,10 +19157,10 @@ function UserMessageCell({ node, sessionId, useRoster, actions, renderMessageIma
const steering = node.kind === "steering";
const groupTime = roster.roomsById?.[sessionId] && Number.isFinite(data.time) ? messageClock(data.time) : void 0;
const { shown, folded, control } = useFold(text, { markdown: false });
- const body = (0, import_react46.createElement)(UserText, { text: shown, bots: mentionableBots(roster, sessionId), referenceLabels, skillNames: data.skillNames ?? [], references, actions });
- const row = (0, import_react46.useRef)(null);
- const [toolsOpen, setToolsOpen] = (0, import_react46.useState)(false);
- (0, import_react46.useEffect)(() => {
+ const body = (0, import_react47.createElement)(UserText, { text: shown, bots: mentionableBots(roster, sessionId), referenceLabels, skillNames: data.skillNames ?? [], references, actions });
+ const row = (0, import_react47.useRef)(null);
+ const [toolsOpen, setToolsOpen] = (0, import_react47.useState)(false);
+ (0, import_react47.useEffect)(() => {
if (!toolsOpen) return void 0;
const onDown = (event) => {
if (!row.current?.contains(event.target)) setToolsOpen(false);
@@ -19077,24 +19176,24 @@ function UserMessageCell({ node, sessionId, useRoster, actions, renderMessageIma
if (!noHover() || event.target instanceof Element && event.target.closest("a,button,input,textarea,select,pre,code,[role=button],[role=link]")) return;
setToolsOpen((value) => !value);
};
- return (0, import_react46.createElement)(
+ return (0, import_react47.createElement)(
"div",
{ ref: row, className: "bt-umsg", "data-steering": steering || void 0 },
- (0, import_react46.createElement)(
+ (0, import_react47.createElement)(
"div",
{ className: "bt-ustack" },
- attachments.length > 0 ? (0, import_react46.createElement)(Attachments, { attachments, renderMessageImages }) : null,
- text !== "" || rest.length > 0 ? (0, import_react46.createElement)(
+ attachments.length > 0 ? (0, import_react47.createElement)(Attachments, { attachments, renderMessageImages }) : null,
+ text !== "" || rest.length > 0 ? (0, import_react47.createElement)(
"div",
{ className: "bt-ubody", title: clock3(data.time), "data-folded": folded || void 0, onClick: tapTools },
- control ? (0, import_react46.createElement)("div", { className: "bt-fold-body" }, body) : body,
+ control ? (0, import_react47.createElement)("div", { className: "bt-fold-body" }, body) : body,
control,
- rest.map((block, index) => ui.JsonBlock ? (0, import_react46.createElement)(ui.JsonBlock, { key: index, label: t("Content"), payload: block }) : null)
+ rest.map((block, index) => ui.JsonBlock ? (0, import_react47.createElement)(ui.JsonBlock, { key: index, label: t("Content"), payload: block }) : null)
) : null,
- referenceLabels.length > 0 ? (0, import_react46.createElement)("div", { className: "bt-urefs" }, t("Referenced {labels}", { labels: referenceLabels.join(t(", ")) })) : null,
- groupTime ? (0, import_react46.createElement)("div", { className: "bt-utime", title: clockLabel(data.time) }, groupTime) : null
+ referenceLabels.length > 0 ? (0, import_react47.createElement)("div", { className: "bt-urefs" }, t("Referenced {labels}", { labels: referenceLabels.join(t(", ")) })) : null,
+ groupTime ? (0, import_react47.createElement)("div", { className: "bt-utime", title: clockLabel(data.time) }, groupTime) : null
),
- text === "" ? null : (0, import_react46.createElement)("div", { className: "bt-utools", "data-open": toolsOpen || void 0 }, (0, import_react46.createElement)(CopyButton, { text }))
+ text === "" ? null : (0, import_react47.createElement)("div", { className: "bt-utools", "data-open": toolsOpen || void 0 }, (0, import_react47.createElement)(CopyButton, { text }))
);
}
var USER_MESSAGE_CSS = `
@@ -19283,6 +19382,7 @@ function apply(ctx) {
const live2 = {};
if (value.connectors !== void 0 && JSON.stringify(value.connectors) !== JSON.stringify(current.connectors ?? [])) live2.connectors = value.connectors;
if (value.capabilityStatuses !== void 0 && JSON.stringify(value.capabilityStatuses) !== JSON.stringify(current.capabilityStatuses ?? {})) live2.capabilityStatuses = value.capabilityStatuses;
+ if (value.learning !== void 0 && JSON.stringify(value.learning) !== JSON.stringify(current.learning ?? {})) live2.learning = value.learning;
if (value.btwCards !== void 0 && JSON.stringify(value.btwCards) !== JSON.stringify(current.btwCards ?? {})) live2.btwCards = value.btwCards;
if (value.sideChats !== void 0 && JSON.stringify(value.sideChats) !== JSON.stringify(current.sideChats ?? {})) live2.sideChats = value.sideChats;
if (Object.keys(live2).length > 0 && current.ready) roster.set({ ...current, ...live2 });
@@ -19550,6 +19650,7 @@ function apply(ctx) {
};
const actions = {
...teamActions({ call, after, partFor, roster, ui: ui2, live, cards, heldCards, turnClock }),
+ learningAnswer: (id, approve) => after(call("learning-answer", { id, approve })),
// The full-pane overlays sit above the conversation, so opening one must dismiss them
// or the click appears to do nothing.
isAgentSession: (id) => isAgentSessionId(id),
@@ -19921,6 +20022,7 @@ function apply(ctx) {
() => installCompiledStyles(SHELL_CSS, "shell"),
() => installCompiledStyles(SIDEBAR_CSS, "sidebar"),
() => installCompiledStyles(SECRET_CSS, "secrets"),
+ () => installCompiledStyles(LEARNING_CSS, "learning"),
() => installCompiledStyles(MODEL_CSS + FOLDER_CSS + BOT_SETTINGS_CSS + GROUP_SETTINGS_CSS + AVATAR_LIBRARY_CSS, "models"),
() => installCompiledStyles(USAGE_CSS, "usage"),
() => installCompiledStyles(SCHEDULE_CSS, "schedules"),
@@ -19988,6 +20090,7 @@ function apply(ctx) {
() => shadow("conversation.input.dock", { id: "bot-question", order: -100, inject: face }, QuestionDock),
() => shadow("conversation.input.dock", { id: "bot-view", order: -95, inject: face }, WidgetDock),
() => shadow("conversation.input.dock", { id: "bot-secret", order: -90, inject: face }, SecretDock),
+ () => shadow("conversation.input.dock", { id: "bot-learning", order: -80, inject: face }, LearningDock),
() => shadow("conversation.input.dock", { id: "bot-handoff", order: -85, inject: face }, HandoffDock)
];
ctx.effect(() => installCompiledStyles(RETURN_CSS, "return"), "ds-bot: return styles");
diff --git a/ds-bot/dev/learning-eval.mjs b/ds-bot/dev/learning-eval.mjs
new file mode 100644
index 00000000..ce7776ae
--- /dev/null
+++ b/ds-bot/dev/learning-eval.mjs
@@ -0,0 +1,114 @@
+// Real-model check of the skill review in src/host/learning.js, on an OpenAI-compatible
+// endpoint. Two stages per case in learning-eval/skill-cases.json:
+// decide: does the review propose the expected action (none, create, update) with the
+// rules the work showed and none of the one-off details?
+// effect: for the skills it creates, do held-out tasks of the same class come out
+// better with the skill than without, and does an unrelated task stay clean?
+// Usage (no defaults; nothing is saved but the report):
+// LEARN_EVAL_BASE_URL=… LEARN_EVAL_MODEL=… LEARN_EVAL_KEY=… \
+// node --import ./test/harness-resolve.mjs dev/learning-eval.mjs [--reps 3] [--cases skill-cases.json] [--only id,id] [--out report.json]
+import { readFileSync, writeFileSync } from 'node:fs'
+import { parseArgs } from 'node:util'
+import { parseJson, skillProblem, skillRequest, skillSystem, LEARNING } from '../src/host/learning.js'
+import { writtenIn } from '../src/host/memory.js'
+
+const { values: args } = parseArgs({ options: { reps: { type: 'string', default: '3' }, only: { type: 'string' }, cases: { type: 'string', default: 'skill-cases.json' }, out: { type: 'string' }, stage: { type: 'string', default: 'decide,effect' } } })
+const base = process.env.LEARN_EVAL_BASE_URL
+const model = process.env.LEARN_EVAL_MODEL
+const key = process.env.LEARN_EVAL_KEY
+if (!base || !model || !key) throw new Error('Set LEARN_EVAL_BASE_URL, LEARN_EVAL_MODEL and LEARN_EVAL_KEY.')
+const reps = Number(args.reps)
+const stages = new Set(args.stage.split(','))
+const cases = JSON.parse(readFileSync(new URL(`./learning-eval/${args.cases}`, import.meta.url), 'utf8')).cases
+ .filter(entry => !args.only || args.only.split(',').includes(entry.id))
+
+async function chat(system, user) {
+ for (let attempt = 1; ; attempt += 1) {
+ try {
+ const response = await fetch(`${base.replace(/\/$/, '')}/chat/completions`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
+ body: JSON.stringify({ model, messages: [{ role: 'system', content: system }, { role: 'user', content: user }], max_tokens: 8192, reasoning_effort: 'none' }),
+ signal: AbortSignal.timeout(180_000),
+ })
+ if (!response.ok) throw new Error(`HTTP ${response.status}`)
+ const body = await response.json()
+ return { text: body.choices?.[0]?.message?.content ?? '', tokens: body.usage?.total_tokens ?? 0 }
+ } catch (error) {
+ if (attempt >= 4) throw error
+ await new Promise(resolve => setTimeout(resolve, 2000 * attempt))
+ }
+ }
+}
+
+const re = source => new RegExp(source, 'i')
+let tokens = 0
+
+async function decide(entry) {
+ const learned = entry.learned ?? []
+ const skills = Object.fromEntries([...learned, ...(entry.library ?? [])].map(skill => [skill.name, { kind: 'skill', content: '', ...skill }]))
+ const user = entry.lines.filter(line => line.startsWith('User: ')).map(line => line.slice(6))
+ const request = skillRequest({ bot: entry.bot, lines: entry.lines, learned, library: entry.library ?? [], declined: entry.declined ?? [], language: writtenIn(user, 'English') })
+ const reply = await chat(skillSystem(), request)
+ tokens += reply.tokens
+ const proposal = parseJson(reply.text)
+ const problem = skillProblem(proposal, { skills, learned: learned.map(skill => skill.name), limit: LEARNING.learnedMax })
+ const action = problem === undefined ? proposal.action : 'invalid'
+ const text = `${proposal?.description ?? ''}\n${proposal?.content ?? ''}`
+ const misses = []
+ if (action !== entry.expect) misses.push(`action ${action}${problem ? ` (${problem})` : ''}, expected ${entry.expect}`)
+ if (action === entry.expect && action !== 'none') {
+ if (entry.target && proposal.name !== entry.target) misses.push(`updated ${proposal.name}, expected ${entry.target}`)
+ for (const must of entry.must ?? []) if (!re(must).test(text)) misses.push(`missing /${must}/`)
+ for (const bad of entry.mustNot ?? []) if (re(bad).test(text)) misses.push(`has /${bad}/`)
+ }
+ return { pass: misses.length === 0, action, misses, proposal: action === 'invalid' ? reply.text : proposal }
+}
+
+const withSkill = skill => `You are a capable assistant.\n\nA skill applies to this kind of task; follow it:\n<skill name="${skill.name}">\ndescription: ${skill.description}\n${skill.content}\n</skill>`
+const plain = 'You are a capable assistant.'
+function gradeTask(task, answer) {
+ const misses = []
+ for (const must of task.expect ?? []) if (!re(must).test(answer)) misses.push(`missing /${must}/`)
+ for (const bad of task.mustNot ?? []) if (re(bad).test(answer)) misses.push(`has /${bad}/`)
+ const kept = answer.split('\n').filter(line => line.trim() !== '')
+ if (task.lastLine && !re(task.lastLine).test(kept.at(-1) ?? '')) misses.push(`last line lacks /${task.lastLine}/`)
+ if (task.maxLines && answer.split('\n').filter(line => line.trim() !== '').length > task.maxLines) misses.push(`more than ${task.maxLines} lines`)
+ return misses
+}
+
+async function effect(entry, skill) {
+ const arms = { without: plain, with: withSkill(skill) }
+ const rows = []
+ for (const [arm, system] of Object.entries(arms)) {
+ for (const task of [...entry.heldOut, { ...entry.unrelated, unrelated: true }]) {
+ for (let rep = 0; rep < reps; rep += 1) {
+ const reply = await chat(system, task.task)
+ tokens += reply.tokens
+ const misses = gradeTask(task, reply.text)
+ rows.push({ arm, unrelated: task.unrelated === true, task: task.task.slice(0, 40), pass: misses.length === 0, misses })
+ }
+ }
+ }
+ const rate = (arm, unrelated) => {
+ const picked = rows.filter(row => row.arm === arm && row.unrelated === unrelated)
+ return `${picked.filter(row => row.pass).length}/${picked.length}`
+ }
+ return { heldOut: { without: rate('without', false), with: rate('with', false) }, unrelated: { without: rate('without', true), with: rate('with', true) }, rows }
+}
+
+const report = { model, reps, cases: [] }
+for (const entry of cases) {
+ const result = { id: entry.id, expect: entry.expect, decide: [] }
+ if (stages.has('decide')) for (let rep = 0; rep < reps; rep += 1) result.decide.push(await decide(entry))
+ const made = result.decide.find(run => run.pass && run.action !== 'none')?.proposal
+ if (stages.has('effect') && made && entry.heldOut) result.effect = await effect(entry, made)
+ report.cases.push(result)
+ const passed = result.decide.filter(run => run.pass).length
+ console.log(`${entry.id.padEnd(22)} decide ${passed}/${result.decide.length}${result.effect ? ` held-out without ${result.effect.heldOut.without} with ${result.effect.heldOut.with} unrelated with ${result.effect.unrelated.with}` : ''}`)
+ for (const run of result.decide.filter(item => !item.pass)) console.log(` - ${run.misses.join('; ')}`)
+}
+const all = report.cases.flatMap(entry => entry.decide)
+const quiet = report.cases.filter(entry => entry.expect === 'none').flatMap(entry => entry.decide)
+console.log(`decide total ${all.filter(run => run.pass).length}/${all.length}; restraint ${quiet.filter(run => run.pass).length}/${quiet.length}; tokens ${tokens}`)
+if (args.out) writeFileSync(args.out, JSON.stringify(report, null, 1))
diff --git a/ds-bot/dev/learning-eval/skill-cases.json b/ds-bot/dev/learning-eval/skill-cases.json
new file mode 100644
index 00000000..6657ddb7
--- /dev/null
+++ b/ds-bot/dev/learning-eval/skill-cases.json
@@ -0,0 +1,242 @@
+{
+ "note": "Skill review cases. expect: none | create | update. must/mustNot: regexes (case-insensitive) on description+content. heldOut: tasks of the same class, answered with and without the learned skill; unrelated: a task the skill must not leak into.",
+ "cases": [
+ {
+ "id": "routine-rename",
+ "expect": "none",
+ "bot": {
+ "name": "Coder",
+ "role": "Developer"
+ },
+ "lines": [
+ "User: Rename the variable `cnt` to `count` in src/stats.js.",
+ "Renamed `cnt` to `count` in src/stats.js (4 places) and ran the tests: all pass.",
+ "User: Thanks."
+ ]
+ },
+ {
+ "id": "one-off-bug",
+ "expect": "none",
+ "bot": {
+ "name": "Coder",
+ "role": "Developer"
+ },
+ "lines": [
+ "User: The export button crashes with \"Cannot read properties of undefined (reading 'rows')\".",
+ "Found it: exportTable() runs before the table loads, so table is undefined. I added a guard that waits for the load event, and added a test.",
+ "User: Works now, thanks."
+ ]
+ },
+ {
+ "id": "personal-fact",
+ "expect": "none",
+ "bot": {
+ "name": "Assistant",
+ "role": "Personal assistant"
+ },
+ "lines": [
+ "User: 帮我订周五去上海的高铁,我一般坐靠窗。",
+ "已查到周五 9:00 G7 次有靠窗座位,已帮你预订。",
+ "User: 好的。我老家在杭州,以后过年也要订票。"
+ ]
+ },
+ {
+ "id": "release-corrections",
+ "expect": "create",
+ "bot": {
+ "name": "Shipper",
+ "role": "Release engineer"
+ },
+ "lines": [
+ "User: 发 0.4.0。",
+ "已把改动直接推到 main,并打了 v0.4.0 标签。",
+ "User: 不对。不要直接推 main,先开 PR 到 dev,合并要我同意。标签也删掉。",
+ "已删除标签,回退 main,在 dev 上开了 PR。",
+ "User: 发版前还要跑 node dev/release/scan.mjs 扫描泄露,必须是 must fix: 0。你漏了。",
+ "已运行扫描,结果 must fix: 0。",
+ "User: 还有,版本号在 ds-bot/package.json,CHANGELOG 要同时填日期,两个一起改,别只改一个。",
+ "已同时更新 package.json 版本号和 CHANGELOG 日期,并推到 PR 分支。",
+ "User: 上次发 0.3.0 你也是只改了版本号。记住这套流程。"
+ ],
+ "must": [
+ "dev",
+ "scan\\.mjs",
+ "must fix: ?0",
+ "CHANGELOG",
+ "package\\.json"
+ ],
+ "mustNot": [
+ "#\\d{2,}",
+ "20\\d\\d-\\d\\d-\\d\\d",
+ "0\\.4\\.0"
+ ],
+ "heldOut": [
+ {
+ "task": "我修好了一个小 bug,准备发 0.5.2。告诉我你接下来按什么步骤做,列出来。",
+ "expect": [
+ "scan\\.mjs",
+ "CHANGELOG",
+ "(PR|pull request)",
+ "package\\.json"
+ ],
+ "mustNot": [
+ "(直接|先)推(到)? ?main"
+ ]
+ },
+ {
+ "task": "Ship version 1.2.0 of this repo. List the exact steps you will take, in order.",
+ "expect": [
+ "scan\\.mjs",
+ "CHANGELOG",
+ "dev"
+ ]
+ }
+ ],
+ "unrelated": {
+ "task": "Write a Python function that reverses a string, with one example.",
+ "mustNot": [
+ "scan\\.mjs",
+ "CHANGELOG"
+ ]
+ }
+ },
+ {
+ "id": "review-format",
+ "expect": "create",
+ "bot": {
+ "name": "Reviewer",
+ "role": "Code reviewer"
+ },
+ "lines": [
+ "User: Review my diff in src/auth.js.",
+ "Great work overall! The code is clean. A few thoughts: maybe consider renaming `tok`, the error message could be friendlier, and line 42 compares the password with == which may allow timing attacks.",
+ "User: Stop praising. Only list what must be fixed before merge, most severe first. Each item: file:line, the problem, the fix. Skip style nits.",
+ "1. src/auth.js:42 — password compared with ==, open to timing attacks. Fix: use crypto.timingSafeEqual.",
+ "User: Better. Next time review like this from the start. Also say \"No blocking issues\" when there are none instead of inventing some.",
+ "Understood.",
+ "User: Review src/session.js too.",
+ "No blocking issues."
+ ],
+ "must": [
+ "sever",
+ "line",
+ "(nit|style)",
+ "no blocking issues"
+ ],
+ "mustNot": [
+ "auth\\.js",
+ "timingSafeEqual"
+ ],
+ "heldOut": [
+ {
+ "task": "Review this change before merge:\n```js\nfunction getUser(id) {\n const q = 'SELECT * FROM users WHERE id = ' + id\n return db.query(q)\n}\nconst x = 1;\n```\nFile is src/users.js, the function starts at line 10.",
+ "expect": [
+ "users\\.js:1\\d",
+ "(sql|inject)"
+ ],
+ "mustNot": [
+ "(great|nice|good) (work|job)",
+ "well done"
+ ]
+ },
+ {
+ "task": "Review this change before merge, file src/math.js line 3:\n```js\nexport const add = (a, b) => a + b\n```",
+ "expect": [
+ "no blocking issues"
+ ]
+ }
+ ],
+ "unrelated": {
+ "task": "Explain in two sentences what a hash table is.",
+ "mustNot": [
+ "no blocking issues",
+ "file:line"
+ ]
+ }
+ },
+ {
+ "id": "weekly-update",
+ "expect": "update",
+ "target": "weekly-report",
+ "bot": {
+ "name": "Writer",
+ "role": "Report writer"
+ },
+ "learned": [
+ {
+ "name": "weekly-report",
+ "description": "Use when writing the user's weekly work report.",
+ "content": "- Group items by project.\n- List what was done, then what is next."
+ }
+ ],
+ "lines": [
+ "User: 写本周周报。",
+ "(按项目分组的周报,约 30 行,列出了完成项和下周计划)",
+ "User: 太长了。先写一句结论,再列要点,全文最多 10 行。风险单独一行放最后。",
+ "已改为:结论一句 + 7 行要点 + 风险一行。",
+ "User: 对,以后都这样。"
+ ],
+ "must": [
+ "结论|conclusion",
+ "10",
+ "风险|risk",
+ "项目|project"
+ ],
+ "mustNot": [
+ "20\\d\\d-\\d\\d-\\d\\d"
+ ],
+ "heldOut": [
+ {
+ "task": "写本周周报。本周:A 项目完成登录页、修了 3 个 bug;B 项目接口联调完成一半,第三方接口不稳定;下周:A 上线,B 完成联调。",
+ "maxLines": 12,
+ "lastLine": "风险"
+ }
+ ],
+ "unrelated": {
+ "task": "把这句话翻成英文:今天天气很好。",
+ "mustNot": [
+ "结论",
+ "风险"
+ ]
+ }
+ },
+ {
+ "id": "already-covered",
+ "expect": "none",
+ "bot": {
+ "name": "Shipper",
+ "role": "Release engineer"
+ },
+ "library": [
+ {
+ "name": "release-checks",
+ "description": "Use when releasing a new version: PR to dev, leak scan, version and CHANGELOG together."
+ }
+ ],
+ "lines": [
+ "User: 发 0.6.1。",
+ "按流程:在 dev 开 PR,跑了 node dev/release/scan.mjs(must fix: 0),同时改了 package.json 版本号和 CHANGELOG 日期,等你同意合并。",
+ "User: 好,合并吧。"
+ ]
+ },
+ {
+ "id": "declined-before",
+ "expect": "none",
+ "bot": {
+ "name": "Coder",
+ "role": "Developer"
+ },
+ "declined": [
+ {
+ "name": "commit-messages",
+ "description": "Use when writing commit messages: imperative mood, under 72 characters."
+ }
+ ],
+ "lines": [
+ "User: Commit this.",
+ "Committed: \"Add retry to the upload client\".",
+ "User: ok"
+ ]
+ }
+ ]
+}
diff --git a/ds-bot/index.js b/ds-bot/index.js
index 4325b78b..d5938d42 100644
--- a/ds-bot/index.js
+++ b/ds-bot/index.js
@@ -30,6 +30,7 @@ import * as browserPlugins from './src/host/browser-plugins.js'
import * as secrets from './src/host/secrets.js'
import * as connectors from './src/host/connectors.js'
import * as capabilities from './src/host/capabilities.js'
+import * as learning from './src/host/learning.js'
import * as schedules from './src/host/schedules.js'
import * as usage from './src/host/usage.js'
import * as update from './src/host/update.js'
@@ -47,7 +48,7 @@ export const inject = ['llm', 'tools', 'systemPrompt', 'sessionController', 'ses
// the functions other modules call; they call them through `rt` at run time, never during
// install. Listeners run and tools are listed in registration order, so this order holds:
// usage meters model requests after every listener that rewrites them.
-const MODULES = [diagnostics, store, models, providers, permissions, turns, handoffs, requests, delivery, roster, folders, agents, prompt, timeContext, tools, chatIndex, groups, approvals, resume, memory, parts, images, avatarLibrary, secrets, connectors, capabilities, schedules, browser, browserPlugins, usage, update, dshUpdate, thinking, feed, channel]
+const MODULES = [diagnostics, store, models, providers, permissions, turns, handoffs, requests, delivery, roster, folders, agents, prompt, timeContext, tools, chatIndex, groups, approvals, resume, memory, parts, images, avatarLibrary, secrets, connectors, capabilities, learning, schedules, browser, browserPlugins, usage, update, dshUpdate, thinking, feed, channel]
export function apply(ctx, config = {}) {
createRuntime(ctx, config)
diff --git a/ds-bot/src/client/i18n.js b/ds-bot/src/client/i18n.js
index 32ba8c99..d99862e1 100644
--- a/ds-bot/src/client/i18n.js
+++ b/ds-bot/src/client/i18n.js
@@ -81,6 +81,11 @@ const ZH = {
URL: 'URL',
'Headers (JSON)': '请求头(JSON)',
'Skill instructions': '技能说明',
+ 'New skill': '新技能',
+ 'Skill update': '技能更新',
+ Preview: '预览',
+ Approve: '批准',
+ Decline: '拒绝',
'Save changes': '保存更改',
Edit: '编辑',
'Click again to delete.': '再点一次删除。',
diff --git a/ds-bot/src/client/index.js b/ds-bot/src/client/index.js
index daf7dd4a..0563fb87 100644
--- a/ds-bot/src/client/index.js
+++ b/ds-bot/src/client/index.js
@@ -24,6 +24,7 @@ import { Overlays } from './overlays.js'
import { createBotBrowsers, serveBrowserReads } from './bot-browsers.js'
import { BROWSER_CSS } from './browser-pane.js'
import { SECRET_CSS, SecretDock } from './secret-card.js'
+import { LearningDock, LEARNING_CSS } from './learning-card.js'
import { MODEL_CSS, ModelChoiceDock, DeepSeekSignIn } from './model-menu.js'
import { BOT_SETTINGS_CSS } from './bot-settings.js'
import { AVATAR_LIBRARY_CSS } from './avatar-library.js'
@@ -159,6 +160,7 @@ export function apply(ctx) {
const live = {}
if (value.connectors !== undefined && JSON.stringify(value.connectors) !== JSON.stringify(current.connectors ?? [])) live.connectors = value.connectors
if (value.capabilityStatuses !== undefined && JSON.stringify(value.capabilityStatuses) !== JSON.stringify(current.capabilityStatuses ?? {})) live.capabilityStatuses = value.capabilityStatuses
+ if (value.learning !== undefined && JSON.stringify(value.learning) !== JSON.stringify(current.learning ?? {})) live.learning = value.learning
if (value.btwCards !== undefined && JSON.stringify(value.btwCards) !== JSON.stringify(current.btwCards ?? {})) live.btwCards = value.btwCards
if (value.sideChats !== undefined && JSON.stringify(value.sideChats) !== JSON.stringify(current.sideChats ?? {})) live.sideChats = value.sideChats
if (Object.keys(live).length > 0 && current.ready) roster.set({ ...current, ...live })
@@ -400,6 +402,7 @@ export function apply(ctx) {
}
const actions = {
...teamActions({ call, after, partFor, roster, ui, live, cards, heldCards, turnClock }),
+ learningAnswer: (id, approve) => after(call('learning-answer', { id, approve })),
// The full-pane overlays sit above the conversation, so opening one must dismiss them
// or the click appears to do nothing.
isAgentSession: id => isAgentSessionId(id),
@@ -789,6 +792,7 @@ export function apply(ctx) {
() => installCompiledStyles(SHELL_CSS, 'shell'),
() => installCompiledStyles(SIDEBAR_CSS, 'sidebar'),
() => installCompiledStyles(SECRET_CSS, 'secrets'),
+ () => installCompiledStyles(LEARNING_CSS, 'learning'),
() => installCompiledStyles(MODEL_CSS + FOLDER_CSS + BOT_SETTINGS_CSS + GROUP_SETTINGS_CSS + AVATAR_LIBRARY_CSS, 'models'),
() => installCompiledStyles(USAGE_CSS, 'usage'),
() => installCompiledStyles(SCHEDULE_CSS, 'schedules'),
@@ -856,6 +860,7 @@ export function apply(ctx) {
() => shadow('conversation.input.dock', { id: 'bot-question', order: -100, inject: face }, QuestionDock),
() => shadow('conversation.input.dock', { id: 'bot-view', order: -95, inject: face }, WidgetDock),
() => shadow('conversation.input.dock', { id: 'bot-secret', order: -90, inject: face }, SecretDock),
+ () => shadow('conversation.input.dock', { id: 'bot-learning', order: -80, inject: face }, LearningDock),
() => shadow('conversation.input.dock', { id: 'bot-handoff', order: -85, inject: face }, HandoffDock),
]
// The return button belongs to neither group: it renders only in Agent mode.
diff --git a/ds-bot/src/client/learning-card.js b/ds-bot/src/client/learning-card.js
new file mode 100644
index 00000000..14d9098f
--- /dev/null
+++ b/ds-bot/src/client/learning-card.js
@@ -0,0 +1,78 @@
+import { createElement as h, useId, useRef, useState } from 'react'
+import { currentPart } from './text.js'
+import { t } from './i18n.js'
+
+function ProposalCard({ proposal, initialError = '', onFailure, onResolved, actions }) {
+ const [busy, setBusy] = useState(false)
+ const [error, setError] = useState(initialError)
+ const busyRef = useRef(false)
+ const titleId = useId()
+ const answer = approve => {
+ if (busyRef.current) return
+ busyRef.current = true
+ setBusy(true)
+ setError('')
+ void Promise.resolve().then(() => actions.learningAnswer(proposal.id, approve))
+ .then(() => onResolved(proposal.id), failure => {
+ const message = failure?.message ?? String(failure)
+ setError(message)
+ onFailure(proposal, message)
+ })
+ .finally(() => {
+ busyRef.current = false
+ setBusy(false)
+ })
+ }
+ return h('section', { className: 'bt-q bt-secret bt-learning-card', role: 'group', 'aria-labelledby': titleId, 'aria-busy': busy },
+ h('div', { className: 'bt-q-head' },
+ h('div', { className: 'bt-q-text' },
+ h('p', { className: 'bt-q-title', id: titleId }, t(proposal.action === 'update' ? 'Skill update' : 'New skill')),
+ h('div', { className: 'bt-q-detail' }, h('code', { className: 'bt-secret-name' }, proposal.name)))),
+ h('div', { className: 'bt-secret-body' },
+ h('p', { className: 'bt-learning-description' }, proposal.description),
+ proposal.why ? h('p', { className: 'bt-learning-why' }, proposal.why) : null,
+ h('details', { className: 'bt-learning-preview' },
+ h('summary', null, t('Preview')),
+ h('pre', null, proposal.content))),
+ error ? h('div', { className: 'bt-secret-error', role: 'alert' }, error) : null,
+ h('div', { className: 'bt-q-foot' },
+ h('button', { type: 'button', className: 'bt-q-btn', disabled: busy, onClick: () => answer(false) }, t('Decline')),
+ h('button', { type: 'button', className: 'bt-q-submit', disabled: busy, onClick: () => answer(true) }, t('Approve'))))
+}
+
+export function LearningDock({ sessionId, useRoster, actions }) {
+ const roster = useRoster(value => value)
+ const [failed, setFailed] = useState({})
+ const bot = roster.byId?.[sessionId]
+ if (!bot || currentPart(bot) !== sessionId) return null
+ const proposals = roster.learning?.[bot.id] ?? []
+ const visible = [...proposals]
+ for (const entry of Object.values(failed)) {
+ if (entry.proposal.botId === bot.id && !proposals.some(proposal => proposal.id === entry.proposal.id)) visible.push(entry.proposal)
+ }
+ if (visible.length === 0) return null
+ const onFailure = (proposal, error) => setFailed(current => ({ ...current, [proposal.id]: { proposal, error } }))
+ const onResolved = id => setFailed(current => {
+ if (!(id in current)) return current
+ const next = { ...current }
+ delete next[id]
+ return next
+ })
+ return h('div', { className: 'bt-q-dock bt-secret-dock bt-learning-dock' },
+ visible.map(proposal => h(ProposalCard, {
+ key: proposal.id,
+ proposal,
+ initialError: failed[proposal.id]?.error,
+ onFailure,
+ onResolved,
+ actions,
+ })))
+}
+
+export const LEARNING_CSS = `
+.bt-learning-description{margin:0;font-size:13px;line-height:18px;color:var(--bt-ink)}
+.bt-learning-why{margin:0;font-size:12px;line-height:16px;color:var(--bt-ink-2)}
+.bt-learning-preview{font-size:12px;line-height:16px;color:var(--bt-ink-2)}
+.bt-learning-preview summary{width:fit-content;cursor:pointer}
+.bt-learning-preview pre{max-height:220px;overflow:auto;margin:6px 0 0;padding:8px;border:1px solid var(--bt-line);border-radius:8px;background:var(--bt-card);color:var(--bt-ink);font-family:ui-monospace,monospace;font-size:12px;line-height:16px;white-space:pre-wrap;overflow-wrap:anywhere}
+`
diff --git a/ds-bot/src/client/sources.js b/ds-bot/src/client/sources.js
index ff5db0df..36e9b579 100644
--- a/ds-bot/src/client/sources.js
+++ b/ds-bot/src/client/sources.js
@@ -16,7 +16,7 @@ export function createSource(initial) {
}
}
-export const EMPTY_ROSTER = { revision: -1, mainBotId: null, mainBotIds: [], bots: [], rooms: [], agents: [], exchangeTurns: {}, questions: {}, btwCards: {}, sideChats: {}, secrets: [], capabilities: [], capabilityStatuses: {}, secretRequests: {}, models: [], defaultModel: null, modelsKey: null, prefs: {}, byId: {}, roomsById: {}, agentsById: {}, agentOf: {}, ready: false }
+export const EMPTY_ROSTER = { revision: -1, mainBotId: null, mainBotIds: [], bots: [], rooms: [], agents: [], exchangeTurns: {}, questions: {}, btwCards: {}, sideChats: {}, learning: {}, secrets: [], capabilities: [], capabilityStatuses: {}, secretRequests: {}, models: [], defaultModel: null, modelsKey: null, prefs: {}, byId: {}, roomsById: {}, agentsById: {}, agentOf: {}, ready: false }
// Any Main Bot holds the Main Bot powers; `mainBotId`, the first one, is the sidebar tile.
export const isMainOf = (roster, id) => id !== undefined && (roster.mainBotIds ?? []).includes(id)
diff --git a/ds-bot/src/host/channel.js b/ds-bot/src/host/channel.js
index d254700d..e27822de 100644
--- a/ds-bot/src/host/channel.js
+++ b/ds-bot/src/host/channel.js
@@ -80,7 +80,8 @@ export function install(rt) {
// Set while another dsh host holds the team (store.js): why this one cannot change it.
readOnly: rt.readOnly() ?? null,
})
- // What changes without a new revision, sent with every state answer. `memoryNews`:
+ // What changes without a new revision, sent with every state answer. `learning`:
+ // each Bot's pending skill proposals. `memoryNews`:
// per Bot or group chat, its latest memory change; `memoryReviews`: memory reviews waiting or
// running; `work`: per Bot, the turn it runs and its tool; per group, the members whose
// turn there runs; the approvals waiting; per Bot, who it waits for to hand back work;
@@ -90,6 +91,7 @@ export function install(rt) {
work: { bots: rt.workView(), rooms: rt.roomWorkView(), approvals: rt.approvalsView(), waiting: rt.waitingView?.() ?? {}, turns: rt.turnsView?.() ?? {}, steered: rt.steeredView?.() ?? {} },
memoryNews: rt.memoryNews(),
memoryReviews: rt.memoryReviews(),
+ learning: rt.learningView(),
connectors: rt.connectorsView(),
capabilityStatuses: rt.capabilityStatusesView(),
})
@@ -363,6 +365,9 @@ export function install(rt) {
case 'memory-forget':
await rt.load()
return ok(await rt.memoryEndpoint(endpoint, body))
+ case 'learning-answer':
+ await rt.load()
+ return ok(await rt.learningEndpoint(endpoint, body))
case 'answer-question': {
if (typeof body.sessionId !== 'string') return fail('Nothing to send', 'bot/invalid')
await rt.load()
diff --git a/ds-bot/src/host/learning.js b/ds-bot/src/host/learning.js
new file mode 100644
index 00000000..923f9424
--- /dev/null
+++ b/ds-bot/src/host/learning.js
@@ -0,0 +1,347 @@
+// Learning from work. After a long turn a Bot's chat is marked for review. When the
+// team has been idle for a while, and at most every few hours, one background pass
+// reads what was said since the last pass and asks the Bot's model whether the work
+// taught a skill worth keeping. Most work teaches nothing, so most passes propose
+// nothing. A proposal never takes effect by itself: it waits as a card in the Bot's
+// chat until the user approves it, and a declined one is remembered so the same idea
+// does not come back.
+import { createHash, randomUUID } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import { mkdir, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { replaceFile } from './files.js'
+import { looksLikeKey, looksSecret, writtenIn } from './memory.js'
+import { validateCapabilityName } from './capabilities.js'
+
+export const LEARNING = {
+ minTurnMs: 3 * 60_000,
+ idleMs: 10 * 60_000,
+ intervalMs: 12 * 60 * 60_000,
+ tickMs: 60_000,
+ readMs: 15_000,
+ textMax: 24_000,
+ userLineMax: 2_000,
+ botLineMax: 1_200,
+ learnedMax: 5,
+ descriptionMax: 300,
+ contentMax: 8_000,
+ declinedMax: 20,
+}
+
+const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} […]` : text)
+export const hashOf = text => createHash('sha256').update(String(text ?? '')).digest('hex').slice(0, 16)
+
+// The answer of the review's model: one JSON object, perhaps in a code fence.
+export function parseJson(text) {
+ const raw = String(text ?? '').trim()
+ // A skill's content may hold code fences of its own, so read the whole answer first
+ // and fall back to a fenced block only when that fails.
+ const fenced = /^```(?:json)?\s*([\s\S]*?)```\s*$/.exec(raw)?.[1]
+ for (const candidate of [raw, fenced]) {
+ if (candidate === undefined) continue
+ const start = candidate.indexOf('{')
+ const end = candidate.lastIndexOf('}')
+ if (start === -1 || end < start) continue
+ try {
+ const value = JSON.parse(candidate.slice(start, end + 1))
+ if (value && typeof value === 'object' && !Array.isArray(value)) return value
+ } catch {}
+ }
+ return undefined
+}
+
+// ---------------------------------------------------------------------------
+// The skill review.
+
+export function skillSystem() {
+ return [
+ 'You review a stretch of finished work by an AI Bot and decide whether it taught a skill worth keeping.',
+ 'A skill is a short document that a future Bot loads when a task of the same class comes up. Strong models already do most tasks well without one, so a weak or narrow skill only adds noise. Most work teaches nothing new: answer "none" unless every test below passes.',
+ '',
+ 'Tests:',
+ '1. Reusable: tasks of this class will come up again in later, different work. A rule that only fits this task, file, bug, or pull request fails.',
+ '2. Missing knowledge: it gives constraints or context that a capable model does not have or gets wrong: requirements and corrections the user stated, project or tool conventions found by trial and error, pitfalls that cost time, how this user wants the result checked or delivered. General know-how fails, and so does anything that reading the code or the docs shows at once.',
+ '3. Earned: the work shows it. The user asked for it or corrected the Bot, or a non-obvious way was found after one that failed. Never invent rules the work did not show.',
+ '4. Not covered: no listed skill already says it. When a learned skill of this Bot covers the class, update that skill instead of making a new one. Also update a learned skill that the work showed to be wrong, outdated, or missing a step.',
+ '',
+ 'Never put in a skill: one-off facts, task progress, file lists, dates, version, pull request or issue numbers, pasted errors or logs, facts about the user that are not about how to work (memory keeps those), or keys, passwords, and tokens.',
+ 'Write rules that stand alone, not the story of this work: a reason says why the rule matters, never when or how it was broken ("the user corrected this before", "version 1.2 missed it"). Examples must be general, not the files, values, or fixes of this task.',
+ '',
+ 'Form:',
+ '- name: the class of task, lowercase letters, digits, and single hyphens, 32 characters at most, e.g. "release-checks", never "fix-login-bug".',
+ '- description: one sentence, 200 characters at most, that says when to use the skill. Be concrete about the trigger.',
+ '- content: Markdown, 600 words at most. The rules as short imperative lines, each with a brief reason when it is not obvious; numbered steps only where order matters; how to check the work is done. Keep what is still true when you update, and remove what the work showed to be wrong.',
+ '',
+ 'Answer with one JSON object only, one of:',
+ '{"action":"none","reason":"<one sentence>"}',
+ '{"action":"create","name":"…","description":"…","content":"…","why":"<one sentence: what in this work shows the skill is needed>"}',
+ '{"action":"update","name":"<a learned skill listed>","description":"…","content":"<the whole new content>","why":"<one sentence>"}',
+ ].join('\n')
+}
+
+// `learned`: this Bot's learned skills, in full. `library`: every other skill on the
+// team, by name and description. `declined`: proposals the user turned down.
+export function skillRequest({ bot, lines, learned = [], library = [], declined = [], language, limit = LEARNING.learnedMax }) {
+ const parts = [`The Bot: ${bot.name}${bot.role ? `, ${bot.role}` : ''}.`]
+ parts.push(learned.length === 0
+ ? 'This Bot has no learned skills yet.'
+ : `This Bot's learned skills (${learned.length} of ${limit}${learned.length >= limit ? '; at the limit, only update one' : ''}):\n${learned.map(skill => `<skill name="${skill.name}">\ndescription: ${skill.description}\n${skill.content}\n</skill>`).join('\n')}`)
+ if (library.length > 0) parts.push(`Other skills on the team:\n${library.map(skill => `- ${skill.name}: ${skill.description}`).join('\n')}`)
+ if (declined.length > 0) parts.push(`Proposals the user declined. Do not propose them again unless this work shows something new:\n${declined.map(entry => `- ${entry.name}: ${entry.description}`).join('\n')}`)
+ if (language) parts.push(`Write the description and the content in ${language}.`)
+ parts.push(`The work, oldest first ("User:" lines are the user's own words):\n<work>\n${lines.join('\n')}\n</work>`)
+ return parts.join('\n\n')
+}
+
+// What keeps a proposal from the user, or undefined when it may be shown.
+// `skills`: the team's skills by name; `learned`: names this Bot learned.
+// Skills often talk about code such as `password == input`; read keys everywhere, but
+// password wording only outside code.
+const holdsSecret = text => looksLikeKey(text) || looksSecret(text.replace(/```[\s\S]*?```/g, ' ').replace(/`[^`\n]*`/g, ' '))
+
+export function skillProblem(proposal, { skills, learned, limit = LEARNING.learnedMax }) {
+ if (proposal === undefined) return 'the answer is not a JSON object'
+ const { action } = proposal
+ if (action === 'none') return undefined
+ if (action !== 'create' && action !== 'update') return `unknown action ${String(action)}`
+ let name
+ try {
+ name = validateCapabilityName(proposal.name)
+ } catch (error) {
+ return error.message
+ }
+ const description = typeof proposal.description === 'string' ? proposal.description.trim() : ''
+ const content = typeof proposal.content === 'string' ? proposal.content.trim() : ''
+ if (description === '' || description.length > LEARNING.descriptionMax) return `the description must have 1 to ${LEARNING.descriptionMax} characters`
+ if (content === '' || content.length > LEARNING.contentMax) return `the content must have 1 to ${LEARNING.contentMax} characters`
+ if (holdsSecret(description) || holdsSecret(content)) return 'it looks like it holds a secret'
+ const existing = Object.hasOwn(skills, name) ? skills[name] : undefined
+ if (action === 'create') {
+ if (existing !== undefined) return `${name} already exists`
+ if (learned.length >= limit) return `the Bot already keeps ${limit} learned skills`
+ } else {
+ if (existing === undefined || existing.kind !== 'skill' || !learned.includes(name)) return `${name} is not a learned skill of this Bot`
+ if (existing.content.trim() === content && existing.description.trim() === description) return 'the update changes nothing'
+ }
+ return undefined
+}
+
+// ---------------------------------------------------------------------------
+
+export function install(rt) {
+ const { ctx, config, home, state } = rt
+ const settings = { ...LEARNING, ...(config.learningTimes ?? {}) }
+ const on = config.learning !== false && config.memory !== false
+ const view = () => ({})
+ if (!on) {
+ Object.assign(rt, { learningView: view, learningEndpoint: async () => { throw new Error('Learning is turned off on this host') }, learningRun: async () => 0 })
+ return
+ }
+
+ const root = join(home, 'learning')
+ const path = join(root, 'state.json')
+ // `pending`: Bot id → the chat Sessions with a long turn since the last pass.
+ // `cursors`: Session id → the newest line a pass has read. `proposals`: waiting
+ // for the user, by id. `declined`: the latest proposals the user turned down.
+ let saved
+ const data = () => {
+ if (saved !== undefined) return saved
+ try {
+ saved = JSON.parse(readFileSync(path, 'utf8'))
+ } catch (error) {
+ if (error?.code !== 'ENOENT') rt.warn('ds-bot: unreadable learning state %s: %s', path, error?.message ?? error)
+ saved = {}
+ }
+ saved.pending ??= {}
+ saved.cursors ??= {}
+ saved.proposals ??= {}
+ saved.declined ??= []
+ return saved
+ }
+ let writing = Promise.resolve()
+ const persist = () => {
+ const text = JSON.stringify(data(), null, 1)
+ writing = writing.then(async () => {
+ await mkdir(dirname(path), { recursive: true })
+ const temp = `${path}.${process.pid}.tmp`
+ await writeFile(temp, text, 'utf8')
+ await replaceFile(temp, path)
+ }).catch(error => rt.warn('ds-bot: saving the learning state failed: %s', error?.message ?? error))
+ return writing
+ }
+
+ // A turn in a Bot's own chat that ran long marks the chat for the next pass.
+ const started = new Map()
+ let lastActivity = Date.now()
+ ctx.on('agent/status', ({ agent, status }) => {
+ const id = agent?.id
+ if (id === undefined || !rt.isBotSession(id)) return
+ lastActivity = Date.now()
+ if (status !== 'idle') {
+ if (!started.has(id)) started.set(id, Date.now())
+ return
+ }
+ const since = started.get(id)
+ started.delete(id)
+ const botId = rt.chatOwners.get(id)
+ if (since === undefined || botId === undefined || Date.now() - since < settings.minTurnMs) return
+ const pending = data().pending
+ pending[botId] = [...new Set([...(pending[botId] ?? []), id])]
+ void persist()
+ })
+
+ // What was said in a chat since the last pass, newest lines first until the budget.
+ async function linesSince(sessionId) {
+ const cursor = data().cursors[sessionId]
+ const kept = []
+ let used = 0
+ let through
+ for await (const line of rt.partLines(sessionId, AbortSignal.timeout(settings.readMs), { pages: Infinity })) {
+ through ??= line.seq
+ if (cursor !== undefined && line.seq <= cursor) break
+ const text = clip(line.text, line.who === 'user' ? settings.userLineMax : settings.botLineMax)
+ if (kept.length > 0 && used + text.length > settings.textMax) break
+ kept.push(text)
+ used += text.length
+ }
+ return { lines: kept.reverse(), through }
+ }
+
+ const skillsOf = () => Object.fromEntries(Object.entries(state.capabilities?.items ?? {}).filter(([, item]) => item.kind === 'skill'))
+ const learnedOf = bot => (bot.capabilities ?? []).filter(name => skillsOf()[name]?.learned === true)
+ const waitingFor = botId => Object.values(data().proposals).filter(entry => entry.botId === botId)
+
+ async function reviewSkill(bot, lines) {
+ const skills = skillsOf()
+ const learned = learnedOf(bot)
+ const library = Object.values(skills).filter(item => !learned.includes(item.name)).map(item => ({ name: item.name, description: item.description }))
+ const text = skillRequest({
+ bot,
+ lines,
+ learned: learned.map(name => skills[name]),
+ library,
+ declined: data().declined,
+ language: writtenIn(lines.filter(line => line.startsWith('User: ')).map(line => line.slice(6)), rt.uiLanguage?.()),
+ limit: settings.learnedMax,
+ })
+ const reply = await rt.botShot(bot, skillSystem(), text, { sessionId: rt.chatOf(bot.id), kind: 'memory' })
+ const proposal = parseJson(reply)
+ const problem = skillProblem(proposal, { skills, learned, limit: settings.learnedMax })
+ if (problem !== undefined) {
+ rt.warn('ds-bot: the skill review of %s proposed nothing usable: %s', bot.name, problem)
+ return undefined
+ }
+ if (proposal.action === 'none') return undefined
+ const name = proposal.name.trim()
+ const previous = proposal.action === 'update' ? skills[name] : undefined
+ return {
+ id: randomUUID(),
+ kind: 'skill',
+ action: proposal.action,
+ botId: bot.id,
+ at: Date.now(),
+ name,
+ description: proposal.description.trim(),
+ content: proposal.content.trim(),
+ why: typeof proposal.why === 'string' ? clip(proposal.why.trim(), 300) : '',
+ ...(previous ? { previous: { description: previous.description, content: previous.content }, base: hashOf(previous.content) } : {}),
+ }
+ }
+
+ // One pass over every Bot with a long turn since the last one. A Bot with a
+ // proposal still waiting is skipped: its chat is read again in the next pass.
+ let running
+ async function run() {
+ if (running) return running
+ running = (async () => {
+ const store = data()
+ store.lastRun = Date.now()
+ let made = 0
+ for (const [botId, sessions] of Object.entries(store.pending)) {
+ const bot = rt.botOf(botId)
+ if (bot === undefined) {
+ delete store.pending[botId]
+ continue
+ }
+ if (waitingFor(botId).length > 0) continue
+ const read = []
+ for (const sessionId of sessions) read.push({ sessionId, ...await linesSince(sessionId) })
+ const lines = read.flatMap(entry => entry.lines)
+ try {
+ const proposal = lines.length === 0 ? undefined : await reviewSkill(bot, lines)
+ if (proposal !== undefined) {
+ store.proposals[proposal.id] = proposal
+ made += 1
+ }
+ } catch (error) {
+ rt.warn('ds-bot: the skill review of %s failed: %s', bot.name, error?.message ?? error)
+ continue
+ }
+ for (const entry of read) if (entry.through !== undefined) store.cursors[entry.sessionId] = entry.through
+ delete store.pending[botId]
+ }
+ await persist()
+ return made
+ })().finally(() => { running = undefined })
+ return running
+ }
+
+ const due = () => {
+ const store = data()
+ if (Object.keys(store.pending).length === 0 || rt.readOnly() !== undefined) return false
+ if (Object.keys(rt.workView()).length > 0 || Date.now() - lastActivity < settings.idleMs) return false
+ return store.lastRun === undefined || Date.now() - store.lastRun >= settings.intervalMs
+ }
+ ctx.effect(() => {
+ const timer = setInterval(() => {
+ if (due()) void run().catch(error => rt.warn('ds-bot: learning pass failed: %s', error?.message ?? error))
+ }, settings.tickMs)
+ timer.unref?.()
+ return () => clearInterval(timer)
+ }, 'ds-bot: learning')
+
+ // The user's answer to a card.
+ async function answer({ id, approve }) {
+ const store = data()
+ const proposal = Object.hasOwn(store.proposals, id) ? store.proposals[id] : undefined
+ if (proposal === undefined) throw Object.assign(new Error('This proposal is no longer open'), { code: 'bot/stale' })
+ delete store.proposals[id]
+ if (approve !== true) {
+ store.declined = [...store.declined, { name: proposal.name, description: proposal.description, at: Date.now() }].slice(-settings.declinedMax)
+ await persist()
+ return { applied: false }
+ }
+ try {
+ await applySkill(proposal)
+ } finally {
+ await persist()
+ }
+ return { applied: true }
+ }
+
+ async function applySkill(proposal) {
+ const bot = rt.botOf(proposal.botId)
+ if (bot === undefined) throw new Error('The Bot that learned this skill is gone')
+ const current = skillsOf()[proposal.name]
+ if (proposal.action === 'create' && current !== undefined) throw new Error(`A skill named ${proposal.name} exists now`)
+ if (proposal.action === 'update' && (current === undefined || hashOf(current.content) !== proposal.base)) throw new Error(`${proposal.name} changed since this was proposed`)
+ const item = await rt.capabilitySave({ name: proposal.name, kind: 'skill', description: proposal.description, content: proposal.content }, { botId: bot.id, ui: false })
+ item.learned = true
+ await rt.save()
+ if (!(bot.capabilities ?? []).includes(item.name)) await rt.capabilityEndpoint('bot-capability', { botId: bot.id, name: item.name, attached: true })
+ }
+
+ async function learningEndpoint(endpoint, body) {
+ if (endpoint === 'learning-answer') return answer({ id: String(body.id ?? ''), approve: body.approve === true })
+ throw new Error(`Unknown endpoint ${endpoint}`)
+ }
+
+ // Per Bot, its proposals waiting for the user.
+ const learningView = () => {
+ const byBot = {}
+ for (const entry of Object.values(data().proposals)) (byBot[entry.botId] ??= []).push(entry)
+ return byBot
+ }
+
+ Object.assign(rt, { learningView, learningEndpoint, learningRun: run })
+}
diff --git a/ds-bot/src/host/memory.js b/ds-bot/src/host/memory.js
index de583e91..21a1ad82 100644
--- a/ds-bot/src/host/memory.js
+++ b/ds-bot/src/host/memory.js
@@ -1743,6 +1743,6 @@ export function install(rt) {
Object.assign(rt, {
memoryEnabled: true, renewMemory, archiveBotMemory, archiveGroupMemory, memoryEndpoint, memoryReviews,
- memoryNews: () => Object.fromEntries(news),
+ memoryNews: () => Object.fromEntries(news), botShot,
})
}
diff --git a/ds-bot/test/install.test.js b/ds-bot/test/install.test.js
index 2330ae1f..23e62c07 100644
--- a/ds-bot/test/install.test.js
+++ b/ds-bot/test/install.test.js
@@ -103,6 +103,8 @@ test('apply registers listeners, tools, and effects in a fixed order', (t) => {
'effect ds-bot: connectors',
'on agent/created',
'effect ds-bot: capabilities',
+ 'on agent/status',
+ 'effect ds-bot: learning',
'effect ds-bot: usage ledger',
'on llm/stream',
'effect ds-bot: update check',
diff --git a/ds-bot/test/learning.test.js b/ds-bot/test/learning.test.js
new file mode 100644
index 00000000..9f01e976
--- /dev/null
+++ b/ds-bot/test/learning.test.js
@@ -0,0 +1,277 @@
+import assert from 'node:assert/strict'
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'node:test'
+import { apply } from '../index.js'
+import { LEARNING, parseJson, skillProblem } from '../src/host/learning.js'
+import { fakeContext } from './fake-ctx.js'
+
+const TEAM = {
+ version: 1,
+ mainBotIds: ['m1'],
+ revision: 1,
+ whaleStored: true,
+ bots: {
+ m1: { id: 'm1', sessionId: 'm1', name: 'Chief', role: 'Chief of Staff', model: 'deepseek/v4', instructions: '', capabilities: [], createdAt: 1 },
+ b2: { id: 'b2', sessionId: 'b2', name: 'Writer', role: 'Writing', model: 'deepseek/v4', instructions: '', capabilities: [], createdAt: 2 },
+ },
+ rooms: {},
+ capabilities: { items: {} },
+ exchanges: [],
+ exchangeTurns: {},
+ questions: {},
+}
+
+const ownChat = [
+ { event: { seq: 10, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Please review this work' }] } } },
+ { event: { seq: 11, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: 'I finished the work.' }] } } } },
+]
+
+const createReply = JSON.stringify({
+ action: 'create',
+ name: 'release-checks',
+ description: 'Use when checking a release.',
+ content: 'Run the release checks.',
+ why: 'The repeated work showed that the checks need to be kept.',
+})
+const noneReply = JSON.stringify({ action: 'none', reason: 'This work taught nothing reusable.' })
+
+function start({ replies = [], config = {}, history = {}, seed = TEAM } = {}) {
+ const home = mkdtempSync(join(tmpdir(), 'ds-bot-learning-'))
+ writeFileSync(join(home, 'state.json'), JSON.stringify(JSON.parse(JSON.stringify(seed))))
+ const ctx = fakeContext()
+ const requests = []
+ Object.assign(ctx, {
+ llm: {
+ listProviders: () => [{ id: 'deepseek', name: 'DeepSeek' }],
+ listModels: async () => [{ id: 'v4' }],
+ resolveModelInfo: async () => ({ reasoning: { efforts: [{ id: 'off' }, { id: 'high' }] } }),
+ async* stream(request) {
+ requests.push(request)
+ const reply = await replies.shift()
+ if (reply instanceof Error) throw reply
+ const out = typeof reply === 'object' && reply !== null ? reply : { text: reply ?? '' }
+ yield { type: 'text-delta', index: 0, text: out.text ?? '' }
+ yield { type: 'finish', reason: { kind: out.finish ?? 'stop', ...(out.failure ? { failure: { message: out.failure } } : {}) } }
+ },
+ },
+ sessionController: {
+ resolveAgent: async sessionId => ({
+ agent: {
+ id: sessionId,
+ session: {
+ id: sessionId,
+ seq: (history[sessionId] ?? ownChat).at(-1).event.seq + 1,
+ requestHeader: () => ({ config: { provider: 'deepseek', model: 'v4' } }),
+ },
+ },
+ }),
+ selectModel: async () => {},
+ page: async ({ address, beforeSeq, maxMessages }) => {
+ const all = (history[address.sessionId] ?? ownChat).filter(record => beforeSeq === undefined || record.event.seq < beforeSeq)
+ const records = maxMessages === undefined ? all : all.slice(-maxMessages)
+ return { records, hasMore: records.length < all.length }
+ },
+ },
+ sessions: { messageProjections: [] },
+ workspaceRegistry: { archiveSession: async () => {} },
+ })
+ let route
+ const register = ctx.connection.fetch.register
+ ctx.connection.fetch.register = options => {
+ route = options
+ return register(options)
+ }
+ const settings = {
+ memoryReviewDelayMs: 60_000,
+ ...config,
+ learningTimes: { minTurnMs: 0, idleMs: 0, intervalMs: 0, tickMs: 10, ...config.learningTimes },
+ }
+ apply(ctx, { home, ...settings })
+ const call = async (endpoint, payload = {}) => {
+ const response = await route.fetch(new Request('http://localhost/api/bot', {
+ method: 'POST',
+ body: JSON.stringify({ endpoint, payload }),
+ }))
+ return response.json()
+ }
+ const learningPath = join(home, 'learning', 'state.json')
+ const learningState = () => existsSync(learningPath) ? JSON.parse(readFileSync(learningPath, 'utf8')) : {}
+ const teamState = () => JSON.parse(readFileSync(join(home, 'state.json'), 'utf8'))
+ const stop = async () => {
+ await ctx.dispose()
+ rmSync(home, { recursive: true, force: true, maxRetries: 5 })
+ }
+ return { ctx, home, call, requests, history, learningPath, learningState, teamState, stop }
+}
+
+const settle = () => new Promise(resolve => setTimeout(resolve, 20))
+
+async function until(check, message = 'condition did not hold within 4 s') {
+ for (let tries = 0; tries < 200; tries += 1) {
+ if (await check()) return
+ await settle()
+ }
+ assert.fail(message)
+}
+
+async function startRuntime(options) {
+ const runtime = start(options)
+ await runtime.call('state')
+ return runtime
+}
+
+async function finishTurn(ctx, { id = 'b2', duration = 0 } = {}) {
+ const { agent } = await ctx.sessionController.resolveAgent(id)
+ ctx.emit('agent/status', { agent, status: 'running' })
+ if (duration > 0) await new Promise(resolve => setTimeout(resolve, duration))
+ ctx.emit('agent/status', { agent, status: 'idle' })
+}
+
+function addNextTurn(history) {
+ history.b2.push(
+ { event: { seq: 12, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Please check the release process too.' }] } } },
+ { event: { seq: 13, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: 'The release process is checked.' }] } } } },
+ )
+}
+
+test('parseJson accepts fenced and unfenced objects and ignores garbage', () => {
+ assert.deepEqual(parseJson('```json\n{"action":"none"}\n```'), { action: 'none' })
+ assert.deepEqual(parseJson('Answer: {"action":"none"}'), { action: 'none' })
+ const answer = JSON.stringify({ action: 'create', content: 'Use this:\n```js\nvalue\n```' })
+ const expected = JSON.parse(answer)
+ assert.deepEqual(parseJson(answer), expected)
+ assert.deepEqual(parseJson('```json\n' + answer + '\n```'), expected)
+ assert.equal(parseJson('No JSON was returned.'), undefined)
+ assert.equal(parseJson('```json\n{"action":\n```'), undefined)
+})
+
+test('skillProblem validates actions, learned-skill limits, and secret-looking content', () => {
+ const proposal = {
+ action: 'create',
+ name: 'release-checks',
+ description: 'Use when checking a release.',
+ content: 'Run the release checks.',
+ }
+ assert.equal(skillProblem({ action: 'none' }, { skills: {}, learned: [] }), undefined)
+ assert.match(skillProblem({ ...proposal, name: 'Bad_Name' }, { skills: {}, learned: [] }), /capability name/)
+ assert.match(skillProblem(proposal, { skills: { 'release-checks': { kind: 'skill' } }, learned: [] }), /already exists/)
+ assert.match(skillProblem(proposal, { skills: {}, learned: ['one', 'two'], limit: 2 }), /learned skills/)
+ assert.match(skillProblem({ ...proposal, action: 'update' }, { skills: { 'release-checks': { kind: 'skill', content: 'Old', description: 'Old' } }, learned: [] }), /not a learned skill/)
+ assert.match(skillProblem({ ...proposal, action: 'update' }, {
+ skills: { 'release-checks': { kind: 'skill', content: proposal.content, description: proposal.description } },
+ learned: ['release-checks'],
+ }), /changes nothing/)
+ assert.match(skillProblem({ ...proposal, content: 'The root password is hunter2' }, { skills: {}, learned: [] }), /secret/)
+ assert.equal(skillProblem({ ...proposal, content: 'Compare `password == input` with a timing-safe check' }, { skills: {}, learned: [] }), undefined)
+ assert.match(skillProblem({ ...proposal, description: 'x'.repeat(LEARNING.descriptionMax + 1) }, { skills: {}, learned: [] }), /description/)
+})
+
+test('short turns do not mark a Bot chat; a longer turn marks it for review', async t => {
+ const short = await startRuntime({ config: { learningTimes: { minTurnMs: 60_000, idleMs: 60_000 } } })
+ t.after(() => short.stop())
+ await finishTurn(short.ctx)
+ await settle()
+ assert.equal(existsSync(short.learningPath), false)
+
+ const long = await startRuntime({ config: { learningTimes: { minTurnMs: 5, idleMs: 60_000 } } })
+ t.after(() => long.stop())
+ await finishTurn(long.ctx, { duration: 15 })
+ await until(() => long.learningState().pending?.b2?.includes('b2'))
+ assert.deepEqual(long.learningState().pending.b2, ['b2'])
+})
+
+test('a none reply makes no proposal and advances the chat cursor', async t => {
+ const runtime = await startRuntime({ replies: [noneReply] })
+ t.after(() => runtime.stop())
+ await finishTurn(runtime.ctx)
+ await until(() => runtime.learningState().cursors?.b2 === 11 && runtime.learningState().pending?.b2 === undefined)
+ assert.equal(runtime.requests.length, 1)
+ assert.deepEqual((await runtime.call('state')).value.learning ?? {}, {})
+})
+
+test('a waiting proposal is skipped and approval saves and attaches a learned skill', async t => {
+ const history = { b2: [...ownChat] }
+ const runtime = await startRuntime({ history, replies: [createReply, noneReply] })
+ t.after(() => runtime.stop())
+ await finishTurn(runtime.ctx)
+ await until(() => Object.keys(runtime.learningState().proposals ?? {}).length === 1)
+ const proposal = (await runtime.call('state')).value.learning.b2[0]
+ assert.equal(proposal.name, 'release-checks')
+
+ addNextTurn(history)
+ await finishTurn(runtime.ctx)
+ await until(() => runtime.learningState().pending?.b2?.includes('b2'))
+ await settle()
+ assert.equal(runtime.requests.length, 1)
+
+ const answer = await runtime.call('learning-answer', { id: proposal.id, approve: true })
+ assert.equal(answer.value.applied, true)
+ await until(() => runtime.learningState().cursors?.b2 === 13 && runtime.learningState().pending?.b2 === undefined)
+ const saved = runtime.teamState()
+ assert.equal(saved.capabilities.items['release-checks'].learned, true)
+ assert.ok(saved.bots.b2.capabilities.includes('release-checks'))
+ assert.equal(runtime.requests.length, 2)
+})
+
+test('declined proposals are recorded and included in the next skill request', async t => {
+ const history = { b2: [...ownChat] }
+ const runtime = await startRuntime({ history, replies: [createReply, noneReply] })
+ t.after(() => runtime.stop())
+ await finishTurn(runtime.ctx)
+ await until(() => Object.keys(runtime.learningState().proposals ?? {}).length === 1)
+ const proposal = (await runtime.call('state')).value.learning.b2[0]
+
+ addNextTurn(history)
+ await finishTurn(runtime.ctx)
+ await until(() => runtime.learningState().pending?.b2?.includes('b2'))
+ await settle()
+ assert.equal(runtime.requests.length, 1)
+
+ const answer = await runtime.call('learning-answer', { id: proposal.id, approve: false })
+ assert.equal(answer.value.applied, false)
+ await until(() => runtime.learningState().cursors?.b2 === 13 && runtime.learningState().pending?.b2 === undefined)
+ assert.equal(runtime.teamState().capabilities.items['release-checks'], undefined)
+ assert.equal(runtime.learningState().declined[0].name, 'release-checks')
+ assert.match(runtime.requests[1].messages[0].content[0].text, /Proposals the user declined[\s\S]*- release-checks: Use when checking a release\./)
+})
+
+test('an update approval fails if the learned skill changed after the proposal', async t => {
+ const seed = JSON.parse(JSON.stringify(TEAM))
+ seed.bots.b2.capabilities = ['release-checks']
+ seed.capabilities.items['release-checks'] = {
+ name: 'release-checks',
+ kind: 'skill',
+ description: 'Use when checking a release.',
+ content: 'Run the original release checks.',
+ createdBy: 'b2',
+ createdAt: 3,
+ updatedAt: 3,
+ learned: true,
+ }
+ const updateReply = JSON.stringify({
+ action: 'update',
+ name: 'release-checks',
+ description: 'Use when checking a release.',
+ content: 'Run the new release checks.',
+ why: 'The work showed an additional step.',
+ })
+ const runtime = await startRuntime({ seed, replies: [updateReply] })
+ t.after(() => runtime.stop())
+ await finishTurn(runtime.ctx)
+ await until(() => Object.keys(runtime.learningState().proposals ?? {}).length === 1)
+ const proposal = (await runtime.call('state')).value.learning.b2[0]
+
+ await runtime.call('capability-save', {
+ name: 'release-checks',
+ kind: 'skill',
+ description: 'Use when checking a release.',
+ content: 'Changed independently after the proposal.',
+ })
+ const answer = await runtime.call('learning-answer', { id: proposal.id, approve: true })
+ assert.equal(answer.ok, false)
+ assert.match(JSON.stringify(answer), /release-checks changed since this was proposed/)
+ assert.equal(runtime.teamState().capabilities.items['release-checks'].content, 'Changed independently after the proposal.')
+ assert.equal(Object.keys(runtime.learningState().proposals).length, 0)
+})
diff --git a/ds-bot/test/memory.test.js b/ds-bot/test/memory.test.js
index 707b624b..ae8a798a 100644
--- a/ds-bot/test/memory.test.js
+++ b/ds-bot/test/memory.test.js
@@ -1147,7 +1147,7 @@ test('a turn whose Session moves to a fresh part before its review is still revi
test('config.memoryReview false leaves memory to the Bots\' own remember calls', async (t) => {
const { ctx, call, section, requests, stop } = start({ config: { memoryReview: false, memoryReviewDelayMs: 0 } })
t.after(() => stop())
- assert.equal(ctx.calls.filter(entry => entry === 'on agent/status').length, 3)
+ assert.equal(ctx.calls.filter(entry => entry === 'on agent/status').length, 4)
await call('state', {})
assert.match(section('b2'), /Call it in that same turn, before you reply/)
ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' })