Stop paying the multimodal tax. Your AI agent keeps every image it has seen but only sends the ones it needs: lighter chats, no endless reconnecting, no starting over. Works with Codex, Claude Code, Cline, Gemini CLI, OpenCode, WorkBuddy and more. Install once, forget it.
TypeScript ai-agentsai-imageclaude-codeclaude-code-pluginclinecodexcodex-plugincomputer-usecontext-engineeringcontext-managementdeveloper-toolsgemini-cliimage-generationmcpmultimodalmultimodal-aiopencodeqwen-codetoken-optimizationvision-language-model
test(v0.7): 子代理和旁聊的自测
chipfighteree795ff7bbb7
diff --git a/spike/src/testkit.ts b/spike/src/testkit.tsindex ce342e0..d8db1ec 100644--- a/spike/src/testkit.ts+++ b/spike/src/testkit.ts@@ -49,6 +49,34 @@ export const testImages = {},};+// v0.3 self-tests (moved here for v0.7's): a background, a shape in the middle, and a digit in one corner that a+// one-line description leaves out.+const FONT: Record<string, string[]> = {+ "3": ["11111", "00010", "00100", "00010", "00001", "10001", "01110"],+ "5": ["11111", "10000", "11110", "00001", "00001", "10001", "01110"],+ "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"],+};+const digit = (ch: string, left: number, top: number, scale = 8) => (x: number, y: number) => {+ const col = Math.floor((x - left) / scale);+ const row = Math.floor((y - top) / scale);+ return x >= left && y >= top && col < 5 && row < 7 && FONT[ch][row][col] === "1";+};+export const CORNER_IMAGES: Record<string, () => Buffer> = {+ // White, a red square in the middle, a black 7 in the bottom-right corner.+ "a.png": () => { const seven = digit("7", 440, 420); return png(512, 512, (x, y) => (seven(x, y) ? [0, 0, 0] : x >= 136 && x < 376 && y >= 136 && y < 376 ? [220, 30, 30] : [255, 255, 255])); },+ // Yellow, a blue disc in the middle, a green 3 in the top-left corner.+ "b.png": () => { const three = digit("3", 30, 30); return png(512, 512, (x, y) => (three(x, y) ? [20, 120, 40] : (x - 256) ** 2 + (y - 256) ** 2 < 130 ** 2 ? [40, 70, 210] : [250, 220, 60])); },+ // Purple, a white triangle pointing up, an orange 5 in the top-right corner.+ "c.png": () => { const five = digit("5", 442, 30); return png(512, 512, (x, y) => (five(x, y) ? [250, 140, 20] : y > 120 && y < 400 && Math.abs(x - 256) < (y - 120) * 0.6 ? [255, 255, 255] : [120, 50, 160])); },+};++// A number in the reply, as a numeral or a word, ignoring image ids.+const NUMBER_WORDS: Record<number, RegExp> = { 3: /三|three/i, 5: /五|five/i, 7: /七|seven/i, 10: /十|ten\b/i, 12: /十二|twelve/i };+export const says = (reply: string, n: number) => {+ const clean = reply.replace(/IMG-\d{3}/g, "");+ return new RegExp(`(^|[^0-9])${n}([^0-9]|$)`).test(clean) || NUMBER_WORDS[n].test(clean);+};+// Tests use gpt-6-sol at low effort to save the user's tokens (AGENTS.md). v0.1-24: CAM_TEST_MODEL and CAM_TEST_EFFORT// run the same test on another model, to check that the text for the model works for every GPT-6 model.export const TEST_MODEL = process.env.CAM_TEST_MODEL?.trim() || "gpt-6-sol";diff --git a/spike/src/v03-auto-selftest.ts b/spike/src/v03-auto-selftest.tsindex 9a23272..a56b908 100644--- a/spike/src/v03-auto-selftest.ts+++ b/spike/src/v03-auto-selftest.ts@@ -26,32 +26,12 @@ import { requestStatsDirOf, selectionDirOf } from "../../plugin/src/paths.ts";import { readRequestStats } from "../../plugin/src/request-stats.ts";import { writeSelection } from "../../plugin/src/selection.ts";import { loadThreadIndex } from "../../plugin/src/thread-index.ts";-import { archiveThreads, png, runTurn, TEST_EFFORT, TEST_MODEL, text } from "./testkit.ts";+import { archiveThreads, CORNER_IMAGES, runTurn, says, TEST_EFFORT, TEST_MODEL, text } from "./testkit.ts";type Json = Record<string, any>;const here = dirname(fileURLToPath(import.meta.url));const localRoot = join(resolve(here, "../.."), "local");-// ---- test images: a background, a shape in the middle, and a digit in one corner-const FONT: Record<string, string[]> = {- "3": ["11111", "00010", "00100", "00010", "00001", "10001", "01110"],- "5": ["11111", "10000", "11110", "00001", "00001", "10001", "01110"],- "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"],-};-const digit = (ch: string, left: number, top: number, scale = 8) => (x: number, y: number) => {- const col = Math.floor((x - left) / scale);- const row = Math.floor((y - top) / scale);- return x >= left && y >= top && col < 5 && row < 7 && FONT[ch][row][col] === "1";-};-const IMAGES: Record<string, () => Buffer> = {- // White, a red square in the middle, a black 7 in the bottom-right corner.- "a.png": () => { const seven = digit("7", 440, 420); return png(512, 512, (x, y) => (seven(x, y) ? [0, 0, 0] : x >= 136 && x < 376 && y >= 136 && y < 376 ? [220, 30, 30] : [255, 255, 255])); },- // Yellow, a blue disc in the middle, a green 3 in the top-left corner.- "b.png": () => { const three = digit("3", 30, 30); return png(512, 512, (x, y) => (three(x, y) ? [20, 120, 40] : (x - 256) ** 2 + (y - 256) ** 2 < 130 ** 2 ? [40, 70, 210] : [250, 220, 60])); },- // Purple, a white triangle pointing up, an orange 5 in the top-right corner.- "c.png": () => { const five = digit("5", 442, 30); return png(512, 512, (x, y) => (five(x, y) ? [250, 140, 20] : y > 120 && y < 400 && Math.abs(x - 256) < (y - 120) * 0.6 ? [255, 255, 255] : [120, 50, 160])); },-};-const SCRIPT = {zh: {name: (label: string, thread: string) => `[CAM测试] 自动选图自测 ${thread} ${label}`,@@ -89,13 +69,6 @@ const SCRIPT = {},};-// A number in the reply, as a numeral or a word, ignoring image ids.-const NUMBER_WORDS: Record<number, RegExp> = { 3: /三|three/i, 5: /五|five/i, 7: /七|seven/i, 10: /十|ten\b/i, 12: /十二|twelve/i };-const says = (reply: string, n: number) => {- const clean = reply.replace(/IMG-\d{3}/g, "");- return new RegExp(`(^|[^0-9])${n}([^0-9]|$)`).test(clean) || NUMBER_WORDS[n].test(clean);-};-async function main(): Promise<void> {const option = (name: string, fallback: string) => (process.argv.includes(name) ? process.argv[process.argv.indexOf(name) + 1] : fallback);const label = option("--label", "run1");@@ -108,7 +81,7 @@ async function main(): Promise<void> {// A folder per run, so runs side by side never rewrite an image another one is uploading.const work = join(root, "work", label);mkdirSync(work, { recursive: true });- for (const [name, draw] of Object.entries(IMAGES)) writeFileSync(join(work, name), draw());+ for (const [name, draw] of Object.entries(CORNER_IMAGES)) writeFileSync(join(work, name), draw());// What the panel would have told the plugin: Codex shows this language.rememberLang(lang, "codex", languageFileOf(data));const sessionsDir = join(codexHome(), "sessions");diff --git a/spike/src/v07-children-selftest.ts b/spike/src/v07-children-selftest.tsnew file mode 100644index 0000000..337ffbe--- /dev/null+++ b/spike/src/v07-children-selftest.ts@@ -0,0 +1,197 @@+// Purpose: v0.7 — subagents and side chats end to end (docs/v0.7/plan.md §5). A side chat's requests leave out the+// images of the task it came from, and the model there fetches one from that task when the answer needs it; a subagent+// started without history has the images of its earlier turns left out and fetches one back; with the task's switch for+// them off, their requests go out as they are; the task's panel lists them.+// Set up like v03-auto-selftest.ts: a test engine of this checkout (its own port and data folder), this checkout's+// plugin server attached to the test app-server for cam_view_image, Codex's multi_agent_v2 on (spawn_agent's+// fork_turns), the tools that act on the machine off, a read-only sandbox, and the tasks archived at the end (their+// subagents go with them; a side chat is kept in memory only). The side chat is opened as Codex opens one (an ephemeral+// fork), with a short instruction like the one Codex gives it.+// Thread S: a side chat asks a text question, then about a detail of an image. Thread C: a subagent views an image in+// its first turn and is asked about a detail of it in its second, which the main task gives it with followup_task+// (Codex takes no input for a multi_agent_v2 subagent from the app-server). Thread O: the same with the switch off.+// Input: [--label name] [--lang zh|en] [--port 17898] [--threads S,C,O]; CAM_TEST_MODEL, CAM_TEST_EFFORT.+// Output: local/v07/children-<label>.json (synthetic images only).++import { spawn } from "node:child_process";+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { setTimeout as sleep } from "node:timers/promises";+import { fileURLToPath } from "node:url";+import { AppServer, findBundledCodex } from "./appserver.ts";+import { codexHome } from "../../plugin/src/codexconfig.ts";+import { engineHealth } from "../../plugin/src/engine.ts";+import { languageFileOf, rememberLang, type Lang } from "../../plugin/src/language.ts";+import { loadPanelState } from "../../plugin/src/panel-state.ts";+import { requestStatsDirOf, selectionDirOf } from "../../plugin/src/paths.ts";+import { readRequestStats } from "../../plugin/src/request-stats.ts";+import { writeSelection } from "../../plugin/src/selection.ts";+import { loadThreadIndex, threadsStartedSince } from "../../plugin/src/thread-index.ts";+import { archiveThreads, CORNER_IMAGES, runTurn, says, TEST_EFFORT, TEST_MODEL, text } from "./testkit.ts";++type Json = Record<string, any>;+const here = dirname(fileURLToPath(import.meta.url));+const localRoot = join(resolve(here, "../.."), "local");++const SCRIPT = {+ zh: {+ name: (label: string, thread: string) => `[CAM测试] 子代理和旁聊自测 ${thread} ${label}`,+ uploadTwo: "这是两张测试图。分别用一句话说说它们的背景色和中间的主要形状,只说这两样,不要提角落里的东西。",+ sideNote: "这是从主任务分出来的旁聊,用来问一个小问题。在这之前的内容都是从主任务继承来的,只作参考,不要接着做里面的事。只回答在这之后提的问题。",+ sideText: "用一句话总结我们刚才在做什么。",+ detail: "a.png 右下角的数字是几?",+ spawn: "用 spawn_agent 开一个子代理,fork_turns 设为 \"none\",交给它这个任务:“用 view_image 工具查看当前目录下的 a.png,然后用一句话说说它的背景色和中间的主要形状,只说这两样,不要提角落里的东西。”等它做完,把它的回答原样告诉我。",+ followup: "用 followup_task 给你刚才开的那个子代理发一个后续任务:“a.png 右下角的数字是几?”等它答完,把它的回答原样告诉我。",+ },+ en: {+ name: (label: string, thread: string) => `[CAM test] subagents and side chats ${thread} ${label}`,+ uploadTwo: "Here are two test images. In one sentence each, give their background color and the main shape in the middle. Only those two things: don't mention anything in the corners.",+ sideNote: "This is a side chat forked from the main task for a quick question. Everything before this point is inherited from the main task and is for reference only; do not continue anything in it. Answer only what is asked after this point.",+ sideText: "In one sentence, sum up what we were just doing.",+ detail: "What is the number in the bottom-right corner of a.png?",+ spawn: "Use spawn_agent to start one subagent with fork_turns set to \"none\" and this task: \"Use the view_image tool on a.png in the current folder, then give its background color and the main shape in the middle in one sentence. Only those two things: don't mention anything in the corners.\" Wait until it finishes, then tell me its answer as it is.",+ followup: "Use followup_task to give the subagent you just started a follow-up task: \"What is the number in the bottom-right corner of a.png?\" Wait until it answers, then tell me its answer as it is.",+ },+};++async function main(): Promise<void> {+ const option = (name: string, fallback: string) => (process.argv.includes(name) ? process.argv[process.argv.indexOf(name) + 1] : fallback);+ const label = option("--label", "run1");+ const lang: Lang = option("--lang", "zh") === "en" ? "en" : "zh";+ const port = Number(option("--port", "17898"));+ const only = option("--threads", "S,C,O").split(",");+ const words = SCRIPT[lang];+ const root = join(localRoot, "v07");+ const data = join(root, `data-${label}`);+ const work = join(root, "work", label);+ mkdirSync(work, { recursive: true });+ for (const name of ["a.png", "c.png"]) writeFileSync(join(work, name), CORNER_IMAGES[name]());+ rememberLang(lang, "codex", languageFileOf(data));+ const sessionsDir = join(codexHome(), "sessions");+ const startedMs = Date.now() - 5_000;++ const engine = spawn(process.execPath, [join(here, "..", "..", "plugin", "src", "proxy.ts"), "--port", String(port), "--stay"], { env: { ...process.env, CAM_DATA_DIR: data }, stdio: "ignore", windowsHide: true });+ for (let i = 0; i < 50 && !(await engineHealth(port)); i++) await sleep(200);+ const off = ["computer-use@openai-bundled", "unified-computer-use@openai-bundled", "chrome@openai-bundled", "browser@openai-bundled", "codex-app-tools@openai-bundled"]+ .flatMap((plugin) => ["-c", `plugins.${plugin}.enabled=false`]);+ const env = { CODEX_HOME: codexHome(), CAM_DATA_DIR: data, CAM_NO_ENGINE: "1" };+ const plugin = [+ "-c", `mcp_servers.cam_dev.command=${JSON.stringify(process.execPath)}`,+ "-c", `mcp_servers.cam_dev.args=${JSON.stringify([join(here, "..", "..", "plugin", "src", "plugin-server.ts")])}`,+ "-c", `mcp_servers.cam_dev.env={ ${Object.entries(env).map(([name, value]) => `${name} = ${JSON.stringify(value)}`).join(", ")} }`,+ "-c", `mcp_servers.cam_dev.default_tools_approval_mode="approve"`,+ "-c", "mcp_servers.cam_dev.startup_timeout_sec=20",+ ];+ const server = new AppServer(findBundledCodex(), ["-c", `openai_base_url="http://localhost:${port}/backend-api/codex"`, "-c", "features.respect_system_proxy=true", "-c", "features.multi_agent_v2=true", "-c", "notify=[]", "-c", "mcp_servers.node_repl.enabled=false", ...off, ...plugin], { NO_PROXY: undefined, no_proxy: undefined }, join(root, `appserver-${label}.log`));+ const result: Json = { createdAt: new Date().toISOString(), label, lang, model: TEST_MODEL, effort: TEST_EFFORT, threads: {} };+ const created: string[] = [];+ // What the engine did to a thread's last full request, and how it went out.+ const lastOf = (threadId: string) => {+ const stats = readRequestStats(threadId, requestStatsDirOf(data));+ const rewrite = stats?.lastHttp?.rewrite ?? null;+ return { transport: stats?.latest?.transport ?? null, images: stats?.lastHttp?.images ?? null, rewrite: rewrite && { auto: rewrite.auto ?? false, sideChatOf: rewrite.sideChatOf ?? null, off: rewrite.off ?? false, replaced: (rewrite.replaced ?? []).map((r: Json) => r.id), skipped: rewrite.skipped ?? null } };+ };+ // The fetches the plugin server answered (its log), for a thread or the task a side chat came from.+ const fetches = () => {+ const file = join(data, "plugin-server.jsonl");+ if (!existsSync(file)) return [];+ return readFileSync(file, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line)).filter((entry) => entry.event === "fetch");+ };+ const subagentsOf = (threadId: string) => threadsStartedSince(sessionsDir, startedMs).filter((entry) => entry.meta?.source?.subagent?.thread_spawn?.parent_thread_id === threadId).map((entry) => entry.threadId);+ try {+ await server.initialize();+ const start = async (name: string) => {+ const { thread } = await server.request<{ thread: Json }>("thread/start", { model: TEST_MODEL, cwd: work, approvalPolicy: "never", sandbox: "read-only", ephemeral: false });+ created.push(thread.id);+ await server.request("thread/name/set", { threadId: thread.id, name: words.name(label, name) });+ return thread.id as string;+ };+ // The engine records a request once its response has ended, which can be just after the turn completes.+ const turn = async (threadId: string, prompt: string, images: string[] = []) => {+ const done = await runTurn(server, threadId, [text(prompt), ...images.map((file) => ({ type: "localImage", path: join(work, file) }))]);+ await sleep(1500);+ return { ...done, last: lastOf(threadId) };+ };+ const sideChat = async (threadId: string) => (await server.request<{ thread: Json }>("thread/fork", { threadId, ephemeral: true, excludeTurns: true, developerInstructions: words.sideNote })).thread.id as string;+ const threads: Record<string, () => Promise<Json>> = {+ S: async () => {+ const id = await start("S");+ const turns: Json = { upload: await turn(id, words.uploadTwo, ["a.png", "c.png"]) };+ const side = await sideChat(id);+ turns.sideText = await turn(side, words.sideText);+ turns.sideDetail = await turn(side, words.detail);+ return { threadId: id, side, turns, fetches: fetches().filter((entry) => entry.threadId === side) };+ },+ C: async () => {+ const id = await start("C");+ const turns: Json = { spawn: await turn(id, words.spawn) };+ const [child] = subagentsOf(id);+ if (!child) return { threadId: id, turns, error: "no subagent started" };+ turns.childFirst = lastOf(child);+ // The subagent's second turn.+ turns.followup = await turn(id, words.followup);+ turns.childSecond = lastOf(child);+ return { threadId: id, child, turns, childImages: loadThreadIndex(sessionsDir, child).images.map((image) => image.id), fetches: fetches().filter((entry) => entry.threadId === child) };+ },+ O: async () => {+ const id = await start("O");+ const turns: Json = { upload: await turn(id, words.uploadTwo, ["a.png", "c.png"]) };+ // As the panel's switch does.+ writeSelection({ threadId: id, unchecked: {}, childrenAuto: false }, selectionDirOf(data));+ await sleep(2500);+ const side = await sideChat(id);+ turns.sideText = await turn(side, words.sideText);+ turns.spawn = await turn(id, words.spawn);+ const [child] = subagentsOf(id);+ if (child) turns.followup = await turn(id, words.followup);+ return { threadId: id, side, child: child ?? null, turns, childLast: child ? lastOf(child) : null };+ },+ };+ for (const name of only) if (threads[name]) result.threads[name] = await threads[name]();+ // What each task's panel lists (as the panel would read it).+ for (const thread of Object.values(result.threads) as Json[]) {+ thread.panel = loadPanelState(thread.threadId, { sessionsDir, dataRoot: data }).children.map((child) => ({ kind: child.kind, name: child.name, auto: child.auto, images: child.images, replaced: child.last?.replaced ?? null }));+ }+ } finally {+ result.notArchived = await archiveThreads(server, created);+ await server.stop();+ engine.kill();+ }++ const S = result.threads.S ?? null;+ const C = result.threads.C ?? null;+ const O = result.threads.O ?? null;+ const fetched = (thread: Json | null, id: string) => (thread?.fetches ?? []).some((entry: Json) => (entry.found ?? []).includes(id));+ result.checks = {+ ...(S ? {+ // The side chat's text question: both of the task's images left out, as placeholders.+ sideLeavesOut: S.turns.sideText.last.rewrite?.sideChatOf === S.threadId && ["IMG-001", "IMG-002"].every((id) => S.turns.sideText.last.rewrite?.replaced.includes(id)),+ sideTextNoFetch: !S.turns.sideText.tools.some((tool: string) => tool.startsWith("mcpToolCall")),+ sideFetches: fetched(S, "IMG-001") && (S.fetches ?? []).every((entry: Json) => entry.from === S.threadId),+ sideAnswers: says(S.turns.sideDetail.reply, 7),+ sideListed: S.panel.some((child: Json) => child.kind === "side" && child.auto),+ } : {}),+ ...(C ? {+ childAuto: C.turns.childSecond?.rewrite?.auto === true,+ childLeavesOut: (C.childImages ?? []).length > 0 && !!C.turns.childSecond?.rewrite?.replaced?.includes(C.childImages[0]),+ childFetches: fetched(C, C.childImages?.[0] ?? "IMG-001"),+ childAnswers: says(C.turns.followup?.reply ?? "", 7),+ childListed: C.panel.some((child: Json) => child.kind === "subagent" && child.auto),+ } : {}),+ ...(O ? {+ offSideAsIs: !O.turns.sideText.last.rewrite?.replaced?.length && !O.turns.sideText.last.rewrite?.auto,+ offChildAsIs: !!O.childLast && (O.childLast.images ?? 1) > 0 && !O.childLast.rewrite?.replaced?.length && !O.childLast.rewrite?.auto,+ offListed: O.panel.length > 0 && O.panel.every((child: Json) => !child.auto),+ } : {}),+ };+ mkdirSync(root, { recursive: true });+ writeFileSync(join(root, `children-${label}.json`), JSON.stringify(result, null, 2));+ const brief = (t: Json) => (t && "reply" in t ? { status: t.status, tools: t.tools, reply: String(t.reply).slice(0, 120), last: t.last } : t);+ console.log(JSON.stringify({+ label, lang, model: TEST_MODEL, notArchived: result.notArchived, checks: result.checks,+ turns: Object.fromEntries(Object.entries(result.threads as Record<string, Json>).map(([name, thread]) => [name, Object.fromEntries(Object.entries(thread.turns as Record<string, Json>).map(([key, t]) => [key, brief(t)]))])),+ panels: Object.fromEntries(Object.entries(result.threads as Record<string, Json>).map(([name, thread]) => [name, thread.panel])),+ }, null, 2));+}++await main();