A local plugin for image-heavy chats in Codex, Claude Code and other agents: earlier images stop going out with every message, and the model fetches the one it needs by its ID, also after compaction.
TypeScript ai-agentsclaude-codeclaude-code-plugincodexcodex-plugincontext-managementdeveloper-toolsimage-generationmcpmultimodal
test(spike): 没有名字的工具截图全部取消后,模型能否按编号要图(中英文)
chipfighter15f9a626d8ba
diff --git a/spike/src/appserver.ts b/spike/src/appserver.tsindex f26cc3b..e9de9b2 100644--- a/spike/src/appserver.ts+++ b/spike/src/appserver.ts@@ -124,10 +124,11 @@ export class AppServer {});}- async initialize(): Promise<Json> {+ // capabilities: e.g. { experimentalApi: true, requestAttestation: false } for thread/resume with a given history.+ async initialize(capabilities: Json | null = null): Promise<Json> {const result = await this.request("initialize", {clientInfo: { name: "codex-attachment-manager-phase0", title: "Codex Attachment Manager (phase 0)", version: "0.0.0" },- capabilities: null,+ capabilities,});this.notify("initialized");return result;diff --git a/spike/src/v01-needs-selftest.ts b/spike/src/v01-needs-selftest.tsnew file mode 100644index 0000000..149ae3c--- /dev/null+++ b/spike/src/v01-needs-selftest.ts@@ -0,0 +1,176 @@+// Purpose: v0.1-17 follow-up — when the images the model needs are unnamed screenshots it took with a tool (as with+// computer use) and the user unchecked them all, does it ask for them by id ("需要 IMG-xxx")? Replays the shape of the+// user's report of 2026-09-26: the model changed a setting in a desktop app, taking screenshots after each step (each+// call has a title, no screenshot has a name, and the model never described them); later all of them are unchecked.+// Thread 1 asks what the user asked then ("can you look at the images, how are the settings now?", "which images do+// you need?"), then a detail only an old screenshot shows; thread 2 asks only that detail, in a fresh context.+// The history is given to app-server's thread/resume (history: an unstable parameter), which starts a new thread+// holding it. Computer use, the browsers and the other plugins that act on the machine are switched off for this+// app-server, and it runs read-only with no approvals, so nothing on the machine is touched.+// Input: [--label name] [--port 17898] [--lang zh|en]. Output: local/v01/needs-<label>.json (synthetic threads only).++import { spawn } from "node:child_process";+import { randomUUID } from "node:crypto";+import { mkdirSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { setTimeout as sleep } from "node:timers/promises";+import { fileURLToPath } from "node:url";+import { AppServer, findBundledCodex } from "./appserver.ts";+import { codexHome } from "../../plugin/src/codexconfig.ts";+import { engineHealth } from "../../plugin/src/engine.ts";+import { languageFileOf, rememberLang } from "../../plugin/src/language.ts";+import { askedIn } from "../../plugin/src/panel-state.ts";+import { selectionDirOf } from "../../plugin/src/paths.ts";+import { readRequestStats } from "../../plugin/src/request-stats.ts";+import { writeSelection } from "../../plugin/src/selection.ts";+import { loadThreadIndex } from "../../plugin/src/thread-index.ts";+import { png, runTurn, TEST_MODEL, text } from "./testkit.ts";++type Json = Record<string, any>;+const here = dirname(fileURLToPath(import.meta.url));+const localRoot = join(resolve(here, "../.."), "local");++// A 640×400 app window: title bar, a list on the left with one entry highlighted (or none), and one shape on the right.+type Rgb = [number, number, number];+function screen(selected: number | null, shape: (x: number, y: number) => boolean, color: Rgb): Buffer {+ return png(640, 400, (x, y) => {+ if (y < 28) return [45, 45, 50];+ if (shape(x, y)) return color;+ if (selected !== null && x < 150) return y >= 60 + selected * 36 && y < 90 + selected * 36 ? [70, 110, 200] : [235, 235, 238];+ return [250, 250, 250];+ });+}+const SHOTS = [+ // Main window: a dark preview area with a green disc in it.+ () => png(640, 400, (x, y) => (y < 28 ? [45, 45, 50] : x > 40 && x < 600 && y > 50 && y < 300 ? ((x - 320) ** 2 + (y - 175) ** 2 < 55 ** 2 ? [40, 200, 90] : [20, 20, 24]) : [230, 230, 232])),+ // Settings, first page: a red square.+ () => screen(0, (x, y) => x > 300 && x < 380 && y > 150 && y < 230, [220, 40, 40]),+ // Settings, output page: a blue bar.+ () => screen(1, (x, y) => x > 200 && x < 560 && y > 180 && y < 204, [40, 90, 220]),+ // Settings, video page: an orange triangle in the top-right corner.+ () => screen(2, (x, y) => y > 50 && y < 120 && x > 520 && Math.abs(x - 560) < (y - 50) * 0.6, [245, 140, 20]),+];++const SCRIPT = {+ zh: {+ name: (label: string, n: number) => `[CAM测试] 截图编号自测 ${label}-${n}`,+ ask: "帮我把这个录屏软件的录像保存路径改到 D:\\录像,改完告诉我。",+ plan: "我来打开软件,在设置里修改录像路径并核对。",+ titles: ["查看主界面", "打开设置对话框", "切到输出页", "填入新的录像路径", "切到视频页"],+ pathText: (path: string) => `录像路径 Value: ${path}`,+ done: "已经把录像路径改成 D:\\录像,也看过了视频设置页。",+ lookNow: "你能看看图片,现在设置如何的?",+ whichImages: "你需要哪些图片?",+ detail: "之前你截的视频设置页里,右上角那个图形是什么颜色?",+ newShots: /截图|截个图|截一张|发.{0,6}图|重新截/,+ },+ en: {+ name: (label: string, n: number) => `[CAM test] screenshot id self-test ${label}-${n}`,+ ask: "Please change this screen recorder's recording folder to D:\\Recordings and tell me when it's done.",+ plan: "I'll open the app, change the recording path in its settings, and check it.",+ titles: ["Look at the main window", "Open the settings dialog", "Go to the Output page", "Enter the new recording path", "Go to the Video page"],+ pathText: (path: string) => `Recording Path Value: ${path}`,+ done: "The recording path is now D:\\Recordings, and I also looked at the video settings page.",+ lookNow: "Can you look at the images? How are the settings now?",+ whichImages: "Which images do you need?",+ detail: "In the video settings page you took a screenshot of earlier, what color is the shape in the top-right corner?",+ newShots: /screenshot|send (me )?(a |an |the )?(new |fresh )?(image|picture)/i,+ },+};++// The history as Codex records it: the model's calls have titles; each screenshot is a tool output with no name.+function history(words: typeof SCRIPT.zh): Json[] {+ const url = (bytes: Buffer) => `data:image/png;base64,${bytes.toString("base64")}`;+ const call = (n: number, title: string, output: Json[]): Json[] => [+ { type: "custom_tool_call", status: "completed", call_id: `call_shot_${n}`, name: "exec", input: `const r=await tools.mcp__node_repl__js({code:"state=await sky.get_window_state({window:targetWindow,include_screenshot:${output.some((part) => part.type === "input_image")}});",title:${JSON.stringify(title)}}); for(const c of r.content||[]){if(c.type==="image") image(c); else if(c.type==="text") text(c.text)}` },+ { type: "custom_tool_call_output", call_id: `call_shot_${n}`, output: [{ type: "input_text", text: "Script completed\nWall time 1.0 seconds\nOutput:\n" }, ...output] },+ ];+ const shot = (i: number) => ({ type: "input_image", image_url: url(SHOTS[i]()) });+ return [+ { type: "message", role: "user", content: [{ type: "input_text", text: words.ask }] },+ { type: "message", role: "assistant", content: [{ type: "output_text", text: words.plan }] },+ ...call(1, words.titles[0], [shot(0)]),+ ...call(2, words.titles[1], [shot(1)]),+ ...call(3, words.titles[2], [{ type: "input_text", text: words.pathText("C:\\Users\\me\\Videos") }, shot(2)]),+ ...call(4, words.titles[3], [{ type: "input_text", text: words.pathText(words.ask.includes("D:\\录像") ? "D:\\录像" : "D:\\Recordings") }]),+ ...call(5, words.titles[4], [shot(3)]),+ { type: "message", role: "assistant", content: [{ type: "output_text", text: words.done }] },+ ];+}++async function main(): Promise<void> {+ const option = (name: string, fallback: string) => (process.argv.includes(name) ? process.argv[process.argv.indexOf(name) + 1] : fallback);+ const label = option("--label", "run1");+ const port = Number(option("--port", "17898"));+ const lang = option("--lang", "zh") === "en" ? "en" : "zh";+ const words = SCRIPT[lang];+ const root = join(localRoot, "v01");+ const data = join(root, `needs-data-${label}`);+ const work = join(root, "work");+ mkdirSync(work, { recursive: true });+ rememberLang(lang, "codex", languageFileOf(data));++ const engine = spawn(process.execPath, [join(here, "..", "..", "plugin", "src", "proxy.ts"), "--port", String(port), "--stay"], { env: { ...process.env, CAM_DATA_DIR: data }, stdio: "ignore", windowsHide: true });+ for (let i = 0; i < 50 && !(await engineHealth(port)); i++) await sleep(200);+ // Nothing that acts on the machine: no computer use, browsers or app tools, no notify program.+ const off = ["computer-use@openai-bundled", "unified-computer-use@openai-bundled", "chrome@openai-bundled", "browser@openai-bundled", "codex-app-tools@openai-bundled"]+ .flatMap((plugin) => ["-c", `plugins.${plugin}.enabled=false`]);+ const server = new AppServer(findBundledCodex(), ["-c", `openai_base_url="http://localhost:${port}/backend-api/codex"`, "-c", "features.respect_system_proxy=true", "-c", "notify=[]", "-c", "mcp_servers.node_repl.enabled=false", ...off], { NO_PROXY: undefined, no_proxy: undefined }, join(root, `needs-appserver-${label}.log`));+ const result: Json = { createdAt: new Date().toISOString(), label, lang, threads: [] };+ const sessionsDir = join(codexHome(), "sessions");+ try {+ // thread/resume takes a history only from a client that opts into the experimental API.+ await server.initialize({ experimentalApi: true, requestAttestation: false });+ // Starts a thread holding the history, with every screenshot unchecked, and asks the questions in turn.+ const run = async (n: number, questions: Array<[string, string]>) => {+ const { thread } = await server.request<{ thread: Json }>("thread/resume", { threadId: randomUUID(), history: history(words), model: TEST_MODEL, cwd: work, approvalPolicy: "never", sandbox: "read-only" });+ const threadId: string = thread.id;+ await server.request("thread/name/set", { threadId, name: words.name(label, n) });+ const entry: Json = { threadId, turns: {} };+ // The index comes from the thread's record; if Codex has not written it yet, one short turn (images still in) does.+ const indexed = async () => {+ for (let i = 0; i < 25; i++) {+ try { const index = loadThreadIndex(sessionsDir, threadId); if (index.images.length === SHOTS.length) return index; } catch { /* not written yet */ }+ await sleep(200);+ }+ return null;+ };+ let index = await indexed();+ if (!index) {+ entry.warmUp = await runTurn(server, threadId, [text(lang === "zh" ? "先不用做别的,只回复“好”。" : "Nothing else for now; just reply \"OK\".")]);+ index = await indexed();+ }+ if (!index) throw new Error(`thread ${threadId}: its screenshots never showed up in the index`);+ const unchecked = Object.fromEntries(index.images.map((image) => [image.key, { id: image.id, at: new Date().toISOString() }]));+ writeSelection({ threadId, unchecked }, selectionDirOf(data));+ await sleep(2500);+ entry.images = index.images.map((image) => `${image.id}:${image.kind}:${image.name ?? "-"}`);+ for (const [name, prompt] of questions) entry.turns[name] = await runTurn(server, threadId, [text(prompt)]);+ entry.lastRequest = readRequestStats(threadId, join(data, "state", "requests"))?.lastHttp?.rewrite ?? null;+ result.threads.push(entry);+ return entry;+ };+ await run(1, [["lookNow", words.lookNow], ["whichImages", words.whichImages], ["detail", words.detail]]);+ await run(2, [["detail", words.detail]]);+ } finally {+ await server.stop();+ engine.kill();+ }++ const summary = result.threads.map((entry: Json) => ({+ threadId: entry.threadId,+ warmUp: entry.warmUp ? entry.warmUp.reply : null,+ images: entry.images,+ replaced: entry.lastRequest?.replaced?.map((r: Json) => `${r.id}:${r.mode}`) ?? null,+ turns: Object.fromEntries(Object.entries(entry.turns as Record<string, Json>).map(([name, t]) => [name, {+ // What the panel picks up from this reply (its bar and tags), next to every id the reply mentions.+ panelSees: askedIn(String(t.reply)), ids: [...new Set(String(t.reply).match(/IMG-\d{3}/g) ?? [])], asksForNewShots: words.newShots.test(t.reply),+ tools: t.tools, status: t.status, errors: t.errors, reply: t.reply,+ }])),+ }));+ result.summary = summary;+ writeFileSync(join(root, `needs-${label}.json`), JSON.stringify(result, null, 2));+ console.log(JSON.stringify(summary, null, 2));+}++await main();