A local plugin for image-heavy chats in Codex, Claude Code and other agents: earlier images stop going out with every message, and the model fetches the one it needs by its ID, also after compaction.
TypeScript ai-agentsclaude-codeclaude-code-plugincodexcodex-plugincontext-managementdeveloper-toolsimage-generationmcpmultimodal
// Purpose: v0.1-17 follow-up — when the images the model needs are unnamed screenshots it took with a tool (as with
// computer use) and the user unchecked them all, does it ask for them by id ("需要 IMG-xxx")? Replays the shape of the
// user's report of 2026-09-26: the model changed a setting in a desktop app, taking screenshots after each step (each
// call has a title, no screenshot has a name, and the model never described them); later all of them are unchecked.
// Thread 1 asks what the user asked then ("can you look at the images, how are the settings now?", "which images do
// you need?"), then a detail only an old screenshot shows; thread 2 asks only that detail, in a fresh context.
// The history is given to app-server's thread/resume (history: an unstable parameter), which starts a new thread
// holding it. Computer use, the browsers and the other plugins that act on the machine are switched off for this
// app-server, and it runs read-only with no approvals, so nothing on the machine is touched.
// Input: [--label name] [--port 17898] [--lang zh|en]. Output: local/v01/needs-<label>.json (synthetic threads only).
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import { AppServer, findBundledCodex } from "./appserver.ts";
import { codexHome } from "../../plugin/src/adapters/codex/codexconfig.ts";
import { engineHealth } from "../../plugin/src/engine.ts";
import { languageFileOf } from "../../plugin/src/core/language.ts";
import { rememberLang } from "../../plugin/src/adapters/codex/language.ts";
import { askedIn } from "../../plugin/src/panel/panel-state.ts";
import { selectionDirOf } from "../../plugin/src/core/paths.ts";
import { readRequestStats } from "../../plugin/src/core/request-stats.ts";
import { writeSelection } from "../../plugin/src/core/selection.ts";
import { loadThreadIndex } from "../../plugin/src/adapters/codex/thread-index.ts";
import { archiveThreads, png, runTurn, TEST_EFFORT, TEST_MODEL, text } from "./testkit.ts";
type Json = Record<string, any>;
const here = dirname(fileURLToPath(import.meta.url));
const localRoot = join(resolve(here, "../.."), "local");
// A 640×400 app window: title bar, a list on the left with one entry highlighted (or none), and one shape on the right.
type Rgb = [number, number, number];
function screen(selected: number | null, shape: (x: number, y: number) => boolean, color: Rgb): Buffer {
return png(640, 400, (x, y) => {
if (y < 28) return [45, 45, 50];
if (shape(x, y)) return color;
if (selected !== null && x < 150) return y >= 60 + selected * 36 && y < 90 + selected * 36 ? [70, 110, 200] : [235, 235, 238];
return [250, 250, 250];
});
}
const SHOTS = [
// Main window: a dark preview area with a green disc in it.
() => png(640, 400, (x, y) => (y < 28 ? [45, 45, 50] : x > 40 && x < 600 && y > 50 && y < 300 ? ((x - 320) ** 2 + (y - 175) ** 2 < 55 ** 2 ? [40, 200, 90] : [20, 20, 24]) : [230, 230, 232])),
// Settings, first page: a red square.
() => screen(0, (x, y) => x > 300 && x < 380 && y > 150 && y < 230, [220, 40, 40]),
// Settings, output page: a blue bar.
() => screen(1, (x, y) => x > 200 && x < 560 && y > 180 && y < 204, [40, 90, 220]),
// Settings, video page: an orange triangle in the top-right corner.
() => screen(2, (x, y) => y > 50 && y < 120 && x > 520 && Math.abs(x - 560) < (y - 50) * 0.6, [245, 140, 20]),
];
const SCRIPT = {
zh: {
name: (label: string, n: number) => `[CAM测试] 截图编号自测 ${label}-${n}`,
ask: "帮我把这个录屏软件的录像保存路径改到 D:\\录像,改完告诉我。",
plan: "我来打开软件,在设置里修改录像路径并核对。",
titles: ["查看主界面", "打开设置对话框", "切到输出页", "填入新的录像路径", "切到视频页"],
pathText: (path: string) => `录像路径 Value: ${path}`,
done: "已经把录像路径改成 D:\\录像,也看过了视频设置页。",
lookNow: "你能看看图片,现在设置如何的?",
whichImages: "你需要哪些图片?",
detail: "之前你截的视频设置页里,右上角那个图形是什么颜色?",
recheck: "我已经勾上了。那个图形是什么颜色?",
color: /橙/,
newShots: /截图|截个图|截一张|发.{0,6}图|重新截/,
},
en: {
name: (label: string, n: number) => `[CAM test] screenshot id self-test ${label}-${n}`,
ask: "Please change this screen recorder's recording folder to D:\\Recordings and tell me when it's done.",
plan: "I'll open the app, change the recording path in its settings, and check it.",
titles: ["Look at the main window", "Open the settings dialog", "Go to the Output page", "Enter the new recording path", "Go to the Video page"],
pathText: (path: string) => `Recording Path Value: ${path}`,
done: "The recording path is now D:\\Recordings, and I also looked at the video settings page.",
lookNow: "Can you look at the images? How are the settings now?",
whichImages: "Which images do you need?",
detail: "In the video settings page you took a screenshot of earlier, what color is the shape in the top-right corner?",
recheck: "Checked it. What color is the shape?",
color: /orange/i,
newShots: /screenshot|send (me )?(a |an |the )?(new |fresh )?(image|picture)/i,
},
};
// The history as Codex records it: the model's calls have titles; each screenshot is a tool output with no name.
function history(words: typeof SCRIPT.zh): Json[] {
const url = (bytes: Buffer) => `data:image/png;base64,${bytes.toString("base64")}`;
const call = (n: number, title: string, output: Json[]): Json[] => [
{ type: "custom_tool_call", status: "completed", call_id: `call_shot_${n}`, name: "exec", input: `const r=await tools.mcp__node_repl__js({code:"state=await sky.get_window_state({window:targetWindow,include_screenshot:${output.some((part) => part.type === "input_image")}});",title:${JSON.stringify(title)}}); for(const c of r.content||[]){if(c.type==="image") image(c); else if(c.type==="text") text(c.text)}` },
{ type: "custom_tool_call_output", call_id: `call_shot_${n}`, output: [{ type: "input_text", text: "Script completed\nWall time 1.0 seconds\nOutput:\n" }, ...output] },
];
const shot = (i: number) => ({ type: "input_image", image_url: url(SHOTS[i]()) });
return [
{ type: "message", role: "user", content: [{ type: "input_text", text: words.ask }] },
{ type: "message", role: "assistant", content: [{ type: "output_text", text: words.plan }] },
...call(1, words.titles[0], [shot(0)]),
...call(2, words.titles[1], [shot(1)]),
...call(3, words.titles[2], [{ type: "input_text", text: words.pathText("C:\\Users\\me\\Videos") }, shot(2)]),
...call(4, words.titles[3], [{ type: "input_text", text: words.pathText(words.ask.includes("D:\\录像") ? "D:\\录像" : "D:\\Recordings") }]),
...call(5, words.titles[4], [shot(3)]),
{ type: "message", role: "assistant", content: [{ type: "output_text", text: words.done }] },
];
}
async function main(): Promise<void> {
const option = (name: string, fallback: string) => (process.argv.includes(name) ? process.argv[process.argv.indexOf(name) + 1] : fallback);
const label = option("--label", "run1");
const port = Number(option("--port", "17898"));
const lang = option("--lang", "zh") === "en" ? "en" : "zh";
const words = SCRIPT[lang];
const root = join(localRoot, "v01");
const data = join(root, `needs-data-${label}`);
const work = join(root, "work");
mkdirSync(work, { recursive: true });
rememberLang(lang, "codex", languageFileOf(data));
const engine = spawn(process.execPath, [join(here, "..", "..", "plugin", "src", "proxy.ts"), "--port", String(port), "--stay"], { env: { ...process.env, CAM_DATA_DIR: data }, stdio: "ignore", windowsHide: true });
for (let i = 0; i < 50 && !(await engineHealth(port)); i++) await sleep(200);
// Nothing that acts on the machine: no computer use, browsers or app tools, no notify program.
const off = ["computer-use@openai-bundled", "unified-computer-use@openai-bundled", "chrome@openai-bundled", "browser@openai-bundled", "codex-app-tools@openai-bundled"]
.flatMap((plugin) => ["-c", `plugins.${plugin}.enabled=false`]);
const server = new AppServer(findBundledCodex(), ["-c", `openai_base_url="http://localhost:${port}/backend-api/codex"`, "-c", "features.respect_system_proxy=true", "-c", "notify=[]", "-c", "mcp_servers.node_repl.enabled=false", ...off], { NO_PROXY: undefined, no_proxy: undefined }, join(root, `needs-appserver-${label}.log`));
const result: Json = { createdAt: new Date().toISOString(), label, lang, model: TEST_MODEL, effort: TEST_EFFORT, threads: [] };
const sessionsDir = join(codexHome(), "sessions");
const created: string[] = [];
try {
// thread/resume takes a history only from a client that opts into the experimental API.
await server.initialize({ experimentalApi: true, requestAttestation: false });
// Starts a thread holding the history, with every screenshot unchecked, and asks the questions in turn.
const run = async (n: number, questions: Array<[string, string]>, recheck = false) => {
const { thread } = await server.request<{ thread: Json }>("thread/resume", { threadId: randomUUID(), history: history(words), model: TEST_MODEL, cwd: work, approvalPolicy: "never", sandbox: "read-only" });
const threadId: string = thread.id;
created.push(threadId);
await server.request("thread/name/set", { threadId, name: words.name(label, n) });
const entry: Json = { threadId, turns: {} };
// The index comes from the thread's record; if Codex has not written it yet, one short turn (images still in) does.
const indexed = async () => {
for (let i = 0; i < 25; i++) {
try { const index = loadThreadIndex(sessionsDir, threadId); if (index.images.length === SHOTS.length) return index; } catch { /* not written yet */ }
await sleep(200);
}
return null;
};
let index = await indexed();
if (!index) {
entry.warmUp = await runTurn(server, threadId, [text(lang === "zh" ? "先不用做别的,只回复“好”。" : "Nothing else for now; just reply \"OK\".")]);
index = await indexed();
}
if (!index) throw new Error(`thread ${threadId}: its screenshots never showed up in the index`);
const unchecked = Object.fromEntries(index.images.map((image) => [image.key, { id: image.id, at: new Date().toISOString() }]));
// Manual mode on purpose: from v0.6 a new task has automatic selection on by default.
writeSelection({ threadId, unchecked, auto: false }, selectionDirOf(data));
await sleep(2500);
entry.images = index.images.map((image) => `${image.id}:${image.kind}:${image.name ?? "-"}`);
for (const [name, prompt] of questions) entry.turns[name] = await runTurn(server, threadId, [text(prompt)]);
// v0.1-25 (user 2026-09-28): the user checks only the images the model asked for, the others stay unchecked, and
// asks again; the model must now answer from the image (GPT-6 Luna kept saying it could not see it).
const asked = askedIn(String(entry.turns[questions[questions.length - 1][0]]?.reply ?? ""));
if (recheck && asked.length) {
writeSelection({ threadId, unchecked: Object.fromEntries(Object.entries(unchecked).filter(([, value]) => !asked.includes(value.id))), auto: false }, selectionDirOf(data));
await sleep(2500);
entry.rechecked = asked;
entry.turns.recheck = await runTurn(server, threadId, [text(words.recheck)]);
}
// The engine records a request once its response has ended, which can be just after the turn completes.
await sleep(1500);
entry.lastRequest = readRequestStats(threadId, join(data, "state", "requests"))?.lastHttp?.rewrite ?? null;
result.threads.push(entry);
return entry;
};
await run(1, [["lookNow", words.lookNow], ["whichImages", words.whichImages], ["detail", words.detail]]);
await run(2, [["detail", words.detail]], true);
} finally {
result.notArchived = await archiveThreads(server, created);
await server.stop();
engine.kill();
}
const summary = result.threads.map((entry: Json) => ({
threadId: entry.threadId,
warmUp: entry.warmUp ? entry.warmUp.reply : null,
images: entry.images,
replaced: entry.lastRequest?.replaced?.map((r: Json) => `${r.id}:${r.mode}`) ?? null,
turns: Object.fromEntries(Object.entries(entry.turns as Record<string, Json>).map(([name, t]) => [name, {
// What the panel picks up from this reply (its bar and tags), next to every id the reply mentions.
panelSees: askedIn(String(t.reply)), ids: [...new Set(String(t.reply).match(/IMG-\d{3}/g) ?? [])], asksForNewShots: words.newShots.test(t.reply),
// v0.1-24: whether the reply uses the words the note asks for ("需要 IMG-xxx" / "need IMG-xxx").
writesNeed: /需要\s*[“"*]*\s*IMG-\d{3}|\bneed\s*[“"*]*\s*IMG-\d{3}/i.test(String(t.reply)),
...(name === "recheck" ? { rechecked: entry.rechecked, answersColor: words.color.test(String(t.reply)) } : {}),
tools: t.tools, status: t.status, errors: t.errors, reply: t.reply,
}])),
}));
result.summary = summary;
writeFileSync(join(root, `needs-${label}.json`), JSON.stringify(result, null, 2));
console.log(JSON.stringify(summary, null, 2));
}
await main();