A local plugin for image-heavy chats in Codex, Claude Code and other agents: earlier images stop going out with every message, and the model fetches the one it needs by its ID, also after compaction.
TypeScript ai-agentsclaude-codeclaude-code-plugincodexcodex-plugincontext-managementdeveloper-toolsimage-generationmcpmultimodal
eval: strict scoring, session totals, and stop on repeated API errors
chipfightercf97be8aa385
diff --git a/spike/src/eval-long-scene.test.ts b/spike/src/eval-long-scene.test.tsindex 18835ff..244be39 100644--- a/spike/src/eval-long-scene.test.ts+++ b/spike/src/eval-long-scene.test.ts@@ -44,4 +44,9 @@ test("eval-long: scoring", () => {assert.equal(score(sit, "坐着,坐在椅子上").verdict, "right");assert.equal(score(sit, "站着").verdict, "wrong");assert.equal(score(count, "一共 3 版").said, "3");+ // Strict: more than one answer is not right, even with the right one among them.+ assert.equal(score(q("Q1"), "不是 s04_v2.png,是 s04_v1.png").verdict, "ambiguous");+ assert.equal(score(q("Q1"), "s04_v1.png 或 s03_v1.png").verdict, "wrong");+ assert.equal(score(digit, "第 1 镜两版女主都在画面中间,没有站在左边的版本。v1 角落数字是 90,v2 是 61。").verdict, "ambiguous");+ assert.equal(score(sit, "不是站着,是坐着").verdict, "ambiguous");});diff --git a/spike/src/eval-long-scene.ts b/spike/src/eval-long-scene.tsindex 29da763..8e7ea17 100644--- a/spike/src/eval-long-scene.ts+++ b/spike/src/eval-long-scene.ts@@ -434,22 +434,30 @@ export function scene(): Scene {export const visible = (reply: string) => reply.replace(/\[\/\/\]:[ \t]*#[ \t]*\(cam[^)\n]*\)/g, "").trim();const CN: Record<string, string> = { 零: "0", 一: "1", 二: "2", 两: "2", 三: "3", 四: "4", 五: "5", 六: "6", 七: "7", 八: "8", 九: "9", 十: "10" };-// right, wrong (another file, number or word), or none (no answer of the asked kind: "I can't tell" and the like).-export function score(question: Question, reply: string): { verdict: "right" | "wrong" | "none"; said: string | null } {+// right, wrong (another file, number or word), ambiguous (more than one, the right one among them: "not s04_v2.png+// but s04_v1.png", or two numbers for two versions), or none (no answer of the asked kind: "I can't tell" and the+// like). Only right counts. A reply that takes back an earlier answer is marked by the summary for a look by hand.+export type Verdict = "right" | "wrong" | "ambiguous" | "none";+export function score(question: Question, reply: string): { verdict: Verdict; said: string | null } {const shown = visible(reply);- const judge = (said: string | null) => ({ verdict: said === null ? "none" as const : said === question.answer ? "right" as const : "wrong" as const, said });+ const judge = (found: string[]) => {+ const said = [...new Set(found)];+ if (!said.length) return { verdict: "none" as const, said: null };+ const verdict: Verdict = said.length > 1 ? (said.includes(question.answer) ? "ambiguous" : "wrong") : said[0] === question.answer ? "right" : "wrong";+ return { verdict, said: said.join(",") };+ };switch (question.expect) {- case "file": return judge(/[\w-]+\.png\b/i.exec(shown)?.[0].toLowerCase() ?? null);+ case "file": return judge([...shown.matchAll(/[\w-]+\.png\b/gi)].map((match) => match[0].toLowerCase()));case "number": {const clean = shown.replace(/[\w-]+\.png\b/gi, "").replace(/IMG-\d+/g, "").replace(/set_\d+/g, "").replace(/第\s*\d+\s*[镜版张段批轮行帧]/g, "").replace(/\bv\d+\b/gi, "");// Digits first: a Chinese numeral is often just a word ("一共", "一张").- const match = /(?<![0-9])[0-9]{1,3}(?![0-9])/.exec(clean)?.[0] ?? /[零一二两三四五六七八九十]/.exec(clean)?.[0] ?? null;- return judge(match === null ? null : CN[match] ?? match);+ const digits = [...clean.matchAll(/(?<![0-9])[0-9]{1,3}(?![0-9])/g)].map((match) => match[0]);+ const words = digits.length ? [] : [...clean.matchAll(/[零一二两三四五六七八九十]/g)].slice(0, 1).map((match) => CN[match[0]]);+ return judge([...digits, ...words]);}case "word": {const options = question.answer === "坐" ? ["坐", "站", "举", "指", "跳"] : ["左", "右"];- const hit = options.map((word) => ({ word, at: shown.indexOf(word) })).filter((item) => item.at >= 0).sort((a, b) => a.at - b.at)[0];- return judge(hit?.word ?? null);+ return judge(options.filter((word) => shown.includes(word)));}case "text": return { verdict: /weather/i.test(shown) ? "right" : "wrong", said: shown.slice(0, 80) };}diff --git a/spike/src/eval-long-summary.ts b/spike/src/eval-long-summary.tsindex 06c4bdb..8b5a578 100644--- a/spike/src/eval-long-summary.ts+++ b/spike/src/eval-long-summary.ts@@ -1,11 +1,18 @@// Purpose: eval — sums up the runs of eval-long.ts for the report (kept in local/eval-long/report until it goes into-// eval/): per run, how many questions were answered right before and after the compactions and in the fork, how many-// images the model fetched or read again, the feature lines it wrote, and what the requests cost. The summary keeps-// the replies (synthetic content) without session ids or local paths.+// eval/): per run, how many questions were answered right before and after the compactions and in the fork, how the+// detail questions were answered (an original fetched, the file read again, or what the model had noted), the+// feature lines it wrote, and what the session cost. The summary keeps the replies (synthetic content) without+// session ids or local paths.+// Answers are scored again here with the scene's current rules (eval-long-scene.ts, strict: one answer, the right+// one), so older runs count the same way; replies that take an earlier answer back are listed for a look by hand.+// Tokens and cost are Claude Code's own totals for the session (the last result's modelUsage and total_cost_usd,+// running totals a fork carries on; compactions included). Per request, from the stream's first event, only the input+// side is final: what was read from the cache and written to it, for the requests that read only the system prompt.// Input: --label name [--out <dir>]. Output: a console summary; with --out, <dir>/summary.json.import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";import { join } from "node:path";+import { scene, score } from "./eval-long-scene.ts";import { evalRoot } from "./eval-long-setup.ts";type Json = Record<string, any>;@@ -13,17 +20,25 @@ const option = (name: string, fallback: string | null) => (process.argv.includesconst label = option("--label", "run1")!;const out = option("--out", null);const dir = join(evalRoot, label);+const questionsByKey = new Map(scene().questions.map((question) => [question.key, question]));// A run is its file: a run done again keeps the earlier one under another name (v0.10-haiku.before-wording.json).const runs: Json[] = readdirSync(dir).filter((name) => name.endsWith(".json")).map((name) => ({ ...JSON.parse(readFileSync(join(dir, name), "utf8")), file: name.replace(/\.json$/, "") }));-// What Claude Code reports a session cost at API prices (total_cost_usd, a running total that a fork carries on).-function costOf(run: Json): number | null {++// The session's last result event: the fork's when there is one.+function lastResult(run: Json): Json | null {const last = (file: string) => {if (!existsSync(file)) return null;- const totals = readFileSync(file, "utf8").split("\n").flatMap((line) => { try { const event = JSON.parse(line); return event.type === "result" && typeof event.total_cost_usd === "number" ? [event.total_cost_usd] : []; } catch { return []; } });- return totals.at(-1) ?? null;+ const results = readFileSync(file, "utf8").split("\n").flatMap((line) => { try { const event = JSON.parse(line); return event.type === "result" ? [event] : []; } catch { return []; } });+ return results.at(-1) ?? null;};- const total = last(join(dir, run.file, "events-fork.jsonl")) ?? last(join(dir, run.file, "events-main.jsonl"));- return total === null ? null : Math.round(total * 10000) / 10000;+ return last(join(dir, run.file, "events-fork.jsonl")) ?? last(join(dir, run.file, "events-main.jsonl"));+}+function sessionTotals(run: Json): Json | null {+ const result = lastResult(run);+ if (!result) return null;+ const sum = (field: string) => Object.values(result.modelUsage ?? {}).reduce((total: number, usage: any) => total + (usage?.[field] ?? 0), 0);+ const input = sum("inputTokens"), cacheRead = sum("cacheReadInputTokens"), cacheWrite = sum("cacheCreationInputTokens");+ return { input, cacheRead, cacheWrite, output: sum("outputTokens"), fromCache: Math.round((cacheRead / (input + cacheRead + cacheWrite)) * 1000) / 10, costUsd: Math.round((result.total_cost_usd ?? 0) * 10000) / 10000 };}// Tokens a request carried: new input, read from the cache, written to the cache.@@ -31,16 +46,23 @@ const context = (request: Json | undefined) => (request ? request.input + reques// Calibrated on Claude's own output (docs/v0.10/plan.md, gate 6): a Han character ~1.3 tokens, any other ~0.45.const listTokens = (list: Json | null) => (list ? Math.round(list.han * 1.3 + (list.chars - list.han) * 0.45) : null);const strip = (text: string) => text.replace(/[A-Za-z]:\\[^\s"'))]*|\/(?:Users|home|tmp)\/[^\s"'))]*/g, "<path>");+const visible = (reply: string) => reply.replace(/\[\/\/\]:[ \t]*#[ \t]*\(cam[^)\n]*\)/g, "").trim();+// A reply that takes back something said before.+const TAKES_BACK = /更正|撤回|不可靠|收回|之前.{0,12}(?:错|不对)|I was wrong|retract|correction/i;function summarize(run: Json): Json {const steps: Json[] = run.steps;- const questions = steps.filter((step) => step.kind === "question");+ const questions = steps.filter((step) => step.kind === "question").map((step) => {+ const question = questionsByKey.get(step.step.replace(/^K-/, ""));+ return question ? { ...step, ...score(question, String(step.reply ?? "")) } : step;+ });const block = (prefix: string) => {const mine = questions.filter((step) => step.step.startsWith(prefix));- return { asked: mine.length, right: mine.filter((step) => step.verdict === "right").length, wrong: mine.filter((step) => step.verdict === "wrong").length, none: mine.filter((step) => step.verdict === "none").length };+ const count = (verdict: string) => mine.filter((step) => step.verdict === verdict).length;+ return { asked: mine.length, right: count("right"), wrong: count("wrong"), ambiguous: count("ambiguous"), none: count("none") };};const turns = steps.filter((step) => step.kind === "turn");- const requests = steps.flatMap((step) => step.requests as Json[]);+ const requests = steps.flatMap((step) => (step.requests as Json[]) ?? []);// The first request after each compaction: the next step's first.const afterCompaction = steps.flatMap((step, at) => (step.kind === "compact" ? [{ compaction: step.step, tokens: context(steps[at + 1]?.requests?.[0]) }] : []));// Lists the engine put back, in order: the first after each compaction.@@ -48,29 +70,40 @@ function summarize(run: Json): Json {const firstLists: Json[] = [];for (const entry of lists) if (!firstLists.some((seen) => seen.compacted.listed === entry.compacted.listed && seen.session === entry.session)) firstLists.push(entry);const featureLines = new Set(turns.flatMap((step) => [...String(step.reply).matchAll(/\[\/\/\]:\s*#\s*\(cam (IMG-\d+)/g)].map((match) => match[1])));+ // How a question whose answer is in the original only was answered: the original fetched, the file read again+ // (Read), or neither: from what the model had written before (a feature line, its reply).+ const howAnswered = (step: Json) => {+ if (!step.needsOriginal) return null;+ const file = step.target ? `${step.target}.png` : null;+ if (file && step.fetched?.includes(file)) return "fetched";+ if (file && step.reads?.some((read: string) => read === file)) return "read again";+ return "from notes";+ };return {run: run.file, arm: run.arm, model: run.model, effort: run.effort, images: run.images, startedAt: run.startedAt, finishedAt: run.finishedAt,turnsDone: turns.length, turnsFailed: turns.filter((step) => step.timedOut || step.error).length,questions: { mid: block("M"), end: block("Q"), fork: block("K"), all: block("") },fetches: { calls: questions.reduce((sum, step) => sum + step.fetchCalls.length, 0), images: questions.reduce((sum, step) => sum + step.fetched.length, 0), reads: questions.reduce((sum, step) => sum + step.reads.length, 0) },- featureLines: featureLines.size, costUsd: costOf(run),+ featureLines: featureLines.size,+ session: sessionTotals(run),+ requests: { count: requests.length, onlySystem: requests.filter((request) => request.cacheRead <= 4300 && (context(request) ?? 0) > 8000).length, largest: Math.max(0, ...requests.map((request) => context(request) ?? 0)) },// Images the model read in each turn (a turn that names files and reads none did not look at them).turnReads: turns.map((step) => step.reads.length),- tokens: { requests: requests.length, input: requests.reduce((sum, r) => sum + r.input, 0), cacheRead: requests.reduce((sum, r) => sum + r.cacheRead, 0), cacheWrite: requests.reduce((sum, r) => sum + r.cacheWrite, 0), output: requests.reduce((sum, r) => sum + r.output, 0), largest: Math.max(0, ...requests.map((r) => context(r) ?? 0)) },afterCompaction: afterCompaction.map((item, at) => {const list = firstLists[at] ?? null;const tokens = listTokens(list?.list ?? null);return { ...item, listed: list?.compacted?.listed ?? null, listTokens: tokens, listShare: tokens && item.tokens ? Math.round((tokens / item.tokens) * 1000) / 10 : null };}),- answers: questions.map((step) => ({ q: step.step, verdict: step.verdict, said: step.said, expected: step.expected, needsOriginal: step.needsOriginal, fetched: step.fetched, reads: step.reads.length, seconds: Math.round(step.ms / 1000), reply: strip(String(step.reply)).slice(0, 600) })),+ takesBack: questions.filter((step) => TAKES_BACK.test(visible(String(step.reply ?? "")))).map((step) => step.step),+ answers: questions.map((step) => ({ q: step.step, verdict: step.verdict, said: step.said, expected: step.expected, needsOriginal: step.needsOriginal, how: howAnswered(step), fetched: step.fetched, reads: step.reads.length, seconds: Math.round(step.ms / 1000), reply: strip(String(step.reply)).slice(0, 600) })),};}const summary = runs.map(summarize).sort((a, b) => a.run.localeCompare(b.run));for (const run of summary) {- console.log(`\n== ${run.run}: ${run.images} images, ${run.turnsDone} turns (${run.turnsFailed} failed), ${run.featureLines} feature lines`);+ console.log(`\n== ${run.run}: ${run.images} images, ${run.turnsDone} turns (${run.turnsFailed} failed), ${run.featureLines} feature lines; takes back in ${run.takesBack.join(",") || "none"}`);console.table({ mid: run.questions.mid, end: run.questions.end, fork: run.questions.fork, all: run.questions.all });- console.log("fetches", run.fetches, "tokens", run.tokens);+ console.log("session", run.session, "requests", run.requests, "fetches", run.fetches);console.table(run.afterCompaction);console.table(run.answers.map(({ reply, ...rest }: Json) => ({ ...rest, fetched: rest.fetched.join(",") })));}diff --git a/spike/src/eval-long.ts b/spike/src/eval-long.tsindex 0af4313..20d77c8 100644--- a/spike/src/eval-long.ts+++ b/spike/src/eval-long.ts@@ -200,11 +200,20 @@ async function runOne(run: Run, label: string, port: number, effort: string, quiconst result: Json = { run: run.name, arm, model: MODELS[model], effort, images: images.length, startedAt: new Date().toISOString(), steps: [] };let session = new Session(args(), env, work, join(runDir, "events-main.jsonl"));const record = (step: Json) => { result.steps.push(step); writeFileSync(join(root, `${run.name}.json`), JSON.stringify(result, null, 2)); };+ // An API error that is not the account's limit (the network, the upstream): the same step once more a minute later;+ // twice in a row ends the run, whose further steps would only fail too (2026-11-01: 36 steps of 502s).+ let broken = false;+ const sendStep = async (content: Json[], timeoutMs: number) => {+ const apiError = (sent: Awaited<ReturnType<Session["send"]>>) => !!sent.result?.is_error && /API Error/i.test(String(sent.result.result)) && !/limit/i.test(String(sent.result.result));+ let sent = await session.send(content, timeoutMs);+ if (apiError(sent)) { await sleep(60_000); sent = await session.send(content, timeoutMs); broken = apiError(sent); }+ return sent;+ };const ask = async (question: Question) => {- const sent = await session.send([{ type: "text", text: question.say }], QUESTION_TIMEOUT);+ const sent = await sendStep([{ type: "text", text: question.say }], QUESTION_TIMEOUT);const reply = sent.result?.result ?? "";record({ step: question.key, kind: "question", ms: sent.ms, timedOut: sent.timedOut, error: sent.result?.is_error ?? null, reply, ...score(question, reply), expected: question.answer, target: question.target, needsOriginal: question.needsOriginal, ...turnOf(sent.events) });- return sent.result !== null;+ return sent.result !== null && !broken;};result.quotaAtStart = { ...quota };@@ -216,15 +225,15 @@ async function runOne(run: Run, label: string, port: number, effort: string, quiif (step.type === "turn") {turnNo++;const content = [{ type: "text", text: step.say }, ...step.upload.map(upload)];- let sent = await session.send(content, TURN_TIMEOUT);+ let sent = await sendStep(content, TURN_TIMEOUT);// Refused for the account's limit: the same turn again once the window has started again.if (sent.result?.is_error && /limit/i.test(String(sent.result.result))) { await waitForWindow(record, true); sent = await session.send(content, TURN_TIMEOUT); }record({ step: `T${turnNo}`, kind: "turn", ms: sent.ms, timedOut: sent.timedOut, error: sent.result?.is_error ?? null, reply: sent.result?.result ?? "", ...turnOf(sent.events) });- if (!sent.result) break;+ if (!sent.result || broken) break;} else if (step.type === "compact") {- const sent = await session.send([{ type: "text", text: "/compact" }], TURN_TIMEOUT);+ const sent = await sendStep([{ type: "text", text: "/compact" }], TURN_TIMEOUT);record({ step: `compact-${step.label}`, kind: "compact", ms: sent.ms, timedOut: sent.timedOut, error: sent.result?.is_error ?? null, ...turnOf(sent.events) });- if (!sent.result) break;+ if (!sent.result || broken) break;} else if (step.block === "fork") {const from = session.sessionId;await session.close();@@ -240,6 +249,7 @@ async function runOne(run: Run, label: string, port: number, effort: string, qui}if (!result.session) result.session = session.sessionId;}+ if (broken) record({ step: "stopped", kind: "stopped", reason: "API errors twice in a row" });} finally {await session.close();engine?.kill();