Skip to content

A local plugin for image-heavy chats in Codex, Claude Code and other agents: earlier images stop going out with every message, and the model fetches the one it needs by its ID, also after compaction.

eval: strict scoring, session totals, and stop on repeated API errors

chipfightercf97be8aa385
diff --git a/spike/src/eval-long-scene.test.ts b/spike/src/eval-long-scene.test.ts
index 18835ff..244be39 100644
--- a/spike/src/eval-long-scene.test.ts
+++ b/spike/src/eval-long-scene.test.ts
@@ -44,4 +44,9 @@ test("eval-long: scoring", () => {
assert.equal(score(sit, "坐着,坐在椅子上").verdict, "right");
assert.equal(score(sit, "站着").verdict, "wrong");
assert.equal(score(count, "一共 3 版").said, "3");
+ // Strict: more than one answer is not right, even with the right one among them.
+ assert.equal(score(q("Q1"), "不是 s04_v2.png,是 s04_v1.png").verdict, "ambiguous");
+ assert.equal(score(q("Q1"), "s04_v1.png 或 s03_v1.png").verdict, "wrong");
+ assert.equal(score(digit, "第 1 镜两版女主都在画面中间,没有站在左边的版本。v1 角落数字是 90,v2 是 61。").verdict, "ambiguous");
+ assert.equal(score(sit, "不是站着,是坐着").verdict, "ambiguous");
});
diff --git a/spike/src/eval-long-scene.ts b/spike/src/eval-long-scene.ts
index 29da763..8e7ea17 100644
--- a/spike/src/eval-long-scene.ts
+++ b/spike/src/eval-long-scene.ts
@@ -434,22 +434,30 @@ export function scene(): Scene {
export const visible = (reply: string) => reply.replace(/\[\/\/\]:[ \t]*#[ \t]*\(cam[^)\n]*\)/g, "").trim();
const CN: Record<string, string> = { 零: "0", 一: "1", 二: "2", 两: "2", 三: "3", 四: "4", 五: "5", 六: "6", 七: "7", 八: "8", 九: "9", 十: "10" };
-// right, wrong (another file, number or word), or none (no answer of the asked kind: "I can't tell" and the like).
-export function score(question: Question, reply: string): { verdict: "right" | "wrong" | "none"; said: string | null } {
+// right, wrong (another file, number or word), ambiguous (more than one, the right one among them: "not s04_v2.png
+// but s04_v1.png", or two numbers for two versions), or none (no answer of the asked kind: "I can't tell" and the
+// like). Only right counts. A reply that takes back an earlier answer is marked by the summary for a look by hand.
+export type Verdict = "right" | "wrong" | "ambiguous" | "none";
+export function score(question: Question, reply: string): { verdict: Verdict; said: string | null } {
const shown = visible(reply);
- const judge = (said: string | null) => ({ verdict: said === null ? "none" as const : said === question.answer ? "right" as const : "wrong" as const, said });
+ const judge = (found: string[]) => {
+ const said = [...new Set(found)];
+ if (!said.length) return { verdict: "none" as const, said: null };
+ const verdict: Verdict = said.length > 1 ? (said.includes(question.answer) ? "ambiguous" : "wrong") : said[0] === question.answer ? "right" : "wrong";
+ return { verdict, said: said.join(",") };
+ };
switch (question.expect) {
- case "file": return judge(/[\w-]+\.png\b/i.exec(shown)?.[0].toLowerCase() ?? null);
+ case "file": return judge([...shown.matchAll(/[\w-]+\.png\b/gi)].map((match) => match[0].toLowerCase()));
case "number": {
const clean = shown.replace(/[\w-]+\.png\b/gi, "").replace(/IMG-\d+/g, "").replace(/set_\d+/g, "").replace(/第\s*\d+\s*[镜版张段批轮行帧]/g, "").replace(/\bv\d+\b/gi, "");
// Digits first: a Chinese numeral is often just a word ("一共", "一张").
- const match = /(?<![0-9])[0-9]{1,3}(?![0-9])/.exec(clean)?.[0] ?? /[零一二两三四五六七八九十]/.exec(clean)?.[0] ?? null;
- return judge(match === null ? null : CN[match] ?? match);
+ const digits = [...clean.matchAll(/(?<![0-9])[0-9]{1,3}(?![0-9])/g)].map((match) => match[0]);
+ const words = digits.length ? [] : [...clean.matchAll(/[零一二两三四五六七八九十]/g)].slice(0, 1).map((match) => CN[match[0]]);
+ return judge([...digits, ...words]);
}
case "word": {
const options = question.answer === "坐" ? ["坐", "站", "举", "指", "跳"] : ["左", "右"];
- const hit = options.map((word) => ({ word, at: shown.indexOf(word) })).filter((item) => item.at >= 0).sort((a, b) => a.at - b.at)[0];
- return judge(hit?.word ?? null);
+ return judge(options.filter((word) => shown.includes(word)));
}
case "text": return { verdict: /weather/i.test(shown) ? "right" : "wrong", said: shown.slice(0, 80) };
}
diff --git a/spike/src/eval-long-summary.ts b/spike/src/eval-long-summary.ts
index 06c4bdb..8b5a578 100644
--- a/spike/src/eval-long-summary.ts
+++ b/spike/src/eval-long-summary.ts
@@ -1,11 +1,18 @@
// Purpose: eval — sums up the runs of eval-long.ts for the report (kept in local/eval-long/report until it goes into
-// eval/): per run, how many questions were answered right before and after the compactions and in the fork, how many
-// images the model fetched or read again, the feature lines it wrote, and what the requests cost. The summary keeps
-// the replies (synthetic content) without session ids or local paths.
+// eval/): per run, how many questions were answered right before and after the compactions and in the fork, how the
+// detail questions were answered (an original fetched, the file read again, or what the model had noted), the
+// feature lines it wrote, and what the session cost. The summary keeps the replies (synthetic content) without
+// session ids or local paths.
+// Answers are scored again here with the scene's current rules (eval-long-scene.ts, strict: one answer, the right
+// one), so older runs count the same way; replies that take an earlier answer back are listed for a look by hand.
+// Tokens and cost are Claude Code's own totals for the session (the last result's modelUsage and total_cost_usd,
+// running totals a fork carries on; compactions included). Per request, from the stream's first event, only the input
+// side is final: what was read from the cache and written to it, for the requests that read only the system prompt.
// Input: --label name [--out <dir>]. Output: a console summary; with --out, <dir>/summary.json.
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
+import { scene, score } from "./eval-long-scene.ts";
import { evalRoot } from "./eval-long-setup.ts";
type Json = Record<string, any>;
@@ -13,17 +20,25 @@ const option = (name: string, fallback: string | null) => (process.argv.includes
const label = option("--label", "run1")!;
const out = option("--out", null);
const dir = join(evalRoot, label);
+const questionsByKey = new Map(scene().questions.map((question) => [question.key, question]));
// A run is its file: a run done again keeps the earlier one under another name (v0.10-haiku.before-wording.json).
const runs: Json[] = readdirSync(dir).filter((name) => name.endsWith(".json")).map((name) => ({ ...JSON.parse(readFileSync(join(dir, name), "utf8")), file: name.replace(/\.json$/, "") }));
-// What Claude Code reports a session cost at API prices (total_cost_usd, a running total that a fork carries on).
-function costOf(run: Json): number | null {
+
+// The session's last result event: the fork's when there is one.
+function lastResult(run: Json): Json | null {
const last = (file: string) => {
if (!existsSync(file)) return null;
- const totals = readFileSync(file, "utf8").split("\n").flatMap((line) => { try { const event = JSON.parse(line); return event.type === "result" && typeof event.total_cost_usd === "number" ? [event.total_cost_usd] : []; } catch { return []; } });
- return totals.at(-1) ?? null;
+ const results = readFileSync(file, "utf8").split("\n").flatMap((line) => { try { const event = JSON.parse(line); return event.type === "result" ? [event] : []; } catch { return []; } });
+ return results.at(-1) ?? null;
};
- const total = last(join(dir, run.file, "events-fork.jsonl")) ?? last(join(dir, run.file, "events-main.jsonl"));
- return total === null ? null : Math.round(total * 10000) / 10000;
+ return last(join(dir, run.file, "events-fork.jsonl")) ?? last(join(dir, run.file, "events-main.jsonl"));
+}
+function sessionTotals(run: Json): Json | null {
+ const result = lastResult(run);
+ if (!result) return null;
+ const sum = (field: string) => Object.values(result.modelUsage ?? {}).reduce((total: number, usage: any) => total + (usage?.[field] ?? 0), 0);
+ const input = sum("inputTokens"), cacheRead = sum("cacheReadInputTokens"), cacheWrite = sum("cacheCreationInputTokens");
+ return { input, cacheRead, cacheWrite, output: sum("outputTokens"), fromCache: Math.round((cacheRead / (input + cacheRead + cacheWrite)) * 1000) / 10, costUsd: Math.round((result.total_cost_usd ?? 0) * 10000) / 10000 };
}
// Tokens a request carried: new input, read from the cache, written to the cache.
@@ -31,16 +46,23 @@ const context = (request: Json | undefined) => (request ? request.input + reques
// Calibrated on Claude's own output (docs/v0.10/plan.md, gate 6): a Han character ~1.3 tokens, any other ~0.45.
const listTokens = (list: Json | null) => (list ? Math.round(list.han * 1.3 + (list.chars - list.han) * 0.45) : null);
const strip = (text: string) => text.replace(/[A-Za-z]:\\[^\s"'))]*|\/(?:Users|home|tmp)\/[^\s"'))]*/g, "<path>");
+const visible = (reply: string) => reply.replace(/\[\/\/\]:[ \t]*#[ \t]*\(cam[^)\n]*\)/g, "").trim();
+// A reply that takes back something said before.
+const TAKES_BACK = /更正|撤回|不可靠|收回|之前.{0,12}(?:错|不对)|I was wrong|retract|correction/i;
function summarize(run: Json): Json {
const steps: Json[] = run.steps;
- const questions = steps.filter((step) => step.kind === "question");
+ const questions = steps.filter((step) => step.kind === "question").map((step) => {
+ const question = questionsByKey.get(step.step.replace(/^K-/, ""));
+ return question ? { ...step, ...score(question, String(step.reply ?? "")) } : step;
+ });
const block = (prefix: string) => {
const mine = questions.filter((step) => step.step.startsWith(prefix));
- return { asked: mine.length, right: mine.filter((step) => step.verdict === "right").length, wrong: mine.filter((step) => step.verdict === "wrong").length, none: mine.filter((step) => step.verdict === "none").length };
+ const count = (verdict: string) => mine.filter((step) => step.verdict === verdict).length;
+ return { asked: mine.length, right: count("right"), wrong: count("wrong"), ambiguous: count("ambiguous"), none: count("none") };
};
const turns = steps.filter((step) => step.kind === "turn");
- const requests = steps.flatMap((step) => step.requests as Json[]);
+ const requests = steps.flatMap((step) => (step.requests as Json[]) ?? []);
// The first request after each compaction: the next step's first.
const afterCompaction = steps.flatMap((step, at) => (step.kind === "compact" ? [{ compaction: step.step, tokens: context(steps[at + 1]?.requests?.[0]) }] : []));
// Lists the engine put back, in order: the first after each compaction.
@@ -48,29 +70,40 @@ function summarize(run: Json): Json {
const firstLists: Json[] = [];
for (const entry of lists) if (!firstLists.some((seen) => seen.compacted.listed === entry.compacted.listed && seen.session === entry.session)) firstLists.push(entry);
const featureLines = new Set(turns.flatMap((step) => [...String(step.reply).matchAll(/\[\/\/\]:\s*#\s*\(cam (IMG-\d+)/g)].map((match) => match[1])));
+ // How a question whose answer is in the original only was answered: the original fetched, the file read again
+ // (Read), or neither: from what the model had written before (a feature line, its reply).
+ const howAnswered = (step: Json) => {
+ if (!step.needsOriginal) return null;
+ const file = step.target ? `${step.target}.png` : null;
+ if (file && step.fetched?.includes(file)) return "fetched";
+ if (file && step.reads?.some((read: string) => read === file)) return "read again";
+ return "from notes";
+ };
return {
run: run.file, arm: run.arm, model: run.model, effort: run.effort, images: run.images, startedAt: run.startedAt, finishedAt: run.finishedAt,
turnsDone: turns.length, turnsFailed: turns.filter((step) => step.timedOut || step.error).length,
questions: { mid: block("M"), end: block("Q"), fork: block("K"), all: block("") },
fetches: { calls: questions.reduce((sum, step) => sum + step.fetchCalls.length, 0), images: questions.reduce((sum, step) => sum + step.fetched.length, 0), reads: questions.reduce((sum, step) => sum + step.reads.length, 0) },
- featureLines: featureLines.size, costUsd: costOf(run),
+ featureLines: featureLines.size,
+ session: sessionTotals(run),
+ requests: { count: requests.length, onlySystem: requests.filter((request) => request.cacheRead <= 4300 && (context(request) ?? 0) > 8000).length, largest: Math.max(0, ...requests.map((request) => context(request) ?? 0)) },
// Images the model read in each turn (a turn that names files and reads none did not look at them).
turnReads: turns.map((step) => step.reads.length),
- tokens: { requests: requests.length, input: requests.reduce((sum, r) => sum + r.input, 0), cacheRead: requests.reduce((sum, r) => sum + r.cacheRead, 0), cacheWrite: requests.reduce((sum, r) => sum + r.cacheWrite, 0), output: requests.reduce((sum, r) => sum + r.output, 0), largest: Math.max(0, ...requests.map((r) => context(r) ?? 0)) },
afterCompaction: afterCompaction.map((item, at) => {
const list = firstLists[at] ?? null;
const tokens = listTokens(list?.list ?? null);
return { ...item, listed: list?.compacted?.listed ?? null, listTokens: tokens, listShare: tokens && item.tokens ? Math.round((tokens / item.tokens) * 1000) / 10 : null };
}),
- answers: questions.map((step) => ({ q: step.step, verdict: step.verdict, said: step.said, expected: step.expected, needsOriginal: step.needsOriginal, fetched: step.fetched, reads: step.reads.length, seconds: Math.round(step.ms / 1000), reply: strip(String(step.reply)).slice(0, 600) })),
+ takesBack: questions.filter((step) => TAKES_BACK.test(visible(String(step.reply ?? "")))).map((step) => step.step),
+ answers: questions.map((step) => ({ q: step.step, verdict: step.verdict, said: step.said, expected: step.expected, needsOriginal: step.needsOriginal, how: howAnswered(step), fetched: step.fetched, reads: step.reads.length, seconds: Math.round(step.ms / 1000), reply: strip(String(step.reply)).slice(0, 600) })),
};
}
const summary = runs.map(summarize).sort((a, b) => a.run.localeCompare(b.run));
for (const run of summary) {
- console.log(`\n== ${run.run}: ${run.images} images, ${run.turnsDone} turns (${run.turnsFailed} failed), ${run.featureLines} feature lines`);
+ console.log(`\n== ${run.run}: ${run.images} images, ${run.turnsDone} turns (${run.turnsFailed} failed), ${run.featureLines} feature lines; takes back in ${run.takesBack.join(",") || "none"}`);
console.table({ mid: run.questions.mid, end: run.questions.end, fork: run.questions.fork, all: run.questions.all });
- console.log("fetches", run.fetches, "tokens", run.tokens);
+ console.log("session", run.session, "requests", run.requests, "fetches", run.fetches);
console.table(run.afterCompaction);
console.table(run.answers.map(({ reply, ...rest }: Json) => ({ ...rest, fetched: rest.fetched.join(",") })));
}
diff --git a/spike/src/eval-long.ts b/spike/src/eval-long.ts
index 0af4313..20d77c8 100644
--- a/spike/src/eval-long.ts
+++ b/spike/src/eval-long.ts
@@ -200,11 +200,20 @@ async function runOne(run: Run, label: string, port: number, effort: string, qui
const result: Json = { run: run.name, arm, model: MODELS[model], effort, images: images.length, startedAt: new Date().toISOString(), steps: [] };
let session = new Session(args(), env, work, join(runDir, "events-main.jsonl"));
const record = (step: Json) => { result.steps.push(step); writeFileSync(join(root, `${run.name}.json`), JSON.stringify(result, null, 2)); };
+ // An API error that is not the account's limit (the network, the upstream): the same step once more a minute later;
+ // twice in a row ends the run, whose further steps would only fail too (2026-11-01: 36 steps of 502s).
+ let broken = false;
+ const sendStep = async (content: Json[], timeoutMs: number) => {
+ const apiError = (sent: Awaited<ReturnType<Session["send"]>>) => !!sent.result?.is_error && /API Error/i.test(String(sent.result.result)) && !/limit/i.test(String(sent.result.result));
+ let sent = await session.send(content, timeoutMs);
+ if (apiError(sent)) { await sleep(60_000); sent = await session.send(content, timeoutMs); broken = apiError(sent); }
+ return sent;
+ };
const ask = async (question: Question) => {
- const sent = await session.send([{ type: "text", text: question.say }], QUESTION_TIMEOUT);
+ const sent = await sendStep([{ type: "text", text: question.say }], QUESTION_TIMEOUT);
const reply = sent.result?.result ?? "";
record({ step: question.key, kind: "question", ms: sent.ms, timedOut: sent.timedOut, error: sent.result?.is_error ?? null, reply, ...score(question, reply), expected: question.answer, target: question.target, needsOriginal: question.needsOriginal, ...turnOf(sent.events) });
- return sent.result !== null;
+ return sent.result !== null && !broken;
};
result.quotaAtStart = { ...quota };
@@ -216,15 +225,15 @@ async function runOne(run: Run, label: string, port: number, effort: string, qui
if (step.type === "turn") {
turnNo++;
const content = [{ type: "text", text: step.say }, ...step.upload.map(upload)];
- let sent = await session.send(content, TURN_TIMEOUT);
+ let sent = await sendStep(content, TURN_TIMEOUT);
// Refused for the account's limit: the same turn again once the window has started again.
if (sent.result?.is_error && /limit/i.test(String(sent.result.result))) { await waitForWindow(record, true); sent = await session.send(content, TURN_TIMEOUT); }
record({ step: `T${turnNo}`, kind: "turn", ms: sent.ms, timedOut: sent.timedOut, error: sent.result?.is_error ?? null, reply: sent.result?.result ?? "", ...turnOf(sent.events) });
- if (!sent.result) break;
+ if (!sent.result || broken) break;
} else if (step.type === "compact") {
- const sent = await session.send([{ type: "text", text: "/compact" }], TURN_TIMEOUT);
+ const sent = await sendStep([{ type: "text", text: "/compact" }], TURN_TIMEOUT);
record({ step: `compact-${step.label}`, kind: "compact", ms: sent.ms, timedOut: sent.timedOut, error: sent.result?.is_error ?? null, ...turnOf(sent.events) });
- if (!sent.result) break;
+ if (!sent.result || broken) break;
} else if (step.block === "fork") {
const from = session.sessionId;
await session.close();
@@ -240,6 +249,7 @@ async function runOne(run: Run, label: string, port: number, effort: string, qui
}
if (!result.session) result.session = session.sessionId;
}
+ if (broken) record({ step: "stopped", kind: "stopped", reason: "API errors twice in a row" });
} finally {
await session.close();
engine?.kill();