Stop paying the multimodal tax. Your AI agent keeps every image it has seen but only sends the ones it needs: lighter chats, no endless reconnecting, no starting over. Works with Codex, Claude Code, Cline, Gemini CLI, OpenCode, WorkBuddy and more. Install once, forget it.
TypeScript ai-agentsai-imageclaude-codeclaude-code-pluginclinecodexcodex-plugincomputer-usecontext-engineeringcontext-managementdeveloper-toolsgemini-cliimage-generationmcpmultimodalmultimodal-aiopencodeqwen-codetoken-optimizationvision-language-model
fix(claude): wait briefly for the turn's newest images to reach the transcript
chipfighter983a3cec6f49
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b65b5a3..65cbc8a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -22,6 +22,7 @@ Earlier images come back after compaction and resume, with the same IDs, in Code- Resumed and forked Claude sessions start with the previous session's checked, pinned and Auto-select settings.- Long feature lines and ratings are cut by width rather than by character count, whatever the language setting, so English feature lines in a Chinese setup are no longer cut down to the place alone. Ratings now keep as much as feature lines (30 Chinese characters or 100 letters, up from 15 characters), so the user's words are no longer cut in half.- In Claude Code, when the model read several images at once and the results came back out of order, an image could get no ID (since 0.4): it went out as an original in every request and was lost after a compaction. Every such image now gets its ID, in the order it was read; feature lines written before are matched to the new IDs.+- In Claude Code, a request sometimes went out a moment before Claude Code had written the turn's newest images to its transcript (about 1 request in 75 in the logs on the development machine). Those images went without IDs, so the model could not note them in that reply. The engine now reads the transcript again for up to 1.5 seconds when that happens.### Notesdiff --git a/plugin/src/claude-rewrite.ts b/plugin/src/claude-rewrite.tsindex 56082bc..44ece91 100644--- a/plugin/src/claude-rewrite.ts+++ b/plugin/src/claude-rewrite.ts@@ -119,8 +119,13 @@ export function rewriteMessages(const sent = refs.filter((ref) => !drop.includes(ref) && !copied.includes(ref));report.sentContentIds = sent.map((ref) => ref.contentId);// A copy the model fetched this turn is known as one, not as an image.- const unknown = refs.filter((ref) => !copied.includes(ref) && !(ref.key && index.copies?.has(ref.key)) && !describe(ref)).length;- if (unknown) report.unknown = unknown;+ const unknown = refs.filter((ref) => !copied.includes(ref) && !(ref.key && index.copies?.has(ref.key)) && !describe(ref));+ if (unknown.length) report.unknown = unknown.length;+ // v0.10: unknown images after the model's latest reply: Claude Code can send a request a moment before those records+ // reach its transcript, and they would go out without ids (the engine reads the transcript again, proxy.ts).+ const lastReply = messages.map((message) => message?.role).lastIndexOf("assistant");+ const late = unknown.filter((ref) => ref.msg > lastReply).length;+ if (late) report.late = late;// v0.8: the first image the index knows; the feature line note follows its message.const first = features ? refs.find((ref) => describe(ref)) : undefined;if (!drop.length && !copied.length && !first) return { messages, report };diff --git a/plugin/src/proxy.ts b/plugin/src/proxy.tsindex c414d8d..33e6b64 100644--- a/plugin/src/proxy.ts+++ b/plugin/src/proxy.ts@@ -23,6 +23,7 @@ import http, { type IncomingHttpHeaders } from "node:http";import net from "node:net";import { dirname, join } from "node:path";import type { Duplex } from "node:stream";+import { setTimeout as sleep } from "node:timers/promises";import tls from "node:tls";import { fileURLToPath } from "node:url";import zlib from "node:zlib";@@ -428,6 +429,11 @@ export function thinkingRejected(status: number, body: string): boolean {// once the attempt goes through (null: not the session's conversation); made: the histories its thinking was made// after; sure: the record says all its thinking fits, so a refusal means the record misses what went out.export type ClaudeAttempt = { body: Buffer; report: Json; history: string | null; sent: string | null; made: string[]; sure: boolean };+// v0.10: how long a request waits for its newest images to reach the transcript (a long-task eval caught 3 of a turn's+// images going out without ids, the transcript written a moment after the request; the user's logs had 119 such of+// 8904 requests). A request whose images never reach it waits the whole time once.+const LATE_WAIT_MS = 1500;+const LATE_STEP_MS = 100;// v0.4: the same rules as rewriteBody, for a Claude Code request (Anthropic Messages, claude-rewrite.ts). The current// turn is the transcript's latest prompt: Claude Code writes it before it sends the request.@@ -498,6 +504,7 @@ export function rewriteClaudeBody(original: Buffer, encoding: string | undefined...(compacted ? { compacted } : {}),...(report.featureAt !== undefined ? { featureAt: report.featureAt, labeled: report.labeled ?? 0 } : {}),...(report.unknown ? { unknown: report.unknown } : {}),+ ...(report.late ? { late: report.late } : {}),};const changed = report.replaced.length > 0 || report.copies.length > 0 || compacted !== null || report.featureAt !== undefined;// Once the engine has sent the session a changed history, thinking made after it may be in every later request, even@@ -923,13 +930,20 @@ async function main(): Promise<void> {}); };if (!rewrite && !claudeRewrite) { send([{ body: null, report: null, edited: false }]); return; }collect();- req.on("end", () => {+ req.on("end", async () => {const original = Buffer.concat(chunks);const encoding = req.headers["content-encoding"] as string | undefined;- let out: { body: Buffer; report: Json; attempts?: ClaudeAttempt[] };- try { out = claudeRewrite ? rewriteClaudeBody(original, encoding, claudeSession!, undefined, undefined, undefined, thinking) : rewriteBody(original, encoding, identity.threadId!, sessionsDir, selectionDir, identity.turnId, identity.forkedFrom); }- catch (error) { out = { body: original, report: { skipped: `rewrite failed: ${String(error)}` } }; }- const attempts: Attempt[] = out.attempts?.length ? out.attempts.map((attempt) => ({ ...attempt, edited: attempt.body !== original })) : [{ body: out.body, report: out.report, edited: out.body !== original }];+ const rewriteNow = (): { body: Buffer; report: Json; attempts?: ClaudeAttempt[] } => {+ try { return claudeRewrite ? rewriteClaudeBody(original, encoding, claudeSession!, undefined, undefined, undefined, thinking) : rewriteBody(original, encoding, identity.threadId!, sessionsDir, selectionDir, identity.turnId, identity.forkedFrom); }+ catch (error) { return { body: original, report: { skipped: `rewrite failed: ${String(error)}` } }; }+ };+ let out = rewriteNow();+ // v0.10: the newest images not in the transcript yet (claude-rewrite.ts, late): read it again for a moment, so+ // they go out with their ids now and the model can note them in this reply.+ let waited = 0;+ for (; claudeRewrite && out.report?.late && waited < LATE_WAIT_MS; waited += LATE_STEP_MS) { await sleep(LATE_STEP_MS); out = rewriteNow(); }+ const withWait = (report: Json) => (waited ? { ...report, waitedMs: waited } : report);+ const attempts: Attempt[] = out.attempts?.length ? out.attempts.map((attempt) => ({ ...attempt, report: withWait(attempt.report), edited: attempt.body !== original })) : [{ body: out.body, report: withWait(out.report), edited: out.body !== original }];send(attempts);});};diff --git a/plugin/src/rewrite.ts b/plugin/src/rewrite.tsindex 6844c53..0ca0e35 100644--- a/plugin/src/rewrite.ts+++ b/plugin/src/rewrite.ts@@ -29,7 +29,7 @@ export type Replacement = { id: string; key: string; kind: ImageKind; mode: "pla// noteAt: where the developer message explaining omitted images was inserted, if one was. copies: the ids whose copies// fetched in an earlier turn were left out. v0.8: featureAt, where the feature line note went; labeled, how many// images sent got their id; unknown, how many images of the request the index did not know (e.g. not recorded yet).-export type RewriteReport = { images: number; replaced: Replacement[]; locked: string[]; sentContentIds: string[]; copies: string[]; noteAt?: number; featureAt?: number; labeled?: number; unknown?: number };+export type RewriteReport = { images: number; replaced: Replacement[]; locked: string[]; sentContentIds: string[]; copies: string[]; noteAt?: number; featureAt?: number; labeled?: number; unknown?: number; late?: number };type FetchWords = { title: string; description: string; ids: string; missing: (id: string) => string; unavailable: (id: string) => string; off: string };type Words = {diff --git a/spike/src/claude-rewrite.test.ts b/spike/src/claude-rewrite.test.tsindex 0f71f05..4bfa152 100644--- a/spike/src/claude-rewrite.test.ts+++ b/spike/src/claude-rewrite.test.ts@@ -30,6 +30,19 @@ test("a pasted image not in the transcript yet has no key; one matching no recorassert.equal(findRequestImages(messages, index).at(-1)!.key, null);});+test("v0.10: unknown images after the model's latest reply are counted as late (the transcript may not have them yet)", () => {+ const { t } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const fresh = { role: "user", content: [image(pngB), text("new one")] };+ const now = rewriteMessages([...usualRequest(), fresh], index, new Set(), noPixels).report;+ assert.equal(now.unknown, 1);+ assert.equal(now.late, 1);+ // Once the model has answered after it, an image the transcript still lacks is not waited for.+ const after = rewriteMessages([...usualRequest(), fresh, { role: "assistant", content: [text("ok")] }, { role: "user", content: [text("next")] }], index, new Set(), noPixels).report;+ assert.equal(after.unknown, 1);+ assert.equal(after.late, undefined);+});+test("records joined into one message still match, in order", () => {const t = transcript();const p1 = t.prompt([image(pngA), text("one")]);