A local plugin for image-heavy chats in Codex, Claude Code and other agents: earlier images stop going out with every message, and the model fetches the one it needs by its ID, also after compaction.
TypeScript ai-agentsclaude-codeclaude-code-plugincodexcodex-plugincontext-managementdeveloper-toolsimage-generationmcpmultimodal
feat(claude): Anthropic Messages 请求的占位符改写
chipfighter16bb979f09a9
diff --git a/plugin/src/claude-rewrite.ts b/plugin/src/claude-rewrite.tsnew file mode 100644index 0000000..86d62ec--- /dev/null+++ b/plugin/src/claude-rewrite.ts@@ -0,0 +1,179 @@+// Purpose: v0.4 — the engine's rewrite of a Claude Code request (Anthropic Messages), with the same rules and words as+// rewrite.ts for Codex: an image the user unchecked becomes a placeholder (a "duplicate" one when the same content is+// still sent), every image still sent gets an "included" label once anything is left out, and a context management+// note explains the placeholders. Automatic selection (v0.3) leaves out earlier turns' images that are not pinned;+// copies the model fetched are sent in their own turn only.+// What changes for Claude: images sit in user messages (pasted) and inside tool_result blocks; the note is a+// mid-conversation system message, which Anthropic allows only right after a user message, so it goes right after+// the message holding the first omitted image instead of before its turn (docs/v0.4/plan.md, D04-03). Everything else+// — thinking blocks included — is passed on unchanged.+// Matching to the index (claude-index.ts): an image inside a tool result by its tool_use_id; pasted images by+// content, a request message to the transcript record(s) holding the same images in the same order.+// Input: the request's messages, the session's index, the keys to leave out. Output: new messages plus a report.++import type { ClaudeIndex, ClaudeImage } from "./claude-index.ts";+import { imageDigest, type ImageDigest, type ImageKind } from "./images.ts";+import type { Lang } from "./language.ts";+import { autoNote, autoPlaceholder, copyNote, duplicatePlaceholder, includedLabel, newImageWord, omissionNote, plainPlaceholder, type Described, type RewriteReport } from "./rewrite.ts";++type Json = Record<string, any>;+// msg: the message; part: the block in its content; sub: for an image in a tool result, its position in that result.+export type RequestImage = ImageDigest & { msg: number; part: number; sub: number | null; key: string | null; kind: ImageKind; block: Json };++const sameList = (a: string[], b: string[]) => a.length === b.length && a.every((value, i) => value === b[i]);++// Every image of the request, with the index key it matches (null: not in the index, e.g. pasted in this turn).+export function findRequestImages(messages: Json[], index: Pick<ClaudeIndex, "uploads">): RequestImage[] {+ const found: RequestImage[] = [];+ const pastedBy = new Map<number, RequestImage[]>();+ messages.forEach((message, msg) => {+ if (message?.role !== "user" || !Array.isArray(message.content)) return;+ const perTool = new Map<string, number>();+ message.content.forEach((block: Json, part: number) => {+ if (block?.type === "image" && typeof block.source?.data === "string") {+ const image: RequestImage = { ...imageDigest(block, block.source.data, block.source.media_type ?? null), msg, part, sub: null, key: null, kind: "upload", block };+ found.push(image);+ pastedBy.set(msg, [...(pastedBy.get(msg) ?? []), image]);+ }+ if (block?.type === "tool_result" && Array.isArray(block.content)) {+ block.content.forEach((inner: Json, sub: number) => {+ if (inner?.type !== "image" || typeof inner.source?.data !== "string") return;+ const id = String(block.tool_use_id ?? "");+ const m = perTool.get(id) ?? 0;+ perTool.set(id, m + 1);+ found.push({ ...imageDigest(inner, inner.source.data, inner.source.media_type ?? null), msg, part, sub, key: `tool:${id}#${m}`, kind: "tool", block: inner });+ });+ }+ });+ });+ // Pasted images: each message, in order, to the next record(s) after the last match holding the same images in the+ // same order (Claude Code may join consecutive records into one message). Records before the latest compaction are+ // looked at only when nothing later matches.+ const uploads = index.uploads;+ const used = new Set<number>();+ let next = uploads.findIndex((upload) => !upload.compacted);+ if (next < 0) next = uploads.length;+ const matchAt = (start: number, ids: string[]): number[] | null => {+ const taken: number[] = [];+ let at = 0;+ for (let i = start; i < uploads.length && at < ids.length; i++) {+ if (used.has(i)) return null;+ const part = uploads[i].contentIds;+ if (!sameList(part, ids.slice(at, at + part.length))) return null;+ taken.push(i);+ at += part.length;+ }+ return at === ids.length ? taken : null;+ };+ for (const [, images] of [...pastedBy].sort((a, b) => a[0] - b[0])) {+ const ids = images.map((image) => image.contentId);+ let taken: number[] | null = null;+ for (let start = next; start < uploads.length && !taken; start++) taken = matchAt(start, ids);+ for (let start = 0; start < next && !taken; start++) taken = matchAt(start, ids);+ if (!taken) continue;+ let position = 0;+ for (const i of taken) {+ uploads[i].contentIds.forEach((_, n) => { images[position++].key = `${uploads[i].uuid}#${n}`; });+ used.add(i);+ }+ next = Math.max(next, taken.at(-1)! + 1);+ }+ return found;+}++export type ClaudeRewriteOptions = {+ lang?: Lang;+ // Copies the model fetched in an earlier turn (key → the id they copy): left out in either mode.+ copies?: Map<string, string>;+ // Automatic selection: its wording; placeholders point to, and labels mark, only earlier turns' (pinned) images.+ auto?: { currentTurn: number };+};++export function rewriteMessages(+ messages: Json[],+ index: Pick<ClaudeIndex, "uploads" | "byKey">,+ leaveOut: Set<string>,+ pixelHash: (image: RequestImage) => string | null,+ options: ClaudeRewriteOptions = {},+): { messages: Json[]; report: RewriteReport } {+ const { lang = "zh", copies = new Map<string, string>(), auto } = options;+ const refs = findRequestImages(messages, index);+ const describe = (ref: RequestImage): ClaudeImage | undefined => (ref.key ? index.byKey.get(ref.key) : undefined);+ const report: RewriteReport = { images: refs.length, replaced: [], locked: [], sentContentIds: [], copies: [] };+ const copied = refs.filter((ref) => ref.key !== null && copies.has(ref.key));+ const drop = refs.filter((ref) => ref.key !== null && !copies.has(ref.key) && leaveOut.has(ref.key) && describe(ref));+ const sent = refs.filter((ref) => !drop.includes(ref) && !copied.includes(ref));+ report.sentContentIds = sent.map((ref) => ref.contentId);+ if (!drop.length && !copied.length) return { messages, report };+ const earlier = (ref: RequestImage) => { const turn = describe(ref)?.turn; return auto !== undefined && turn !== null && turn !== undefined && turn < auto.currentTurn; };+ const lasting = auto ? sent.filter(earlier) : sent;+ const same = (a: RequestImage, b: RequestImage) => {+ if (a.contentId === b.contentId) return true;+ if (a.width !== b.width || a.height !== b.height) return false;+ const pixels = pixelHash(a);+ return pixels !== null && pixels === pixelHash(b);+ };+ // Edits per content array: the message's own content ("3"), or a tool result's ("3:1"). Each puts text blocks at+ // `at`, in place of `remove` blocks (none for a label).+ type Edit = { at: number; remove: number; blocks: Json[] };+ const edits = new Map<string, Edit[]>();+ const edit = (ref: RequestImage, remove: number, text: string, before = false) => {+ const where = ref.sub === null ? `${ref.msg}` : `${ref.msg}:${ref.part}`;+ const at = ref.sub === null ? ref.part : ref.sub;+ // A placeholder keeps the cache breakpoint the image carried, so Claude Code's caching stays where it put it.+ const carried = !before && ref.block.cache_control ? { cache_control: ref.block.cache_control } : {};+ edits.set(where, [...(edits.get(where) ?? []), { at, remove, blocks: [{ type: "text", text, ...carried }] }]);+ };+ let firstPlain: number | null = null;+ for (const ref of drop) {+ const image = describe(ref)!;+ const copy = lasting.find((other) => same(other, ref));+ const copyImage: Described | null = copy ? describe(copy) ?? { id: newImageWord(lang), name: null, label: null, kind: copy.kind, turn: null, width: copy.width, height: copy.height } : null;+ const text = copyImage ? duplicatePlaceholder(image, copyImage, lang) : auto ? autoPlaceholder(image, lang) : plainPlaceholder(image, lang);+ if (!copyImage && firstPlain === null) firstPlain = ref.msg;+ report.replaced.push({ id: image.id, key: ref.key!, kind: image.kind, mode: copyImage ? "duplicate" : "plain", sameAs: copyImage?.id ?? null, base64Chars: ref.base64Chars });+ edit(ref, 1, text);+ }+ for (const ref of copied) {+ edit(ref, 1, copyNote(copies.get(ref.key!)!, lang));+ report.copies.push(copies.get(ref.key!)!);+ }+ // v0.1-25: once anything is left out, every image still sent says which id it is.+ for (const ref of lasting) {+ const image = describe(ref);+ if (image) edit(ref, 0, includedLabel(image, lang), true);+ }+ const apply = (blocks: Json[], list: Edit[]) => {+ const out = [...blocks];+ // Later positions first; at one position, a replacement before the label that goes in front of it.+ for (const { at, remove, blocks: added } of [...list].sort((a, b) => b.at - a.at || b.remove - a.remove)) out.splice(at, remove, ...added);+ return out;+ };+ const next = messages.map((message, msg) => {+ const own = edits.get(`${msg}`);+ const inner = [...edits.keys()].filter((where) => where.startsWith(`${msg}:`));+ if (!own && !inner.length) return message;+ let content: Json[] = [...message.content];+ for (const where of inner) {+ const part = Number(where.split(":")[1]);+ content[part] = { ...content[part], content: apply(content[part].content, edits.get(where)!) };+ }+ if (own) content = apply(content, own);+ return { ...message, content };+ });+ // Only plain placeholders need the note (a duplicate's content is still in view). Right after the user message+ // holding the first one; when Claude Code put a system message there already, the note joins it, since two system+ // messages may not follow each other.+ if (firstPlain !== null) {+ const text = auto ? autoNote(lang) : omissionNote(lang);+ const after = next[firstPlain + 1];+ if (after?.role === "system") {+ const content = typeof after.content === "string" ? [{ type: "text", text: after.content }] : after.content;+ next[firstPlain + 1] = { ...after, content: [{ type: "text", text }, ...content] };+ } else {+ next.splice(firstPlain + 1, 0, { role: "system", content: text });+ }+ report.noteAt = firstPlain + 1;+ }+ return { messages: next, report };+}diff --git a/plugin/src/rewrite.ts b/plugin/src/rewrite.tsindex 908be68..4000cde 100644--- a/plugin/src/rewrite.ts+++ b/plugin/src/rewrite.ts@@ -144,6 +144,9 @@ export function includedLabel(image: Described, lang: Lang = "zh"): string {// v0.3: automatic selection.export const autoNote = (lang: Lang) => WORDS[lang].autoNote;export const fetchWords = (lang: Lang): FetchWords => WORDS[lang].fetch;+// v0.4: the same words for a Claude Code request (claude-rewrite.ts).+export const copyNote = (id: string, lang: Lang) => WORDS[lang].copy(id);+export const newImageWord = (lang: Lang) => WORDS[lang].newImage;export function autoPlaceholder(image: Described, lang: Lang = "zh"): string {const words = WORDS[lang];diff --git a/spike/src/claude-rewrite.test.ts b/spike/src/claude-rewrite.test.tsnew file mode 100644index 0000000..db544dc--- /dev/null+++ b/spike/src/claude-rewrite.test.ts@@ -0,0 +1,163 @@+// Purpose: v0.4 — the rewrite of a Claude Code request (claude-rewrite.ts): matching request images to the index,+// placeholders in user messages and inside tool results, "included" labels, the note as a system message right after+// the first omitted image's message, cache breakpoints kept, thinking untouched, automatic selection and copies.+// Input: synthetic transcripts and requests (claude-fixtures.ts); output: Node test assertions only.++import assert from "node:assert/strict";+import test from "node:test";+import { buildClaudeIndex } from "../../plugin/src/claude-index.ts";+import { findRequestImages, rewriteMessages } from "../../plugin/src/claude-rewrite.ts";+import { omissionNote } from "../../plugin/src/rewrite.ts";+import { image, pngA, pngB, pngC, text, transcript, usualRequest, usualSession } from "./claude-fixtures.ts";++const noPixels = () => null;++test("request images match the index: tool results by tool_use_id, pasted images by content and order", () => {+ const { t, p1, p3 } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const refs = findRequestImages(usualRequest(), index);+ assert.deepEqual(refs.map((ref) => [ref.msg, ref.part, ref.sub, ref.key]), [+ [0, 0, null, `${p1}#0`], [0, 1, null, `${p1}#1`], [4, 0, 0, "tool:toolu_1#0"], [6, 0, null, `${p3}#0`],+ ]);+});++test("a pasted image not in the transcript yet has no key; one matching no record is left alone", () => {+ const { t } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const messages = [...usualRequest().slice(0, 8), { role: "user", content: [image(pngB), text("new one")] }];+ assert.equal(findRequestImages(messages, index).at(-1)!.key, null);+});++test("records joined into one message still match, in order", () => {+ const t = transcript();+ const p1 = t.prompt([image(pngA), text("one")]);+ const p2 = t.prompt([image(pngB), text("two")]);+ const index = buildClaudeIndex("s", t.records);+ const refs = findRequestImages([{ role: "user", content: [image(pngA), text("one"), image(pngB), text("two")] }], index);+ assert.deepEqual(refs.map((ref) => ref.key), [`${p1}#0`, `${p2}#0`]);+});++test("after compaction, pasted images match the records after the boundary first", () => {+ const t = transcript();+ t.prompt([image(pngA), text("before")]);+ t.assistant(text("ok"));+ t.boundary();+ t.summary();+ const p2 = t.prompt([image(pngA), text("after")]);+ const index = buildClaudeIndex("s", t.records);+ const refs = findRequestImages([{ role: "user", content: [text("summary")] }, { role: "assistant", content: [text("ok")] }, { role: "user", content: [image(pngA), text("after")] }], index);+ assert.deepEqual(refs.map((ref) => ref.key), [`${p2}#0`]);+});++test("an unchecked pasted image becomes a placeholder; the note follows its message as a system message", () => {+ const { t, p1 } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const request = usualRequest();+ const before = JSON.stringify(request);+ const { messages, report } = rewriteMessages(request, index, new Set([`${p1}#1`]), noPixels, { lang: "zh" });+ assert.equal(JSON.stringify(request), before, "the input is not changed");+ assert.deepEqual(report.replaced.map((entry) => [entry.id, entry.mode]), [["IMG-002", "plain"]]);+ assert.equal(report.noteAt, 1);+ // Message 0: label + A, the placeholder in place of B, then the text as it was.+ assert.match(messages[0].content[0].text, /^\[图片 IMG-001 已提供|未命名|用户上传|第 1 轮|8×8\]$/);+ assert.equal(messages[0].content[1].source.data, pngA);+ assert.match(messages[0].content[2].text, /^\[图片 IMG-002 已省略|未命名|用户上传|第 1 轮|8×8\]\n.*需要 IMG-002/s);+ assert.equal(messages[0].content[3].text, "What are these?");+ assert.deepEqual(messages[1], { role: "system", content: omissionNote("zh") });+ // The rest keeps its order; the thinking block and the cache breakpoint are untouched.+ assert.deepEqual(messages[2], request[1]);+ assert.equal(messages.length, request.length + 1);+ assert.deepEqual(messages.at(-1), request.at(-1));+ // Images still sent keep their labels: the tool result's and turn 3's.+ assert.match(messages[5].content[0].content[0].text, /IMG-003 已提供|c\.png|工具查看的图片/);+ assert.equal(messages[5].content[0].content[1].source.data, pngC);+ assert.match(messages[7].content[0].text, /IMG-004 已提供/);+});++test("an image in a tool result is replaced inside the result; a duplicate points to the copy still sent", () => {+ const { t, p1 } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const { messages, report } = rewriteMessages(usualRequest(), index, new Set(["tool:toolu_1#0", `${p1}#0`]), noPixels, { lang: "en" });+ assert.deepEqual(report.replaced.map((entry) => [entry.id, entry.mode, entry.sameAs]), [["IMG-001", "duplicate", "IMG-004"], ["IMG-003", "plain", null]]);+ const result = messages.find((message) => message.content?.[0]?.type === "tool_result")!.content[0];+ assert.equal(result.tool_use_id, "toolu_1");+ assert.equal(result.content.length, 1);+ assert.match(result.content[0].text, /^\[Image IMG-003 omitted \| c\.png \| viewed with a tool \| turn 2 \| 16×8\]/);+ // The note goes right after the tool result's message, the first with a plain placeholder.+ const at = messages.indexOf(messages.find((message) => message.content?.[0]?.type === "tool_result")!);+ assert.equal(messages[at + 1].role, "system");+ assert.equal(messages[at + 2].role, "assistant");+ assert.match(messages[0].content[0].text, /^\[Image IMG-001 not repeated/);+});++test("a placeholder keeps the cache breakpoint the image had; a system message already there takes the note", () => {+ const t = transcript();+ const p1 = t.prompt([image(pngA, { cache_control: { type: "ephemeral" } }), text("one")]);+ t.assistant(text("ok"));+ const index = buildClaudeIndex("s", t.records);+ const request = [+ { role: "user", content: [image(pngA, { cache_control: { type: "ephemeral" } }), text("one")] },+ { role: "system", content: "Claude Code's own" },+ { role: "assistant", content: [text("ok")] },+ ];+ const { messages } = rewriteMessages(request, index, new Set([`${p1}#0`]), noPixels, { lang: "zh" });+ assert.deepEqual(messages[0].content[0].cache_control, { type: "ephemeral" });+ assert.equal(messages.length, 3);+ assert.equal(messages[1].role, "system");+ assert.equal(messages[1].content[0].text, omissionNote("zh"));+ assert.deepEqual(messages[1].content[1], { type: "text", text: "Claude Code's own" });+});++test("nothing to leave out: the same messages come back", () => {+ const { t } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const request = usualRequest();+ const { messages, report } = rewriteMessages(request, index, new Set(), noPixels);+ assert.equal(messages, request);+ assert.equal(report.replaced.length, 0);+ assert.equal(report.images, 4);+});++test("automatic selection: earlier turns' images out, this turn's in, only pinned earlier images labelled", () => {+ const { t, p1, p3 } = usualSession();+ const index = buildClaudeIndex("s", t.records);+ const leaveOut = new Set(index.images.filter((entry) => entry.turn! < 3 && entry.key !== `${p1}#1`).map((entry) => entry.key));+ const { messages, report } = rewriteMessages(usualRequest(), index, leaveOut, noPixels, { lang: "zh", auto: { currentTurn: 3 } });+ assert.deepEqual(report.replaced.map((entry) => entry.id), ["IMG-001", "IMG-003"]);+ assert.match(messages[0].content[0].text, /IMG-001 已省略[^]*cam_view_image/);+ // B (pinned, earlier turn) is labelled; turn 3's own image is not, so a fetch mid-turn changes nothing before it.+ assert.match(messages[0].content[1].text, /IMG-002 已提供/);+ const lastPaste = messages.find((message) => message.content?.some?.((block: any) => block.type === "text" && block.text === "Again?"));+ assert.equal(lastPaste.content[0].type, "image");+ assert.equal(messages[1].role, "system");+ assert.match(messages[1].content, /自动选图/);+ void p3;+});++test("a copy the model fetched in an earlier turn is left out, in either mode", () => {+ const t = transcript();+ t.prompt([image(pngA), text("look")]);+ t.assistant(text("red"));+ t.prompt("again");+ t.assistant({ type: "tool_use", id: "toolu_9", name: "mcp__x__cam_view_image", input: { ids: ["IMG-001"] } });+ t.toolResult("toolu_9", [text("[Image IMG-001 fetched original | unnamed | uploaded by the user | turn 1 | 8×8]"), image(pngA)]);+ t.assistant(text("still red"));+ t.prompt("third");+ const index = buildClaudeIndex("s", t.records);+ const request = [+ { role: "user", content: [image(pngA), text("look")] },+ { role: "assistant", content: [text("red")] },+ { role: "user", content: [text("again")] },+ { role: "assistant", content: [{ type: "tool_use", id: "toolu_9", name: "mcp__x__cam_view_image", input: { ids: ["IMG-001"] } }] },+ { role: "user", content: [{ type: "tool_result", tool_use_id: "toolu_9", content: [text("[Image IMG-001 fetched original | unnamed | uploaded by the user | turn 1 | 8×8]"), image(pngA)] }] },+ { role: "assistant", content: [text("still red")] },+ { role: "user", content: [text("third")] },+ ];+ const copies = new Map([...index.copies.values()].map((copy) => [copy.key, copy.of]));+ const { messages, report } = rewriteMessages(request, index, new Set(), noPixels, { lang: "en", copies });+ assert.deepEqual(report.copies, ["IMG-001"]);+ assert.equal(messages[4].content[0].content[1].text, "[Original of IMG-001: fetched by the model with cam_view_image, included only in that turn, now omitted]");+ // The original stays, labelled; no plain placeholder, so no note.+ assert.match(messages[0].content[0].text, /IMG-001 included/);+ assert.equal(report.noteAt, undefined);+});