Stop paying the multimodal tax. Your AI agent keeps every image it has seen but only sends the ones it needs: lighter chats, no endless reconnecting, no starting over. Works with Codex, Claude Code, Cline, Gemini CLI, OpenCode, WorkBuddy and more. Install once, forget it.
TypeScript ai-agentsai-imageclaude-codeclaude-code-pluginclinecodexcodex-plugincomputer-usecontext-engineeringcontext-managementdeveloper-toolsgemini-cliimage-generationmcpmultimodalmultimodal-aiopencodeqwen-codetoken-optimizationvision-language-model
refactor(core): one auto-select rule, any session id, any platform keeps the engine
chipfighter8a678224b6f4
diff --git a/plugin/src/adapters/claude/engine.ts b/plugin/src/adapters/claude/engine.tsindex d8f210f..01c8b5e 100644--- a/plugin/src/adapters/claude/engine.ts+++ b/plugin/src/adapters/claude/engine.ts@@ -12,7 +12,7 @@ import type { Lang } from "../../core/lang.ts";import { currentLang } from "../../core/language.ts";import { claudeSelection } from "./selection.ts";import { stableIdsSince } from "../../core/selection.ts";-import { awaitingLine } from "../../core/session.ts";+import { autoChoice, earlierCopies } from "../../core/session.ts";import { pixelHashOfData } from "../../core/pixels.ts";import { keepCacheRead, marksOutside, sentPrefixOf, type SentPrefix } from "../protocols/anthropic-messages/cache.ts";import { addCompacted, summaryAt, type RequestImage, type Resend } from "../protocols/anthropic-messages/rewrite.ts";@@ -96,14 +96,12 @@ export function rewriteClaudeBody(original: Buffer, encoding: string | undefinedtry { index = loadClaudeIndex(sessionId, home, stableIdsSince(dir)); } catch (error) { return asIs({ skipped: `thread index: ${String(error)}` }); }const selection = claudeSelection(sessionId, dir, claudeStartedAt(index), index.previous);const current = index.turns;- const copies = new Map([...index.copies.values()].filter((copy) => copy.turn === null || copy.turn < current).map((copy) => [copy.key, copy.of]));- let leaveOut = new Set(Object.keys(selection.unchecked));- // v0.10: images of an interrupted turn without a feature line yet are sent once more (awaitingLine).- const awaiting = new Set(selection.auto ? index.images.filter((image) => awaitingLine(index, image, current)).map((image) => image.key) : []);- if (selection.auto) {- const pinned = selection.pinned ?? {};- leaveOut = new Set(index.images.filter((image) => image.replaceable && image.turn !== null && image.turn < current && !pinned[image.key] && !awaiting.has(image.key)).map((image) => image.key));- }+ const copies = earlierCopies(index, current);+ // v0.3: automatic selection (core/session.ts); v0.10: images of an interrupted turn without a feature line yet are+ // sent once more.+ const chosen = selection.auto ? autoChoice(index, current, selection.pinned ?? {}, true) : null;+ const leaveOut = chosen?.leaveOut ?? new Set(Object.keys(selection.unchecked));+ const awaiting = chosen?.awaiting ?? new Set<string>();const pixels = (ref: RequestImage) => (ref.key ? index.byKey.get(ref.key)?.pixelSha256 : undefined) ?? pixelHashOfData(ref, ref.block.source.data);// v0.5: after a compaction, the images from before it go back right after the summary (anthropic-messages/rewrite.ts): the ones// the user keeps sending (checked, or pinned with automatic selection) read from the transcript, the others asdiff --git a/plugin/src/adapters/codex/engine.ts b/plugin/src/adapters/codex/engine.tsindex 903d199..5984379 100644--- a/plugin/src/adapters/codex/engine.ts+++ b/plugin/src/adapters/codex/engine.ts@@ -16,7 +16,7 @@ import { selectionDir } from "../../core/selection.ts";import { findImages, type ImageRef } from "../protocols/openai-responses/images.ts";import { addCodexCompacted, rewriteItems } from "../protocols/openai-responses/rewrite.ts";import { codexHome } from "./codexconfig.ts";-import { awaitingLine } from "../../core/session.ts";+import { autoChoice, earlierCopies } from "../../core/session.ts";import { imageData, loadThreadIndex, pixelHashOf, sideChatOrigin, writesNotes, type ThreadIndex } from "./thread-index.ts";type Json = Record<string, any>;@@ -83,8 +83,15 @@ export function rewriteBody(original: Buffer, encoding: string | undefined, threlet index: ThreadIndex;try { index = loadThreadIndex(sessionsDir, origin ?? threadId); } catch (error) { return { body: original, report: { skipped: `thread index: ${String(error)}` } }; }const selection = side ?? effectiveSelection(threadId, sessionsDir, dir);- // v0.10: images sent once more for their feature line (awaitingLine), marked in their labels.- const awaiting = new Set<string>();+ // v0.8: feature lines in a task's own requests, not a side chat's or a subagent's.+ const features = !origin && writesNotes(sessionsDir, threadId);+ const current = side ? index.turns + 1 : currentTurnOf(index, turnId);+ const copies = earlierCopies(index, current);+ // v0.3: automatic selection (core/session.ts); v0.10: images sent once more for their feature line, marked in their+ // labels.+ const chosen = selection.auto ? autoChoice(index, current, selection.pinned ?? {}, features) : null;+ const leaveOut = chosen?.leaveOut ?? new Set(Object.keys(selection.unchecked));+ const awaiting = chosen?.awaiting ?? new Set<string>();const describe = (ref: ImageRef): Described | undefined => {const known = index.byKey.get(ref.key);// v0.8: with what the model noted about it, for its placeholder.@@ -94,16 +101,6 @@ export function rewriteBody(original: Buffer, encoding: string | undefined, threreturn stored ? { id: stored.id, name: ref.name, label: ref.label, kind: ref.kind, turn: null, width: ref.width, height: ref.height } : undefined;};const pixels = (ref: ImageRef) => index.byKey.get(ref.key)?.pixelSha256 ?? pixelHashOf(json.input, ref);- const current = side ? index.turns + 1 : currentTurnOf(index, turnId);- const copies = new Map([...index.copies.values()].filter((copy) => copy.turn === null || copy.turn < current).map((copy) => [copy.key, copy.of]));- let leaveOut = new Set(Object.keys(selection.unchecked));- // v0.8: feature lines in a task's own requests, not a side chat's or a subagent's.- const features = !origin && writesNotes(sessionsDir, threadId);- if (selection.auto) {- const pinned = selection.pinned ?? {};- for (const image of index.images) if (features && awaitingLine(index, image, current)) awaiting.add(image.key);- leaveOut = new Set(index.images.filter((image) => image.replaceable && image.turn !== null && image.turn < current && !pinned[image.key] && !awaiting.has(image.key)).map((image) => image.key));- }const rewritten = rewriteItems(json.input, describe, leaveOut, pixels, { lang: currentLang(), copies, features, ...(selection.auto ? { auto: { currentTurn: current } } : {}) });const { report } = rewritten;let items = rewritten.items;diff --git a/plugin/src/core/request-stats.ts b/plugin/src/core/request-stats.tsindex 77dd740..21c9717 100644--- a/plugin/src/core/request-stats.ts+++ b/plugin/src/core/request-stats.ts@@ -6,14 +6,14 @@import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";import { join } from "node:path";import { requestStatsDirOf } from "./paths.ts";+import { SESSION_KEY } from "./selection.ts";type Json = Record<string, any>;export type RequestStats = { latest: Json; lastHttp: Json | null };-const THREAD_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;-+// v0.11: under the session's key (selection.ts, sessionKeyOf), whatever the platform.export function readRequestStats(threadId: string, dir = requestStatsDirOf()): RequestStats | null {- if (!THREAD_ID.test(threadId)) return null;+ if (!SESSION_KEY.test(threadId)) return null;const file = join(dir, `${threadId}.json`);if (!existsSync(file)) return null;try {@@ -26,7 +26,7 @@ export function readRequestStats(threadId: string, dir = requestStatsDirOf()): R}export function recordRequest(threadId: string | null, stats: Json, dir = requestStatsDirOf()): void {- if (!threadId || !THREAD_ID.test(threadId)) return;+ if (!threadId || !SESSION_KEY.test(threadId)) return;mkdirSync(dir, { recursive: true });const file = join(dir, `${threadId}.json`);const next: RequestStats = { latest: stats, lastHttp: stats.transport === "http" ? stats : readRequestStats(threadId, dir)?.lastHttp ?? null };diff --git a/plugin/src/core/routes.ts b/plugin/src/core/routes.tsindex faed1cd..bf2df21 100644--- a/plugin/src/core/routes.ts+++ b/plugin/src/core/routes.ts@@ -5,9 +5,11 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";import { join } from "node:path";import { dataDir } from "./paths.ts";-export type Platform = "codex" | "claude";+// v0.11: a platform's name as its adapter registers it (platforms.ts): codex, claude, workbuddy, …+export type Platform = string;export type ProviderRoute = { id: string; platform: Platform; baseUrl: string };const ID = /^[a-f0-9]{32}$/;+export const PLATFORM_NAME = /^[a-z][a-z0-9-]{1,31}$/;export const routingPort = (enginePort: number): number => {if (!Number.isInteger(enginePort) || enginePort < 1 || enginePort > 65534) throw new Error("invalid engine port");@@ -27,7 +29,7 @@ export function validateBaseUrl(value: string, enginePort?: number): URL {}export function registerRoute(input: { platform: Platform; baseUrl: string }, root = dataDir()): ProviderRoute {- if (input.platform !== "codex" && input.platform !== "claude") throw new Error("invalid provider platform");+ if (typeof input.platform !== "string" || !PLATFORM_NAME.test(input.platform)) throw new Error("invalid provider platform");const baseUrl = validateBaseUrl(input.baseUrl).href;const dir = join(root, "routes");mkdirSync(dir, { recursive: true });@@ -52,7 +54,7 @@ export function readRoute(id: string, root = dataDir()): ProviderRoute | null {if (!ID.test(id)) return null;try {const value = JSON.parse(readFileSync(join(root, "routes", `${id}.json`), "utf8"));- if (value.id !== id || !["codex", "claude"].includes(value.platform)) return null;+ if (value.id !== id || typeof value.platform !== "string" || !PLATFORM_NAME.test(value.platform)) return null;return { id, platform: value.platform, baseUrl: validateBaseUrl(value.baseUrl).href };} catch { return null; }}diff --git a/plugin/src/core/selection.ts b/plugin/src/core/selection.tsindex 5fbd340..302aa69 100644--- a/plugin/src/core/selection.ts+++ b/plugin/src/core/selection.ts@@ -13,6 +13,7 @@// Input/Output: <data dir>/selection/<thread id>.json, e.g. { "threadId": "…", "unchecked": { "<key>": { "id": "IMG-003", "at": "…" } } },// plus "auto", "autoAt", "autoSince" and "pinned" (same shape as "unchecked") once automatic selection has been used.+import { createHash } from "node:crypto";import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";import { join } from "node:path";import { selectionDirOf } from "./paths.ts";@@ -21,11 +22,16 @@ export type Mark = { id: string; at: string };export type Selection = { threadId: string; unchecked: Record<string, Mark>; auto?: boolean; autoAt?: string; autoSince?: string; pinned?: Record<string, Mark>; inheritedFrom?: string };export const selectionDir = selectionDirOf();+// Codex and Claude Code name their sessions with UUIDs.export const THREAD_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;+// v0.11: the name of a session's files, whatever the platform calls it: a UUID as it is, any other id as 32 hex+// characters of its hash, which can be neither a path nor one of the files kept next to the selections.+export const SESSION_KEY = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{32})$/;+export const sessionKeyOf = (id: string): string => (THREAD_ID.test(id) ? id : createHash("sha256").update(id).digest("hex").slice(0, 32));const cache = new Map<string, { mtimeMs: number; size: number; selection: Selection }>();export function fileFor(threadId: string, dir: string): string {- if (!THREAD_ID.test(threadId)) throw new Error(`not a thread id: ${threadId}`);+ if (!SESSION_KEY.test(threadId)) throw new Error(`not a thread id: ${threadId}`);return join(dir, `${threadId}.json`);}diff --git a/plugin/src/core/session.ts b/plugin/src/core/session.tsindex 528339a..fe9253d 100644--- a/plugin/src/core/session.ts+++ b/plugin/src/core/session.ts@@ -31,3 +31,17 @@ export function awaitingLine(index: SessionIndex<IndexedImage>, image: IndexedImfor (let turn = image.turn + 1; turn < current; turn++) if (!index.interrupted.has(turn) && !index.skipped?.has(turn)) return false;return true;}++// v0.3: the copies the model fetched before the request's turn (key → the id each copies), left out in either mode.+export function earlierCopies(index: SessionIndex<IndexedImage>, current: number): Map<string, string> {+ return new Map([...index.copies.values()].filter((copy) => copy.turn === null || copy.turn < current).map((copy) => [copy.key, copy.of]));+}++// v0.3: what automatic selection leaves out of a request of turn `current`: the images of earlier turns that can become+// text and are not pinned. v0.10: where feature lines are written, an image waiting for its line (awaitingLine) is sent+// once more instead. v0.11: decided here for every platform (it was written out in the Codex and the Claude engine).+export function autoChoice(index: SessionIndex<IndexedImage>, current: number, pinned: Record<string, unknown>, features: boolean): { leaveOut: Set<string>; awaiting: Set<string> } {+ const awaiting = new Set(features ? index.images.filter((image) => awaitingLine(index, image, current)).map((image) => image.key) : []);+ const leaveOut = new Set(index.images.filter((image) => image.replaceable && image.turn !== null && image.turn < current && !pinned[image.key] && !awaiting.has(image.key)).map((image) => image.key));+ return { leaveOut, awaiting };+}diff --git a/plugin/src/engine.ts b/plugin/src/engine.tsindex 34fb63e..0320b8d 100644--- a/plugin/src/engine.ts+++ b/plugin/src/engine.ts@@ -1,5 +1,6 @@// Purpose: P3-2 — lifecycle of the engine (the local proxy): one instance per machine, `ensure` starts it in the-// background when it is not running, and it exits by itself once no Codex process is left.+// background when it is not running, and it exits by itself once no Codex process is left (v0.11: and no plugin service+// of any platform checks in).// v0.1-3 — an engine reports which code it runs (build and version). After a plugin update, the new plugin's `ensure`// asks the older engine still running to retire: it stops listening at once, finishes what is in flight, then exits,// and an engine of the new code takes the port.@@ -26,10 +27,11 @@ export function isEngineHealth(value: unknown): value is Health {return !!health && health.ok === true && health.service === ENGINE_SERVICE && typeof health.pid === "number";}-// Loopback only, and never through a system proxy: plain http.get on 127.0.0.1.-export async function engineHealth(port = DEFAULT_PORT, timeoutMs = 800): Promise<Health | null> {+// Loopback only, and never through a system proxy: plain http.get on 127.0.0.1. v0.11: `client`, the platform whose+// plugin service is asking (ensureEngine): the engine stays while such checks keep coming (watchForClients).+export async function engineHealth(port = DEFAULT_PORT, timeoutMs = 800, client?: string): Promise<Health | null> {try {- const response = await fetch(`http://127.0.0.1:${port}/__cam/health`, { signal: AbortSignal.timeout(timeoutMs) });+ const response = await fetch(`http://127.0.0.1:${port}/__cam/health`, { signal: AbortSignal.timeout(timeoutMs), ...(client ? { headers: { "x-cam-client": client } } : {}) });const body = await response.json();return isEngineHealth(body) ? body : null;} catch {@@ -108,10 +110,10 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));// The build of each runtime folder, read again once for every different build a running engine reports.const knownBuilds = new Map<string, { build: string | null; against: string | null | undefined }>();-export async function ensureEngine(options: { port?: number; waitMs?: number; extraArgs?: string[]; dir?: string; force?: boolean } = {}): Promise<{ state: EngineState; health: Health | null; replaced?: Health }> {+export async function ensureEngine(options: { port?: number; waitMs?: number; extraArgs?: string[]; dir?: string; force?: boolean; client?: string } = {}): Promise<{ state: EngineState; health: Health | null; replaced?: Health }> {const port = options.port ?? DEFAULT_PORT;const dir = options.dir ?? here;- const existing = await engineHealth(port);+ const existing = await engineHealth(port, 800, options.client);let replaced: Health | undefined;if (existing) {const known = knownBuilds.get(dir);@@ -166,10 +168,16 @@ export function listCodexProcesses(): Promise<number> {});}-// Exit only after `misses` checks in a row found no Codex (v0.4: and no Claude Code), so a restart does not stop the-// engine.-export function watchForCodex(onGone: () => void, options: { intervalMs?: number; misses?: number; count?: () => Promise<number> } = {}): () => void {- const count = options.count ?? listCodexProcesses;+// v0.11: a plugin service of any platform checks the engine every few seconds while its platform runs (plugin-server.ts);+// one checked within this long counts as a client still there.+export const CLIENT_SEEN_MS = 60_000;++// Exit only after `misses` checks in a row found no Codex (v0.4: and no Claude Code; v0.11: and no plugin service of+// another platform checking in, `lastClient`), so a restart does not stop the engine.+export function watchForClients(onGone: () => void, options: { intervalMs?: number; misses?: number; count?: () => Promise<number>; lastClient?: () => number } = {}): () => void {+ const processes = options.count ?? listCodexProcesses;+ const lastClient = options.lastClient ?? (() => 0);+ const count = async () => (Date.now() - lastClient() < CLIENT_SEEN_MS ? 1 : await processes());const needed = options.misses ?? 2;let missed = 0;const timer = setInterval(async () => {diff --git a/plugin/src/plugin-server.ts b/plugin/src/plugin-server.tsindex 17a43c5..891016a 100644--- a/plugin/src/plugin-server.ts+++ b/plugin/src/plugin-server.ts@@ -298,7 +298,7 @@ function main(): void {let engineState = "";const supervise = async () => {if (process.env.CAM_NO_ENGINE === "1") return;- const result = await ensureEngine({ port: enginePort });+ const result = await ensureEngine({ port: enginePort, client: HOST });const replaced = result.replaced ? { replacedPid: result.replaced.pid, replacedBuild: result.replaced.build ?? null } : {};if (result.state !== "running" || engineState !== "running") log({ event: "engine", host: HOST, state: result.state, enginePid: result.health?.pid ?? null, build: result.health?.build ?? null, ...replaced });engineState = result.state;diff --git a/plugin/src/proxy.ts b/plugin/src/proxy.tsindex ae1d0ae..351abce 100644--- a/plugin/src/proxy.ts+++ b/plugin/src/proxy.ts@@ -42,7 +42,7 @@ import { migrateDataOnce } from "./core/migrate-data.ts";import { claudeRequestStatsDirOf, claudeSelectionDirOf, dataDir, pixelCacheDirOf, selectionDirOf } from "./core/paths.ts";import { recordRequest } from "./core/request-stats.ts";import { readRoute, registerRoute, routeBaseUrl, routingPort, validateBaseUrl, type Platform } from "./core/routes.ts";-import { buildOf, DEFAULT_PORT, ENGINE_SERVICE, engineHealth, versionOf, watchForCodex } from "./engine.ts";+import { buildOf, DEFAULT_PORT, ENGINE_SERVICE, engineHealth, versionOf, watchForClients } from "./engine.ts";import { currentLang } from "./adapters/codex/language.ts";import { say } from "./core/messages.ts";import { ensureAutoDefault, ensureStableIds, selectionDir, stableIdsSince } from "./core/selection.ts";@@ -160,6 +160,9 @@ async function main(): Promise<void> {return vias.get(upstream)!;};const startedAt = new Date().toISOString();+ // v0.11: when a plugin service last checked in (engine.ts, watchForClients); a platform other than Codex and Claude+ // Code keeps the engine this way.+ let lastClient = 0;// v0.1-3: which code this engine runs, so the plugin can tell when an update needs a new engine.const build = buildOf();const version = versionOf();@@ -209,11 +212,11 @@ async function main(): Promise<void> {}};setInterval(() => checkPlugin(false), 15_000).unref();- watchForCodex(() => {+ watchForClients(() => {checkPlugin(true);- log({ at: new Date().toISOString(), event: "engine-exit", pid: process.pid, reason: "no Codex or Claude Code process left" });+ log({ at: new Date().toISOString(), event: "engine-exit", pid: process.pid, reason: "no Codex or Claude Code process left, and no plugin service checked in" });process.exit(0);- });+ }, { lastClient: () => lastClient });}// Open WebSocket connections per thread. Over WebSocket Codex only sends new items and the server keeps the rest,// so a thread with unchecked images must move to HTTP: its idle connections are closed, and its next upgrade gets 426.@@ -265,6 +268,7 @@ async function main(): Promise<void> {const onRequest = (req: http.IncomingMessage, res: http.ServerResponse) => {if (req.url === "/__cam/health") {+ if (req.headers["x-cam-client"]) lastClient = Date.now();res.writeHead(200, { "content-type": "application/json" });res.end(JSON.stringify({ ok: true, service: ENGINE_SERVICE, pid: process.pid, startedAt, port, build, version, routingVersion: 1, routingPort: providerPort, upstream: UPSTREAM_HOST, upstreams: [UPSTREAM_HOST, CLAUDE_HOST, OPENAI_API_HOST], via: via ? `${via.host}:${via.port}` : "direct" }));return;diff --git a/spike/src/engine.test.ts b/spike/src/engine.test.tsindex f2387e6..0918400 100644--- a/spike/src/engine.test.ts+++ b/spike/src/engine.test.ts@@ -13,7 +13,7 @@ import { dirname, join } from "node:path";import test from "node:test";import { setTimeout as sleep } from "node:timers/promises";import { fileURLToPath } from "node:url";-import { buildOf, compareVersions, countClaudeInPs, countClaudeProcesses, countCodexInPs, countCodexProcesses, engineHealth, ensureEngine, isEngineHealth, shouldReplace, watchForCodex, type Health } from "../../plugin/src/engine.ts";+import { buildOf, compareVersions, countClaudeInPs, countClaudeProcesses, countCodexInPs, countCodexProcesses, engineHealth, ensureEngine, isEngineHealth, shouldReplace, watchForClients, type Health } from "../../plugin/src/engine.ts";import { selectionDirOf } from "../../plugin/src/core/paths.ts";import { readRequestStats, recordRequest } from "../../plugin/src/core/request-stats.ts";import { autoDefaultSince, writeSelection } from "../../plugin/src/core/selection.ts";@@ -46,7 +46,7 @@ test("on macOS and Linux, Codex processes are counted from ps by executable nametest("the engine gives up only after consecutive checks without Codex", async () => {const run = (counts: number[]) => new Promise<number>((resolve) => {let calls = 0;- const stop = watchForCodex(() => resolve(calls), { intervalMs: 5, misses: 2, count: async () => counts[Math.min(calls++, counts.length - 1)] });+ const stop = watchForClients(() => resolve(calls), { intervalMs: 5, misses: 2, count: async () => counts[Math.min(calls++, counts.length - 1)] });setTimeout(() => { stop(); resolve(-1); }, 500);});assert.equal(await run([1, 0, 0]), 3);@@ -54,6 +54,15 @@ test("the engine gives up only after consecutive checks without Codex", async ()assert.equal(await run([1]), -1, "never exits while Codex runs");});+test("v0.11: a plugin service of another platform checking in keeps the engine without any Codex process", async () => {+ const run = (lastClient: () => number) => new Promise<string>((resolve) => {+ const stop = watchForClients(() => resolve("exited"), { intervalMs: 5, misses: 2, count: async () => 0, lastClient });+ setTimeout(() => { stop(); resolve("stayed"); }, 200);+ });+ assert.equal(await run(() => Date.now()), "stayed");+ assert.equal(await run(() => Date.now() - 10 * 60_000), "exited", "a check-in long ago does not count");+});+test("ensure starts one engine; a second start on the same port leaves it alone", async (t) => {const port = 20000 + Math.floor(Math.random() * 20000);const data = mkdtempSync(join(tmpdir(), "cam-engine-"));