A local plugin for image-heavy chats in Codex, Claude Code and other agents: earlier images stop going out with every message, and the model fetches the one it needs by its ID, also after compaction.
TypeScript ai-agentsclaude-codeclaude-code-plugincodexcodex-plugincontext-managementdeveloper-toolsimage-generationmcpmultimodal
// Purpose: P2 — the image timeline of one thread, read from its rollout (read-only): every image occurrence in
// history order with a stable id (IMG-001…), turn number, name, size and content hashes. The CLI and the proxy both
// derive it from the same files, so they agree on ids without sharing state.
// v0.1-11 — a panel read on a new process used to walk every rollout folder on each call and decode every PNG for its
// pixel fingerprint (1.2 s for 12 screenshots). Now the tree is walked in full once a minute and only the folders of
// today and yesterday in between; fingerprints can be shared between processes through a cache folder. Rollouts
// Codex compressed after a week without activity (.jsonl.zst) are read as well.
// v0.1-10 — the first line of recent rollouts, so a panel opened on a new chat can find the thread that started there.
// v0.1-12 — archived tasks (<CODEX_HOME>/archived_sessions) are looked up too: a fork's history may start in a page of
// a task the user has archived since. v0.1-13 — if that page is gone for good (its task deleted), the rest is read and
// the gap reported, instead of failing.
// v0.3 — an image the model fetched with our tool (cam_view_image) is kept apart as a copy of the original: no id.
// v0.8 — what the model noted about the images in record lines at the end of its replies (notes.ts), by image id.
// Input: the sessions directory and a thread id. Output: ThreadIndex (in memory; files are parsed incrementally).
import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
import { basename, dirname, join } from "node:path";
import { zstdDecompressSync } from "node:zlib";
import { findImages, type ImageRef } from "../protocols/openai-responses/images.ts";
import { addNotes, type Notes } from "../../core/notes.ts";
import { pixelHashOfData } from "../../core/pixels.ts";
import type { FetchedCopy, IndexedImage as SessionImage, SessionIndex } from "../../core/session.ts";
type Json = Record<string, any>;
// v0.11: the shape of the index and awaitingLine are the core's (core/session.ts); a Codex image also carries where it
// sits in its item (ImageRef).
export type IndexedImage = ImageRef & SessionImage;
export type ThreadIndex = SessionIndex<IndexedImage>;
export type Parsed = { offset: number; type: string; payload: Json; timestamp: string | null };
type FileCache = { size: number; mtimeMs: number; parsedTo: number; records: Parsed[] };
const SEGMENT_NAME = /^rollout-[0-9T:-]+-([0-9a-f-]{36})(?:_([0-9a-f-]{36}))?\.jsonl(?:\.zst)?$/;
const files = new Map<string, FileCache>();
function walk(directory: string): string[] {
let entries;
try { entries = readdirSync(directory, { withFileTypes: true }); } catch { return []; }
return entries.flatMap((entry) => {
const full = join(directory, entry.name);
return entry.isDirectory() ? walk(full) : entry.isFile() ? [full] : [];
});
}
function parseLines(buffer: Buffer, base: number, cache: FileCache): number {
let start = 0;
for (let end = buffer.indexOf(0x0a); end >= 0; end = buffer.indexOf(0x0a, start)) {
const text = buffer.toString("utf8", start, end).replace(/\r$/, "");
// A damaged line is skipped rather than failing the whole index; unknown images are sent unchanged anyway.
let record: Json | null = null;
if (text.trim()) { try { record = JSON.parse(text); } catch { record = null; } }
if (record) cache.records.push({ offset: base + start, type: record.type, payload: record.payload ?? {}, timestamp: typeof record.timestamp === "string" ? record.timestamp : null });
start = end + 1;
}
return start;
}
const missing = (plain: string) => Object.assign(new Error(`rollout missing: ${basename(plain)}`), { code: "ENOENT" });
// A rollout is kept under its plain name; Codex may have compressed it (.jsonl.zst), or turned a compressed one back
// into plain text to append to it.
function existing(plain: string): string | null {
return existsSync(plain) ? plain : existsSync(`${plain}.zst`) ? `${plain}.zst` : null;
}
// Rollouts are append-only: parse only the complete lines added since the last read. A compressed one does not change
// until Codex turns it back into plain text, so it is read whole once.
function records(plain: string): Parsed[] {
const file = existing(plain);
if (!file) throw missing(plain);
const stat = statSync(file);
let cache = files.get(file);
if (file.endsWith(".zst")) {
if (!cache || cache.size !== stat.size || cache.mtimeMs !== stat.mtimeMs) {
cache = { size: stat.size, mtimeMs: stat.mtimeMs, parsedTo: 0, records: [] };
cache.parsedTo = parseLines(zstdDecompressSync(readFileSync(file)), 0, cache);
files.set(file, cache);
}
return cache.records;
}
if (!cache || stat.size < cache.parsedTo) cache = { size: 0, mtimeMs: 0, parsedTo: 0, records: [] };
if (stat.size !== cache.size || stat.mtimeMs !== cache.mtimeMs) {
const length = stat.size - cache.parsedTo;
const buffer = Buffer.alloc(length);
const fd = openSync(file, "r");
try { readSync(fd, buffer, 0, length, cache.parsedTo); } finally { closeSync(fd); }
cache.parsedTo += parseLines(buffer, cache.parsedTo, cache);
cache.size = stat.size;
cache.mtimeMs = stat.mtimeMs;
files.set(file, cache);
}
return cache.records;
}
// Every rollout segment by id: a thread's first segment uses the thread id itself, later pages are named
// <thread>_<segment>. A fork's history_base points into its parent's segments, so bases are looked up globally.
// Codex files each rollout under <sessions>/YYYY/MM/DD of the day it was created (local time), so new threads, forks
// and later pages all land in today's folder: between full walks only today's and yesterday's folders (local and UTC)
// are listed again. Archived tasks move to <CODEX_HOME>/archived_sessions, which full walks include (a task's own
// folder wins). A file moved elsewhere is found by the next full walk, or at once when a read misses it (at most one
// such extra walk every 5 s, so a history whose page was deleted does not walk the tree on every poll).
type Tree = { walkedAt: number; missedAt: number; segments: Map<string, string>; threadOf: Map<string, string> };
const FULL_WALK_MS = 60_000;
const MISSED_WALK_MS = 5_000;
const trees = new Map<string, Tree>();
const archivedOf = (sessionsDir: string) => join(dirname(sessionsDir), "archived_sessions");
export function recentFolders(sessionsDir: string, now = Date.now()): string[] {
const pad = (n: number) => String(n).padStart(2, "0");
const folders = new Set<string>();
for (const at of [now, now - 86_400_000]) {
const day = new Date(at);
folders.add(join(sessionsDir, String(day.getFullYear()), pad(day.getMonth() + 1), pad(day.getDate())));
folders.add(join(sessionsDir, String(day.getUTCFullYear()), pad(day.getUTCMonth() + 1), pad(day.getUTCDate())));
}
return [...folders];
}
function addSegment(tree: Tree, file: string): void {
const match = SEGMENT_NAME.exec(basename(file));
if (!match) return;
tree.segments.set(match[2] ?? match[1], file.replace(/\.zst$/, ""));
tree.threadOf.set(match[2] ?? match[1], match[1]);
}
function allSegments(sessionsDir: string, missed = false): Tree {
const now = Date.now();
let tree = trees.get(sessionsDir);
const again = missed && (!tree || now - tree.missedAt >= MISSED_WALK_MS);
if (!tree || again || now - tree.walkedAt > FULL_WALK_MS) {
tree = { walkedAt: now, missedAt: again ? now : tree?.missedAt ?? 0, segments: new Map(), threadOf: new Map() };
for (const file of [...walk(archivedOf(sessionsDir)), ...walk(sessionsDir)]) addSegment(tree, file);
trees.set(sessionsDir, tree);
return tree;
}
for (const folder of recentFolders(sessionsDir, now)) {
let names: string[];
try { names = readdirSync(folder); } catch { continue; }
for (const name of names) addSegment(tree, join(folder, name));
}
return tree;
}
// Reads that miss a file (moved or deleted since the last full walk) walk the tree again once.
function withSegments<T>(sessionsDir: string, read: (tree: Tree) => T): T {
try { return read(allSegments(sessionsDir)); } catch (error) {
if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") throw error;
return read(allSegments(sessionsDir, true));
}
}
const baseOf = (file: string): Json | null => (records(file)[0]?.type === "session_meta" ? records(file)[0].payload.history_base ?? null : null);
// With `missing` (the second look, after walking the tree again), a page the history starts in that is still not found
// is left out and its id added to `missing`; without it, the miss is thrown for the caller to look again.
function effective(segments: Map<string, string>, segmentId: string, endOffset = Number.POSITIVE_INFINITY, missing: string[] | null = null): Parsed[] {
const file = segments.get(segmentId);
// Not listed (yet): the caller walks the tree again once.
if (!file) throw Object.assign(new Error(`rollout segment not found: ${segmentId}`), { code: "ENOENT" });
const base = baseOf(file);
let inherited: Parsed[] = [];
if (base) {
try { inherited = effective(segments, base.thread_id, base.end_byte_offset, missing); } catch (error) {
if (!missing || (error as NodeJS.ErrnoException)?.code !== "ENOENT") throw error;
if (!missing.includes(base.thread_id)) missing.push(base.thread_id);
}
}
return [...inherited, ...records(file).filter((record) => record.offset < endOffset)];
}
export function imageData(items: Json[], ref: ImageRef): string | null {
const item = items[ref.item];
if (ref.part === null) return typeof item?.result === "string" ? item.result : null;
const url = (item?.content ?? item?.output)?.[ref.part]?.image_url;
return typeof url === "string" ? url.slice(url.indexOf(",") + 1) : null;
}
export function pixelHashOf(items: Json[], ref: ImageRef): string | null {
const value = imageData(items, ref);
return value === null ? null : pixelHashOfData(ref, value);
}
export function buildIndex(threadId: string, history: Parsed[]): ThreadIndex {
const turnNumbers = new Map<string, number>();
const items: Json[] = [];
const activeTurn: Array<string | null> = [];
const notes = new Map<string, Notes>();
let current: string | null = null;
// v0.10: the latest compaction of Codex: how many items came before it, and the images of the history it kept.
let compaction: { before: number; kept: Set<string> } | null = null;
const interrupted = new Set<number>();
for (const record of history) {
if (record.type === "compacted" && Array.isArray(record.payload.replacement_history)) {
compaction = { before: items.length, kept: new Set(findImages(record.payload.replacement_history).map((ref) => ref.key)) };
}
if (record.type === "event_msg" && record.payload.type === "task_started" && record.payload.turn_id) {
current = record.payload.turn_id;
if (!turnNumbers.has(current!)) turnNumbers.set(current!, turnNumbers.size + 1);
}
if (record.type === "event_msg" && record.payload.type === "turn_aborted" && current) interrupted.add(turnNumbers.get(current)!);
if (record.type !== "response_item") continue;
items.push(record.payload);
activeTurn.push(current);
// v0.8: only the model's own replies.
const { type, role, content } = record.payload;
if (type === "message" && role === "assistant" && Array.isArray(content)) {
const turn = current ? turnNumbers.get(current) ?? null : null;
for (const part of content) if (part?.type === "output_text" && typeof part.text === "string") addNotes(notes, part.text, turn);
}
}
const images: IndexedImage[] = [];
const byKey = new Map<string, IndexedImage>();
const copies = new Map<string, FetchedCopy>();
for (const ref of findImages(items)) {
if (byKey.has(ref.key) || copies.has(ref.key)) continue;
const turnId = ref.turnId ?? activeTurn[ref.item];
const turn = turnId ? turnNumbers.get(turnId) ?? null : null;
const pixelSha256 = pixelHashOf(items, ref);
// v0.3: what our fetch tool brought back is the same image again, not a new occurrence, so it takes no id (the ids
// after it stay as they were) and the panel lists it on the original's row. Its content must match that id's.
const original = ref.fetchedId ? images.find((image) => image.id === ref.fetchedId) : undefined;
if (original && (original.contentId === ref.contentId || (pixelSha256 !== null && original.pixelSha256 === pixelSha256))) {
copies.set(ref.key, { key: ref.key, of: original.id, turn });
continue;
}
const entry: IndexedImage = { ...ref, id: `IMG-${String(images.length + 1).padStart(3, "0")}`, turn, bytes: Math.floor((ref.base64Chars * 3) / 4), pixelSha256 };
images.push(entry);
byKey.set(ref.key, entry);
}
// An image whose own name is unknown (e.g. viewed through a variable path) takes the name of an identical one.
for (const entry of images) {
if (entry.name) continue;
const twin = images.find((other) => other.name && (other.contentId === entry.contentId || (entry.pixelSha256 !== null && other.pixelSha256 === entry.pixelSha256)));
if (twin) entry.name = twin.name;
}
if (compaction) for (const entry of images) if (entry.item < compaction.before && !compaction.kept.has(entry.key)) entry.compacted = true;
return { threadId, images, byKey, turns: turnNumbers.size, turnNumbers, copies, notes, items, interrupted };
}
// The first line alone (session_meta can be long), read in chunks without parsing the rest of the file.
function firstLine(file: string): string {
const fd = openSync(file, "r");
try {
const chunks: Buffer[] = [];
const chunk = Buffer.alloc(64 * 1024);
for (let position = 0; ; position += chunk.length) {
const read = readSync(fd, chunk, 0, chunk.length, position);
if (read <= 0) break;
const end = chunk.subarray(0, read).indexOf(0x0a);
chunks.push(Buffer.from(chunk.subarray(0, end >= 0 ? end : read)));
if (end >= 0 || read < chunk.length) break;
}
return Buffer.concat(chunks).toString("utf8");
} finally {
closeSync(fd);
}
}
// The first record of a rollout: read from a plain file without parsing the rest; a compressed one is read whole.
function firstRecord(plain: string): Parsed | null {
if (!existsSync(plain)) return records(plain)[0] ?? null;
try {
const raw = JSON.parse(firstLine(plain));
return { offset: 0, type: raw.type, payload: raw.payload ?? {}, timestamp: typeof raw.timestamp === "string" ? raw.timestamp : null };
} catch { return null; }
}
const origins = new Map<string, { parent: string; at: string } | null>();
// A forked thread's session_meta names the thread it came from and when it was forked. Cached: it never changes.
export function forkOrigin(sessionsDir: string, threadId: string): { parent: string; at: string } | null {
if (origins.has(threadId)) return origins.get(threadId)!;
const meta = withSegments(sessionsDir, ({ segments }) => {
const first = segments.get(threadId);
return first ? { found: true, record: firstRecord(first) } : { found: false, record: null };
});
// No rollout yet: do not cache, it may appear once the thread starts. v0.7: nor while its first line is not written
// yet: Codex creates the file a moment before, and a request can come in between.
if (!meta.found || !meta.record) return null;
const origin = meta.record.type === "session_meta" && meta.record.payload.forked_from_id ? { parent: meta.record.payload.forked_from_id, at: meta.record.payload.timestamp ?? "" } : null;
origins.set(threadId, origin);
return origin;
}
const spawners = new Map<string, string | null>();
// v0.7: the thread that started a subagent without history (session_meta source.subagent.thread_spawn); null for
// anything else: the user's tasks, forks (a subagent started with history is one), Codex's background tasks (review,
// compaction, memory). Cached once the rollout's first line is there (as forkOrigin): it never changes.
export function spawnParent(sessionsDir: string, threadId: string): string | null {
const key = `${sessionsDir}\0${threadId}`;
if (spawners.has(key)) return spawners.get(key)!;
const meta = withSegments(sessionsDir, ({ segments }) => {
const first = segments.get(threadId);
return first ? { found: true, record: firstRecord(first) } : { found: false, record: null };
});
if (!meta.found || !meta.record) return null;
const payload = meta.record.type === "session_meta" ? meta.record.payload : null;
const spawn = payload?.source?.subagent?.thread_spawn;
const parent = payload && !payload.forked_from_id && spawn ? spawn.parent_thread_id ?? payload.parent_thread_id : null;
spawners.set(key, typeof parent === "string" ? parent : null);
return spawners.get(key)!;
}
const starts = new Map<string, string | null>();
// v0.6: when a thread started, from its session_meta; a fork counts from the thread it came from (followed to the
// first one; the fork's own time when that one is gone). null while the thread has no rollout, or no first line yet.
// Cached once known.
export function threadStartedAt(sessionsDir: string, threadId: string, depth = 0): string | null {
const origin = forkOrigin(sessionsDir, threadId);
if (origin && depth < 16) return threadStartedAt(sessionsDir, origin.parent, depth + 1) ?? (origin.at || null);
const key = `${sessionsDir}\0${threadId}`;
if (starts.has(key)) return starts.get(key)!;
const meta = withSegments(sessionsDir, ({ segments }) => {
const first = segments.get(threadId);
return first ? { found: true, record: firstRecord(first) } : { found: false, record: null };
});
if (!meta.found || !meta.record) return null;
const at = (meta.record.type === "session_meta" ? meta.record.payload.timestamp : null) ?? meta.record.timestamp ?? null;
starts.set(key, typeof at === "string" ? at : null);
return starts.get(key)!;
}
// Whether the thread has a rollout at all: a thread Codex prepared for a new chat has none until its first message.
export function hasRollout(sessionsDir: string, threadId: string): boolean {
return allSegments(sessionsDir).segments.has(threadId);
}
// v0.7: the task a side chat came from. Codex keeps a side chat in memory only: its requests and tool calls name the
// task it was forked from (forked_from_thread_id), and it has no rollout of its own (a fork the user made has one, and
// follows the fork rules). null for any other thread.
export function sideChatOrigin(threadId: string, forkedFrom: string | null | undefined, sessionsDir: string): string | null {
return forkedFrom && forkedFrom !== threadId && !hasRollout(sessionsDir, threadId) ? forkedFrom : null;
}
// The thread's session_meta (where it came from: source, thread_source, forked_from_id, parent_thread_id), or null.
export function sessionMetaOf(sessionsDir: string, threadId: string): Json | null {
return withSegments(sessionsDir, ({ segments }) => {
const first = segments.get(threadId);
const record = first ? firstRecord(first) : null;
return record?.type === "session_meta" ? record.payload : null;
});
}
const noteTakers = new Map<string, boolean>();
// v0.8: whether a thread's model writes feature lines (docs/v0.8/plan.md): the user's own tasks and their forks, not a
// subagent (started with history or without) or one of Codex's internal tasks. Without a first line yet, yes, and not
// cached: a subagent's first request can come before its rollout is written (v0.7); a user's task's rarely does.
export function writesNotes(sessionsDir: string, threadId: string): boolean {
const key = `${sessionsDir}\0${threadId}`;
if (noteTakers.has(key)) return noteTakers.get(key)!;
const meta = sessionMetaOf(sessionsDir, threadId);
if (!meta) return true;
const source = meta.source;
const own = !(source && typeof source === "object" && ("subagent" in source || "internal" in source));
noteTakers.set(key, own);
return own;
}
// Threads whose rollout was first written at or after `since` (ms), with their session_meta, newest last. Only
// today's and yesterday's folders are looked at. For a thread started by a message, the first line is written when
// that message is sent (Codex writes rollouts lazily), so its timestamp tells when the thread started.
export function threadsStartedSince(sessionsDir: string, since: number): Array<{ threadId: string; startedAt: number; meta: Json }> {
const found: Array<{ threadId: string; startedAt: number; meta: Json }> = [];
for (const folder of recentFolders(sessionsDir)) {
let names: string[];
try { names = readdirSync(folder); } catch { continue; }
for (const name of names) {
const match = SEGMENT_NAME.exec(name);
// First segments only (named after the thread); plain text, since a compressed rollout is a week old.
if (!match || match[2] || name.endsWith(".zst")) continue;
const file = join(folder, name);
try { if (statSync(file).mtimeMs < since) continue; } catch { continue; }
const record = firstRecord(file);
if (record?.type !== "session_meta") continue;
const startedAt = Date.parse(record.timestamp ?? record.payload.timestamp ?? "");
if (Number.isFinite(startedAt) && startedAt >= since && !found.some((entry) => entry.threadId === match[1])) found.push({ threadId: match[1], startedAt, meta: record.payload });
}
}
return found.sort((a, b) => a.startedAt - b.startedAt);
}
// The thread's model-visible history records, following paginated segments back through history_base.
// A page the history starts in may be gone for good (its task deleted rather than archived): after walking the tree
// again, what is left is returned and the missing page ids are added to `missing`. The thread's own latest page must
// be there.
export function readThreadHistory(sessionsDir: string, threadId: string, missing: string[] = []): Parsed[] {
const read = ({ segments, threadOf }: Tree, gaps: string[] | null) => {
const own = [...segments.keys()].filter((id) => threadOf.get(id) === threadId);
if (!own.length) throw new Error(`no rollout found for thread ${threadId}`);
const bases = new Set(own.map((id) => baseOf(segments.get(id)!)?.thread_id).filter(Boolean));
const heads = own.filter((id) => !bases.has(id));
if (heads.length !== 1) throw new Error(`expected one latest rollout segment for ${threadId}, found ${heads.length}`);
return effective(segments, heads[0], Number.POSITIVE_INFINITY, gaps);
};
try { return read(allSegments(sessionsDir), null); } catch (error) {
if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") throw error;
return read(allSegments(sessionsDir, true), missing);
}
}
export function loadThreadIndex(sessionsDir: string, threadId: string): ThreadIndex {
return buildIndex(threadId, readThreadHistory(sessionsDir, threadId));
}