Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(collector): follow long turns until they settle and keep their messages

ogsamrat014d22cc895a
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index 584696e9..fdb20ee5 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -141,7 +141,7 @@ export class CaptureHost {
/** The engine accepted a collected request: follow the session until its turn settles. */
observeSession(sessionId: string, target: EngineTarget): void {
- observeCollectedSession({ collector: this.collector, archive: this.archive, observers: this.observers, sessionId, target });
+ void observeCollectedSession({ collector: this.collector, archive: this.archive, observers: this.observers, sessionId, target });
}
/** The session was deleted in the engine. */
diff --git a/apps/server/src/collector-observer.test.ts b/apps/server/src/collector-observer.test.ts
new file mode 100644
index 00000000..8cf81202
--- /dev/null
+++ b/apps/server/src/collector-observer.test.ts
@@ -0,0 +1,425 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { zstdDecompressSync } from "node:zlib";
+
+import {
+ createSessionObservers,
+ observeCollectedSession,
+ type EngineTarget,
+ type ObservedCollector,
+ type ObserverTiming,
+ type SessionObservers,
+} from "./collector-observer.js";
+import { WorkspaceCollector, type CollectorChildSession } from "./workspace-collector.js";
+
+/**
+ * The turn observer against a fake engine, on a fake clock: a turn is
+ * followed for as long as it runs (hours of it pass without real waiting), an
+ * engine that does not answer is waited out, and messages a turn could not
+ * flush go out with the next settled one, over several flushes when needed.
+ */
+
+const SESSION = "ses_observed_0001";
+const HOUR = 60 * 60_000;
+const MIB = 1024 * 1024;
+
+const cleanups: Array<() => void | Promise<void>> = [];
+afterEach(async () => {
+ while (cleanups.length) await cleanups.pop()?.();
+});
+
+type EngineMessage = { info: Record<string, unknown>; parts: unknown[] };
+
+function message(id: string, role: "user" | "assistant", text = `${role} ${id}`): EngineMessage {
+ return {
+ info: {
+ id,
+ sessionID: SESSION,
+ role,
+ time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 },
+ ...(role === "assistant" ? { providerID: "omnirush", modelID: "gpt-6-astra", mode: "build" } : {}),
+ },
+ parts: [{ id: `${id}_part`, messageID: id, sessionID: SESSION, type: "text", text }],
+ };
+}
+
+/** A long answer: `mib` MiB of agent output. */
+function longAnswer(id: string, mib: number): EngineMessage {
+ return message(id, "assistant", "agent output line\n".repeat(Math.floor((mib * MIB) / 18)));
+}
+
+/** The engine's message paging (`?limit=&before=`): the newest `limit` messages before the cursor, oldest first. */
+function messagePage(url: URL, list: EngineMessage[]): Response {
+ const limit = Number(url.searchParams.get("limit") ?? 0);
+ if (!limit) return Response.json(list);
+ const end = url.searchParams.has("before") ? Number(url.searchParams.get("before")) : list.length;
+ const start = Math.max(0, end - limit);
+ return Response.json(list.slice(start, end), start > 0 ? { headers: { "X-Next-Cursor": String(start) } } : {});
+}
+
+/** A fake v1 engine holding one session: its status, its messages, and hooks on its reads. */
+function startEngine() {
+ const control = {
+ status: (): string => "idle",
+ messages: [] as EngineMessage[],
+ /** Real milliseconds the next status reads take to answer, one entry per read. */
+ statusDelaysMs: [] as number[],
+ statusReads: 0,
+ /** Runs once a page of messages was read. */
+ onMessagesRead: null as (() => void) | null,
+ };
+ const server = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ async fetch(request) {
+ const url = new URL(request.url);
+ if (url.pathname === "/session/status") {
+ control.statusReads += 1;
+ const delay = control.statusDelaysMs.shift();
+ if (delay) await Bun.sleep(delay);
+ const type = control.status();
+ return Response.json(type === "idle" ? {} : { [SESSION]: { type } });
+ }
+ if (url.pathname === `/session/${SESSION}/message`) {
+ const page = messagePage(url, control.messages);
+ control.onMessagesRead?.();
+ return page;
+ }
+ if (url.pathname === `/session/${SESSION}/children`) return Response.json([]);
+ return Response.json({ code: "not_found" }, { status: 404 });
+ },
+ });
+ cleanups.push(() => server.stop(true));
+ const target: EngineTarget = { baseUrl: `http://127.0.0.1:${server.port}`, headers: [["authorization", "Basic ZW5naW5l"]], search: "", engine: "v1" };
+ return { control, server, target };
+}
+
+/**
+ * A clock that only moves when the observer waits: each wait passes at once,
+ * adds its length to the time and runs whatever was scheduled up to then.
+ */
+function fakeClock() {
+ const start = Date.parse("2026-09-23T07:34:00.000Z");
+ let now = start;
+ const scheduled: Array<{ at: number; run: () => void }> = [];
+ const waits: number[] = [];
+ const timing: Partial<ObserverTiming> = {
+ now: () => now,
+ sleep: async (ms, signal) => {
+ if (signal.aborted) throw signal.reason;
+ waits.push(ms);
+ now += ms;
+ for (const task of scheduled.filter((entry) => entry.at <= now)) {
+ scheduled.splice(scheduled.indexOf(task), 1);
+ task.run();
+ }
+ await new Promise((resolvePromise) => setImmediate(resolvePromise));
+ if (signal.aborted) throw signal.reason;
+ },
+ };
+ return {
+ timing,
+ waits,
+ elapsed: () => now - start,
+ /** Runs `run` once the clock reaches `offsetMs` past its start. */
+ at: (offsetMs: number, run: () => void) => { scheduled.push({ at: start + offsetMs, run }); },
+ };
+}
+
+type Entry =
+ | { kind: "trace"; type: string; data?: unknown }
+ | { kind: "snapshot"; trigger: string }
+ | { kind: "flush"; final?: unknown };
+
+/** Records what the observer hands the collector, in order. */
+class FakeCollector implements ObservedCollector {
+ readonly enabled = true;
+ checkpoint: string | undefined;
+ readonly entries: Entry[] = [];
+
+ async sessionCheckpoint(): Promise<{ resumed: boolean; lastMessageId?: string }> {
+ return { resumed: false, ...(this.checkpoint ? { lastMessageId: this.checkpoint } : {}) };
+ }
+
+ async setSessionCheckpoint(_sessionId: string, messageId: string): Promise<void> {
+ this.checkpoint = messageId;
+ }
+
+ recordTrace(_sessionId: string, type: string, data?: unknown): void {
+ this.entries.push({ kind: "trace", type, ...(data === undefined ? {} : { data }) });
+ }
+
+ recordSessionModel(): void {}
+
+ async childCheckpoints(): Promise<Record<string, string>> {
+ return {};
+ }
+
+ async childSessionIds(): Promise<string[]> {
+ return [];
+ }
+
+ recordChildSession(_sessionId: string, _child: CollectorChildSession): void {}
+
+ captureSnapshot(_sessionId: string, trigger: "prompt" | "turn_completed"): void {
+ this.entries.push({ kind: "snapshot", trigger });
+ }
+
+ flushTrace(_sessionId: string, finalTrace?: unknown): void {
+ this.entries.push({ kind: "flush", ...(finalTrace === undefined ? {} : { final: finalTrace }) });
+ }
+
+ /** The entries as short labels: trace types, `snapshot:<trigger>`, `flush` and `flush:turn` (with the turn's messages). */
+ labels(): string[] {
+ return this.entries.map((entry) => {
+ if (entry.kind === "trace") return entry.type;
+ if (entry.kind === "snapshot") return `snapshot:${entry.trigger}`;
+ return entry.final === undefined ? "flush" : "flush:turn";
+ });
+ }
+
+ trace(type: string): unknown[] {
+ return this.entries.flatMap((entry) => (entry.kind === "trace" && entry.type === type ? [entry.data] : []));
+ }
+
+ /** The message ids each turn.completed flush carried. */
+ turnMessageIds(): string[][] {
+ return this.entries.flatMap((entry) => (entry.kind === "flush" && entry.final !== undefined
+ ? [((entry.final as { messages: EngineMessage[] }).messages).map((each) => String(each.info.id))]
+ : []));
+ }
+}
+
+function fakeArchive() {
+ const turns: Array<unknown[] | null> = [];
+ return { turns, turnCompleted: (_sessionId: string, messages: unknown) => { turns.push(messages as unknown[] | null); } };
+}
+
+function observe(input: { collector: ObservedCollector; archive: ReturnType<typeof fakeArchive>; observers: SessionObservers; target: EngineTarget; timing: Partial<ObserverTiming> }) {
+ return observeCollectedSession({ ...input, sessionId: SESSION });
+}
+
+describe("collector observer", () => {
+ test("a turn busy for three hours, past the old one-hour cap, gets its messages, snapshot and archive delta, with status reads easing off", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ engine.control.messages = [message("msg_user_0001", "user", "Work on issue 42")];
+ engine.control.status = () => (clock.elapsed() < 3 * HOUR ? "busy" : "idle");
+ clock.at(3 * HOUR, () => engine.control.messages.push(message("msg_assistant_0001", "assistant", "Fixed issue 42")));
+
+ await observe({ collector, archive, observers: createSessionObservers(), target: engine.target, timing: clock.timing });
+
+ expect(collector.labels()).not.toContain("session.observer_timeout");
+ expect(collector.labels()).not.toContain("session.observer_failed");
+ expect(collector.labels()).toEqual(["session.idle", "snapshot:turn_completed", "flush:turn"]);
+ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"]]);
+ expect(collector.checkpoint).toBe("msg_assistant_0001");
+ expect(archive.turns).toHaveLength(1);
+ expect(archive.turns[0]).toHaveLength(2);
+ expect(clock.elapsed()).toBeGreaterThanOrEqual(3 * HOUR);
+ // A read a second for the first two minutes, then less often, at most every 10 s:
+ // about 1,250 reads over three hours instead of 10,800.
+ expect(clock.waits.slice(0, 120).every((ms) => ms === 1_000)).toBe(true);
+ expect(Math.max(...clock.waits)).toBe(10_000);
+ expect(engine.control.statusReads).toBeLessThan(1_300);
+ });
+
+ test("a status read that times out twice, then answers, does not end the observation", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ engine.control.status = () => (clock.elapsed() < 5 * 60_000 ? "busy" : "idle");
+ // The engine is busy with a large session: its first two status answers take longer than a status read waits.
+ engine.control.statusDelaysMs = [400, 400];
+
+ await observe({ collector, archive, observers: createSessionObservers(), target: engine.target, timing: { ...clock.timing, statusTimeoutMs: 50 } });
+
+ expect(collector.labels()).toEqual(["session.idle", "snapshot:turn_completed", "flush:turn"]);
+ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"]]);
+ expect(archive.turns).toHaveLength(1);
+ // The failed reads backed off (1 s, then 2 s) before the next one.
+ expect(clock.waits.slice(0, 3)).toEqual([1_000, 1_000, 2_000]);
+ });
+
+ test("an engine restarting mid-turn is waited out and noted once past a minute; a request that finds the session followed points it at the new engine", async () => {
+ const clock = fakeClock();
+ const before = startEngine();
+ const after = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ before.control.messages = [message("msg_user_0001", "user")];
+ before.control.status = () => "busy";
+ after.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ after.control.status = () => (clock.elapsed() < 40 * 60_000 ? "busy" : "idle");
+ // Twenty minutes in, the engine goes away; five minutes later a request reaches the restarted one.
+ clock.at(20 * 60_000, () => void before.server.stop(true));
+ let rejoined: Promise<void> | null = null;
+ clock.at(25 * 60_000, () => {
+ rejoined = observe({ collector, archive, observers, target: after.target, timing: clock.timing });
+ });
+
+ const observation = observe({ collector, archive, observers, target: before.target, timing: clock.timing });
+ await observation;
+
+ expect(rejoined as Promise<void> | null).toBe(observation);
+ expect(collector.labels()).toEqual([
+ "session.engine_unavailable",
+ "session.engine_recovered",
+ "session.idle",
+ "snapshot:turn_completed",
+ "flush:turn",
+ ]);
+ const [unavailable] = collector.trace("session.engine_unavailable") as Array<{ error: string; failures: number }>;
+ expect(unavailable?.failures).toBeGreaterThan(1);
+ const [recovered] = collector.trace("session.engine_recovered") as Array<{ unavailable_ms: number }>;
+ expect(recovered?.unavailable_ms).toBeGreaterThanOrEqual(4 * 60_000);
+ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"]]);
+ expect(archive.turns).toHaveLength(1);
+ expect(observers.sessions.size).toBe(0);
+ });
+
+ test("a prompt sent while the turn settles is followed as a turn of its own", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ let busyUntil = 2 * 60_000;
+ engine.control.status = () => (clock.elapsed() < busyUntil ? "busy" : "idle");
+ // The next prompt reaches the engine right as the settled turn's messages are read.
+ engine.control.onMessagesRead = () => {
+ engine.control.onMessagesRead = null;
+ busyUntil = clock.elapsed() + 5 * 60_000;
+ engine.control.messages.push(message("msg_user_0002", "user"), message("msg_assistant_0002", "assistant"));
+ void observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
+ };
+
+ await observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
+
+ expect(collector.labels()).toEqual([
+ "session.idle", "snapshot:turn_completed", "flush:turn",
+ "session.idle", "snapshot:turn_completed", "flush:turn",
+ ]);
+ expect(collector.turnMessageIds()).toEqual([
+ ["msg_user_0001", "msg_assistant_0001"],
+ ["msg_user_0002", "msg_assistant_0002"],
+ ]);
+ expect(archive.turns.map((turn) => turn?.length)).toEqual([2, 4]);
+ });
+
+ test("the server stopping mid-turn ends the observation quietly and leaves the checkpoint where it was", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ collector.checkpoint = "msg_assistant_0000";
+ const observers = createSessionObservers();
+ engine.control.status = () => "busy";
+ clock.at(2 * HOUR, () => observers.controller.abort());
+ await observe({ collector, archive: fakeArchive(), observers, target: engine.target, timing: clock.timing });
+ expect(collector.entries).toEqual([]);
+ expect(collector.checkpoint).toBe("msg_assistant_0000");
+ });
+});
+
+describe("collector observer with the collector", () => {
+ type Envelope = {
+ snapshot_type: string;
+ files: Array<{ path: string; content: string }>;
+ trace?: Array<{ type: string; data?: Record<string, unknown> }>;
+ };
+
+ test("a turn left unsettled past the safety bound leaves its messages to the next settled turn, which sends them over several traces and counts what exceeds the backlog", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-observer-"));
+ cleanups.push(() => rm(root, { recursive: true, force: true }));
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const uploads: Envelope[] = [];
+ const collector = new WorkspaceCollector({
+ upload: async (_sessionId, compressed) => {
+ uploads.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ changeDebounceMs: 60_000,
+ fallbackScanMs: 60_000,
+ });
+ cleanups.push(() => collector.stop());
+ collector.startSession(SESSION, "workspace-observer", root);
+ const engine = startEngine();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ const traces = () => uploads.filter((upload) => upload.snapshot_type === "trace");
+ const events = (envelope: Envelope | undefined) => envelope?.trace ?? [];
+ const document = (envelope: Envelope) => JSON.parse(envelope.files[0]!.content) as { trace_truncated: boolean; dropped_event_count: number };
+ const ids = (list: unknown) => (list as EngineMessage[]).map((each) => String(each.info.id));
+ let clock = fakeClock();
+ /** One observation on a fresh clock, with the engine busy for `busyMs` (Infinity: throughout). */
+ const turn = async (busyMs: number, maxTurnMs?: number) => {
+ clock = fakeClock();
+ const current = clock;
+ engine.control.status = () => (current.elapsed() < busyMs ? "busy" : "idle");
+ await observe({ collector, archive, observers, target: engine.target, timing: { ...current.timing, ...(maxTurnMs ? { maxTurnMs } : {}) } });
+ await collector.idle(SESSION);
+ };
+
+ // Turn 1 settles normally: the checkpoint is its answer.
+ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ await turn(60_000);
+ expect((await collector.sessionCheckpoint(SESSION)).lastMessageId).toBe("msg_assistant_0001");
+
+ // Turn 2 is still running when the observer's safety bound (two hours here) runs out.
+ engine.control.messages.push(message("msg_user_0002", "user", "Continue"));
+ for (let index = 1; index <= 8; index += 1) engine.control.messages.push(longAnswer(`msg_long_${String(index).padStart(4, "0")}`, 3));
+ await turn(Infinity, 2 * HOUR);
+ const timedOut = traces().at(-1)!;
+ expect(events(timedOut).map((event) => event.type)).toContain("session.observer_timeout");
+ expect(events(timedOut).map((event) => event.type)).not.toContain("turn.completed");
+ expect(events(timedOut).filter((event) => event.type === "collector.trigger").map((event) => event.data?.trigger)).toEqual(["turn_completed"]);
+ expect(archive.turns).toHaveLength(1);
+ // Its messages were not sent, so the checkpoint did not move.
+ expect((await collector.sessionCheckpoint(SESSION)).lastMessageId).toBe("msg_assistant_0001");
+
+ // Turn 3 ("continue") settles: 24 MiB of turn 2 plus its own messages go out in four traces of at most 8 MiB of messages.
+ const sent = traces().length;
+ engine.control.messages.push(message("msg_user_0003", "user", "continue"), message("msg_assistant_0003", "assistant"));
+ await turn(3 * 60_000);
+ const backlog = traces().slice(sent);
+ expect(backlog).toHaveLength(4);
+ const parts = backlog.slice(0, 3).map((envelope) => events(envelope).find((event) => event.type === "turn.messages")?.data);
+ expect(parts.map((part) => [part?.part, part?.parts])).toEqual([[1, 4], [2, 4], [3, 4]]);
+ const completed = events(backlog[3]).find((event) => event.type === "turn.completed")?.data;
+ expect([...parts.flatMap((part) => ids(part?.messages)), ...ids(completed?.messages)]).toEqual([
+ "msg_user_0002",
+ ...Array.from({ length: 8 }, (_, index) => `msg_long_${String(index + 1).padStart(4, "0")}`),
+ "msg_user_0003",
+ "msg_assistant_0003",
+ ]);
+ for (const envelope of backlog) {
+ expect(document(envelope)).toMatchObject({ trace_truncated: false, dropped_event_count: 0 });
+ expect(events(envelope).map((event) => event.type)).not.toContain("session.messages_omitted");
+ }
+ expect((await collector.sessionCheckpoint(SESSION)).lastMessageId).toBe("msg_assistant_0003");
+ // The archive delta counts every completed turn the engine holds.
+ expect(archive.turns.at(-1)).toHaveLength(13);
+
+ // A backlog past 32 MiB keeps the newest messages and counts the rest, as a single turn did before.
+ const beforeOmission = traces().length;
+ engine.control.messages.push(message("msg_user_0004", "user"));
+ for (let index = 1; index <= 12; index += 1) engine.control.messages.push(longAnswer(`msg_huge_${String(index).padStart(4, "0")}`, 3));
+ await turn(60_000);
+ const omitted = traces().slice(beforeOmission);
+ expect(omitted.flatMap((envelope) => events(envelope)).filter((event) => event.type === "session.messages_omitted").map((event) => event.data))
+ .toEqual([{ count: 3 }]);
+ const kept = omitted.flatMap((envelope) => events(envelope))
+ .filter((event) => event.type === "turn.messages" || event.type === "turn.completed")
+ .flatMap((event) => ids(event.data?.messages));
+ expect(kept).toEqual(Array.from({ length: 10 }, (_, index) => `msg_huge_${String(index + 3).padStart(4, "0")}`));
+ for (const envelope of omitted) expect(document(envelope)).toMatchObject({ trace_truncated: false, dropped_event_count: 0 });
+ }, 60_000);
+});
diff --git a/apps/server/src/collector-observer.ts b/apps/server/src/collector-observer.ts
index 7bbf8095..73f77a86 100644
--- a/apps/server/src/collector-observer.ts
+++ b/apps/server/src/collector-observer.ts
@@ -20,17 +20,39 @@ export type EngineTarget = {
engine: "v1" | "v2";
};
+/**
+ * A session being observed: the engine its latest collected request went to
+ * (an engine that restarted may answer on another port), how many requests
+ * asked for it while it was, and the observation itself.
+ */
+type ObservedSession = { target: EngineTarget; requests: number; done: Promise<void> };
+
/** Turn observers of one server: the sessions being observed, their last seen message, and the stop signal. */
export type SessionObservers = {
- sessions: Set<string>;
+ sessions: Map<string, ObservedSession>;
lastMessageIds: Map<string, string>;
controller: AbortController;
};
export function createSessionObservers(): SessionObservers {
- return { sessions: new Set(), lastMessageIds: new Map(), controller: new AbortController() };
+ return { sessions: new Map(), lastMessageIds: new Map(), controller: new AbortController() };
}
+/** What the observer asks of the collector. */
+export type ObservedCollector = Pick<
+ WorkspaceCollector,
+ | "enabled"
+ | "sessionCheckpoint"
+ | "setSessionCheckpoint"
+ | "recordTrace"
+ | "recordSessionModel"
+ | "childCheckpoints"
+ | "childSessionIds"
+ | "recordChildSession"
+ | "captureSnapshot"
+ | "flushTrace"
+>;
+
/** An engine target from a proxied request's headers, without the body headers of that request. */
export function engineTarget(baseUrl: string, headers: Headers, search: string, engine: "v1" | "v2" = "v1"): EngineTarget {
const copy = new Headers(headers);
@@ -93,17 +115,48 @@ function turnModelFromMessages(messages: unknown): CollectorSessionModel | null
const MESSAGE_PAGE_SIZE = 50;
/** One engine response (a page of messages, a session record, a status map) is parsed only up to this size. */
const MAX_ENGINE_READ_BYTES = 8 * 1024 * 1024;
-/** The messages of one turn kept whole for its trace, the newest first. */
+/** The messages one trace flush carries at most (a trace document holds 16 MiB in all). */
const MAX_TURN_MESSAGE_BYTES = 8 * 1024 * 1024;
+/**
+ * The messages after the checkpoint kept whole when a settled turn is read,
+ * the newest first: a turn longer than one flush's share, or one that follows
+ * turns the observer never saw settle, goes out over several flushes.
+ */
+const MAX_BACKLOG_MESSAGE_BYTES = 4 * MAX_TURN_MESSAGE_BYTES;
+/** The longest an observer follows one turn: a safety bound far past any real turn. */
+const MAX_OBSERVED_TURN_MS = 24 * 60 * 60_000;
+/** Status reads are a second apart for a turn's first two minutes, then a second more for every minute it has run, up to this. */
+const MAX_STATUS_POLL_MS = 10_000;
+/** Failed status reads back off, doubling from a second, up to this. */
+const MAX_STATUS_RETRY_MS = 30_000;
+/** An engine that has not answered for this long is recorded in the trace; the observer keeps waiting for it. */
+const ENGINE_UNAVAILABLE_TRACE_MS = 60_000;
+/** A session never seen busy that stays idle this long settles without a finished answer (its prompt never ran). */
+const NEVER_BUSY_SETTLE_MS = 10 * 60_000;
+/** A settled turn whose messages the engine did not answer for (a timeout, a dropped connection) is read again after these waits. */
+const HISTORY_RETRY_DELAYS_MS = [2_000, 5_000];
+
+/** The observer's clock and timeouts; tests pass their own so that hours of a turn pass without real waiting. */
+export type ObserverTiming = {
+ now: () => number;
+ sleep: (ms: number, signal: AbortSignal) => Promise<void>;
+ /** One status read. */
+ statusTimeoutMs: number;
+ /** One other engine read (a page of messages, a children list). */
+ readTimeoutMs: number;
+ /** How long one turn is followed at most. */
+ maxTurnMs: number;
+};
type EngineFetch = (path: string, query?: Record<string, string>) => Promise<Response>;
-/** Requests to the engine a collected request went to, with its headers and query plus `query`, each under a fresh `signal()`. */
-function engineFetch(target: EngineTarget, signal: () => AbortSignal): EngineFetch {
+/** Requests to the engine a collected request went to (the current `target()`), with its headers and query plus `query`, each under a fresh `signal()`. */
+function engineFetch(target: () => EngineTarget, signal: () => AbortSignal): EngineFetch {
return (path, query = {}) => {
- const url = new URL(buildOpencodeProxyUrl(target.baseUrl, path, target.search));
+ const { baseUrl, headers, search } = target();
+ const url = new URL(buildOpencodeProxyUrl(baseUrl, path, search));
for (const [key, value] of Object.entries(query)) url.searchParams.set(key, value);
- return loopbackFetch(url.toString(), { headers: new Headers(target.headers), signal: signal() });
+ return loopbackFetch(url.toString(), { headers: new Headers(headers), signal: signal() });
};
}
@@ -136,10 +189,11 @@ function nullOnErrorStatus(error: unknown): null {
/**
* Walks the task-tool subagent tree below a settled root session: one
* "session.child" event per child carrying only the messages that are new
- * since that child's checkpoint, recursively for grandchildren.
+ * since that child's checkpoint, recursively for grandchildren. `beforeRecord`
+ * hears the size of each event's messages before it is recorded.
*/
async function captureChildSessions(input: {
- collector: WorkspaceCollector;
+ collector: ObservedCollector;
rootSessionId: string;
parentSessionId: string;
depth: number;
@@ -147,6 +201,7 @@ async function captureChildSessions(input: {
checkpoints: Record<string, string>;
known: Set<string>;
visited: Set<string>;
+ beforeRecord: (bytes: number) => void;
}): Promise<void> {
if (input.depth > MAX_COLLECTOR_CHILD_SESSION_DEPTH) return;
const children = await readEngineJson(input.fetchEngine, `/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024)
@@ -163,6 +218,7 @@ async function captureChildSessions(input: {
input.collector.recordTrace(input.rootSessionId, "session.messages_omitted", { child_session_id: childId, count: history.omitted });
}
if (newMessages.length > 0 || !input.known.has(childId)) {
+ input.beforeRecord(history ? sum(history.sizes) : 0);
input.collector.recordChildSession(input.rootSessionId, {
childSessionId: childId,
parentSessionId: input.parentSessionId,
@@ -176,6 +232,14 @@ async function captureChildSessions(input: {
}
}
+function sum(values: readonly number[]): number {
+ return values.reduce((total, value) => total + value, 0);
+}
+
+function errorMessage(error: unknown): string {
+ return error instanceof Error ? error.message : "unknown";
+}
+
function collectorDelay(ms: number, signal: AbortSignal): Promise<void> {
return new Promise((resolvePromise, reject) => {
if (signal.aborted) {
@@ -353,20 +417,28 @@ function traceMessageId(message: unknown): string | null {
type EngineHistory = {
/** Every message in outline, oldest first. */
outline: unknown[];
- /** The messages after the checkpoint, oldest first and whole: the newest of them, up to MAX_TURN_MESSAGE_BYTES. */
+ /** The messages after the checkpoint, oldest first and whole: the newest of them, up to the read's budget. */
delta: unknown[];
- /** Messages after the checkpoint left out of `delta`: too large to read, or past the turn's budget. */
+ /** The JSON size of each message in `delta`. */
+ sizes: number[];
+ /** Messages after the checkpoint left out of `delta`: too large to read, or past the budget. */
omitted: number;
};
/**
* Reads the whole history a page at a time: the messages after the
- * checkpoint message (all of them without one) whole, every message in
- * outline, so a history of any length keeps its transcript and turn count.
+ * checkpoint message (all of them without one) whole, up to `budget` bytes of
+ * them, and every message in outline, so a history of any length keeps its
+ * transcript and turn count.
*/
-async function readEngineHistory(messages: AsyncIterable<EngineMessage>, checkpoint: string | undefined): Promise<EngineHistory> {
+async function readEngineHistory(
+ messages: AsyncIterable<EngineMessage>,
+ checkpoint: string | undefined,
+ budget = MAX_TURN_MESSAGE_BYTES,
+): Promise<EngineHistory> {
const outline: unknown[] = [];
const delta: unknown[] = [];
+ const sizes: number[] = [];
let deltaBytes = 0;
let budgetSpent = false;
let omitted = 0;
@@ -376,8 +448,9 @@ async function readEngineHistory(messages: AsyncIterable<EngineMessage>, checkpo
outline.push(messageOutline(message));
if (!afterCheckpoint) continue;
const bytes = whole && !budgetSpent ? Buffer.byteLength(JSON.stringify(message)) : 0;
- if (whole && !budgetSpent && deltaBytes + bytes <= MAX_TURN_MESSAGE_BYTES) {
+ if (whole && !budgetSpent && deltaBytes + bytes <= budget) {
delta.push(message);
+ sizes.push(bytes);
deltaBytes += bytes;
continue;
}
@@ -385,7 +458,38 @@ async function readEngineHistory(messages: AsyncIterable<EngineMessage>, checkpo
if (whole) budgetSpent = true;
omitted += 1;
}
- return { outline: outline.reverse(), delta: delta.reverse(), omitted };
+ return { outline: outline.reverse(), delta: delta.reverse(), sizes: sizes.reverse(), omitted };
+}
+
+/** Messages one trace flush carries, oldest first, and their JSON size. */
+type MessagePart = { messages: unknown[]; bytes: number };
+
+/**
+ * A settled turn's messages (oldest first) cut into the parts its trace
+ * flushes carry, oldest part first: the newest messages up to `maxBytes` make
+ * the last part, and each earlier part takes the next older ones. One part,
+ * empty, when there are no messages.
+ */
+function messageParts(messages: readonly unknown[], sizes: readonly number[], maxBytes = MAX_TURN_MESSAGE_BYTES): MessagePart[] {
+ const parts: MessagePart[] = [];
+ let part: MessagePart = { messages: [], bytes: 0 };
+ for (let index = messages.length - 1; index >= 0; index -= 1) {
+ const bytes = sizes[index] ?? 0;
+ if (part.messages.length > 0 && part.bytes + bytes > maxBytes) {
+ parts.push(part);
+ part = { messages: [], bytes: 0 };
+ }
+ part.messages.push(messages[index]);
+ part.bytes += bytes;
+ }
+ parts.push(part);
+ for (const each of parts) each.messages.reverse();
+ return parts.reverse();
+}
+
+/** The wait before a status read: a second for a turn's first two minutes, then a second more per minute it has run, at most MAX_STATUS_POLL_MS. */
+function statusPollDelay(elapsedMs: number): number {
+ return Math.min(MAX_STATUS_POLL_MS, Math.max(1_000, Math.floor(elapsedMs / 60_000) * 1_000));
}
/**
@@ -395,7 +499,7 @@ async function readEngineHistory(messages: AsyncIterable<EngineMessage>, checkpo
*/
export function projectArchiveEngineReads(target: EngineTarget, sessionId: string): ArchiveEngineReads {
const v2 = target.engine === "v2";
- const fetchEngine = engineFetch(target, () => AbortSignal.timeout(20_000));
+ const fetchEngine = engineFetch(() => target, () => AbortSignal.timeout(20_000));
const session = `${v2 ? "/api/session" : "/session"}/${encodeURIComponent(sessionId)}`;
return {
session: async () => {
@@ -412,131 +516,241 @@ export function projectArchiveEngineReads(target: EngineTarget, sessionId: strin
* turn's model, subagents and messages, takes the turn's change snapshot,
* flushes the trace and hands the engine's messages to the project archive.
* The snapshot, the trace and the archive delta never depend on reading
- * the messages: a turn whose transcript cannot be read still gets them. An
- * observer that stops following (after an hour, or on an error) still takes
- * the turn's snapshot, with its turn.diff, and flushes the trace.
+ * the messages: a turn whose transcript cannot be read still gets them.
+ *
+ * A turn is followed for as long as it runs, reading the engine's status
+ * less often as the turn grows long. The engine failing to answer (a
+ * timeout, a dropped connection, an error status, a restart) is waited out
+ * with backoff, and a request that finds the session already followed points
+ * the observer at that request's engine. Nothing but the server stopping,
+ * the safety bound (MAX_OBSERVED_TURN_MS) or an unexpected error ends an
+ * observation before its turn settles; the last two still take the turn's
+ * snapshot, with its turn.diff, and flush the trace. The messages of a turn
+ * that did not settle stay after the checkpoint, so the session's next
+ * settled turn (after an app restart too) carries them, over several flushes
+ * when they are more than one holds. A request that came in while a turn
+ * settled starts the next observation. Resolves once the observation ended.
*/
export function observeCollectedSession(input: {
- collector: WorkspaceCollector;
+ collector: ObservedCollector;
archive: Pick<ProjectArchiveLifecycle, "turnCompleted">;
observers: SessionObservers;
sessionId: string;
target: EngineTarget;
-}) {
- if (!input.collector.enabled) return;
+ timing?: Partial<ObserverTiming>;
+}): Promise<void> {
+ if (!input.collector.enabled) return Promise.resolve();
const observer = input.observers;
- if (observer.sessions.has(input.sessionId)) return;
- observer.sessions.add(input.sessionId);
- const { baseUrl, search } = input.target;
+ const { collector, sessionId } = input;
+ const observed = observer.sessions.get(sessionId);
+ if (observed) {
+ observed.target = input.target;
+ observed.requests += 1;
+ return observed.done;
+ }
+ const session: ObservedSession = { target: input.target, requests: 0, done: Promise.resolve() };
+ observer.sessions.set(sessionId, session);
+ const timing: ObserverTiming = {
+ now: () => Date.now(),
+ sleep: collectorDelay,
+ statusTimeoutMs: 10_000,
+ readTimeoutMs: 20_000,
+ maxTurnMs: MAX_OBSERVED_TURN_MS,
+ ...input.timing,
+ };
+ const stopped = observer.controller.signal;
const v2 = input.target.engine === "v2";
- const headers = new Headers(input.target.headers);
- const statusUrl = buildOpencodeProxyUrl(baseUrl, v2 ? "/api/session/active" : "/session/status", search);
- const fetchEngine = engineFetch(input.target, () => AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]));
- const messagesPath = v2 ? `/api/session/${encodeURIComponent(input.sessionId)}/context` : `/session/${encodeURIComponent(input.sessionId)}/message`;
+ const target = () => session.target;
+ const fetchStatus = engineFetch(target, () => AbortSignal.any([stopped, AbortSignal.timeout(timing.statusTimeoutMs)]));
+ const fetchEngine = engineFetch(target, () => AbortSignal.any([stopped, AbortSignal.timeout(timing.readTimeoutMs)]));
+ const messagesPath = v2 ? `/api/session/${encodeURIComponent(sessionId)}/context` : `/session/${encodeURIComponent(sessionId)}/message`;
const messages = () => engineMessages(fetchEngine, messagesPath, !v2);
const enginePayload = (payload: unknown) => (v2 && isRecord(payload) && "data" in payload ? payload.data : payload);
const falseUnlessStopped = (error: unknown): false => {
- if (observer.controller.signal.aborted) throw error;
+ if (stopped.aborted) throw error;
return false;
};
// Whether the turn's snapshot was taken: a failure after it must not take a second.
let turnCaptured = false;
- void (async () => {
- let observedBusy = false;
- let consecutiveSettled = 0;
- const checkpoint = await input.collector.sessionCheckpoint(input.sessionId);
- if (checkpoint.lastMessageId) observer.lastMessageIds.set(input.sessionId, checkpoint.lastMessageId);
- for (let attempt = 0; attempt < 3_600; attempt += 1) {
- await collectorDelay(1_000, observer.controller.signal);
- const response = await loopbackFetch(statusUrl, {
- headers,
- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(10_000)]),
- });
- if (!response.ok) {
- await response.body?.cancel().catch(() => undefined);
- consecutiveSettled = 0;
- continue;
+
+ /** The session's status ("idle" when the engine does not list it); throws when the engine does not answer. */
+ const readStatus = async (): Promise<string> => {
+ const statuses = enginePayload(await readEngineJson(fetchStatus, v2 ? "/api/session/active" : "/session/status", 1024 * 1024));
+ const status = isRecord(statuses) ? statuses[sessionId] : undefined;
+ return isRecord(status) && typeof status.type === "string" ? status.type : "idle";
+ };
+
+ /** The history as the settled turn left it; a read the engine did not answer is tried again. */
+ const readHistory = async (): Promise<EngineHistory> => {
+ for (let attempt = 0; ; attempt += 1) {
+ try {
+ return await readEngineHistory(messages(), observer.lastMessageIds.get(sessionId), MAX_BACKLOG_MESSAGE_BYTES);
+ } catch (error) {
+ // An error status is the engine's answer; a timeout or a dropped connection is worth another read.
+ const delay = HISTORY_RETRY_DELAYS_MS[attempt];
+ if (stopped.aborted || error instanceof EngineReadError || delay === undefined) throw error;
+ await timing.sleep(delay, stopped);
}
- const statuses = enginePayload(await readCollectorResponse(response, 1024 * 1024));
- const session = isRecord(statuses) ? statuses[input.sessionId] : undefined;
- const statusType = isRecord(session) && typeof session.type === "string" ? session.type : "idle";
- if (statusType !== "idle") {
- observedBusy = true;
- consecutiveSettled = 0;
- continue;
+ }
+ };
+
+ /** Everything a settled turn records, in order: messages, subagents, the idle event, the snapshot, the trace and the archive delta. */
+ const settle = async (status: string): Promise<void> => {
+ let history: EngineHistory | null = null;
+ let unavailable: Record<string, unknown> = { unavailable: true };
+ try {
+ history = await readHistory();
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ if (error instanceof EngineReadError) unavailable = { status: error.status, unavailable: true };
+ collector.recordTrace(sessionId, "session.messages_failed", { error: errorMessage(error) });
+ }
+ let parts: MessagePart[] = [{ messages: [], bytes: 0 }];
+ if (history) {
+ const lastId = traceMessageId(history.outline.at(-1));
+ if (lastId) {
+ observer.lastMessageIds.set(sessionId, lastId);
+ void collector.setSessionCheckpoint(sessionId, lastId);
}
- consecutiveSettled += 1;
- if (consecutiveSettled < 2 || attempt < 3) continue;
- // Never seen busy: the turn settled only once its answer ended (the prompt may not have started yet).
- if (!observedBusy && !(await newestAssistantFinished(messages()).catch(falseUnlessStopped))) continue;
- let history: EngineHistory | null = null;
- let unavailable: Record<string, unknown> = { unavailable: true };
+ if (history.omitted > 0) collector.recordTrace(sessionId, "session.messages_omitted", { count: history.omitted });
+ const model = turnModelFromMessages(history.delta);
+ if (model) collector.recordSessionModel(sessionId, model);
+ // More messages than one flush holds: the older ones go ahead, a flush per part.
+ parts = messageParts(history.delta, history.sizes);
+ for (const [index, part] of parts.slice(0, -1).entries()) {
+ collector.recordTrace(sessionId, "turn.messages", { messages: part.messages, part: index + 1, parts: parts.length });
+ collector.flushTrace(sessionId);
+ }
+ }
+ const newest = parts.at(-1)!;
+ // Message bytes in the trace since its last flush: subagent messages
+ // flush on their own before they would crowd the turn's out of it.
+ let pending = 0;
+ const room = (bytes: number) => {
+ if (pending > 0 && pending + bytes > MAX_TURN_MESSAGE_BYTES) {
+ collector.flushTrace(sessionId);
+ pending = 0;
+ }
+ pending += bytes;
+ };
+ // The v2 daemon has no subagent children route; only its root turn is captured.
+ if (!v2) {
try {
- history = await readEngineHistory(messages(), observer.lastMessageIds.get(input.sessionId));
- } catch (error) {
- if (observer.controller.signal.aborted) throw error;
- if (error instanceof EngineReadError) unavailable = { status: error.status, unavailable: true };
- input.collector.recordTrace(input.sessionId, "session.messages_failed", {
- error: error instanceof Error ? error.message : "unknown",
+ await captureChildSessions({
+ collector,
+ rootSessionId: sessionId,
+ parentSessionId: sessionId,
+ depth: 1,
+ fetchEngine,
+ checkpoints: await collector.childCheckpoints(sessionId),
+ known: new Set(await collector.childSessionIds(sessionId)),
+ visited: new Set([sessionId]),
+ beforeRecord: room,
});
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ collector.recordTrace(sessionId, "session.children_failed", { error: errorMessage(error) });
}
- if (history) {
- const lastId = traceMessageId(history.outline.at(-1));
- if (lastId) {
- observer.lastMessageIds.set(input.sessionId, lastId);
- void input.collector.setSessionCheckpoint(input.sessionId, lastId);
+ }
+ room(newest.bytes);
+ collector.recordTrace(sessionId, "session.idle", { status });
+ // The turn snapshot runs first so the artifacts it discovers are part of
+ // the trace flushed right behind it.
+ turnCaptured = true;
+ collector.captureSnapshot(sessionId, "turn_completed");
+ collector.flushTrace(sessionId, { messages: history ? newest.messages : unavailable });
+ // The delta's turn number is the engine's completed-turn count, which
+ // survives app restarts; without the messages the archiver numbers it
+ // right after the last archived turn.
+ input.archive.turnCompleted(sessionId, history ? history.outline : null);
+ };
+
+ /**
+ * Follows one turn until it settles: resolves with the requests counted
+ * when the session was first seen idle, or null past the safety bound.
+ */
+ const followTurn = async (): Promise<number | null> => {
+ const startedAt = timing.now();
+ let observedBusy = false;
+ let reads = 0;
+ let idleReads = 0;
+ let idleSince = startedAt;
+ let requestsWhenIdle = session.requests;
+ let failures = 0;
+ let unavailableSince: number | null = null;
+ let unavailableTraced = false;
+ while (timing.now() - startedAt < timing.maxTurnMs) {
+ const elapsed = timing.now() - startedAt;
+ const delay = failures > 0
+ ? Math.max(statusPollDelay(elapsed), Math.min(MAX_STATUS_RETRY_MS, 1_000 * 2 ** (failures - 1)))
+ : idleReads === 1 ? 1_000 : statusPollDelay(elapsed);
+ await timing.sleep(delay, stopped);
+ reads += 1;
+ let status: string;
+ try {
+ status = await readStatus();
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ failures += 1;
+ idleReads = 0;
+ unavailableSince ??= timing.now();
+ if (!unavailableTraced && timing.now() - unavailableSince >= ENGINE_UNAVAILABLE_TRACE_MS) {
+ unavailableTraced = true;
+ collector.recordTrace(sessionId, "session.engine_unavailable", { error: errorMessage(error), failures });
}
- if (history.omitted > 0) input.collector.recordTrace(input.sessionId, "session.messages_omitted", { count: history.omitted });
- const model = turnModelFromMessages(history.delta);
- if (model) input.collector.recordSessionModel(input.sessionId, model);
+ continue;
}
- // The v2 daemon has no subagent children route; only its root turn is captured.
- if (!v2) {
- try {
- await captureChildSessions({
- collector: input.collector,
- rootSessionId: input.sessionId,
- parentSessionId: input.sessionId,
- depth: 1,
- fetchEngine,
- checkpoints: await input.collector.childCheckpoints(input.sessionId),
- known: new Set(await input.collector.childSessionIds(input.sessionId)),
- visited: new Set([input.sessionId]),
- });
- } catch (error) {
- if (observer.controller.signal.aborted) throw error;
- input.collector.recordTrace(input.sessionId, "session.children_failed", {
- error: error instanceof Error ? error.message : "unknown",
- });
- }
+ if (unavailableTraced && unavailableSince !== null) {
+ collector.recordTrace(sessionId, "session.engine_recovered", { failures, unavailable_ms: timing.now() - unavailableSince });
}
- input.collector.recordTrace(input.sessionId, "session.idle", { status: statusType });
- // The turn snapshot runs first so the artifacts it discovers are part of
- // the trace flushed right behind it.
- turnCaptured = true;
- input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId, { messages: history ? history.delta : unavailable });
- // The delta's turn number is the engine's completed-turn count, which
- // survives app restarts; without the messages the archiver numbers it
- // right after the last archived turn.
- input.archive.turnCompleted(input.sessionId, history ? history.outline : null);
- return;
+ failures = 0;
+ unavailableSince = null;
+ unavailableTraced = false;
+ if (status !== "idle") {
+ observedBusy = true;
+ idleReads = 0;
+ continue;
+ }
+ if (idleReads === 0) {
+ idleSince = timing.now();
+ requestsWhenIdle = session.requests;
+ }
+ idleReads += 1;
+ if (idleReads < 2 || reads < 4) continue;
+ // Never seen busy: the turn settled only once its answer ended (the prompt may not have started yet).
+ if (!observedBusy && timing.now() - idleSince < NEVER_BUSY_SETTLE_MS && !(await newestAssistantFinished(messages()).catch(falseUnlessStopped))) {
+ continue;
+ }
+ await settle(status);
+ return requestsWhenIdle;
}
- input.collector.recordTrace(input.sessionId, "session.observer_timeout");
+ collector.recordTrace(sessionId, "session.observer_timeout", { waited_ms: timing.now() - startedAt });
// No longer followed, the turn still gets its snapshot and turn.diff: the
// next prompt would otherwise measure its own turn from past these edits.
+ // Its messages are left after the checkpoint for the next settled turn.
turnCaptured = true;
- input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ collector.flushTrace(sessionId);
+ return null;
+ };
+
+ session.done = (async () => {
+ const checkpoint = await collector.sessionCheckpoint(sessionId);
+ if (checkpoint.lastMessageId) observer.lastMessageIds.set(sessionId, checkpoint.lastMessageId);
+ while (true) {
+ turnCaptured = false;
+ const requests = await followTurn();
+ // A request that came in once the session was idle (the next prompt) started a turn of its own.
+ if (requests === null || session.requests === requests) return;
+ }
})().catch((error: unknown) => {
- if (!observer.controller.signal.aborted) {
- input.collector.recordTrace(input.sessionId, "session.observer_failed", {
- error: error instanceof Error ? error.message : "unknown",
- });
- if (!turnCaptured) input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId);
+ if (!stopped.aborted) {
+ collector.recordTrace(sessionId, "session.observer_failed", { error: errorMessage(error) });
+ if (!turnCaptured) collector.captureSnapshot(sessionId, "turn_completed");
+ collector.flushTrace(sessionId);
}
}).finally(() => {
- observer.sessions.delete(input.sessionId);
+ observer.sessions.delete(sessionId);
});
+ return session.done;
}
diff --git a/apps/server/src/session-archive/lifecycle.test.ts b/apps/server/src/session-archive/lifecycle.test.ts
index ec51e596..ae97c26d 100644
--- a/apps/server/src/session-archive/lifecycle.test.ts
+++ b/apps/server/src/session-archive/lifecycle.test.ts
@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterEach, describe, expect, jest, test } from "bun:test";
import { execFile } from "node:child_process";
import { mkdir, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -394,6 +394,91 @@ describe("ProjectArchiveLifecycle", () => {
expect(child.reads).toEqual({ session: 2, messages: 0 });
});
+ test("an engine too slow to answer at session start is read again a minute later, so a session whose first turn runs for hours still gets its base", async () => {
+ jest.useFakeTimers();
+ try {
+ const archiver = new FakeArchiver();
+ const { subject, logs } = lifecycle(archiver);
+ const root = await tempDir("root");
+ let engineUp = false;
+ const slow = engine({
+ session: () => {
+ if (!engineUp) throw new Error("TimeoutError: the engine did not answer");
+ return { id: "ses_retried_0001" };
+ },
+ messages: () => messages(2, true),
+ });
+ subject.sessionStarted({ sessionId: "ses_retried_0001", root, engine: slow.reader });
+ await subject.settled();
+ expect(archiver.captures()).toEqual([]);
+ expect(logs.map((log) => [log.attributes?.reason, log.attributes?.retryInMs])).toEqual([["the engine session could not be read", 60_000]]);
+
+ // The engine answers by the time the retry runs, with the first turn of this app run still going.
+ engineUp = true;
+ jest.advanceTimersByTime(59_999);
+ await subject.settled();
+ expect(slow.reads.session).toBe(1);
+ jest.advanceTimersByTime(1);
+ await subject.settled();
+ expect(archiver.captures()).toEqual([`base ses_retried_0001 ${root} 2`]);
+ // Resolved: no further retry, and the turn's delta follows the base.
+ jest.advanceTimersByTime(60 * 60_000);
+ subject.turnCompleted("ses_retried_0001", messages(3));
+ await subject.settled();
+ expect(archiver.captures()).toEqual([`base ses_retried_0001 ${root} 2`, `delta ses_retried_0001 ${root} 3`]);
+ expect(slow.reads).toEqual({ session: 2, messages: 1 });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test("an engine that stays unreadable is read again after 1, 2, 5 and 10 minutes, then at the session's next prompt or turn", async () => {
+ jest.useFakeTimers();
+ try {
+ const archiver = new FakeArchiver();
+ const { subject, logs } = lifecycle(archiver);
+ const root = await tempDir("root");
+ let engineUp = false;
+ const down = engine({ session: () => { if (!engineUp) throw new Error("ECONNREFUSED"); return { id: "ses_down_retry_01" }; } });
+ subject.sessionStarted({ sessionId: "ses_down_retry_01", root, engine: down.reader });
+ await subject.settled();
+ // A prompt while a retry is waiting reads the engine at once but does not add another retry.
+ subject.sessionStarted({ sessionId: "ses_down_retry_01", root, engine: down.reader });
+ await subject.settled();
+ expect(down.reads.session).toBe(2);
+ for (const minutes of [1, 2, 5, 10]) {
+ const reads = down.reads.session;
+ jest.advanceTimersByTime(minutes * 60_000 - 1);
+ await subject.settled();
+ expect(down.reads.session).toBe(reads);
+ jest.advanceTimersByTime(1);
+ await subject.settled();
+ expect(down.reads.session).toBe(reads + 1);
+ }
+ jest.advanceTimersByTime(24 * 60 * 60_000);
+ await subject.settled();
+ expect(down.reads.session).toBe(6);
+ expect(logs.map((log) => log.attributes?.retryInMs)).toEqual([60_000, undefined, 120_000, 300_000, 600_000, undefined]);
+
+ // Past the retries, the next completed turn still resolves it.
+ engineUp = true;
+ subject.turnCompleted("ses_down_retry_01", messages(1));
+ await subject.settled();
+ expect(archiver.captures()).toEqual([`base ses_down_retry_01 ${root} 1`, `delta ses_down_retry_01 ${root} 1`]);
+
+ // A retry never runs once archiving stopped.
+ const stopped = engine({ session: () => { throw new Error("ECONNREFUSED"); } });
+ subject.sessionStarted({ sessionId: "ses_stopped_0001", root, engine: stopped.reader });
+ await subject.settled();
+ await subject.stop();
+ jest.advanceTimersByTime(60_000);
+ await subject.settled();
+ expect(stopped.reads.session).toBe(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
test("with the real archiver: base at session start, a delta per changed turn, and the next app run resumes the upload", async () => {
const server = new FakeArchiveServer();
const root = await tempDir("project");
diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.ts
index 89409a7b..cef7da7c 100644
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@ -115,8 +115,13 @@ type SessionRecord = {
start: { root: string; engine: ArchiveEngineReads } | null;
/** A start step is queued or running. */
starting: boolean;
+ /** Engine reads retried on their own while unresolved, and the retry waiting (see unresolved()). */
+ retries?: { count: number; timer: ReturnType<typeof setTimeout> | null };
};
+/** An unresolved session start reads the engine again after these waits, then only at its next prompt or turn. */
+const UNRESOLVED_RETRY_DELAYS_MS = [60_000, 2 * 60_000, 5 * 60_000, 10 * 60_000];
+
export class ProjectArchiveLifecycle {
private readonly archiver: ProjectArchiver;
private readonly log: ArchiveLifecycleLog;
@@ -256,13 +261,14 @@ export class ProjectArchiveLifecycle {
* archived), at which real path, and its base with `turns` completed turns
* (read from the engine when null). The base is a no-op ("exists") when an
* earlier app run captured it. False while the engine cannot be read: the
- * session stays unresolved for its next prompt or completed turn.
+ * session stays unresolved, and is read again after a while (see
+ * unresolved()) and at its next prompt or completed turn.
*/
private async resolve(sessionId: string, record: SessionRecord, turns: number | null): Promise<boolean> {
const start = record.start;
if (!start) return true;
const parent = parentSessionId(await start.engine.session().catch(() => null));
- if (parent === undefined) return this.unresolved(sessionId, "the engine session could not be read");
+ if (parent === undefined) return this.unresolved(sessionId, record, "the engine session could not be read");
if (parent !== null) {
record.start = null;
return true;
@@ -273,7 +279,7 @@ export class ProjectArchiveLifecycle {
return true;
}
const count = turns ?? completedTurnCount(await start.engine.messages().catch(() => null));
- if (count === null) return this.unresolved(sessionId, "the engine messages could not be read");
+ if (count === null) return this.unresolved(sessionId, record, "the engine messages could not be read");
record.start = null;
record.root = root;
const result = await this.archiver.captureBase(sessionId, root, count);
@@ -283,8 +289,37 @@ export class ProjectArchiveLifecycle {
return true;
}
- private unresolved(sessionId: string, reason: string): false {
- this.log("warn", "OmniRush project archive could not read the engine for a session; its next prompt or turn tries again", { sessionId, reason });
+ /**
+ * The engine could not be read for a session start (it can still be
+ * starting, or be busy with a large session). Besides the session's next
+ * prompt or completed turn, the start step runs again on its own after 1, 2,
+ * 5 and 10 minutes, so a session whose first turn runs for hours still gets
+ * its base; one retry waits at a time.
+ */
+ private unresolved(sessionId: string, record: SessionRecord, reason: string): false {
+ const retries = record.retries ??= { count: 0, timer: null };
+ const delay = retries.timer ? undefined : UNRESOLVED_RETRY_DELAYS_MS[retries.count];
+ if (delay !== undefined) {
+ retries.count += 1;
+ retries.timer = setTimeout(() => {
+ retries.timer = null;
+ if (!this.active || this.consentOff || this.sessions.get(sessionId) !== record || !record.start || record.starting) return;
+ record.starting = true;
+ this.schedule(sessionId, "base", async () => {
+ try {
+ if (this.sessions.get(sessionId) === record && record.start) await this.resolve(sessionId, record, null);
+ } finally {
+ record.starting = false;
+ }
+ });
+ }, delay);
+ retries.timer.unref?.();
+ }
+ this.log("warn", "OmniRush project archive could not read the engine for a session; its next prompt or turn tries again", {
+ sessionId,
+ reason,
+ ...(delay !== undefined ? { retryInMs: delay } : {}),
+ });
return false;
}
diff --git a/apps/server/src/workspace-collector.test.ts b/apps/server/src/workspace-collector.test.ts
index 71b8ed00..ed7d8182 100644
--- a/apps/server/src/workspace-collector.test.ts
+++ b/apps/server/src/workspace-collector.test.ts
@@ -2523,6 +2523,69 @@ describe("workspace collector incremental snapshots", () => {
await collector.stop();
});
+ test("two chats on one folder: only the one running a turn uploads the edits made meanwhile, and each still captures its own milestones", async () => {
+ const root = await workspace("shared", 4);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 150, minChangeIntervalMs: 0 });
+ const working = "session-shared-working-1";
+ const idle = "session-shared-idle-0001";
+ const both = [working, idle];
+ for (const sessionId of both) collector.startSession(sessionId, "workspace-shared", root);
+ for (const sessionId of both) await collector.idle(sessionId);
+ const settle = async () => {
+ // Past the debounce and several reconcile passes.
+ await sleep(600);
+ for (const sessionId of both) await collector.idle(sessionId);
+ };
+ const changesOf = (sessionId: string) => changes(uploads)
+ .filter((item) => item.session_id === sessionId)
+ .map((item) => [item.trigger, contentPaths(item).sort()]);
+ // Whichever of the watcher and the reconcile pass saw the edit first.
+ const edit = expect.stringMatching(/^(?:fs_change|periodic)$/);
+
+ collector.captureSnapshot(working, "prompt");
+ await collector.idle(working);
+ await writeFile(join(root, "src", "f00.txt"), "the agent's edit\n");
+ await settle();
+ expect(changesOf(working)).toEqual([[edit, ["src/f00.txt"]]]);
+ expect(changesOf(idle)).toEqual([]);
+ expect(collector.metrics.capturesHeld).toBeGreaterThan(0);
+
+ // The turn ends: the other chat still leaves that turn's edits alone, reconcile passes included.
+ collector.captureSnapshot(working, "turn_completed");
+ await settle();
+ expect(changesOf(idle)).toEqual([]);
+
+ // An edit between turns is uploaded once, by the chat that ran the last turn.
+ await writeFile(join(root, "README.md"), "# edited by hand\n");
+ await settle();
+ expect(changesOf(working).at(-1)).toEqual([edit, ["README.md"]]);
+ expect(changesOf(idle)).toEqual([]);
+
+ // The other chat's own prompt carries what it has not sent yet; while it
+ // runs its turn, the first chat holds back in turn.
+ collector.captureSnapshot(idle, "prompt");
+ await collector.idle(idle);
+ expect(changesOf(idle)).toEqual([["prompt", ["README.md", "src/f00.txt"]]]);
+ const sentByWorking = changesOf(working).length;
+ await writeFile(join(root, "src", "f02.txt"), "the other agent's edit\n");
+ await settle();
+ expect(changesOf(idle).at(-1)).toEqual([edit, ["src/f02.txt"]]);
+ expect(changesOf(working)).toHaveLength(sentByWorking);
+ collector.captureSnapshot(idle, "turn_completed");
+ await collector.idle(idle);
+ for (const sessionId of both) collector.flushTrace(sessionId);
+ await collector.stop();
+ for (const sessionId of both) {
+ const milestones = uploads
+ .filter((item) => item.snapshot_type === "trace" && item.session_id === sessionId)
+ .flatMap((item) => item.trace ?? [])
+ .filter((event) => event.type === "collector.trigger")
+ .map((event) => event.data?.trigger);
+ expect(milestones).toEqual(["prompt", "turn_completed"]);
+ }
+ });
+
test("keeps gitignored files out of change snapshots and the journal in a workspace git does not manage", async () => {
const root = await mkdtemp(join(tmpdir(), "omnirush-collector-nogit-"));
roots.push(root);
diff --git a/apps/server/src/workspace-collector.ts b/apps/server/src/workspace-collector.ts
index 3a685d9d..6a43b855 100644
--- a/apps/server/src/workspace-collector.ts
+++ b/apps/server/src/workspace-collector.ts
@@ -325,6 +325,14 @@ type SessionState = {
turnManifest: TurnBaseline | null;
/** When the current turn's prompt was sent (null before the first). */
turnStartedAt: number | null;
+ /** A prompt was dispatched and its turn has not completed (its turn_completed milestone ends it). */
+ turnInProgress: boolean;
+ /**
+ * Edits were made on this session's root while it had no turn in progress
+ * and another session on the root had one: they are that turn's, and this
+ * session's own filesystem and periodic captures wait for its next milestone.
+ */
+ changesHeld: boolean;
/**
* Files the journal saw the turn write that may be binary or over the size
* cap, with their mtime: "skipped" in its "turn.diff".
@@ -2203,6 +2211,8 @@ export type CollectorMetrics = {
reconciles: number;
capturesSkipped: number;
capturesDeferred: number;
+ /** Filesystem and periodic captures left to another session on the same root (its turn made the edits). */
+ capturesHeld: number;
ignoreCheckSpawns: number;
watchEvents: number;
envelopesWritten: number;
@@ -2213,7 +2223,7 @@ export type CollectorMetrics = {
function freshMetrics(): CollectorMetrics {
return {
fileStats: 0, fileReads: 0, fileRedactions: 0, fullScans: 0, dirtyScans: 0, reconciles: 0,
- capturesSkipped: 0, capturesDeferred: 0, ignoreCheckSpawns: 0, watchEvents: 0, envelopesWritten: 0, redactedTextHits: 0,
+ capturesSkipped: 0, capturesDeferred: 0, capturesHeld: 0, ignoreCheckSpawns: 0, watchEvents: 0, envelopesWritten: 0, redactedTextHits: 0,
};
}
@@ -3126,6 +3136,8 @@ export class WorkspaceCollector {
manifest: null,
turnManifest: null,
turnStartedAt: null,
+ turnInProgress: false,
+ changesHeld: false,
turnSkipped: new Map(),
cache: this.acquireCache(root),
listing: { denied: 0, truncated: false },
@@ -3710,6 +3722,9 @@ export class WorkspaceCollector {
// just before may carry a later fraction of the same millisecond.
if (trigger === "prompt") state.turnStartedAt = Date.now() + 1;
const startedAt = state.turnStartedAt;
+ state.turnInProgress = trigger === "prompt";
+ // The milestone carries whatever edits were held back for another session's turn.
+ state.changesHeld = false;
if (state.changeTimer) {
clearTimeout(state.changeTimer);
state.changeTimer = null;
@@ -3899,6 +3914,7 @@ export class WorkspaceCollector {
private scheduleChange(state: SessionState, trigger: Extract<ChangeTrigger, "fs_change" | "periodic">): void {
if (state.finished) return;
+ if (!state.turnInProgress && this.turnOnRootElsewhere(state)) state.changesHeld = true;
if (state.changeTimer) clearTimeout(state.changeTimer);
// A filesystem change is the more specific reason; keep it when the
// periodic scan also notices the same edit.
@@ -3929,6 +3945,10 @@ export class WorkspaceCollector {
// A finished session is about to upload its end snapshot, which already
// carries everything a queued change capture would.
if (state.finished) return;
+ if (!milestone && this.changesLeftToOthers(state)) {
+ this.metrics.capturesHeld += 1;
+ return;
+ }
if (trigger === "turn_completed") {
await this.captureArtifacts(state).catch((error: unknown) => {
this.log("warn", "OmniRush artifact capture failed", {
@@ -3951,6 +3971,40 @@ export class WorkspaceCollector {
if (milestone) this.appendTrace(state, "collector.trigger", { trigger, captured });
}
+ /** Whether another live session on `state`'s root has a turn in progress. */
+ private turnOnRootElsewhere(state: SessionState): boolean {
+ for (const other of this.sessions.values()) {
+ if (other !== state && !other.finished && other.cache === state.cache && other.turnInProgress) return true;
+ }
+ return false;
+ }
+
+ /**
+ * Whether a filesystem or periodic capture of `state` is left to the other
+ * sessions open on its root, so that one agent's edits are not uploaded
+ * again by every chat open on the same folder. It is when `state` has no
+ * turn in progress and another session on the root has one, and, once
+ * edits were held back that way, until `state`'s own next milestone
+ * (which carries them) as long as another session on the root still
+ * uploads its own changes. A session alone on its root is never held.
+ */
+ private changesLeftToOthers(state: SessionState): boolean {
+ if (state.turnInProgress) return false;
+ let uploading = false;
+ let shared = false;
+ for (const other of this.sessions.values()) {
+ if (other === state || other.finished || other.cache !== state.cache) continue;
+ if (other.turnInProgress) {
+ state.changesHeld = true;
+ return true;
+ }
+ shared = true;
+ if (!other.changesHeld) uploading = true;
+ }
+ if (!shared) state.changesHeld = false;
+ return state.changesHeld && uploading;
+ }
+
/** Whether anything could have moved since the last accepted snapshot; one `git rev-parse` at most. */
private async hasPendingChanges(state: SessionState): Promise<boolean> {
if (state.dirtyOverflow || state.watchMode !== "watching" || state.dirty.size > 0 || this.journalHasChanges(state)) return true;
diff --git a/docs/workspace-collector-privacy.md b/docs/workspace-collector-privacy.md
index db9a0f04..426b2ad3 100644
--- a/docs/workspace-collector-privacy.md
+++ b/docs/workspace-collector-privacy.md
@@ -17,8 +17,8 @@ Neither field changes what may leave the machine: `files[]`, `changed_paths` and
There is no per-session storage limit: a session keeps uploading snapshots and traces for as long as it runs, however much it has already sent, and the privacy block carries no `max_session_bytes`. Every remaining limit applies to one file or one request and matches what the backend accepts per request: 4 MiB per file, 64 MiB per snapshot envelope (uncompressed, and again once compressed), 16 MiB per trace document, 2 MiB of git diff and 50,000 manifest entries. These bound the collector's memory and never stop a session: a snapshot that would not fit still goes out with the files it has room for (touched and git-changed files first, smallest first within each tier), the full manifest, and the rest counted in `snapshot_cap_omitted_count` and `snapshot_cap_omitted_bytes`. The collector still counts the bytes each session has sent, in its local session ledger, for diagnostics only.
-The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once.
+The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once. When several chats are open on the same folder, the edits made while one of them runs a turn are that turn's: only a chat with a turn in progress uploads filesystem- and reconcile-driven change snapshots for them, and a chat without one leaves them to its own next prompt or turn snapshot (or its end snapshot), so one agent's edits are not uploaded again by every chat open on the folder. Each chat's own prompt and turn milestones capture as before, and a chat alone on its folder is unaffected.
## Per-turn diffs: the `turn.diff` trace event
-At the end of each turn the trace gets one `turn.diff` event (`apps/server/src/turn-diff.ts`), pairing the turn with what it changed in the workspace: every file whose redacted sha256 moved between the workspace as the turn's prompt found it and the last accepted manifest. The turn begins when its prompt is sent, not when the queued prompt snapshot runs: a file written after that moment (an edit landing while an earlier upload still holds the queue) is measured from what the collector last saw of it before the prompt, and one it only ever saw already written by the turn is reported as `no_base` without a "before" digest. Likewise, when the next prompt goes out before a turn's own snapshot has run, what the next turn wrote is left to it. A turn the observer stops following (after an hour, or on an error) still gets its snapshot and `turn.diff`. Each entry carries the workspace-relative redacted path, a `status` (`added`, `modified`, `deleted`, `skipped` for a binary or over-4-MiB file the turn wrote, sent without content, or `no_base`, for a changed or deleted file, when there is no text from before the turn to diff against: no longer held, or never seen), both redacted digests, the `+`/`-` line counts and a unified diff of the scrubbed text sent before the turn against the scrubbed text sent after it. A binary or oversized file the turn only read (a traced read, an attachment) is not an entry: only one written since the prompt counts. The diff work is bounded: one file's line matching stops after 5 million comparisons and writes the rest of its change as removed and added lines (still a patch that turns the old text into the new one, its counts then an upper bound), and a turn's diffs get 250 ms of work in all, after which the files still waiting for a diff are only counted in `omitted_file_count`. A file diff is cut at 256 KiB and the whole event, envelope included, at 2 MiB (JSON-encoded), both marked `truncated`; files past the event cap are only counted in `omitted_file_count`, while entries without diff text (`skipped`, `no_base`) still go in as long as they fit. The diff is built from text the collector already sent, so the denylist and scrubber apply unchanged, and the trace is scrubbed again as a whole like every other event. To have the "before" side, the collector keeps the scrubbed texts it sent (start snapshots seed it) in a content-addressed store under the app state directory (`omnirush-collector-bases`, owner-only files, 64 MiB at most, least recently used out first), so bases survive a restart; a sign-out deletes it with the spool. Version 1.0.10 and earlier neither read nor delete that directory: after a downgrade it stays on disk, up to 64 MiB of scrubbed workspace text, until it is removed by hand or a later version signs out.
+At the end of each turn the trace gets one `turn.diff` event (`apps/server/src/turn-diff.ts`), pairing the turn with what it changed in the workspace: every file whose redacted sha256 moved between the workspace as the turn's prompt found it and the last accepted manifest. The turn begins when its prompt is sent, not when the queued prompt snapshot runs: a file written after that moment (an edit landing while an earlier upload still holds the queue) is measured from what the collector last saw of it before the prompt, and one it only ever saw already written by the turn is reported as `no_base` without a "before" digest. Likewise, when the next prompt goes out before a turn's own snapshot has run, what the next turn wrote is left to it. The turn observer follows a turn until it settles, however long it runs, reading the engine's status every second at first and less often as the turn goes on (every 10 seconds once it has run ten minutes); it waits out an engine that times out, drops the connection or restarts, noting `session.engine_unavailable` once the engine has not answered for a minute and `session.engine_recovered` when it answers again, and stops following only after 24 hours (`session.observer_timeout`) or on an unexpected error. A turn it stops following still gets its snapshot and `turn.diff`; its messages, like those of a turn cut short by the app quitting, stay after the session's checkpoint and go out with the session's next settled turn, after an app restart too. One trace carries at most 8 MiB of messages: the older messages of a longer turn, or of such an unsent backlog, go ahead in `turn.messages` events (`part` of `parts`), each flushed as its own trace, up to 32 MiB in all, and any older ones are counted in `session.messages_omitted`. Each entry carries the workspace-relative redacted path, a `status` (`added`, `modified`, `deleted`, `skipped` for a binary or over-4-MiB file the turn wrote, sent without content, or `no_base`, for a changed or deleted file, when there is no text from before the turn to diff against: no longer held, or never seen), both redacted digests, the `+`/`-` line counts and a unified diff of the scrubbed text sent before the turn against the scrubbed text sent after it. A binary or oversized file the turn only read (a traced read, an attachment) is not an entry: only one written since the prompt counts. The diff work is bounded: one file's line matching stops after 5 million comparisons and writes the rest of its change as removed and added lines (still a patch that turns the old text into the new one, its counts then an upper bound), and a turn's diffs get 250 ms of work in all, after which the files still waiting for a diff are only counted in `omitted_file_count`. A file diff is cut at 256 KiB and the whole event, envelope included, at 2 MiB (JSON-encoded), both marked `truncated`; files past the event cap are only counted in `omitted_file_count`, while entries without diff text (`skipped`, `no_base`) still go in as long as they fit. The diff is built from text the collector already sent, so the denylist and scrubber apply unchanged, and the trace is scrubbed again as a whole like every other event. To have the "before" side, the collector keeps the scrubbed texts it sent (start snapshots seed it) in a content-addressed store under the app state directory (`omnirush-collector-bases`, owner-only files, 64 MiB at most, least recently used out first), so bases survive a restart; a sign-out deletes it with the spool. Version 1.0.10 and earlier neither read nor delete that directory: after a downgrade it stays on disk, up to 64 MiB of scrubbed workspace text, until it is removed by hand or a later version signs out.