Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

Merge branch 'fix/v1.0.11-observer-long-turns' into feat/v1.0.11-capture

ogsamrat03bf6da518a2
diff --cc apps/server/src/collector-observer.test.ts
index 00000000,8cf81202..d67289d3
mode 000000,100644..100644
--- a/apps/server/src/collector-observer.test.ts
+++ b/apps/server/src/collector-observer.test.ts
@@@ -1,0 -1,425 +1,625 @@@
+ import { afterEach, describe, expect, test } from "bun:test";
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
++import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+ import { tmpdir } from "node:os";
+ import { join } from "node:path";
+ import { zstdDecompressSync } from "node:zlib";
+
+ import {
+ createSessionObservers,
+ observeCollectedSession,
++ projectArchiveEngineReads,
+ type EngineTarget,
+ type ObservedCollector,
+ type ObserverTiming,
+ type SessionObservers,
+ } from "./collector-observer.js";
++import type { CaptureResult, DrainResult, FinalReason } from "./session-archive/index.js";
++import { ProjectArchiveLifecycle, type ProjectArchiver } from "./session-archive/lifecycle.js";
+ import { WorkspaceCollector, type CollectorChildSession } from "./workspace-collector.js";
+
+ /**
+ * The turn observer against a fake engine, on a fake clock: a turn is
+ * followed for as long as it runs (hours of it pass without real waiting), an
+ * engine that does not answer is waited out, and messages a turn could not
+ * flush go out with the next settled one, over several flushes when needed.
+ */
+
+ const SESSION = "ses_observed_0001";
+ const HOUR = 60 * 60_000;
+ const MIB = 1024 * 1024;
+
+ const cleanups: Array<() => void | Promise<void>> = [];
+ afterEach(async () => {
+ while (cleanups.length) await cleanups.pop()?.();
+ });
+
+ type EngineMessage = { info: Record<string, unknown>; parts: unknown[] };
+
+ function message(id: string, role: "user" | "assistant", text = `${role} ${id}`): EngineMessage {
+ return {
+ info: {
+ id,
+ sessionID: SESSION,
+ role,
+ time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 },
+ ...(role === "assistant" ? { providerID: "omnirush", modelID: "gpt-6-astra", mode: "build" } : {}),
+ },
+ parts: [{ id: `${id}_part`, messageID: id, sessionID: SESSION, type: "text", text }],
+ };
+ }
+
+ /** A long answer: `mib` MiB of agent output. */
+ function longAnswer(id: string, mib: number): EngineMessage {
+ return message(id, "assistant", "agent output line\n".repeat(Math.floor((mib * MIB) / 18)));
+ }
+
+ /** The engine's message paging (`?limit=&before=`): the newest `limit` messages before the cursor, oldest first. */
+ function messagePage(url: URL, list: EngineMessage[]): Response {
+ const limit = Number(url.searchParams.get("limit") ?? 0);
+ if (!limit) return Response.json(list);
+ const end = url.searchParams.has("before") ? Number(url.searchParams.get("before")) : list.length;
+ const start = Math.max(0, end - limit);
+ return Response.json(list.slice(start, end), start > 0 ? { headers: { "X-Next-Cursor": String(start) } } : {});
+ }
+
+ /** A fake v1 engine holding one session: its status, its messages, and hooks on its reads. */
+ function startEngine() {
+ const control = {
+ status: (): string => "idle",
+ messages: [] as EngineMessage[],
+ /** Real milliseconds the next status reads take to answer, one entry per read. */
+ statusDelaysMs: [] as number[],
+ statusReads: 0,
+ /** Runs once a page of messages was read. */
+ onMessagesRead: null as (() => void) | null,
+ };
+ const server = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ async fetch(request) {
+ const url = new URL(request.url);
+ if (url.pathname === "/session/status") {
+ control.statusReads += 1;
+ const delay = control.statusDelaysMs.shift();
+ if (delay) await Bun.sleep(delay);
+ const type = control.status();
+ return Response.json(type === "idle" ? {} : { [SESSION]: { type } });
+ }
+ if (url.pathname === `/session/${SESSION}/message`) {
+ const page = messagePage(url, control.messages);
+ control.onMessagesRead?.();
+ return page;
+ }
+ if (url.pathname === `/session/${SESSION}/children`) return Response.json([]);
++ if (url.pathname === `/session/${SESSION}`) return Response.json({ id: SESSION });
+ return Response.json({ code: "not_found" }, { status: 404 });
+ },
+ });
+ cleanups.push(() => server.stop(true));
+ const target: EngineTarget = { baseUrl: `http://127.0.0.1:${server.port}`, headers: [["authorization", "Basic ZW5naW5l"]], search: "", engine: "v1" };
+ return { control, server, target };
+ }
+
+ /**
+ * A clock that only moves when the observer waits: each wait passes at once,
+ * adds its length to the time and runs whatever was scheduled up to then.
+ */
+ function fakeClock() {
+ const start = Date.parse("2026-09-23T07:34:00.000Z");
+ let now = start;
+ const scheduled: Array<{ at: number; run: () => void }> = [];
+ const waits: number[] = [];
+ const timing: Partial<ObserverTiming> = {
+ now: () => now,
+ sleep: async (ms, signal) => {
+ if (signal.aborted) throw signal.reason;
+ waits.push(ms);
+ now += ms;
+ for (const task of scheduled.filter((entry) => entry.at <= now)) {
+ scheduled.splice(scheduled.indexOf(task), 1);
+ task.run();
+ }
+ await new Promise((resolvePromise) => setImmediate(resolvePromise));
+ if (signal.aborted) throw signal.reason;
+ },
+ };
+ return {
+ timing,
+ waits,
+ elapsed: () => now - start,
+ /** Runs `run` once the clock reaches `offsetMs` past its start. */
+ at: (offsetMs: number, run: () => void) => { scheduled.push({ at: start + offsetMs, run }); },
+ };
+ }
+
+ type Entry =
+ | { kind: "trace"; type: string; data?: unknown }
+ | { kind: "snapshot"; trigger: string }
+ | { kind: "flush"; final?: unknown };
+
+ /** Records what the observer hands the collector, in order. */
+ class FakeCollector implements ObservedCollector {
+ readonly enabled = true;
+ checkpoint: string | undefined;
+ readonly entries: Entry[] = [];
+
+ async sessionCheckpoint(): Promise<{ resumed: boolean; lastMessageId?: string }> {
+ return { resumed: false, ...(this.checkpoint ? { lastMessageId: this.checkpoint } : {}) };
+ }
+
+ async setSessionCheckpoint(_sessionId: string, messageId: string): Promise<void> {
+ this.checkpoint = messageId;
+ }
+
+ recordTrace(_sessionId: string, type: string, data?: unknown): void {
+ this.entries.push({ kind: "trace", type, ...(data === undefined ? {} : { data }) });
+ }
+
+ recordSessionModel(): void {}
+
+ async childCheckpoints(): Promise<Record<string, string>> {
+ return {};
+ }
+
+ async childSessionIds(): Promise<string[]> {
+ return [];
+ }
+
+ recordChildSession(_sessionId: string, _child: CollectorChildSession): void {}
+
+ captureSnapshot(_sessionId: string, trigger: "prompt" | "turn_completed"): void {
+ this.entries.push({ kind: "snapshot", trigger });
+ }
+
+ flushTrace(_sessionId: string, finalTrace?: unknown): void {
+ this.entries.push({ kind: "flush", ...(finalTrace === undefined ? {} : { final: finalTrace }) });
+ }
+
+ /** The entries as short labels: trace types, `snapshot:<trigger>`, `flush` and `flush:turn` (with the turn's messages). */
+ labels(): string[] {
+ return this.entries.map((entry) => {
+ if (entry.kind === "trace") return entry.type;
+ if (entry.kind === "snapshot") return `snapshot:${entry.trigger}`;
+ return entry.final === undefined ? "flush" : "flush:turn";
+ });
+ }
+
+ trace(type: string): unknown[] {
+ return this.entries.flatMap((entry) => (entry.kind === "trace" && entry.type === type ? [entry.data] : []));
+ }
+
+ /** The message ids each turn.completed flush carried. */
+ turnMessageIds(): string[][] {
+ return this.entries.flatMap((entry) => (entry.kind === "flush" && entry.final !== undefined
+ ? [((entry.final as { messages: EngineMessage[] }).messages).map((each) => String(each.info.id))]
+ : []));
+ }
+ }
+
++/** What the observer tells the project archive: the messages of each completed turn, and every call in order. */
+ function fakeArchive() {
+ const turns: Array<unknown[] | null> = [];
- return { turns, turnCompleted: (_sessionId: string, messages: unknown) => { turns.push(messages as unknown[] | null); } };
++ const calls: Array<"followed" | "completed" | "incomplete"> = [];
++ return {
++ turns,
++ calls,
++ turnFollowed: () => { calls.push("followed"); },
++ turnCompleted: (_sessionId: string, messages: unknown) => {
++ turns.push(messages as unknown[] | null);
++ calls.push("completed");
++ },
++ turnIncomplete: () => { calls.push("incomplete"); },
++ };
+ }
+
-function observe(input: { collector: ObservedCollector; archive: ReturnType<typeof fakeArchive>; observers: SessionObservers; target: EngineTarget; timing: Partial<ObserverTiming> }) {
++type ObservedArchive = Parameters<typeof observeCollectedSession>[0]["archive"];
++
++function observe(input: { collector: ObservedCollector; archive: ObservedArchive; observers: SessionObservers; target: EngineTarget; timing: Partial<ObserverTiming> }) {
+ return observeCollectedSession({ ...input, sessionId: SESSION });
+ }
+
++/** A project archiver that records what the real lifecycle asks of it (bases and deltas numbered like the real one). */
++class RecordingArchiver implements ProjectArchiver {
++ readonly calls: string[] = [];
++ private lastTurn = -1;
++
++ async captureBase(sessionId: string, root: string, turn = 0): Promise<CaptureResult> {
++ this.calls.push(`base ${sessionId} ${root} ${turn}`);
++ this.lastTurn = turn;
++ return { status: "queued", archiveId: "archive-base", kind: "base", sequence: 0, size: 1 };
++ }
++
++ async captureDelta(sessionId: string, root: string, turn: number | null): Promise<CaptureResult> {
++ this.calls.push(`delta ${sessionId} ${root} ${turn}`);
++ const next = turn ?? this.lastTurn + 1;
++ if (next <= this.lastTurn) return { status: "skipped", reason: "stale_turn" };
++ this.lastTurn = next;
++ return { status: "queued", archiveId: `archive-delta-${next}`, kind: "delta", sequence: next, size: 1 };
++ }
++
++ async captureFinal(sessionId: string, reason: FinalReason): Promise<CaptureResult> {
++ this.calls.push(`final ${sessionId} ${reason}`);
++ return { status: "queued", archiveId: `archive-final-${this.calls.length}`, kind: "delta", sequence: this.calls.length, size: 1 };
++ }
++
++ async startFinalCandidates(): Promise<string[]> {
++ return [];
++ }
++
++ async drain(): Promise<DrainResult> {
++ return { uploaded: 0, pending: 0, dropped: 0, blocked: null, disabled: false };
++ }
++
++ async signOut(): Promise<void> {}
++
++ async stop(): Promise<void> {}
++}
++
++/** The real project archive lifecycle on a recording archiver, a session of the fake engine started on a fresh folder. */
++async function archivedSession(engine: ReturnType<typeof startEngine>, finalIdleMs: number) {
++ const folder = await mkdtemp(join(tmpdir(), "omnirush-observer-archive-"));
++ cleanups.push(() => rm(folder, { recursive: true, force: true }));
++ const root = await realpath(folder);
++ const archiver = new RecordingArchiver();
++ const archive = new ProjectArchiveLifecycle({ archiver, enabled: true, log: () => undefined, finalIdleMs });
++ cleanups.push(() => archive.stop({ finals: false }));
++ archive.sessionStarted({ sessionId: SESSION, root, engine: projectArchiveEngineReads(engine.target, SESSION) });
++ await archive.settled();
++ return { root, archiver, archive };
++}
++
+ describe("collector observer", () => {
+ test("a turn busy for three hours, past the old one-hour cap, gets its messages, snapshot and archive delta, with status reads easing off", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ engine.control.messages = [message("msg_user_0001", "user", "Work on issue 42")];
+ engine.control.status = () => (clock.elapsed() < 3 * HOUR ? "busy" : "idle");
+ clock.at(3 * HOUR, () => engine.control.messages.push(message("msg_assistant_0001", "assistant", "Fixed issue 42")));
+
+ await observe({ collector, archive, observers: createSessionObservers(), target: engine.target, timing: clock.timing });
+
+ expect(collector.labels()).not.toContain("session.observer_timeout");
+ expect(collector.labels()).not.toContain("session.observer_failed");
+ expect(collector.labels()).toEqual(["session.idle", "snapshot:turn_completed", "flush:turn"]);
+ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"]]);
+ expect(collector.checkpoint).toBe("msg_assistant_0001");
+ expect(archive.turns).toHaveLength(1);
+ expect(archive.turns[0]).toHaveLength(2);
++ // A settled turn, however long, is a completed one for the project archive.
++ expect(archive.calls).toEqual(["followed", "completed"]);
+ expect(clock.elapsed()).toBeGreaterThanOrEqual(3 * HOUR);
+ // A read a second for the first two minutes, then less often, at most every 10 s:
+ // about 1,250 reads over three hours instead of 10,800.
+ expect(clock.waits.slice(0, 120).every((ms) => ms === 1_000)).toBe(true);
+ expect(Math.max(...clock.waits)).toBe(10_000);
+ expect(engine.control.statusReads).toBeLessThan(1_300);
+ });
+
+ test("a status read that times out twice, then answers, does not end the observation", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ engine.control.status = () => (clock.elapsed() < 5 * 60_000 ? "busy" : "idle");
+ // The engine is busy with a large session: its first two status answers take longer than a status read waits.
+ engine.control.statusDelaysMs = [400, 400];
+
+ await observe({ collector, archive, observers: createSessionObservers(), target: engine.target, timing: { ...clock.timing, statusTimeoutMs: 50 } });
+
+ expect(collector.labels()).toEqual(["session.idle", "snapshot:turn_completed", "flush:turn"]);
+ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"]]);
- expect(archive.turns).toHaveLength(1);
++ expect(archive.calls).toEqual(["followed", "completed"]);
+ // The failed reads backed off (1 s, then 2 s) before the next one.
+ expect(clock.waits.slice(0, 3)).toEqual([1_000, 1_000, 2_000]);
+ });
+
+ test("an engine restarting mid-turn is waited out and noted once past a minute; a request that finds the session followed points it at the new engine", async () => {
+ const clock = fakeClock();
+ const before = startEngine();
+ const after = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ before.control.messages = [message("msg_user_0001", "user")];
+ before.control.status = () => "busy";
+ after.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ after.control.status = () => (clock.elapsed() < 40 * 60_000 ? "busy" : "idle");
+ // Twenty minutes in, the engine goes away; five minutes later a request reaches the restarted one.
+ clock.at(20 * 60_000, () => void before.server.stop(true));
+ let rejoined: Promise<void> | null = null;
+ clock.at(25 * 60_000, () => {
+ rejoined = observe({ collector, archive, observers, target: after.target, timing: clock.timing });
+ });
+
+ const observation = observe({ collector, archive, observers, target: before.target, timing: clock.timing });
+ await observation;
+
+ expect(rejoined as Promise<void> | null).toBe(observation);
+ expect(collector.labels()).toEqual([
+ "session.engine_unavailable",
+ "session.engine_recovered",
+ "session.idle",
+ "snapshot:turn_completed",
+ "flush:turn",
+ ]);
+ const [unavailable] = collector.trace("session.engine_unavailable") as Array<{ error: string; failures: number }>;
+ expect(unavailable?.failures).toBeGreaterThan(1);
+ const [recovered] = collector.trace("session.engine_recovered") as Array<{ unavailable_ms: number }>;
+ expect(recovered?.unavailable_ms).toBeGreaterThanOrEqual(4 * 60_000);
+ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"]]);
+ expect(archive.turns).toHaveLength(1);
++ // Waiting out the engine is not an incomplete turn, and the request that rejoined followed no turn of its own.
++ expect(archive.calls).toEqual(["followed", "completed"]);
+ expect(observers.sessions.size).toBe(0);
+ });
+
+ test("a prompt sent while the turn settles is followed as a turn of its own", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ let busyUntil = 2 * 60_000;
+ engine.control.status = () => (clock.elapsed() < busyUntil ? "busy" : "idle");
+ // The next prompt reaches the engine right as the settled turn's messages are read.
+ engine.control.onMessagesRead = () => {
+ engine.control.onMessagesRead = null;
+ busyUntil = clock.elapsed() + 5 * 60_000;
+ engine.control.messages.push(message("msg_user_0002", "user"), message("msg_assistant_0002", "assistant"));
+ void observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
+ };
+
+ await observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
+
+ expect(collector.labels()).toEqual([
+ "session.idle", "snapshot:turn_completed", "flush:turn",
+ "session.idle", "snapshot:turn_completed", "flush:turn",
+ ]);
+ expect(collector.turnMessageIds()).toEqual([
+ ["msg_user_0001", "msg_assistant_0001"],
+ ["msg_user_0002", "msg_assistant_0002"],
+ ]);
+ expect(archive.turns.map((turn) => turn?.length)).toEqual([2, 4]);
++ // The second turn is announced before it is followed: the idle final archive the first one armed is off.
++ expect(archive.calls).toEqual(["followed", "completed", "followed", "completed"]);
+ });
+
+ test("the server stopping mid-turn ends the observation quietly and leaves the checkpoint where it was", async () => {
+ const clock = fakeClock();
+ const engine = startEngine();
+ const collector = new FakeCollector();
+ collector.checkpoint = "msg_assistant_0000";
+ const observers = createSessionObservers();
+ engine.control.status = () => "busy";
+ clock.at(2 * HOUR, () => observers.controller.abort());
- await observe({ collector, archive: fakeArchive(), observers, target: engine.target, timing: clock.timing });
++ const archive = fakeArchive();
++ await observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
+ expect(collector.entries).toEqual([]);
+ expect(collector.checkpoint).toBe("msg_assistant_0000");
++ // A quit is no incomplete turn: the project archive's stop() packs its final archives.
++ expect(archive.calls).toEqual(["followed"]);
++ });
++
++ test("an unexpected error ends the observation with the turn's snapshot and trace, and the project archive hears once that the turn did not complete", async () => {
++ const clock = fakeClock();
++ const engine = startEngine();
++ const archive = fakeArchive();
++ const observers = createSessionObservers();
++ // A collector that fails on the second turn's idle event, before that turn's snapshot.
++ let idles = 0;
++ const collector = new (class extends FakeCollector {
++ override recordTrace(sessionId: string, type: string, data?: unknown): void {
++ if (type === "session.idle" && ++idles === 2) throw new Error("the collector failed");
++ super.recordTrace(sessionId, type, data);
++ }
++ })();
++ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
++ let busyUntil = 2 * 60_000;
++ engine.control.status = () => (clock.elapsed() < busyUntil ? "busy" : "idle");
++ engine.control.onMessagesRead = () => {
++ engine.control.onMessagesRead = null;
++ busyUntil = clock.elapsed() + 5 * 60_000;
++ engine.control.messages.push(message("msg_user_0002", "user"), message("msg_assistant_0002", "assistant"));
++ void observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
++ };
++
++ await observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
++
++ expect(collector.labels()).toEqual([
++ "session.idle", "snapshot:turn_completed", "flush:turn",
++ "session.observer_failed", "snapshot:turn_completed", "flush",
++ ]);
++ // The first turn completed; the second, which failed, is incomplete, and neither is told twice.
++ expect(archive.calls).toEqual(["followed", "completed", "followed", "incomplete"]);
++ expect(observers.sessions.size).toBe(0);
++ });
++});
++
++describe("collector observer with the project archive", () => {
++ const user = (id: string) => message(`msg_user_${id}`, "user");
++ const answer = (id: string) => message(`msg_assistant_${id}`, "assistant");
++
++ test("a turn that runs past the old one-hour cap and then settles gets its delta, and the idle final archive once the session stays quiet", async () => {
++ const clock = fakeClock();
++ const engine = startEngine();
++ engine.control.messages = [user("0001")];
++ const { root, archiver, archive } = await archivedSession(engine, 100);
++ expect(archiver.calls).toEqual([`base ${SESSION} ${root} 0`]);
++ engine.control.status = () => (clock.elapsed() < 3 * HOUR ? "busy" : "idle");
++ clock.at(3 * HOUR, () => engine.control.messages.push(answer("0001")));
++
++ const collector = new FakeCollector();
++ await observe({ collector, archive, observers: createSessionObservers(), target: engine.target, timing: clock.timing });
++ await archive.settled();
++
++ expect(clock.elapsed()).toBeGreaterThanOrEqual(3 * HOUR);
++ expect(collector.labels()).toEqual(["session.idle", "snapshot:turn_completed", "flush:turn"]);
++ expect(archiver.calls).toEqual([`base ${SESSION} ${root} 0`, `delta ${SESSION} ${root} 1`]);
++ // No prompt follows: the quiet window after the turn ends in a final archive of the folder.
++ await Bun.sleep(250);
++ await archive.settled();
++ expect(archiver.calls).toEqual([`base ${SESSION} ${root} 0`, `delta ${SESSION} ${root} 1`, `final ${SESSION} idle`]);
++ });
++
++ test("a turn still busy at the 24-hour safety bound gets its delta when the engine's count moved, then the idle final archive", async () => {
++ const clock = fakeClock();
++ const engine = startEngine();
++ engine.control.messages = [user("0001")];
++ const { root, archiver, archive } = await archivedSession(engine, 100);
++ // The answer ends five hours in, but the engine never reports the session idle again.
++ engine.control.status = () => "busy";
++ clock.at(5 * HOUR, () => engine.control.messages.push(answer("0001")));
++
++ const collector = new FakeCollector();
++ await observe({ collector, archive, observers: createSessionObservers(), target: engine.target, timing: clock.timing });
++ await archive.settled();
++
++ expect(clock.elapsed()).toBeGreaterThanOrEqual(24 * HOUR);
++ expect(collector.labels()).toEqual(["session.observer_timeout", "snapshot:turn_completed", "flush"]);
++ // turnIncomplete read the engine's count again: it moved, so the turn gets its delta.
++ expect(archiver.calls).toEqual([`base ${SESSION} ${root} 0`, `delta ${SESSION} ${root} 1`]);
++ await Bun.sleep(250);
++ await archive.settled();
++ expect(archiver.calls).toEqual([`base ${SESSION} ${root} 0`, `delta ${SESSION} ${root} 1`, `final ${SESSION} idle`]);
++ }, 60_000);
++
++ test("a prompt sent while a turn settles keeps the idle final archive off while its own turn runs", async () => {
++ const clock = fakeClock();
++ const engine = startEngine();
++ engine.control.messages = [user("0001")];
++ const { root, archiver, archive } = await archivedSession(engine, 100);
++ const collector = new FakeCollector();
++ const observers = createSessionObservers();
++ let busyUntil = 2 * 60_000;
++ engine.control.status = () => (clock.elapsed() < busyUntil ? "busy" : "idle");
++ clock.at(busyUntil, () => engine.control.messages.push(answer("0001")));
++ // The next prompt reaches the server while turn 1's messages are read: its session start
++ // reaches the archive before turn 1's end does.
++ engine.control.onMessagesRead = () => {
++ engine.control.onMessagesRead = null;
++ busyUntil = clock.elapsed() + 30 * 60_000;
++ engine.control.messages.push(user("0002"));
++ clock.at(busyUntil, () => engine.control.messages.push(answer("0002")));
++ archive.sessionStarted({ sessionId: SESSION, root, engine: projectArchiveEngineReads(engine.target, SESSION) });
++ void observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
++ // Turn 2's first status answer takes longer than the quiet window.
++ engine.control.statusDelaysMs = [300];
++ };
++
++ await observe({ collector, archive, observers, target: engine.target, timing: clock.timing });
++ await archive.settled();
++
++ expect(collector.turnMessageIds()).toEqual([["msg_user_0001", "msg_assistant_0001"], ["msg_user_0002", "msg_assistant_0002"]]);
++ // No idle final archive while turn 2 ran, only after it.
++ expect(archiver.calls).toEqual([`base ${SESSION} ${root} 0`, `delta ${SESSION} ${root} 1`, `delta ${SESSION} ${root} 2`]);
++ await Bun.sleep(250);
++ await archive.settled();
++ expect(archiver.calls).toEqual([
++ `base ${SESSION} ${root} 0`,
++ `delta ${SESSION} ${root} 1`,
++ `delta ${SESSION} ${root} 2`,
++ `final ${SESSION} idle`,
++ ]);
+ });
+ });
+
+ describe("collector observer with the collector", () => {
+ type Envelope = {
+ snapshot_type: string;
+ files: Array<{ path: string; content: string }>;
+ trace?: Array<{ type: string; data?: Record<string, unknown> }>;
+ };
+
+ test("a turn left unsettled past the safety bound leaves its messages to the next settled turn, which sends them over several traces and counts what exceeds the backlog", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-observer-"));
+ cleanups.push(() => rm(root, { recursive: true, force: true }));
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const uploads: Envelope[] = [];
+ const collector = new WorkspaceCollector({
+ upload: async (_sessionId, compressed) => {
+ uploads.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ changeDebounceMs: 60_000,
+ fallbackScanMs: 60_000,
+ });
+ cleanups.push(() => collector.stop());
+ collector.startSession(SESSION, "workspace-observer", root);
+ const engine = startEngine();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ const traces = () => uploads.filter((upload) => upload.snapshot_type === "trace");
+ const events = (envelope: Envelope | undefined) => envelope?.trace ?? [];
+ const document = (envelope: Envelope) => JSON.parse(envelope.files[0]!.content) as { trace_truncated: boolean; dropped_event_count: number };
+ const ids = (list: unknown) => (list as EngineMessage[]).map((each) => String(each.info.id));
+ let clock = fakeClock();
+ /** One observation on a fresh clock, with the engine busy for `busyMs` (Infinity: throughout). */
+ const turn = async (busyMs: number, maxTurnMs?: number) => {
+ clock = fakeClock();
+ const current = clock;
+ engine.control.status = () => (current.elapsed() < busyMs ? "busy" : "idle");
+ await observe({ collector, archive, observers, target: engine.target, timing: { ...current.timing, ...(maxTurnMs ? { maxTurnMs } : {}) } });
+ await collector.idle(SESSION);
+ };
+
+ // Turn 1 settles normally: the checkpoint is its answer.
+ engine.control.messages = [message("msg_user_0001", "user"), message("msg_assistant_0001", "assistant")];
+ await turn(60_000);
+ expect((await collector.sessionCheckpoint(SESSION)).lastMessageId).toBe("msg_assistant_0001");
+
+ // Turn 2 is still running when the observer's safety bound (two hours here) runs out.
+ engine.control.messages.push(message("msg_user_0002", "user", "Continue"));
+ for (let index = 1; index <= 8; index += 1) engine.control.messages.push(longAnswer(`msg_long_${String(index).padStart(4, "0")}`, 3));
+ await turn(Infinity, 2 * HOUR);
+ const timedOut = traces().at(-1)!;
+ expect(events(timedOut).map((event) => event.type)).toContain("session.observer_timeout");
+ expect(events(timedOut).map((event) => event.type)).not.toContain("turn.completed");
+ expect(events(timedOut).filter((event) => event.type === "collector.trigger").map((event) => event.data?.trigger)).toEqual(["turn_completed"]);
+ expect(archive.turns).toHaveLength(1);
++ // The project archive hears the turn ended without completing (its delta, or a final archive).
++ expect(archive.calls).toEqual(["followed", "completed", "followed", "incomplete"]);
+ // Its messages were not sent, so the checkpoint did not move.
+ expect((await collector.sessionCheckpoint(SESSION)).lastMessageId).toBe("msg_assistant_0001");
+
+ // Turn 3 ("continue") settles: 24 MiB of turn 2 plus its own messages go out in four traces of at most 8 MiB of messages.
+ const sent = traces().length;
+ engine.control.messages.push(message("msg_user_0003", "user", "continue"), message("msg_assistant_0003", "assistant"));
+ await turn(3 * 60_000);
+ const backlog = traces().slice(sent);
+ expect(backlog).toHaveLength(4);
+ const parts = backlog.slice(0, 3).map((envelope) => events(envelope).find((event) => event.type === "turn.messages")?.data);
+ expect(parts.map((part) => [part?.part, part?.parts])).toEqual([[1, 4], [2, 4], [3, 4]]);
+ const completed = events(backlog[3]).find((event) => event.type === "turn.completed")?.data;
+ expect([...parts.flatMap((part) => ids(part?.messages)), ...ids(completed?.messages)]).toEqual([
+ "msg_user_0002",
+ ...Array.from({ length: 8 }, (_, index) => `msg_long_${String(index + 1).padStart(4, "0")}`),
+ "msg_user_0003",
+ "msg_assistant_0003",
+ ]);
+ for (const envelope of backlog) {
+ expect(document(envelope)).toMatchObject({ trace_truncated: false, dropped_event_count: 0 });
+ expect(events(envelope).map((event) => event.type)).not.toContain("session.messages_omitted");
+ }
+ expect((await collector.sessionCheckpoint(SESSION)).lastMessageId).toBe("msg_assistant_0003");
+ // The archive delta counts every completed turn the engine holds.
+ expect(archive.turns.at(-1)).toHaveLength(13);
+
+ // A backlog past 32 MiB keeps the newest messages and counts the rest, as a single turn did before.
+ const beforeOmission = traces().length;
+ engine.control.messages.push(message("msg_user_0004", "user"));
+ for (let index = 1; index <= 12; index += 1) engine.control.messages.push(longAnswer(`msg_huge_${String(index).padStart(4, "0")}`, 3));
+ await turn(60_000);
+ const omitted = traces().slice(beforeOmission);
+ expect(omitted.flatMap((envelope) => events(envelope)).filter((event) => event.type === "session.messages_omitted").map((event) => event.data))
+ .toEqual([{ count: 3 }]);
+ const kept = omitted.flatMap((envelope) => events(envelope))
+ .filter((event) => event.type === "turn.messages" || event.type === "turn.completed")
+ .flatMap((event) => ids(event.data?.messages));
+ expect(kept).toEqual(Array.from({ length: 10 }, (_, index) => `msg_huge_${String(index + 3).padStart(4, "0")}`));
+ for (const envelope of omitted) expect(document(envelope)).toMatchObject({ trace_truncated: false, dropped_event_count: 0 });
+ }, 60_000);
+ });
diff --cc apps/server/src/collector-observer.ts
index 33f9c9a9,73f77a86..1884dcc6
--- a/apps/server/src/collector-observer.ts
+++ b/apps/server/src/collector-observer.ts
@@@ -412,14 -516,24 +516,30 @@@ export function projectArchiveEngineRea
* turn's model, subagents and messages, takes the turn's change snapshot,
* flushes the trace and hands the engine's messages to the project archive.
* The snapshot, the trace and the archive delta never depend on reading
- * the messages: a turn whose transcript cannot be read still gets them. An
- * observer that stops following (after an hour, or on an error) still takes
- * the turn's snapshot, with its turn.diff, flushes the trace and tells the
- * project archive the turn ended without completing.
+ * the messages: a turn whose transcript cannot be read still gets them.
+ *
+ * A turn is followed for as long as it runs, reading the engine's status
+ * less often as the turn grows long. The engine failing to answer (a
+ * timeout, a dropped connection, an error status, a restart) is waited out
+ * with backoff, and a request that finds the session already followed points
+ * the observer at that request's engine. Nothing but the server stopping,
+ * the safety bound (MAX_OBSERVED_TURN_MS) or an unexpected error ends an
+ * observation before its turn settles; the last two still take the turn's
- * snapshot, with its turn.diff, and flush the trace. The messages of a turn
++ * snapshot, with its turn.diff, flush the trace and tell the project archive
++ * the turn ended without completing (turnIncomplete: its delta, or a final
++ * archive of the folder). A server stop tells it nothing: the archive's own
++ * stop() packs the final archives of a quit. The messages of a turn
+ * that did not settle stay after the checkpoint, so the session's next
+ * settled turn (after an app restart too) carries them, over several flushes
+ * when they are more than one holds. A request that came in while a turn
- * settled starts the next observation. Resolves once the observation ended.
++ * settled starts the next observation. Every turn followed tells the project
++ * archive first (turnFollowed), so the idle final archive that the previous
++ * turn's end armed never runs while this one does. Resolves once the
++ * observation ended.
*/
export function observeCollectedSession(input: {
- collector: WorkspaceCollector;
- archive: Pick<ProjectArchiveLifecycle, "turnCompleted" | "turnIncomplete">;
+ collector: ObservedCollector;
- archive: Pick<ProjectArchiveLifecycle, "turnCompleted">;
++ archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete">;
observers: SessionObservers;
sessionId: string;
target: EngineTarget;
@@@ -437,114 -567,190 +573,204 @@@
const messages = () => engineMessages(fetchEngine, messagesPath, !v2);
const enginePayload = (payload: unknown) => (v2 && isRecord(payload) && "data" in payload ? payload.data : payload);
const falseUnlessStopped = (error: unknown): false => {
- if (observer.controller.signal.aborted) throw error;
+ if (stopped.aborted) throw error;
return false;
};
- // Whether the turn's snapshot was taken: a failure after it must not take a second.
+ // Whether the turn's snapshot was taken, and whether the project archive
+ // heard of the turn's end: a failure after either must not repeat it.
let turnCaptured = false;
+ let turnArchived = false;
- void (async () => {
- let observedBusy = false;
- let consecutiveSettled = 0;
- const checkpoint = await input.collector.sessionCheckpoint(input.sessionId);
- if (checkpoint.lastMessageId) observer.lastMessageIds.set(input.sessionId, checkpoint.lastMessageId);
- for (let attempt = 0; attempt < 3_600; attempt += 1) {
- await collectorDelay(1_000, observer.controller.signal);
- const response = await loopbackFetch(statusUrl, {
- headers,
- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(10_000)]),
- });
- if (!response.ok) {
- await response.body?.cancel().catch(() => undefined);
- consecutiveSettled = 0;
- continue;
+
+ /** The session's status ("idle" when the engine does not list it); throws when the engine does not answer. */
+ const readStatus = async (): Promise<string> => {
+ const statuses = enginePayload(await readEngineJson(fetchStatus, v2 ? "/api/session/active" : "/session/status", 1024 * 1024));
+ const status = isRecord(statuses) ? statuses[sessionId] : undefined;
+ return isRecord(status) && typeof status.type === "string" ? status.type : "idle";
+ };
+
+ /** The history as the settled turn left it; a read the engine did not answer is tried again. */
+ const readHistory = async (): Promise<EngineHistory> => {
+ for (let attempt = 0; ; attempt += 1) {
+ try {
+ return await readEngineHistory(messages(), observer.lastMessageIds.get(sessionId), MAX_BACKLOG_MESSAGE_BYTES);
+ } catch (error) {
+ // An error status is the engine's answer; a timeout or a dropped connection is worth another read.
+ const delay = HISTORY_RETRY_DELAYS_MS[attempt];
+ if (stopped.aborted || error instanceof EngineReadError || delay === undefined) throw error;
+ await timing.sleep(delay, stopped);
}
- const statuses = enginePayload(await readCollectorResponse(response, 1024 * 1024));
- const session = isRecord(statuses) ? statuses[input.sessionId] : undefined;
- const statusType = isRecord(session) && typeof session.type === "string" ? session.type : "idle";
- if (statusType !== "idle") {
- observedBusy = true;
- consecutiveSettled = 0;
- continue;
+ }
+ };
+
+ /** Everything a settled turn records, in order: messages, subagents, the idle event, the snapshot, the trace and the archive delta. */
+ const settle = async (status: string): Promise<void> => {
+ let history: EngineHistory | null = null;
+ let unavailable: Record<string, unknown> = { unavailable: true };
+ try {
+ history = await readHistory();
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ if (error instanceof EngineReadError) unavailable = { status: error.status, unavailable: true };
+ collector.recordTrace(sessionId, "session.messages_failed", { error: errorMessage(error) });
+ }
+ let parts: MessagePart[] = [{ messages: [], bytes: 0 }];
+ if (history) {
+ const lastId = traceMessageId(history.outline.at(-1));
+ if (lastId) {
+ observer.lastMessageIds.set(sessionId, lastId);
+ void collector.setSessionCheckpoint(sessionId, lastId);
}
- consecutiveSettled += 1;
- if (consecutiveSettled < 2 || attempt < 3) continue;
- // Never seen busy: the turn settled only once its answer ended (the prompt may not have started yet).
- if (!observedBusy && !(await newestAssistantFinished(messages()).catch(falseUnlessStopped))) continue;
- let history: EngineHistory | null = null;
- let unavailable: Record<string, unknown> = { unavailable: true };
+ if (history.omitted > 0) collector.recordTrace(sessionId, "session.messages_omitted", { count: history.omitted });
+ const model = turnModelFromMessages(history.delta);
+ if (model) collector.recordSessionModel(sessionId, model);
+ // More messages than one flush holds: the older ones go ahead, a flush per part.
+ parts = messageParts(history.delta, history.sizes);
+ for (const [index, part] of parts.slice(0, -1).entries()) {
+ collector.recordTrace(sessionId, "turn.messages", { messages: part.messages, part: index + 1, parts: parts.length });
+ collector.flushTrace(sessionId);
+ }
+ }
+ const newest = parts.at(-1)!;
+ // Message bytes in the trace since its last flush: subagent messages
+ // flush on their own before they would crowd the turn's out of it.
+ let pending = 0;
+ const room = (bytes: number) => {
+ if (pending > 0 && pending + bytes > MAX_TURN_MESSAGE_BYTES) {
+ collector.flushTrace(sessionId);
+ pending = 0;
+ }
+ pending += bytes;
+ };
+ // The v2 daemon has no subagent children route; only its root turn is captured.
+ if (!v2) {
try {
- history = await readEngineHistory(messages(), observer.lastMessageIds.get(input.sessionId));
- } catch (error) {
- if (observer.controller.signal.aborted) throw error;
- if (error instanceof EngineReadError) unavailable = { status: error.status, unavailable: true };
- input.collector.recordTrace(input.sessionId, "session.messages_failed", {
- error: error instanceof Error ? error.message : "unknown",
+ await captureChildSessions({
+ collector,
+ rootSessionId: sessionId,
+ parentSessionId: sessionId,
+ depth: 1,
+ fetchEngine,
+ checkpoints: await collector.childCheckpoints(sessionId),
+ known: new Set(await collector.childSessionIds(sessionId)),
+ visited: new Set([sessionId]),
+ beforeRecord: room,
});
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ collector.recordTrace(sessionId, "session.children_failed", { error: errorMessage(error) });
}
- if (history) {
- const lastId = traceMessageId(history.outline.at(-1));
- if (lastId) {
- observer.lastMessageIds.set(input.sessionId, lastId);
- void input.collector.setSessionCheckpoint(input.sessionId, lastId);
+ }
+ room(newest.bytes);
+ collector.recordTrace(sessionId, "session.idle", { status });
+ // The turn snapshot runs first so the artifacts it discovers are part of
+ // the trace flushed right behind it.
+ turnCaptured = true;
+ collector.captureSnapshot(sessionId, "turn_completed");
+ collector.flushTrace(sessionId, { messages: history ? newest.messages : unavailable });
+ // The delta's turn number is the engine's completed-turn count, which
+ // survives app restarts; without the messages the archiver numbers it
- // right after the last archived turn.
++ // right after the last archived turn. It also arms the idle final archive.
++ turnArchived = true;
+ input.archive.turnCompleted(sessionId, history ? history.outline : null);
+ };
+
+ /**
+ * Follows one turn until it settles: resolves with the requests counted
+ * when the session was first seen idle, or null past the safety bound.
+ */
+ const followTurn = async (): Promise<number | null> => {
++ // A turn is running: the previous turn's idle final archive, if armed, is off.
++ input.archive.turnFollowed(sessionId);
+ const startedAt = timing.now();
+ let observedBusy = false;
+ let reads = 0;
+ let idleReads = 0;
+ let idleSince = startedAt;
+ let requestsWhenIdle = session.requests;
+ let failures = 0;
+ let unavailableSince: number | null = null;
+ let unavailableTraced = false;
+ while (timing.now() - startedAt < timing.maxTurnMs) {
+ const elapsed = timing.now() - startedAt;
+ const delay = failures > 0
+ ? Math.max(statusPollDelay(elapsed), Math.min(MAX_STATUS_RETRY_MS, 1_000 * 2 ** (failures - 1)))
+ : idleReads === 1 ? 1_000 : statusPollDelay(elapsed);
+ await timing.sleep(delay, stopped);
+ reads += 1;
+ let status: string;
+ try {
+ status = await readStatus();
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ failures += 1;
+ idleReads = 0;
+ unavailableSince ??= timing.now();
+ if (!unavailableTraced && timing.now() - unavailableSince >= ENGINE_UNAVAILABLE_TRACE_MS) {
+ unavailableTraced = true;
+ collector.recordTrace(sessionId, "session.engine_unavailable", { error: errorMessage(error), failures });
}
- if (history.omitted > 0) input.collector.recordTrace(input.sessionId, "session.messages_omitted", { count: history.omitted });
- const model = turnModelFromMessages(history.delta);
- if (model) input.collector.recordSessionModel(input.sessionId, model);
+ continue;
}
- // The v2 daemon has no subagent children route; only its root turn is captured.
- if (!v2) {
- try {
- await captureChildSessions({
- collector: input.collector,
- rootSessionId: input.sessionId,
- parentSessionId: input.sessionId,
- depth: 1,
- fetchEngine,
- checkpoints: await input.collector.childCheckpoints(input.sessionId),
- known: new Set(await input.collector.childSessionIds(input.sessionId)),
- visited: new Set([input.sessionId]),
- });
- } catch (error) {
- if (observer.controller.signal.aborted) throw error;
- input.collector.recordTrace(input.sessionId, "session.children_failed", {
- error: error instanceof Error ? error.message : "unknown",
- });
- }
+ if (unavailableTraced && unavailableSince !== null) {
+ collector.recordTrace(sessionId, "session.engine_recovered", { failures, unavailable_ms: timing.now() - unavailableSince });
}
- input.collector.recordTrace(input.sessionId, "session.idle", { status: statusType });
- // The turn snapshot runs first so the artifacts it discovers are part of
- // the trace flushed right behind it.
- turnCaptured = true;
- input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId, { messages: history ? history.delta : unavailable });
- // The delta's turn number is the engine's completed-turn count, which
- // survives app restarts; without the messages the archiver numbers it
- // right after the last archived turn.
- turnArchived = true;
- input.archive.turnCompleted(input.sessionId, history ? history.outline : null);
- return;
+ failures = 0;
+ unavailableSince = null;
+ unavailableTraced = false;
+ if (status !== "idle") {
+ observedBusy = true;
+ idleReads = 0;
+ continue;
+ }
+ if (idleReads === 0) {
+ idleSince = timing.now();
+ requestsWhenIdle = session.requests;
+ }
+ idleReads += 1;
+ if (idleReads < 2 || reads < 4) continue;
+ // Never seen busy: the turn settled only once its answer ended (the prompt may not have started yet).
+ if (!observedBusy && timing.now() - idleSince < NEVER_BUSY_SETTLE_MS && !(await newestAssistantFinished(messages()).catch(falseUnlessStopped))) {
+ continue;
+ }
+ await settle(status);
+ return requestsWhenIdle;
}
- input.collector.recordTrace(input.sessionId, "session.observer_timeout");
++ // The server stopping is no timeout: it ends the observation quietly, and
++ // the project archive's own stop() packs the final archives of a quit.
++ stopped.throwIfAborted();
+ collector.recordTrace(sessionId, "session.observer_timeout", { waited_ms: timing.now() - startedAt });
// No longer followed, the turn still gets its snapshot and turn.diff: the
// next prompt would otherwise measure its own turn from past these edits.
+ // Its messages are left after the checkpoint for the next settled turn.
turnCaptured = true;
- input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ collector.flushTrace(sessionId);
+ // And the project archive its delta, or a final archive of the folder.
+ turnArchived = true;
- input.archive.turnIncomplete(input.sessionId);
++ input.archive.turnIncomplete(sessionId);
+ return null;
+ };
+
+ session.done = (async () => {
+ const checkpoint = await collector.sessionCheckpoint(sessionId);
+ if (checkpoint.lastMessageId) observer.lastMessageIds.set(sessionId, checkpoint.lastMessageId);
+ while (true) {
+ turnCaptured = false;
++ turnArchived = false;
+ const requests = await followTurn();
+ // A request that came in once the session was idle (the next prompt) started a turn of its own.
+ if (requests === null || session.requests === requests) return;
+ }
})().catch((error: unknown) => {
- if (!observer.controller.signal.aborted) {
- input.collector.recordTrace(input.sessionId, "session.observer_failed", {
- error: error instanceof Error ? error.message : "unknown",
- });
- if (!turnCaptured) input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId);
- if (!turnArchived) input.archive.turnIncomplete(input.sessionId);
+ if (!stopped.aborted) {
+ collector.recordTrace(sessionId, "session.observer_failed", { error: errorMessage(error) });
+ if (!turnCaptured) collector.captureSnapshot(sessionId, "turn_completed");
+ collector.flushTrace(sessionId);
++ // The project archive hears of the turn's end once: its delta, or a final archive of the folder.
++ if (!turnArchived) input.archive.turnIncomplete(sessionId);
}
}).finally(() => {
- observer.sessions.delete(input.sessionId);
+ observer.sessions.delete(sessionId);
});
+ return session.done;
}
diff --cc apps/server/src/session-archive/README.md
index 33e22f81,84c24203..2bccb371
--- a/apps/server/src/session-archive/README.md
+++ b/apps/server/src/session-archive/README.md
@@@ -114,49 -105,22 +114,51 @@@ in `captureServicesByServer` and calls
| Server hook | Lifecycle call |
| --- | --- |
-| `startServer` (`startCaptureService`) | `start()`: a drain when enabled; otherwise `signOut()` clears what a previous run left (a no-op, touching nothing, when there is no `omnirush-archive/`) |
+| `startServer` (`startCaptureService`) | `start()`: a drain and the app-start final archives when enabled; otherwise `signOut()` clears what a previous run left (a no-op, touching nothing, when there is no `omnirush-archive/`) |
| A prompt dispatch on a local workspace (v1 `prompt_async`/`prompt`/`command`, v2 `prompt`/`prompt_async`/`command`/`generate`), after the collector's `startSession` | `sessionStarted({sessionId, root: workspace.path, engine})`, the engine reads built on the worker from the request's engine target |
--| The collector observer's `turn_completed` (on the worker), with the engine messages it just read (null when it could not read them) | `turnCompleted(sessionId, messages)` |
- | The collector observer stops following a turn without seeing it settle: `session.observer_timeout` (after an hour) or `session.observer_failed` (for example the engine went away) | `turnIncomplete(sessionId)` |
-| `collector.finishSession` (session deleted) | `sessionEnded(sessionId)`: forgets the session, kicks a drain |
++| The collector observer starts following a turn (on the worker): the first of an observation, or the next one after a request that came in while a turn settled | `turnFollowed(sessionId)`: the session's idle final archive is off until that turn ends |
++| The collector observer's `turn_completed` (on the worker), with the engine messages it just read (null when it could not read them), however long the turn ran | `turnCompleted(sessionId, messages)` |
++| The collector observer stops following a turn without seeing it settle: `session.observer_timeout` (its 24-hour safety bound) or `session.observer_failed` (an unexpected error; an engine that times out, drops the connection or restarts is waited out). A server stop ends an observation without this call | `turnIncomplete(sessionId)` |
+| `collector.finishSession` (session deleted) | `sessionEnded(sessionId)`: a final archive (`session_deleted`), then the session is forgotten; kicks a drain |
| The broker's `invalidate` hook (revoked or expired account), before `clearSpool()` | `signOut()` |
-| Server shutdown and a failed start | `stop()`, started first: the main thread aborts the part uploads it is making for the worker at once, before the task-recovery checkpoint (up to 10 s), then the worker stops the archiver and the collector; the worker is terminated after 20 s at the latest. A user sign-out reaches the server this way too: the desktop clears the account, then restarts the server |
+| Server shutdown and a failed start | `stop({finals})`, started first: the main thread aborts the part uploads it is making for the worker at once, before the task-recovery checkpoint (up to 10 s), then the worker packs the final archives (at most `QUIT_FINAL_BUDGET_MS`, 5 s) and stops the archiver and the collector; the worker is terminated after 20 s at the latest. A user sign-out reaches the server this way too: the desktop clears the account, then restarts the server. So `server.ts` asks the account store (`omnirushGatewayCredentials.latest()`, at most 1 s) whether the account is still there, and without it no final archive is packed |
What the lifecycle adds on top of the calls below:
- **Every call returns at once.** Engine reads and captures run in the background, one at a time (`concurrency`, default 1), and each session's steps run in call order, so a turn's delta never overtakes the session's base. A thrown error or a rejected read costs one `warn` line and never reaches the request.
- **A base waits for a quiet moment.** The server passes `baseIdleMs` 2 s and `baseMaxDeferMs` 10 s (`PROJECT_ARCHIVE_BASE_IDLE_MS`, `PROJECT_ARCHIVE_BASE_MAX_DEFER_MS`): a base is packed once no prompt has been dispatched on any session for 2 s, and at the latest 10 s after its own prompt. Chats opened and prompted one after another are packed once the burst is over, one at a time; a lone chat's base still shows the folder as its first turn began. The wait holds no concurrency slot, and `stop()` or `signOut()` ends it without packing. Each session still gets its own base (sequence 0 of its own chain, section 7 of the backend spec); a second chat on the same folder packs from the root's persisted hash cache, so pass 1 of an unchanged tree reads no file (pass 2 streams and hashes the bytes it packs, as section 5.9 requires).
--- **Root sessions only.** The start step reads the engine's session record (v1 `/session/:id`, v2 `/api/session/:id`); one with a `parentID` is a child and is never archived. An unreadable record, or unreadable messages, leave the session unresolved: its next prompt or its next completed turn reads the engine again. After an app restart the engine can take a minute to answer, longer than the 20 s read timeout, while the first prompt's turn still runs; at the turn's end the observer's messages give the count, the base is captured with it (a no-op, `exists`, when an earlier run captured it) and the turn's delta follows, so that turn is not lost.
++- **Root sessions only.** The start step reads the engine's session record (v1 `/session/:id`, v2 `/api/session/:id`); one with a `parentID` is a child and is never archived. An unreadable record, or unreadable messages, leave the session unresolved: its next prompt or its next completed turn reads the engine again, and the start step also runs again on its own after 1, 2, 5 and 10 minutes (`UNRESOLVED_RETRY_DELAYS_MS`; one retry waits at a time, and each failed read logs one `warn`, with `retryInMs` when it armed a retry). A deleted session, a sign-out and a shutdown clear the waiting retry, and a start read that fails after one of them arms none. After an app restart the engine can take a minute to answer, longer than the 20 s read timeout, while the first prompt's turn still runs, possibly for hours; a retry captures the base once the engine answers, or at the latest the turn's end does, with the count from the observer's messages (a no-op, `exists`, when an earlier run captured it), and the turn's delta follows, so that turn is not lost.
- **Turn numbers from the engine.** `completedTurnCount(messages)` counts prompts answered by an assistant message that ended (a completion time, a finish reason, an error or a finish part; several steps of one turn count once), from v1 `/session/:id/message` or the v2 `/api/session/:id/context` list. The v1 list is read a page at a time (`?limit=&before=`, each page under the 8 MiB read cap) and kept only in outline (each message's info and finish parts), so a history of any length is counted; a message over the cap on its own is counted from the info that leads it. The base gets the count read when its step runs; a delta gets the count the observer read when the turn settled. When the observer could not read the messages at all, the delta still runs with no count (`captureDelta(sessionId, root, null)`), numbered right after the last archived turn. The count survives restarts. A reverted or compacted history can lower it; the archiver then skips those deltas as `stale_turn` and the changes go into the first delta whose count is higher again.
- **Real path.** The root is `realpath`ed before `captureBase`, so the gate sees the real folder (a symlinked workspace is archived as its target; the gate still refuses a symlinked root it is handed).
- **Consent off.** A `disabled` capture result, or a drain that reports `disabled`, turns the lifecycle off for 10 minutes (`consentRecheckMs`): no capture, key check or drain. After that, the next prompt on a session without a base checks `/archives/key` again. No log line: the archiver logs the one `info`.
- - **Signed out stays out.** After `signOut()` or `stop()`, queued steps do not run and nothing new is scheduled. `SessionArchiver.stop()` also bumps the generation once its final archives are done or their budget is spent, so a capture still packing after that never commits (the server that replaces this one after a sign-out must not find it queued; a sign-out packs no final archive in the first place).
-- **Signed out stays out.** After `signOut()` or `stop()`, queued steps do not run and nothing new is scheduled. `SessionArchiver.stop()` also bumps the generation, so a capture still packing at shutdown never commits (the server that replaces this one after a sign-out must not find it queued).
++- **Signed out stays out.** After `signOut()` or `stop()`, queued steps do not run and nothing new is scheduled; every session's timers (the idle final archive's quiet window, an unresolved start's retry) are cleared, as `sessionEnded` clears a deleted session's. `SessionArchiver.stop()` also bumps the generation once its final archives are done or their budget is spent, so a capture still packing after that never commits (the server that replaces this one after a sign-out must not find it queued; a sign-out packs no final archive in the first place).
+- **Final archives.** See "Final archives" below: the quiet window after a turn (`finalIdleMs`, default `FINAL_IDLE_MS`, 10 minutes), a turn that ends without completing, a deleted session, shutdown (`quitBudgetMs`, default `QUIT_FINAL_BUDGET_MS`, 5 s) and app start. They wait for a concurrency slot behind every other waiting step, so a base or a turn's delta is never held up by them.
+
+### Final archives
+
+The base and the turn deltas show the folder as each turn left it. What
+changed after the last completed turn (the user's own edits, a turn that was
+aborted or failed, edits while the app was closed) would otherwise only
+reach the archive with the next turn, or never. A final archive captures it:
+
+- **Wire.** An ordinary `kind: "delta"` create request: `sequence` is the
+ parent's plus one, `parent_archive_id` the previous archive, and `turn`
+ **equal** to the parent's (the last completed turn N; the next turn delta
+ is still N+1). Several final archives for one turn may follow each other.
+ Only the sealed `manifest.json` tells them apart: every delta carries
+ `"trigger": "turn"` or `"trigger": "final"`, and a final archive its
+ `"reason"`: `idle`, `turn_incomplete`, `session_deleted`, `app_quit` or
+ `app_start`. The other manifest fields are unchanged, and a base has
+ neither key. The queue record keeps the trigger too.
+- **When** (`lifecycle.ts`):
- - `idle`: a turn ended (completed or not) and no prompt followed on that session within `finalIdleMs` (10 minutes). One per quiet window; a new prompt clears the timer and aborts a final archive that is queued or packing (`cancelled`).
- - `turn_incomplete`: the observer stopped following a turn (`turnIncomplete`). The lifecycle reads the engine's messages again: a completed-turn count that moved gets the turn's delta, as if `turnCompleted` had fired; a count that did not, or messages that cannot be read, get a final archive. A `turnCompleted` whose count did not move (a prompt aborted before any answer, a reverted history: `stale_turn`) gets one too.
++ - `idle`: a turn ended (completed or not) and no prompt followed on that session within `finalIdleMs` (10 minutes). One per quiet window; a new prompt clears the timer and aborts a final archive that is queued or packing (`cancelled`), and so does the observer following the session's next turn (`turnFollowed`): a prompt sent while the previous turn settled reaches the lifecycle before that turn's `turnCompleted` does, and the quiet window armed then must not run through the next turn. The observer follows a turn for as long as it runs, so after a turn of several hours the window starts when it ends.
++ - `turn_incomplete`: the observer stopped following a turn (`turnIncomplete`: past its 24-hour safety bound, or on an unexpected error; never at a server stop, whose final archives are `app_quit`). The lifecycle reads the engine's messages again: a completed-turn count that moved gets the turn's delta, as if `turnCompleted` had fired; a count that did not, or messages that cannot be read, get a final archive. A `turnCompleted` whose count did not move (a prompt aborted before any answer, a reverted history: `stale_turn`) gets one too.
+ - `session_deleted`: `sessionEnded` queues one before the session is forgotten, also for a session this app run has not seen (the archiver answers `no_base` when it never had a base). The session's record is marked `ended`.
+ - `app_quit`: `stop()` passes the sessions of this app run (resolved, or unresolved but maybe with a base from an earlier run), the most recent first, to `SessionArchiver.stop({finals, budgetMs})`. The drain and its retry timer stop first; the final archives are captured one after the other (a session whose capture is still running goes last), each with an abort signal that the scan and the tar writer check between entries and blocks. When the budget (5 s) runs out, the signal aborts, and the generation is bumped: nothing commits after that. They are uploaded at the next start.
+ - `app_start`: `start()` asks `startFinalCandidates()` for the sessions whose chain moved within the last 7 days, not stopped and not ended: every one with a turn captured since its last final archive (`final_due` in its record: the last shutdown did not get to it, or the app was killed), and the most recent one on each other folder (edits while the app was closed). At most 10, the most recent first.
+- **Guards.** `captureFinal` needs a base (`no_base` otherwise, also while the base is pending), a session that is not stopped and archiving on (`disabled`). It runs the gate of section 4 again on the session root recorded at the base: a folder that is gone, or that the gate refuses now, gets nothing (`not_archivable`). A folder that did not change uploads nothing (`unchanged`: a stat-only scan with the root's hash cache). Only the session root is scanned, with the same exclusions, credential filter and pass 2 checks as every other archive. The lifecycle's rules hold as for any capture: consent off, signed out, `OMNIRUSH_ARCHIVE_ENABLED`, child sessions, the bounded queue. The server's per-upload and per-chat limits count final archives like any other (a 413 stops the session).
++- **Several chats on one folder.** Each session archives the folder in its own chain, so its final archive, like its turn deltas, carries whatever changed in the folder since its own previous archive, including another chat's edits, and each session's quiet window is its own (one timer per session, re-armed by each turn end, never doubled). The collector's rule for a shared root (a chat without a turn in progress leaves the filesystem changes made during another chat's turn to that turn's snapshots, `workspace-collector.ts`) concerns its scrubbed snapshots only: the archive neither waits for it nor schedules anything from it.
+- **A server without final archives.** A backend from before 1.0.11 answers `409 archive_parent_mismatch` to a delta whose turn equals its parent's. Until the server has accepted a final archive in this app run, the first final archive in a chain keeps the chain point before it (`rewind` in the session record, with that archive's baseline, which `start()` keeps too). On that 409 for a final archive, the drain holds the session and, under its lock, puts the chain back there, drops the final archive and every job chained on it, and captures one turn delta again on top (with the highest turn among the dropped turn deltas), so later turns keep uploading. It logs once, and no final archive is captured again until the app restarts (`unsupported`). Once a final archive is uploaded, the rewind point goes (its baseline is deleted) and a later `archive_parent_mismatch` ends the chain as before.
The contract the lifecycle follows is the original wiring note below.
@@@ -347,11 -312,10 +349,12 @@@ cd apps/server && bun --conditions=deve
- replay, chain conflicts and an unknown kid;
- backoff;
- the broker request hook.
- - `lifecycle.test.ts` (fake archiver and engine, plus runs on the real archiver): the turn count on v1 and v2 message shapes; the feature flag; one base per root session per app run at the real path; child sessions ignored; deltas numbered from the engine and ordered after the base, and after the last archived turn when the messages could not be read; restart resume; a session whose start could not read the engine resolved at its next completed turn (its delta kept, also on the real archiver across an app restart), and a child resolved that way still ignored; signed-out start clearing; sign-out stopping queued steps; 428 off until the recheck; failures contained. Final archives: one idle final per quiet window, a new prompt cancelling a pending or packing one; a turn that ends without completing (delta when the engine's count moved, final otherwise or when the engine cannot be read, and a completed turn whose count did not move); a deleted session (child never); shutdown passing this run's sessions most recent first with the budget, and nothing without a budget, with consent off or with the account gone; app-start finals queued behind other steps; on the real archiver, the idle, quit and app-start finals of one session with the next turn's delta after them.
-- `lifecycle.test.ts` (fake archiver and engine, plus one run on the real archiver): the turn count on v1 and v2 message shapes; the feature flag; one base per root session per app run at the real path; child sessions ignored; deltas numbered from the engine and ordered after the base, and after the last archived turn when the messages could not be read; restart resume; a session whose start could not read the engine resolved at its next completed turn (its delta kept, also on the real archiver across an app restart), and a child resolved that way still ignored; signed-out start clearing; sign-out stopping queued steps; 428 off until the recheck; failures contained.
++- `lifecycle.test.ts` (fake archiver and engine, plus runs on the real archiver): the turn count on v1 and v2 message shapes; the feature flag; one base per root session per app run at the real path; child sessions ignored; deltas numbered from the engine and ordered after the base, and after the last archived turn when the messages could not be read; restart resume; a session whose start could not read the engine resolved at its next completed turn (its delta kept, also on the real archiver across an app restart), and a child resolved that way still ignored; an unreadable start retried after 1, 2, 5 and 10 minutes, one retry at a time, and never once archiving stopped; signed-out start clearing; sign-out stopping queued steps; 428 off until the recheck; failures contained. Final archives: one idle final per quiet window, a new prompt cancelling a pending or packing one, and so does a turn the observer follows (also when that turn's prompt came before the previous turn's end); a deleted session, a sign-out and a shutdown clearing both the idle final and the start retry, and a start read that fails after them arming no retry; a turn that ends without completing (delta when the engine's count moved, final otherwise or when the engine cannot be read, and a completed turn whose count did not move); a deleted session (child never); shutdown passing this run's sessions most recent first with the budget, and nothing without a budget, with consent off or with the account gone; app-start finals queued behind other steps; on the real archiver, the idle, quit and app-start finals of one session with the next turn's delta after them.
- `../project-archive.test.ts`: archives go through the broker's `archiveRequest` into `<state dir>/omnirush-archive/`; `OMNIRUSH_ARCHIVE_ENABLED=0` and a signed-out start clear leftovers without any network; a sign-out aborts a slow part PUT within a second and aborts the upload through the broker.
+- `../capture-client.test.ts`: stopping the capture worker packs a final archive, and none when the account is gone.
++- `../collector-observer.test.ts` ("with the project archive", the real lifecycle on a recording archiver, the observer on a fake clock): a turn of three hours, past the old one-hour cap, gets its delta and then the idle final; a turn still busy at the 24-hour safety bound gets `turnIncomplete` (its delta, the engine's count having moved) and then the idle final; a prompt sent while a turn settles keeps the idle final off while its own turn runs. On a fake archive: `turnFollowed`, then `turnCompleted` or `turnIncomplete` once per turn, including a turn after a completed one that fails, and nothing more at a server stop.
- `../omnirush-gateway-broker.test.ts`: `archiveRequest` URL, bearer, 401 refresh and path allowlist.
- - `../workspace-collector.server.e2e.test.ts` ("project archive wiring"): through the real proxy and observer, one base then a delta numbered 2 after a changed turn; a turn whose observer fails still gets its delta, numbered from the engine; a history past the 8 MiB read cap (one message over it alone) still gets its base, its turns' transcripts and deltas, and a turn whose messages cannot be read still gets its snapshot and delta; 428 checked once with the chat unaffected; the flag off sends nothing.
-- `../workspace-collector.server.e2e.test.ts` ("project archive wiring"): through the real proxy and observer, one base then a delta numbered 2 after a changed turn; a history past the 8 MiB read cap (one message over it alone) still gets its base, its turns' transcripts and deltas, and a turn whose messages cannot be read still gets its snapshot and delta; 428 checked once with the chat unaffected; the flag off sends nothing.
++- `../workspace-collector.server.e2e.test.ts` ("project archive wiring"): through the real proxy and observer, one base then a delta numbered 2 after a changed turn; a turn whose status the engine stops answering is waited out (no `session.observer_failed`) and still gets its delta, numbered from the engine; a history past the 8 MiB read cap (one message over it alone) still gets its base, its turns' transcripts and deltas, and a turn whose messages cannot be read still gets its snapshot and delta; 428 checked once with the chat unaffected; the flag off sends nothing.
- `index.test.ts`:
- the whole folder: `.git/` and gitignored `node_modules/` and `dist/` present, `.env` and `id_rsa` absent unless `archiveIncludeCredentialFiles`; the base then a delta, each decrypted and checked;
- no delta when nothing changed, `stale_turn` and `exists`;
diff --cc apps/server/src/session-archive/lifecycle.test.ts
index 82252786,ae97c26d..f379b278
--- a/apps/server/src/session-archive/lifecycle.test.ts
+++ b/apps/server/src/session-archive/lifecycle.test.ts
@@@ -579,236 -643,3 +664,348 @@@ describe("ProjectArchiveLifecycle base
}
});
});
+
+describe("ProjectArchiveLifecycle final archives", () => {
+ const sleep = (ms: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
+
+ /** Turn deltas numbered like the real archiver: a count that did not move is stale. */
+ function turnArchiver(): FakeArchiver {
+ const archiver = new FakeArchiver();
+ const lastTurn = new Map<string, number>();
+ archiver.base = async (sessionId, turn) => {
+ lastTurn.set(sessionId, turn);
+ return queued("base");
+ };
+ archiver.delta = async (sessionId, turn) => {
+ const next = turn ?? (lastTurn.get(sessionId) ?? 0) + 1;
+ if (next <= (lastTurn.get(sessionId) ?? -1)) return skipped("stale_turn");
+ lastTurn.set(sessionId, next);
+ return queued("delta", next);
+ };
+ return archiver;
+ }
+
+ test("a session quiet after its turn gets one idle final archive; a new prompt cancels the pending one", async () => {
+ const archiver = turnArchiver();
+ const { subject } = lifecycle(archiver, { finalIdleMs: 300 });
+ const root = await tempDir("root");
+ const reads = engine({ messages: () => messages(0, true) }).reader;
+ subject.sessionStarted({ sessionId: "ses_idle_000001", root, engine: reads });
+ await subject.settled();
+ // The next prompt comes within the quiet window: no final archive.
+ subject.turnCompleted("ses_idle_000001", messages(1));
+ await sleep(30);
+ subject.sessionStarted({ sessionId: "ses_idle_000001", root, engine: reads });
+ await sleep(400);
+ await subject.settled();
+ expect(archiver.finals()).toEqual([]);
+
+ // Turn 2 ends and nothing follows: one final archive, however long the quiet lasts.
+ subject.turnCompleted("ses_idle_000001", messages(2));
+ await sleep(400);
+ await subject.settled();
+ expect(archiver.finals()).toEqual(["final ses_idle_000001 idle"]);
+ await sleep(400);
+ await subject.settled();
+ expect(archiver.finals()).toHaveLength(1);
+ expect(archiver.captures()).toEqual([`base ses_idle_000001 ${root} 0`, `delta ses_idle_000001 ${root} 1`, `delta ses_idle_000001 ${root} 2`]);
+
+ // A prompt while the idle final is packing cancels it.
+ let signal: AbortSignal | undefined;
+ archiver.final = (_sessionId, _reason, given) => new Promise((resolvePromise) => {
+ signal = given;
+ given?.addEventListener("abort", () => resolvePromise(skipped("cancelled")), { once: true });
+ });
+ subject.turnCompleted("ses_idle_000001", messages(3));
+ await sleep(400);
+ expect(archiver.finals()).toHaveLength(2);
+ expect(signal?.aborted).toBe(false);
+ subject.sessionStarted({ sessionId: "ses_idle_000001", root, engine: reads });
+ await subject.settled();
+ expect(signal?.aborted).toBe(true);
+ });
+
++ test("a turn the observer follows keeps the idle final archive off until it ends, even when its prompt came before the previous turn's end", async () => {
++ const archiver = turnArchiver();
++ const { subject } = lifecycle(archiver, { finalIdleMs: 100 });
++ const root = await tempDir("root");
++ const reads = engine().reader;
++ subject.sessionStarted({ sessionId: "ses_followed_001", root, engine: reads });
++ await subject.settled();
++ // The next prompt went out while turn 1 settled: its session start comes first, then turn 1's end,
++ // then the observer follows turn 2.
++ subject.sessionStarted({ sessionId: "ses_followed_001", root, engine: reads });
++ subject.turnCompleted("ses_followed_001", messages(1));
++ subject.turnFollowed("ses_followed_001");
++ await sleep(250);
++ await subject.settled();
++ expect(archiver.finals()).toEqual([]);
++ // Turn 2 ends: its quiet window gets the idle final archive.
++ subject.turnCompleted("ses_followed_001", messages(2));
++ await sleep(250);
++ await subject.settled();
++ expect(archiver.finals()).toEqual(["final ses_followed_001 idle"]);
++ expect(archiver.captures()).toEqual([`base ses_followed_001 ${root} 0`, `delta ses_followed_001 ${root} 1`, `delta ses_followed_001 ${root} 2`]);
++
++ // An idle final already packing when a turn is followed is cancelled like one a prompt ends.
++ let signal: AbortSignal | undefined;
++ archiver.final = (_sessionId, _reason, given) => new Promise((resolvePromise) => {
++ signal = given;
++ given?.addEventListener("abort", () => resolvePromise(skipped("cancelled")), { once: true });
++ });
++ subject.turnCompleted("ses_followed_001", messages(3));
++ await sleep(250);
++ expect(signal?.aborted).toBe(false);
++ subject.turnFollowed("ses_followed_001");
++ await subject.settled();
++ expect(signal?.aborted).toBe(true);
++ // A session this run does not know is left alone.
++ subject.turnFollowed("ses_unknown_0002");
++ await subject.settled();
++ expect(archiver.finals()).toHaveLength(2);
++ });
++
++ test("a session deleted, a sign-out and a shutdown clear both an idle final archive and an unresolved start's retry; neither runs afterwards", async () => {
++ const root = await tempDir("root");
++ jest.useFakeTimers();
++ try {
++ for (const end of ["sessionEnded", "signOut", "stop"] as const) {
++ const archiver = turnArchiver();
++ const { subject } = lifecycle(archiver);
++ // One session's turn ended: its idle final archive waits for FINAL_IDLE_MS.
++ subject.sessionStarted({ sessionId: "ses_timer_idle01", root, engine: engine().reader });
++ await subject.settled();
++ subject.turnCompleted("ses_timer_idle01", messages(1));
++ await subject.settled();
++ // Another's engine could not be read at its start: its retry waits a minute.
++ const down = engine({ session: () => { throw new Error("ECONNREFUSED"); } });
++ subject.sessionStarted({ sessionId: "ses_timer_retry1", root, engine: down.reader });
++ await subject.settled();
++ expect(jest.getTimerCount()).toBe(2);
++
++ if (end === "sessionEnded") {
++ subject.sessionEnded("ses_timer_idle01");
++ subject.sessionEnded("ses_timer_retry1");
++ } else if (end === "signOut") {
++ await subject.signOut();
++ } else {
++ await subject.stop();
++ }
++ await subject.settled();
++ expect(jest.getTimerCount()).toBe(0);
++ jest.advanceTimersByTime(24 * 60 * 60_000);
++ await subject.settled();
++ expect(down.reads.session).toBe(1);
++ // Only a deleted session gets its own (session_deleted) final archive; shutdown hands its finals to the archiver.
++ expect(archiver.finals().sort()).toEqual(end === "sessionEnded"
++ ? ["final ses_timer_idle01 session_deleted", "final ses_timer_retry1 session_deleted"]
++ : []);
++ expect(archiver.captures()).toEqual([`base ses_timer_idle01 ${root} 0`, `delta ses_timer_idle01 ${root} 1`]);
++ }
++ } finally {
++ jest.useRealTimers();
++ }
++ });
++
++ test("a start read still running when the session is deleted, at a sign-out or at shutdown arms no retry when it fails", async () => {
++ const root = await tempDir("root");
++ jest.useFakeTimers();
++ try {
++ for (const end of ["sessionEnded", "signOut", "stop"] as const) {
++ const archiver = turnArchiver();
++ const { subject, logs } = lifecycle(archiver);
++ const read = deferred<unknown>();
++ const slow = engine({ session: () => read.promise });
++ subject.sessionStarted({ sessionId: "ses_inflight_001", root, engine: slow.reader });
++ for (let tick = 0; tick < 100 && slow.reads.session === 0; tick += 1) await Promise.resolve();
++ expect(slow.reads.session).toBe(1);
++ if (end === "sessionEnded") subject.sessionEnded("ses_inflight_001");
++ else if (end === "signOut") await subject.signOut();
++ else await subject.stop();
++ // The engine gives up only now.
++ read.resolve(null);
++ await subject.settled();
++ expect(logs.filter((log) => log.attributes?.reason === "the engine session could not be read").map((log) => log.attributes?.retryInMs)).toEqual([undefined]);
++ expect(jest.getTimerCount()).toBe(0);
++ jest.advanceTimersByTime(24 * 60 * 60_000);
++ await subject.settled();
++ expect(slow.reads.session).toBe(1);
++ expect(archiver.captures()).toEqual([]);
++ }
++ } finally {
++ jest.useRealTimers();
++ }
++ });
++
+ test("a turn that ends without completing: its delta when the engine's count moved, else a final archive", async () => {
+ const archiver = turnArchiver();
+ const { subject } = lifecycle(archiver, { finalIdleMs: 60_000 });
+ const root = await tempDir("root");
+ let count: number | null = 0;
+ const reads = engine({ messages: () => (count === null ? Promise.reject(new Error("ECONNREFUSED")) : messages(count)) });
+ subject.sessionStarted({ sessionId: "ses_abnormal_001", root, engine: reads.reader });
+ await subject.settled();
+ // The observer gave up (or failed) after the turn had completed in the engine.
+ count = 1;
+ subject.turnIncomplete("ses_abnormal_001");
+ await subject.settled();
+ // Aborted before any answer: the count did not move.
+ subject.turnIncomplete("ses_abnormal_001");
+ await subject.settled();
+ // The engine went away: no count to go by.
+ count = null;
+ subject.turnIncomplete("ses_abnormal_001");
+ await subject.settled();
+ // A settled turn whose count did not move (a prompt aborted at once) is final too.
+ subject.turnCompleted("ses_abnormal_001", messages(1, true));
+ await subject.settled();
+ expect(archiver.calls.filter((call) => call !== "drain")).toEqual([
+ `base ses_abnormal_001 ${root} 0`,
+ `delta ses_abnormal_001 ${root} 1`,
+ `delta ses_abnormal_001 ${root} 1`,
+ "final ses_abnormal_001 turn_incomplete",
+ "final ses_abnormal_001 turn_incomplete",
+ `delta ses_abnormal_001 ${root} 1`,
+ "final ses_abnormal_001 turn_incomplete",
+ ]);
+ // Unknown sessions are left alone.
+ subject.turnIncomplete("ses_unknown_0001");
+ await subject.settled();
+ expect(archiver.finals()).toHaveLength(3);
+ });
+
+ test("a deleted session gets a final archive before it is forgotten; a child never, an unknown one if the archiver has its base", async () => {
+ const archiver = turnArchiver();
+ const { subject } = lifecycle(archiver);
+ const root = await tempDir("root");
+ subject.sessionStarted({ sessionId: "ses_deleted_0001", root, engine: engine().reader });
+ subject.sessionStarted({ sessionId: "ses_child_00001", root, engine: engine({ parentID: "ses_deleted_0001" }).reader });
+ await subject.settled();
+ subject.turnCompleted("ses_deleted_0001", messages(1));
+ subject.sessionEnded("ses_deleted_0001");
+ subject.sessionEnded("ses_child_00001");
+ // A session this app run has not seen (the archiver answers no_base when it never had one).
+ subject.sessionEnded("ses_earlier_0001");
+ await subject.settled();
+ expect(archiver.finals().sort()).toEqual(["final ses_deleted_0001 session_deleted", "final ses_earlier_0001 session_deleted"]);
+ // Forgotten: its pending turn step was dropped, and later calls are ignored.
+ subject.turnCompleted("ses_deleted_0001", messages(2));
+ await subject.settled();
+ expect(archiver.captures()).toEqual([`base ses_deleted_0001 ${root} 0`]);
+ });
+
+ test("shutdown hands this run's sessions to the archiver's final archives, most recent first, within the budget", async () => {
+ let now = 1_000;
+ const archiver = turnArchiver();
+ const { subject } = lifecycle(archiver, { now: () => now });
+ const root = await tempDir("root");
+ subject.sessionStarted({ sessionId: "ses_quit_old_001", root, engine: engine().reader });
+ now += 1_000;
+ subject.sessionStarted({ sessionId: "ses_quit_new_001", root, engine: engine().reader });
+ subject.sessionStarted({ sessionId: "ses_quit_child_1", root, engine: engine({ parentID: "ses_quit_new_001" }).reader });
+ await subject.settled();
+ now += 1_000;
+ subject.turnCompleted("ses_quit_old_001", messages(1));
+ await subject.settled();
+ await subject.stop();
+ expect(archiver.stops).toEqual([{ finals: ["ses_quit_old_001", "ses_quit_new_001"], budgetMs: 5_000 }]);
+
+ // No budget, consent off, or the account gone (a user sign-out restarts the server): a plain stop.
+ const noBudget = turnArchiver();
+ const off = lifecycle(noBudget, { quitBudgetMs: 0 });
+ off.subject.sessionStarted({ sessionId: "ses_quit_none_01", root, engine: engine().reader });
+ await off.subject.settled();
+ await off.subject.stop();
+ const refused = turnArchiver();
+ refused.base = async () => skipped("disabled");
+ const consent = lifecycle(refused);
+ consent.subject.sessionStarted({ sessionId: "ses_quit_none_02", root, engine: engine().reader });
+ await consent.subject.settled();
+ await consent.subject.stop();
+ const gone = turnArchiver();
+ const signedOut = lifecycle(gone);
+ signedOut.subject.sessionStarted({ sessionId: "ses_quit_none_03", root, engine: engine().reader });
+ await signedOut.subject.settled();
+ await signedOut.subject.stop({ finals: false });
+ expect([...noBudget.stops, ...refused.stops, ...gone.stops]).toEqual([{}, {}, {}]);
+ });
+
+ test("app start: the sessions the archiver names get a final archive, behind every other waiting step", async () => {
+ const archiver = turnArchiver();
+ const { subject } = lifecycle(archiver);
+ const root = await tempDir("root");
+ const first = deferred<CaptureResult>();
+ archiver.base = (sessionId) => (sessionId === "ses_start_first1" ? first.promise : Promise.resolve(queued("base")));
+ // A base is packing when the app-start check comes back.
+ subject.sessionStarted({ sessionId: "ses_start_first1", root, engine: engine().reader });
+ await sleep(10);
+ expect(archiver.captures()).toEqual([`base ses_start_first1 ${root} 0`]);
+ archiver.startFinals = ["ses_start_final1", "ses_start_final2"];
+ subject.start();
+ await sleep(10);
+ subject.sessionStarted({ sessionId: "ses_start_next01", root, engine: engine().reader });
+ first.resolve(queued("base"));
+ await subject.settled();
+ expect(archiver.calls.filter((call) => call !== "drain")).toEqual([
+ `base ses_start_first1 ${root} 0`,
+ `base ses_start_next01 ${root} 0`,
+ "final ses_start_final1 app_start",
+ "final ses_start_final2 app_start",
+ ]);
+ });
+
+ test("with the real archiver: after the last turn, the folder is archived when the chat goes quiet, when the app quits and at the next start", async () => {
+ const server = new FakeArchiveServer();
+ const root = await tempDir("project");
+ const git = (...args: string[]) => execFileAsync("git", ["-C", root, "-c", "user.name=t", "-c", "[email protected]", "-c", "commit.gpgsign=false", ...args]);
+ await git("init", "-q", "-b", "main");
+ await writeFile(join(root, "app.ts"), "export const app = 1;\n");
+ await git("add", "app.ts");
+ await git("commit", "-q", "-m", "initial");
+ const state = await tempDir("state");
+ const archiverFor = () => new SessionArchiver({ gatewayUrl: server.gatewayUrl, accessToken: server.token, fetch: server.respond, stateDir: state, retry: { baseMs: 1, maxMs: 2, attempts: 2 } });
+ const id = "ses_final_real_1";
+
+ const first = lifecycle(archiverFor(), { finalIdleMs: 100 });
+ first.subject.start();
+ first.subject.sessionStarted({ sessionId: id, root, engine: engine({ messages: () => messages(0, true) }).reader });
+ await first.subject.settled();
+ await writeFile(join(root, "turn.txt"), "turn 1\n");
+ first.subject.turnCompleted(id, messages(1));
+ await first.subject.settled();
+ // The user edits after the turn; the chat stays quiet.
+ await writeFile(join(root, "idle.txt"), "after the turn\n");
+ await sleep(200);
+ await first.subject.settled();
+ // More edits, then the app quits.
+ await writeFile(join(root, "quit.txt"), "before quitting\n");
+ await first.subject.stop();
+ // And edits while it is closed.
+ await writeFile(join(root, "closed.txt"), "while closed\n");
+
+ const second = lifecycle(archiverFor());
+ second.subject.start();
+ await second.subject.settled();
+ second.subject.sessionStarted({ sessionId: id, root, engine: engine({ messages: () => messages(1, true) }).reader });
+ await second.subject.settled();
+ await writeFile(join(root, "turn.txt"), "turn 2\n");
+ second.subject.turnCompleted(id, messages(2));
+ await second.subject.settled();
+ await second.subject.stop();
+
+ const objects = server.objects();
+ expect(objects.map((object) => [object.request.kind, object.request.sequence, object.request.turn])).toEqual([
+ ["base", 0, 0], ["delta", 1, 1], ["delta", 2, 1], ["delta", 3, 1], ["delta", 4, 1], ["delta", 5, 2],
+ ]);
+ const manifests = await Promise.all(objects.map(async (object) => {
+ const members = await openArchive(object.object!);
+ return { manifest: manifestOf(members), names: members.slice(1).map((member) => member.name) };
+ }));
+ expect(manifests.slice(1).map(({ manifest }) => [manifest.trigger, manifest.reason])).toEqual([["turn", undefined], ["final", "idle"], ["final", "app_quit"], ["final", "app_start"], ["turn", undefined]]);
+ expect(manifests[2]!.names).toContain("idle.txt");
+ expect(manifests[3]!.names).toContain("quit.txt");
+ expect(manifests[4]!.names).toContain("closed.txt");
+ expect(manifests[5]!.names).toContain("turn.txt");
+ expect([...first.logs, ...second.logs].filter((log) => log.level === "warn")).toEqual([]);
+ });
+});
diff --cc apps/server/src/session-archive/lifecycle.ts
index 9bede8f6,cef7da7c..2383f140
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@@ -131,16 -115,13 +131,21 @@@ type SessionRecord =
start: { root: string; engine: ArchiveEngineReads } | null;
/** A start step is queued or running. */
starting: boolean;
+ /** The engine reads of the latest prompt: a turn that ends without completing reads its turn count there. */
+ engine: ArchiveEngineReads;
+ /** When a prompt was last dispatched or a turn last ended (shutdown takes the most recent sessions first). */
+ activeAt: number;
+ /** The quiet window after the last turn; its end queues the idle final archive. */
+ idleTimer: ReturnType<typeof setTimeout> | null;
+ /** Cancels the idle final archive while it is queued or packing: a new prompt ends the quiet period. */
+ idleFinal: AbortController | null;
+ /** Engine reads retried on their own while unresolved, and the retry waiting (see unresolved()). */
+ retries?: { count: number; timer: ReturnType<typeof setTimeout> | null };
};
+ /** An unresolved session start reads the engine again after these waits, then only at its next prompt or turn. */
+ const UNRESOLVED_RETRY_DELAYS_MS = [60_000, 2 * 60_000, 5 * 60_000, 10 * 60_000];
+
export class ProjectArchiveLifecycle {
private readonly archiver: ProjectArchiver;
private readonly log: ArchiveLifecycleLog;
@@@ -258,43 -219,16 +263,56 @@@
this.schedule(sessionId, "delta", async () => {
if (this.sessions.get(sessionId) !== record) return;
if (record.start && !(await this.resolve(sessionId, record, turns))) return;
- if (!record.root || this.sessions.get(sessionId) !== record || this.consentOff) return;
- const result = await this.archiver.captureDelta(sessionId, record.root, turns);
- if (result.status === "skipped" && result.reason === "stopped") record.root = null;
- this.settle(result);
+ await this.captureTurn(sessionId, record, turns);
+ });
+ this.armIdleFinal(sessionId, record);
+ }
+
++ /**
++ * The observer follows a turn of the session: it is not quiet. A turn that
++ * ended just before (its turnCompleted can come after the next prompt's
++ * sessionStarted, when that prompt went out while the turn settled) must
++ * not leave its idle final archive armed through this turn, however long
++ * it runs; this turn's own end arms the next one.
++ */
++ turnFollowed(sessionId: string): void {
++ const record = this.sessions.get(sessionId);
++ if (record) this.cancelIdleFinal(record);
++ }
++
+ /**
+ * The observer stopped following the session's turn without seeing it
- * complete (it gave up after an hour, or failed, such as when the engine
- * went away). The engine's turn count is read again: when it moved, the
- * turn gets its delta as if it had completed; otherwise (or when the
- * engine cannot be read) the folder gets a final archive.
++ * complete: past its 24-hour safety bound, or on an unexpected error (an
++ * engine that does not answer is waited out, and a server stop is not
++ * this). The engine's turn count is read again: when it moved, the turn
++ * gets its delta as if it had completed; otherwise (or when the engine
++ * cannot be read) the folder gets a final archive.
+ */
+ turnIncomplete(sessionId: string): void {
+ const record = this.sessions.get(sessionId);
+ if (!this.active || this.consentOff || !record) return;
+ this.schedule(sessionId, "turn", async () => {
+ if (this.sessions.get(sessionId) !== record) return;
+ if (record.start && !(await this.resolve(sessionId, record, null))) return;
+ if (!record.root) return;
+ const turns = completedTurnCount(await record.engine.messages().catch(() => null));
+ await this.captureTurn(sessionId, record, turns, turns === null);
});
+ this.armIdleFinal(sessionId, record);
}
- /** The session was deleted: nothing to capture, the queue keeps uploading. */
+ /**
+ * The session was deleted: a last final archive of its folder (when it
- * has a base and the folder is still there), then it is forgotten; the
- * queue keeps uploading.
++ * has a base and the folder is still there), then it is forgotten, with
++ * its idle final and its start retry; the queue keeps uploading.
+ */
sessionEnded(sessionId: string): void {
+ const record = this.sessions.get(sessionId);
- if (record) this.cancelIdleFinal(record);
++ if (record) this.release(record);
this.sessions.delete(sessionId);
+ // A resolved session without a root is a child, not archivable or stopped. One this app
+ // run has not seen may still have a base from an earlier run; the archiver knows.
+ if (this.active && !this.consentOff && (!record || record.root || record.start)) this.final(sessionId, "session_deleted");
this.kick();
}
@@@ -306,25 -240,15 +324,25 @@@
async signOut(): Promise<void> {
this.signedOut = true;
this.wakeSleepers();
- for (const record of this.sessions.values()) this.cancelIdleFinal(record);
++ for (const record of this.sessions.values()) this.release(record);
this.sessions.clear();
await this.archiver.signOut().catch((error: unknown) => this.warn("sign-out", error));
}
- /** Server shutdown: no new steps; the queue stays on disk for the next start. */
- async stop(): Promise<void> {
+ /**
+ * Server shutdown: no new steps, and the queue stays on disk for the next
+ * start. Unless `finals` is false (the account is gone), the sessions of
+ * this app run get a final archive first, the most recent first, within
+ * quitBudgetMs in all (what does not fit is checked at the next start).
+ */
+ async stop(options: { finals?: boolean } = {}): Promise<void> {
+ const finals = options.finals !== false && this.active && !this.consentOff && this.quitBudgetMs > 0
+ ? [...this.sessions].filter(([, record]) => record.root || record.start).sort(([, left], [, right]) => right.activeAt - left.activeAt).map(([sessionId]) => sessionId)
+ : [];
this.stopped = true;
this.wakeSleepers();
- for (const record of this.sessions.values()) this.cancelIdleFinal(record);
- await this.archiver.stop().catch((error: unknown) => this.warn("stop", error));
++ for (const record of this.sessions.values()) this.release(record);
+ await this.archiver.stop(finals.length > 0 ? { finals, budgetMs: this.quitBudgetMs } : {}).catch((error: unknown) => this.warn("stop", error));
}
/** Resolves once every scheduled step and drain has finished (tests). */
@@@ -364,70 -289,37 +383,113 @@@
return true;
}
+ /**
+ * The capture at a turn's end: its delta, numbered with the engine's
+ * completed-turn count (null: after the last archived turn). A count that
+ * did not move (the prompt was aborted before any answer, or the history
+ * was reverted) gets a final archive instead, as does a turn whose count
+ * is unknown when `finalWithoutCount`.
+ */
+ private async captureTurn(sessionId: string, record: SessionRecord, turns: number | null, finalWithoutCount = false): Promise<void> {
+ if (!record.root || this.sessions.get(sessionId) !== record || this.consentOff) return;
+ let result: CaptureResult | null = null;
+ if (!finalWithoutCount) {
+ result = await this.archiver.captureDelta(sessionId, record.root, turns);
+ if (result.status === "skipped" && result.reason === "stopped") record.root = null;
+ this.settle(result);
+ }
+ if (result && !(result.status === "skipped" && result.reason === "stale_turn")) return;
+ if (!record.root || this.consentOff) return;
+ const final = await this.archiver.captureFinal(sessionId, "turn_incomplete");
+ if (final.status === "skipped" && final.reason === "stopped") record.root = null;
+ this.settle(final);
+ }
+
+ /**
+ * Queues a final archive for the session, behind its earlier steps and
+ * behind other sessions' waiting steps. With `record`, only while that
+ * record is current and has a root (an idle final); `signal` cancels it.
+ */
+ private final(sessionId: string, reason: FinalReason, record?: SessionRecord, signal?: AbortSignal): void {
+ this.schedule(sessionId, "final", async () => {
+ if (signal?.aborted || this.consentOff) return;
+ if (record && (this.sessions.get(sessionId) !== record || !record.root)) return;
+ const result = await this.archiver.captureFinal(sessionId, reason, signal ? { signal } : {});
+ if (record && result.status === "skipped" && result.reason === "stopped") record.root = null;
+ this.settle(result);
+ }, undefined, true);
+ }
+
+ /** A turn ended: the quiet window starts; without a prompt by its end, an idle final archive is queued. At most one per window. */
+ private armIdleFinal(sessionId: string, record: SessionRecord): void {
+ this.cancelIdleFinal(record);
+ record.activeAt = this.now();
+ // Known not to be archived (a child, not archivable, stopped): nothing to arm.
+ if (!record.root && !record.start) return;
+ const timer = setTimeout(() => {
+ if (record.idleTimer !== timer) return;
+ record.idleTimer = null;
+ if (!this.active || this.sessions.get(sessionId) !== record) return;
+ const controller = new AbortController();
+ record.idleFinal = controller;
+ this.final(sessionId, "idle", record, controller.signal);
+ }, this.finalIdleMs);
+ timer.unref?.();
+ record.idleTimer = timer;
+ }
+
+ private cancelIdleFinal(record: SessionRecord): void {
+ if (record.idleTimer) clearTimeout(record.idleTimer);
+ record.idleTimer = null;
+ record.idleFinal?.abort();
+ record.idleFinal = null;
+ }
+
- private unresolved(sessionId: string, reason: string): false {
- this.log("warn", "OmniRush project archive could not read the engine for a session; its next prompt or turn tries again", { sessionId, reason });
++ /** An unresolved start's retry waiting, if any, never runs. */
++ private cancelRetry(record: SessionRecord): void {
++ if (record.retries?.timer) clearTimeout(record.retries.timer);
++ if (record.retries) record.retries.timer = null;
++ }
++
++ /** The session is forgotten (deleted, signed out, shutdown): none of its timers runs any more. */
++ private release(record: SessionRecord): void {
++ this.cancelIdleFinal(record);
++ this.cancelRetry(record);
++ }
++
+ /**
+ * The engine could not be read for a session start (it can still be
+ * starting, or be busy with a large session). Besides the session's next
+ * prompt or completed turn, the start step runs again on its own after 1, 2,
+ * 5 and 10 minutes, so a session whose first turn runs for hours still gets
- * its base; one retry waits at a time.
++ * its base; one retry waits at a time, and none once the session is
++ * forgotten or archiving stopped.
+ */
+ private unresolved(sessionId: string, record: SessionRecord, reason: string): false {
+ const retries = record.retries ??= { count: 0, timer: null };
- const delay = retries.timer ? undefined : UNRESOLVED_RETRY_DELAYS_MS[retries.count];
++ const current = this.active && this.sessions.get(sessionId) === record;
++ const delay = retries.timer || !current ? undefined : UNRESOLVED_RETRY_DELAYS_MS[retries.count];
+ if (delay !== undefined) {
+ retries.count += 1;
+ retries.timer = setTimeout(() => {
+ retries.timer = null;
+ if (!this.active || this.consentOff || this.sessions.get(sessionId) !== record || !record.start || record.starting) return;
+ record.starting = true;
+ this.schedule(sessionId, "base", async () => {
+ try {
+ if (this.sessions.get(sessionId) === record && record.start) await this.resolve(sessionId, record, null);
+ } finally {
+ record.starting = false;
+ }
+ });
+ }, delay);
+ retries.timer.unref?.();
+ }
+ this.log("warn", "OmniRush project archive could not read the engine for a session; its next prompt or turn tries again", {
+ sessionId,
+ reason,
+ ...(delay !== undefined ? { retryInMs: delay } : {}),
+ });
return false;
}
diff --cc apps/server/src/workspace-collector.server.e2e.test.ts
index eac8add8,30e227b1..65f28356
--- a/apps/server/src/workspace-collector.server.e2e.test.ts
+++ b/apps/server/src/workspace-collector.server.e2e.test.ts
@@@ -897,26 -892,6 +897,36 @@@ describe("project archive wiring", () =
expect(archive.calls.filter((call) => call.path === "archives/key")).toHaveLength(1);
}, 60_000);
- test("a turn the observer stops following (it failed) still gets its delta, numbered from the engine", async () => {
++ test("a turn whose status the engine stops answering is waited out, not given up: once it answers, the turn gets its delta, numbered from the engine", async () => {
+ const { root, engine, archive, gateway, omnirush } = await archivedServer();
+ expect((await prompt(omnirush.base)).status).toBe(204);
+ await waitFor(() => (traces(gateway.uploads).length >= 1 ? true : undefined));
+ await waitFor(() => (archive.objects().length >= 1 ? true : undefined));
- // The status route stops answering JSON: the observer fails before it sees the turn settle.
++ // The status route stops answering JSON: the observer retries with backoff instead of failing.
+ engine.control.statusBody = "not json";
+ await writeFile(join(root, "feature.txt"), "new work\n");
++ const statusReads = () => engine.received.filter((request) => request === "GET /session/status").length;
++ const before = statusReads();
+ expect((await prompt(omnirush.base)).status).toBe(204);
++ await waitFor(() => (statusReads() >= before + 2 ? true : undefined));
++ engine.control.statusBody = null;
+ const objects = await waitFor(() => (archive.objects().some((object) => object.request.turn === 2) ? archive.objects() : undefined));
- const failed = await waitFor(() => traces(gateway.uploads).find((envelope) => events(envelope).some((event) => event.type === "session.observer_failed")));
++ const settled = await waitFor(() => traces(gateway.uploads).slice(1).find((envelope) => events(envelope).some((event) => event.type === "turn.completed")));
+ await omnirush.stop();
+
- expect(events(failed).some((event) => event.type === "session.observer_failed")).toBe(true);
++ // The turn settled as any other: its messages went out, and the observer neither failed nor timed out.
++ const completed = events(settled).find((event) => event.type === "turn.completed")?.data?.messages as Array<{ info: { id: string } }>;
++ expect(completed.map((entry) => entry.info.id)).toEqual(["user_2", "assistant_2"]);
++ for (const envelope of traces(gateway.uploads)) {
++ expect(events(envelope).map((event) => event.type)).not.toContain("session.observer_failed");
++ expect(events(envelope).map((event) => event.type)).not.toContain("session.observer_timeout");
++ }
+ expect(objects.map((object) => [object.request.kind, object.request.turn])).toEqual([["base", 1], ["delta", 2]]);
+ const members = await openArchive(objects[1]!.object!);
+ expect(manifestOf(members)).toMatchObject({ kind: "delta", turn: 2, trigger: "turn" });
+ expect(members.map((member) => member.name)).toContain("feature.txt");
+ }, 60_000);
+
test("a chat whose history is past the 8 MiB read cap keeps each turn's transcript, turn snapshot and archive delta, and gets its base; a turn whose messages cannot be read keeps its snapshot and delta", async () => {
const message = (id: string, role: "user" | "assistant", part: Record<string, unknown>): EngineMessage => ({
info: { id, sessionID: "ses_root", role, time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 } },
diff --cc docs/project-archive.md
index 2bf24fd9,2b24782a..2f66b5ad
--- a/docs/project-archive.md
+++ b/docs/project-archive.md
@@@ -51,35 -49,15 +51,46 @@@ archived
ignores, such as `node_modules/`, build output, media and other binaries.
The copy is taken once the app has had no new message for about two
seconds (at most ten seconds after the chat's first message), so opening
-- and starting several chats in a row is not slowed down by it. A chat
++ and starting several chats in a row is not slowed down by it. If the
++ app cannot read the chat from its engine then (the engine can still be
++ starting after an app restart), it tries again after 1, 2, 5 and 10
++ minutes, and with the chat's next message or completed turn, so a chat
++ whose first turn runs for hours still gets its full copy. A chat
that already has this full copy, for example one you continue after
restarting the app, does not get a second one.
- **After each turn that changed something:** only the files that were
added or changed, plus the list of deleted paths. A turn that changed
-- nothing uploads nothing.
++ nothing uploads nothing. The app follows a turn for as long as it runs
++ (up to 24 hours), so a turn of several hours gets its upload when it
++ ends.
+- **Once more after the last turn (a final copy):** the files added,
+ changed or deleted since the previous upload, in the same form, when the
+ folder changed after the chat's last completed turn, for example because
+ you edited files yourself, or a turn was stopped or failed before it
+ finished. The app looks for such changes:
+ - when the chat has had no new message for 10 minutes after a turn (a
- new message before then cancels it);
- - when a turn ends without completing (the app stops following it after
- an hour, or loses track of it);
++ new message before then cancels it, also one sent while the turn was
++ still finishing);
++ - when a turn ends without the app seeing it complete (the app stops
++ following a turn after 24 hours, or on an unexpected error; it waits
++ for an engine that stops answering or restarts);
+ - when you delete the chat in the app, if its folder still exists;
+ - when you quit the app. The app spends at most about 5 seconds on this
+ while it shuts down; the copy is uploaded the next time the app runs;
+ - when the app starts again, for chats archived in the last 7 days:
+ anything the last shutdown did not get to, and changes made while the
+ app was closed (for the most recent chat on each folder).
+
+ A folder that did not change uploads nothing. A final copy is only taken
+ for a chat that already has its full copy, and only while the folder is
- still one that qualifies (see [When it runs](#when-it-runs)).
++ still one that qualifies (see [When it runs](#when-it-runs)). When
++ several chats work in the same folder, each has its own copies, so one
++ chat's final copy (like its turn uploads) also holds what another chat
++ changed in the folder since that chat's previous upload.
Each upload records the chat and its turn number, so the folder can be
-rebuilt as it was after any turn.
+rebuilt as it was after any turn. A final copy carries the number of the
+last completed turn again and is marked as final, with what prompted it.
## Size limits
diff --cc docs/workspace-collector-privacy.md
index db9a0f04,426b2ad3..aabf60da
--- a/docs/workspace-collector-privacy.md
+++ b/docs/workspace-collector-privacy.md
@@@ -17,8 -17,8 +17,8 @@@ Neither field changes what may leave th
There is no per-session storage limit: a session keeps uploading snapshots and traces for as long as it runs, however much it has already sent, and the privacy block carries no `max_session_bytes`. Every remaining limit applies to one file or one request and matches what the backend accepts per request: 4 MiB per file, 64 MiB per snapshot envelope (uncompressed, and again once compressed), 16 MiB per trace document, 2 MiB of git diff and 50,000 manifest entries. These bound the collector's memory and never stop a session: a snapshot that would not fit still goes out with the files it has room for (touched and git-changed files first, smallest first within each tier), the full manifest, and the rest counted in `snapshot_cap_omitted_count` and `snapshot_cap_omitted_bytes`. The collector still counts the bytes each session has sent, in its local session ledger, for diagnostics only.
- The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once.
-The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once. When several chats are open on the same folder, the edits made while one of them runs a turn are that turn's: only a chat with a turn in progress uploads filesystem- and reconcile-driven change snapshots for them, and a chat without one leaves them to its own next prompt or turn snapshot (or its end snapshot), so one agent's edits are not uploaded again by every chat open on the folder. Each chat's own prompt and turn milestones capture as before, and a chat alone on its folder is unaffected.
++The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path outside the workspace root (a parent folder, another Windows drive or a network share), and none reached through a symlinked directory, is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once. When several chats are open on the same folder, the edits made while one of them runs a turn are that turn's: only a chat with a turn in progress uploads filesystem- and reconcile-driven change snapshots for them, and a chat without one leaves them to its own next prompt or turn snapshot (or its end snapshot), so one agent's edits are not uploaded again by every chat open on the folder. Each chat's own prompt and turn milestones capture as before, and a chat alone on its folder is unaffected. This concerns the collector's scrubbed snapshots only: the project archive keeps its own copies of the folder for each chat ([project-archive.md](project-archive.md)).
## Per-turn diffs: the `turn.diff` trace event
- At the end of each turn the trace gets one `turn.diff` event (`apps/server/src/turn-diff.ts`), pairing the turn with what it changed in the workspace: every file whose redacted sha256 moved between the workspace as the turn's prompt found it and the last accepted manifest. The turn begins when its prompt is sent, not when the queued prompt snapshot runs: a file written after that moment (an edit landing while an earlier upload still holds the queue) is measured from what the collector last saw of it before the prompt, and one it only ever saw already written by the turn is reported as `no_base` without a "before" digest. Likewise, when the next prompt goes out before a turn's own snapshot has run, what the next turn wrote is left to it. A turn the observer stops following (after an hour, or on an error) still gets its snapshot and `turn.diff`. Each entry carries the workspace-relative redacted path, a `status` (`added`, `modified`, `deleted`, `skipped` for a binary or over-4-MiB file the turn wrote, sent without content, or `no_base`, for a changed or deleted file, when there is no text from before the turn to diff against: no longer held, or never seen), both redacted digests, the `+`/`-` line counts and a unified diff of the scrubbed text sent before the turn against the scrubbed text sent after it. A binary or oversized file the turn only read (a traced read, an attachment) is not an entry: only one written since the prompt counts. The diff work is bounded: one file's line matching stops after 5 million comparisons and writes the rest of its change as removed and added lines (still a patch that turns the old text into the new one, its counts then an upper bound), and a turn's diffs get 250 ms of work in all, after which the files still waiting for a diff are only counted in `omitted_file_count`. A file diff is cut at 256 KiB and the whole event, envelope included, at 2 MiB (JSON-encoded), both marked `truncated`; files past the event cap are only counted in `omitted_file_count`, while entries without diff text (`skipped`, `no_base`) still go in as long as they fit. The diff is built from text the collector already sent, so the denylist and scrubber apply unchanged, and the trace is scrubbed again as a whole like every other event. To have the "before" side, the collector keeps the scrubbed texts it sent (start snapshots seed it) in a content-addressed store under the app state directory (`omnirush-collector-bases`, owner-only files, 64 MiB at most, least recently used out first), so bases survive a restart; a sign-out deletes it with the spool. Version 1.0.10 and earlier neither read nor delete that directory: after a downgrade it stays on disk, up to 64 MiB of scrubbed workspace text, until it is removed by hand or a later version signs out.
-At the end of each turn the trace gets one `turn.diff` event (`apps/server/src/turn-diff.ts`), pairing the turn with what it changed in the workspace: every file whose redacted sha256 moved between the workspace as the turn's prompt found it and the last accepted manifest. The turn begins when its prompt is sent, not when the queued prompt snapshot runs: a file written after that moment (an edit landing while an earlier upload still holds the queue) is measured from what the collector last saw of it before the prompt, and one it only ever saw already written by the turn is reported as `no_base` without a "before" digest. Likewise, when the next prompt goes out before a turn's own snapshot has run, what the next turn wrote is left to it. The turn observer follows a turn until it settles, however long it runs, reading the engine's status every second at first and less often as the turn goes on (every 10 seconds once it has run ten minutes); it waits out an engine that times out, drops the connection or restarts, noting `session.engine_unavailable` once the engine has not answered for a minute and `session.engine_recovered` when it answers again, and stops following only after 24 hours (`session.observer_timeout`) or on an unexpected error. A turn it stops following still gets its snapshot and `turn.diff`; its messages, like those of a turn cut short by the app quitting, stay after the session's checkpoint and go out with the session's next settled turn, after an app restart too. One trace carries at most 8 MiB of messages: the older messages of a longer turn, or of such an unsent backlog, go ahead in `turn.messages` events (`part` of `parts`), each flushed as its own trace, up to 32 MiB in all, and any older ones are counted in `session.messages_omitted`. Each entry carries the workspace-relative redacted path, a `status` (`added`, `modified`, `deleted`, `skipped` for a binary or over-4-MiB file the turn wrote, sent without content, or `no_base`, for a changed or deleted file, when there is no text from before the turn to diff against: no longer held, or never seen), both redacted digests, the `+`/`-` line counts and a unified diff of the scrubbed text sent before the turn against the scrubbed text sent after it. A binary or oversized file the turn only read (a traced read, an attachment) is not an entry: only one written since the prompt counts. The diff work is bounded: one file's line matching stops after 5 million comparisons and writes the rest of its change as removed and added lines (still a patch that turns the old text into the new one, its counts then an upper bound), and a turn's diffs get 250 ms of work in all, after which the files still waiting for a diff are only counted in `omitted_file_count`. A file diff is cut at 256 KiB and the whole event, envelope included, at 2 MiB (JSON-encoded), both marked `truncated`; files past the event cap are only counted in `omitted_file_count`, while entries without diff text (`skipped`, `no_base`) still go in as long as they fit. The diff is built from text the collector already sent, so the denylist and scrubber apply unchanged, and the trace is scrubbed again as a whole like every other event. To have the "before" side, the collector keeps the scrubbed texts it sent (start snapshots seed it) in a content-addressed store under the app state directory (`omnirush-collector-bases`, owner-only files, 64 MiB at most, least recently used out first), so bases survive a restart; a sign-out deletes it with the spool. Version 1.0.10 and earlier neither read nor delete that directory: after a downgrade it stays on disk, up to 64 MiB of scrubbed workspace text, until it is removed by hand or a later version signs out.
++At the end of each turn the trace gets one `turn.diff` event (`apps/server/src/turn-diff.ts`), pairing the turn with what it changed in the workspace: every file whose redacted sha256 moved between the workspace as the turn's prompt found it and the last accepted manifest. The turn begins when its prompt is sent, not when the queued prompt snapshot runs: a file written after that moment (an edit landing while an earlier upload still holds the queue) is measured from what the collector last saw of it before the prompt, and one it only ever saw already written by the turn is reported as `no_base` without a "before" digest. Likewise, when the next prompt goes out before a turn's own snapshot has run, what the next turn wrote is left to it. The turn observer follows a turn until it settles, however long it runs, reading the engine's status every second at first and less often as the turn goes on (every 10 seconds once it has run ten minutes); it waits out an engine that times out, drops the connection or restarts, noting `session.engine_unavailable` once the engine has not answered for a minute and `session.engine_recovered` when it answers again, and stops following only after 24 hours (`session.observer_timeout`) or on an unexpected error. A turn it stops following still gets its snapshot and `turn.diff`, and the project archive is told the turn ended without completing (see [project-archive.md](project-archive.md)); its messages, like those of a turn cut short by the app quitting, stay after the session's checkpoint and go out with the session's next settled turn, after an app restart too. One trace carries at most 8 MiB of messages: the older messages of a longer turn, or of such an unsent backlog, go ahead in `turn.messages` events (`part` of `parts`), each flushed as its own trace, up to 32 MiB in all, and any older ones are counted in `session.messages_omitted`. Each entry carries the workspace-relative redacted path, a `status` (`added`, `modified`, `deleted`, `skipped` for a binary or over-4-MiB file the turn wrote, sent without content, or `no_base`, for a changed or deleted file, when there is no text from before the turn to diff against: no longer held, or never seen), both redacted digests, the `+`/`-` line counts and a unified diff of the scrubbed text sent before the turn against the scrubbed text sent after it. A binary or oversized file the turn only read (a traced read, an attachment) is not an entry: only one written since the prompt counts. The diff work is bounded: one file's line matching stops after 5 million comparisons and writes the rest of its change as removed and added lines (still a patch that turns the old text into the new one, its counts then an upper bound), and a turn's diffs get 250 ms of work in all, after which the files still waiting for a diff are only counted in `omitted_file_count`. A file diff is cut at 256 KiB and the whole event, envelope included, at 2 MiB (JSON-encoded), both marked `truncated`; files past the event cap are only counted in `omitted_file_count`, while entries without diff text (`skipped`, `no_base`) still go in as long as they fit. The diff is built from text the collector already sent, so the denylist and scrubber apply unchanged, and the trace is scrubbed again as a whole like every other event. To have the "before" side, the collector keeps the scrubbed texts it sent (start snapshots seed it) in a content-addressed store under the app state directory (`omnirush-collector-bases`, owner-only files, 64 MiB at most, least recently used out first), so bases survive a restart; a sign-out deletes it with the spool. Version 1.0.10 and earlier neither read nor delete that directory: after a downgrade it stays on disk, up to 64 MiB of scrubbed workspace text, until it is removed by hand or a later version signs out.