Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: keep long turns in sync while they run

ogsamratadb248b9c84b
diff --git a/apps/server/src/capture-context.test.ts b/apps/server/src/capture-context.test.ts
index 3597aec7..bed233a7 100644
--- a/apps/server/src/capture-context.test.ts
+++ b/apps/server/src/capture-context.test.ts
@@ -844,12 +844,12 @@ test("CLI/desktop parity: the context modules match PARITY.sha256 (the same list
for (const name of files) assert.equal(createHash("sha256").update(fs.readFileSync(path.join(contextDir, name))).digest("hex"), listed.get(name), `${name} changed: port it to the CLI and update PARITY.sha256 in both`);
});
-test("CLI/desktop parity: the shared session-archive modules match their PARITY.sha256 (account park, auto compact, code files, late writes, tool start, turn writers, turn prompts)", async () => {
+test("CLI/desktop parity: the shared session-archive modules match their PARITY.sha256 (account park, auto compact, code files, late writes, tool start, turn writers, turn prompts, turn progress)", async () => {
const { createHash } = await import("node:crypto");
const dir = path.join(import.meta.dir, "session-archive");
const lines = fs.readFileSync(path.join(dir, "PARITY.sha256"), "utf8").split("\n").filter((line) => line && !line.startsWith("#"));
const listed = new Map(lines.map((line) => { const [hash, name] = line.split(/\s+/); return [name, hash]; }));
- assert.deepEqual([...listed.keys()].sort(), ["account-park.ts", "auto-compact.ts", "code-files.ts", "late-writes.ts", "tool-start.ts", "turn-overlap.ts", "turn-prompts.ts", "turn-writers.ts"]);
+ assert.deepEqual([...listed.keys()].sort(), ["account-park.ts", "auto-compact.ts", "code-files.ts", "late-writes.ts", "tool-start.ts", "turn-overlap.ts", "turn-progress.ts", "turn-prompts.ts", "turn-writers.ts"]);
for (const [name, hash] of listed) assert.equal(createHash("sha256").update(fs.readFileSync(path.join(dir, name))).digest("hex"), hash, `${name} changed: port it to the CLI and update PARITY.sha256 in both`);
});
diff --git a/apps/server/src/session-archive/PARITY.sha256 b/apps/server/src/session-archive/PARITY.sha256
index 3586b9e1..dcd6d1b0 100644
--- a/apps/server/src/session-archive/PARITY.sha256
+++ b/apps/server/src/session-archive/PARITY.sha256
@@ -8,3 +8,4 @@ c7f2e9a9c126ea9dfef714d36385e611e2190b0af8043cd52368a08bd0590af5 auto-compact.t
55e5e322b2c372b77714eb3677bee6ea0f3e0a11120ba6edc7d202c264a3d7e8 turn-prompts.ts
ca3bd7e89143925e6d35ae37757e736b9f8c20f6e5911a4f66433684fa185f1c code-files.ts
21459ba1d35b71195e98c7b5385277838248eb6bae316ad16fa9fcc93646d706 account-park.ts
+11431f5e1bcb96bbbbd792afc036602da79a939c19c540ffb86c1e4e9454c716 turn-progress.ts
diff --git a/apps/server/src/session-archive/turn-progress.ts b/apps/server/src/session-archive/turn-progress.ts
new file mode 100644
index 00000000..b2c70d78
--- /dev/null
+++ b/apps/server/src/session-archive/turn-progress.ts
@@ -0,0 +1,203 @@
+/**
+ * A long turn's messages go out while it runs. The same file ships in
+ * omnirush-cli and omnirush-gui (PARITY.sha256): both observers follow a
+ * running turn the same way.
+ *
+ * A turn is uploaded when it ends. One that runs for many minutes (an agent
+ * that starts sub-agents and waits for them) would otherwise send nothing
+ * between its start and its end, and nothing at all when the app is closed
+ * or killed before it ends. While the engine says the session is busy, the
+ * observer sends what is final so far:
+ *
+ * - every TURN_PROGRESS_INTERVAL_MS once the turn is TURN_PROGRESS_MIN_AGE_MS
+ * old, and as soon as a sub-agent of a turn that old ends (a session the
+ * engine listed as busy is not any more), at most once per
+ * TURN_PROGRESS_MIN_GAP_MS;
+ * - only settled messages, oldest first, up to the first one that is not:
+ * a user message, or an assistant message that ended with no tool call
+ * still pending or running, and never a session's newest message (the
+ * engine still adds to a step that just ended: its file patch). A message
+ * still being written is sent when the turn ends, with its final content.
+ *
+ * The root session's settled messages go ahead as `turn.messages` (the
+ * events a long turn's backlog already uses, `progress: true`), each
+ * sub-agent's as a `session.child` event. No checkpoint moves: the turn's
+ * end reads every session from where the turn started, as without
+ * progress, and sends only the messages not sent ahead, or sent ahead and
+ * changed since (their size differs, as a prompt does once the engine adds
+ * its summary): each message goes up once more at most, in its final
+ * form. Everything the turn's end derives from its messages (outcome,
+ * writers, files read and used, the archive, the checkpoints) still reads
+ * all of them.
+ */
+export const TURN_PROGRESS_INTERVAL_MS = 3 * 60_000;
+export const TURN_PROGRESS_MIN_AGE_MS = 60_000;
+export const TURN_PROGRESS_MIN_GAP_MS = 15_000;
+/** The shortest interval an override may set. */
+export const TURN_PROGRESS_FLOOR_MS = 10_000;
+
+export type TurnProgressReason = "interval" | "sub_agent_ended";
+
+type MessageRecord = Record<string, unknown>;
+
+function isRecord(value: unknown): value is MessageRecord {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function infoOf(message: MessageRecord): MessageRecord {
+ return isRecord(message.info) ? message.info : message;
+}
+
+/** The interval set by OMNIRUSH_TURN_PROGRESS_MS: "0" turns it off, anything unreadable keeps the default. */
+export function turnProgressIntervalMs(value: string | undefined): number {
+ if (value === undefined || value.trim() === "") return TURN_PROGRESS_INTERVAL_MS;
+ const parsed = Number(value);
+ if (!Number.isFinite(parsed) || parsed < 0) return TURN_PROGRESS_INTERVAL_MS;
+ if (parsed === 0) return 0;
+ return Math.max(TURN_PROGRESS_FLOOR_MS, Math.floor(parsed));
+}
+
+/** An engine message's id, or null. */
+export function progressMessageId(message: unknown): string | null {
+ if (!isRecord(message)) return null;
+ const id = infoOf(message).id;
+ return typeof id === "string" && id ? id : null;
+}
+
+/** Whether a message is final: a user message, or an assistant message that ended with no tool call still open. */
+export function settledMessage(message: unknown): boolean {
+ if (!isRecord(message)) return false;
+ const info = infoOf(message);
+ const role = info.role ?? info.type;
+ if (role === "user") return true;
+ if (role !== "assistant") return false;
+ const time = isRecord(info.time) ? info.time : null;
+ const ended = (time !== null && (typeof time.completed === "number" || typeof time.completed === "string")) || info.error != null;
+ if (!ended) return false;
+ const parts = Array.isArray(message.parts) ? message.parts : [];
+ return !parts.some((part) => {
+ if (!isRecord(part) || part.type !== "tool") return false;
+ const state = isRecord(part.state) ? part.state.status : undefined;
+ return state === "pending" || state === "running";
+ });
+}
+
+/** How many messages, from the oldest, are settled before the first one that is not. */
+export function settledPrefix(messages: readonly unknown[]): number {
+ let count = 0;
+ while (count < messages.length && settledMessage(messages[count])) count += 1;
+ return count;
+}
+
+/**
+ * One followed turn's progress: when to send, and which of the root's
+ * messages went ahead. Fed with every status read of the engine while the
+ * turn runs.
+ */
+export class TurnProgress {
+ /** Messages sent ahead of the turn's end, by session ("" for the root): message id → size when sent. */
+ private readonly sent = new Map<string, Map<string, number>>();
+ private readonly startedAt: number;
+ private readonly intervalMs: number;
+ private lastAt: number | null = null;
+ private busy = new Set<string>();
+ private ended = false;
+
+ constructor(startedAt: number, intervalMs: number) {
+ this.startedAt = startedAt;
+ this.intervalMs = intervalMs;
+ }
+
+ get enabled(): boolean {
+ return this.intervalMs > 0;
+ }
+
+ /**
+ * One read of the engine's statuses (session id → { type }): a session
+ * other than the root that was busy and is no longer listed busy ended
+ * (a sub-agent, as far as the root's children say when they are read).
+ */
+ noteStatuses(statuses: unknown, rootSessionId: string): void {
+ if (!isRecord(statuses)) return;
+ const now = new Set<string>();
+ for (const [id, status] of Object.entries(statuses)) {
+ if (id === rootSessionId) continue;
+ const type = isRecord(status) ? status.type : undefined;
+ if (typeof type === "string" && type !== "idle") now.add(id);
+ }
+ for (const id of this.busy) {
+ if (!now.has(id)) this.ended = true;
+ }
+ this.busy = now;
+ }
+
+ /** Why the turn's progress is due now, or null. */
+ due(now: number): TurnProgressReason | null {
+ if (!this.enabled) return null;
+ if (now - this.startedAt < TURN_PROGRESS_MIN_AGE_MS) return null;
+ const since = this.lastAt ?? this.startedAt;
+ if (this.ended && (this.lastAt === null || now - this.lastAt >= TURN_PROGRESS_MIN_GAP_MS)) return "sub_agent_ended";
+ if (now - since >= this.intervalMs) return "interval";
+ return null;
+ }
+
+ /** The progress went out (or was tried): the next one is measured from now. */
+ flushed(now: number): void {
+ this.lastAt = now;
+ this.ended = false;
+ }
+
+ /** How many messages went ahead in all. */
+ get sentCount(): number {
+ let count = 0;
+ for (const session of this.sent.values()) count += session.size;
+ return count;
+ }
+
+ /**
+ * A session's settled messages not sent ahead yet, with their sizes
+ * (`sizes` matches `messages`; `session` is "" for the root, else the
+ * sub-agent's id). Its newest message is never among them.
+ */
+ ahead(messages: readonly unknown[], sizes: readonly number[], session = ""): { messages: unknown[]; sizes: number[] } {
+ const count = settledPrefix(messages.slice(0, -1));
+ const sent = this.sent.get(session);
+ const out: unknown[] = [];
+ const outSizes: number[] = [];
+ for (let index = 0; index < count; index += 1) {
+ const id = progressMessageId(messages[index]);
+ if (!id || sent?.has(id)) continue;
+ out.push(messages[index]);
+ outSizes.push(sizes[index] ?? 0);
+ }
+ return { messages: out, sizes: outSizes };
+ }
+
+ /** Marks a session's messages as sent ahead (`sizes` matches `messages`). */
+ markSent(messages: readonly unknown[], sizes: readonly number[], session = ""): void {
+ let sent = this.sent.get(session);
+ if (!sent) {
+ sent = new Map();
+ this.sent.set(session, sent);
+ }
+ for (const [index, message] of messages.entries()) {
+ const id = progressMessageId(message);
+ if (id) sent.set(id, sizes[index] ?? 0);
+ }
+ }
+
+ /** The turn's messages its end still has to send: those not sent ahead, or changed since, with their sizes. */
+ unsent(messages: readonly unknown[], sizes: readonly number[], session = ""): { messages: unknown[]; sizes: number[] } {
+ const sent = this.sent.get(session);
+ if (!sent || sent.size === 0) return { messages: messages.slice(), sizes: sizes.slice() };
+ const out: unknown[] = [];
+ const outSizes: number[] = [];
+ for (const [index, message] of messages.entries()) {
+ const id = progressMessageId(message);
+ if (id && sent.get(id) === (sizes[index] ?? 0)) continue;
+ out.push(message);
+ outSizes.push(sizes[index] ?? 0);
+ }
+ return { messages: out, sizes: outSizes };
+ }
+}
diff --git a/apps/server/src/session-upload-observer-progress.test.ts b/apps/server/src/session-upload-observer-progress.test.ts
new file mode 100644
index 00000000..6f96ecd3
--- /dev/null
+++ b/apps/server/src/session-upload-observer-progress.test.ts
@@ -0,0 +1,237 @@
+import { afterEach, expect, test } from "bun:test";
+
+import { createSessionObservers, observeUploadedSession, type EngineTarget, type ObservedUploader } from "./session-upload-observer.js";
+import { settledMessage, settledPrefix, TurnProgress, turnProgressIntervalMs, TURN_PROGRESS_FLOOR_MS, TURN_PROGRESS_INTERVAL_MS } from "./session-archive/turn-progress.js";
+
+// A long turn sends what is final while it runs (session-archive/turn-progress.ts,
+// the same file as the CLI's, PARITY.sha256): the parent's settled messages
+// ahead as turn.messages, a sub-agent's as session.child as soon as it ends,
+// and the turn's end only what was not sent ahead (or changed since). The
+// CLI's test/turn-progress.test.js runs the same cases.
+
+type Message = { info: Record<string, unknown> & { id: string }; parts: unknown[] };
+
+const user = (id: string, text: string, created: number): Message => ({ info: { id, role: "user", time: { created } }, parts: [{ type: "text", text }] });
+const running = (id: string, created: number, tool = "task"): Message => ({ info: { id, role: "assistant", time: { created } }, parts: [{ type: "tool", tool, state: { status: "running" } }] });
+const done = (id: string, created: number, extra: unknown[] = []): Message => ({ info: { id, role: "assistant", finish: "stop", time: { created, completed: created + 1 } }, parts: [{ type: "text", text: `answer ${id}` }, { type: "step-finish" }, ...extra] });
+
+const cleanups: Array<() => void> = [];
+afterEach(() => {
+ while (cleanups.length > 0) cleanups.pop()!();
+});
+
+test("settled messages: a user message, an ended answer with no open tool call", () => {
+ expect(settledMessage(user("u", "hi", 1))).toBe(true);
+ expect(settledMessage(done("a", 1))).toBe(true);
+ expect(settledMessage(running("b", 1))).toBe(false);
+ expect(settledMessage({ info: { id: "c", role: "assistant", time: { created: 1, completed: 2 } }, parts: [{ type: "tool", state: { status: "pending" } }] })).toBe(false);
+ expect(settledPrefix([user("u", "x", 1), done("a", 2), running("b", 3), done("c", 4)])).toBe(2);
+});
+
+test("the interval: default, off, floor, unreadable", () => {
+ expect(turnProgressIntervalMs(undefined)).toBe(TURN_PROGRESS_INTERVAL_MS);
+ expect(turnProgressIntervalMs("0")).toBe(0);
+ expect(turnProgressIntervalMs("10")).toBe(TURN_PROGRESS_FLOOR_MS);
+ expect(turnProgressIntervalMs("120000")).toBe(120_000);
+ expect(turnProgressIntervalMs("soon")).toBe(TURN_PROGRESS_INTERVAL_MS);
+});
+
+test("when progress is due: never in the first minute, then on the interval or a sub-agent's end", () => {
+ const progress = new TurnProgress(0, 180_000);
+ progress.noteStatuses({ root: { type: "busy" }, c1: { type: "busy" } }, "root");
+ progress.noteStatuses({ root: { type: "busy" } }, "root");
+ expect(progress.due(30_000)).toBeNull();
+ expect(progress.due(61_000)).toBe("sub_agent_ended");
+ progress.flushed(61_000);
+ expect(progress.due(200_000)).toBeNull();
+ expect(progress.due(241_000)).toBe("interval");
+ progress.flushed(241_000);
+ progress.noteStatuses({ root: { type: "busy" }, c2: { type: "busy" } }, "root");
+ progress.noteStatuses({ root: { type: "busy" } }, "root");
+ expect(progress.due(250_000)).toBeNull();
+ expect(progress.due(256_000)).toBe("sub_agent_ended");
+ const off = new TurnProgress(0, 0);
+ expect(off.enabled).toBe(false);
+ expect(off.due(10 * 60_000)).toBeNull();
+});
+
+test("sent ahead once, never a session's newest message; changed since: sent again at the end", () => {
+ const progress = new TurnProgress(0, 180_000);
+ const messages = [user("u1", "go", 1), done("a1", 2), running("a2", 3)];
+ const ahead = progress.ahead(messages, [10, 20, 30]);
+ expect(ahead.messages.map((m) => (m as Message).info.id)).toEqual(["u1", "a1"]);
+ progress.markSent(ahead.messages, ahead.sizes);
+ expect(progress.ahead(messages, [10, 20, 30]).messages).toEqual([]);
+ const unsent = progress.unsent(messages, [10, 25, 30]);
+ expect(unsent.messages.map((m) => (m as Message).info.id)).toEqual(["a1", "a2"]);
+ expect(unsent.sizes).toEqual([25, 30]);
+ // A session that ended: its newest message waits for the turn's end (the engine still adds its patch).
+ const child = [user("cu", "go", 1), done("ca1", 2), done("ca2", 3)];
+ expect(progress.ahead(child, [1, 2, 3], "ses_c").messages.map((m) => (m as Message).info.id)).toEqual(["cu", "ca1"]);
+ expect(progress.unsent(child, [1, 2, 3], "ses_c").messages.length).toBe(3);
+ expect(progress.sentCount).toBe(2);
+});
+
+type Phase = { statuses: Record<string, unknown>; children: Record<string, Array<{ id: string; title?: string }>>; messages: Record<string, Message[]> };
+
+/** A scripted engine: `phase()` gives the statuses, the children and every session's messages. */
+function scriptedEngine(phase: () => Phase): EngineTarget {
+ const server = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ fetch(request) {
+ const url = new URL(request.url);
+ const now = phase();
+ if (url.pathname === "/session/status") return Response.json(now.statuses);
+ const children = /^\/session\/([^/]+)\/children$/.exec(url.pathname);
+ if (children) return Response.json(now.children[decodeURIComponent(children[1]!)] ?? []);
+ const list = /^\/session\/([^/]+)\/message$/.exec(url.pathname);
+ if (list) {
+ const all = now.messages[decodeURIComponent(list[1]!)] ?? [];
+ const limit = Number(url.searchParams.get("limit") ?? all.length);
+ const before = url.searchParams.get("before");
+ const end = before ? all.findIndex((message) => message.info.id === before) : all.length;
+ const page = all.slice(Math.max(0, end - limit), end);
+ return Response.json(page, { headers: end - limit > 0 ? { "x-next-cursor": page[0]!.info.id } : {} });
+ }
+ return Response.json({ id: "ses_root" });
+ },
+ });
+ cleanups.push(() => server.stop(true));
+ return { baseUrl: `http://127.0.0.1:${server.port}`, headers: [], search: "", engine: "v1" };
+}
+
+type Event = { at: number; type: string; data?: unknown };
+
+function recordingUploader(clock: () => number, events: Event[], flushes: Array<{ at: number; final: unknown }>, checkpoint: string | null): ObservedUploader {
+ const childCheckpoints: Record<string, string> = {};
+ const childIds: string[] = [];
+ return {
+ enabled: true,
+ sessionCheckpoint: async () => (checkpoint ? { lastMessageId: checkpoint } : {}),
+ setSessionCheckpoint: async () => undefined,
+ recordTrace: (_id: string, type: string, data?: unknown) => { events.push({ at: clock(), type, data }); },
+ recordSessionModel: () => undefined,
+ childCheckpoints: async () => ({ ...childCheckpoints }),
+ childSessionIds: async () => [...childIds],
+ recordChildSession: (_id: string, child: { childSessionId: string; lastMessageId?: string | null }) => {
+ if (!childIds.includes(child.childSessionId)) childIds.push(child.childSessionId);
+ if (child.lastMessageId) childCheckpoints[child.childSessionId] = child.lastMessageId;
+ events.push({ at: clock(), type: "session.child", data: child });
+ },
+ noteChildSessionIds: () => 0,
+ captureSnapshot: () => undefined,
+ flushTrace: (_id: string, data?: unknown) => { flushes.push({ at: clock(), final: data ?? null }); },
+ } as unknown as ObservedUploader;
+}
+
+const archive = { turnFollowed: () => undefined, turnCompleted: () => undefined, turnIncomplete: () => { throw new Error("incomplete"); } };
+const ids = (messages: unknown): string[] => (messages as Message[]).map((message) => message.info.id);
+
+test("a long turn with a sub-agent: progress while it runs, the end sends the rest once", async () => {
+ let clock = 0;
+ const before = [user("u0", "earlier", 0), done("a0", 1)];
+ const childFinal = [user("cu1", "research", 10), done("ca1", 11), done("ca2", 12), done("ca3", 13)];
+ // 0-70 s: the parent waits on the sub-agent, which works; from 70 s the sub-agent is done;
+ // from 150 s the parent's first step ended and its second runs; from 250 s that first step
+ // carries one more part (it changed after it went ahead); from 300 s the turn is over.
+ const target = scriptedEngine(() => {
+ const childRunning = clock < 70_000;
+ const a1 = clock < 150_000 ? running("a1", 3) : clock < 250_000 ? done("a1", 3) : done("a1", 3, [{ type: "patch", hash: "h" }]);
+ const root = [...before, user("u1", "deep research", 2), a1];
+ if (clock >= 150_000) root.push(clock < 300_000 ? running("a2", 4, "bash") : done("a2", 4));
+ return {
+ statuses: clock >= 300_000 ? {} : childRunning ? { ses_root: { type: "busy" }, ses_c1: { type: "busy" } } : { ses_root: { type: "busy" } },
+ children: { ses_root: [{ id: "ses_c1", title: "helper" }] },
+ messages: { ses_root: root, ses_c1: childRunning ? [...childFinal.slice(0, 2), running("ca2", 12, "bash")] : childFinal },
+ };
+ });
+ const events: Event[] = [];
+ const flushes: Array<{ at: number; final: unknown }> = [];
+ const observers = createSessionObservers();
+ cleanups.push(() => observers.controller.abort());
+ await observeUploadedSession({
+ sessionUploader: recordingUploader(() => clock, events, flushes, "a0"),
+ archive,
+ observers,
+ sessionId: "ses_root",
+ target,
+ timing: { now: () => clock, sleep: async (ms: number) => { clock += ms; }, progressMs: 100_000 },
+ });
+
+ const progress = events.filter((event) => event.type === "turn.progress");
+ expect(progress.map((event) => (event.data as { reason: string }).reason)).toEqual(["sub_agent_ended", "interval"]);
+ expect(progress[0]!.at).toBeGreaterThanOrEqual(70_000);
+ expect(progress[0]!.at).toBeLessThan(85_000);
+
+ // The sub-agent: all but its newest message as soon as it ended, the newest at the turn's end;
+ // its checkpoint moves only then, as without progress.
+ const childEvents = events.filter((event) => event.type === "session.child");
+ expect(childEvents.map((event) => ids((event.data as { messages: unknown }).messages))).toEqual([["cu1", "ca1", "ca2"], ["ca3"]]);
+ expect(childEvents[0]!.at).toBeLessThan(85_000);
+ expect(childEvents[1]!.at).toBeGreaterThanOrEqual(300_000);
+ expect(childEvents.map((event) => (event.data as { lastMessageId: string | null }).lastMessageId)).toEqual([null, "ca3"]);
+
+ const ahead = events.filter((event) => event.type === "turn.messages" && (event.data as { progress?: boolean }).progress === true);
+ expect(ahead.map((event) => ids((event.data as { messages: unknown }).messages))).toEqual([["u1"], ["a1"]]);
+ const end = flushes.filter((flush) => flush.final);
+ expect(end.length).toBe(1);
+ const final = end[0]!.final as { messages: Message[] };
+ expect(ids(final.messages)).toEqual(["a1", "a2"]);
+ expect((final.messages[0]!.parts.at(-1) as { type: string }).type).toBe("patch");
+ expect(end[0]!.at).toBeGreaterThanOrEqual(300_000);
+ expect(end[0]!.at).toBeLessThan(315_000);
+ for (const event of progress) expect(flushes.some((flush) => flush.at === event.at && !flush.final)).toBe(true);
+});
+
+test("progress off (OMNIRUSH_TURN_PROGRESS_MS=0): the turn's end alone, as before", async () => {
+ let clock = 0;
+ const target = scriptedEngine(() => ({
+ statuses: clock >= 300_000 ? {} : { ses_root: { type: "busy" }, ses_c1: { type: clock < 70_000 ? "busy" : "idle" } },
+ children: { ses_root: [{ id: "ses_c1", title: "helper" }] },
+ messages: {
+ ses_root: [user("u1", "go", 2), clock < 300_000 ? running("a1", 3) : done("a1", 3)],
+ ses_c1: [user("cu1", "research", 10), done("ca1", 11)],
+ },
+ }));
+ const events: Event[] = [];
+ const flushes: Array<{ at: number; final: unknown }> = [];
+ const observers = createSessionObservers();
+ cleanups.push(() => observers.controller.abort());
+ await observeUploadedSession({
+ sessionUploader: recordingUploader(() => clock, events, flushes, null),
+ archive,
+ observers,
+ sessionId: "ses_root",
+ target,
+ timing: { now: () => clock, sleep: async (ms: number) => { clock += ms; }, progressMs: 0 },
+ });
+ expect(events.filter((event) => event.type === "turn.progress" || (event.type === "turn.messages" && (event.data as { progress?: boolean }).progress)).length).toBe(0);
+ expect(events.filter((event) => event.type === "session.child").length).toBe(1);
+ expect(ids((flushes.filter((flush) => flush.final).at(-1)!.final as { messages: unknown }).messages)).toEqual(["u1", "a1"]);
+});
+
+test("what the turn's context collects after its end goes out once it is in, not at the next prompt", async () => {
+ let clock = 0;
+ const target = scriptedEngine(() => ({ statuses: clock >= 5_000 ? {} : { ses_root: { type: "busy" } }, children: {}, messages: { ses_root: [user("u1", "go", 1), clock >= 5_000 ? done("a1", 2) : running("a1", 2)] } }));
+ const events: Event[] = [];
+ const flushes: Array<{ at: number; final: unknown }> = [];
+ let release: () => void = () => undefined;
+ const contextIn = new Promise<void>((resolve) => { release = resolve; });
+ const uploader = recordingUploader(() => clock, events, flushes, null) as unknown as Record<string, unknown>;
+ uploader.context = { settled: async (_id: string, waitMs: number) => { expect(waitMs).toBeGreaterThanOrEqual(10_000); await contextIn; } };
+ const observers = createSessionObservers();
+ cleanups.push(() => observers.controller.abort());
+ await observeUploadedSession({
+ sessionUploader: uploader as unknown as ObservedUploader,
+ archive,
+ observers,
+ sessionId: "ses_root",
+ target,
+ timing: { now: () => clock, sleep: async (ms: number) => { clock += ms; } },
+ });
+ expect(flushes.map((flush) => (flush.final ? "turn" : "plain"))).toEqual(["turn"]);
+ release();
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(flushes.map((flush) => (flush.final ? "turn" : "plain"))).toEqual(["turn", "plain"]);
+});
diff --git a/apps/server/src/session-upload-observer.test.ts b/apps/server/src/session-upload-observer.test.ts
index 51978a6d..5c15fb23 100644
--- a/apps/server/src/session-upload-observer.test.ts
+++ b/apps/server/src/session-upload-observer.test.ts
@@ -154,6 +154,8 @@ function fakeClock() {
await new Promise((resolvePromise) => setImmediate(resolvePromise));
if (signal.aborted) throw signal.reason;
},
+ // These cases follow a turn to its end only; a running turn's progress has its own (session-upload-observer-progress.test.ts).
+ progressMs: 0,
};
return {
timing,
diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.ts
index f345e20a..f8bdd1aa 100644
--- a/apps/server/src/session-upload-observer.ts
+++ b/apps/server/src/session-upload-observer.ts
@@ -16,6 +16,7 @@ import { turnWriters, type WriterSessionMessages } from "./session-archive/turn-
import { watchToolStarts } from "./session-archive/tool-start.js";
import { watchToolEvents } from "./context/tool-events.js";
import { closeOpenToolParts } from "./context/open-tools.js";
+import { TurnProgress, turnProgressIntervalMs, type TurnProgressReason } from "./session-archive/turn-progress.js";
import type { ContextCapture } from "./context/index.js";
/** The engine a captured request went to: base URL, request headers (the engine's auth), query and API generation. */
@@ -260,6 +261,8 @@ const CUT_PROMPT_WAIT_MS = 15_000;
const CHILD_IDS_INTERVAL_MS = 60_000;
/** A settled turn whose messages the engine did not answer for (a timeout, a dropped connection) is read again after these waits. */
const HISTORY_RETRY_DELAYS_MS = [2_000, 5_000];
+/** How long a settled turn's context may still collect before what it found is flushed. */
+const TURN_CONTEXT_WAIT_MS = 30_000;
/** The observer's clock and timeouts; tests pass their own so that hours of a turn pass without real waiting. */
export type ObserverTiming = {
@@ -271,6 +274,8 @@ export type ObserverTiming = {
readTimeoutMs: number;
/** How long one turn is followed at most. */
maxTurnMs: number;
+ /** A running turn's settled messages go out this often (session-archive/turn-progress.ts); 0: only at its end. */
+ progressMs: number;
};
type EngineFetch = (path: string, query?: Record<string, string>) => Promise<Response>;
@@ -335,6 +340,15 @@ async function captureChildSessions(input: {
* only those of answers that ended; `closed` counts what was closed.
*/
toolsClosed?: { engineIdle: boolean; closed: number };
+ /**
+ * The turn's progress (session-archive/turn-progress.ts). With `ahead`
+ * the turn still runs: each child's settled messages not sent yet go out
+ * and no checkpoint moves (a child over the read budget is left to the
+ * turn's end). Without it (the turn's end), a child's messages sent ahead
+ * and unchanged are not sent again.
+ */
+ progress?: TurnProgress | null;
+ ahead?: boolean;
}): Promise<void> {
if (input.depth > MAX_UPLOAD_CHILD_SESSION_DEPTH) return;
const children = await readEngineJson(input.fetchEngine, `/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024)
@@ -346,6 +360,27 @@ async function captureChildSessions(input: {
input.visited.add(childId);
const history = await readEngineHistory(engineMessages(input.fetchEngine, `/session/${encodeURIComponent(childId)}/message`, true), input.checkpoints[childId])
.catch(nullOnErrorStatus);
+ if (input.ahead) {
+ if (history && input.progress && history.omitted === 0 && !history.checkpointMissing) {
+ const sending = input.progress.ahead(history.delta, history.sizes, childId);
+ if (sending.messages.length > 0 || !input.known.has(childId)) {
+ input.beforeRecord(sum(sending.sizes));
+ input.sessionUploader.recordChildSession(input.rootSessionId, {
+ childSessionId: childId,
+ parentSessionId: input.parentSessionId,
+ depth: input.depth,
+ title: isRecord(child) ? optionalTraceString(child.title) : null,
+ agent: turnModelFromMessages(history.outline)?.agent ?? null,
+ messages: sending.messages,
+ lastMessageId: null,
+ });
+ input.progress.markSent(sending.messages, sending.sizes, childId);
+ if (sending.messages.length > 0) input.onChild?.(childId, input.depth, sending.messages);
+ }
+ }
+ await captureChildSessions({ ...input, parentSessionId: childId, depth: input.depth + 1 });
+ continue;
+ }
// A sub-agent may still run: unless the engine no longer does, only the calls of its answers that ended are closed.
const closing = history ? closeOpenToolParts(history.delta, { engineIdle: input.toolsClosed?.engineIdle ?? false }) : null;
if (closing && input.toolsClosed) input.toolsClosed.closed += closing.closed;
@@ -355,14 +390,16 @@ async function captureChildSessions(input: {
input.sessionUploader.recordTrace(input.rootSessionId, "session.messages_omitted", { child_session_id: childId, count: history.omitted });
}
if (newMessages.length > 0 || !input.known.has(childId)) {
- input.beforeRecord(history ? sum(history.sizes) : 0);
+ // Sent ahead while the turn ran and unchanged: not again.
+ const sending = input.progress ? input.progress.unsent(newMessages, history?.sizes ?? [], childId) : { messages: newMessages, sizes: history?.sizes ?? [] };
+ input.beforeRecord(sum(sending.sizes));
input.sessionUploader.recordChildSession(input.rootSessionId, {
childSessionId: childId,
parentSessionId: input.parentSessionId,
depth: input.depth,
title: isRecord(child) ? optionalTraceString(child.title) : null,
agent: turnModelFromMessages(history?.outline ?? [])?.agent ?? null,
- messages: newMessages,
+ messages: sending.messages,
lastMessageId: traceMessageId(history?.outline.at(-1)),
});
}
@@ -996,6 +1033,7 @@ export function observeUploadedSession(input: {
statusTimeoutMs: 10_000,
readTimeoutMs: 20_000,
maxTurnMs: MAX_OBSERVED_TURN_MS,
+ progressMs: turnProgressIntervalMs(process.env.OMNIRUSH_TURN_PROGRESS_MS),
...input.timing,
};
const stopped = observer.controller.signal;
@@ -1024,9 +1062,12 @@ export function observeUploadedSession(input: {
else sessionUploader.captureSnapshot(sessionId, "turn_completed");
};
+ // Every session's status at the last read: a running turn's progress sees its sub-agents end there.
+ let lastStatuses: unknown = null;
/** The session's status ("idle" when the engine does not list it); throws when the engine does not answer. */
const readStatus = async (): Promise<string> => {
const statuses = enginePayload(await readEngineJson(fetchStatus, v2 ? "/api/session/active" : "/session/status", 1024 * 1024));
+ lastStatuses = statuses;
const status = isRecord(statuses) ? statuses[sessionId] : undefined;
return isRecord(status) && typeof status.type === "string" ? status.type : "idle";
};
@@ -1086,6 +1127,80 @@ export function observeUploadedSession(input: {
return false;
};
+ // The followed turn's progress (session-archive/turn-progress.ts).
+ let progress: TurnProgress | null = null;
+ let progressStartedAt = timing.now();
+ /**
+ * What the turn's context collects once it ended (its files outside the
+ * folder, its network, its installs) goes out as soon as it is in, not with
+ * the next prompt.
+ */
+ const flushTurnContext = (): void => {
+ const context = (sessionUploader as Partial<Pick<SessionUploader, "context">>).context;
+ if (!context) return;
+ void context.settled(sessionId, TURN_CONTEXT_WAIT_MS).then(() => {
+ if (!stopped.aborted) sessionUploader.flushTrace(sessionId);
+ }, () => undefined);
+ };
+ /**
+ * The running turn's settled messages so far: the root's as `turn.messages`
+ * (progress), the sub-agents' as `session.child`, then a flush. The turn's
+ * checkpoint stays where it started.
+ */
+ const sendProgress = async (current: TurnProgress, reason: TurnProgressReason): Promise<void> => {
+ let rootMessages = 0;
+ let childMessages = 0;
+ let pending = 0;
+ const room = (bytes: number) => {
+ if (pending > 0 && pending + bytes > MAX_TURN_MESSAGE_BYTES) {
+ sessionUploader.flushTrace(sessionId);
+ pending = 0;
+ }
+ pending += bytes;
+ };
+ try {
+ const history = await readEngineHistory(messages(), observer.lastMessageIds.get(sessionId), MAX_BACKLOG_MESSAGE_BYTES);
+ if (history.omitted === 0 && !history.checkpointMissing) {
+ const ahead = current.ahead(history.delta, history.sizes);
+ for (const part of ahead.messages.length > 0 ? messageParts(ahead.messages, ahead.sizes) : []) {
+ room(part.bytes);
+ sessionUploader.recordTrace(sessionId, "turn.messages", { messages: part.messages, progress: true });
+ }
+ current.markSent(ahead.messages, ahead.sizes);
+ rootMessages = ahead.messages.length;
+ }
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ }
+ if (!v2) {
+ try {
+ await captureChildSessions({
+ onChild: (_childId, _depth, sent) => {
+ childMessages += sent.length;
+ },
+ sessionUploader,
+ rootSessionId: sessionId,
+ parentSessionId: sessionId,
+ depth: 1,
+ fetchEngine,
+ checkpoints: await sessionUploader.childCheckpoints(sessionId),
+ known: new Set(await sessionUploader.childSessionIds(sessionId)),
+ visited: new Set([sessionId]),
+ beforeRecord: room,
+ progress: current,
+ ahead: true,
+ });
+ } catch (error) {
+ if (stopped.aborted) throw error;
+ }
+ }
+ if (rootMessages + childMessages > 0) {
+ sessionUploader.recordTrace(sessionId, "turn.progress", { reason, elapsed_ms: Math.max(0, timing.now() - progressStartedAt), messages: rootMessages, sub_agent_messages: childMessages });
+ }
+ // Whatever else the turn recorded so far (its context, its sub-agents) goes with it.
+ sessionUploader.flushTrace(sessionId);
+ };
+
let recovering = input.recovered === true;
const settle = async (status: string, read?: EngineHistory): Promise<void> => {
// Once per turn: an app quit may settle it while the follow loop is about to.
@@ -1136,7 +1251,9 @@ export function observeUploadedSession(input: {
const model = turnModelFromMessages(history.delta);
if (model) sessionUploader.recordSessionModel(sessionId, model);
// More messages than one flush holds: the older ones go ahead, a flush per part.
- parts = messageParts(history.delta, history.sizes);
+ // Messages sent ahead while the turn ran (turn-progress.ts) are not sent again, unless they changed since.
+ const unsent = progress ? progress.unsent(history.delta, history.sizes) : { messages: history.delta, sizes: history.sizes };
+ parts = messageParts(unsent.messages, unsent.sizes);
for (const [index, part] of parts.slice(0, -1).entries()) {
sessionUploader.recordTrace(sessionId, "turn.messages", { messages: part.messages, part: index + 1, parts: parts.length });
sessionUploader.flushTrace(sessionId);
@@ -1171,6 +1288,7 @@ export function observeUploadedSession(input: {
visited: new Set([sessionId]),
beforeRecord: room,
toolsClosed: childTools,
+ progress,
});
} catch (error) {
if (stopped.aborted) throw error;
@@ -1200,6 +1318,7 @@ export function observeUploadedSession(input: {
...(recovered ? { recovered: true } : {}),
...(status === "quit" ? { reason: "app_quit" } : {}),
});
+ if (status !== "quit") flushTurnContext();
// The delta's turn number is the engine's completed-turn count, which
// survives app restarts; without the messages the archiver numbers it
// right after the last archived turn. It also arms the idle final archive.
@@ -1220,6 +1339,9 @@ export function observeUploadedSession(input: {
// A turn is running: the previous turn's idle final archive, if armed, is off.
input.archive.turnFollowed(sessionId);
const startedAt = timing.now();
+ progressStartedAt = startedAt;
+ const current = new TurnProgress(startedAt, timing.progressMs);
+ progress = current;
let observedBusy = false;
let reads = 0;
let idleReads = 0;
@@ -1297,6 +1419,15 @@ export function observeUploadedSession(input: {
observedBusy = true;
idleReads = 0;
turnState().idleAt = null;
+ // A long turn: what is final so far goes out now (turn-progress.ts).
+ if (current.enabled) {
+ current.noteStatuses(lastStatuses, sessionId);
+ const reason = current.due(timing.now());
+ if (reason) {
+ await sendProgress(current, reason);
+ current.flushed(timing.now());
+ }
+ }
// The v2 daemon has no children route: nothing to list there.
if (!v2 && timing.now() - childIdsAt >= CHILD_IDS_INTERVAL_MS) {
childIdsAt = timing.now();