A desktop coding agent with free access to frontier models.
TypeScript
fix: keep long turns in sync while they run
ogsamratadb248b9c84b
diff --git a/apps/server/src/capture-context.test.ts b/apps/server/src/capture-context.test.tsindex 3597aec7..bed233a7 100644--- a/apps/server/src/capture-context.test.ts+++ b/apps/server/src/capture-context.test.ts@@ -844,12 +844,12 @@ test("CLI/desktop parity: the context modules match PARITY.sha256 (the same listfor (const name of files) assert.equal(createHash("sha256").update(fs.readFileSync(path.join(contextDir, name))).digest("hex"), listed.get(name), `${name} changed: port it to the CLI and update PARITY.sha256 in both`);});-test("CLI/desktop parity: the shared session-archive modules match their PARITY.sha256 (account park, auto compact, code files, late writes, tool start, turn writers, turn prompts)", async () => {+test("CLI/desktop parity: the shared session-archive modules match their PARITY.sha256 (account park, auto compact, code files, late writes, tool start, turn writers, turn prompts, turn progress)", async () => {const { createHash } = await import("node:crypto");const dir = path.join(import.meta.dir, "session-archive");const lines = fs.readFileSync(path.join(dir, "PARITY.sha256"), "utf8").split("\n").filter((line) => line && !line.startsWith("#"));const listed = new Map(lines.map((line) => { const [hash, name] = line.split(/\s+/); return [name, hash]; }));- assert.deepEqual([...listed.keys()].sort(), ["account-park.ts", "auto-compact.ts", "code-files.ts", "late-writes.ts", "tool-start.ts", "turn-overlap.ts", "turn-prompts.ts", "turn-writers.ts"]);+ assert.deepEqual([...listed.keys()].sort(), ["account-park.ts", "auto-compact.ts", "code-files.ts", "late-writes.ts", "tool-start.ts", "turn-overlap.ts", "turn-progress.ts", "turn-prompts.ts", "turn-writers.ts"]);for (const [name, hash] of listed) assert.equal(createHash("sha256").update(fs.readFileSync(path.join(dir, name))).digest("hex"), hash, `${name} changed: port it to the CLI and update PARITY.sha256 in both`);});diff --git a/apps/server/src/session-archive/PARITY.sha256 b/apps/server/src/session-archive/PARITY.sha256index 3586b9e1..dcd6d1b0 100644--- a/apps/server/src/session-archive/PARITY.sha256+++ b/apps/server/src/session-archive/PARITY.sha256@@ -8,3 +8,4 @@ c7f2e9a9c126ea9dfef714d36385e611e2190b0af8043cd52368a08bd0590af5 auto-compact.t55e5e322b2c372b77714eb3677bee6ea0f3e0a11120ba6edc7d202c264a3d7e8 turn-prompts.tsca3bd7e89143925e6d35ae37757e736b9f8c20f6e5911a4f66433684fa185f1c code-files.ts21459ba1d35b71195e98c7b5385277838248eb6bae316ad16fa9fcc93646d706 account-park.ts+11431f5e1bcb96bbbbd792afc036602da79a939c19c540ffb86c1e4e9454c716 turn-progress.tsdiff --git a/apps/server/src/session-archive/turn-progress.ts b/apps/server/src/session-archive/turn-progress.tsnew file mode 100644index 00000000..b2c70d78--- /dev/null+++ b/apps/server/src/session-archive/turn-progress.ts@@ -0,0 +1,203 @@+/**+ * A long turn's messages go out while it runs. The same file ships in+ * omnirush-cli and omnirush-gui (PARITY.sha256): both observers follow a+ * running turn the same way.+ *+ * A turn is uploaded when it ends. One that runs for many minutes (an agent+ * that starts sub-agents and waits for them) would otherwise send nothing+ * between its start and its end, and nothing at all when the app is closed+ * or killed before it ends. While the engine says the session is busy, the+ * observer sends what is final so far:+ *+ * - every TURN_PROGRESS_INTERVAL_MS once the turn is TURN_PROGRESS_MIN_AGE_MS+ * old, and as soon as a sub-agent of a turn that old ends (a session the+ * engine listed as busy is not any more), at most once per+ * TURN_PROGRESS_MIN_GAP_MS;+ * - only settled messages, oldest first, up to the first one that is not:+ * a user message, or an assistant message that ended with no tool call+ * still pending or running, and never a session's newest message (the+ * engine still adds to a step that just ended: its file patch). A message+ * still being written is sent when the turn ends, with its final content.+ *+ * The root session's settled messages go ahead as `turn.messages` (the+ * events a long turn's backlog already uses, `progress: true`), each+ * sub-agent's as a `session.child` event. No checkpoint moves: the turn's+ * end reads every session from where the turn started, as without+ * progress, and sends only the messages not sent ahead, or sent ahead and+ * changed since (their size differs, as a prompt does once the engine adds+ * its summary): each message goes up once more at most, in its final+ * form. Everything the turn's end derives from its messages (outcome,+ * writers, files read and used, the archive, the checkpoints) still reads+ * all of them.+ */+export const TURN_PROGRESS_INTERVAL_MS = 3 * 60_000;+export const TURN_PROGRESS_MIN_AGE_MS = 60_000;+export const TURN_PROGRESS_MIN_GAP_MS = 15_000;+/** The shortest interval an override may set. */+export const TURN_PROGRESS_FLOOR_MS = 10_000;++export type TurnProgressReason = "interval" | "sub_agent_ended";++type MessageRecord = Record<string, unknown>;++function isRecord(value: unknown): value is MessageRecord {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function infoOf(message: MessageRecord): MessageRecord {+ return isRecord(message.info) ? message.info : message;+}++/** The interval set by OMNIRUSH_TURN_PROGRESS_MS: "0" turns it off, anything unreadable keeps the default. */+export function turnProgressIntervalMs(value: string | undefined): number {+ if (value === undefined || value.trim() === "") return TURN_PROGRESS_INTERVAL_MS;+ const parsed = Number(value);+ if (!Number.isFinite(parsed) || parsed < 0) return TURN_PROGRESS_INTERVAL_MS;+ if (parsed === 0) return 0;+ return Math.max(TURN_PROGRESS_FLOOR_MS, Math.floor(parsed));+}++/** An engine message's id, or null. */+export function progressMessageId(message: unknown): string | null {+ if (!isRecord(message)) return null;+ const id = infoOf(message).id;+ return typeof id === "string" && id ? id : null;+}++/** Whether a message is final: a user message, or an assistant message that ended with no tool call still open. */+export function settledMessage(message: unknown): boolean {+ if (!isRecord(message)) return false;+ const info = infoOf(message);+ const role = info.role ?? info.type;+ if (role === "user") return true;+ if (role !== "assistant") return false;+ const time = isRecord(info.time) ? info.time : null;+ const ended = (time !== null && (typeof time.completed === "number" || typeof time.completed === "string")) || info.error != null;+ if (!ended) return false;+ const parts = Array.isArray(message.parts) ? message.parts : [];+ return !parts.some((part) => {+ if (!isRecord(part) || part.type !== "tool") return false;+ const state = isRecord(part.state) ? part.state.status : undefined;+ return state === "pending" || state === "running";+ });+}++/** How many messages, from the oldest, are settled before the first one that is not. */+export function settledPrefix(messages: readonly unknown[]): number {+ let count = 0;+ while (count < messages.length && settledMessage(messages[count])) count += 1;+ return count;+}++/**+ * One followed turn's progress: when to send, and which of the root's+ * messages went ahead. Fed with every status read of the engine while the+ * turn runs.+ */+export class TurnProgress {+ /** Messages sent ahead of the turn's end, by session ("" for the root): message id → size when sent. */+ private readonly sent = new Map<string, Map<string, number>>();+ private readonly startedAt: number;+ private readonly intervalMs: number;+ private lastAt: number | null = null;+ private busy = new Set<string>();+ private ended = false;++ constructor(startedAt: number, intervalMs: number) {+ this.startedAt = startedAt;+ this.intervalMs = intervalMs;+ }++ get enabled(): boolean {+ return this.intervalMs > 0;+ }++ /**+ * One read of the engine's statuses (session id → { type }): a session+ * other than the root that was busy and is no longer listed busy ended+ * (a sub-agent, as far as the root's children say when they are read).+ */+ noteStatuses(statuses: unknown, rootSessionId: string): void {+ if (!isRecord(statuses)) return;+ const now = new Set<string>();+ for (const [id, status] of Object.entries(statuses)) {+ if (id === rootSessionId) continue;+ const type = isRecord(status) ? status.type : undefined;+ if (typeof type === "string" && type !== "idle") now.add(id);+ }+ for (const id of this.busy) {+ if (!now.has(id)) this.ended = true;+ }+ this.busy = now;+ }++ /** Why the turn's progress is due now, or null. */+ due(now: number): TurnProgressReason | null {+ if (!this.enabled) return null;+ if (now - this.startedAt < TURN_PROGRESS_MIN_AGE_MS) return null;+ const since = this.lastAt ?? this.startedAt;+ if (this.ended && (this.lastAt === null || now - this.lastAt >= TURN_PROGRESS_MIN_GAP_MS)) return "sub_agent_ended";+ if (now - since >= this.intervalMs) return "interval";+ return null;+ }++ /** The progress went out (or was tried): the next one is measured from now. */+ flushed(now: number): void {+ this.lastAt = now;+ this.ended = false;+ }++ /** How many messages went ahead in all. */+ get sentCount(): number {+ let count = 0;+ for (const session of this.sent.values()) count += session.size;+ return count;+ }++ /**+ * A session's settled messages not sent ahead yet, with their sizes+ * (`sizes` matches `messages`; `session` is "" for the root, else the+ * sub-agent's id). Its newest message is never among them.+ */+ ahead(messages: readonly unknown[], sizes: readonly number[], session = ""): { messages: unknown[]; sizes: number[] } {+ const count = settledPrefix(messages.slice(0, -1));+ const sent = this.sent.get(session);+ const out: unknown[] = [];+ const outSizes: number[] = [];+ for (let index = 0; index < count; index += 1) {+ const id = progressMessageId(messages[index]);+ if (!id || sent?.has(id)) continue;+ out.push(messages[index]);+ outSizes.push(sizes[index] ?? 0);+ }+ return { messages: out, sizes: outSizes };+ }++ /** Marks a session's messages as sent ahead (`sizes` matches `messages`). */+ markSent(messages: readonly unknown[], sizes: readonly number[], session = ""): void {+ let sent = this.sent.get(session);+ if (!sent) {+ sent = new Map();+ this.sent.set(session, sent);+ }+ for (const [index, message] of messages.entries()) {+ const id = progressMessageId(message);+ if (id) sent.set(id, sizes[index] ?? 0);+ }+ }++ /** The turn's messages its end still has to send: those not sent ahead, or changed since, with their sizes. */+ unsent(messages: readonly unknown[], sizes: readonly number[], session = ""): { messages: unknown[]; sizes: number[] } {+ const sent = this.sent.get(session);+ if (!sent || sent.size === 0) return { messages: messages.slice(), sizes: sizes.slice() };+ const out: unknown[] = [];+ const outSizes: number[] = [];+ for (const [index, message] of messages.entries()) {+ const id = progressMessageId(message);+ if (id && sent.get(id) === (sizes[index] ?? 0)) continue;+ out.push(message);+ outSizes.push(sizes[index] ?? 0);+ }+ return { messages: out, sizes: outSizes };+ }+}diff --git a/apps/server/src/session-upload-observer-progress.test.ts b/apps/server/src/session-upload-observer-progress.test.tsnew file mode 100644index 00000000..6f96ecd3--- /dev/null+++ b/apps/server/src/session-upload-observer-progress.test.ts@@ -0,0 +1,237 @@+import { afterEach, expect, test } from "bun:test";++import { createSessionObservers, observeUploadedSession, type EngineTarget, type ObservedUploader } from "./session-upload-observer.js";+import { settledMessage, settledPrefix, TurnProgress, turnProgressIntervalMs, TURN_PROGRESS_FLOOR_MS, TURN_PROGRESS_INTERVAL_MS } from "./session-archive/turn-progress.js";++// A long turn sends what is final while it runs (session-archive/turn-progress.ts,+// the same file as the CLI's, PARITY.sha256): the parent's settled messages+// ahead as turn.messages, a sub-agent's as session.child as soon as it ends,+// and the turn's end only what was not sent ahead (or changed since). The+// CLI's test/turn-progress.test.js runs the same cases.++type Message = { info: Record<string, unknown> & { id: string }; parts: unknown[] };++const user = (id: string, text: string, created: number): Message => ({ info: { id, role: "user", time: { created } }, parts: [{ type: "text", text }] });+const running = (id: string, created: number, tool = "task"): Message => ({ info: { id, role: "assistant", time: { created } }, parts: [{ type: "tool", tool, state: { status: "running" } }] });+const done = (id: string, created: number, extra: unknown[] = []): Message => ({ info: { id, role: "assistant", finish: "stop", time: { created, completed: created + 1 } }, parts: [{ type: "text", text: `answer ${id}` }, { type: "step-finish" }, ...extra] });++const cleanups: Array<() => void> = [];+afterEach(() => {+ while (cleanups.length > 0) cleanups.pop()!();+});++test("settled messages: a user message, an ended answer with no open tool call", () => {+ expect(settledMessage(user("u", "hi", 1))).toBe(true);+ expect(settledMessage(done("a", 1))).toBe(true);+ expect(settledMessage(running("b", 1))).toBe(false);+ expect(settledMessage({ info: { id: "c", role: "assistant", time: { created: 1, completed: 2 } }, parts: [{ type: "tool", state: { status: "pending" } }] })).toBe(false);+ expect(settledPrefix([user("u", "x", 1), done("a", 2), running("b", 3), done("c", 4)])).toBe(2);+});++test("the interval: default, off, floor, unreadable", () => {+ expect(turnProgressIntervalMs(undefined)).toBe(TURN_PROGRESS_INTERVAL_MS);+ expect(turnProgressIntervalMs("0")).toBe(0);+ expect(turnProgressIntervalMs("10")).toBe(TURN_PROGRESS_FLOOR_MS);+ expect(turnProgressIntervalMs("120000")).toBe(120_000);+ expect(turnProgressIntervalMs("soon")).toBe(TURN_PROGRESS_INTERVAL_MS);+});++test("when progress is due: never in the first minute, then on the interval or a sub-agent's end", () => {+ const progress = new TurnProgress(0, 180_000);+ progress.noteStatuses({ root: { type: "busy" }, c1: { type: "busy" } }, "root");+ progress.noteStatuses({ root: { type: "busy" } }, "root");+ expect(progress.due(30_000)).toBeNull();+ expect(progress.due(61_000)).toBe("sub_agent_ended");+ progress.flushed(61_000);+ expect(progress.due(200_000)).toBeNull();+ expect(progress.due(241_000)).toBe("interval");+ progress.flushed(241_000);+ progress.noteStatuses({ root: { type: "busy" }, c2: { type: "busy" } }, "root");+ progress.noteStatuses({ root: { type: "busy" } }, "root");+ expect(progress.due(250_000)).toBeNull();+ expect(progress.due(256_000)).toBe("sub_agent_ended");+ const off = new TurnProgress(0, 0);+ expect(off.enabled).toBe(false);+ expect(off.due(10 * 60_000)).toBeNull();+});++test("sent ahead once, never a session's newest message; changed since: sent again at the end", () => {+ const progress = new TurnProgress(0, 180_000);+ const messages = [user("u1", "go", 1), done("a1", 2), running("a2", 3)];+ const ahead = progress.ahead(messages, [10, 20, 30]);+ expect(ahead.messages.map((m) => (m as Message).info.id)).toEqual(["u1", "a1"]);+ progress.markSent(ahead.messages, ahead.sizes);+ expect(progress.ahead(messages, [10, 20, 30]).messages).toEqual([]);+ const unsent = progress.unsent(messages, [10, 25, 30]);+ expect(unsent.messages.map((m) => (m as Message).info.id)).toEqual(["a1", "a2"]);+ expect(unsent.sizes).toEqual([25, 30]);+ // A session that ended: its newest message waits for the turn's end (the engine still adds its patch).+ const child = [user("cu", "go", 1), done("ca1", 2), done("ca2", 3)];+ expect(progress.ahead(child, [1, 2, 3], "ses_c").messages.map((m) => (m as Message).info.id)).toEqual(["cu", "ca1"]);+ expect(progress.unsent(child, [1, 2, 3], "ses_c").messages.length).toBe(3);+ expect(progress.sentCount).toBe(2);+});++type Phase = { statuses: Record<string, unknown>; children: Record<string, Array<{ id: string; title?: string }>>; messages: Record<string, Message[]> };++/** A scripted engine: `phase()` gives the statuses, the children and every session's messages. */+function scriptedEngine(phase: () => Phase): EngineTarget {+ const server = Bun.serve({+ hostname: "127.0.0.1",+ port: 0,+ fetch(request) {+ const url = new URL(request.url);+ const now = phase();+ if (url.pathname === "/session/status") return Response.json(now.statuses);+ const children = /^\/session\/([^/]+)\/children$/.exec(url.pathname);+ if (children) return Response.json(now.children[decodeURIComponent(children[1]!)] ?? []);+ const list = /^\/session\/([^/]+)\/message$/.exec(url.pathname);+ if (list) {+ const all = now.messages[decodeURIComponent(list[1]!)] ?? [];+ const limit = Number(url.searchParams.get("limit") ?? all.length);+ const before = url.searchParams.get("before");+ const end = before ? all.findIndex((message) => message.info.id === before) : all.length;+ const page = all.slice(Math.max(0, end - limit), end);+ return Response.json(page, { headers: end - limit > 0 ? { "x-next-cursor": page[0]!.info.id } : {} });+ }+ return Response.json({ id: "ses_root" });+ },+ });+ cleanups.push(() => server.stop(true));+ return { baseUrl: `http://127.0.0.1:${server.port}`, headers: [], search: "", engine: "v1" };+}++type Event = { at: number; type: string; data?: unknown };++function recordingUploader(clock: () => number, events: Event[], flushes: Array<{ at: number; final: unknown }>, checkpoint: string | null): ObservedUploader {+ const childCheckpoints: Record<string, string> = {};+ const childIds: string[] = [];+ return {+ enabled: true,+ sessionCheckpoint: async () => (checkpoint ? { lastMessageId: checkpoint } : {}),+ setSessionCheckpoint: async () => undefined,+ recordTrace: (_id: string, type: string, data?: unknown) => { events.push({ at: clock(), type, data }); },+ recordSessionModel: () => undefined,+ childCheckpoints: async () => ({ ...childCheckpoints }),+ childSessionIds: async () => [...childIds],+ recordChildSession: (_id: string, child: { childSessionId: string; lastMessageId?: string | null }) => {+ if (!childIds.includes(child.childSessionId)) childIds.push(child.childSessionId);+ if (child.lastMessageId) childCheckpoints[child.childSessionId] = child.lastMessageId;+ events.push({ at: clock(), type: "session.child", data: child });+ },+ noteChildSessionIds: () => 0,+ captureSnapshot: () => undefined,+ flushTrace: (_id: string, data?: unknown) => { flushes.push({ at: clock(), final: data ?? null }); },+ } as unknown as ObservedUploader;+}++const archive = { turnFollowed: () => undefined, turnCompleted: () => undefined, turnIncomplete: () => { throw new Error("incomplete"); } };+const ids = (messages: unknown): string[] => (messages as Message[]).map((message) => message.info.id);++test("a long turn with a sub-agent: progress while it runs, the end sends the rest once", async () => {+ let clock = 0;+ const before = [user("u0", "earlier", 0), done("a0", 1)];+ const childFinal = [user("cu1", "research", 10), done("ca1", 11), done("ca2", 12), done("ca3", 13)];+ // 0-70 s: the parent waits on the sub-agent, which works; from 70 s the sub-agent is done;+ // from 150 s the parent's first step ended and its second runs; from 250 s that first step+ // carries one more part (it changed after it went ahead); from 300 s the turn is over.+ const target = scriptedEngine(() => {+ const childRunning = clock < 70_000;+ const a1 = clock < 150_000 ? running("a1", 3) : clock < 250_000 ? done("a1", 3) : done("a1", 3, [{ type: "patch", hash: "h" }]);+ const root = [...before, user("u1", "deep research", 2), a1];+ if (clock >= 150_000) root.push(clock < 300_000 ? running("a2", 4, "bash") : done("a2", 4));+ return {+ statuses: clock >= 300_000 ? {} : childRunning ? { ses_root: { type: "busy" }, ses_c1: { type: "busy" } } : { ses_root: { type: "busy" } },+ children: { ses_root: [{ id: "ses_c1", title: "helper" }] },+ messages: { ses_root: root, ses_c1: childRunning ? [...childFinal.slice(0, 2), running("ca2", 12, "bash")] : childFinal },+ };+ });+ const events: Event[] = [];+ const flushes: Array<{ at: number; final: unknown }> = [];+ const observers = createSessionObservers();+ cleanups.push(() => observers.controller.abort());+ await observeUploadedSession({+ sessionUploader: recordingUploader(() => clock, events, flushes, "a0"),+ archive,+ observers,+ sessionId: "ses_root",+ target,+ timing: { now: () => clock, sleep: async (ms: number) => { clock += ms; }, progressMs: 100_000 },+ });++ const progress = events.filter((event) => event.type === "turn.progress");+ expect(progress.map((event) => (event.data as { reason: string }).reason)).toEqual(["sub_agent_ended", "interval"]);+ expect(progress[0]!.at).toBeGreaterThanOrEqual(70_000);+ expect(progress[0]!.at).toBeLessThan(85_000);++ // The sub-agent: all but its newest message as soon as it ended, the newest at the turn's end;+ // its checkpoint moves only then, as without progress.+ const childEvents = events.filter((event) => event.type === "session.child");+ expect(childEvents.map((event) => ids((event.data as { messages: unknown }).messages))).toEqual([["cu1", "ca1", "ca2"], ["ca3"]]);+ expect(childEvents[0]!.at).toBeLessThan(85_000);+ expect(childEvents[1]!.at).toBeGreaterThanOrEqual(300_000);+ expect(childEvents.map((event) => (event.data as { lastMessageId: string | null }).lastMessageId)).toEqual([null, "ca3"]);++ const ahead = events.filter((event) => event.type === "turn.messages" && (event.data as { progress?: boolean }).progress === true);+ expect(ahead.map((event) => ids((event.data as { messages: unknown }).messages))).toEqual([["u1"], ["a1"]]);+ const end = flushes.filter((flush) => flush.final);+ expect(end.length).toBe(1);+ const final = end[0]!.final as { messages: Message[] };+ expect(ids(final.messages)).toEqual(["a1", "a2"]);+ expect((final.messages[0]!.parts.at(-1) as { type: string }).type).toBe("patch");+ expect(end[0]!.at).toBeGreaterThanOrEqual(300_000);+ expect(end[0]!.at).toBeLessThan(315_000);+ for (const event of progress) expect(flushes.some((flush) => flush.at === event.at && !flush.final)).toBe(true);+});++test("progress off (OMNIRUSH_TURN_PROGRESS_MS=0): the turn's end alone, as before", async () => {+ let clock = 0;+ const target = scriptedEngine(() => ({+ statuses: clock >= 300_000 ? {} : { ses_root: { type: "busy" }, ses_c1: { type: clock < 70_000 ? "busy" : "idle" } },+ children: { ses_root: [{ id: "ses_c1", title: "helper" }] },+ messages: {+ ses_root: [user("u1", "go", 2), clock < 300_000 ? running("a1", 3) : done("a1", 3)],+ ses_c1: [user("cu1", "research", 10), done("ca1", 11)],+ },+ }));+ const events: Event[] = [];+ const flushes: Array<{ at: number; final: unknown }> = [];+ const observers = createSessionObservers();+ cleanups.push(() => observers.controller.abort());+ await observeUploadedSession({+ sessionUploader: recordingUploader(() => clock, events, flushes, null),+ archive,+ observers,+ sessionId: "ses_root",+ target,+ timing: { now: () => clock, sleep: async (ms: number) => { clock += ms; }, progressMs: 0 },+ });+ expect(events.filter((event) => event.type === "turn.progress" || (event.type === "turn.messages" && (event.data as { progress?: boolean }).progress)).length).toBe(0);+ expect(events.filter((event) => event.type === "session.child").length).toBe(1);+ expect(ids((flushes.filter((flush) => flush.final).at(-1)!.final as { messages: unknown }).messages)).toEqual(["u1", "a1"]);+});++test("what the turn's context collects after its end goes out once it is in, not at the next prompt", async () => {+ let clock = 0;+ const target = scriptedEngine(() => ({ statuses: clock >= 5_000 ? {} : { ses_root: { type: "busy" } }, children: {}, messages: { ses_root: [user("u1", "go", 1), clock >= 5_000 ? done("a1", 2) : running("a1", 2)] } }));+ const events: Event[] = [];+ const flushes: Array<{ at: number; final: unknown }> = [];+ let release: () => void = () => undefined;+ const contextIn = new Promise<void>((resolve) => { release = resolve; });+ const uploader = recordingUploader(() => clock, events, flushes, null) as unknown as Record<string, unknown>;+ uploader.context = { settled: async (_id: string, waitMs: number) => { expect(waitMs).toBeGreaterThanOrEqual(10_000); await contextIn; } };+ const observers = createSessionObservers();+ cleanups.push(() => observers.controller.abort());+ await observeUploadedSession({+ sessionUploader: uploader as unknown as ObservedUploader,+ archive,+ observers,+ sessionId: "ses_root",+ target,+ timing: { now: () => clock, sleep: async (ms: number) => { clock += ms; } },+ });+ expect(flushes.map((flush) => (flush.final ? "turn" : "plain"))).toEqual(["turn"]);+ release();+ await new Promise((resolve) => setImmediate(resolve));+ expect(flushes.map((flush) => (flush.final ? "turn" : "plain"))).toEqual(["turn", "plain"]);+});diff --git a/apps/server/src/session-upload-observer.test.ts b/apps/server/src/session-upload-observer.test.tsindex 51978a6d..5c15fb23 100644--- a/apps/server/src/session-upload-observer.test.ts+++ b/apps/server/src/session-upload-observer.test.ts@@ -154,6 +154,8 @@ function fakeClock() {await new Promise((resolvePromise) => setImmediate(resolvePromise));if (signal.aborted) throw signal.reason;},+ // These cases follow a turn to its end only; a running turn's progress has its own (session-upload-observer-progress.test.ts).+ progressMs: 0,};return {timing,diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.tsindex f345e20a..f8bdd1aa 100644--- a/apps/server/src/session-upload-observer.ts+++ b/apps/server/src/session-upload-observer.ts@@ -16,6 +16,7 @@ import { turnWriters, type WriterSessionMessages } from "./session-archive/turn-import { watchToolStarts } from "./session-archive/tool-start.js";import { watchToolEvents } from "./context/tool-events.js";import { closeOpenToolParts } from "./context/open-tools.js";+import { TurnProgress, turnProgressIntervalMs, type TurnProgressReason } from "./session-archive/turn-progress.js";import type { ContextCapture } from "./context/index.js";/** The engine a captured request went to: base URL, request headers (the engine's auth), query and API generation. */@@ -260,6 +261,8 @@ const CUT_PROMPT_WAIT_MS = 15_000;const CHILD_IDS_INTERVAL_MS = 60_000;/** A settled turn whose messages the engine did not answer for (a timeout, a dropped connection) is read again after these waits. */const HISTORY_RETRY_DELAYS_MS = [2_000, 5_000];+/** How long a settled turn's context may still collect before what it found is flushed. */+const TURN_CONTEXT_WAIT_MS = 30_000;/** The observer's clock and timeouts; tests pass their own so that hours of a turn pass without real waiting. */export type ObserverTiming = {@@ -271,6 +274,8 @@ export type ObserverTiming = {readTimeoutMs: number;/** How long one turn is followed at most. */maxTurnMs: number;+ /** A running turn's settled messages go out this often (session-archive/turn-progress.ts); 0: only at its end. */+ progressMs: number;};type EngineFetch = (path: string, query?: Record<string, string>) => Promise<Response>;@@ -335,6 +340,15 @@ async function captureChildSessions(input: {* only those of answers that ended; `closed` counts what was closed.*/toolsClosed?: { engineIdle: boolean; closed: number };+ /**+ * The turn's progress (session-archive/turn-progress.ts). With `ahead`+ * the turn still runs: each child's settled messages not sent yet go out+ * and no checkpoint moves (a child over the read budget is left to the+ * turn's end). Without it (the turn's end), a child's messages sent ahead+ * and unchanged are not sent again.+ */+ progress?: TurnProgress | null;+ ahead?: boolean;}): Promise<void> {if (input.depth > MAX_UPLOAD_CHILD_SESSION_DEPTH) return;const children = await readEngineJson(input.fetchEngine, `/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024)@@ -346,6 +360,27 @@ async function captureChildSessions(input: {input.visited.add(childId);const history = await readEngineHistory(engineMessages(input.fetchEngine, `/session/${encodeURIComponent(childId)}/message`, true), input.checkpoints[childId]).catch(nullOnErrorStatus);+ if (input.ahead) {+ if (history && input.progress && history.omitted === 0 && !history.checkpointMissing) {+ const sending = input.progress.ahead(history.delta, history.sizes, childId);+ if (sending.messages.length > 0 || !input.known.has(childId)) {+ input.beforeRecord(sum(sending.sizes));+ input.sessionUploader.recordChildSession(input.rootSessionId, {+ childSessionId: childId,+ parentSessionId: input.parentSessionId,+ depth: input.depth,+ title: isRecord(child) ? optionalTraceString(child.title) : null,+ agent: turnModelFromMessages(history.outline)?.agent ?? null,+ messages: sending.messages,+ lastMessageId: null,+ });+ input.progress.markSent(sending.messages, sending.sizes, childId);+ if (sending.messages.length > 0) input.onChild?.(childId, input.depth, sending.messages);+ }+ }+ await captureChildSessions({ ...input, parentSessionId: childId, depth: input.depth + 1 });+ continue;+ }// A sub-agent may still run: unless the engine no longer does, only the calls of its answers that ended are closed.const closing = history ? closeOpenToolParts(history.delta, { engineIdle: input.toolsClosed?.engineIdle ?? false }) : null;if (closing && input.toolsClosed) input.toolsClosed.closed += closing.closed;@@ -355,14 +390,16 @@ async function captureChildSessions(input: {input.sessionUploader.recordTrace(input.rootSessionId, "session.messages_omitted", { child_session_id: childId, count: history.omitted });}if (newMessages.length > 0 || !input.known.has(childId)) {- input.beforeRecord(history ? sum(history.sizes) : 0);+ // Sent ahead while the turn ran and unchanged: not again.+ const sending = input.progress ? input.progress.unsent(newMessages, history?.sizes ?? [], childId) : { messages: newMessages, sizes: history?.sizes ?? [] };+ input.beforeRecord(sum(sending.sizes));input.sessionUploader.recordChildSession(input.rootSessionId, {childSessionId: childId,parentSessionId: input.parentSessionId,depth: input.depth,title: isRecord(child) ? optionalTraceString(child.title) : null,agent: turnModelFromMessages(history?.outline ?? [])?.agent ?? null,- messages: newMessages,+ messages: sending.messages,lastMessageId: traceMessageId(history?.outline.at(-1)),});}@@ -996,6 +1033,7 @@ export function observeUploadedSession(input: {statusTimeoutMs: 10_000,readTimeoutMs: 20_000,maxTurnMs: MAX_OBSERVED_TURN_MS,+ progressMs: turnProgressIntervalMs(process.env.OMNIRUSH_TURN_PROGRESS_MS),...input.timing,};const stopped = observer.controller.signal;@@ -1024,9 +1062,12 @@ export function observeUploadedSession(input: {else sessionUploader.captureSnapshot(sessionId, "turn_completed");};+ // Every session's status at the last read: a running turn's progress sees its sub-agents end there.+ let lastStatuses: unknown = null;/** The session's status ("idle" when the engine does not list it); throws when the engine does not answer. */const readStatus = async (): Promise<string> => {const statuses = enginePayload(await readEngineJson(fetchStatus, v2 ? "/api/session/active" : "/session/status", 1024 * 1024));+ lastStatuses = statuses;const status = isRecord(statuses) ? statuses[sessionId] : undefined;return isRecord(status) && typeof status.type === "string" ? status.type : "idle";};@@ -1086,6 +1127,80 @@ export function observeUploadedSession(input: {return false;};+ // The followed turn's progress (session-archive/turn-progress.ts).+ let progress: TurnProgress | null = null;+ let progressStartedAt = timing.now();+ /**+ * What the turn's context collects once it ended (its files outside the+ * folder, its network, its installs) goes out as soon as it is in, not with+ * the next prompt.+ */+ const flushTurnContext = (): void => {+ const context = (sessionUploader as Partial<Pick<SessionUploader, "context">>).context;+ if (!context) return;+ void context.settled(sessionId, TURN_CONTEXT_WAIT_MS).then(() => {+ if (!stopped.aborted) sessionUploader.flushTrace(sessionId);+ }, () => undefined);+ };+ /**+ * The running turn's settled messages so far: the root's as `turn.messages`+ * (progress), the sub-agents' as `session.child`, then a flush. The turn's+ * checkpoint stays where it started.+ */+ const sendProgress = async (current: TurnProgress, reason: TurnProgressReason): Promise<void> => {+ let rootMessages = 0;+ let childMessages = 0;+ let pending = 0;+ const room = (bytes: number) => {+ if (pending > 0 && pending + bytes > MAX_TURN_MESSAGE_BYTES) {+ sessionUploader.flushTrace(sessionId);+ pending = 0;+ }+ pending += bytes;+ };+ try {+ const history = await readEngineHistory(messages(), observer.lastMessageIds.get(sessionId), MAX_BACKLOG_MESSAGE_BYTES);+ if (history.omitted === 0 && !history.checkpointMissing) {+ const ahead = current.ahead(history.delta, history.sizes);+ for (const part of ahead.messages.length > 0 ? messageParts(ahead.messages, ahead.sizes) : []) {+ room(part.bytes);+ sessionUploader.recordTrace(sessionId, "turn.messages", { messages: part.messages, progress: true });+ }+ current.markSent(ahead.messages, ahead.sizes);+ rootMessages = ahead.messages.length;+ }+ } catch (error) {+ if (stopped.aborted) throw error;+ }+ if (!v2) {+ try {+ await captureChildSessions({+ onChild: (_childId, _depth, sent) => {+ childMessages += sent.length;+ },+ sessionUploader,+ rootSessionId: sessionId,+ parentSessionId: sessionId,+ depth: 1,+ fetchEngine,+ checkpoints: await sessionUploader.childCheckpoints(sessionId),+ known: new Set(await sessionUploader.childSessionIds(sessionId)),+ visited: new Set([sessionId]),+ beforeRecord: room,+ progress: current,+ ahead: true,+ });+ } catch (error) {+ if (stopped.aborted) throw error;+ }+ }+ if (rootMessages + childMessages > 0) {+ sessionUploader.recordTrace(sessionId, "turn.progress", { reason, elapsed_ms: Math.max(0, timing.now() - progressStartedAt), messages: rootMessages, sub_agent_messages: childMessages });+ }+ // Whatever else the turn recorded so far (its context, its sub-agents) goes with it.+ sessionUploader.flushTrace(sessionId);+ };+let recovering = input.recovered === true;const settle = async (status: string, read?: EngineHistory): Promise<void> => {// Once per turn: an app quit may settle it while the follow loop is about to.@@ -1136,7 +1251,9 @@ export function observeUploadedSession(input: {const model = turnModelFromMessages(history.delta);if (model) sessionUploader.recordSessionModel(sessionId, model);// More messages than one flush holds: the older ones go ahead, a flush per part.- parts = messageParts(history.delta, history.sizes);+ // Messages sent ahead while the turn ran (turn-progress.ts) are not sent again, unless they changed since.+ const unsent = progress ? progress.unsent(history.delta, history.sizes) : { messages: history.delta, sizes: history.sizes };+ parts = messageParts(unsent.messages, unsent.sizes);for (const [index, part] of parts.slice(0, -1).entries()) {sessionUploader.recordTrace(sessionId, "turn.messages", { messages: part.messages, part: index + 1, parts: parts.length });sessionUploader.flushTrace(sessionId);@@ -1171,6 +1288,7 @@ export function observeUploadedSession(input: {visited: new Set([sessionId]),beforeRecord: room,toolsClosed: childTools,+ progress,});} catch (error) {if (stopped.aborted) throw error;@@ -1200,6 +1318,7 @@ export function observeUploadedSession(input: {...(recovered ? { recovered: true } : {}),...(status === "quit" ? { reason: "app_quit" } : {}),});+ if (status !== "quit") flushTurnContext();// The delta's turn number is the engine's completed-turn count, which// survives app restarts; without the messages the archiver numbers it// right after the last archived turn. It also arms the idle final archive.@@ -1220,6 +1339,9 @@ export function observeUploadedSession(input: {// A turn is running: the previous turn's idle final archive, if armed, is off.input.archive.turnFollowed(sessionId);const startedAt = timing.now();+ progressStartedAt = startedAt;+ const current = new TurnProgress(startedAt, timing.progressMs);+ progress = current;let observedBusy = false;let reads = 0;let idleReads = 0;@@ -1297,6 +1419,15 @@ export function observeUploadedSession(input: {observedBusy = true;idleReads = 0;turnState().idleAt = null;+ // A long turn: what is final so far goes out now (turn-progress.ts).+ if (current.enabled) {+ current.noteStatuses(lastStatuses, sessionId);+ const reason = current.due(timing.now());+ if (reason) {+ await sendProgress(current, reason);+ current.flushed(timing.now());+ }+ }// The v2 daemon has no children route: nothing to list there.if (!v2 && timing.now() - childIdsAt >= CHILD_IDS_INTERVAL_MS) {childIdsAt = timing.now();