A desktop coding agent with free access to frontier models.
TypeScript
fix: keep turn ends across app crashes
ogsamrat08f072a6bc9b
diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.tsindex 5a9cd7b8..0ebc8d73 100644--- a/apps/server/src/session-uploader.test.ts+++ b/apps/server/src/session-uploader.test.ts@@ -407,6 +407,186 @@ describe("session uploader privacy", () => {expect(envelopes.slice(3).map((item) => [item.snapshot_type, item.session_segment, item.sequence])).toEqual([["start", 2, 4], ["trace", 2, 5], ["end", 2, 6]]);}, 20_000);+ test("a crash after a turn settled, while its trace is on its way, still sends the turn's end and every event at the next start", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-turnend-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-turnend-state-"));+ roots.push(root, stateDir);+ await writeFile(join(root, "app.txt"), "hello\n");+ const envelopes: Array<Record<string, unknown>> = [];+ const parse = (compressed: Uint8Array) => withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>;+ let traces = 0;+ const hangs: Array<() => void> = [];+ let released = false;+ // An upload that never gets through while the run lives; once it "died", any upload fails at once.+ const stall = () => released ? Promise.resolve(new Response(null, { status: 503 })) : new Promise<Response>((resolve) => hangs.push(() => resolve(new Response(null, { status: 503 }))));+ const crashed = new SessionUploader({+ stateDir,+ upload: async (_sessionId, compressed) => {+ const envelope = parse(compressed);+ if (envelope.snapshot_type === "trace") {+ traces += 1;+ // The turn's own trace never gets through before the app dies (a slow link, a big transcript).+ if (traces > 1) return stall();+ // The flush before it is slow: the turn's end is taken while it is on its way.+ await new Promise((resolve) => setTimeout(resolve, 300));+ }+ envelopes.push(envelope);+ return Response.json({ ok: true }, { status: 201 });+ },+ fallbackScanMs: 60_000,+ });+ const sessionId = "session-turnend-1234";+ crashed.startSession(sessionId, "workspace-turnend", root);+ await crashed.idle(sessionId);+ crashed.captureSnapshot(sessionId, "prompt");+ crashed.recordTrace(sessionId, "tool.call", { n: 1 });+ crashed.flushTrace(sessionId);+ crashed.recordTrace(sessionId, "tool.call", { n: 2 });+ crashed.recordTrace(sessionId, "session.idle", { status: "idle", outcome: "completed" });+ crashed.captureSnapshot(sessionId, "turn_completed");+ crashed.flushTrace(sessionId, { messages: [{ info: { id: "msg_user", role: "user" } }, { info: { id: "msg_answer", role: "assistant", parentID: "msg_user" } }] });+ for (let waited = 0; waited < 5_000 && !envelopes.some((item) => item.snapshot_type === "trace"); waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));+ await new Promise((resolve) => setTimeout(resolve, 600));+ // The process dies here: the turn's trace (its end, its messages) is still on its way.+ const sent = envelopes.length;+ const sentEvents = envelopes.filter((item) => item.snapshot_type === "trace").flatMap((item) => (item.trace as Array<{ type: string; data?: Record<string, unknown> }>) ?? []);+ expect(sentEvents.filter((event) => event.type === "tool.call").map((event) => event.data?.n)).toEqual([1]);+ expect(sentEvents.map((event) => event.type)).not.toContain("turn.completed");++ const next = new SessionUploader({+ stateDir,+ upload: async (_sessionId, compressed) => {+ envelopes.push(parse(compressed));+ return Response.json({ ok: true }, { status: 201 });+ },+ fallbackScanMs: 60_000,+ });+ for (let waited = 0; waited < 5_000 && envelopes.length <= sent; waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));+ const recovered = envelopes[sent]!;+ expect(recovered).toMatchObject({ snapshot_type: "trace", session_segment: 1 });+ const events = recovered.trace as Array<{ type: string; data?: Record<string, unknown> }>;+ expect(events[0]?.type).toBe("collector.recovered");+ expect(events[0]?.data).toMatchObject({ turn_open: false });+ expect(events.filter((event) => event.type === "tool.call").map((event) => event.data?.n)).toEqual([2]);+ const end = events.find((event) => event.type === "turn.completed");+ expect((end?.data?.messages as unknown[] | undefined)?.length).toBe(2);+ await next.stop();+ // The stalled uploads of the run that "died" end, so they hold no upload slot.+ released = true;+ for (const release of hangs) release();+ await crashed.stop().catch(() => undefined);+ }, 20_000);++ test("an upload on its way when the app died leaves no gap: the journal that holds its events goes out under its sequence", async () => {+ for (const serverKnows of [true, false]) {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-unsent-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-unsent-state-"));+ roots.push(root, stateDir);+ await writeFile(join(root, "app.txt"), "hello\n");+ const stored: Array<Record<string, unknown>> = [];+ const hangs: Array<() => void> = [];+ let released = false;+ // An upload that never gets through while the run lives; once it "died", any upload fails at once.+ const stall = () => released ? Promise.resolve(new Response(null, { status: 503 })) : new Promise<Response>((resolve) => hangs.push(() => resolve(new Response(null, { status: 503 }))));+ const options = {+ stateDir,+ fallbackScanMs: 60_000,+ capabilities: async () => ({ schema_versions: [1, 2], canonical_trace: false, idempotency_key: true, integrity: true, integrity_summary: true }),+ sessionIntegrity: async () => serverKnows+ ? Response.json(stored.length === 0+ ? { summary: true, max_segment: null, max_sequence: null, segments: [] }+ : { summary: true, max_segment: 1, max_sequence: Math.max(...stored.map((item) => Number(item.sequence))), segments: [] })+ : Response.json({ detail: "unavailable" }, { status: 503 }),+ };+ const crashed = new SessionUploader({+ ...options,+ upload: async (_sessionId, compressed) => {+ const envelope = withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>;+ // The trace flush is on its way when the app dies: the server never stores it.+ if (envelope.snapshot_type === "trace") return stall();+ stored.push(envelope);+ return Response.json({ ok: true }, { status: 201 });+ },+ });+ const sessionId = "session-unsent-1234";+ crashed.startSession(sessionId, "workspace-unsent", root);+ await crashed.idle(sessionId);+ crashed.recordTrace(sessionId, "tool.call", { n: 1 });+ crashed.flushTrace(sessionId);+ await new Promise((resolve) => setTimeout(resolve, 600));+ expect(stored.map((item) => [item.snapshot_type, item.sequence])).toEqual([["start", 1]]);++ const next = new SessionUploader({+ ...options,+ upload: async (_sessionId, compressed) => {+ stored.push(withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>);+ return Response.json({ ok: true }, { status: 201 });+ },+ });+ for (let waited = 0; waited < 5_000 && !stored.some((item) => item.snapshot_type === "trace"); waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));+ const recovered = stored.find((item) => item.snapshot_type === "trace")!;+ const marker = (recovered.trace as Array<{ type: string; data: Record<string, unknown> }>)[0]!;+ expect(marker.type).toBe("collector.recovered");+ expect((recovered.trace as Array<{ type: string; data: Record<string, unknown> }>).filter((event) => event.type === "tool.call")).toHaveLength(1);+ if (serverKnows) {+ // Sequence 2 never reached the server: the recovered trace takes it.+ expect(recovered).toMatchObject({ session_segment: 1, sequence: 2 });+ expect(marker.data.unsent_sequences).toEqual({ from: 2, to: 2 });+ } else {+ // The server cannot say: the numbering goes on past it, as before.+ expect(recovered).toMatchObject({ session_segment: 1, sequence: 3 });+ expect(marker.data.unsent_sequences).toBeUndefined();+ }+ await next.stop();+ released = true;+ for (const release of hangs) release();+ await crashed.stop().catch(() => undefined);+ }+ }, 30_000);++ test("a recovered journal is a recovered segment when the server takes one, so a run killed before its start snapshot reads as such", async () => {+ for (const recoveredSegments of [true, false]) {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-recseg-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-recseg-state-"));+ roots.push(root, stateDir);+ await writeFile(join(root, "app.txt"), "hello\n");+ const envelopes: Array<Record<string, unknown>> = [];+ const hangs: Array<() => void> = [];+ let released = false;+ // An upload that never gets through while the run lives; once it "died", any upload fails at once.+ const stall = () => released ? Promise.resolve(new Response(null, { status: 503 })) : new Promise<Response>((resolve) => hangs.push(() => resolve(new Response(null, { status: 503 }))));+ const options = {+ stateDir,+ fallbackScanMs: 60_000,+ capabilities: async () => ({ schema_versions: [1, 2], canonical_trace: false, idempotency_key: true, recovered_segments: recoveredSegments }),+ };+ const crashed = new SessionUploader({+ ...options,+ // The start snapshot never goes out: the app dies while it uploads.+ upload: async () => stall(),+ });+ const sessionId = "session-recseg-1234";+ crashed.startSession(sessionId, "workspace-recseg", root);+ crashed.recordTrace(sessionId, "tool.call", { n: 1 });+ await new Promise((resolve) => setTimeout(resolve, 600));+ const next = new SessionUploader({+ ...options,+ upload: async (_sessionId, compressed) => {+ envelopes.push(withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>);+ return Response.json({ ok: true }, { status: 201 });+ },+ });+ for (let waited = 0; waited < 5_000 && !envelopes.some((item) => item.snapshot_type === "trace"); waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));+ const recovered = envelopes.find((item) => item.snapshot_type === "trace");+ expect(recovered).toMatchObject({ trigger: recoveredSegments ? "session_recovered" : "trace_flush" });+ expect((recovered?.trace as Array<{ type: string }>)[0]?.type).toBe("collector.recovered");+ await next.stop();+ released = true;+ for (const release of hangs) release();+ await crashed.stop().catch(() => undefined);+ }+ }, 30_000);+test("an app that went away after a finished turn: the next launch sends the chat's end (marked recovered), once", async () => {const root = await mkdtemp(join(tmpdir(), "omnirush-upload-endrec-"));const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-endrec-state-"));diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex bf198cd9..dc0b7046 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -239,7 +239,11 @@ export type UploadTrigger =| "fs_change"| "periodic"| "session_end"- | "trace_flush";+ | "trace_flush"+ // The trace of a run that ended without its uploads (a crash, a kill),+ // sent from its journal at the next start: its segment may have no start+ // snapshot (the backend's recovered segment).+ | "session_recovered";/** How a turn ended, when not as answered: "aborted" is a turn the user stopped (Esc, Stop). */export type TurnEndOutcome = "aborted";@@ -674,6 +678,13 @@ type JournalState = {full: boolean;/** A recovery's stand-in state: never journaled itself. */off: boolean;+ /**+ * Trace events taken off the chat's trace for an upload that has not+ * settled yet (queued behind a snapshot, or on its way), oldest first: a+ * rewrite after an earlier upload keeps them, so a crash before they are+ * sent loses none of them.+ */+ inFlight: TraceEvent[][];/*** Lines are written once the chat's own recovery is done (prepareSession):* a journal a previous process left under the same name is read first.@@ -4503,7 +4514,7 @@ export function segmentEnd(payload: unknown, segment: number): "missing" | "pres}/** The /collect features this client uses; without a capability answer, none. */-type UploadFeatures = { idempotencyKey: boolean; integrity: boolean };+type UploadFeatures = { idempotencyKey: boolean; integrity: boolean; recoveredSegments?: boolean };const NO_UPLOAD_FEATURES: UploadFeatures = { idempotencyKey: false, integrity: false };/** What a /collect upload carries besides its body. */@@ -4642,7 +4653,11 @@ export class SessionUploader {server: () => this.uploadUrl ?? "gateway",...(this.capabilityProbe ? { probe: this.capabilityProbe } : {}),onAnswer: (capabilities) => {- this.probedFeatures = { idempotencyKey: capabilities.idempotency_key === true, integrity: capabilities.integrity === true };+ this.probedFeatures = {+ idempotencyKey: capabilities.idempotency_key === true,+ integrity: capabilities.integrity === true,+ recoveredSegments: capabilities.recovered_segments === true || (capabilities.triggers?.includes("session_recovered") ?? false),+ };},log: (level, message, attributes) => this.log(level, message, attributes),});@@ -5172,7 +5187,7 @@ export class SessionUploader {failureCount: 0,ready: Promise.resolve(),tail: Promise.resolve(),- journal: { pending: [], timer: null, tail: Promise.resolve(), bytes: 0, full: false, off: false, ready: false },+ journal: { pending: [], timer: null, tail: Promise.resolve(), bytes: 0, full: false, off: false, ready: false, inFlight: [] },refused: [],idleEndTimer: null,turns: 0,@@ -6135,16 +6150,25 @@ export class SessionUploader {}// Capture context: the turn's last messages (its tool calls) arrive here, not as turn.messages.if (isRecord(finalTrace)) this.contextOf(sessionId)?.turnMessages(sessionId, finalTrace.messages, true);- pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });+ const completed: TraceEvent = { at: new Date().toISOString(), type: "turn.completed", data: finalTrace };+ pendingTrace.push(completed);+ // The turn's end is on disk at once: the journal already says the turn+ // is closed (its turn_completed milestone), and the upload may wait+ // behind that snapshot's. A crash meanwhile still sends the turn's end+ // and messages at the next start.+ this.journalEvent(state, completed, true);}if (pendingTrace.length === 0) return;+ state.journal.inFlight.push(pendingTrace);this.enqueue(state, async () => {await state.ready;if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));try {await this.uploadTrace(state, pendingTrace);} finally {- // Uploaded, spooled or refused: the journal keeps only what came after.+ // Uploaded, spooled or refused: the journal keeps only what came after+ // (and what other uploads have not sent yet).+ state.journal.inFlight = state.journal.inFlight.filter((batch) => batch !== pendingTrace);await this.journalRewrite(state);}});@@ -6712,7 +6736,7 @@ export class SessionUploader {}}- private async uploadTrace(state: SessionState, events: TraceEvent[]): Promise<void> {+ private async uploadTrace(state: SessionState, events: TraceEvent[], trigger: Extract<UploadTrigger, "trace_flush" | "session_recovered"> = "trace_flush"): Promise<void> {if (events.length === 0) return;// A session imported from the opencode 2.x engine's store says so in each// trace upload (an opaque event type: kept as sent), so a reader can tell@@ -6743,13 +6767,13 @@ export class SessionUploader {// One form per schema: the server reads the top-level trace for 3, and only trace.json for 1 and 2....(version === TRACE_SCHEMA_VERSION ? { trace: batch.events } : {}),});- let uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));- if (uploaded === "key_reused") uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));+ let uploaded = await this.uploadEnvelope(state, "trace", trigger, await makeBody(schemaVersion));+ if (uploaded === "key_reused") uploaded = await this.uploadEnvelope(state, "trace", trigger, await makeBody(schemaVersion));if (uploaded === "unsupported_schema" && schemaVersion === TRACE_SCHEMA_VERSION && !downgraded) {downgraded = true;schemaVersion = 2;this.downgradeTraceSchema();- uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));+ uploaded = await this.uploadEnvelope(state, "trace", trigger, await makeBody(schemaVersion));}}}@@ -7269,7 +7293,7 @@ export class SessionUploader {this.scheduleJournal(state, now);}- private journalEvent(state: SessionState, event: TraceEvent): void {+ private journalEvent(state: SessionState, event: TraceEvent, now = false): void {if (!this.journalDir || state.journal.full || state.journal.off) return;let line: string;try {@@ -7283,7 +7307,7 @@ export class SessionUploader {}state.journal.bytes += line.length + 1;state.journal.pending.push(line);- this.scheduleJournal(state, false);+ this.scheduleJournal(state, now);}private scheduleJournal(state: SessionState, now: boolean): void {@@ -7325,7 +7349,7 @@ export class SessionUploader {const lines = [this.journalMetaLine(state)];state.journal.bytes = 0;state.journal.full = false;- for (const event of state.trace) {+ for (const event of [...state.journal.inFlight.flat(), ...state.trace]) {const line = JSON.stringify({ e: event });if (state.journal.bytes + line.length > MAX_JOURNAL_BYTES) {state.journal.full = true;@@ -7415,6 +7439,25 @@ export class SessionUploader {void Promise.allSettled(recovered);}+ /**+ * The sequences a previous process took for this chat that the server+ * never stored and no spooled upload holds (an upload on its way when the+ * process died): the run after the server's highest and the spool's. Null+ * when there are none, or when the server cannot say (offline, an older+ * backend, a chat it holds nothing of): the numbering then goes on as it was.+ */+ private async unsentSequences(state: SessionState): Promise<{ from: number; to: number } | null> {+ if (state.sequence <= 0) return null;+ const server = await this.serverPosition(state.id);+ if (!server || server.segment > state.segment || server.sequence >= state.sequence) return null;+ const spooled = this.spoolDir+ ? await this.spoolLocked(() => this.listSpool()).catch(() => null)+ : [];+ if (spooled === null) return null;+ const floor = Math.max(server.sequence, ...spooled.filter((entry) => entry.session_id === state.id).map((entry) => entry.sequence));+ return floor < state.sequence ? { from: floor + 1, to: state.sequence } : null;+ }+/*** A chat's journal outlived the app and the ledger has no end for it: the* end never went out. A turn left open too (its messages are settled in the@@ -7515,17 +7558,29 @@ export class SessionUploader {if (path && !isUploadPathDenied(path) && state.touchedPaths.size < MAX_TOUCHED_PATHS) state.touchedPaths.add(path);}}+ // The upload the process had on its way when it went away took a+ // sequence the server never stored; its events are this journal's.+ // They go out under that number again, so the segment shows no gap.+ const unsent = events.length > 0 ? await this.unsentSequences(state) : null;+ if (unsent) state.sequence = unsent.from - 1;const marker: TraceEvent = {at: new Date().toISOString(),type: "collector.recovered",- data: { reason: "restart", turn_open: meta.turn_open, events: events.length, journal_at: meta.at, session_segment: state.segment },+ data: {+ reason: "restart", turn_open: meta.turn_open, events: events.length, journal_at: meta.at, session_segment: state.segment,+ ...(unsent ? { unsent_sequences: unsent } : {}),+ },};// The end that follows was taken at this launch, not when the app went away.const endMarker: TraceEvent[] = endMissing? [{ at: marker.at, type: "session.end_recovered", data: { reason: journal.reason ?? "run_ended_without_end", last_message_id: state.lastMessageId ?? null, ...(meta.turn_open ? { turn_open: true } : {}), ...(record ? {} : { ledger: "missing" }) } }]: [];+ // A run killed before its segment's start snapshot went out leaves a+ // segment of traces only: the backend takes it as a recovered segment+ // when the trace says so (its trigger), not as a missing start.+ const trigger = (await this.uploadFeatures()).recoveredSegments ? "session_recovered" as const : "trace_flush" as const;try {- await this.uploadTrace(state, [marker, ...events, ...endMarker]);+ await this.uploadTrace(state, [marker, ...events, ...endMarker], trigger);} catch (error) {// Refused (a 422): recovered data is never dropped; the next start tries again, up to the age limit.this.log("warn", "OmniRush session journal upload refused; kept for the next start", { sessionId: meta.session_id, error: error instanceof Error ? error.message : "unknown" });