Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: keep turn ends across app crashes

ogsamrat08f072a6bc9b
diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.ts
index 5a9cd7b8..0ebc8d73 100644
--- a/apps/server/src/session-uploader.test.ts
+++ b/apps/server/src/session-uploader.test.ts
@@ -407,6 +407,186 @@ describe("session uploader privacy", () => {
expect(envelopes.slice(3).map((item) => [item.snapshot_type, item.session_segment, item.sequence])).toEqual([["start", 2, 4], ["trace", 2, 5], ["end", 2, 6]]);
}, 20_000);
+ test("a crash after a turn settled, while its trace is on its way, still sends the turn's end and every event at the next start", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-turnend-"));
+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-turnend-state-"));
+ roots.push(root, stateDir);
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const envelopes: Array<Record<string, unknown>> = [];
+ const parse = (compressed: Uint8Array) => withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>;
+ let traces = 0;
+ const hangs: Array<() => void> = [];
+ let released = false;
+ // An upload that never gets through while the run lives; once it "died", any upload fails at once.
+ const stall = () => released ? Promise.resolve(new Response(null, { status: 503 })) : new Promise<Response>((resolve) => hangs.push(() => resolve(new Response(null, { status: 503 }))));
+ const crashed = new SessionUploader({
+ stateDir,
+ upload: async (_sessionId, compressed) => {
+ const envelope = parse(compressed);
+ if (envelope.snapshot_type === "trace") {
+ traces += 1;
+ // The turn's own trace never gets through before the app dies (a slow link, a big transcript).
+ if (traces > 1) return stall();
+ // The flush before it is slow: the turn's end is taken while it is on its way.
+ await new Promise((resolve) => setTimeout(resolve, 300));
+ }
+ envelopes.push(envelope);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ fallbackScanMs: 60_000,
+ });
+ const sessionId = "session-turnend-1234";
+ crashed.startSession(sessionId, "workspace-turnend", root);
+ await crashed.idle(sessionId);
+ crashed.captureSnapshot(sessionId, "prompt");
+ crashed.recordTrace(sessionId, "tool.call", { n: 1 });
+ crashed.flushTrace(sessionId);
+ crashed.recordTrace(sessionId, "tool.call", { n: 2 });
+ crashed.recordTrace(sessionId, "session.idle", { status: "idle", outcome: "completed" });
+ crashed.captureSnapshot(sessionId, "turn_completed");
+ crashed.flushTrace(sessionId, { messages: [{ info: { id: "msg_user", role: "user" } }, { info: { id: "msg_answer", role: "assistant", parentID: "msg_user" } }] });
+ for (let waited = 0; waited < 5_000 && !envelopes.some((item) => item.snapshot_type === "trace"); waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));
+ await new Promise((resolve) => setTimeout(resolve, 600));
+ // The process dies here: the turn's trace (its end, its messages) is still on its way.
+ const sent = envelopes.length;
+ const sentEvents = envelopes.filter((item) => item.snapshot_type === "trace").flatMap((item) => (item.trace as Array<{ type: string; data?: Record<string, unknown> }>) ?? []);
+ expect(sentEvents.filter((event) => event.type === "tool.call").map((event) => event.data?.n)).toEqual([1]);
+ expect(sentEvents.map((event) => event.type)).not.toContain("turn.completed");
+
+ const next = new SessionUploader({
+ stateDir,
+ upload: async (_sessionId, compressed) => {
+ envelopes.push(parse(compressed));
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ fallbackScanMs: 60_000,
+ });
+ for (let waited = 0; waited < 5_000 && envelopes.length <= sent; waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));
+ const recovered = envelopes[sent]!;
+ expect(recovered).toMatchObject({ snapshot_type: "trace", session_segment: 1 });
+ const events = recovered.trace as Array<{ type: string; data?: Record<string, unknown> }>;
+ expect(events[0]?.type).toBe("collector.recovered");
+ expect(events[0]?.data).toMatchObject({ turn_open: false });
+ expect(events.filter((event) => event.type === "tool.call").map((event) => event.data?.n)).toEqual([2]);
+ const end = events.find((event) => event.type === "turn.completed");
+ expect((end?.data?.messages as unknown[] | undefined)?.length).toBe(2);
+ await next.stop();
+ // The stalled uploads of the run that "died" end, so they hold no upload slot.
+ released = true;
+ for (const release of hangs) release();
+ await crashed.stop().catch(() => undefined);
+ }, 20_000);
+
+ test("an upload on its way when the app died leaves no gap: the journal that holds its events goes out under its sequence", async () => {
+ for (const serverKnows of [true, false]) {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-unsent-"));
+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-unsent-state-"));
+ roots.push(root, stateDir);
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const stored: Array<Record<string, unknown>> = [];
+ const hangs: Array<() => void> = [];
+ let released = false;
+ // An upload that never gets through while the run lives; once it "died", any upload fails at once.
+ const stall = () => released ? Promise.resolve(new Response(null, { status: 503 })) : new Promise<Response>((resolve) => hangs.push(() => resolve(new Response(null, { status: 503 }))));
+ const options = {
+ stateDir,
+ fallbackScanMs: 60_000,
+ capabilities: async () => ({ schema_versions: [1, 2], canonical_trace: false, idempotency_key: true, integrity: true, integrity_summary: true }),
+ sessionIntegrity: async () => serverKnows
+ ? Response.json(stored.length === 0
+ ? { summary: true, max_segment: null, max_sequence: null, segments: [] }
+ : { summary: true, max_segment: 1, max_sequence: Math.max(...stored.map((item) => Number(item.sequence))), segments: [] })
+ : Response.json({ detail: "unavailable" }, { status: 503 }),
+ };
+ const crashed = new SessionUploader({
+ ...options,
+ upload: async (_sessionId, compressed) => {
+ const envelope = withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>;
+ // The trace flush is on its way when the app dies: the server never stores it.
+ if (envelope.snapshot_type === "trace") return stall();
+ stored.push(envelope);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ });
+ const sessionId = "session-unsent-1234";
+ crashed.startSession(sessionId, "workspace-unsent", root);
+ await crashed.idle(sessionId);
+ crashed.recordTrace(sessionId, "tool.call", { n: 1 });
+ crashed.flushTrace(sessionId);
+ await new Promise((resolve) => setTimeout(resolve, 600));
+ expect(stored.map((item) => [item.snapshot_type, item.sequence])).toEqual([["start", 1]]);
+
+ const next = new SessionUploader({
+ ...options,
+ upload: async (_sessionId, compressed) => {
+ stored.push(withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ });
+ for (let waited = 0; waited < 5_000 && !stored.some((item) => item.snapshot_type === "trace"); waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));
+ const recovered = stored.find((item) => item.snapshot_type === "trace")!;
+ const marker = (recovered.trace as Array<{ type: string; data: Record<string, unknown> }>)[0]!;
+ expect(marker.type).toBe("collector.recovered");
+ expect((recovered.trace as Array<{ type: string; data: Record<string, unknown> }>).filter((event) => event.type === "tool.call")).toHaveLength(1);
+ if (serverKnows) {
+ // Sequence 2 never reached the server: the recovered trace takes it.
+ expect(recovered).toMatchObject({ session_segment: 1, sequence: 2 });
+ expect(marker.data.unsent_sequences).toEqual({ from: 2, to: 2 });
+ } else {
+ // The server cannot say: the numbering goes on past it, as before.
+ expect(recovered).toMatchObject({ session_segment: 1, sequence: 3 });
+ expect(marker.data.unsent_sequences).toBeUndefined();
+ }
+ await next.stop();
+ released = true;
+ for (const release of hangs) release();
+ await crashed.stop().catch(() => undefined);
+ }
+ }, 30_000);
+
+ test("a recovered journal is a recovered segment when the server takes one, so a run killed before its start snapshot reads as such", async () => {
+ for (const recoveredSegments of [true, false]) {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-recseg-"));
+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-recseg-state-"));
+ roots.push(root, stateDir);
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const envelopes: Array<Record<string, unknown>> = [];
+ const hangs: Array<() => void> = [];
+ let released = false;
+ // An upload that never gets through while the run lives; once it "died", any upload fails at once.
+ const stall = () => released ? Promise.resolve(new Response(null, { status: 503 })) : new Promise<Response>((resolve) => hangs.push(() => resolve(new Response(null, { status: 503 }))));
+ const options = {
+ stateDir,
+ fallbackScanMs: 60_000,
+ capabilities: async () => ({ schema_versions: [1, 2], canonical_trace: false, idempotency_key: true, recovered_segments: recoveredSegments }),
+ };
+ const crashed = new SessionUploader({
+ ...options,
+ // The start snapshot never goes out: the app dies while it uploads.
+ upload: async () => stall(),
+ });
+ const sessionId = "session-recseg-1234";
+ crashed.startSession(sessionId, "workspace-recseg", root);
+ crashed.recordTrace(sessionId, "tool.call", { n: 1 });
+ await new Promise((resolve) => setTimeout(resolve, 600));
+ const next = new SessionUploader({
+ ...options,
+ upload: async (_sessionId, compressed) => {
+ envelopes.push(withTraceEvents(JSON.parse(zstdDecompressSync(compressed).toString("utf8"))) as Record<string, unknown>);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ });
+ for (let waited = 0; waited < 5_000 && !envelopes.some((item) => item.snapshot_type === "trace"); waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));
+ const recovered = envelopes.find((item) => item.snapshot_type === "trace");
+ expect(recovered).toMatchObject({ trigger: recoveredSegments ? "session_recovered" : "trace_flush" });
+ expect((recovered?.trace as Array<{ type: string }>)[0]?.type).toBe("collector.recovered");
+ await next.stop();
+ released = true;
+ for (const release of hangs) release();
+ await crashed.stop().catch(() => undefined);
+ }
+ }, 30_000);
+
test("an app that went away after a finished turn: the next launch sends the chat's end (marked recovered), once", async () => {
const root = await mkdtemp(join(tmpdir(), "omnirush-upload-endrec-"));
const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-endrec-state-"));
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index bf198cd9..dc0b7046 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -239,7 +239,11 @@ export type UploadTrigger =
| "fs_change"
| "periodic"
| "session_end"
- | "trace_flush";
+ | "trace_flush"
+ // The trace of a run that ended without its uploads (a crash, a kill),
+ // sent from its journal at the next start: its segment may have no start
+ // snapshot (the backend's recovered segment).
+ | "session_recovered";
/** How a turn ended, when not as answered: "aborted" is a turn the user stopped (Esc, Stop). */
export type TurnEndOutcome = "aborted";
@@ -674,6 +678,13 @@ type JournalState = {
full: boolean;
/** A recovery's stand-in state: never journaled itself. */
off: boolean;
+ /**
+ * Trace events taken off the chat's trace for an upload that has not
+ * settled yet (queued behind a snapshot, or on its way), oldest first: a
+ * rewrite after an earlier upload keeps them, so a crash before they are
+ * sent loses none of them.
+ */
+ inFlight: TraceEvent[][];
/**
* Lines are written once the chat's own recovery is done (prepareSession):
* a journal a previous process left under the same name is read first.
@@ -4503,7 +4514,7 @@ export function segmentEnd(payload: unknown, segment: number): "missing" | "pres
}
/** The /collect features this client uses; without a capability answer, none. */
-type UploadFeatures = { idempotencyKey: boolean; integrity: boolean };
+type UploadFeatures = { idempotencyKey: boolean; integrity: boolean; recoveredSegments?: boolean };
const NO_UPLOAD_FEATURES: UploadFeatures = { idempotencyKey: false, integrity: false };
/** What a /collect upload carries besides its body. */
@@ -4642,7 +4653,11 @@ export class SessionUploader {
server: () => this.uploadUrl ?? "gateway",
...(this.capabilityProbe ? { probe: this.capabilityProbe } : {}),
onAnswer: (capabilities) => {
- this.probedFeatures = { idempotencyKey: capabilities.idempotency_key === true, integrity: capabilities.integrity === true };
+ this.probedFeatures = {
+ idempotencyKey: capabilities.idempotency_key === true,
+ integrity: capabilities.integrity === true,
+ recoveredSegments: capabilities.recovered_segments === true || (capabilities.triggers?.includes("session_recovered") ?? false),
+ };
},
log: (level, message, attributes) => this.log(level, message, attributes),
});
@@ -5172,7 +5187,7 @@ export class SessionUploader {
failureCount: 0,
ready: Promise.resolve(),
tail: Promise.resolve(),
- journal: { pending: [], timer: null, tail: Promise.resolve(), bytes: 0, full: false, off: false, ready: false },
+ journal: { pending: [], timer: null, tail: Promise.resolve(), bytes: 0, full: false, off: false, ready: false, inFlight: [] },
refused: [],
idleEndTimer: null,
turns: 0,
@@ -6135,16 +6150,25 @@ export class SessionUploader {
}
// Capture context: the turn's last messages (its tool calls) arrive here, not as turn.messages.
if (isRecord(finalTrace)) this.contextOf(sessionId)?.turnMessages(sessionId, finalTrace.messages, true);
- pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });
+ const completed: TraceEvent = { at: new Date().toISOString(), type: "turn.completed", data: finalTrace };
+ pendingTrace.push(completed);
+ // The turn's end is on disk at once: the journal already says the turn
+ // is closed (its turn_completed milestone), and the upload may wait
+ // behind that snapshot's. A crash meanwhile still sends the turn's end
+ // and messages at the next start.
+ this.journalEvent(state, completed, true);
}
if (pendingTrace.length === 0) return;
+ state.journal.inFlight.push(pendingTrace);
this.enqueue(state, async () => {
await state.ready;
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
try {
await this.uploadTrace(state, pendingTrace);
} finally {
- // Uploaded, spooled or refused: the journal keeps only what came after.
+ // Uploaded, spooled or refused: the journal keeps only what came after
+ // (and what other uploads have not sent yet).
+ state.journal.inFlight = state.journal.inFlight.filter((batch) => batch !== pendingTrace);
await this.journalRewrite(state);
}
});
@@ -6712,7 +6736,7 @@ export class SessionUploader {
}
}
- private async uploadTrace(state: SessionState, events: TraceEvent[]): Promise<void> {
+ private async uploadTrace(state: SessionState, events: TraceEvent[], trigger: Extract<UploadTrigger, "trace_flush" | "session_recovered"> = "trace_flush"): Promise<void> {
if (events.length === 0) return;
// A session imported from the opencode 2.x engine's store says so in each
// trace upload (an opaque event type: kept as sent), so a reader can tell
@@ -6743,13 +6767,13 @@ export class SessionUploader {
// One form per schema: the server reads the top-level trace for 3, and only trace.json for 1 and 2.
...(version === TRACE_SCHEMA_VERSION ? { trace: batch.events } : {}),
});
- let uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));
- if (uploaded === "key_reused") uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));
+ let uploaded = await this.uploadEnvelope(state, "trace", trigger, await makeBody(schemaVersion));
+ if (uploaded === "key_reused") uploaded = await this.uploadEnvelope(state, "trace", trigger, await makeBody(schemaVersion));
if (uploaded === "unsupported_schema" && schemaVersion === TRACE_SCHEMA_VERSION && !downgraded) {
downgraded = true;
schemaVersion = 2;
this.downgradeTraceSchema();
- uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));
+ uploaded = await this.uploadEnvelope(state, "trace", trigger, await makeBody(schemaVersion));
}
}
}
@@ -7269,7 +7293,7 @@ export class SessionUploader {
this.scheduleJournal(state, now);
}
- private journalEvent(state: SessionState, event: TraceEvent): void {
+ private journalEvent(state: SessionState, event: TraceEvent, now = false): void {
if (!this.journalDir || state.journal.full || state.journal.off) return;
let line: string;
try {
@@ -7283,7 +7307,7 @@ export class SessionUploader {
}
state.journal.bytes += line.length + 1;
state.journal.pending.push(line);
- this.scheduleJournal(state, false);
+ this.scheduleJournal(state, now);
}
private scheduleJournal(state: SessionState, now: boolean): void {
@@ -7325,7 +7349,7 @@ export class SessionUploader {
const lines = [this.journalMetaLine(state)];
state.journal.bytes = 0;
state.journal.full = false;
- for (const event of state.trace) {
+ for (const event of [...state.journal.inFlight.flat(), ...state.trace]) {
const line = JSON.stringify({ e: event });
if (state.journal.bytes + line.length > MAX_JOURNAL_BYTES) {
state.journal.full = true;
@@ -7415,6 +7439,25 @@ export class SessionUploader {
void Promise.allSettled(recovered);
}
+ /**
+ * The sequences a previous process took for this chat that the server
+ * never stored and no spooled upload holds (an upload on its way when the
+ * process died): the run after the server's highest and the spool's. Null
+ * when there are none, or when the server cannot say (offline, an older
+ * backend, a chat it holds nothing of): the numbering then goes on as it was.
+ */
+ private async unsentSequences(state: SessionState): Promise<{ from: number; to: number } | null> {
+ if (state.sequence <= 0) return null;
+ const server = await this.serverPosition(state.id);
+ if (!server || server.segment > state.segment || server.sequence >= state.sequence) return null;
+ const spooled = this.spoolDir
+ ? await this.spoolLocked(() => this.listSpool()).catch(() => null)
+ : [];
+ if (spooled === null) return null;
+ const floor = Math.max(server.sequence, ...spooled.filter((entry) => entry.session_id === state.id).map((entry) => entry.sequence));
+ return floor < state.sequence ? { from: floor + 1, to: state.sequence } : null;
+ }
+
/**
* A chat's journal outlived the app and the ledger has no end for it: the
* end never went out. A turn left open too (its messages are settled in the
@@ -7515,17 +7558,29 @@ export class SessionUploader {
if (path && !isUploadPathDenied(path) && state.touchedPaths.size < MAX_TOUCHED_PATHS) state.touchedPaths.add(path);
}
}
+ // The upload the process had on its way when it went away took a
+ // sequence the server never stored; its events are this journal's.
+ // They go out under that number again, so the segment shows no gap.
+ const unsent = events.length > 0 ? await this.unsentSequences(state) : null;
+ if (unsent) state.sequence = unsent.from - 1;
const marker: TraceEvent = {
at: new Date().toISOString(),
type: "collector.recovered",
- data: { reason: "restart", turn_open: meta.turn_open, events: events.length, journal_at: meta.at, session_segment: state.segment },
+ data: {
+ reason: "restart", turn_open: meta.turn_open, events: events.length, journal_at: meta.at, session_segment: state.segment,
+ ...(unsent ? { unsent_sequences: unsent } : {}),
+ },
};
// The end that follows was taken at this launch, not when the app went away.
const endMarker: TraceEvent[] = endMissing
? [{ at: marker.at, type: "session.end_recovered", data: { reason: journal.reason ?? "run_ended_without_end", last_message_id: state.lastMessageId ?? null, ...(meta.turn_open ? { turn_open: true } : {}), ...(record ? {} : { ledger: "missing" }) } }]
: [];
+ // A run killed before its segment's start snapshot went out leaves a
+ // segment of traces only: the backend takes it as a recovered segment
+ // when the trace says so (its trigger), not as a missing start.
+ const trigger = (await this.uploadFeatures()).recoveredSegments ? "session_recovered" as const : "trace_flush" as const;
try {
- await this.uploadTrace(state, [marker, ...events, ...endMarker]);
+ await this.uploadTrace(state, [marker, ...events, ...endMarker], trigger);
} catch (error) {
// Refused (a 422): recovered data is never dropped; the next start tries again, up to the age limit.
this.log("warn", "OmniRush session journal upload refused; kept for the next start", { sessionId: meta.session_id, error: error instanceof Error ? error.message : "unknown" });