Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: recover unfinished turns

ogsamrata11e98d28c37
diff --git a/apps/server/src/capture-client.test.ts b/apps/server/src/capture-client.test.ts
index 39deb3a4..5d52a94e 100644
--- a/apps/server/src/capture-client.test.ts
+++ b/apps/server/src/capture-client.test.ts
@@ -576,3 +576,99 @@ describe("capture worker restarts", () => {
expect(sink.envelopes().map((item) => [item.session_id, item.snapshot_type])).toEqual([["session-restart-0001", "start"]]);
}, 30_000);
});
+
+describe("turns left open: worker restarts and quits", () => {
+ let crashSwitch: string | undefined;
+ let contextSwitch: string | undefined;
+ beforeAll(() => {
+ crashSwitch = process.env.OMNIRUSH_CAPTURE_TEST_CRASH;
+ contextSwitch = process.env.OMNIRUSH_CAPTURE_CONTEXT;
+ process.env.OMNIRUSH_CAPTURE_TEST_CRASH = "1";
+ process.env.OMNIRUSH_CAPTURE_CONTEXT = "0";
+ });
+ afterAll(() => {
+ if (crashSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_TEST_CRASH;
+ else process.env.OMNIRUSH_CAPTURE_TEST_CRASH = crashSwitch;
+ if (contextSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_CONTEXT;
+ else process.env.OMNIRUSH_CAPTURE_CONTEXT = contextSwitch;
+ });
+
+ const SESSION = "ses_recovered_0001";
+
+ /** A fake engine with one chat: its status and its messages. */
+ function fakeEngine() {
+ const control = { busy: true, messages: [] as Array<{ info: Record<string, unknown>; parts: unknown[] }> };
+ const message = (id: string, role: "user" | "assistant") => ({
+ info: { id, sessionID: SESSION, role, time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 } },
+ parts: [{ id: `${id}_p`, messageID: id, sessionID: SESSION, type: "text", text: `${role} ${id}` }],
+ });
+ const server = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ fetch(request) {
+ const url = new URL(request.url);
+ if (url.pathname === "/session/status") return Response.json(control.busy ? { [SESSION]: { type: "busy" } } : {});
+ if (url.pathname === `/session/${SESSION}/message`) return Response.json(control.messages);
+ if (url.pathname === `/session/${SESSION}/children`) return Response.json([]);
+ if (url.pathname === `/session/${SESSION}`) return Response.json({ id: SESSION });
+ return Response.json({ code: "not_found" }, { status: 404 });
+ },
+ });
+ cleanups.push(() => server.stop(true));
+ const target = { baseUrl: `http://127.0.0.1:${server.port}`, headers: [] as Array<[string, string]>, search: "", engine: "v1" as const };
+ return { control, message, target };
+ }
+
+ test("a worker killed mid-turn: the next worker uploads the turn's journaled events, marked recovered, and settles the turn from the engine", async () => {
+ const root = await tempDir("recover-root");
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const sink = uploadSink();
+ const engine = fakeEngine();
+ engine.control.messages = [engine.message("msg_1", "user")];
+ const capture = service({
+ stateDir: await tempDir("recover-state"),
+ sessionUploader: { upload: sink.upload },
+ restartBackoffMs: { baseMs: 50, maxMs: 50 },
+ });
+ capture.startSession(SESSION, "workspace-recover", root);
+ capture.captureSnapshot(SESSION, "prompt");
+ capture.recordTrace(SESSION, "engine.request", { body: "do the task" });
+ capture.recordTrace(SESSION, "tool.call", { name: "edit", marker: "before-the-crash" });
+ capture.observeSession(SESSION, engine.target);
+ // The journal is on disk within its flush interval; then the worker dies.
+ await new Promise((resolvePromise) => setTimeout(resolvePromise, 800));
+ capture.recordTrace(SESSION, "omnirush.test.worker_crash");
+ await until(() => capture.mode() === "restarting", 10_000, "the worker's exit");
+ // The turn finishes while the new worker comes up.
+ engine.control.messages.push(engine.message("msg_2", "assistant"));
+ engine.control.busy = false;
+ await until(() => sink.envelopes().some((item) => JSON.stringify(item).includes("\"recovered\":true")), 30_000, "the settled recovered turn");
+ const envelopes = sink.envelopes() as unknown as Array<{ snapshot_type: string; trace?: Array<{ type: string; data?: unknown }> }>;
+ const recovered = envelopes.find((item) => item.trace?.[0]?.type === "collector.recovered");
+ expect(recovered).toBeDefined();
+ expect(JSON.stringify(recovered)).toContain("before-the-crash");
+ const settled = envelopes.find((item) => item.trace?.some((event) => event.type === "turn.completed"));
+ const completed = settled?.trace?.find((event) => event.type === "turn.completed");
+ expect(completed?.data).toMatchObject({ recovered: true });
+ expect(JSON.stringify(completed)).toContain("msg_2");
+ }, 60_000);
+
+ test("an app quit settles a turn still running before the chat ends", async () => {
+ const root = await tempDir("quit-root");
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const sink = uploadSink();
+ const engine = fakeEngine();
+ engine.control.messages = [engine.message("msg_1", "user"), engine.message("msg_2", "assistant")];
+ const capture = service({ stateDir: await tempDir("quit-state"), sessionUploader: { upload: sink.upload } });
+ capture.startSession(SESSION, "workspace-quit", root);
+ capture.captureSnapshot(SESSION, "prompt");
+ capture.observeSession(SESSION, engine.target);
+ await new Promise((resolvePromise) => setTimeout(resolvePromise, 500));
+ await capture.stop();
+ const envelopes = sink.envelopes() as unknown as Array<{ snapshot_type: string; trace?: Array<{ type: string; data?: unknown }> }>;
+ const completed = envelopes.flatMap((item) => item.trace ?? []).find((event) => event.type === "turn.completed");
+ expect(completed?.data).toMatchObject({ reason: "app_quit" });
+ expect(JSON.stringify(completed)).toContain("msg_2");
+ expect(envelopes.at(-1)?.snapshot_type).toBe("end");
+ }, 60_000);
+});
diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.ts
index 6ac7f771..ed63aca0 100644
--- a/apps/server/src/capture-client.ts
+++ b/apps/server/src/capture-client.ts
@@ -47,6 +47,8 @@ const RESTART_STABLE_MS = 5 * 60_000;
/** Calls waiting for a worker to start; past this the oldest one nobody waits on is dropped. */
const MAX_QUEUED_CALLS = 20_000;
const SESSION_ID_PATTERN = /^[A-Za-z0-9._:-]{8,128}$/;
+/** Chats whose engine is remembered for a restarted worker. */
+const MAX_OBSERVED_TARGETS = 256;
export type CaptureServiceOptions = Omit<CaptureHostOptions, "onSessionClosed"> & {
/** Run capture on a worker thread; default on unless OMNIRUSH_CAPTURE_WORKER is 0, false, no or off. */
@@ -149,6 +151,8 @@ class CaptureClient implements CaptureService {
/** Only the first start falls back to capturing in-process; later failures start new workers. */
private canRunLocally = true;
private overflowing = false;
+ /** The engine each recently observed chat's requests went to: a restarted worker settles their open turns there. */
+ private readonly observedTargets = new Map<string, EngineTarget>();
private stopping: Promise<void> | null = null;
constructor(private readonly options: CaptureServiceOptions) {
@@ -234,6 +238,9 @@ class CaptureClient implements CaptureService {
}
observeSession(sessionId: string, target: EngineTarget): void {
+ this.observedTargets.delete(sessionId);
+ this.observedTargets.set(sessionId, target);
+ if (this.observedTargets.size > MAX_OBSERVED_TARGETS) this.observedTargets.delete(this.observedTargets.keys().next().value as string);
this.send({ kind: "call", id: null, method: "observeSession", args: [sessionId, target] });
}
@@ -382,6 +389,8 @@ class CaptureClient implements CaptureService {
engineVersion: this.options.engineVersion,
accountId: Boolean(this.options.accountId),
sessionIntegrity: Boolean(this.options.sessionIntegrity),
+ // Always: the chats this thread saw observed are known here even without a resolver.
+ engineTarget: true,
sessionUploader: {
upload: Boolean(sessionUploader.upload),
uploadFile: Boolean(sessionUploader.uploadFile),
@@ -582,6 +591,15 @@ class CaptureClient implements CaptureService {
...(request.turns !== undefined ? { turns: request.turns } : {}),
})) };
}
+ case "engineTarget":
+ // A chat this server saw go to an engine (a restarted worker's), else the workspace's engine.
+ return {
+ kind: "result",
+ id,
+ ok: true,
+ response: null,
+ target: this.observedTargets.get(request.sessionId) ?? (this.options.engineTarget ? await this.options.engineTarget(request.sessionId, request.workspaceId) : null),
+ };
case "accountId":
return { kind: "result", id, ok: true, response: null, account: this.options.accountId ? await this.options.accountId() : null };
case "refreshAccessToken": {
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index e563adbb..299b495b 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -25,7 +25,7 @@ import {
import type { CaptureStopOptions } from "./capture-protocol.js";
import { SessionArchiver, type FilesUsedStatus, type SessionArchiverOptions } from "./session-archive/index.js";
import { ProjectArchiveLifecycle, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";
-import { SessionUploader, type TraceCapabilities, type UploadMetrics, type UploadRequestOptions, type UploadWebVisit } from "./session-uploader.js";
+import { SessionUploader, type RecoveredTurn, type TraceCapabilities, type UploadMetrics, type UploadRequestOptions, type UploadWebVisit } from "./session-uploader.js";
export type { EngineReplacement, EngineTarget } from "./session-upload-observer.js";
@@ -33,6 +33,10 @@ export type CaptureLog = ArchiveLifecycleLog;
/** Prompt bodies larger than this are not parsed for the trace (nor for attachments). */
const MAX_TRACED_REQUEST_BYTES = 4 * 1024 * 1024;
+/** An app quit gives the turns still open this long in all to be settled from the engine. */
+const QUIT_SETTLE_BUDGET_MS = 5_000;
+/** A recovered turn asks for its engine again after these waits (the engine may still be starting). */
+const RECOVERY_TARGET_DELAYS_MS = [0, 5_000, 15_000, 30_000, 60_000];
/** A captured engine request, as the "engine.request" trace event and the prompt's attachments are built from it. */
export type PromptRecord = {
@@ -60,6 +64,12 @@ export type CaptureHostOptions = {
log: CaptureLog;
/** The signed-in account's id (null: not known now); spooled uploads are stamped with it. */
accountId?: () => Promise<string | null>;
+ /**
+ * The engine holding a chat of this workspace, for settling a turn a
+ * previous process left open; null when there is none (remote, unknown,
+ * or the engine does not have the chat).
+ */
+ engineTarget?: (sessionId: string, workspaceId: string) => Promise<EngineTarget | null>;
/** GET /omnirush/me/sessions/{id}/integrity with the device bearer. */
sessionIntegrity?: (sessionId: string, options?: { summary?: boolean; turns?: number }) => Promise<Response>;
sessionUploader: {
@@ -107,8 +117,10 @@ export class CaptureHost {
private readonly sessionRoots = new Map<string, string>();
/** The state dir and the app's data/config/cache dirs: never reported or recorded as a touched file. */
private readonly appDirs: string[];
+ private readonly engineTarget?: CaptureHostOptions["engineTarget"];
constructor(options: CaptureHostOptions) {
+ this.engineTarget = options.engineTarget;
this.sessionUploader = new SessionUploader({
...options.sessionUploader,
stateDir: options.stateDir,
@@ -117,6 +129,8 @@ export class CaptureHost {
log: options.log,
...(options.accountId ? { accountId: options.accountId } : {}),
...(options.sessionIntegrity ? { sessionIntegrity: options.sessionIntegrity } : {}),
+ // A turn a crashed process or worker left open is settled from the engine's messages.
+ onRecoveredTurn: (turn) => void this.recoverTurn(turn),
// Capture context (context/); OMNIRUSH_CAPTURE_CONTEXT=0 turns it off.
context: options.sessionUploader?.context ?? {},
...(options.onSessionClosed ? { onSessionClosed: options.onSessionClosed } : {}),
@@ -212,7 +226,7 @@ export class CaptureHost {
}
/** The engine accepted a captured request: follow the session until its turn settles. */
- observeSession(sessionId: string, target: EngineTarget): void {
+ observeSession(sessionId: string, target: EngineTarget, recovered = false): void {
const root = this.sessionRoots.get(sessionId);
void observeUploadedSession({
sessionUploader: this.sessionUploader,
@@ -221,9 +235,31 @@ export class CaptureHost {
observers: this.observers,
sessionId,
target,
+ ...(recovered ? { recovered } : {}),
});
}
+ /**
+ * A turn a previous process (or worker) left open: the chat resumes here
+ * and its observer settles the turn from the engine's messages, or follows
+ * it to its end when it is still running. Without an engine that has the
+ * chat, its messages wait for its next prompt (the checkpoint did not move).
+ */
+ private async recoverTurn(turn: RecoveredTurn): Promise<void> {
+ if (!this.engineTarget) return;
+ const stopped = this.observers.controller.signal;
+ for (const delay of RECOVERY_TARGET_DELAYS_MS) {
+ if (delay > 0) await new Promise((resolvePromise) => setTimeout(resolvePromise, delay).unref?.());
+ if (stopped.aborted || this.sessionUploader.hasSession(turn.sessionId)) return;
+ const target = await this.engineTarget(turn.sessionId, turn.workspaceId).catch(() => null);
+ if (stopped.aborted || this.sessionUploader.hasSession(turn.sessionId)) return;
+ if (!target) continue;
+ this.startSession(turn.sessionId, turn.workspaceId, turn.root);
+ this.observeSession(turn.sessionId, target, true);
+ return;
+ }
+ }
+
/** Files used, for the app's one-time notice: what omnirush.ai says. */
filesUsedStatus(): Promise<FilesUsedStatus> {
return this.archive.filesUsedStatus();
@@ -250,12 +286,29 @@ export class CaptureHost {
* final archives (unless `archiveFinals` is false: the account is gone).
*/
async stop(options: CaptureStopOptions = { archiveFinals: true }): Promise<void> {
+ // Turns still open get their messages and their end first, within a bounded budget.
+ await this.settleOpenTurns(QUIT_SETTLE_BUDGET_MS);
const archiveStopped = this.archive.stop({ finals: options.archiveFinals });
this.observers.controller.abort();
await this.sessionUploader.stop().catch(() => undefined);
await archiveStopped;
}
+ /** Settles every followed turn as the engine has it now, waiting at most `budgetMs`. */
+ private async settleOpenTurns(budgetMs: number): Promise<void> {
+ const open = [...this.observers.sessions.values()].map((session) => session.settleNow?.().catch(() => undefined));
+ if (open.length === 0) return;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ await Promise.race([
+ Promise.allSettled(open),
+ new Promise((resolvePromise) => {
+ timer = setTimeout(resolvePromise, budgetMs);
+ timer.unref?.();
+ }),
+ ]);
+ clearTimeout(timer);
+ }
+
/** Resolves once every queued capture, upload and archive step has settled (tests, profiling). */
async idle(): Promise<void> {
await this.sessionUploader.idleAll();
diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.ts
index 2663ada1..fea734f9 100644
--- a/apps/server/src/capture-protocol.ts
+++ b/apps/server/src/capture-protocol.ts
@@ -46,6 +46,7 @@ export type HostRequest =
| { type: "uploadFile"; sessionId: string; path: string; size: number; options?: UploadRequestOptions }
| { type: "capabilities" }
| { type: "accountId" }
+ | { type: "engineTarget"; sessionId: string; workspaceId: string }
| { type: "sessionIntegrity"; sessionId: string; summary?: boolean; turns?: number }
| { type: "refreshAccessToken" }
| { type: "archiveRequest"; path: string; method: "GET" | "POST"; body?: string; refresh?: false }
@@ -56,7 +57,7 @@ export type RequestChannel = "uploader" | "archive";
export type SerializedResponse = { status: number; statusText: string; headers: Array<[string, string]>; body: ArrayBuffer | null };
export type RequestResult =
- | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null; value?: TraceCapabilities; account?: string | null }
+ | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null; value?: TraceCapabilities; account?: string | null; target?: EngineTarget | null }
| { kind: "result"; id: number; ok: false; error: string; name: string };
export type ToWorker = CaptureCall | RequestResult;
@@ -79,6 +80,8 @@ export type CaptureWorkerInit = {
accountId: boolean;
/** The main thread can read the server's integrity record of a chat. */
sessionIntegrity: boolean;
+ /** The main thread can name the engine holding a chat (to settle a turn a previous process left open). */
+ engineTarget: boolean;
sessionUploader: { upload: boolean; uploadFile: boolean; capabilities: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };
archive: {
enabled: boolean;
diff --git a/apps/server/src/capture-worker.ts b/apps/server/src/capture-worker.ts
index d76b54f8..0c47dc6d 100644
--- a/apps/server/src/capture-worker.ts
+++ b/apps/server/src/capture-worker.ts
@@ -116,6 +116,9 @@ const host = new CaptureHost({
engineVersion: init.engineVersion,
log,
...(init.accountId ? { accountId: async () => (await ask("uploader", { type: "accountId" })).account ?? null } : {}),
+ ...(init.engineTarget
+ ? { engineTarget: async (sessionId: string, workspaceId: string) => (await ask("uploader", { type: "engineTarget", sessionId, workspaceId })).target ?? null }
+ : {}),
...(init.sessionIntegrity
? {
sessionIntegrity: async (sessionId: string, options: { summary?: boolean; turns?: number } = {}) => {
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index 71d2de94..8df606e1 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -193,7 +193,7 @@ import { CloudProviderSync, parseCloudProviderDenSession } from "./cloud-provide
import { createEngineV2Preview, type EngineV2Preview } from "./engine-v2-preview.js";
import { sessionUploaderEnabled } from "./session-uploader.js";
import { startCaptureService, type CaptureService } from "./capture-client.js";
-import { buildOpencodeProxyUrl, engineTarget } from "./session-upload-observer.js";
+import { buildOpencodeProxyUrl, engineTarget, type EngineTarget } from "./session-upload-observer.js";
import { OmniRushGatewayBroker } from "./omnirush-gateway-broker.js";
import { awaitOmniRushModelCatalogSettled, startOmniRushModelCatalogSync } from "./omnirush-model-catalog-sync.js";
import { OmniRushVoiceService, voiceProjectContext } from "./omnirush-voice.js";
@@ -901,6 +901,27 @@ function uploadDeletedSessionId(method: string, proxyPath: string): string | nul
type EngineRequestTarget = { baseUrl: string; headers: Headers };
+/**
+ * The engine holding a chat of a local workspace, scoped like the proxy's
+ * requests, for settling a turn a previous process left open; null for a
+ * remote or unknown workspace, no engine, or an engine without the chat.
+ */
+async function captureEngineTarget(config: ServerConfig, sessionId: string, workspaceId: string): Promise<EngineTarget | null> {
+ const workspace = config.workspaces.find((item) => item.id === workspaceId);
+ if (!workspace || workspace.workspaceType === "remote") return null;
+ const route = enginePoolForConfig(config)?.routeRequest("GET", `/session/${sessionId}/message`) ?? null;
+ const connection = route ? null : resolveWorkspaceOpencodeConnection(config, workspace);
+ const baseUrl = route?.target.baseUrl ?? connection?.baseUrl?.trim() ?? "";
+ if (!baseUrl) return null;
+ const auth = route ? buildEngineAuthProbeHeader(route.target.username, route.target.password) : connection?.authHeader ?? null;
+ const directory = resolveOpencodeDirectory(workspace);
+ if (!(await engineHasSession(baseUrl, auth, directory, sessionId))) return null;
+ const headers = new Headers();
+ if (auth) headers.set("Authorization", auth);
+ const scoped = scopeWorkspaceOpencodeRequest(headers, "", directory);
+ return engineTarget(baseUrl, scoped.headers, scoped.search);
+}
+
/** How long the existence check before a chat's first captured prompt waits for the engine. */
const ENGINE_SESSION_CHECK_TIMEOUT_MS = 3_000;
@@ -1060,6 +1081,8 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
// "opencode/1.18.32-r2": the bundled engine, as every upload names it (engine-identity.ts).
engineVersion: BUNDLED_ENGINE_VERSION,
log: (level, message, attributes) => logger.log(level, message, attributes),
+ // A turn a crashed process or worker left open is settled from the engine that has its chat.
+ engineTarget: (sessionId: string, workspaceId: string) => captureEngineTarget(config, sessionId, workspaceId),
...(gatewayBroker.enabled
? { accountId: () => gatewayBroker.accountId(), sessionIntegrity: (sessionId: string, options?: { summary?: boolean; turns?: number }) => gatewayBroker.sessionIntegrity(sessionId, options) }
: {}),
diff --git a/apps/server/src/session-upload-observer.test.ts b/apps/server/src/session-upload-observer.test.ts
index 16fa5219..796cc23e 100644
--- a/apps/server/src/session-upload-observer.test.ts
+++ b/apps/server/src/session-upload-observer.test.ts
@@ -885,3 +885,57 @@ describe("message outlines and turn outcomes", () => {
])).toBe("stopped");
});
});
+
+describe("turns left open by an ended process, and turns open at quit", () => {
+ test("a recovered turn on an idle session settles at the first read, with its messages, marked recovered", async () => {
+ const { control, target } = startEngine();
+ control.messages = [message("msg_1", "user"), message("msg_2", "assistant")];
+ const uploader = new FakeUploader();
+ const archive = fakeArchive();
+ const clock = fakeClock();
+ await observeUploadedSession({ sessionUploader: uploader, archive, observers: createSessionObservers(), target, timing: clock.timing, sessionId: SESSION, recovered: true });
+ expect(control.statusReads).toBe(1);
+ expect(uploader.turnMessageIds()).toEqual([["msg_1", "msg_2"]]);
+ const final = uploader.entries.find((entry) => entry.kind === "flush" && entry.final !== undefined);
+ expect(final).toMatchObject({ final: { recovered: true } });
+ expect(uploader.checkpoint).toBe("msg_2");
+ expect(archive.calls).toEqual(["followed", "completed"]);
+ });
+
+ test("a recovered turn still running (a restarted worker) is followed to its end as usual", async () => {
+ const { control, target } = startEngine();
+ let busy = true;
+ control.status = () => (busy ? "busy" : "idle");
+ control.messages = [message("msg_1", "user")];
+ const uploader = new FakeUploader();
+ const archive = fakeArchive();
+ const clock = fakeClock();
+ clock.at(30_000, () => {
+ busy = false;
+ control.messages.push(message("msg_2", "assistant"));
+ });
+ await observeUploadedSession({ sessionUploader: uploader, archive, observers: createSessionObservers(), target, timing: clock.timing, sessionId: SESSION, recovered: true });
+ expect(clock.elapsed()).toBeGreaterThanOrEqual(30_000);
+ expect(uploader.turnMessageIds()).toEqual([["msg_1", "msg_2"]]);
+ });
+
+ test("settleNow (an app quit) ends a running turn with the messages so far, once", async () => {
+ const { control, target } = startEngine();
+ control.status = () => "busy";
+ control.messages = [message("msg_1", "user"), message("msg_2", "assistant")];
+ const uploader = new FakeUploader();
+ const archive = fakeArchive();
+ const observers = createSessionObservers();
+ // A real clock: the observer is between status reads when the app quits.
+ void observeUploadedSession({ sessionUploader: uploader, archive, observers, target, sessionId: SESSION });
+ await Bun.sleep(50);
+ const observed = observers.sessions.get(SESSION);
+ expect(observed?.settleNow).toBeDefined();
+ await observed!.settleNow!();
+ await observed!.settleNow!();
+ observers.controller.abort();
+ expect(uploader.turnMessageIds()).toEqual([["msg_1", "msg_2"]]);
+ expect(uploader.labels()).toContain("session.quit_settled");
+ expect(uploader.entries.find((entry) => entry.kind === "flush" && entry.final !== undefined)).toMatchObject({ final: { reason: "app_quit" } });
+ });
+});
diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.ts
index d757bcda..638712b2 100644
--- a/apps/server/src/session-upload-observer.ts
+++ b/apps/server/src/session-upload-observer.ts
@@ -32,14 +32,21 @@ export type EngineTarget = {
* snapshot is taken right as the prompt goes out, and its messages stop
* before the prompt's own.
*/
-type FollowedTurn = { snapshotTaken: boolean; cutAt: number | null };
+type FollowedTurn = { snapshotTaken: boolean; cutAt: number | null; settled?: boolean };
/**
* A session being observed: the engine its latest captured request went to
* (an engine that restarted may answer on another port), how many requests
* asked for it while it was, the turn being followed, and the observation itself.
*/
-type ObservedSession = { target: EngineTarget; requests: number; turn: FollowedTurn | null; done: Promise<void> };
+type ObservedSession = {
+ target: EngineTarget;
+ requests: number;
+ turn: FollowedTurn | null;
+ done: Promise<void>;
+ /** Settles the turn being followed now, as the engine has it (an app quit); nothing when none is open. */
+ settleNow?: () => Promise<void>;
+};
/** The engine that took over from a closed one: its base URL and its Authorization header. */
export type EngineReplacement = { baseUrl: string; authorization: string | null };
@@ -797,6 +804,12 @@ export function observeUploadedSession(input: {
sessionId: string;
target: EngineTarget;
timing?: Partial<ObserverTiming>;
+ /**
+ * The turn was left open by a process that ended (a crash, a killed
+ * worker): an idle session settles at its first read, and the turn's end
+ * says it was recovered.
+ */
+ recovered?: boolean;
}): Promise<void> {
if (!input.sessionUploader.enabled) return Promise.resolve();
const observer = input.observers;
@@ -872,7 +885,14 @@ export function observeUploadedSession(input: {
* history when it was read already; a turn ended by the next prompt keeps
* only the messages from before that prompt.
*/
+ let recovering = input.recovered === true;
const settle = async (status: string, read?: EngineHistory): Promise<void> => {
+ // Once per turn: an app quit may settle it while the follow loop is about to.
+ const turn = turnState();
+ if (turn.settled) return;
+ turn.settled = true;
+ const recovered = recovering;
+ recovering = false;
let history: EngineHistory | null = read ?? null;
let unavailable: Record<string, unknown> = { unavailable: true };
try {
@@ -953,7 +973,12 @@ export function observeUploadedSession(input: {
// A turn the user stopped (Esc, Stop) says so on its trigger and its turn.completed event too.
const aborted = outcome === "stopped" ? ("aborted" as const) : undefined;
captureTurnSnapshot(aborted);
- sessionUploader.flushTrace(sessionId, { messages: history ? newest.messages : unavailable, ...(aborted ? { outcome: aborted } : {}) });
+ sessionUploader.flushTrace(sessionId, {
+ messages: history ? newest.messages : unavailable,
+ ...(aborted ? { outcome: aborted } : {}),
+ ...(recovered ? { recovered: true } : {}),
+ ...(status === "quit" ? { reason: "app_quit" } : {}),
+ });
// The delta's turn number is the engine's completed-turn count, which
// survives app restarts; without the messages the archiver numbers it
// right after the last archived turn. It also arms the idle final archive.
@@ -1016,6 +1041,11 @@ export function observeUploadedSession(input: {
if (unavailableTraced && unavailableSince !== null) {
sessionUploader.recordTrace(sessionId, "session.engine_recovered", { failures, unavailable_ms: timing.now() - unavailableSince });
}
+ // A turn a previous process left open: an idle session has nothing more to wait for.
+ if (recovering && status === "idle") {
+ await settle(status);
+ return session.requests;
+ }
failures = 0;
unavailableSince = null;
unavailableTraced = false;
@@ -1084,6 +1114,11 @@ export function observeUploadedSession(input: {
return null;
};
+ session.settleNow = async () => {
+ if (!session.turn || session.turn.settled) return;
+ sessionUploader.recordTrace(sessionId, "session.quit_settled", {});
+ await settle("quit");
+ };
session.done = (async () => {
const checkpoint = await sessionUploader.sessionCheckpoint(sessionId);
if (checkpoint.lastMessageId) observer.lastMessageIds.set(sessionId, checkpoint.lastMessageId);
diff --git a/apps/server/src/session-upload-state.ts b/apps/server/src/session-upload-state.ts
index 5047ea0b..d79faaf2 100644
--- a/apps/server/src/session-upload-state.ts
+++ b/apps/server/src/session-upload-state.ts
@@ -18,6 +18,8 @@ export const UPLOAD_TEMP_DIRECTORY = "omnirush-upload-tmp";
export const UPLOAD_SESSION_LEDGER_FILE = "omnirush-upload-sessions.json";
export const UPLOAD_SPOOL_DIRECTORY = "omnirush-upload-spool";
export const UPLOAD_BASE_DIRECTORY = "omnirush-upload-bases";
+/** Each live chat's trace events not yet handed to an upload, so a crash loses none (session-uploader.ts). */
+export const UPLOAD_JOURNAL_DIRECTORY = "omnirush-upload-journal";
/** The same files under the names used up to 2.2.1. */
export const LEGACY_UPLOAD_STATE_NAMES = {
diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.ts
index 00a9d68c..1735df9f 100644
--- a/apps/server/src/session-uploader.test.ts
+++ b/apps/server/src/session-uploader.test.ts
@@ -19,6 +19,7 @@ import {
MAX_SPOOL_AGE_MS,
uploadIdempotencyKey,
serverSessionPosition,
+ parseJournal,
SessionUploader,
isRetryableUploadStatus,
mapBounded,
@@ -348,6 +349,79 @@ describe("session uploader privacy", () => {
expect(Object.values(ledger.sessions).filter((record) => record.childSessionIds)).toHaveLength(512);
});
+ test("a crash mid-turn loses no trace event: the next start uploads the journal in its own segment, marked recovered, and settles the open turn", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-journal-"));
+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-journal-state-"));
+ roots.push(root, stateDir);
+ await writeFile(join(root, "app.txt"), "hello\n");
+ const envelopes: Array<Record<string, unknown>> = [];
+ const upload = async (_sessionId: string, compressed: Uint8Array) => {
+ envelopes.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Record<string, unknown>);
+ return Response.json({ ok: true }, { status: 201 });
+ };
+ const crashed = new SessionUploader({ stateDir, upload, fallbackScanMs: 60_000 });
+ const sessionId = "session-journal-1234";
+ crashed.startSession(sessionId, "workspace-journal", root);
+ await crashed.idle(sessionId);
+ crashed.captureSnapshot(sessionId, "prompt");
+ crashed.recordTrace(sessionId, "engine.request", { body: "do the task" });
+ crashed.recordTrace(sessionId, "tool.call", { name: "edit" });
+ await new Promise((resolve) => setTimeout(resolve, 600));
+ // The process dies here: no stop(), nothing flushed.
+ expect(envelopes.map((item) => item.snapshot_type)).toEqual(["start"]);
+ const journals = await readdir(join(stateDir, "omnirush-upload-journal"));
+ expect(journals).toHaveLength(1);
+
+ const recoveredTurns: Array<{ sessionId: string; workspaceId: string; root: string }> = [];
+ const next = new SessionUploader({ stateDir, upload, fallbackScanMs: 60_000, onRecoveredTurn: (turn) => recoveredTurns.push(turn) });
+ for (let waited = 0; waited < 5_000 && envelopes.length < 2; waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));
+ const recovered = envelopes[1]!;
+ expect(recovered).toMatchObject({ snapshot_type: "trace", session_segment: 1, sequence: 2 });
+ const types = (recovered.trace as Array<{ type: string }>).map((event) => event.type);
+ expect(types[0]).toBe("collector.recovered");
+ expect(types).toContain("engine.request");
+ expect(types).toContain("tool.call");
+ expect(recoveredTurns).toEqual([{ sessionId, workspaceId: "workspace-journal", root }]);
+ expect(await readdir(join(stateDir, "omnirush-upload-journal"))).toEqual([]);
+ // The chat's next segment starts after the recovered upload.
+ next.startSession(sessionId, "workspace-journal", root);
+ await next.stop();
+ expect(envelopes.slice(2).map((item) => [item.snapshot_type, item.session_segment, item.sequence])).toEqual([["start", 2, 3], ["trace", 2, 4], ["end", 2, 5]]);
+ }, 20_000);
+
+ test("the journal holds only what is not yet handed to an upload, and is gone once the chat ends", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-journal2-"));
+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-journal2-state-"));
+ roots.push(root, stateDir);
+ const sessionUploader = new SessionUploader({
+ stateDir,
+ upload: async () => Response.json({ ok: true }, { status: 201 }),
+ fallbackScanMs: 60_000,
+ });
+ const sessionId = "session-journal2-1234";
+ sessionUploader.startSession(sessionId, "workspace-journal2", root);
+ sessionUploader.captureSnapshot(sessionId, "prompt");
+ sessionUploader.recordTrace(sessionId, "tool.call", { n: 1 });
+ sessionUploader.captureSnapshot(sessionId, "turn_completed");
+ sessionUploader.flushTrace(sessionId, { messages: [] });
+ await sessionUploader.idle(sessionId);
+ await new Promise((resolve) => setTimeout(resolve, 400));
+ const dir = join(stateDir, "omnirush-upload-journal");
+ const [name] = await readdir(dir);
+ const journal = parseJournal(await readFile(join(dir, name!), "utf8"));
+ expect(journal?.meta).toMatchObject({ session_id: sessionId, turn_open: false, segment: 1 });
+ expect(journal?.events.map((event) => event.type) ?? []).not.toContain("tool.call");
+ await sessionUploader.stop();
+ expect(await readdir(dir)).toEqual([]);
+ }, 20_000);
+
+ test("reads a journal cut off mid-line by a crash", () => {
+ const meta = { v: 1 as const, session_id: "session-cut-1234", workspace_id: "ws", root: "/w", segment: 3, turn_open: true, at: "2026-10-10T00:00:00.000Z" };
+ const text = `${JSON.stringify({ m: meta })}\n${JSON.stringify({ e: { at: "t", type: "tool.call", data: { a: 1 } } })}\n{"e":{"at":"t","ty`;
+ expect(parseJournal(text)).toEqual({ meta, events: [{ at: "t", type: "tool.call", data: { a: 1 } }] });
+ expect(parseJournal(`${JSON.stringify({ e: { at: "t", type: "x" } })}\n`)).toBeNull();
+ });
+
test("persists session segments and message checkpoints across a resume", async () => {
const root = await mkdtemp(join(tmpdir(), "omnirush-upload-resume-"));
const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-state-"));
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index 39562746..e7ab98bb 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -1,7 +1,7 @@
import { createHash, randomBytes } from "node:crypto";
import { execFile, spawn } from "node:child_process";
import { createReadStream, createWriteStream, watch, type FSWatcher, type WriteStream } from "node:fs";
-import { lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";
+import { appendFile, lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";
import { homedir, release as osRelease, tmpdir } from "node:os";
import nodePath, { basename, dirname, extname, join, relative, resolve, sep, type PlatformPath } from "node:path";
import { promisify } from "node:util";
@@ -13,6 +13,7 @@ import { SESSION_UPLOAD_BUDGET, SESSION_UPLOAD_ENDPOINT_PATH, sessionUploadTimeo
import { externalFetch } from "./server-fetch.js";
import {
UPLOAD_BASE_DIRECTORY,
+ UPLOAD_JOURNAL_DIRECTORY,
UPLOAD_SESSION_LEDGER_FILE,
UPLOAD_SPOOL_DIRECTORY,
UPLOAD_TEMP_DIRECTORY,
@@ -145,6 +146,10 @@ const MAX_SPOOL_TRACE_BYTES = 128 * 1024 * 1024;
const MAX_SPOOL_TRACE_ENTRIES = 2_000;
/** A spooled upload that keeps failing with a retryable answer is kept this long, then dropped. */
export const MAX_SPOOL_AGE_MS = 7 * 24 * 60 * 60_000;
+/** Trace events reach the chat's journal on disk within this long. */
+const JOURNAL_FLUSH_MS = 250;
+/** A journal stops taking events past this size (its chat's meta lines still go in). */
+const MAX_JOURNAL_BYTES = 256 * 1024 * 1024;
/** How long spooling or a drain waits to learn which account is signed in. */
const ACCOUNT_LOOKUP_TIMEOUT_MS = 5_000;
const UPLOAD_ATTEMPTS = 3;
@@ -560,8 +565,77 @@ type SessionState = {
extras: Map<string, ExtraManifestEntry>;
/** The numbering already moved past the server's once (an Idempotency-Key it held with other bytes). */
repositioned?: boolean;
+ /** Trace events not yet handed to an upload, as written to the chat's journal on disk. */
+ journal: JournalState;
};
+type JournalState = {
+ pending: string[];
+ timer: ReturnType<typeof setTimeout> | null;
+ tail: Promise<void>;
+ bytes: number;
+ full: boolean;
+ /** A recovery's stand-in state: never journaled itself. */
+ off: boolean;
+ /**
+ * Lines are written once the chat's own recovery is done (prepareSession):
+ * a journal a previous process left under the same name is read first.
+ */
+ ready: boolean;
+};
+
+/** What a chat's journal says about it (its last meta line). */
+type JournalMeta = {
+ v: 1;
+ session_id: string;
+ workspace_id: string;
+ root: string;
+ segment: number;
+ /** A prompt went out and its turn has not ended. */
+ turn_open: boolean;
+ at: string;
+};
+
+function parseJournalMeta(value: unknown): JournalMeta | null {
+ if (!isRecord(value) || value.v !== 1 || typeof value.session_id !== "string" || typeof value.workspace_id !== "string"
+ || typeof value.root !== "string" || !Number.isSafeInteger(value.segment) || typeof value.turn_open !== "boolean"
+ || typeof value.at !== "string") return null;
+ return {
+ v: 1,
+ session_id: value.session_id,
+ workspace_id: value.workspace_id,
+ root: value.root,
+ segment: Number(value.segment),
+ turn_open: value.turn_open,
+ at: value.at,
+ };
+}
+
+/** A journal file's meta (the last meta line) and its events; null when it holds no meta. */
+export function parseJournal(text: string): { meta: JournalMeta; events: TraceEvent[] } | null {
+ let meta: JournalMeta | null = null;
+ const events: TraceEvent[] = [];
+ for (const line of text.split("\n")) {
+ if (!line) continue;
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(line);
+ } catch {
+ // A line cut off by the crash.
+ continue;
+ }
+ if (!isRecord(parsed)) continue;
+ if ("m" in parsed) meta = parseJournalMeta(parsed.m) ?? meta;
+ else if (isRecord(parsed.e) && typeof parsed.e.at === "string" && typeof parsed.e.type === "string") {
+ events.push({ at: parsed.e.at, type: parsed.e.type, ...("data" in parsed.e ? { data: parsed.e.data } : {}) });
+ }
+ }
+ return meta ? { meta, events } : null;
+}
+
+/** Where the turns a previous process left open are told: their chat is settled from the engine (capture-host.ts). */
+export type RecoveredTurn = { sessionId: string; workspaceId: string; root: string };
+
type SessionUploaderOptions = {
gatewayUrl?: string;
accessToken?: string;
@@ -576,6 +650,12 @@ type SessionUploaderOptions = {
* this hook the spool is the one account's.
*/
accountId?: () => Promise<string | null>;
+ /**
+ * A chat a previous process left with its turn open (a crash, a killed
+ * worker): its journaled events are uploaded, then the host settles the
+ * turn from the engine's messages.
+ */
+ onRecoveredTurn?: (turn: RecoveredTurn) => void;
/** GET /omnirush/me/sessions/{id}/integrity for this account (where the server's copy of a chat stands). */
sessionIntegrity?: (sessionId: string, options?: { summary?: boolean; turns?: number }) => Promise<Response>;
/** Negotiates the canonical structured trace envelope. Failures fall back to schema v2. */
@@ -4276,6 +4356,13 @@ export class SessionUploader {
private readonly spoolMaxAgeMs: number;
private readonly accountIdHook?: SessionUploaderOptions["accountId"];
private readonly sessionIntegrity?: SessionUploaderOptions["sessionIntegrity"];
+ private readonly onRecoveredTurn?: SessionUploaderOptions["onRecoveredTurn"];
+ /** Each live chat's journal of trace events (null without a state dir). */
+ private readonly journalDir: string | null;
+ /** The journals a previous process left, read at start; a chat's own start waits for its recovery. */
+ private readonly journalsRead: Promise<void>;
+ private readonly recovering = new Map<string, Promise<void>>();
+ private journalWarned = false;
/** The signed-in account, once known (it never changes within a process: an account change restarts the server). */
private accountKnown: string | null = null;
private accountLookup: Promise<string | null> | null = null;
@@ -4344,6 +4431,7 @@ export class SessionUploader {
this.stateDir = stateDir;
this.ledgerPath = stateDir ? join(stateDir, UPLOAD_SESSION_LEDGER_FILE) : null;
this.spoolDir = stateDir ? join(stateDir, UPLOAD_SPOOL_DIRECTORY) : null;
+ this.journalDir = stateDir ? join(stateDir, UPLOAD_JOURNAL_DIRECTORY) : null;
// State left under the pre-2.2.2 names is moved before anything reads it.
this.stateMigrated = stateDir
? migrateLegacyUploadState(stateDir, (message, details) => this.log("warn", message, details))
@@ -4364,6 +4452,7 @@ export class SessionUploader {
this.spoolMaxAgeMs = options.spoolMaxAgeMs ?? MAX_SPOOL_AGE_MS;
this.accountIdHook = options.accountId;
this.sessionIntegrity = options.sessionIntegrity;
+ this.onRecoveredTurn = options.onRecoveredTurn;
this.snapshotMaxBytes = options.snapshotMaxBytes ?? MAX_SNAPSHOT_BYTES;
this.minChangeIntervalMs = options.minChangeIntervalMs ?? MIN_UPLOAD_CHANGE_INTERVAL_MS;
this.maxWatchedFiles = options.maxWatchedFiles ?? MAX_UPLOAD_WATCHED_FILES;
@@ -4390,6 +4479,8 @@ export class SessionUploader {
// Without an account they wait (up to the spool's age limit) for the
// account to come back: only a user sign-out clears them (clearSpool()).
if (this.spoolDir && this.enabled) this.scheduleRetry(this.retryBaseMs);
+ // So do the journals a previous process left: their turns are settled now.
+ this.journalsRead = this.enabled ? this.recoverJournals() : Promise.resolve();
}
/** Capture context wired to this uploader: its trace, its scrub, the project archive and the archive's exclusions. */
@@ -4491,6 +4582,10 @@ export class SessionUploader {
private async prepareSession(state: SessionState): Promise<void> {
await this.ledgerReady;
+ // What a previous process left of this chat goes out first, numbered before this segment.
+ await this.journalsRead;
+ await this.recovering.get(state.id);
+ state.journal.ready = true;
const recorded = this.ledger.sessions[state.id];
// No local record: a new chat, or one this device's ledger lost. The
// server says where its copy stands, so the numbering goes on after it
@@ -4520,6 +4615,7 @@ export class SessionUploader {
}
this.ledger.sessions[state.id] = this.ledgerRecord(state);
await this.saveLedger();
+ this.journalMeta(state);
}
/**
@@ -4701,7 +4797,32 @@ export class SessionUploader {
startSession(sessionId: string, workspaceId: string, root: string): void {
if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;
void this.toolchains?.get(root).catch(() => null); // toolchain.ts: collected in the background
- const state: SessionState = {
+ const state = this.newSessionState(sessionId, workspaceId, root, this.acquireCache(root));
+ state.ready = this.prepareSession(state);
+ this.sessions.set(sessionId, state);
+ this.journalMeta(state);
+ // Capture context collects once the start snapshot is out.
+ let startSnapshotDone: () => void = () => undefined;
+ const startSnapshot = new Promise<void>((resolveStart) => (startSnapshotDone = resolveStart));
+ this.context?.sessionStarted(sessionId, root, startSnapshot);
+ this.enqueue(state, async () => {
+ await state.ready;
+ try {
+ // The watchers go up as soon as the start snapshot's listing is in,
+ // before any file is read: an edit made during the scan is either
+ // seen by the scan or reported by a watcher.
+ await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");
+ } finally {
+ state.started = true;
+ startSnapshotDone();
+ // No listing reached the plan (a failed scan): poll.
+ if (state.watchMode === "starting") this.installWatchers(state, null);
+ }
+ });
+ }
+
+ private newSessionState(sessionId: string, workspaceId: string, root: string, cache: Map<string, HashCacheEntry>): SessionState {
+ return {
id: sessionId,
root,
workspaceId,
@@ -4738,7 +4859,7 @@ export class SessionUploader {
turnSkipped: new Map(),
turnWrittenPaths: new Map(),
relevance: new Map(),
- cache: this.acquireCache(root),
+ cache,
listing: { denied: 0, truncated: false },
dirty: new Set(),
// Nothing is known about the tree yet: the first change capture rescans it.
@@ -4753,27 +4874,8 @@ export class SessionUploader {
failureCount: 0,
ready: Promise.resolve(),
tail: Promise.resolve(),
+ journal: { pending: [], timer: null, tail: Promise.resolve(), bytes: 0, full: false, off: false, ready: false },
};
- state.ready = this.prepareSession(state);
- this.sessions.set(sessionId, state);
- // Capture context collects once the start snapshot is out.
- let startSnapshotDone: () => void = () => undefined;
- const startSnapshot = new Promise<void>((resolveStart) => (startSnapshotDone = resolveStart));
- this.context?.sessionStarted(sessionId, root, startSnapshot);
- this.enqueue(state, async () => {
- await state.ready;
- try {
- // The watchers go up as soon as the start snapshot's listing is in,
- // before any file is read: an edit made during the scan is either
- // seen by the scan or reported by a watcher.
- await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");
- } finally {
- state.started = true;
- startSnapshotDone();
- // No listing reached the plan (a failed scan): poll.
- if (state.watchMode === "starting") this.installWatchers(state, null);
- }
- });
}
// --- watcher and dirty tracking --------------------------------------------
@@ -5210,7 +5312,9 @@ export class SessionUploader {
/** Appends one trace event, keeping only the newest MAX_UPLOAD_TRACE_EVENTS. */
private appendTrace(state: SessionState, type: string, data?: unknown): void {
- state.trace.push({ at: new Date().toISOString(), type, ...(data === undefined ? {} : { data }) });
+ const event: TraceEvent = { at: new Date().toISOString(), type, ...(data === undefined ? {} : { data }) };
+ state.trace.push(event);
+ this.journalEvent(state, event);
if (state.trace.length > MAX_UPLOAD_TRACE_EVENTS) state.trace.splice(0, state.trace.length - MAX_UPLOAD_TRACE_EVENTS);
}
@@ -5438,6 +5542,8 @@ export class SessionUploader {
if (trigger === "prompt") state.turnStartedAt = Date.now() + 1;
const startedAt = state.turnStartedAt;
state.turnInProgress = trigger === "prompt";
+ // A crash from here on leaves the turn open in the journal (or no longer).
+ this.journalMeta(state, true);
// The milestone carries whatever edits were held back for another session's turn.
state.changesHeld = false;
if (state.changeTimer) {
@@ -5587,6 +5693,7 @@ export class SessionUploader {
await this.uploadTrace(state, pendingTrace);
await this.uploadWorkspace(state, "end", "session_end");
} finally {
+ await this.journalRemove(state);
this.releaseCache(state);
this.sessions.delete(sessionId);
this.onSessionClosed?.(sessionId);
@@ -5610,7 +5717,12 @@ export class SessionUploader {
this.enqueue(state, async () => {
await state.ready;
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
- await this.uploadTrace(state, pendingTrace);
+ try {
+ await this.uploadTrace(state, pendingTrace);
+ } finally {
+ // Uploaded, spooled or refused: the journal keeps only what came after.
+ await this.journalRewrite(state);
+ }
});
}
@@ -6536,6 +6648,217 @@ export class SessionUploader {
}
}
+ // --- trace journal -------------------------------------------------------
+ // Each live chat's trace events not yet handed to an upload are appended to
+ // a journal on disk (JSON lines: {"m": meta} and {"e": event}) within
+ // JOURNAL_FLUSH_MS, and its turn's opening and end at once. A crash or a
+ // killed worker loses none of them: the next start uploads what a journal
+ // holds, marked recovered, and has the host settle a turn left open.
+
+ private journalPath(sessionId: string): string {
+ return join(this.journalDir!, `${createHash("sha256").update(sessionId).digest("hex").slice(0, 40)}.jsonl`);
+ }
+
+ private journalMetaLine(state: SessionState): string {
+ const meta: JournalMeta = {
+ v: 1,
+ session_id: state.id,
+ workspace_id: state.workspaceId,
+ root: state.root,
+ segment: state.segment,
+ turn_open: state.turnInProgress,
+ at: new Date().toISOString(),
+ };
+ return JSON.stringify({ m: meta });
+ }
+
+ private journalMeta(state: SessionState, now = false): void {
+ if (!this.journalDir || state.finished || state.journal.off) return;
+ state.journal.pending.push(this.journalMetaLine(state));
+ this.scheduleJournal(state, now);
+ }
+
+ private journalEvent(state: SessionState, event: TraceEvent): void {
+ if (!this.journalDir || state.journal.full || state.journal.off) return;
+ let line: string;
+ try {
+ line = JSON.stringify({ e: event });
+ } catch {
+ return;
+ }
+ if (state.journal.bytes + line.length > MAX_JOURNAL_BYTES) {
+ state.journal.full = true;
+ return;
+ }
+ state.journal.bytes += line.length + 1;
+ state.journal.pending.push(line);
+ this.scheduleJournal(state, false);
+ }
+
+ private scheduleJournal(state: SessionState, now: boolean): void {
+ if (now) {
+ if (state.journal.timer) clearTimeout(state.journal.timer);
+ state.journal.timer = null;
+ void this.flushJournal(state);
+ return;
+ }
+ if (state.journal.timer) return;
+ state.journal.timer = setTimeout(() => {
+ state.journal.timer = null;
+ void this.flushJournal(state);
+ }, JOURNAL_FLUSH_MS);
+ state.journal.timer.unref?.();
+ }
+
+ /** Appends the lines waiting for the journal; resolves once they are on disk. */
+ private flushJournal(state: SessionState): Promise<void> {
+ if (!state.journal.ready) return state.journal.tail;
+ const lines = state.journal.pending.splice(0);
+ if (lines.length === 0 || !this.journalDir) return state.journal.tail;
+ const path = this.journalPath(state.id);
+ state.journal.tail = state.journal.tail
+ .then(async () => {
+ await mkdir(this.journalDir!, { recursive: true, mode: 0o700 });
+ await appendFile(path, `${lines.join("\n")}\n`, { mode: 0o600 });
+ })
+ .catch((error: unknown) => this.journalFailed(state, error));
+ return state.journal.tail;
+ }
+
+ /** The events went to an upload: the journal keeps its meta and what was recorded since. */
+ private journalRewrite(state: SessionState): Promise<void> {
+ if (!this.journalDir || state.finished || state.journal.off || !state.journal.ready) return state.journal.tail;
+ if (state.journal.timer) clearTimeout(state.journal.timer);
+ state.journal.timer = null;
+ state.journal.pending = [];
+ const lines = [this.journalMetaLine(state)];
+ state.journal.bytes = 0;
+ state.journal.full = false;
+ for (const event of state.trace) {
+ const line = JSON.stringify({ e: event });
+ if (state.journal.bytes + line.length > MAX_JOURNAL_BYTES) {
+ state.journal.full = true;
+ break;
+ }
+ state.journal.bytes += line.length + 1;
+ lines.push(line);
+ }
+ const path = this.journalPath(state.id);
+ state.journal.tail = state.journal.tail
+ .then(async () => {
+ await mkdir(this.journalDir!, { recursive: true, mode: 0o700 });
+ await writeFileAtomic(path, `${lines.join("\n")}\n`, { mode: 0o600 });
+ })
+ .catch((error: unknown) => this.journalFailed(state, error));
+ return state.journal.tail;
+ }
+
+ /** The chat's last upload settled: nothing is left to recover. */
+ private journalRemove(state: SessionState): Promise<void> {
+ if (state.journal.timer) clearTimeout(state.journal.timer);
+ state.journal.timer = null;
+ state.journal.pending = [];
+ if (!this.journalDir || state.journal.off) return state.journal.tail;
+ const path = this.journalPath(state.id);
+ state.journal.tail = state.journal.tail.then(() => rm(path, { force: true })).catch((error: unknown) => this.journalFailed(state, error));
+ return state.journal.tail;
+ }
+
+ private journalFailed(state: SessionState, error: unknown): void {
+ if (this.journalWarned) return;
+ this.journalWarned = true;
+ this.log("warn", "OmniRush session journal write failed", { sessionId: state.id, error: error instanceof Error ? error.message : "unknown" });
+ }
+
+ /**
+ * App (or capture worker) start: every journal a previous process left is
+ * uploaded as a trace of the segment it was recorded in, after a
+ * "collector.recovered" event; a turn it left open is handed to the host
+ * (onRecoveredTurn) to be settled from the engine. A chat started meanwhile
+ * waits for its own (prepareSession). Journals past the spool's age limit
+ * are deleted unread.
+ */
+ private async recoverJournals(): Promise<void> {
+ if (!this.journalDir) return;
+ await this.stateMigrated;
+ await this.ledgerReady;
+ let names: string[];
+ try {
+ names = (await readdir(this.journalDir)).filter((name) => name.endsWith(".jsonl"));
+ } catch {
+ return;
+ }
+ const found: Array<{ path: string; meta: JournalMeta; events: TraceEvent[] }> = [];
+ for (const name of names) {
+ const path = join(this.journalDir, name);
+ try {
+ const file = await stat(path);
+ const parsed = Date.now() - file.mtimeMs > this.spoolMaxAgeMs ? null : parseJournal(await readFile(path, "utf8"));
+ if (!parsed || (parsed.events.length === 0 && !parsed.meta.turn_open)) {
+ await rm(path, { force: true });
+ continue;
+ }
+ found.push({ path, ...parsed });
+ } catch {
+ await rm(path, { force: true }).catch(() => undefined);
+ }
+ }
+ // Registered before this resolves, so a chat's start finds its recovery.
+ const recovered: Promise<void>[] = [];
+ let chain = Promise.resolve();
+ for (const journal of found) {
+ chain = chain.then(() => this.recoverJournal(journal)).catch((error: unknown) => {
+ this.log("warn", "OmniRush session journal recovery failed", { sessionId: journal.meta.session_id, error: error instanceof Error ? error.message : "unknown" });
+ });
+ const done = chain;
+ this.recovering.set(journal.meta.session_id, done);
+ recovered.push(done.finally(() => {
+ if (this.recovering.get(journal.meta.session_id) === done) this.recovering.delete(journal.meta.session_id);
+ }));
+ }
+ void Promise.allSettled(recovered);
+ }
+
+ private async recoverJournal(journal: { path: string; meta: JournalMeta; events: TraceEvent[] }): Promise<void> {
+ const { meta, events } = journal;
+ const record = this.ledger.sessions[meta.session_id];
+ if (events.length > 0) {
+ // The segment the events were recorded in, numbered after what it already sent.
+ const state = this.newSessionState(meta.session_id, meta.workspace_id, meta.root, new FileHashCache());
+ state.journal.off = true;
+ state.segment = record?.segment ?? meta.segment;
+ state.sequence = record?.nextSequence ?? 0;
+ state.sentBytes = record?.sentBytes ?? 0;
+ state.lastMessageId = record?.lastMessageId;
+ state.failureCount = record?.failureCount ?? 0;
+ state.lastFailureAt = record?.lastFailureAt;
+ state.lastSuccessAt = record?.lastSuccessAt;
+ state.model = record?.model ?? null;
+ state.childSessionIds = [...(record?.childSessionIds ?? [])];
+ state.childCheckpoints = new Map(Object.entries(record?.childCheckpoints ?? {}));
+ state.resumed = state.segment > 1;
+ for (const event of events) {
+ for (const evidence of knownToolPathEvidence(event.type, event.data)) {
+ const path = workspaceRelativePath(state.root, evidence.path.replaceAll("\\", "/"));
+ if (path && !isUploadPathDenied(path) && state.touchedPaths.size < MAX_TOUCHED_PATHS) state.touchedPaths.add(path);
+ }
+ }
+ const marker: TraceEvent = {
+ at: new Date().toISOString(),
+ type: "collector.recovered",
+ data: { reason: "restart", turn_open: meta.turn_open, events: events.length, journal_at: meta.at, session_segment: state.segment },
+ };
+ await this.uploadTrace(state, [marker, ...events]);
+ this.log("info", "OmniRush session journal recovered", { sessionId: meta.session_id, events: events.length, turnOpen: meta.turn_open });
+ }
+ await rm(journal.path, { force: true });
+ // The open turn's messages are still in the engine: the host settles it once this chat's recovery is done.
+ if (meta.turn_open && this.onRecoveredTurn) {
+ const turn: RecoveredTurn = { sessionId: meta.session_id, workspaceId: meta.workspace_id, root: meta.root };
+ queueMicrotask(() => this.onRecoveredTurn?.(turn));
+ }
+ }
+
// --- durable retry spool -------------------------------------------------
private spoolLocked<T>(operation: () => Promise<T>): Promise<T> {
@@ -6891,8 +7214,9 @@ export class SessionUploader {
this.traceSchemaProbe = null;
this.retryFailures = 0;
return this.spoolLocked(async () => {
- // The turn diffs' bases are workspace content on disk too.
+ // The turn diffs' bases are workspace content on disk too, and so are the journals.
await this.bases.clear();
+ if (this.journalDir) await rm(this.journalDir, { recursive: true, force: true });
if (!this.spoolDir) return;
await rm(this.spoolDir, { recursive: true, force: true });
if (this.stateDir) await removeLegacyUploadContent(this.stateDir);