A desktop coding agent with free access to frontier models.
TypeScript
fix: recover unfinished turns
ogsamrata11e98d28c37
diff --git a/apps/server/src/capture-client.test.ts b/apps/server/src/capture-client.test.tsindex 39deb3a4..5d52a94e 100644--- a/apps/server/src/capture-client.test.ts+++ b/apps/server/src/capture-client.test.ts@@ -576,3 +576,99 @@ describe("capture worker restarts", () => {expect(sink.envelopes().map((item) => [item.session_id, item.snapshot_type])).toEqual([["session-restart-0001", "start"]]);}, 30_000);});++describe("turns left open: worker restarts and quits", () => {+ let crashSwitch: string | undefined;+ let contextSwitch: string | undefined;+ beforeAll(() => {+ crashSwitch = process.env.OMNIRUSH_CAPTURE_TEST_CRASH;+ contextSwitch = process.env.OMNIRUSH_CAPTURE_CONTEXT;+ process.env.OMNIRUSH_CAPTURE_TEST_CRASH = "1";+ process.env.OMNIRUSH_CAPTURE_CONTEXT = "0";+ });+ afterAll(() => {+ if (crashSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_TEST_CRASH;+ else process.env.OMNIRUSH_CAPTURE_TEST_CRASH = crashSwitch;+ if (contextSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_CONTEXT;+ else process.env.OMNIRUSH_CAPTURE_CONTEXT = contextSwitch;+ });++ const SESSION = "ses_recovered_0001";++ /** A fake engine with one chat: its status and its messages. */+ function fakeEngine() {+ const control = { busy: true, messages: [] as Array<{ info: Record<string, unknown>; parts: unknown[] }> };+ const message = (id: string, role: "user" | "assistant") => ({+ info: { id, sessionID: SESSION, role, time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 } },+ parts: [{ id: `${id}_p`, messageID: id, sessionID: SESSION, type: "text", text: `${role} ${id}` }],+ });+ const server = Bun.serve({+ hostname: "127.0.0.1",+ port: 0,+ fetch(request) {+ const url = new URL(request.url);+ if (url.pathname === "/session/status") return Response.json(control.busy ? { [SESSION]: { type: "busy" } } : {});+ if (url.pathname === `/session/${SESSION}/message`) return Response.json(control.messages);+ if (url.pathname === `/session/${SESSION}/children`) return Response.json([]);+ if (url.pathname === `/session/${SESSION}`) return Response.json({ id: SESSION });+ return Response.json({ code: "not_found" }, { status: 404 });+ },+ });+ cleanups.push(() => server.stop(true));+ const target = { baseUrl: `http://127.0.0.1:${server.port}`, headers: [] as Array<[string, string]>, search: "", engine: "v1" as const };+ return { control, message, target };+ }++ test("a worker killed mid-turn: the next worker uploads the turn's journaled events, marked recovered, and settles the turn from the engine", async () => {+ const root = await tempDir("recover-root");+ await writeFile(join(root, "app.txt"), "hello\n");+ const sink = uploadSink();+ const engine = fakeEngine();+ engine.control.messages = [engine.message("msg_1", "user")];+ const capture = service({+ stateDir: await tempDir("recover-state"),+ sessionUploader: { upload: sink.upload },+ restartBackoffMs: { baseMs: 50, maxMs: 50 },+ });+ capture.startSession(SESSION, "workspace-recover", root);+ capture.captureSnapshot(SESSION, "prompt");+ capture.recordTrace(SESSION, "engine.request", { body: "do the task" });+ capture.recordTrace(SESSION, "tool.call", { name: "edit", marker: "before-the-crash" });+ capture.observeSession(SESSION, engine.target);+ // The journal is on disk within its flush interval; then the worker dies.+ await new Promise((resolvePromise) => setTimeout(resolvePromise, 800));+ capture.recordTrace(SESSION, "omnirush.test.worker_crash");+ await until(() => capture.mode() === "restarting", 10_000, "the worker's exit");+ // The turn finishes while the new worker comes up.+ engine.control.messages.push(engine.message("msg_2", "assistant"));+ engine.control.busy = false;+ await until(() => sink.envelopes().some((item) => JSON.stringify(item).includes("\"recovered\":true")), 30_000, "the settled recovered turn");+ const envelopes = sink.envelopes() as unknown as Array<{ snapshot_type: string; trace?: Array<{ type: string; data?: unknown }> }>;+ const recovered = envelopes.find((item) => item.trace?.[0]?.type === "collector.recovered");+ expect(recovered).toBeDefined();+ expect(JSON.stringify(recovered)).toContain("before-the-crash");+ const settled = envelopes.find((item) => item.trace?.some((event) => event.type === "turn.completed"));+ const completed = settled?.trace?.find((event) => event.type === "turn.completed");+ expect(completed?.data).toMatchObject({ recovered: true });+ expect(JSON.stringify(completed)).toContain("msg_2");+ }, 60_000);++ test("an app quit settles a turn still running before the chat ends", async () => {+ const root = await tempDir("quit-root");+ await writeFile(join(root, "app.txt"), "hello\n");+ const sink = uploadSink();+ const engine = fakeEngine();+ engine.control.messages = [engine.message("msg_1", "user"), engine.message("msg_2", "assistant")];+ const capture = service({ stateDir: await tempDir("quit-state"), sessionUploader: { upload: sink.upload } });+ capture.startSession(SESSION, "workspace-quit", root);+ capture.captureSnapshot(SESSION, "prompt");+ capture.observeSession(SESSION, engine.target);+ await new Promise((resolvePromise) => setTimeout(resolvePromise, 500));+ await capture.stop();+ const envelopes = sink.envelopes() as unknown as Array<{ snapshot_type: string; trace?: Array<{ type: string; data?: unknown }> }>;+ const completed = envelopes.flatMap((item) => item.trace ?? []).find((event) => event.type === "turn.completed");+ expect(completed?.data).toMatchObject({ reason: "app_quit" });+ expect(JSON.stringify(completed)).toContain("msg_2");+ expect(envelopes.at(-1)?.snapshot_type).toBe("end");+ }, 60_000);+});diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.tsindex 6ac7f771..ed63aca0 100644--- a/apps/server/src/capture-client.ts+++ b/apps/server/src/capture-client.ts@@ -47,6 +47,8 @@ const RESTART_STABLE_MS = 5 * 60_000;/** Calls waiting for a worker to start; past this the oldest one nobody waits on is dropped. */const MAX_QUEUED_CALLS = 20_000;const SESSION_ID_PATTERN = /^[A-Za-z0-9._:-]{8,128}$/;+/** Chats whose engine is remembered for a restarted worker. */+const MAX_OBSERVED_TARGETS = 256;export type CaptureServiceOptions = Omit<CaptureHostOptions, "onSessionClosed"> & {/** Run capture on a worker thread; default on unless OMNIRUSH_CAPTURE_WORKER is 0, false, no or off. */@@ -149,6 +151,8 @@ class CaptureClient implements CaptureService {/** Only the first start falls back to capturing in-process; later failures start new workers. */private canRunLocally = true;private overflowing = false;+ /** The engine each recently observed chat's requests went to: a restarted worker settles their open turns there. */+ private readonly observedTargets = new Map<string, EngineTarget>();private stopping: Promise<void> | null = null;constructor(private readonly options: CaptureServiceOptions) {@@ -234,6 +238,9 @@ class CaptureClient implements CaptureService {}observeSession(sessionId: string, target: EngineTarget): void {+ this.observedTargets.delete(sessionId);+ this.observedTargets.set(sessionId, target);+ if (this.observedTargets.size > MAX_OBSERVED_TARGETS) this.observedTargets.delete(this.observedTargets.keys().next().value as string);this.send({ kind: "call", id: null, method: "observeSession", args: [sessionId, target] });}@@ -382,6 +389,8 @@ class CaptureClient implements CaptureService {engineVersion: this.options.engineVersion,accountId: Boolean(this.options.accountId),sessionIntegrity: Boolean(this.options.sessionIntegrity),+ // Always: the chats this thread saw observed are known here even without a resolver.+ engineTarget: true,sessionUploader: {upload: Boolean(sessionUploader.upload),uploadFile: Boolean(sessionUploader.uploadFile),@@ -582,6 +591,15 @@ class CaptureClient implements CaptureService {...(request.turns !== undefined ? { turns: request.turns } : {}),})) };}+ case "engineTarget":+ // A chat this server saw go to an engine (a restarted worker's), else the workspace's engine.+ return {+ kind: "result",+ id,+ ok: true,+ response: null,+ target: this.observedTargets.get(request.sessionId) ?? (this.options.engineTarget ? await this.options.engineTarget(request.sessionId, request.workspaceId) : null),+ };case "accountId":return { kind: "result", id, ok: true, response: null, account: this.options.accountId ? await this.options.accountId() : null };case "refreshAccessToken": {diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.tsindex e563adbb..299b495b 100644--- a/apps/server/src/capture-host.ts+++ b/apps/server/src/capture-host.ts@@ -25,7 +25,7 @@ import {import type { CaptureStopOptions } from "./capture-protocol.js";import { SessionArchiver, type FilesUsedStatus, type SessionArchiverOptions } from "./session-archive/index.js";import { ProjectArchiveLifecycle, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";-import { SessionUploader, type TraceCapabilities, type UploadMetrics, type UploadRequestOptions, type UploadWebVisit } from "./session-uploader.js";+import { SessionUploader, type RecoveredTurn, type TraceCapabilities, type UploadMetrics, type UploadRequestOptions, type UploadWebVisit } from "./session-uploader.js";export type { EngineReplacement, EngineTarget } from "./session-upload-observer.js";@@ -33,6 +33,10 @@ export type CaptureLog = ArchiveLifecycleLog;/** Prompt bodies larger than this are not parsed for the trace (nor for attachments). */const MAX_TRACED_REQUEST_BYTES = 4 * 1024 * 1024;+/** An app quit gives the turns still open this long in all to be settled from the engine. */+const QUIT_SETTLE_BUDGET_MS = 5_000;+/** A recovered turn asks for its engine again after these waits (the engine may still be starting). */+const RECOVERY_TARGET_DELAYS_MS = [0, 5_000, 15_000, 30_000, 60_000];/** A captured engine request, as the "engine.request" trace event and the prompt's attachments are built from it. */export type PromptRecord = {@@ -60,6 +64,12 @@ export type CaptureHostOptions = {log: CaptureLog;/** The signed-in account's id (null: not known now); spooled uploads are stamped with it. */accountId?: () => Promise<string | null>;+ /**+ * The engine holding a chat of this workspace, for settling a turn a+ * previous process left open; null when there is none (remote, unknown,+ * or the engine does not have the chat).+ */+ engineTarget?: (sessionId: string, workspaceId: string) => Promise<EngineTarget | null>;/** GET /omnirush/me/sessions/{id}/integrity with the device bearer. */sessionIntegrity?: (sessionId: string, options?: { summary?: boolean; turns?: number }) => Promise<Response>;sessionUploader: {@@ -107,8 +117,10 @@ export class CaptureHost {private readonly sessionRoots = new Map<string, string>();/** The state dir and the app's data/config/cache dirs: never reported or recorded as a touched file. */private readonly appDirs: string[];+ private readonly engineTarget?: CaptureHostOptions["engineTarget"];constructor(options: CaptureHostOptions) {+ this.engineTarget = options.engineTarget;this.sessionUploader = new SessionUploader({...options.sessionUploader,stateDir: options.stateDir,@@ -117,6 +129,8 @@ export class CaptureHost {log: options.log,...(options.accountId ? { accountId: options.accountId } : {}),...(options.sessionIntegrity ? { sessionIntegrity: options.sessionIntegrity } : {}),+ // A turn a crashed process or worker left open is settled from the engine's messages.+ onRecoveredTurn: (turn) => void this.recoverTurn(turn),// Capture context (context/); OMNIRUSH_CAPTURE_CONTEXT=0 turns it off.context: options.sessionUploader?.context ?? {},...(options.onSessionClosed ? { onSessionClosed: options.onSessionClosed } : {}),@@ -212,7 +226,7 @@ export class CaptureHost {}/** The engine accepted a captured request: follow the session until its turn settles. */- observeSession(sessionId: string, target: EngineTarget): void {+ observeSession(sessionId: string, target: EngineTarget, recovered = false): void {const root = this.sessionRoots.get(sessionId);void observeUploadedSession({sessionUploader: this.sessionUploader,@@ -221,9 +235,31 @@ export class CaptureHost {observers: this.observers,sessionId,target,+ ...(recovered ? { recovered } : {}),});}+ /**+ * A turn a previous process (or worker) left open: the chat resumes here+ * and its observer settles the turn from the engine's messages, or follows+ * it to its end when it is still running. Without an engine that has the+ * chat, its messages wait for its next prompt (the checkpoint did not move).+ */+ private async recoverTurn(turn: RecoveredTurn): Promise<void> {+ if (!this.engineTarget) return;+ const stopped = this.observers.controller.signal;+ for (const delay of RECOVERY_TARGET_DELAYS_MS) {+ if (delay > 0) await new Promise((resolvePromise) => setTimeout(resolvePromise, delay).unref?.());+ if (stopped.aborted || this.sessionUploader.hasSession(turn.sessionId)) return;+ const target = await this.engineTarget(turn.sessionId, turn.workspaceId).catch(() => null);+ if (stopped.aborted || this.sessionUploader.hasSession(turn.sessionId)) return;+ if (!target) continue;+ this.startSession(turn.sessionId, turn.workspaceId, turn.root);+ this.observeSession(turn.sessionId, target, true);+ return;+ }+ }+/** Files used, for the app's one-time notice: what omnirush.ai says. */filesUsedStatus(): Promise<FilesUsedStatus> {return this.archive.filesUsedStatus();@@ -250,12 +286,29 @@ export class CaptureHost {* final archives (unless `archiveFinals` is false: the account is gone).*/async stop(options: CaptureStopOptions = { archiveFinals: true }): Promise<void> {+ // Turns still open get their messages and their end first, within a bounded budget.+ await this.settleOpenTurns(QUIT_SETTLE_BUDGET_MS);const archiveStopped = this.archive.stop({ finals: options.archiveFinals });this.observers.controller.abort();await this.sessionUploader.stop().catch(() => undefined);await archiveStopped;}+ /** Settles every followed turn as the engine has it now, waiting at most `budgetMs`. */+ private async settleOpenTurns(budgetMs: number): Promise<void> {+ const open = [...this.observers.sessions.values()].map((session) => session.settleNow?.().catch(() => undefined));+ if (open.length === 0) return;+ let timer: ReturnType<typeof setTimeout> | undefined;+ await Promise.race([+ Promise.allSettled(open),+ new Promise((resolvePromise) => {+ timer = setTimeout(resolvePromise, budgetMs);+ timer.unref?.();+ }),+ ]);+ clearTimeout(timer);+ }+/** Resolves once every queued capture, upload and archive step has settled (tests, profiling). */async idle(): Promise<void> {await this.sessionUploader.idleAll();diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.tsindex 2663ada1..fea734f9 100644--- a/apps/server/src/capture-protocol.ts+++ b/apps/server/src/capture-protocol.ts@@ -46,6 +46,7 @@ export type HostRequest =| { type: "uploadFile"; sessionId: string; path: string; size: number; options?: UploadRequestOptions }| { type: "capabilities" }| { type: "accountId" }+ | { type: "engineTarget"; sessionId: string; workspaceId: string }| { type: "sessionIntegrity"; sessionId: string; summary?: boolean; turns?: number }| { type: "refreshAccessToken" }| { type: "archiveRequest"; path: string; method: "GET" | "POST"; body?: string; refresh?: false }@@ -56,7 +57,7 @@ export type RequestChannel = "uploader" | "archive";export type SerializedResponse = { status: number; statusText: string; headers: Array<[string, string]>; body: ArrayBuffer | null };export type RequestResult =- | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null; value?: TraceCapabilities; account?: string | null }+ | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null; value?: TraceCapabilities; account?: string | null; target?: EngineTarget | null }| { kind: "result"; id: number; ok: false; error: string; name: string };export type ToWorker = CaptureCall | RequestResult;@@ -79,6 +80,8 @@ export type CaptureWorkerInit = {accountId: boolean;/** The main thread can read the server's integrity record of a chat. */sessionIntegrity: boolean;+ /** The main thread can name the engine holding a chat (to settle a turn a previous process left open). */+ engineTarget: boolean;sessionUploader: { upload: boolean; uploadFile: boolean; capabilities: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };archive: {enabled: boolean;diff --git a/apps/server/src/capture-worker.ts b/apps/server/src/capture-worker.tsindex d76b54f8..0c47dc6d 100644--- a/apps/server/src/capture-worker.ts+++ b/apps/server/src/capture-worker.ts@@ -116,6 +116,9 @@ const host = new CaptureHost({engineVersion: init.engineVersion,log,...(init.accountId ? { accountId: async () => (await ask("uploader", { type: "accountId" })).account ?? null } : {}),+ ...(init.engineTarget+ ? { engineTarget: async (sessionId: string, workspaceId: string) => (await ask("uploader", { type: "engineTarget", sessionId, workspaceId })).target ?? null }+ : {}),...(init.sessionIntegrity? {sessionIntegrity: async (sessionId: string, options: { summary?: boolean; turns?: number } = {}) => {diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex 71d2de94..8df606e1 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -193,7 +193,7 @@ import { CloudProviderSync, parseCloudProviderDenSession } from "./cloud-provideimport { createEngineV2Preview, type EngineV2Preview } from "./engine-v2-preview.js";import { sessionUploaderEnabled } from "./session-uploader.js";import { startCaptureService, type CaptureService } from "./capture-client.js";-import { buildOpencodeProxyUrl, engineTarget } from "./session-upload-observer.js";+import { buildOpencodeProxyUrl, engineTarget, type EngineTarget } from "./session-upload-observer.js";import { OmniRushGatewayBroker } from "./omnirush-gateway-broker.js";import { awaitOmniRushModelCatalogSettled, startOmniRushModelCatalogSync } from "./omnirush-model-catalog-sync.js";import { OmniRushVoiceService, voiceProjectContext } from "./omnirush-voice.js";@@ -901,6 +901,27 @@ function uploadDeletedSessionId(method: string, proxyPath: string): string | nultype EngineRequestTarget = { baseUrl: string; headers: Headers };+/**+ * The engine holding a chat of a local workspace, scoped like the proxy's+ * requests, for settling a turn a previous process left open; null for a+ * remote or unknown workspace, no engine, or an engine without the chat.+ */+async function captureEngineTarget(config: ServerConfig, sessionId: string, workspaceId: string): Promise<EngineTarget | null> {+ const workspace = config.workspaces.find((item) => item.id === workspaceId);+ if (!workspace || workspace.workspaceType === "remote") return null;+ const route = enginePoolForConfig(config)?.routeRequest("GET", `/session/${sessionId}/message`) ?? null;+ const connection = route ? null : resolveWorkspaceOpencodeConnection(config, workspace);+ const baseUrl = route?.target.baseUrl ?? connection?.baseUrl?.trim() ?? "";+ if (!baseUrl) return null;+ const auth = route ? buildEngineAuthProbeHeader(route.target.username, route.target.password) : connection?.authHeader ?? null;+ const directory = resolveOpencodeDirectory(workspace);+ if (!(await engineHasSession(baseUrl, auth, directory, sessionId))) return null;+ const headers = new Headers();+ if (auth) headers.set("Authorization", auth);+ const scoped = scopeWorkspaceOpencodeRequest(headers, "", directory);+ return engineTarget(baseUrl, scoped.headers, scoped.search);+}+/** How long the existence check before a chat's first captured prompt waits for the engine. */const ENGINE_SESSION_CHECK_TIMEOUT_MS = 3_000;@@ -1060,6 +1081,8 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {// "opencode/1.18.32-r2": the bundled engine, as every upload names it (engine-identity.ts).engineVersion: BUNDLED_ENGINE_VERSION,log: (level, message, attributes) => logger.log(level, message, attributes),+ // A turn a crashed process or worker left open is settled from the engine that has its chat.+ engineTarget: (sessionId: string, workspaceId: string) => captureEngineTarget(config, sessionId, workspaceId),...(gatewayBroker.enabled? { accountId: () => gatewayBroker.accountId(), sessionIntegrity: (sessionId: string, options?: { summary?: boolean; turns?: number }) => gatewayBroker.sessionIntegrity(sessionId, options) }: {}),diff --git a/apps/server/src/session-upload-observer.test.ts b/apps/server/src/session-upload-observer.test.tsindex 16fa5219..796cc23e 100644--- a/apps/server/src/session-upload-observer.test.ts+++ b/apps/server/src/session-upload-observer.test.ts@@ -885,3 +885,57 @@ describe("message outlines and turn outcomes", () => {])).toBe("stopped");});});++describe("turns left open by an ended process, and turns open at quit", () => {+ test("a recovered turn on an idle session settles at the first read, with its messages, marked recovered", async () => {+ const { control, target } = startEngine();+ control.messages = [message("msg_1", "user"), message("msg_2", "assistant")];+ const uploader = new FakeUploader();+ const archive = fakeArchive();+ const clock = fakeClock();+ await observeUploadedSession({ sessionUploader: uploader, archive, observers: createSessionObservers(), target, timing: clock.timing, sessionId: SESSION, recovered: true });+ expect(control.statusReads).toBe(1);+ expect(uploader.turnMessageIds()).toEqual([["msg_1", "msg_2"]]);+ const final = uploader.entries.find((entry) => entry.kind === "flush" && entry.final !== undefined);+ expect(final).toMatchObject({ final: { recovered: true } });+ expect(uploader.checkpoint).toBe("msg_2");+ expect(archive.calls).toEqual(["followed", "completed"]);+ });++ test("a recovered turn still running (a restarted worker) is followed to its end as usual", async () => {+ const { control, target } = startEngine();+ let busy = true;+ control.status = () => (busy ? "busy" : "idle");+ control.messages = [message("msg_1", "user")];+ const uploader = new FakeUploader();+ const archive = fakeArchive();+ const clock = fakeClock();+ clock.at(30_000, () => {+ busy = false;+ control.messages.push(message("msg_2", "assistant"));+ });+ await observeUploadedSession({ sessionUploader: uploader, archive, observers: createSessionObservers(), target, timing: clock.timing, sessionId: SESSION, recovered: true });+ expect(clock.elapsed()).toBeGreaterThanOrEqual(30_000);+ expect(uploader.turnMessageIds()).toEqual([["msg_1", "msg_2"]]);+ });++ test("settleNow (an app quit) ends a running turn with the messages so far, once", async () => {+ const { control, target } = startEngine();+ control.status = () => "busy";+ control.messages = [message("msg_1", "user"), message("msg_2", "assistant")];+ const uploader = new FakeUploader();+ const archive = fakeArchive();+ const observers = createSessionObservers();+ // A real clock: the observer is between status reads when the app quits.+ void observeUploadedSession({ sessionUploader: uploader, archive, observers, target, sessionId: SESSION });+ await Bun.sleep(50);+ const observed = observers.sessions.get(SESSION);+ expect(observed?.settleNow).toBeDefined();+ await observed!.settleNow!();+ await observed!.settleNow!();+ observers.controller.abort();+ expect(uploader.turnMessageIds()).toEqual([["msg_1", "msg_2"]]);+ expect(uploader.labels()).toContain("session.quit_settled");+ expect(uploader.entries.find((entry) => entry.kind === "flush" && entry.final !== undefined)).toMatchObject({ final: { reason: "app_quit" } });+ });+});diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.tsindex d757bcda..638712b2 100644--- a/apps/server/src/session-upload-observer.ts+++ b/apps/server/src/session-upload-observer.ts@@ -32,14 +32,21 @@ export type EngineTarget = {* snapshot is taken right as the prompt goes out, and its messages stop* before the prompt's own.*/-type FollowedTurn = { snapshotTaken: boolean; cutAt: number | null };+type FollowedTurn = { snapshotTaken: boolean; cutAt: number | null; settled?: boolean };/*** A session being observed: the engine its latest captured request went to* (an engine that restarted may answer on another port), how many requests* asked for it while it was, the turn being followed, and the observation itself.*/-type ObservedSession = { target: EngineTarget; requests: number; turn: FollowedTurn | null; done: Promise<void> };+type ObservedSession = {+ target: EngineTarget;+ requests: number;+ turn: FollowedTurn | null;+ done: Promise<void>;+ /** Settles the turn being followed now, as the engine has it (an app quit); nothing when none is open. */+ settleNow?: () => Promise<void>;+};/** The engine that took over from a closed one: its base URL and its Authorization header. */export type EngineReplacement = { baseUrl: string; authorization: string | null };@@ -797,6 +804,12 @@ export function observeUploadedSession(input: {sessionId: string;target: EngineTarget;timing?: Partial<ObserverTiming>;+ /**+ * The turn was left open by a process that ended (a crash, a killed+ * worker): an idle session settles at its first read, and the turn's end+ * says it was recovered.+ */+ recovered?: boolean;}): Promise<void> {if (!input.sessionUploader.enabled) return Promise.resolve();const observer = input.observers;@@ -872,7 +885,14 @@ export function observeUploadedSession(input: {* history when it was read already; a turn ended by the next prompt keeps* only the messages from before that prompt.*/+ let recovering = input.recovered === true;const settle = async (status: string, read?: EngineHistory): Promise<void> => {+ // Once per turn: an app quit may settle it while the follow loop is about to.+ const turn = turnState();+ if (turn.settled) return;+ turn.settled = true;+ const recovered = recovering;+ recovering = false;let history: EngineHistory | null = read ?? null;let unavailable: Record<string, unknown> = { unavailable: true };try {@@ -953,7 +973,12 @@ export function observeUploadedSession(input: {// A turn the user stopped (Esc, Stop) says so on its trigger and its turn.completed event too.const aborted = outcome === "stopped" ? ("aborted" as const) : undefined;captureTurnSnapshot(aborted);- sessionUploader.flushTrace(sessionId, { messages: history ? newest.messages : unavailable, ...(aborted ? { outcome: aborted } : {}) });+ sessionUploader.flushTrace(sessionId, {+ messages: history ? newest.messages : unavailable,+ ...(aborted ? { outcome: aborted } : {}),+ ...(recovered ? { recovered: true } : {}),+ ...(status === "quit" ? { reason: "app_quit" } : {}),+ });// The delta's turn number is the engine's completed-turn count, which// survives app restarts; without the messages the archiver numbers it// right after the last archived turn. It also arms the idle final archive.@@ -1016,6 +1041,11 @@ export function observeUploadedSession(input: {if (unavailableTraced && unavailableSince !== null) {sessionUploader.recordTrace(sessionId, "session.engine_recovered", { failures, unavailable_ms: timing.now() - unavailableSince });}+ // A turn a previous process left open: an idle session has nothing more to wait for.+ if (recovering && status === "idle") {+ await settle(status);+ return session.requests;+ }failures = 0;unavailableSince = null;unavailableTraced = false;@@ -1084,6 +1114,11 @@ export function observeUploadedSession(input: {return null;};+ session.settleNow = async () => {+ if (!session.turn || session.turn.settled) return;+ sessionUploader.recordTrace(sessionId, "session.quit_settled", {});+ await settle("quit");+ };session.done = (async () => {const checkpoint = await sessionUploader.sessionCheckpoint(sessionId);if (checkpoint.lastMessageId) observer.lastMessageIds.set(sessionId, checkpoint.lastMessageId);diff --git a/apps/server/src/session-upload-state.ts b/apps/server/src/session-upload-state.tsindex 5047ea0b..d79faaf2 100644--- a/apps/server/src/session-upload-state.ts+++ b/apps/server/src/session-upload-state.ts@@ -18,6 +18,8 @@ export const UPLOAD_TEMP_DIRECTORY = "omnirush-upload-tmp";export const UPLOAD_SESSION_LEDGER_FILE = "omnirush-upload-sessions.json";export const UPLOAD_SPOOL_DIRECTORY = "omnirush-upload-spool";export const UPLOAD_BASE_DIRECTORY = "omnirush-upload-bases";+/** Each live chat's trace events not yet handed to an upload, so a crash loses none (session-uploader.ts). */+export const UPLOAD_JOURNAL_DIRECTORY = "omnirush-upload-journal";/** The same files under the names used up to 2.2.1. */export const LEGACY_UPLOAD_STATE_NAMES = {diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.tsindex 00a9d68c..1735df9f 100644--- a/apps/server/src/session-uploader.test.ts+++ b/apps/server/src/session-uploader.test.ts@@ -19,6 +19,7 @@ import {MAX_SPOOL_AGE_MS,uploadIdempotencyKey,serverSessionPosition,+ parseJournal,SessionUploader,isRetryableUploadStatus,mapBounded,@@ -348,6 +349,79 @@ describe("session uploader privacy", () => {expect(Object.values(ledger.sessions).filter((record) => record.childSessionIds)).toHaveLength(512);});+ test("a crash mid-turn loses no trace event: the next start uploads the journal in its own segment, marked recovered, and settles the open turn", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-journal-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-journal-state-"));+ roots.push(root, stateDir);+ await writeFile(join(root, "app.txt"), "hello\n");+ const envelopes: Array<Record<string, unknown>> = [];+ const upload = async (_sessionId: string, compressed: Uint8Array) => {+ envelopes.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Record<string, unknown>);+ return Response.json({ ok: true }, { status: 201 });+ };+ const crashed = new SessionUploader({ stateDir, upload, fallbackScanMs: 60_000 });+ const sessionId = "session-journal-1234";+ crashed.startSession(sessionId, "workspace-journal", root);+ await crashed.idle(sessionId);+ crashed.captureSnapshot(sessionId, "prompt");+ crashed.recordTrace(sessionId, "engine.request", { body: "do the task" });+ crashed.recordTrace(sessionId, "tool.call", { name: "edit" });+ await new Promise((resolve) => setTimeout(resolve, 600));+ // The process dies here: no stop(), nothing flushed.+ expect(envelopes.map((item) => item.snapshot_type)).toEqual(["start"]);+ const journals = await readdir(join(stateDir, "omnirush-upload-journal"));+ expect(journals).toHaveLength(1);++ const recoveredTurns: Array<{ sessionId: string; workspaceId: string; root: string }> = [];+ const next = new SessionUploader({ stateDir, upload, fallbackScanMs: 60_000, onRecoveredTurn: (turn) => recoveredTurns.push(turn) });+ for (let waited = 0; waited < 5_000 && envelopes.length < 2; waited += 50) await new Promise((resolve) => setTimeout(resolve, 50));+ const recovered = envelopes[1]!;+ expect(recovered).toMatchObject({ snapshot_type: "trace", session_segment: 1, sequence: 2 });+ const types = (recovered.trace as Array<{ type: string }>).map((event) => event.type);+ expect(types[0]).toBe("collector.recovered");+ expect(types).toContain("engine.request");+ expect(types).toContain("tool.call");+ expect(recoveredTurns).toEqual([{ sessionId, workspaceId: "workspace-journal", root }]);+ expect(await readdir(join(stateDir, "omnirush-upload-journal"))).toEqual([]);+ // The chat's next segment starts after the recovered upload.+ next.startSession(sessionId, "workspace-journal", root);+ await next.stop();+ expect(envelopes.slice(2).map((item) => [item.snapshot_type, item.session_segment, item.sequence])).toEqual([["start", 2, 3], ["trace", 2, 4], ["end", 2, 5]]);+ }, 20_000);++ test("the journal holds only what is not yet handed to an upload, and is gone once the chat ends", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-journal2-"));+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-journal2-state-"));+ roots.push(root, stateDir);+ const sessionUploader = new SessionUploader({+ stateDir,+ upload: async () => Response.json({ ok: true }, { status: 201 }),+ fallbackScanMs: 60_000,+ });+ const sessionId = "session-journal2-1234";+ sessionUploader.startSession(sessionId, "workspace-journal2", root);+ sessionUploader.captureSnapshot(sessionId, "prompt");+ sessionUploader.recordTrace(sessionId, "tool.call", { n: 1 });+ sessionUploader.captureSnapshot(sessionId, "turn_completed");+ sessionUploader.flushTrace(sessionId, { messages: [] });+ await sessionUploader.idle(sessionId);+ await new Promise((resolve) => setTimeout(resolve, 400));+ const dir = join(stateDir, "omnirush-upload-journal");+ const [name] = await readdir(dir);+ const journal = parseJournal(await readFile(join(dir, name!), "utf8"));+ expect(journal?.meta).toMatchObject({ session_id: sessionId, turn_open: false, segment: 1 });+ expect(journal?.events.map((event) => event.type) ?? []).not.toContain("tool.call");+ await sessionUploader.stop();+ expect(await readdir(dir)).toEqual([]);+ }, 20_000);++ test("reads a journal cut off mid-line by a crash", () => {+ const meta = { v: 1 as const, session_id: "session-cut-1234", workspace_id: "ws", root: "/w", segment: 3, turn_open: true, at: "2026-10-10T00:00:00.000Z" };+ const text = `${JSON.stringify({ m: meta })}\n${JSON.stringify({ e: { at: "t", type: "tool.call", data: { a: 1 } } })}\n{"e":{"at":"t","ty`;+ expect(parseJournal(text)).toEqual({ meta, events: [{ at: "t", type: "tool.call", data: { a: 1 } }] });+ expect(parseJournal(`${JSON.stringify({ e: { at: "t", type: "x" } })}\n`)).toBeNull();+ });+test("persists session segments and message checkpoints across a resume", async () => {const root = await mkdtemp(join(tmpdir(), "omnirush-upload-resume-"));const stateDir = await mkdtemp(join(tmpdir(), "omnirush-upload-state-"));diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 39562746..e7ab98bb 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -1,7 +1,7 @@import { createHash, randomBytes } from "node:crypto";import { execFile, spawn } from "node:child_process";import { createReadStream, createWriteStream, watch, type FSWatcher, type WriteStream } from "node:fs";-import { lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";+import { appendFile, lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";import { homedir, release as osRelease, tmpdir } from "node:os";import nodePath, { basename, dirname, extname, join, relative, resolve, sep, type PlatformPath } from "node:path";import { promisify } from "node:util";@@ -13,6 +13,7 @@ import { SESSION_UPLOAD_BUDGET, SESSION_UPLOAD_ENDPOINT_PATH, sessionUploadTimeoimport { externalFetch } from "./server-fetch.js";import {UPLOAD_BASE_DIRECTORY,+ UPLOAD_JOURNAL_DIRECTORY,UPLOAD_SESSION_LEDGER_FILE,UPLOAD_SPOOL_DIRECTORY,UPLOAD_TEMP_DIRECTORY,@@ -145,6 +146,10 @@ const MAX_SPOOL_TRACE_BYTES = 128 * 1024 * 1024;const MAX_SPOOL_TRACE_ENTRIES = 2_000;/** A spooled upload that keeps failing with a retryable answer is kept this long, then dropped. */export const MAX_SPOOL_AGE_MS = 7 * 24 * 60 * 60_000;+/** Trace events reach the chat's journal on disk within this long. */+const JOURNAL_FLUSH_MS = 250;+/** A journal stops taking events past this size (its chat's meta lines still go in). */+const MAX_JOURNAL_BYTES = 256 * 1024 * 1024;/** How long spooling or a drain waits to learn which account is signed in. */const ACCOUNT_LOOKUP_TIMEOUT_MS = 5_000;const UPLOAD_ATTEMPTS = 3;@@ -560,8 +565,77 @@ type SessionState = {extras: Map<string, ExtraManifestEntry>;/** The numbering already moved past the server's once (an Idempotency-Key it held with other bytes). */repositioned?: boolean;+ /** Trace events not yet handed to an upload, as written to the chat's journal on disk. */+ journal: JournalState;};+type JournalState = {+ pending: string[];+ timer: ReturnType<typeof setTimeout> | null;+ tail: Promise<void>;+ bytes: number;+ full: boolean;+ /** A recovery's stand-in state: never journaled itself. */+ off: boolean;+ /**+ * Lines are written once the chat's own recovery is done (prepareSession):+ * a journal a previous process left under the same name is read first.+ */+ ready: boolean;+};++/** What a chat's journal says about it (its last meta line). */+type JournalMeta = {+ v: 1;+ session_id: string;+ workspace_id: string;+ root: string;+ segment: number;+ /** A prompt went out and its turn has not ended. */+ turn_open: boolean;+ at: string;+};++function parseJournalMeta(value: unknown): JournalMeta | null {+ if (!isRecord(value) || value.v !== 1 || typeof value.session_id !== "string" || typeof value.workspace_id !== "string"+ || typeof value.root !== "string" || !Number.isSafeInteger(value.segment) || typeof value.turn_open !== "boolean"+ || typeof value.at !== "string") return null;+ return {+ v: 1,+ session_id: value.session_id,+ workspace_id: value.workspace_id,+ root: value.root,+ segment: Number(value.segment),+ turn_open: value.turn_open,+ at: value.at,+ };+}++/** A journal file's meta (the last meta line) and its events; null when it holds no meta. */+export function parseJournal(text: string): { meta: JournalMeta; events: TraceEvent[] } | null {+ let meta: JournalMeta | null = null;+ const events: TraceEvent[] = [];+ for (const line of text.split("\n")) {+ if (!line) continue;+ let parsed: unknown;+ try {+ parsed = JSON.parse(line);+ } catch {+ // A line cut off by the crash.+ continue;+ }+ if (!isRecord(parsed)) continue;+ if ("m" in parsed) meta = parseJournalMeta(parsed.m) ?? meta;+ else if (isRecord(parsed.e) && typeof parsed.e.at === "string" && typeof parsed.e.type === "string") {+ events.push({ at: parsed.e.at, type: parsed.e.type, ...("data" in parsed.e ? { data: parsed.e.data } : {}) });+ }+ }+ return meta ? { meta, events } : null;+}++/** Where the turns a previous process left open are told: their chat is settled from the engine (capture-host.ts). */+export type RecoveredTurn = { sessionId: string; workspaceId: string; root: string };+type SessionUploaderOptions = {gatewayUrl?: string;accessToken?: string;@@ -576,6 +650,12 @@ type SessionUploaderOptions = {* this hook the spool is the one account's.*/accountId?: () => Promise<string | null>;+ /**+ * A chat a previous process left with its turn open (a crash, a killed+ * worker): its journaled events are uploaded, then the host settles the+ * turn from the engine's messages.+ */+ onRecoveredTurn?: (turn: RecoveredTurn) => void;/** GET /omnirush/me/sessions/{id}/integrity for this account (where the server's copy of a chat stands). */sessionIntegrity?: (sessionId: string, options?: { summary?: boolean; turns?: number }) => Promise<Response>;/** Negotiates the canonical structured trace envelope. Failures fall back to schema v2. */@@ -4276,6 +4356,13 @@ export class SessionUploader {private readonly spoolMaxAgeMs: number;private readonly accountIdHook?: SessionUploaderOptions["accountId"];private readonly sessionIntegrity?: SessionUploaderOptions["sessionIntegrity"];+ private readonly onRecoveredTurn?: SessionUploaderOptions["onRecoveredTurn"];+ /** Each live chat's journal of trace events (null without a state dir). */+ private readonly journalDir: string | null;+ /** The journals a previous process left, read at start; a chat's own start waits for its recovery. */+ private readonly journalsRead: Promise<void>;+ private readonly recovering = new Map<string, Promise<void>>();+ private journalWarned = false;/** The signed-in account, once known (it never changes within a process: an account change restarts the server). */private accountKnown: string | null = null;private accountLookup: Promise<string | null> | null = null;@@ -4344,6 +4431,7 @@ export class SessionUploader {this.stateDir = stateDir;this.ledgerPath = stateDir ? join(stateDir, UPLOAD_SESSION_LEDGER_FILE) : null;this.spoolDir = stateDir ? join(stateDir, UPLOAD_SPOOL_DIRECTORY) : null;+ this.journalDir = stateDir ? join(stateDir, UPLOAD_JOURNAL_DIRECTORY) : null;// State left under the pre-2.2.2 names is moved before anything reads it.this.stateMigrated = stateDir? migrateLegacyUploadState(stateDir, (message, details) => this.log("warn", message, details))@@ -4364,6 +4452,7 @@ export class SessionUploader {this.spoolMaxAgeMs = options.spoolMaxAgeMs ?? MAX_SPOOL_AGE_MS;this.accountIdHook = options.accountId;this.sessionIntegrity = options.sessionIntegrity;+ this.onRecoveredTurn = options.onRecoveredTurn;this.snapshotMaxBytes = options.snapshotMaxBytes ?? MAX_SNAPSHOT_BYTES;this.minChangeIntervalMs = options.minChangeIntervalMs ?? MIN_UPLOAD_CHANGE_INTERVAL_MS;this.maxWatchedFiles = options.maxWatchedFiles ?? MAX_UPLOAD_WATCHED_FILES;@@ -4390,6 +4479,8 @@ export class SessionUploader {// Without an account they wait (up to the spool's age limit) for the// account to come back: only a user sign-out clears them (clearSpool()).if (this.spoolDir && this.enabled) this.scheduleRetry(this.retryBaseMs);+ // So do the journals a previous process left: their turns are settled now.+ this.journalsRead = this.enabled ? this.recoverJournals() : Promise.resolve();}/** Capture context wired to this uploader: its trace, its scrub, the project archive and the archive's exclusions. */@@ -4491,6 +4582,10 @@ export class SessionUploader {private async prepareSession(state: SessionState): Promise<void> {await this.ledgerReady;+ // What a previous process left of this chat goes out first, numbered before this segment.+ await this.journalsRead;+ await this.recovering.get(state.id);+ state.journal.ready = true;const recorded = this.ledger.sessions[state.id];// No local record: a new chat, or one this device's ledger lost. The// server says where its copy stands, so the numbering goes on after it@@ -4520,6 +4615,7 @@ export class SessionUploader {}this.ledger.sessions[state.id] = this.ledgerRecord(state);await this.saveLedger();+ this.journalMeta(state);}/**@@ -4701,7 +4797,32 @@ export class SessionUploader {startSession(sessionId: string, workspaceId: string, root: string): void {if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;void this.toolchains?.get(root).catch(() => null); // toolchain.ts: collected in the background- const state: SessionState = {+ const state = this.newSessionState(sessionId, workspaceId, root, this.acquireCache(root));+ state.ready = this.prepareSession(state);+ this.sessions.set(sessionId, state);+ this.journalMeta(state);+ // Capture context collects once the start snapshot is out.+ let startSnapshotDone: () => void = () => undefined;+ const startSnapshot = new Promise<void>((resolveStart) => (startSnapshotDone = resolveStart));+ this.context?.sessionStarted(sessionId, root, startSnapshot);+ this.enqueue(state, async () => {+ await state.ready;+ try {+ // The watchers go up as soon as the start snapshot's listing is in,+ // before any file is read: an edit made during the scan is either+ // seen by the scan or reported by a watcher.+ await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");+ } finally {+ state.started = true;+ startSnapshotDone();+ // No listing reached the plan (a failed scan): poll.+ if (state.watchMode === "starting") this.installWatchers(state, null);+ }+ });+ }++ private newSessionState(sessionId: string, workspaceId: string, root: string, cache: Map<string, HashCacheEntry>): SessionState {+ return {id: sessionId,root,workspaceId,@@ -4738,7 +4859,7 @@ export class SessionUploader {turnSkipped: new Map(),turnWrittenPaths: new Map(),relevance: new Map(),- cache: this.acquireCache(root),+ cache,listing: { denied: 0, truncated: false },dirty: new Set(),// Nothing is known about the tree yet: the first change capture rescans it.@@ -4753,27 +4874,8 @@ export class SessionUploader {failureCount: 0,ready: Promise.resolve(),tail: Promise.resolve(),+ journal: { pending: [], timer: null, tail: Promise.resolve(), bytes: 0, full: false, off: false, ready: false },};- state.ready = this.prepareSession(state);- this.sessions.set(sessionId, state);- // Capture context collects once the start snapshot is out.- let startSnapshotDone: () => void = () => undefined;- const startSnapshot = new Promise<void>((resolveStart) => (startSnapshotDone = resolveStart));- this.context?.sessionStarted(sessionId, root, startSnapshot);- this.enqueue(state, async () => {- await state.ready;- try {- // The watchers go up as soon as the start snapshot's listing is in,- // before any file is read: an edit made during the scan is either- // seen by the scan or reported by a watcher.- await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");- } finally {- state.started = true;- startSnapshotDone();- // No listing reached the plan (a failed scan): poll.- if (state.watchMode === "starting") this.installWatchers(state, null);- }- });}// --- watcher and dirty tracking --------------------------------------------@@ -5210,7 +5312,9 @@ export class SessionUploader {/** Appends one trace event, keeping only the newest MAX_UPLOAD_TRACE_EVENTS. */private appendTrace(state: SessionState, type: string, data?: unknown): void {- state.trace.push({ at: new Date().toISOString(), type, ...(data === undefined ? {} : { data }) });+ const event: TraceEvent = { at: new Date().toISOString(), type, ...(data === undefined ? {} : { data }) };+ state.trace.push(event);+ this.journalEvent(state, event);if (state.trace.length > MAX_UPLOAD_TRACE_EVENTS) state.trace.splice(0, state.trace.length - MAX_UPLOAD_TRACE_EVENTS);}@@ -5438,6 +5542,8 @@ export class SessionUploader {if (trigger === "prompt") state.turnStartedAt = Date.now() + 1;const startedAt = state.turnStartedAt;state.turnInProgress = trigger === "prompt";+ // A crash from here on leaves the turn open in the journal (or no longer).+ this.journalMeta(state, true);// The milestone carries whatever edits were held back for another session's turn.state.changesHeld = false;if (state.changeTimer) {@@ -5587,6 +5693,7 @@ export class SessionUploader {await this.uploadTrace(state, pendingTrace);await this.uploadWorkspace(state, "end", "session_end");} finally {+ await this.journalRemove(state);this.releaseCache(state);this.sessions.delete(sessionId);this.onSessionClosed?.(sessionId);@@ -5610,7 +5717,12 @@ export class SessionUploader {this.enqueue(state, async () => {await state.ready;if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));- await this.uploadTrace(state, pendingTrace);+ try {+ await this.uploadTrace(state, pendingTrace);+ } finally {+ // Uploaded, spooled or refused: the journal keeps only what came after.+ await this.journalRewrite(state);+ }});}@@ -6536,6 +6648,217 @@ export class SessionUploader {}}+ // --- trace journal -------------------------------------------------------+ // Each live chat's trace events not yet handed to an upload are appended to+ // a journal on disk (JSON lines: {"m": meta} and {"e": event}) within+ // JOURNAL_FLUSH_MS, and its turn's opening and end at once. A crash or a+ // killed worker loses none of them: the next start uploads what a journal+ // holds, marked recovered, and has the host settle a turn left open.++ private journalPath(sessionId: string): string {+ return join(this.journalDir!, `${createHash("sha256").update(sessionId).digest("hex").slice(0, 40)}.jsonl`);+ }++ private journalMetaLine(state: SessionState): string {+ const meta: JournalMeta = {+ v: 1,+ session_id: state.id,+ workspace_id: state.workspaceId,+ root: state.root,+ segment: state.segment,+ turn_open: state.turnInProgress,+ at: new Date().toISOString(),+ };+ return JSON.stringify({ m: meta });+ }++ private journalMeta(state: SessionState, now = false): void {+ if (!this.journalDir || state.finished || state.journal.off) return;+ state.journal.pending.push(this.journalMetaLine(state));+ this.scheduleJournal(state, now);+ }++ private journalEvent(state: SessionState, event: TraceEvent): void {+ if (!this.journalDir || state.journal.full || state.journal.off) return;+ let line: string;+ try {+ line = JSON.stringify({ e: event });+ } catch {+ return;+ }+ if (state.journal.bytes + line.length > MAX_JOURNAL_BYTES) {+ state.journal.full = true;+ return;+ }+ state.journal.bytes += line.length + 1;+ state.journal.pending.push(line);+ this.scheduleJournal(state, false);+ }++ private scheduleJournal(state: SessionState, now: boolean): void {+ if (now) {+ if (state.journal.timer) clearTimeout(state.journal.timer);+ state.journal.timer = null;+ void this.flushJournal(state);+ return;+ }+ if (state.journal.timer) return;+ state.journal.timer = setTimeout(() => {+ state.journal.timer = null;+ void this.flushJournal(state);+ }, JOURNAL_FLUSH_MS);+ state.journal.timer.unref?.();+ }++ /** Appends the lines waiting for the journal; resolves once they are on disk. */+ private flushJournal(state: SessionState): Promise<void> {+ if (!state.journal.ready) return state.journal.tail;+ const lines = state.journal.pending.splice(0);+ if (lines.length === 0 || !this.journalDir) return state.journal.tail;+ const path = this.journalPath(state.id);+ state.journal.tail = state.journal.tail+ .then(async () => {+ await mkdir(this.journalDir!, { recursive: true, mode: 0o700 });+ await appendFile(path, `${lines.join("\n")}\n`, { mode: 0o600 });+ })+ .catch((error: unknown) => this.journalFailed(state, error));+ return state.journal.tail;+ }++ /** The events went to an upload: the journal keeps its meta and what was recorded since. */+ private journalRewrite(state: SessionState): Promise<void> {+ if (!this.journalDir || state.finished || state.journal.off || !state.journal.ready) return state.journal.tail;+ if (state.journal.timer) clearTimeout(state.journal.timer);+ state.journal.timer = null;+ state.journal.pending = [];+ const lines = [this.journalMetaLine(state)];+ state.journal.bytes = 0;+ state.journal.full = false;+ for (const event of state.trace) {+ const line = JSON.stringify({ e: event });+ if (state.journal.bytes + line.length > MAX_JOURNAL_BYTES) {+ state.journal.full = true;+ break;+ }+ state.journal.bytes += line.length + 1;+ lines.push(line);+ }+ const path = this.journalPath(state.id);+ state.journal.tail = state.journal.tail+ .then(async () => {+ await mkdir(this.journalDir!, { recursive: true, mode: 0o700 });+ await writeFileAtomic(path, `${lines.join("\n")}\n`, { mode: 0o600 });+ })+ .catch((error: unknown) => this.journalFailed(state, error));+ return state.journal.tail;+ }++ /** The chat's last upload settled: nothing is left to recover. */+ private journalRemove(state: SessionState): Promise<void> {+ if (state.journal.timer) clearTimeout(state.journal.timer);+ state.journal.timer = null;+ state.journal.pending = [];+ if (!this.journalDir || state.journal.off) return state.journal.tail;+ const path = this.journalPath(state.id);+ state.journal.tail = state.journal.tail.then(() => rm(path, { force: true })).catch((error: unknown) => this.journalFailed(state, error));+ return state.journal.tail;+ }++ private journalFailed(state: SessionState, error: unknown): void {+ if (this.journalWarned) return;+ this.journalWarned = true;+ this.log("warn", "OmniRush session journal write failed", { sessionId: state.id, error: error instanceof Error ? error.message : "unknown" });+ }++ /**+ * App (or capture worker) start: every journal a previous process left is+ * uploaded as a trace of the segment it was recorded in, after a+ * "collector.recovered" event; a turn it left open is handed to the host+ * (onRecoveredTurn) to be settled from the engine. A chat started meanwhile+ * waits for its own (prepareSession). Journals past the spool's age limit+ * are deleted unread.+ */+ private async recoverJournals(): Promise<void> {+ if (!this.journalDir) return;+ await this.stateMigrated;+ await this.ledgerReady;+ let names: string[];+ try {+ names = (await readdir(this.journalDir)).filter((name) => name.endsWith(".jsonl"));+ } catch {+ return;+ }+ const found: Array<{ path: string; meta: JournalMeta; events: TraceEvent[] }> = [];+ for (const name of names) {+ const path = join(this.journalDir, name);+ try {+ const file = await stat(path);+ const parsed = Date.now() - file.mtimeMs > this.spoolMaxAgeMs ? null : parseJournal(await readFile(path, "utf8"));+ if (!parsed || (parsed.events.length === 0 && !parsed.meta.turn_open)) {+ await rm(path, { force: true });+ continue;+ }+ found.push({ path, ...parsed });+ } catch {+ await rm(path, { force: true }).catch(() => undefined);+ }+ }+ // Registered before this resolves, so a chat's start finds its recovery.+ const recovered: Promise<void>[] = [];+ let chain = Promise.resolve();+ for (const journal of found) {+ chain = chain.then(() => this.recoverJournal(journal)).catch((error: unknown) => {+ this.log("warn", "OmniRush session journal recovery failed", { sessionId: journal.meta.session_id, error: error instanceof Error ? error.message : "unknown" });+ });+ const done = chain;+ this.recovering.set(journal.meta.session_id, done);+ recovered.push(done.finally(() => {+ if (this.recovering.get(journal.meta.session_id) === done) this.recovering.delete(journal.meta.session_id);+ }));+ }+ void Promise.allSettled(recovered);+ }++ private async recoverJournal(journal: { path: string; meta: JournalMeta; events: TraceEvent[] }): Promise<void> {+ const { meta, events } = journal;+ const record = this.ledger.sessions[meta.session_id];+ if (events.length > 0) {+ // The segment the events were recorded in, numbered after what it already sent.+ const state = this.newSessionState(meta.session_id, meta.workspace_id, meta.root, new FileHashCache());+ state.journal.off = true;+ state.segment = record?.segment ?? meta.segment;+ state.sequence = record?.nextSequence ?? 0;+ state.sentBytes = record?.sentBytes ?? 0;+ state.lastMessageId = record?.lastMessageId;+ state.failureCount = record?.failureCount ?? 0;+ state.lastFailureAt = record?.lastFailureAt;+ state.lastSuccessAt = record?.lastSuccessAt;+ state.model = record?.model ?? null;+ state.childSessionIds = [...(record?.childSessionIds ?? [])];+ state.childCheckpoints = new Map(Object.entries(record?.childCheckpoints ?? {}));+ state.resumed = state.segment > 1;+ for (const event of events) {+ for (const evidence of knownToolPathEvidence(event.type, event.data)) {+ const path = workspaceRelativePath(state.root, evidence.path.replaceAll("\\", "/"));+ if (path && !isUploadPathDenied(path) && state.touchedPaths.size < MAX_TOUCHED_PATHS) state.touchedPaths.add(path);+ }+ }+ const marker: TraceEvent = {+ at: new Date().toISOString(),+ type: "collector.recovered",+ data: { reason: "restart", turn_open: meta.turn_open, events: events.length, journal_at: meta.at, session_segment: state.segment },+ };+ await this.uploadTrace(state, [marker, ...events]);+ this.log("info", "OmniRush session journal recovered", { sessionId: meta.session_id, events: events.length, turnOpen: meta.turn_open });+ }+ await rm(journal.path, { force: true });+ // The open turn's messages are still in the engine: the host settles it once this chat's recovery is done.+ if (meta.turn_open && this.onRecoveredTurn) {+ const turn: RecoveredTurn = { sessionId: meta.session_id, workspaceId: meta.workspace_id, root: meta.root };+ queueMicrotask(() => this.onRecoveredTurn?.(turn));+ }+ }+// --- durable retry spool -------------------------------------------------private spoolLocked<T>(operation: () => Promise<T>): Promise<T> {@@ -6891,8 +7214,9 @@ export class SessionUploader {this.traceSchemaProbe = null;this.retryFailures = 0;return this.spoolLocked(async () => {- // The turn diffs' bases are workspace content on disk too.+ // The turn diffs' bases are workspace content on disk too, and so are the journals.await this.bases.clear();+ if (this.journalDir) await rm(this.journalDir, { recursive: true, force: true });if (!this.spoolDir) return;await rm(this.spoolDir, { recursive: true, force: true });if (this.stateDir) await removeLegacyUploadContent(this.stateDir);