Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: wait for late writes before the next turn

ogsamratbe8d2db1fa97
diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.ts
index ed63aca0..096601ad 100644
--- a/apps/server/src/capture-client.ts
+++ b/apps/server/src/capture-client.ts
@@ -14,9 +14,10 @@ import { Worker } from "node:worker_threads";
import { lstat } from "node:fs/promises";
import { resolve } from "node:path";
import { START_GATE_MS } from "./session-archive/capture-v2.js";
+import { LATE_WRITES_CAP_MS } from "./session-archive/late-writes.js";
import type { FilesUsedStatus } from "./session-archive/index.js";
-import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineReplacement, type EngineTarget, type PromptRecord } from "./capture-host.js";
+import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineReplacement, type EngineTarget, type LateWritesGate, type PromptRecord } from "./capture-host.js";
import {
invokeCapture,
serializeResponse,
@@ -88,11 +89,13 @@ export type CaptureService = {
recordWebVisit(sessionId: string, visit: UploadWebVisit): boolean;
archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void;
/**
- * Capture v2: resolves once the project archive took the session-start
+ * Resolves once the turn before's background processes finished writing
+ * (with `lateWrites`, when that turn ended: LATE_WRITES_CAP_MS after its
+ * end at most) and capture v2's project archive took the session-start
* manifest (hash and stat only, START_GATE_MS at most); the prompt goes to
* the engine after it. Never rejects.
*/
- archiveStartGate(sessionId: string, root: string): Promise<void>;
+ archiveStartGate(sessionId: string, root: string, lateWrites?: LateWritesGate | null): Promise<void>;
observeSession(sessionId: string, target: EngineTarget): void;
/** An engine was closed and another took over its sessions: turn observers reading it move there. */
engineReplaced(closedBaseUrl: string, replacement: EngineReplacement): void;
@@ -224,13 +227,13 @@ class CaptureClient implements CaptureService {
this.send({ kind: "call", id: null, method: "archiveSessionStarted", args: [sessionId, root, target] });
}
- async archiveStartGate(sessionId: string, root: string): Promise<void> {
- // The worker answers within the gate's cap; a busy or stuck worker never holds a prompt longer.
+ async archiveStartGate(sessionId: string, root: string, lateWrites?: LateWritesGate | null): Promise<void> {
+ // The worker answers within the gate's cap (plus the late-writes cap); a busy or stuck worker never holds a prompt longer.
let timer: ReturnType<typeof setTimeout> | undefined;
await Promise.race([
- this.query({ kind: "call", id: null, method: "archiveStartGate", args: [sessionId, root] }).catch(() => undefined),
+ this.query({ kind: "call", id: null, method: "archiveStartGate", args: lateWrites ? [sessionId, root, lateWrites] : [sessionId, root] }).catch(() => undefined),
new Promise<void>((resolvePromise) => {
- timer = setTimeout(resolvePromise, START_GATE_MS + START_GATE_SLACK_MS);
+ timer = setTimeout(resolvePromise, START_GATE_MS + START_GATE_SLACK_MS + (lateWrites ? LATE_WRITES_CAP_MS : 0));
timer.unref?.();
}),
]);
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index bcba2e98..97b53184 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -19,13 +19,15 @@ import {
followToolStart, followContextToolEvents, followFilesUsedCalls,
projectArchiveEngineReads,
promptDispatched,
+ turnEndedAt,
type EngineReplacement,
type EngineTarget,
type SessionObservers,
} from "./session-upload-observer.js";
import type { CaptureStopOptions } from "./capture-protocol.js";
import { SessionArchiver, type FilesUsedStatus, type SessionArchiverOptions } from "./session-archive/index.js";
-import { ProjectArchiveLifecycle, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";
+import { ProjectArchiveLifecycle, type ArchiveLifecycleLog, type LateWrites } from "./session-archive/lifecycle.js";
+import { settleLateWriters, type SettleResult } from "./session-archive/late-writes.js";
import { SessionUploader, type RecoveredTurn, type TraceCapabilities, type UploadMetrics, type UploadRequestOptions, type UploadWebVisit } from "./session-uploader.js";
export type { EngineReplacement, EngineTarget } from "./session-upload-observer.js";
@@ -39,6 +41,14 @@ const QUIT_SETTLE_BUDGET_MS = 5_000;
/** A recovered turn asks for its engine again after these waits (the engine may still be starting). */
const RECOVERY_TARGET_DELAYS_MS = [0, 2_000, 5_000, 15_000, 30_000, 60_000];
+/**
+ * A prompt's wait for the turn before's background processes (late-writes.ts):
+ * the engines' processes, never waited for (with this process, whose
+ * descendants they usually are). Left out (no wait) in a sandbox or a remote
+ * workspace, whose processes are not this machine's.
+ */
+export type LateWritesGate = { exclude: number[] };
+
/** A captured engine request, as the "engine.request" trace event and the prompt's attachments are built from it. */
export type PromptRecord = {
method: string;
@@ -119,6 +129,10 @@ export class CaptureHost {
/** The state dir and the app's data/config/cache dirs: never reported or recorded as a touched file. */
private readonly appDirs: string[];
private readonly engineTarget?: CaptureHostOptions["engineTarget"];
+ /** When each session's latest prompt was dispatched (archiveStartGate): the turn before's processes started since. */
+ private readonly promptAt = new Map<string, number>();
+ /** The latest prompt's wait for the turn before's background processes, when it found any ("turn.late"). */
+ private readonly lateSettles = new Map<string, SettleResult>();
constructor(options: CaptureHostOptions) {
this.engineTarget = options.engineTarget;
@@ -157,6 +171,8 @@ export class CaptureHost {
enabled: enabled && this.sessionUploader.enabled,
accountMissing: !this.sessionUploader.enabled,
log: options.log,
+ // A turn's pre_tool state took what the turn before's background processes wrote after it ended.
+ onLateWrites: (late) => this.recordLateWrites(late),
...(baseIdleMs !== undefined ? { baseIdleMs } : {}),
...(baseMaxDeferMs !== undefined ? { baseMaxDeferMs } : {}),
});
@@ -179,8 +195,30 @@ export class CaptureHost {
finishSession(sessionId: string, finalTrace?: unknown): void {
this.sessionUploader.finishSession(sessionId, finalTrace);
this.archive.sessionEnded(sessionId);
+ this.forgetSession(sessionId);
+ }
+
+ private forgetSession(sessionId: string): void {
this.observers.lastMessageIds.delete(sessionId);
+ this.observers.turnEnds?.delete(sessionId);
this.sessionRoots.delete(sessionId);
+ this.promptAt.delete(sessionId);
+ this.lateSettles.delete(sessionId);
+ }
+
+ /** "turn.late": files the turn before's background processes wrote after it ended, taken into this turn's start. */
+ private recordLateWrites(late: LateWrites): void {
+ const settled = this.lateSettles.get(late.sessionId);
+ this.sessionUploader.recordTrace(late.sessionId, "turn.late", {
+ late_from_turn: late.turn > 1 ? late.turn - 1 : null,
+ turn: late.turn,
+ after_end_at: new Date(late.after).toISOString(),
+ tool_start_at: new Date(late.toolStartAt).toISOString(),
+ waited_ms: settled?.waitedMs ?? 0,
+ settled: settled?.settled ?? true,
+ paths: late.paths,
+ processes: settled?.processes ?? [],
+ });
}
/** The "engine.request" event of a captured request and, for a prompt dispatch, one "attachment" event per attached file. */
@@ -218,8 +256,28 @@ export class CaptureHost {
followFilesUsedCalls({ observers: this.observers, archive: this.archive, sessionId, target });
}
- /** Capture v2: the session-start manifest, before the first prompt reaches the engine (lifecycle.ts startGate). */
- async archiveStartGate(sessionId: string, root: string): Promise<void> {
+ /**
+ * Call while a prompt is held, before it reaches the engine. When the
+ * session's turn before has ended (the engine went idle), the background
+ * processes that turn left running in the workspace get to finish writing
+ * first, at most LATE_WRITES_CAP_MS after it ended (late-writes.ts), so the
+ * turn's end snapshot and archive delta, taken as this prompt cuts it, hold
+ * those writes. A prompt typed into a running turn waits for nothing.
+ * Then capture v2's session-start manifest, before the first prompt
+ * reaches the engine (lifecycle.ts startGate).
+ */
+ async archiveStartGate(sessionId: string, root: string, lateWrites?: LateWritesGate | null): Promise<void> {
+ const previousStart = this.promptAt.get(sessionId);
+ const endedAt = turnEndedAt(this.observers, sessionId);
+ this.lateSettles.delete(sessionId);
+ if (lateWrites && previousStart !== undefined && endedAt !== null && endedAt >= previousStart) {
+ const settled = await settleLateWriters({ root, since: previousStart, exclude: [process.pid, ...lateWrites.exclude], endedAt });
+ if (settled.processes.length > 0) {
+ this.lateSettles.set(sessionId, settled);
+ this.sessionUploader.recordTrace(sessionId, "turn.settle", { waited_ms: settled.waitedMs, settled: settled.settled, processes: settled.processes });
+ }
+ }
+ this.promptAt.set(sessionId, Date.now());
await this.archive.startGate(sessionId, root);
}
@@ -280,8 +338,7 @@ export class CaptureHost {
this.sessionUploader.recordTrace(sessionId, "session.deleted");
this.sessionUploader.finishSession(sessionId);
this.archive.sessionEnded(sessionId);
- this.observers.lastMessageIds.delete(sessionId);
- this.sessionRoots.delete(sessionId);
+ this.forgetSession(sessionId);
}
/** The account is gone: queued archives and spooled uploads are deleted, nothing more is archived. */
diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.ts
index fea734f9..dee21eb9 100644
--- a/apps/server/src/capture-protocol.ts
+++ b/apps/server/src/capture-protocol.ts
@@ -7,7 +7,7 @@
* logs and closed sessions. Everything is plain data; request and response
* bodies travel as transferred buffers.
*/
-import type { CaptureHost, EngineReplacement, EngineTarget, PromptRecord } from "./capture-host.js";
+import type { CaptureHost, EngineReplacement, EngineTarget, LateWritesGate, PromptRecord } from "./capture-host.js";
import type { FolderGateOptions } from "./session-archive/detect.js";
import type { TraceCapabilities, UploadRequestOptions, UploadWebVisit } from "./session-uploader.js";
@@ -24,7 +24,7 @@ export type CaptureCalls = {
captureSnapshot: [sessionId: string, trigger: "prompt" | "turn_completed", outcome?: "aborted"];
recordWebVisit: [sessionId: string, visit: UploadWebVisit];
archiveSessionStarted: [sessionId: string, root: string, target: EngineTarget];
- archiveStartGate: [sessionId: string, root: string];
+ archiveStartGate: [sessionId: string, root: string, lateWrites?: LateWritesGate | null];
observeSession: [sessionId: string, target: EngineTarget];
engineReplaced: [closedBaseUrl: string, replacement: EngineReplacement];
sessionDeleted: [sessionId: string];
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index 9a7a2248..b7eb0926 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -196,6 +196,7 @@ import { CloudProviderSync, parseCloudProviderDenSession } from "./cloud-provide
import { createEngineV2Preview, type EngineV2Preview } from "./engine-v2-preview.js";
import { sessionUploaderEnabled } from "./session-uploader.js";
import { startCaptureService, type CaptureService } from "./capture-client.js";
+import type { LateWritesGate } from "./capture-host.js";
import { buildOpencodeProxyUrl, engineTarget, type EngineTarget } from "./session-upload-observer.js";
import { OmniRushGatewayBroker } from "./omnirush-gateway-broker.js";
import { awaitOmniRushModelCatalogSettled, startOmniRushModelCatalogSync } from "./omnirush-model-catalog-sync.js";
@@ -1601,6 +1602,16 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
};
}
+/**
+ * A prompt's wait for the turn before's background processes (capture-host.ts
+ * archiveStartGate): only for a local workspace outside a sandbox, whose
+ * processes run on this machine. The engines' processes are never waited for.
+ */
+function lateWritesGate(workspace: WorkspaceInfo, enginePids: ReadonlyArray<number | null> = []): LateWritesGate | null {
+ if (workspace.workspaceType === "remote" || workspace.sandboxBackend || workspace.sandboxRunId || workspace.sandboxContainerName) return null;
+ return { exclude: enginePids.filter((pid): pid is number => typeof pid === "number" && pid > 0) };
+}
+
/**
* The engine v2 preview mount. A prompt dispatch here is gated and captured
* exactly like one on the v1 mount: refused without a connected account
@@ -1765,8 +1776,9 @@ export async function proxyOpencodeV2Request(input: {
: null;
const engine = capture && uploadedSessionId ? engineTarget(input.connection.url, headers, target.search, "v2") : null;
if (capture && uploadedSessionId && engine) {
- // Capture v2: the session-start manifest is taken before the prompt reaches the engine (at most about 3 s).
- await capture.archiveStartGate(uploadedSessionId, input.workspace.path);
+ // The turn before's background processes finish writing (at most 10 s after it ended), then
+ // capture v2's session-start manifest is taken before the prompt reaches the engine (at most about 3 s).
+ await capture.archiveStartGate(uploadedSessionId, input.workspace.path, lateWritesGate(input.workspace));
capture.startSession(uploadedSessionId, input.workspace.id, input.workspace.path);
capture.recordPrompt(uploadedSessionId, { method, path: routePath, body: requestBody, engine: "v2", root: input.workspace.path, attachments: true, dispatchedAt: Date.now() });
capture.captureSnapshot(uploadedSessionId, "prompt");
@@ -2132,8 +2144,9 @@ export async function proxyOpencodeRequest(input: {
if (capture?.uploadEnabled && uploadedSessionId && workspace && workspace.workspaceType !== "remote"
&& (capture.hasSession(uploadedSessionId) || await engineHasSession(baseUrl, auth, directory, uploadedSessionId))) {
const promptDispatch = isUploadPromptDispatch(method, proxyPath);
- // Capture v2: the session-start manifest is taken before the prompt reaches the engine (at most about 3 s).
- if (promptDispatch) await capture.archiveStartGate(uploadedSessionId, workspace.path);
+ // The turn before's background processes finish writing (at most 10 s after it ended), then
+ // capture v2's session-start manifest is taken before the prompt reaches the engine (at most about 3 s).
+ if (promptDispatch) await capture.archiveStartGate(uploadedSessionId, workspace.path, lateWritesGate(workspace, pool?.snapshot().generations.map((generation) => generation.pid) ?? []));
capture.startSession(uploadedSessionId, workspace.id, workspace.path);
capture.recordPrompt(uploadedSessionId, {
method,
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index a49f62f9..642437cd 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -50,6 +50,7 @@ import {
type ArchiveKind,
type ArchiveTrigger,
type ExcludedCounts,
+ type ArchiveReason,
type FinalReason,
type ScannedEntry,
} from "./manifest.js";
@@ -133,7 +134,7 @@ import {
} from "./upload.js";
export { gitMarkerDetector, gitParentDetector, isArchivableProject, type ArchivableProject, type ProjectMarkerDetector } from "./detect.js";
-export { isArchiveCredentialPath, type ArchiveTrigger, type FinalReason } from "./manifest.js";
+export { isArchiveCredentialPath, type ArchiveReason, type ArchiveTrigger, type FinalReason } from "./manifest.js";
export type { ArchiveApiRequest } from "./upload.js";
/** Subdirectory of the session uploader state dir that holds everything the archiver keeps. */
@@ -236,7 +237,7 @@ export type CaptureSkipReason =
/** Packing paused before commit because the configured disk budget is full. */
export type CaptureResult =
- | { status: "queued"; archiveId: string; kind: ArchiveKind; sequence: number; size: number }
+ | { status: "queued"; archiveId: string; kind: ArchiveKind; sequence: number; size: number; late?: string[] }
| { status: "paused"; reason: "disk_budget"; requiredBytes: number; pendingBytes: number; freeBytes: number | null }
| { status: "skipped"; reason: CaptureSkipReason };
@@ -384,15 +385,65 @@ function markerAllowed(marker: string, policy: ArchivePolicy): boolean {
type CaptureOptions = {
/** Deltas: a completed turn, or a final archive with its reason (a touched-files base a final archive captures says "final" too, without it in its manifest). */
trigger?: ArchiveTrigger;
- reason?: FinalReason;
+ reason?: ArchiveReason;
/** A final archive of a session deleted in the app: its record is marked ended. */
ended?: boolean;
/** Cancels the capture until its commit. */
signal?: AbortSignal;
/** Capture v2: the turn a pre_tool state belongs to (the turn in progress). */
stateTurn?: number;
+ /**
+ * Capture v2, pre_tool: when the turn's first tool call started. A file
+ * changed at or after it is the tool's own write: the state keeps the
+ * chain's copy (none for a new file), and the turn's delta carries it.
+ */
+ before?: number;
+ /** Capture v2, pre_tool: a file changed after this is a late write of the turn before (manifest reason "late_writes"). */
+ lateAfter?: number;
};
+/** Options of a pre_tool state (captureState). */
+export type StateCaptureOptions = { before?: number; lateAfter?: number };
+
+/** The modification time in milliseconds a scanned entry's stat key holds (`size:mtimeNs:ctimeNs:ino`), or null. */
+function entryMtimeMs(entry: ScannedEntry): number | null {
+ const parts = entry.statKey.split(":");
+ if (parts.length < 2 || !/^\d+$/.test(parts[1]!)) return null;
+ return Number(BigInt(parts[1]!) / 1_000_000n);
+}
+
+/**
+ * A pre_tool state's scan as of `before`: an entry changed at or after it
+ * (the first tool's own write, racing the scan) takes the chain's entry
+ * (`held`), or is left out when the chain has none. Returns the paths it
+ * reverted.
+ */
+function scanAsOf(entries: ScannedEntry[], held: ReadonlyMap<string, ArchiveEntry>, before: number): string[] {
+ const reverted: string[] = [];
+ for (let index = entries.length - 1; index >= 0; index -= 1) {
+ const entry = entries[index]!;
+ if (entry.type !== "file" || entry.source !== undefined || entry.content !== undefined) continue;
+ const mtime = entryMtimeMs(entry);
+ if (mtime === null || mtime < before) continue;
+ const previous = held.get(entry.path);
+ if (previous && previous.sha256 === entry.sha256) continue;
+ reverted.push(entry.path);
+ // Not in the delta (it equals the chain's entry): nothing reads it; a blank stat key matches no file.
+ if (previous) entries[index] = { ...previous, mtime: entry.mtime, statKey: "", dev: entry.dev };
+ else entries.splice(index, 1);
+ }
+ return reverted;
+}
+
+/** The files of a delta changed after `after`: late writes of the turn before. */
+function lateFiles(files: readonly ScannedEntry[], after: number): string[] {
+ return files.filter((entry) => {
+ if (entry.type !== "file" || entry.source !== undefined) return false;
+ const mtime = entryMtimeMs(entry);
+ return mtime !== null && mtime > after;
+ }).map((entry) => entry.path);
+}
+
/** Capture v2: the session-start scan taken before the first prompt reached the model (startManifest). */
type Prescan = {
root: string;
@@ -909,7 +960,7 @@ export class SessionArchiver {
* turn (like a final archive), `trigger: "pre_tool"`; `stateTurn` is the
* turn in progress. Only for a v2 chain with a base.
*/
- captureState(sessionId: string, stateTurn: number): Promise<CaptureResult> {
+ captureState(sessionId: string, stateTurn: number, options: StateCaptureOptions = {}): Promise<CaptureResult> {
return this.guard("delta", sessionId, stateTurn, async () => this.withSession(sessionId, async () => {
const generation = this.generation;
if (this.disabled || !this.uploader.configured) return { status: "skipped", reason: "disabled" };
@@ -921,7 +972,7 @@ export class SessionArchiver {
const key = await this.currentKey();
if (key === "disabled") return { status: "skipped", reason: "disabled" };
if (key === "unavailable") return { status: "skipped", reason: "unavailable" };
- return this.captureArchive(state, "delta", state.last_turn, key, generation, { trigger: "pre_tool", stateTurn });
+ return this.captureArchive(state, "delta", state.last_turn, key, generation, { trigger: "pre_tool", stateTurn, ...options });
}));
}
@@ -1414,6 +1465,7 @@ export class SessionArchiver {
mergeScans(scan, await this.scanOutside(state, baseline, signal, filesUsed));
if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));
if (filesUsed) mergeStaged(scan, await this.filesUsed.entries(sessionId));
+ if (options.trigger === "pre_tool" && options.before !== undefined) scanAsOf(scan.entries, new Map(baseline.map((entry) => [entry.path, entry])), options.before);
const change = touchedChange(baseline, scan);
if (change.files.length === 0 && change.deleted.length === 0 && !forced) {
if (cache.changed) await this.saveHashCache(rootKey, cache);
@@ -1449,6 +1501,7 @@ export class SessionArchiver {
enclosing = found.repo;
mergeScans(scan, found);
}
+ if (options.trigger === "pre_tool" && options.before !== undefined) scanAsOf(scan.entries, new Map(baseline.map((entry) => [entry.path, entry])), options.before);
fullScan = scan.entries;
files = scan.entries;
excluded = scan.excluded;
@@ -1466,6 +1519,9 @@ export class SessionArchiver {
}
git = withGit ? await (gitReading ?? readArchiveGit(state.root)) : null;
}
+ // A pre_tool state holding files written after the turn before ended: late writes of that turn.
+ const late = options.trigger === "pre_tool" && options.lateAfter !== undefined ? lateFiles(files, options.lateAfter) : [];
+ const reason = options.reason ?? (late.length > 0 ? "late_writes" as const : undefined);
const archiveId = this.random.uuid();
const sequence = state.next_sequence;
const createdAt = this.now();
@@ -1509,7 +1565,7 @@ export class SessionArchiver {
createdAt,
parentArchiveId: kind === "base" ? null : state.last_archive_id,
...(kind === "delta" && options.trigger ? { trigger: options.trigger } : {}),
- ...(kind === "delta" && options.reason ? { reason: options.reason } : {}),
+ ...(kind === "delta" && reason ? { reason } : {}),
...(touched ? { scope: "touched" as const } : {}),
label: archiveLabel(state.root),
marker: state.marker,
@@ -1648,7 +1704,7 @@ export class SessionArchiver {
...(ignored > 0 ? { gitignored: ignored } : {}),
...(unstable.size > 0 ? { unstable: unstable.size } : {}),
});
- return { status: "queued", archiveId, kind, sequence, size: sealed.size };
+ return { status: "queued", archiveId, kind, sequence, size: sealed.size, ...(late.length > 0 ? { late: late.slice(0, 500) } : {}) };
} catch (error) {
await rm(partial, { force: true });
if (error instanceof DiskBudgetPause) return {
diff --git a/apps/server/src/session-archive/late-writes.test.ts b/apps/server/src/session-archive/late-writes.test.ts
new file mode 100644
index 00000000..99a7369f
--- /dev/null
+++ b/apps/server/src/session-archive/late-writes.test.ts
@@ -0,0 +1,549 @@
+import { afterAll, afterEach, describe, expect, test } from "bun:test";
+import { execFileSync, spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { zstdDecompressSync } from "node:zlib";
+
+import { CaptureHost } from "../capture-host.js";
+import {
+ createSessionObservers,
+ observeUploadedSession,
+ turnEndedAt,
+ type EngineTarget,
+ type ObservedUploader,
+ type ObserverTiming,
+} from "../session-upload-observer.js";
+import { SessionArchiver, type CaptureResult } from "./index.js";
+import { lateWriters, settleLateWriters } from "./late-writes.js";
+import { ProjectArchiveLifecycle, promptTurnAt, type ArchiveEngineReads, type LateWrites, type ProjectArchiver } from "./lifecycle.js";
+import { openSealedBuffer, SEAL_ALG, sealKeyring } from "./seal.js";
+import { readTar } from "./test-helpers.js";
+import { toolStartTime, watchToolStarts } from "./tool-start.js";
+import { turnWriters } from "./turn-writers.js";
+
+// Turn boundaries, the desktop's side of omnirush-cli's late-writes test: a
+// turn's start is taken from the workspace as its first tool call found it,
+// after the turn before's background processes finished writing; what they
+// wrote after that turn ended is marked late; and the files the root session
+// and its sub-agents both wrote in a turn are named.
+
+const cleanups: string[] = [];
+const stops: Array<() => Promise<void> | void> = [];
+afterEach(async () => {
+ while (stops.length) await stops.pop()?.();
+});
+afterAll(() => {
+ for (const dir of cleanups) rmSync(dir, { recursive: true, force: true });
+});
+
+const linux = process.platform === "linux";
+const sleep = (ms: number) => new Promise<void>((resolvePromise) => setTimeout(resolvePromise, ms));
+function git(cwd: string, ...args: string[]): string {
+ return execFileSync("git", ["-C", cwd, "-c", "[email protected]", "-c", "user.name=t", "-c", "init.defaultBranch=main", "-c", "commit.gpgSign=false", ...args], { stdio: ["ignore", "pipe", "pipe"] }).toString();
+}
+function temp(prefix: string): string {
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
+ cleanups.push(dir);
+ return dir;
+}
+function write(file: string, content: string, mtimeMs?: number): void {
+ mkdirSync(dirname(file), { recursive: true });
+ writeFileSync(file, content);
+ if (mtimeMs !== undefined) utimesSync(file, mtimeMs / 1000, mtimeMs / 1000);
+}
+
+type Entry = { path: string };
+type Opened = { manifest: { reason?: string; files: Entry[] }; state: { state: { kind: string; turn: number; reason?: string } } };
+
+function rig() {
+ const stateDir = temp("omnirush-late-state-");
+ const keyring = sealKeyring([randomBytes(32)]);
+ const { publicRaw, kid } = [...keyring.values()][0]!;
+ const request = async (route: string) => route === "archives/key"
+ ? Response.json({ public_key: publicRaw.toString("base64"), kid, alg: SEAL_ALG, policy: { all_folders: true, touched_files: true, capture_v2: true } })
+ : new Response("{}", { status: 404 });
+ const archiver = new SessionArchiver({ stateDir, request, excludedDirs: [stateDir], log: () => undefined, touchedFlushMs: 10 });
+ stops.push(() => archiver.stop({ budgetMs: 1000 }));
+ const open = async (result: CaptureResult): Promise<Opened> => {
+ if (result.status !== "queued") throw new Error(`not queued: ${JSON.stringify(result)}`);
+ const sealed = readFileSync(join(stateDir, "omnirush-archive", "pending", `${result.archiveId}.orseal`));
+ const members = readTar(zstdDecompressSync(await openSealedBuffer(sealed, keyring)));
+ const json = (name: string) => JSON.parse(members.find((member) => member.name === name)!.content.toString("utf8"));
+ return { manifest: json("__omnirush__/manifest.json"), state: json("__omnirush__/state.json") };
+ };
+ return { archiver, open };
+}
+
+/** A project archiver whose pre_tool states report one late file; every capture is recorded. */
+function fakeArchiver(calls: unknown[][]): ProjectArchiver {
+ return {
+ captureBase: async (_id: string, _root: string, turn = 0) => (calls.push(["base", turn]), { status: "queued", archiveId: "b", kind: "base", sequence: 0, size: 1 }),
+ captureDelta: async (_id: string, _root: string, turn: number | null) => (calls.push(["delta", turn]), { status: "queued", archiveId: "d", kind: "delta", sequence: 1, size: 1 }),
+ captureFinal: async () => ({ status: "skipped", reason: "unchanged" }),
+ startFinalCandidates: async () => [],
+ recordTouched: () => undefined,
+ forgetTouched: () => undefined,
+ drain: async () => ({ uploaded: 0, pending: 0, dropped: 0, blocked: null, disabled: false }),
+ signOut: async () => undefined,
+ stop: async () => undefined,
+ startManifest: async () => "complete",
+ hasStartManifest: () => true,
+ captureState: async (_id: string, stateTurn: number, options?: unknown) => (calls.push(["pre_tool", stateTurn, options]), { status: "queued", archiveId: "p", kind: "delta", sequence: 2, size: 1, late: ["late.txt"] }),
+ } as ProjectArchiver;
+}
+
+type Message = { info: Record<string, unknown>; parts: unknown[] };
+
+describe("late writes: the archive", () => {
+ test("a pre_tool state is the folder as the first tool call found it: late writes in, marked; the tool's own writes out", async () => {
+ const { archiver, open } = rig();
+ const root = temp("omnirush-late-ws-");
+ write(join(root, "a.txt"), "turn one\n");
+ write(join(root, "edit.txt"), "v1\n");
+ git(root, "init", "-q");
+ git(root, "add", ".");
+ git(root, "commit", "-qm", "init");
+ const sessionId = "ses_late_pre_tool";
+ await archiver.startManifest(sessionId, root);
+ await open(await archiver.captureBase(sessionId, root, 0));
+ const ended = Date.now() - 5_000;
+ const toolStart = Date.now() - 1_000;
+ // The turn before's background job, after that turn ended and before this turn's first tool call.
+ write(join(root, "late.txt"), "late from turn 1\n", toolStart - 2_000);
+ // The first tool call's own writes, racing the state: a new file and an edit.
+ write(join(root, "tool.txt"), "written by the tool\n", toolStart + 10);
+ write(join(root, "edit.txt"), "v2 by the tool\n", toolStart + 20);
+ const result = await archiver.captureState(sessionId, 2, { before: toolStart, lateAfter: ended });
+ expect(result.status === "queued" ? result.late : null).toEqual(["late.txt"]);
+ const pre = await open(result);
+ expect(pre.state.state).toEqual({ kind: "pre_tool", turn: 2 });
+ expect(pre.manifest.reason).toBe("late_writes");
+ // The tool's own writes are left to the turn's delta.
+ expect(pre.manifest.files.map((entry) => entry.path)).toEqual(["late.txt"]);
+ // The turn's delta then carries them.
+ const after = await open(await archiver.captureDelta(sessionId, root, 2));
+ expect(after.manifest.files.map((entry) => entry.path).sort()).toEqual(["edit.txt", "tool.txt"]);
+ expect(after.manifest.reason).toBeUndefined();
+ });
+
+ test("a pre_tool state with no late write carries no reason", async () => {
+ const { archiver, open } = rig();
+ const root = temp("omnirush-late-ws-");
+ write(join(root, "a.txt"), "x\n");
+ const sessionId = "ses_late_none";
+ await archiver.startManifest(sessionId, root);
+ await open(await archiver.captureBase(sessionId, root, 0));
+ const ended = Date.now();
+ write(join(root, "user.txt"), "the user's edit before the turn ended\n", ended - 3_000);
+ const result = await archiver.captureState(sessionId, 2, { before: Date.now() + 1_000, lateAfter: ended });
+ const pre = await open(result);
+ expect(result.status === "queued" ? result.late : "not queued").toBeUndefined();
+ expect(pre.manifest.reason).toBeUndefined();
+ });
+});
+
+describe("late writes: the lifecycle", () => {
+ test("a prompt sent before the turn before settled is a turn of its own; its pre_tool state is cut at the tool's start and late after the turn before ended", async () => {
+ const calls: unknown[][] = [];
+ const reported: LateWrites[] = [];
+ let clock = 1_000_000;
+ const prompts: Message[] = [];
+ const engine: ArchiveEngineReads = { session: async () => ({ id: "ses_quick" }), messages: async () => prompts };
+ const lifecycle = new ProjectArchiveLifecycle({ archiver: fakeArchiver(calls), enabled: true, log: () => undefined, now: () => clock, onLateWrites: (item) => reported.push(item) });
+ stops.push(() => lifecycle.stop({ finals: false }));
+ prompts.push({ info: { id: "u1", role: "user", time: { created: clock } }, parts: [{ type: "text", text: "one" }] });
+ lifecycle.sessionStarted({ sessionId: "ses_quick", root: tmpdir(), engine });
+ clock += 100;
+ lifecycle.toolStarted("ses_quick", clock);
+ await lifecycle.settled();
+ expect(calls.at(-1)).toEqual(["pre_tool", 1, { before: 1_000_100 }]);
+ // Turn 1 settles (its end), then a second prompt; and a third one sent into turn 2 before it settled.
+ clock += 1_000;
+ const answered = [...prompts, { info: { id: "a1", role: "assistant", time: { completed: clock } }, parts: [{ type: "step-finish" }] }];
+ lifecycle.turnCompleted("ses_quick", answered);
+ prompts.push(answered[1]!);
+ const endedAt = clock;
+ clock += 2_000;
+ prompts.push({ info: { id: "u2", role: "user", time: { created: clock } }, parts: [{ type: "text", text: "two" }] });
+ lifecycle.sessionStarted({ sessionId: "ses_quick", root: tmpdir(), engine });
+ clock += 500;
+ lifecycle.toolStarted("ses_quick", clock);
+ await lifecycle.settled();
+ expect(calls.at(-1)).toEqual(["pre_tool", 2, { before: clock, lateAfter: endedAt }]);
+ expect(reported.at(-1)).toEqual({ sessionId: "ses_quick", turn: 2, after: endedAt, toolStartAt: clock, paths: ["late.txt"] });
+ const thirdAt = clock + 100;
+ clock = thirdAt;
+ prompts.push({ info: { id: "u3", role: "user", time: { created: clock } }, parts: [{ type: "text", text: "three" }] });
+ lifecycle.sessionStarted({ sessionId: "ses_quick", root: tmpdir(), engine });
+ clock += 300;
+ lifecycle.toolStarted("ses_quick", clock);
+ await lifecycle.settled();
+ // Turn 2 never settled (no turnCompleted): turn 3 by the engine's prompts, late after its own prompt.
+ expect(calls.at(-1)).toEqual(["pre_tool", 3, { before: clock, lateAfter: thirdAt }]);
+ });
+
+ test("a turn the next prompt cut settles after that prompt: its writes are late from the prompt on", async () => {
+ const calls: unknown[][] = [];
+ let clock = 2_000_000;
+ const prompts: Message[] = [];
+ const engine: ArchiveEngineReads = { session: async () => ({ id: "ses_cut" }), messages: async () => prompts };
+ const lifecycle = new ProjectArchiveLifecycle({ archiver: fakeArchiver(calls), enabled: true, log: () => undefined, now: () => clock });
+ stops.push(() => lifecycle.stop({ finals: false }));
+ prompts.push({ info: { id: "u1", role: "user", time: { created: clock } }, parts: [{ type: "text", text: "one" }] });
+ lifecycle.sessionStarted({ sessionId: "ses_cut", root: tmpdir(), engine });
+ clock += 100;
+ lifecycle.toolStarted("ses_cut", clock);
+ await lifecycle.settled();
+ // The second prompt goes out first; the observer settles turn 1 (cut) a moment later.
+ clock += 1_000;
+ const secondAt = clock;
+ prompts.push({ info: { id: "u2", role: "user", time: { created: clock } }, parts: [{ type: "text", text: "two" }] });
+ lifecycle.sessionStarted({ sessionId: "ses_cut", root: tmpdir(), engine });
+ clock += 400;
+ lifecycle.turnCompleted("ses_cut", prompts.slice(0, 1));
+ clock += 200;
+ lifecycle.toolStarted("ses_cut", clock);
+ await lifecycle.settled();
+ expect(calls.at(-1)).toEqual(["pre_tool", 2, { before: clock, lateAfter: secondAt }]);
+ });
+
+ test("without readable messages a pre_tool state takes the completed turns plus one", async () => {
+ const calls: unknown[][] = [];
+ let reads = 0;
+ // Readable at the session's start, not at its tool call.
+ const engine: ArchiveEngineReads = {
+ session: async () => ({ id: "ses_unread" }),
+ messages: async () => {
+ if ((reads += 1) > 1) throw new Error("down");
+ return [];
+ },
+ };
+ const lifecycle = new ProjectArchiveLifecycle({ archiver: fakeArchiver(calls), enabled: true, log: () => undefined });
+ stops.push(() => lifecycle.stop({ finals: false }));
+ lifecycle.sessionStarted({ sessionId: "ses_unread", root: tmpdir(), engine });
+ lifecycle.toolStarted("ses_unread", 5);
+ await lifecycle.settled();
+ expect(calls.at(-1)?.[1]).toBe(1);
+ });
+
+ test("promptTurnAt counts the prompts sent up to a moment, not synthetic or compaction messages", () => {
+ const messages = [
+ { info: { role: "user", time: { created: 10 } }, parts: [{ type: "text", text: "a" }] },
+ { info: { role: "assistant", time: { created: 11, completed: 12 } }, parts: [] },
+ { info: { role: "user", time: { created: 13 } }, parts: [{ type: "text", text: "background result", synthetic: true }] },
+ { info: { role: "user", agent: "compaction", time: { created: 14 } }, parts: [{ type: "text", text: "summarize" }] },
+ { info: { role: "user", time: { created: 20 } }, parts: [{ type: "text", text: "b" }] },
+ { info: { role: "user", time: { created: 30 } }, parts: [{ type: "text", text: "c" }] },
+ ];
+ expect(promptTurnAt(messages, 25)).toBe(2);
+ expect(promptTurnAt(messages, 30)).toBe(3);
+ expect(promptTurnAt(messages, 5)).toBeNull();
+ expect(promptTurnAt(null, 5)).toBeNull();
+ });
+});
+
+describe("late writes: tool start", () => {
+ test("the tool-start time: the event's own time, the earliest of the event and the part's start", () => {
+ expect(toolStartTime({ type: "message.part.updated", properties: { part: { type: "tool", state: { status: "pending" } }, time: 1_791_734_333_976 } })).toBe(1_791_734_333_976);
+ expect(toolStartTime({ type: "message.part.updated", properties: { part: { type: "tool", state: { status: "running", time: { start: 1_791_734_333_981 } } }, time: 1_791_734_333_990 } })).toBe(1_791_734_333_981);
+ expect(toolStartTime({ payload: { type: "session.tool.called", data: { sessionID: "s", timestamp: "2026-10-11T16:00:00.000Z" } } })).toBe(Date.parse("2026-10-11T16:00:00.000Z"));
+ // Not epoch milliseconds.
+ expect(toolStartTime({ type: "message.part.updated", properties: { time: 12 } })).toBeNull();
+ expect(toolStartTime(null)).toBeNull();
+ });
+
+ test("the tool-start hook passes when the call started", async () => {
+ const frame = { type: "message.part.updated", properties: { part: { type: "tool", sessionID: "ses_t", state: { status: "pending" } }, time: 1_791_000_000_000 } };
+ const server = http.createServer((_req, res) => {
+ res.writeHead(200, { "content-type": "text/event-stream" });
+ res.write(`data: ${JSON.stringify(frame)}\n\n`);
+ });
+ await new Promise<void>((resolvePromise) => server.listen(0, "127.0.0.1", resolvePromise));
+ stops.push(() => { server.closeAllConnections(); server.close(); });
+ const seen: number[] = [];
+ await watchToolStarts({
+ url: `http://127.0.0.1:${(server.address() as AddressInfo).port}/event`,
+ headers: new Headers(),
+ sessionId: "ses_t",
+ signal: AbortSignal.timeout(5_000),
+ fetch: (url, init) => fetch(url, init),
+ onToolStart: (at) => seen.push(at),
+ });
+ expect(seen).toEqual([1_791_000_000_000]);
+ });
+});
+
+/** Starts `script` in `cwd` in a shell that exits at once: the job is orphaned, as a tool's `cmd &` is once its shell ends. Returns the job's pid. */
+function orphan(cwd: string, script: string): number {
+ const pid = Number(execFileSync("sh", ["-c", `(${script}) >/dev/null 2>&1 </dev/null & echo $!`], { cwd }).toString().trim());
+ stops.push(() => {
+ try {
+ process.kill(pid, "SIGKILL");
+ } catch {
+ // Gone.
+ }
+ });
+ return pid;
+}
+
+describe("late writes: the processes", () => {
+ test.skipIf(!linux)("late writers: the turn's background processes in the workspace, waited for until they exit, never past the cap", async () => {
+ const root = temp("omnirush-late-proc-");
+ const outside = temp("omnirush-late-out-");
+ const since = Date.now() - 1_000;
+ const job = spawn("sh", ["-c", "sleep 1.2; echo late > late.txt"], { cwd: root, detached: true, stdio: "ignore" });
+ const elsewhere = spawn("sh", ["-c", "sleep 5"], { cwd: outside, detached: true, stdio: "ignore" });
+ const excluded = spawn("sh", ["-c", "sleep 5"], { cwd: root, detached: true, stdio: "ignore" });
+ stops.push(() => {
+ for (const child of [job, elsewhere, excluded]) {
+ try {
+ process.kill(child.pid!, "SIGKILL");
+ } catch {
+ // Gone.
+ }
+ }
+ });
+ await sleep(100);
+ const found = await lateWriters({ root, since, exclude: [excluded.pid!] });
+ expect(found.some((item) => item.pid === job.pid)).toBe(true);
+ // Working outside the workspace.
+ expect(found.some((item) => item.pid === elsewhere.pid)).toBe(false);
+ // Excluded (the engine's tree).
+ expect(found.some((item) => item.pid === excluded.pid)).toBe(false);
+ // Started before the turn.
+ expect(await lateWriters({ root, since: Date.now() + 60_000, exclude: [] })).toEqual([]);
+
+ const started = Date.now();
+ const settled = await settleLateWriters({ root, since, exclude: [excluded.pid!], endedAt: Date.now(), capMs: 5_000 });
+ expect(settled.settled).toBe(true);
+ expect(settled.processes.some((item) => item.pid === job.pid)).toBe(true);
+ // The late write landed before the wait ended.
+ expect(readFileSync(join(root, "late.txt"), "utf8")).toBe("late\n");
+ expect(Date.now() - started).toBeLessThan(4_000);
+
+ const capped = Date.now();
+ const stuck = await settleLateWriters({ root, since, exclude: [], endedAt: Date.now() - 9_700, capMs: 10_000 });
+ // A process still running at the cap (a dev server).
+ expect(stuck.settled).toBe(false);
+ // The cap counts from the turn's end.
+ expect(Date.now() - capped).toBeLessThan(1_500);
+ // Linux only.
+ expect(await lateWriters({ root, since, exclude: [], platform: "darwin" })).toEqual([]);
+ });
+});
+
+describe("late writes: turn writers", () => {
+ test("the files the root and a sub-agent both wrote are the overlap; none without a sub-agent's write", () => {
+ const root = "/work/app";
+ const tool = (name: string, input: Record<string, unknown>, start: number, end: number) => ({ type: "tool", tool: name, state: { status: "completed", input, time: { start, end } } });
+ const rootMessages = [{ info: { role: "assistant" }, parts: [
+ tool("write", { filePath: "/work/app/shared.txt" }, 1_000, 1_001),
+ tool("task", { prompt: "help" }, 1_002, 1_500),
+ tool("apply_patch", { patchText: "*** Begin Patch\n*** Update File: shared.txt\n@@\n-a\n+b\n*** Add File: root-only.txt\n+x\n*** End Patch" }, 1_600, 1_601),
+ tool("bash", { command: "echo > ignored.txt" }, 1_700, 1_701),
+ ] }];
+ const child = [{ info: { role: "assistant" }, parts: [
+ tool("write", { filePath: "sub.txt" }, 1_100, 1_101),
+ tool("edit", { filePath: "/work/app/shared.txt" }, 1_200, 1_201),
+ tool("write", { filePath: "/elsewhere/out.txt" }, 1_300, 1_301),
+ { type: "tool", tool: "write", state: { status: "error", input: { filePath: "failed.txt" } } },
+ ] }];
+ const result = turnWriters(root, [{ sessionId: "ses_root", role: "root", depth: 0, messages: rootMessages }, { sessionId: "ses_child", role: "child", depth: 1, messages: child }])!;
+ expect(result.overlap).toEqual(["shared.txt"]);
+ expect(result.files.map((file) => file.path)).toEqual(["root-only.txt", "shared.txt", "sub.txt"]);
+ const shared = result.files.find((file) => file.path === "shared.txt")!;
+ expect(shared.sessions.map((s) => [s.session_id, s.role, s.calls])).toEqual([["ses_root", "root", 2], ["ses_child", "child", 1]]);
+ expect(shared.sessions[1]!.first_at).toBe(new Date(1_200).toISOString());
+ expect(turnWriters(root, [{ sessionId: "ses_root", role: "root", depth: 0, messages: rootMessages }])).toBeNull();
+ });
+});
+
+const SESSION = "ses_late_observed";
+
+/** A fake v1 engine holding one session (and its children): its status and messages. */
+function startEngine() {
+ const control = {
+ status: "busy" as string,
+ messages: [] as Message[],
+ children: [] as Array<{ id: string }>,
+ childMessages: {} as Record<string, Message[]>,
+ };
+ const server = Bun.serve({
+ hostname: "127.0.0.1",
+ port: 0,
+ fetch(request) {
+ const url = new URL(request.url);
+ if (url.pathname === "/session/status") return Response.json(control.status === "idle" ? {} : { [SESSION]: { type: control.status } });
+ if (url.pathname === `/session/${SESSION}/message`) return Response.json(control.messages);
+ if (url.pathname === `/session/${SESSION}/children`) return Response.json(control.children);
+ const child = /^\/session\/([^/]+)\/(message|children)$/.exec(url.pathname);
+ if (child && child[1]! in control.childMessages) return Response.json(child[2] === "message" ? control.childMessages[child[1]!] : []);
+ if (url.pathname === `/session/${SESSION}`) return Response.json({ id: SESSION });
+ return Response.json({ code: "not_found" }, { status: 404 });
+ },
+ });
+ stops.push(() => server.stop(true));
+ const target: EngineTarget = { baseUrl: `http://127.0.0.1:${server.port}`, headers: [], search: "", engine: "v1" };
+ return { control, target };
+}
+
+/** A clock that moves only when the observer waits (each wait yields to the event loop for a moment). */
+function stepClock() {
+ let now = Date.parse("2026-10-11T12:00:00.000Z");
+ const timing: Partial<ObserverTiming> = {
+ now: () => now,
+ sleep: async (ms, signal) => {
+ if (signal.aborted) throw signal.reason;
+ now += ms;
+ await sleep(5);
+ if (signal.aborted) throw signal.reason;
+ },
+ };
+ return { timing, now: () => now };
+}
+
+class TraceUploader implements ObservedUploader {
+ readonly enabled = true;
+ readonly traces: Array<{ type: string; data?: unknown }> = [];
+ async sessionCheckpoint() { return { resumed: false }; }
+ async setSessionCheckpoint() {}
+ recordTrace(_sessionId: string, type: string, data?: unknown) { this.traces.push({ type, data }); }
+ recordSessionModel() {}
+ async childCheckpoints() { return {}; }
+ async childSessionIds() { return []; }
+ recordChildSession() {}
+ noteChildSessionIds(_sessionId: string, ids: readonly string[]) { return ids.length; }
+ captureSnapshot() {}
+ flushTrace() {}
+}
+
+const fakeObservedArchive = () => ({ turnFollowed: () => undefined, turnCompleted: () => undefined, turnIncomplete: () => undefined });
+
+describe("late writes: the observer", () => {
+ test("the turn's end: none while it runs, the first idle read once the engine stops, kept once it settles", async () => {
+ const engine = startEngine();
+ const clock = stepClock();
+ const observers = createSessionObservers();
+ stops.push(() => observers.controller.abort());
+ engine.control.messages = [{ info: { id: "m1", role: "user", time: { created: 1 } }, parts: [{ type: "text", text: "one" }] }];
+ const watching = observeUploadedSession({ sessionUploader: new TraceUploader(), archive: fakeObservedArchive(), observers, sessionId: SESSION, target: engine.target, timing: clock.timing });
+ while (!observers.sessions.get(SESSION)?.turn) await sleep(5);
+ await sleep(50);
+ // Running: a prompt typed now waits for nothing.
+ expect(turnEndedAt(observers, SESSION)).toBeNull();
+ engine.control.messages.push({ info: { id: "m2", role: "assistant", finish: "stop", time: { created: 2, completed: 3 } }, parts: [{ type: "step-finish" }] });
+ engine.control.status = "idle";
+ let idleAt: number | null = null;
+ while (idleAt === null && observers.sessions.has(SESSION)) {
+ idleAt = turnEndedAt(observers, SESSION);
+ await sleep(1);
+ }
+ expect(idleAt).not.toBeNull();
+ await watching;
+ // Settled: the end is the first idle read, not the settle.
+ expect(turnEndedAt(observers, SESSION)).toBe(idleAt);
+ expect(idleAt!).toBeLessThan(clock.now());
+ });
+
+ test("a settled turn whose sub-agent wrote files records who wrote what (turn.writers)", async () => {
+ const engine = startEngine();
+ const clock = stepClock();
+ const root = temp("omnirush-late-writers-");
+ const tool = (name: string, filePath: string, start: number) => ({ type: "tool", tool: name, state: { status: "completed", input: { filePath }, time: { start, end: start + 1 } } });
+ engine.control.status = "idle";
+ engine.control.messages = [
+ { info: { id: "m1", role: "user", time: { created: 1 } }, parts: [{ type: "text", text: "one" }] },
+ { info: { id: "m2", role: "assistant", finish: "stop", time: { created: 2, completed: 3 } }, parts: [tool("write", join(root, "shared.txt"), 1_000), { type: "step-finish" }] },
+ ];
+ engine.control.children = [{ id: "ses_late_child" }];
+ engine.control.childMessages = { ses_late_child: [{ info: { id: "c1", role: "assistant", time: { created: 2, completed: 3 } }, parts: [tool("edit", join(root, "shared.txt"), 1_200), tool("write", "sub.txt", 1_300)] }] };
+ const uploader = new TraceUploader();
+ const observers = createSessionObservers();
+ stops.push(() => observers.controller.abort());
+ await observeUploadedSession({ sessionUploader: uploader, archive: fakeObservedArchive(), turnFiles: { root }, observers, sessionId: SESSION, target: engine.target, timing: clock.timing });
+ const writers = uploader.traces.filter((trace) => trace.type === "turn.writers").map((trace) => trace.data as { overlap: string[]; files: Array<{ path: string }> });
+ expect(writers).toHaveLength(1);
+ expect(writers[0]!.overlap).toEqual(["shared.txt"]);
+ expect(writers[0]!.files.map((file) => file.path)).toEqual(["shared.txt", "sub.txt"]);
+ });
+});
+
+/** A capture host with an upload hook that accepts everything; its traces are recorded. */
+function captureHost() {
+ const stateDir = temp("omnirush-late-host-");
+ const host = new CaptureHost({
+ stateDir,
+ appVersion: "test",
+ engineVersion: "test",
+ log: () => undefined,
+ sessionUploader: { upload: async () => new Response("{}", { status: 200 }), context: false },
+ archive: { enabled: false, excludedDirs: [] },
+ });
+ stops.push(() => host.stop({ archiveFinals: false }));
+ const traces: Array<{ type: string; data?: unknown }> = [];
+ const record = host.sessionUploader.recordTrace.bind(host.sessionUploader);
+ host.sessionUploader.recordTrace = (sessionId, type, data) => {
+ traces.push({ type, data });
+ record(sessionId, type, data);
+ };
+ return { host, traces };
+}
+
+/** Waits until the observer saw the session idle (a status read every second). */
+async function untilIdleSeen(host: CaptureHost): Promise<void> {
+ const observers = (host as unknown as { observers: ReturnType<typeof createSessionObservers> }).observers;
+ for (let waited = 0; turnEndedAt(observers, SESSION) === null; waited += 50) {
+ if (waited > 8_000) throw new Error("the observer never saw the session idle");
+ await sleep(50);
+ }
+}
+
+describe("late writes: the prompt hold", () => {
+ test.skipIf(!linux)("a prompt after the turn before ended waits for that turn's background job, then goes out", async () => {
+ const engine = startEngine();
+ const root = temp("omnirush-late-hold-");
+ const { host, traces } = captureHost();
+ host.startSession(SESSION, "ws", root);
+ await host.archiveStartGate(SESSION, root, { exclude: [] });
+ host.observeSession(SESSION, engine.target);
+ // The turn's `cmd &`, started a while into the turn (/proc start times are whole-second precise): still writing after the turn ended.
+ await sleep(1_500);
+ const job = orphan(root, "sleep 2.5; echo late > late.txt");
+ engine.control.messages = [{ info: { id: "m1", role: "user", time: { created: 1 } }, parts: [{ type: "text", text: "one" }] }, { info: { id: "m2", role: "assistant", finish: "stop", time: { created: 2, completed: 3 } }, parts: [{ type: "step-finish" }] }];
+ engine.control.status = "idle";
+ await untilIdleSeen(host);
+ expect(existsSync(join(root, "late.txt"))).toBe(false);
+ const started = Date.now();
+ await host.archiveStartGate(SESSION, root, { exclude: [] });
+ expect(existsSync(join(root, "late.txt"))).toBe(true);
+ expect(Date.now() - started).toBeLessThan(5_000);
+ const settle = traces.find((trace) => trace.type === "turn.settle")?.data as { settled: boolean; processes: Array<{ pid: number }> } | undefined;
+ expect(settle?.settled).toBe(true);
+ expect(settle?.processes.some((item) => item.pid === job)).toBe(true);
+ }, 20_000);
+
+ test.skipIf(!linux)("a prompt typed into a running turn, or held without the gate (a sandbox), waits for nothing", async () => {
+ const engine = startEngine();
+ const root = temp("omnirush-late-hold-");
+ const { host, traces } = captureHost();
+ host.startSession(SESSION, "ws", root);
+ await host.archiveStartGate(SESSION, root, { exclude: [] });
+ host.observeSession(SESSION, engine.target);
+ await sleep(1_500);
+ orphan(root, "sleep 8");
+ await sleep(1_200);
+ let started = Date.now();
+ await host.archiveStartGate(SESSION, root, { exclude: [] });
+ expect(Date.now() - started).toBeLessThan(500);
+ engine.control.status = "idle";
+ await untilIdleSeen(host);
+ started = Date.now();
+ await host.archiveStartGate(SESSION, root, null);
+ expect(Date.now() - started).toBeLessThan(500);
+ expect(traces.some((trace) => trace.type === "turn.settle")).toBe(false);
+ }, 20_000);
+});
diff --git a/apps/server/src/session-archive/late-writes.ts b/apps/server/src/session-archive/late-writes.ts
new file mode 100644
index 00000000..65d19ac6
--- /dev/null
+++ b/apps/server/src/session-archive/late-writes.ts
@@ -0,0 +1,194 @@
+/**
+ * Late writes: a turn's background processes still writing after the turn
+ * ended (`cmd &`, `npm install &`, a watcher or a dev server writing a
+ * build folder). When the next prompt arrives, its turn must start from the
+ * workspace those writes leave, not from one taken while they were still
+ * landing. lateWriters() finds the processes a turn left running in the
+ * workspace (Linux /proc; elsewhere none are found and nothing is waited
+ * for), and settleLateWriters() waits, at most until a cap measured from the
+ * turn's end, for them to exit. The wait runs while the prompt is being
+ * dispatched, and only when such a process is still running, so a user who
+ * types for longer than the cap never waits. Identical in the CLI and the
+ * desktop app.
+ */
+import { readdir, readFile, readlink } from "node:fs/promises";
+import { sep } from "node:path";
+
+/** How long after a turn ended its next prompt may wait for the turn's background processes. */
+export const LATE_WRITES_CAP_MS = 10_000;
+/** How often the processes are looked at again while waiting. */
+export const LATE_WRITES_POLL_MS = 200;
+/** At most this many processes are reported. */
+const MAX_REPORTED = 16;
+const MAX_COMMAND_CHARS = 120;
+
+export type LateWriter = { pid: number; command: string };
+
+export type LateWritersInput = {
+ /** The workspace: a process whose working directory is in it is a candidate. */
+ root: string;
+ /** When the turn started (epoch ms): older processes (a dev server from an earlier turn, the user's own shell) are not its. */
+ since: number;
+ /** Processes never waited for, with their descendants: this app, the engine. */
+ exclude: readonly number[];
+ /** Tests: the /proc directory and the platform. */
+ proc?: string;
+ platform?: NodeJS.Platform;
+};
+
+type ProcEntry = { pid: number; ppid: number; startedAt: number; uid: number | null; command: string };
+
+/** Boot time (epoch ms) and clock ticks per second, for /proc/<pid>/stat start times. */
+async function bootTime(proc: string): Promise<number | null> {
+ try {
+ const text = await readFile(`${proc}/stat`, "utf8");
+ const match = /^btime (\d+)$/m.exec(text);
+ return match ? Number(match[1]) * 1000 : null;
+ } catch {
+ return null;
+ }
+}
+
+/** USER_HZ: 100 on every Linux the app runs on. */
+const CLOCK_TICKS = 100;
+
+async function procEntry(proc: string, pid: number, boot: number): Promise<ProcEntry | null> {
+ try {
+ const stat = await readFile(`${proc}/${pid}/stat`, "utf8");
+ // `pid (comm) state ppid ...`: comm may hold spaces and parentheses; the fields follow the last `)`.
+ const close = stat.lastIndexOf(")");
+ const open = stat.indexOf("(");
+ if (close < 0 || open < 0) return null;
+ const fields = stat.slice(close + 2).split(" ");
+ const ppid = Number(fields[1]);
+ const ticks = Number(fields[19]);
+ if (!Number.isFinite(ppid) || !Number.isFinite(ticks)) return null;
+ // A zombie has exited: it writes nothing.
+ if (fields[0] === "Z") return null;
+ let uid: number | null = null;
+ try {
+ const status = await readFile(`${proc}/${pid}/status`, "utf8");
+ const match = /^Uid:\s+(\d+)/m.exec(status);
+ uid = match ? Number(match[1]) : null;
+ } catch {
+ uid = null;
+ }
+ let command = stat.slice(open + 1, close);
+ try {
+ const cmdline = (await readFile(`${proc}/${pid}/cmdline`, "utf8")).split("\0").filter(Boolean).join(" ");
+ if (cmdline) command = cmdline;
+ } catch {
+ // The short name will do.
+ }
+ return { pid, ppid, startedAt: boot + (ticks * 1000) / CLOCK_TICKS, uid, command: command.slice(0, MAX_COMMAND_CHARS) };
+ } catch {
+ return null;
+ }
+}
+
+function inside(root: string, path: string): boolean {
+ const base = root.endsWith(sep) ? root : `${root}${sep}`;
+ return path === root || path.startsWith(base);
+}
+
+/**
+ * The processes a turn left running in the workspace: started since the turn
+ * began, working in the workspace, owned by this user, and neither an
+ * excluded process nor one of its descendants (the engine's own helpers, a
+ * tool call still running). Empty off Linux, or when /proc cannot be read.
+ */
+export async function lateWriters(input: LateWritersInput): Promise<LateWriter[]> {
+ const platform = input.platform ?? process.platform;
+ if (platform !== "linux") return [];
+ const proc = input.proc ?? "/proc";
+ const boot = await bootTime(proc);
+ if (boot === null) return [];
+ let names: string[];
+ try {
+ names = await readdir(proc);
+ } catch {
+ return [];
+ }
+ const uid = typeof process.getuid === "function" ? process.getuid() : null;
+ // Only a process working in the workspace is read in full (another user's cwd cannot even be read).
+ const candidates: number[] = [];
+ await Promise.all(names.filter((name) => /^\d+$/.test(name)).map(async (name) => {
+ const pid = Number(name);
+ if (pid === process.pid) return;
+ try {
+ if (inside(input.root, await readlink(`${proc}/${pid}/cwd`))) candidates.push(pid);
+ } catch {
+ // Gone, or not this user's.
+ }
+ }));
+ if (candidates.length === 0) return [];
+ const entries = new Map<number, ProcEntry | null>();
+ const entryOf = async (pid: number): Promise<ProcEntry | null> => {
+ if (!entries.has(pid)) entries.set(pid, await procEntry(proc, pid, boot));
+ return entries.get(pid) ?? null;
+ };
+ const excluded = new Set(input.exclude.filter((pid) => Number.isSafeInteger(pid) && pid > 0));
+ const excludedTree = async (pid: number): Promise<boolean> => {
+ for (let current: number | undefined = pid, hops = 0; current !== undefined && current > 1 && hops < 64; hops += 1) {
+ if (excluded.has(current)) return true;
+ current = (await entryOf(current))?.ppid;
+ }
+ return false;
+ };
+ const found: LateWriter[] = [];
+ for (const pid of candidates) {
+ const entry = await entryOf(pid);
+ if (!entry || entry.startedAt < input.since) continue;
+ if (uid !== null && entry.uid !== null && entry.uid !== uid) continue;
+ if (await excludedTree(pid)) continue;
+ found.push({ pid, command: entry.command });
+ }
+ found.sort((left, right) => left.pid - right.pid);
+ return found.slice(0, MAX_REPORTED);
+}
+
+export type SettleResult = {
+ /** How long the prompt waited. */
+ waitedMs: number;
+ /** No process of the turn was left running (or all of them exited within the cap). */
+ settled: boolean;
+ /** The processes found running when the wait began. */
+ processes: LateWriter[];
+};
+
+/**
+ * Waits for the processes the turn that ended at `endedAt` left running in the
+ * workspace to exit, until `endedAt + capMs` at the latest (never past it,
+ * whenever it is called). Returns at once when none runs. Never throws.
+ */
+export async function settleLateWriters(input: LateWritersInput & {
+ endedAt: number;
+ capMs?: number;
+ pollMs?: number;
+ now?: () => number;
+ sleep?: (ms: number) => Promise<void>;
+ signal?: AbortSignal;
+}): Promise<SettleResult> {
+ const now = input.now ?? Date.now;
+ const sleep = input.sleep ?? ((ms: number) => new Promise<void>((resolvePromise) => setTimeout(resolvePromise, ms).unref?.()));
+ const deadline = input.endedAt + (input.capMs ?? LATE_WRITES_CAP_MS);
+ const started = now();
+ let processes: LateWriter[];
+ try {
+ processes = await lateWriters(input);
+ } catch {
+ return { waitedMs: 0, settled: true, processes: [] };
+ }
+ if (processes.length === 0) return { waitedMs: now() - started, settled: true, processes };
+ let running = processes;
+ while (running.length > 0 && now() < deadline && !input.signal?.aborted) {
+ await sleep(Math.min(input.pollMs ?? LATE_WRITES_POLL_MS, Math.max(0, deadline - now())));
+ const pids = new Set(running.map((process) => process.pid));
+ try {
+ running = (await lateWriters(input)).filter((process) => pids.has(process.pid));
+ } catch {
+ running = [];
+ }
+ }
+ return { waitedMs: now() - started, settled: running.length === 0, processes };
+}
diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.ts
index 2fd2a324..f89b8972 100644
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@ -60,6 +60,8 @@ export type ProjectArchiveLifecycleOptions = {
quitBudgetMs?: number;
/** Tests. */
now?: () => number;
+ /** A pre_tool state took files written after the turn before ended (the session uploader records them). */
+ onLateWrites?: (late: LateWrites) => void;
};
const DEFAULT_CONSENT_RECHECK_MS = 10 * 60_000;
@@ -141,6 +143,36 @@ export function completedTurnCount(messages: unknown): number | null {
return turns;
}
+function createdAtMs(message: Record<string, unknown>): number | null {
+ const info = messageInfo(message);
+ const time = isRecord(info.time) ? info.time.created : info.createdAt ?? info.created_at;
+ if (typeof time === "number" && Number.isFinite(time)) return time;
+ if (typeof time === "string") {
+ const parsed = Date.parse(time);
+ return Number.isFinite(parsed) ? parsed : null;
+ }
+ return null;
+}
+
+/**
+ * The turn a tool call that started at `at` belongs to: the number of prompts
+ * sent up to then (the first prompt is turn 1), counting a prompt without a
+ * creation time as sent. A prompt sent before the previous turn settled (a
+ * quick follow-up, or one typed into a running turn) is its own turn, as the
+ * converter numbers them. Null when the list could not be read or holds no
+ * prompt.
+ */
+export function promptTurnAt(messages: unknown, at: number): number | null {
+ if (!Array.isArray(messages)) return null;
+ let turns = 0;
+ for (const message of messages) {
+ if (!isRecord(message) || !isPromptMessage(message)) continue;
+ const created = createdAtMs(message);
+ if (created === null || created <= at) turns += 1;
+ }
+ return turns > 0 ? turns : null;
+}
+
/** The parent of a session record: a string for a child (sub-agent) session, null for a root one, undefined when unreadable. */
function parentSessionId(session: unknown): string | null | undefined {
if (!isRecord(session)) return undefined;
@@ -179,6 +211,23 @@ type SessionRecord = {
turns?: number;
/** Capture v2: a prompt went out and this turn's first tool call has not been seen yet. */
awaitingTool?: boolean;
+ /** When the latest prompt, and the one before it, were dispatched. */
+ promptedAt?: number;
+ previousPromptAt?: number;
+ /** When a turn of the session last settled (turnCompleted / turnIncomplete). */
+ endedAt?: number;
+};
+
+/** A pre_tool state that took late writes of the turn before (lifecycle onLateWrites). */
+export type LateWrites = {
+ sessionId: string;
+ /** The turn in progress, whose start the files are part of; the turn before wrote them. */
+ turn: number;
+ /** When the turn before ended (or, for a prompt sent into a running turn, when that prompt went out). */
+ after: number;
+ /** When this turn's first tool call started. */
+ toolStartAt: number;
+ paths: string[];
};
/** An unresolved session start reads the engine again after these waits, then only at its next prompt or turn. */
@@ -194,6 +243,7 @@ export class ProjectArchiveLifecycle {
private readonly finalIdleMs: number;
private readonly quitBudgetMs: number;
private readonly now: () => number;
+ private readonly onLateWrites: ((late: LateWrites) => void) | undefined;
private readonly enabled: boolean;
private readonly accountMissing: boolean;
private signedOut = false;
@@ -226,6 +276,7 @@ export class ProjectArchiveLifecycle {
this.finalIdleMs = Math.max(0, options.finalIdleMs ?? FINAL_IDLE_MS);
this.quitBudgetMs = Math.max(0, options.quitBudgetMs ?? QUIT_FINAL_BUDGET_MS);
this.now = options.now ?? Date.now;
+ this.onLateWrites = options.onLateWrites;
}
private get active(): boolean {
@@ -274,11 +325,13 @@ export class ProjectArchiveLifecycle {
known.engine = input.engine;
known.activeAt = promptedAt;
known.awaitingTool = true;
+ known.previousPromptAt = known.promptedAt;
+ known.promptedAt = promptedAt;
}
// Resolved, or its start step is still queued: nothing to add.
if (known && (!known.start || known.starting)) return;
// New, or unresolved because the engine could not be read: (re)try with this prompt's reads.
- const record: SessionRecord = known ?? { root: null, start: null, starting: false, engine: input.engine, activeAt: promptedAt, idleTimer: null, idleFinal: null, awaitingTool: true };
+ const record: SessionRecord = known ?? { root: null, start: null, starting: false, engine: input.engine, activeAt: promptedAt, idleTimer: null, idleFinal: null, awaitingTool: true, promptedAt };
record.start = { root: input.root, engine: input.engine };
record.starting = true;
this.sessions.set(input.sessionId, record);
@@ -312,18 +365,40 @@ export class ProjectArchiveLifecycle {
}
/**
- * Capture v2, #9: a tool call of the session started (tool-start.ts). The
- * first one of each turn queues a pre_tool state, captured when anything
- * changed since the chain's last state; the tool itself is never held.
+ * Capture v2, #9: a tool call of the session started (tool-start.ts) at
+ * `at` (the engine's event time when it has one). The first one of each
+ * turn queues a pre_tool state, captured when anything changed since the
+ * chain's last state; the tool itself is never held, so a file changed at
+ * or after `at` (the tool's own writes) is left out of it and goes to the
+ * turn's delta. What changed between the chain's last state and `at`
+ * (a background process of the turn before still writing, the user's own
+ * edits) is in it, marked `late_writes` when the turn before had ended.
+ * The state is numbered with the prompt the tool answers, read from the
+ * engine: a prompt sent before the previous turn settled is a turn of its
+ * own.
*/
- toolStarted(sessionId: string): void {
+ toolStarted(sessionId: string, at: number = this.now()): void {
const record = this.sessions.get(sessionId);
if (!this.active || this.consentOff || !record || !record.awaitingTool || !this.archiver.captureState) return;
record.awaitingTool = false;
+ // Late: written after the turn before ended; for a prompt sent into a running turn (no end in between), after that prompt.
+ const ended = record.endedAt !== undefined && (record.previousPromptAt === undefined || record.endedAt >= record.previousPromptAt) ? record.endedAt : undefined;
+ // A turn this prompt cut settles after the prompt went out (the observer reads it then): it ended at the prompt at the latest.
+ const lateAfter = ended !== undefined
+ ? (record.promptedAt !== undefined ? Math.min(ended, record.promptedAt) : ended)
+ : record.previousPromptAt !== undefined ? record.promptedAt : undefined;
const capture = this.archiver.captureState.bind(this.archiver);
this.schedule(sessionId, "pre_tool", async () => {
if (this.sessions.get(sessionId) !== record || !record.root) return;
- const result = await capture(sessionId, (record.turns ?? 0) + 1);
+ const turn = promptTurnAt(await record.engine.messages().catch(() => null), at) ?? (record.turns ?? 0) + 1;
+ const result = await capture(sessionId, turn, { before: at, ...(lateAfter !== undefined ? { lateAfter } : {}) });
+ if (result.status === "queued" && result.late && result.late.length > 0 && lateAfter !== undefined) {
+ try {
+ this.onLateWrites?.({ sessionId, turn, after: lateAfter, toolStartAt: at, paths: result.late });
+ } catch {
+ // Bookkeeping only.
+ }
+ }
if (result.status === "skipped" && result.reason === "stopped") record.root = null;
this.settle(result);
});
@@ -360,6 +435,7 @@ export class ProjectArchiveLifecycle {
if (!this.active || this.consentOff || !record) return;
const turns = completedTurnCount(messages);
if (turns !== null) record.turns = turns;
+ record.endedAt = this.now();
this.schedule(sessionId, "delta", async () => {
if (this.sessions.get(sessionId) !== record) return;
if (record.start && !(await this.resolve(sessionId, record, turns))) return;
@@ -456,6 +532,7 @@ export class ProjectArchiveLifecycle {
turnIncomplete(sessionId: string): void {
const record = this.sessions.get(sessionId);
if (!this.active || this.consentOff || !record) return;
+ record.endedAt = this.now();
this.schedule(sessionId, "turn", async () => {
if (this.sessions.get(sessionId) !== record) return;
if (record.start && !(await this.resolve(sessionId, record, null))) return;
diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.ts
index 07ef4eeb..b031ba3b 100644
--- a/apps/server/src/session-archive/manifest.ts
+++ b/apps/server/src/session-archive/manifest.ts
@@ -56,6 +56,13 @@ export type ArchiveKind = "base" | "delta";
export type ArchiveTrigger = "turn" | "final" | "pre_tool";
/** What prompted a final archive (manifest.json `reason`). */
export type FinalReason = "idle" | "turn_incomplete" | "session_deleted" | "app_quit" | "app_start";
+/**
+ * manifest.json `reason`: a final archive's, or "late_writes" on a pre_tool
+ * state holding files written after the turn before ended (a background
+ * process of that turn still writing): they belong to that turn's outcome
+ * and to this turn's start.
+ */
+export type ArchiveReason = FinalReason | "late_writes";
/**
* manifest.json `scope` of a touched-files archive: its `files` are only the
* files the agent touched, and a path it does not list is not deleted (only
@@ -130,7 +137,7 @@ export type ArchiveManifest = {
created_at: string;
parent_archive_id: string | null;
trigger?: ArchiveTrigger;
- reason?: FinalReason;
+ reason?: ArchiveReason;
scope?: ArchiveScope;
workspace: { label: string; marker: string; git: ArchiveGit | null };
files: ArchiveEntry[];
@@ -809,7 +816,7 @@ export type ManifestInput = {
parentArchiveId: string | null;
/** Deltas only: a completed turn, or a final archive (then with its reason). */
trigger?: ArchiveTrigger;
- reason?: FinalReason;
+ reason?: ArchiveReason;
/** Touched-files archives only. */
scope?: ArchiveScope;
label: string;
diff --git a/apps/server/src/session-archive/tool-start.ts b/apps/server/src/session-archive/tool-start.ts
index 4ea14a78..886b6033 100644
--- a/apps/server/src/session-archive/tool-start.ts
+++ b/apps/server/src/session-archive/tool-start.ts
@@ -3,7 +3,8 @@
* follows a turn it listens to the engine's event stream (v1 `/event`, v2
* `/api/event`) and calls `onToolStart` once, at the first tool part of that
* session that is pending or running (v1 `message.part.updated` with a
- * `tool` part, v2 `session.tool.input.started` / `session.tool.called`).
+ * `tool` part, v2 `session.tool.input.started` / `session.tool.called`),
+ * with the time the call started (the files it writes are newer).
* The capture it triggers runs in the background: the tool is never held.
* Identical in the CLI and the desktop app.
*/
@@ -37,6 +38,34 @@ export function toolStartSession(payload: unknown): string | null {
return null;
}
+/**
+ * When the tool call of a tool-start event started, in epoch milliseconds:
+ * the event's own time (v1 `properties.time`, the part's `state.time.start`,
+ * v2 `time`/`timestamp`), or null when it carries none. The earliest one wins:
+ * a `pending` part has no start of its own yet, and its event is sent first.
+ */
+export function toolStartTime(payload: unknown): number | null {
+ if (!isRecord(payload)) return null;
+ if (isRecord(payload.payload) && typeof payload.payload.type === "string") return toolStartTime(payload.payload);
+ const times: unknown[] = [];
+ const properties = isRecord(payload.properties) ? payload.properties : null;
+ const data = isRecord(payload.data) ? payload.data : null;
+ for (const holder of [payload, properties, data]) {
+ if (!holder) continue;
+ times.push(holder.time, holder.timestamp);
+ const part = isRecord(holder.part) ? holder.part : null;
+ const state = part && isRecord(part.state) ? part.state : null;
+ if (state && isRecord(state.time)) times.push(state.time.start);
+ }
+ let earliest: number | null = null;
+ for (const time of times) {
+ const value = typeof time === "number" ? time : typeof time === "string" ? Date.parse(time) : NaN;
+ // Epoch milliseconds from this century only (not seconds, not a monotonic clock).
+ if (Number.isFinite(value) && value > 1_500_000_000_000 && value < 4_000_000_000_000 && (earliest === null || value < earliest)) earliest = value;
+ }
+ return earliest;
+}
+
/** Splits SSE text into the `data:` payloads of complete frames; returns the unfinished rest. */
export function sseFrames(buffer: string): { payloads: string[]; rest: string } {
const payloads: string[] = [];
@@ -66,7 +95,8 @@ export async function watchToolStarts(input: {
sessionId: string;
signal: AbortSignal;
fetch: ToolStartFetch;
- onToolStart: () => void;
+ /** `at`: when the tool call started (toolStartTime, else when its event arrived). */
+ onToolStart: (at: number) => void;
}): Promise<void> {
const controller = new AbortController();
const stop = () => controller.abort();
@@ -97,8 +127,11 @@ export async function watchToolStarts(input: {
continue;
}
if (toolStartSession(payload) === input.sessionId) {
+ const received = Date.now();
+ const at = toolStartTime(payload);
try {
- input.onToolStart();
+ // An event time from a clock running ahead is no later than the event's arrival.
+ input.onToolStart(at === null ? received : Math.min(at, received));
} finally {
controller.abort();
}
diff --git a/apps/server/src/session-archive/turn-writers.ts b/apps/server/src/session-archive/turn-writers.ts
new file mode 100644
index 00000000..af098b8b
--- /dev/null
+++ b/apps/server/src/session-archive/turn-writers.ts
@@ -0,0 +1,118 @@
+/**
+ * Who wrote what in a settled turn: the files the root session's and its
+ * sub-agents' (task tool child sessions') file tools wrote, by session, with
+ * when, and the files more than one session wrote in the turn (`overlap`),
+ * so that a rebuild of the turn can put those edits in the order they
+ * happened. Written as one "turn.writers" event per turn in which a
+ * sub-agent wrote a file. Shell commands name no paths reliably and are not
+ * counted. Identical in the CLI and the desktop app.
+ */
+import { isAbsolute, relative, sep } from "node:path";
+
+const WRITE_TOOLS = new Set(["write", "edit", "multiedit", "patch", "apply_patch", "str_replace_based_edit_tool", "create", "notebookedit"]);
+const PATH_KEYS = ["filePath", "file_path", "path", "filename", "notebook_path"];
+const MAX_FILES = 500;
+const MAX_PATH_CHARS = 1_024;
+
+export type WriterSessionMessages = {
+ sessionId: string;
+ role: "root" | "child";
+ /** 0 for the root, 1 for a child, 2 for a grandchild, ... */
+ depth: number;
+ messages: readonly unknown[];
+};
+
+export type TurnWriterSession = { session_id: string; role: "root" | "child"; depth: number; calls: number; first_at: string | null; last_at: string | null };
+export type TurnWriters = { files: Array<{ path: string; sessions: TurnWriterSession[] }>; overlap: string[] };
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** The paths an apply_patch / patch text adds, updates, deletes or moves to. */
+export function patchTextPaths(text: string): string[] {
+ const paths: string[] = [];
+ for (const match of text.matchAll(/^\*\*\* (?:Add|Update|Delete) File: (.+)$|^\*\*\* Move to: (.+)$/gm)) {
+ const path = (match[1] ?? match[2] ?? "").trim();
+ if (path) paths.push(path);
+ }
+ return paths;
+}
+
+/** Workspace-relative, forward slashes; null for a path outside the workspace or the workspace itself. */
+function workspacePath(root: string, path: string): string | null {
+ if (!path || path.length > MAX_PATH_CHARS || path.includes("\0")) return null;
+ const relativePath = isAbsolute(path) ? relative(root, path) : path.replace(/^\.\//, "");
+ if (!relativePath || relativePath === "." || relativePath.startsWith("..") || isAbsolute(relativePath)) return null;
+ return relativePath.split(sep).join("/").replaceAll("\\", "/");
+}
+
+function toolWrites(part: Record<string, unknown>): string[] {
+ const tool = typeof part.tool === "string" ? part.tool.toLowerCase() : "";
+ if (!WRITE_TOOLS.has(tool)) return [];
+ const state = isRecord(part.state) ? part.state : {};
+ if (state.status === "error") return [];
+ const input = isRecord(state.input) ? state.input : {};
+ const paths: string[] = [];
+ for (const key of PATH_KEYS) if (typeof input[key] === "string") paths.push(input[key] as string);
+ for (const key of ["patchText", "patch", "input"]) if (typeof input[key] === "string") paths.push(...patchTextPaths(input[key] as string));
+ // A patch tool's metadata names the files it changed, relative or absolute.
+ const metadata = isRecord(state.metadata) ? state.metadata : {};
+ if (Array.isArray(metadata.files)) {
+ for (const file of metadata.files) {
+ if (isRecord(file)) for (const key of ["filePath", "relativePath", "movePath"]) if (typeof file[key] === "string") paths.push(file[key] as string);
+ }
+ }
+ return paths;
+}
+
+function timeOf(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
+}
+
+/**
+ * The turn's writers. `root` is the workspace; the root session's messages
+ * come first. Null when no sub-agent wrote a file in the workspace (nothing
+ * to order).
+ */
+export function turnWriters(root: string, sessions: readonly WriterSessionMessages[]): TurnWriters | null {
+ type Seen = { role: "root" | "child"; depth: number; calls: number; first: number | null; last: number | null };
+ const byPath = new Map<string, Map<string, Seen>>();
+ let childWrote = false;
+ for (const session of sessions) {
+ for (const message of session.messages) {
+ if (!isRecord(message) || !Array.isArray(message.parts)) continue;
+ for (const part of message.parts) {
+ if (!isRecord(part) || part.type !== "tool") continue;
+ const paths = [...new Set(toolWrites(part).map((path) => workspacePath(root, path)).filter((path): path is string => path !== null))];
+ if (paths.length === 0) continue;
+ const state = isRecord(part.state) ? part.state : {};
+ const time = isRecord(state.time) ? state.time : {};
+ const start = timeOf(time.start);
+ const end = timeOf(time.end) ?? start;
+ for (const path of paths) {
+ let writers = byPath.get(path);
+ if (!writers) {
+ if (byPath.size >= MAX_FILES) continue;
+ writers = new Map();
+ byPath.set(path, writers);
+ }
+ const seen = writers.get(session.sessionId) ?? { role: session.role, depth: session.depth, calls: 0, first: null, last: null };
+ seen.calls += 1;
+ if (start !== null && (seen.first === null || start < seen.first)) seen.first = start;
+ if (end !== null && (seen.last === null || end > seen.last)) seen.last = end;
+ writers.set(session.sessionId, seen);
+ if (session.role === "child") childWrote = true;
+ }
+ }
+ }
+ }
+ if (!childWrote) return null;
+ const iso = (value: number | null) => (value === null ? null : new Date(value).toISOString());
+ const files = [...byPath].sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)).map(([path, writers]) => ({
+ path,
+ sessions: [...writers].map(([id, seen]) => ({ session_id: id, role: seen.role, depth: seen.depth, calls: seen.calls, first_at: iso(seen.first), last_at: iso(seen.last) }))
+ .sort((left, right) => (left.first_at ?? "").localeCompare(right.first_at ?? "")),
+ }));
+ return { files, overlap: files.filter((file) => file.sessions.length > 1).map((file) => file.path) };
+}
diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.ts
index 7285d736..b499864d 100644
--- a/apps/server/src/session-upload-observer.ts
+++ b/apps/server/src/session-upload-observer.ts
@@ -12,6 +12,7 @@ import { isFinishedAssistantMessage, isPromptMessage, type ArchiveEngineReads, t
import { MAX_UPLOAD_CHILD_SESSION_DEPTH, type UploadSessionModel, type SessionUploader } from "./session-uploader.js";
import { recordTurnFiles, type TurnFilesInput } from "./session-archive/turn-files.js";
import { watchToolCallEnds } from "./session-archive/files-used.js";
+import { turnWriters, type WriterSessionMessages } from "./session-archive/turn-writers.js";
import { watchToolStarts } from "./session-archive/tool-start.js";
import { watchToolEvents } from "./context/tool-events.js";
import type { ContextCapture } from "./context/index.js";
@@ -32,7 +33,13 @@ export type EngineTarget = {
* snapshot is taken right as the prompt goes out, and its messages stop
* before the prompt's own.
*/
-type FollowedTurn = { snapshotTaken: boolean; cutAt: number | null; settled?: boolean };
+type FollowedTurn = {
+ snapshotTaken: boolean;
+ cutAt: number | null;
+ settled?: boolean;
+ /** When a status read first found the session idle (null while it is busy): the turn's end, once it settles. */
+ idleAt?: number | null;
+};
/**
* A session being observed: the engine its latest captured request went to
@@ -64,10 +71,24 @@ export type SessionObservers = {
lastMessageIds: Map<string, string>;
replacedEngines: Map<string, EngineReplacement>;
controller: AbortController;
+ /** When each session's last settled turn was seen to end (the engine went idle), for turnEndedAt(). */
+ turnEnds?: Map<string, number>;
};
export function createSessionObservers(): SessionObservers {
- return { sessions: new Map(), lastMessageIds: new Map(), replacedEngines: new Map(), controller: new AbortController() };
+ return { sessions: new Map(), lastMessageIds: new Map(), replacedEngines: new Map(), controller: new AbortController(), turnEnds: new Map() };
+}
+
+/**
+ * When the session's latest turn ended, as the observer's status reads saw
+ * it (epoch ms): the engine is idle now and the turn has not settled yet, or
+ * the turn settled after the engine went idle. Null while the turn is still
+ * running (a prompt typed into it waits for nothing), or when no read saw it end.
+ */
+export function turnEndedAt(observers: SessionObservers, sessionId: string): number | null {
+ const turn = observers.sessions.get(sessionId)?.turn;
+ if (turn && !turn.settled) return turn.idleAt ?? null;
+ return observers.turnEnds?.get(sessionId) ?? null;
}
function originOf(url: string): string {
@@ -303,6 +324,8 @@ async function captureChildSessions(input: {
known: Set<string>;
visited: Set<string>;
beforeRecord: (bytes: number) => void;
+ /** Each child's new messages, as recorded (turn-writers.ts). */
+ onChild?: (childId: string, depth: number, messages: readonly unknown[]) => void;
}): Promise<void> {
if (input.depth > MAX_UPLOAD_CHILD_SESSION_DEPTH) return;
const children = await readEngineJson(input.fetchEngine, `/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024)
@@ -315,6 +338,7 @@ async function captureChildSessions(input: {
const history = await readEngineHistory(engineMessages(input.fetchEngine, `/session/${encodeURIComponent(childId)}/message`, true), input.checkpoints[childId])
.catch(nullOnErrorStatus);
const newMessages = history?.delta ?? [];
+ if (newMessages.length > 0) input.onChild?.(childId, input.depth, newMessages);
if (history && history.omitted > 0) {
input.sessionUploader.recordTrace(input.rootSessionId, "session.messages_omitted", { child_session_id: childId, count: history.omitted });
}
@@ -908,6 +932,8 @@ export function observeUploadedSession(input: {
const turn = turnState();
if (turn.settled) return;
turn.settled = true;
+ // When the engine went idle (not a cut while it ran): the next prompt's late-writes settle measures from it.
+ if (typeof turn.idleAt === "number") observer.turnEnds?.set(sessionId, turn.idleAt);
const recovered = recovering;
recovering = false;
let history: EngineHistory | null = read ?? null;
@@ -960,10 +986,12 @@ export function observeUploadedSession(input: {
}
pending += bytes;
};
+ const children: WriterSessionMessages[] = [];
// The v2 daemon has no subagent children route; only its root turn is captured.
if (!v2) {
try {
await captureChildSessions({
+ onChild: (childId, depth, childMessages) => children.push({ sessionId: childId, role: "child", depth, messages: childMessages }),
sessionUploader,
rootSessionId: sessionId,
parentSessionId: sessionId,
@@ -979,6 +1007,11 @@ export function observeUploadedSession(input: {
sessionUploader.recordTrace(sessionId, "session.children_failed", { error: errorMessage(error) });
}
}
+ // Sub-agents that wrote files in the workspace: who wrote what, and the files the root and a sub-agent (or two) both wrote.
+ if (history && children.length > 0 && input.turnFiles) {
+ const writers = turnWriters(input.turnFiles.root, [{ sessionId, role: "root", depth: 0, messages: history.delta }, ...children]);
+ if (writers) sessionUploader.recordTrace(sessionId, "turn.writers", writers);
+ }
room(newest.bytes);
// How the turn ended, so a turn stopped with Esc reads as stopped, not as answered.
const outcome = history ? turnOutcome(history.outline) : null;
@@ -1092,6 +1125,7 @@ export function observeUploadedSession(input: {
if (status !== "idle") {
observedBusy = true;
idleReads = 0;
+ turnState().idleAt = null;
// The v2 daemon has no children route: nothing to list there.
if (!v2 && timing.now() - childIdsAt >= CHILD_IDS_INTERVAL_MS) {
childIdsAt = timing.now();
@@ -1106,6 +1140,7 @@ export function observeUploadedSession(input: {
if (idleReads === 0) {
idleSince = timing.now();
requestsWhenIdle = session.requests;
+ turnState().idleAt = idleSince;
}
idleReads += 1;
if (idleReads < 2 || reads < 4) continue;
@@ -1199,7 +1234,7 @@ export function followToolStart(input: {
sessionId: input.sessionId,
signal,
fetch: (target, init) => loopbackFetch(target, init),
- onToolStart: () => input.archive.toolStarted(input.sessionId),
+ onToolStart: (at) => input.archive.toolStarted(input.sessionId, at),
}).finally(() => {
if (map.get(input.sessionId) === controller) map.delete(input.sessionId);
});