A desktop coding agent with free access to frontier models.
TypeScript
Reduce repeated capture work and upload copies
Task Builder70292d3f2e46
diff --git a/apps/desktop/electron/external-fetch.mjs b/apps/desktop/electron/external-fetch.mjsindex df307e1c..16860156 100644--- a/apps/desktop/electron/external-fetch.mjs+++ b/apps/desktop/electron/external-fetch.mjs@@ -12,8 +12,12 @@* Loopback targets (the embedded server, local development APIs) stay on* Node's fetch, where CA trust is irrelevant.*/+import { createReadStream } from "node:fs";+import { readFile } from "node:fs/promises";+import { Readable } from "node:stream";const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]);+const MAX_UPLOAD_RESPONSE_BYTES = 1024 * 1024;/*** @typedef {(input: string | URL | Request, init?: RequestInit) => Promise<Response>} ExternalFetch@@ -65,3 +69,80 @@ export function createExternalFetch({ net, isReady = () => true, nodeFetch = gloreturn net.fetch(/** @type {string | Request} */ (url), { credentials: "omit", ...init });};}++function abortedUpload(signal) {+ return signal?.reason instanceof Error+ ? signal.reason+ : new DOMException("The operation was aborted", "AbortError");+}++async function nodeFileUpload(nodeFetch, url, init) {+ if (init.signal?.aborted) throw abortedUpload(init.signal);+ const stream = createReadStream(init.path, { highWaterMark: 64 * 1024 });+ const abort = () => stream.destroy(abortedUpload(init.signal));+ init.signal?.addEventListener("abort", abort, { once: true });+ try {+ return await nodeFetch(url, {+ method: init.method,+ headers: init.headers,+ body: Readable.toWeb(stream),+ duplex: "half",+ signal: init.signal,+ redirect: "error",+ });+ } finally {+ init.signal?.removeEventListener("abort", abort);+ stream.destroy();+ }+}++async function boundedUploadResponse(response) {+ if (!response.body) return response;+ const reader = response.body.getReader();+ const chunks = [];+ let total = 0;+ try {+ for (;;) {+ const { value, done } = await reader.read();+ if (done) break;+ total += value.byteLength;+ if (total > MAX_UPLOAD_RESPONSE_BYTES) throw new Error("session upload response exceeds its bound");+ chunks.push(Buffer.from(value));+ }+ return new Response(Buffer.concat(chunks), {+ status: response.status, statusText: response.statusText, headers: response.headers,+ });+ } catch (error) {+ try { await reader.cancel(); } catch {}+ throw error;+ } finally {+ reader.releaseLock();+ }+}++/**+ * A file descriptor avoids worker/broker IPC copies. Electron's existing+ * net.fetch preserves its OS certificate verifier and system proxy.+ *+ * Keep one bounded body here: Electron 43's chunked net.request can throw+ * outside the request on interrupted uploads ("two startReading calls").+ * Node's path streams; Chromium buffers until that native issue is fixed.+ */+export function createExternalFileUpload({ net, isReady = () => true, nodeFetch = globalThis.fetch }) {+ return async function externalFileUpload(url, init) {+ if (init.signal?.aborted) throw abortedUpload(init.signal);+ if (isLoopbackUrl(url) || typeof net?.fetch !== "function" || !isReady()) {+ return boundedUploadResponse(await nodeFileUpload(nodeFetch, url, init));+ }+ if (!Number.isSafeInteger(init.size) || init.size < 0 || init.size > 64 * 1024 * 1024) {+ throw new Error("session upload file exceeds its bound");+ }+ const body = await readFile(init.path, { signal: init.signal });+ if (body.byteLength !== init.size) throw new Error("session upload file changed");+ const response = await net.fetch(url, {+ method: init.method, headers: init.headers, body, signal: init.signal,+ credentials: "omit", redirect: "error",+ });+ return boundedUploadResponse(response);+ };+}diff --git a/apps/desktop/electron/external-fetch.test.mjs b/apps/desktop/electron/external-fetch.test.mjsindex 95a9acbe..97adb28b 100644--- a/apps/desktop/electron/external-fetch.test.mjs+++ b/apps/desktop/electron/external-fetch.test.mjs@@ -58,3 +58,113 @@ test("loopback targets, and calls before app ready or without net, stay on Node'"https://omnirush.ai/",]);});++import { mkdtemp, writeFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { createExternalFileUpload } from "./external-fetch.mjs";++async function uploadFixture(t) {+ const dir = await mkdtemp(join(tmpdir(), "capture-transport-"));+ t.after(() => rm(dir, { recursive: true, force: true }));+ const path = join(dir, "payload.zst");+ const bytes = Buffer.alloc(256 * 1024, 73);+ await writeFile(path, bytes);+ return { path, size: bytes.length, bytes };+}++test("file upload uses Chromium credential policy and sends exact bytes", async (t) => {+ const file = await uploadFixture(t);+ const mock = recorder();+ const response = await createExternalFileUpload({ net: { fetch: mock.fn } })("https://collector.example.test/collect", {+ method: "POST", headers: { authorization: "Bearer synthetic-token" }, ...file,+ });+ assert.equal(response.status, 200);+ assert.deepEqual(mock.calls[0].init.body, file.bytes);+ assert.equal(mock.calls[0].init.credentials, "omit");+ assert.equal(mock.calls[0].init.redirect, "error");+});++test("auth retries reopen the full file and use the refreshed bearer", async (t) => {+ const file = await uploadFixture(t);+ const calls = [];+ const net = { fetch: async (_url, init) => {+ calls.push(init);+ return new Response("{}", { status: calls.length === 1 ? 401 : 201 });+ }};+ const upload = createExternalFileUpload({ net });+ assert.equal((await upload("https://collector.example.test/collect", { method: "POST", ...file, headers: { authorization: "Bearer synthetic-old" } })).status, 401);+ assert.equal((await upload("https://collector.example.test/collect", { method: "POST", ...file, headers: { authorization: "Bearer synthetic-refreshed" } })).status, 201);+ assert.notEqual(calls[0].body, calls[1].body);+ assert.deepEqual(calls[0].body, file.bytes);+ assert.deepEqual(calls[1].body, file.bytes);+ assert.equal(calls[1].headers.authorization, "Bearer synthetic-refreshed");+});++test("native file upload accepts a no-content response", async (t) => {+ const file = await uploadFixture(t);+ const net = { fetch: async () => new Response(null, { status: 204 }) };+ const response = await createExternalFileUpload({ net })("https://collector.example.test/collect", { method: "POST", headers: {}, ...file });+ assert.equal(response.status, 204);+ assert.equal(await response.text(), "");+});++test("native file upload rejects truncated and oversized responses", async (t) => {+ const file = await uploadFixture(t);+ for (const broken of [true, false]) {+ let cancelled = false;+ const net = { fetch: async () => new Response(new ReadableStream({+ start(controller) {+ if (broken) controller.error(new Error("response truncated"));+ else controller.enqueue(new Uint8Array(1024 * 1024 + 1));+ },+ cancel() { cancelled = true; },+ })) };+ await assert.rejects(createExternalFileUpload({ net })("https://collector.example.test/collect", { method: "POST", headers: {}, ...file }), /truncated|exceeds its bound/);+ if (!broken) assert.equal(cancelled, true);+ }+});++test("native upload cancellation rejects and pre-aborted uploads never fetch", async (t) => {+ const file = await uploadFixture(t);+ let calls = 0;+ const net = { fetch: async (_url, init) => {+ calls += 1;+ return new Promise((_resolve, reject) => init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true }));+ }};+ const controller = new AbortController();+ const upload = createExternalFileUpload({ net });+ const result = upload("https://collector.example.test/collect", { method: "POST", headers: {}, ...file, signal: controller.signal });+ while (!calls) await new Promise(resolve => setTimeout(resolve, 1));+ controller.abort();+ await assert.rejects(result, { name: "AbortError" });+ await assert.rejects(upload("https://collector.example.test/collect", { method: "POST", headers: {}, ...file, signal: controller.signal }), { name: "AbortError" });+ assert.equal(calls, 1);+});++test("native upload refuses invalid sizes and changed immutable files before fetch", async (t) => {+ const file = await uploadFixture(t);+ const mock = recorder();+ const upload = createExternalFileUpload({ net: { fetch: mock.fn } });+ for (const size of [-1, 64 * 1024 * 1024 + 1, file.size - 1]) {+ await assert.rejects(upload("https://collector.example.test/collect", { method: "POST", headers: {}, ...file, size }), /bound|changed/);+ }+ assert.equal(mock.calls.length, 0);+});++test("loopback file uploads use Node duplex streaming and reopen on retries", async (t) => {+ const file = await uploadFixture(t);+ const bodies = [];+ const nodeFetch = async (_url, init) => {+ assert.equal(init.duplex, "half");+ assert.equal(init.redirect, "error");+ const bytes = await new Response(init.body).arrayBuffer();+ bodies.push(Buffer.from(bytes));+ return new Response("{}", { status: 201 });+ };+ const upload = createExternalFileUpload({ net: null, nodeFetch });+ const init = { method: "POST", headers: {}, ...file };+ await upload("http://127.0.0.1/collect", init);+ await upload("http://127.0.0.1/collect", init);+ assert.deepEqual(bodies, [file.bytes, file.bytes]);+});diff --git a/apps/desktop/electron/main.mjs b/apps/desktop/electron/main.mjsindex de3a0548..e2cf54cf 100644--- a/apps/desktop/electron/main.mjs+++ b/apps/desktop/electron/main.mjs@@ -84,7 +84,7 @@ import { readSkillFolder } from "./skill-folder.mjs";import { createDesktopVaultKeyProvider } from "./secure-vault-key.mjs";import { applyLinuxPasswordStore, recordLinuxPasswordStore } from "./linux-password-store.mjs";import { createDesktopOmniRushAccountStore, legacyKeychainAllowed } from "./omnirush-account.mjs";-import { createExternalFetch } from "./external-fetch.mjs";+import { createExternalFetch, createExternalFileUpload } from "./external-fetch.mjs";import {clearOmniRushSentrySession,initOmniRushSentry,@@ -122,6 +122,7 @@ const pty = require(["node", "pty"].join("-"));// Chromium's network stack (OS trust store, system proxy) for the main// process's own requests off the machine; see external-fetch.mjs.const desktopExternalFetch = createExternalFetch({ net: electronNet, isReady: () => app.isReady() });+const desktopExternalFileUpload = createExternalFileUpload({ net: electronNet, isReady: () => app.isReady() });const NATIVE_DEEP_LINK_EVENT = "omnirush:deep-link-native";const AUTOMATION_RUNNER_CREDENTIAL_REJECTED_EVENT = "omnirush:automation-runner:credential-rejected";const isDevMode = process.env.OMNIRUSH_DEV_MODE === "1";@@ -1360,6 +1361,7 @@ const runtimeManager = createRuntimeManager({app,desktopRoot: path.resolve(__dirname, ".."),externalFetch: desktopExternalFetch,+ captureFileUpload: desktopExternalFileUpload,// Rewrites persisted `npx -y omnirush-ui-mcp` entries to the bundled launch// before the engine starts. Null (bundle missing) removes them instead.omnirushUiMcpLaunch: () => {diff --git a/apps/desktop/electron/runtime.mjs b/apps/desktop/electron/runtime.mjsindex 6bc4a24c..8e121e88 100644--- a/apps/desktop/electron/runtime.mjs+++ b/apps/desktop/electron/runtime.mjs@@ -1418,6 +1418,7 @@ export function createRuntimeManager({workspacePlatform = process.platform,// Chromium-backed fetch for requests off the machine (see external-fetch.mjs).externalFetch = null,+ captureFileUpload = null,}) {const inheritedProcessEnv = { ...process.env };let injectedUserEnvKeys = new Set();@@ -2221,6 +2222,7 @@ export function createRuntimeManager({opencodeCwd: managedOpencodeWorkdir(),localManagedMcpVaultKey,appVersion: typeof appVersion === "string" && appVersion.trim() ? appVersion.trim() : undefined,+ captureFileUpload: typeof captureFileUpload === "function" ? captureFileUpload : undefined,// Bundled UI-control MCP launch for the startup migration of persisted// `npx -y omnirush-ui-mcp` entries; null removes those entries instead.omnirushUiMcp: omnirushUiMcpLaunch() ?? null,diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.tsindex b168cc8a..8839307f 100644--- a/apps/server/src/capture-client.ts+++ b/apps/server/src/capture-client.ts@@ -11,6 +11,8 @@* number of times.*/import { Worker } from "node:worker_threads";+import { lstat } from "node:fs/promises";+import { resolve } from "node:path";import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineReplacement, type EngineTarget, type PromptRecord } from "./capture-host.js";import {@@ -26,7 +28,7 @@ import {type ToWorker,} from "./capture-protocol.js";import { externalFetch } from "./server-fetch.js";-import { isUploadableWebUrl, sessionUploaderEnabled, type UploadWebVisit } from "./session-uploader.js";+import { isUploadableWebUrl, MAX_UPLOAD_COMPRESSED_BYTES, sessionUploaderEnabled, type UploadWebVisit } from "./session-uploader.js";/** A prompt body larger than this is not traced, so it is not copied to the worker either. */const MAX_TRACED_REQUEST_BYTES = 4 * 1024 * 1024;@@ -54,6 +56,8 @@ export type CaptureService = {hasSession(sessionId: string): boolean;startSession(sessionId: string, workspaceId: string, root: string): void;recordTrace(sessionId: string, type: string, data?: unknown): void;+ flushTrace(sessionId: string, finalTrace?: unknown): void;+ finishSession(sessionId: string, finalTrace?: unknown): void;/** The "engine.request" event of a captured request (its body parsed off the main thread), plus a prompt's attachments. */recordPrompt(sessionId: string, prompt: Omit<PromptRecord, "body"> & { body: ArrayBuffer | undefined }): void;captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void;@@ -109,7 +113,7 @@ class CaptureClient implements CaptureService {constructor(private readonly options: CaptureServiceOptions) {this.uploadEnabled = sessionUploaderEnabled({- upload: Boolean(options.sessionUploader.upload),+ upload: Boolean(options.sessionUploader.upload || options.sessionUploader.uploadFile),gatewayUrl: options.sessionUploader.gatewayUrl,accessToken: options.sessionUploader.accessToken,});@@ -135,6 +139,15 @@ class CaptureClient implements CaptureService {this.send({ kind: "call", id: null, method: "recordTrace", args: data === undefined ? [sessionId, type] : [sessionId, type, data] });}+ flushTrace(sessionId: string, finalTrace?: unknown): void {+ this.send({ kind: "call", id: null, method: "flushTrace", args: finalTrace === undefined ? [sessionId] : [sessionId, finalTrace] });+ }++ finishSession(sessionId: string, finalTrace?: unknown): void {+ if (this.sessions.get(sessionId) === true) this.sessions.set(sessionId, false);+ this.send({ kind: "call", id: null, method: "finishSession", args: finalTrace === undefined ? [sessionId] : [sessionId, finalTrace] });+ }+recordPrompt(sessionId: string, prompt: Omit<PromptRecord, "body"> & { body: ArrayBuffer | undefined }): void {// The body is still forwarded to the engine: the worker gets its own copy, moved rather than cloned.const body = prompt.body && prompt.body.byteLength > 0 && prompt.body.byteLength <= MAX_TRACED_REQUEST_BYTES ? new Uint8Array(prompt.body.slice(0)) : null;@@ -272,6 +285,8 @@ class CaptureClient implements CaptureService {engineVersion: this.options.engineVersion,sessionUploader: {upload: Boolean(sessionUploader.upload),+ uploadFile: Boolean(sessionUploader.uploadFile),+ capabilities: Boolean(sessionUploader.capabilities),refreshAccessToken: Boolean(sessionUploader.refreshAccessToken),...(sessionUploader.gatewayUrl !== undefined ? { gatewayUrl: sessionUploader.gatewayUrl } : {}),...(sessionUploader.accessToken !== undefined ? { accessToken: sessionUploader.accessToken } : {}),@@ -436,6 +451,26 @@ class CaptureClient implements CaptureService {if (!sessionUploader.upload) throw new Error("no session upload hook");return { kind: "result", id, ok: true, response: await serializeResponse(await sessionUploader.upload(request.sessionId, request.body, signal)) };}+ case "uploadFile": {+ if (!sessionUploader.uploadFile) throw new Error("no session file upload hook");+ const root = resolve(this.options.stateDir);+ const path = resolve(request.path);+ if (!path.startsWith(`${root}${process.platform === "win32" ? "\\" : "/"}`)) throw new Error("capture file is outside the state directory");+ const file = await lstat(path);+ if (!file.isFile() || file.size !== request.size || file.size > MAX_UPLOAD_COMPRESSED_BYTES) {+ throw new Error("capture file descriptor is invalid");+ }+ return {+ kind: "result",+ id,+ ok: true,+ response: await serializeResponse(await sessionUploader.uploadFile(request.sessionId, path, request.size, signal)),+ };+ }+ case "capabilities": {+ if (!sessionUploader.capabilities) throw new Error("no session capability hook");+ return { kind: "result", id, ok: true, response: null, value: await sessionUploader.capabilities() };+ }case "refreshAccessToken": {const refresh = channel === "archive" ? archive.refreshAccessToken : sessionUploader.refreshAccessToken;return { kind: "result", id, ok: true, response: null, token: refresh ? await refresh() : null };diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.tsindex c172937e..aff82893 100644--- a/apps/server/src/capture-host.ts+++ b/apps/server/src/capture-host.ts@@ -23,7 +23,7 @@ import {import type { CaptureStopOptions } from "./capture-protocol.js";import { SessionArchiver, type SessionArchiverOptions } from "./session-archive/index.js";import { ProjectArchiveLifecycle, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";-import { SessionUploader, type UploadMetrics, type UploadWebVisit } from "./session-uploader.js";+import { SessionUploader, type TraceCapabilities, type UploadMetrics, type UploadWebVisit } from "./session-uploader.js";export type { EngineReplacement, EngineTarget } from "./session-upload-observer.js";@@ -58,6 +58,8 @@ export type CaptureHostOptions = {log: CaptureLog;sessionUploader: {upload?: (sessionId: string, compressed: Uint8Array, signal?: AbortSignal) => Promise<Response>;+ uploadFile?: (sessionId: string, path: string, size: number, signal?: AbortSignal) => Promise<Response>;+ capabilities?: () => Promise<TraceCapabilities>;refreshAccessToken?: () => Promise<string | null>;fetch?: (input: string, init?: RequestInit) => Promise<Response>;gatewayUrl?: string;@@ -124,6 +126,16 @@ export class CaptureHost {this.sessionUploader.recordTrace(sessionId, type, data);}+ flushTrace(sessionId: string, finalTrace?: unknown): void {+ this.sessionUploader.flushTrace(sessionId, finalTrace);+ }++ finishSession(sessionId: string, finalTrace?: unknown): void {+ this.sessionUploader.finishSession(sessionId, finalTrace);+ this.archive.sessionEnded(sessionId);+ this.observers.lastMessageIds.delete(sessionId);+ }+/** The "engine.request" event of a captured request and, for a prompt dispatch, one "attachment" event per attached file. */recordPrompt(sessionId: string, prompt: PromptRecord): void {// The turn this prompt follows takes its end snapshot first, ahead of this prompt's own.diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.tsindex a666596a..5acb654c 100644--- a/apps/server/src/capture-protocol.ts+++ b/apps/server/src/capture-protocol.ts@@ -9,7 +9,7 @@*/import type { CaptureHost, EngineReplacement, EngineTarget, PromptRecord } from "./capture-host.js";import type { FolderGateOptions } from "./session-archive/detect.js";-import type { UploadWebVisit } from "./session-uploader.js";+import type { TraceCapabilities, UploadWebVisit } from "./session-uploader.js";/** How the capture stops: `archiveFinals` false (the account is gone) packs no final project archives. */export type CaptureStopOptions = { archiveFinals: boolean };@@ -18,6 +18,8 @@ export type CaptureStopOptions = { archiveFinals: boolean };export type CaptureCalls = {startSession: [sessionId: string, workspaceId: string, root: string];recordTrace: [sessionId: string, type: string, data?: unknown];+ flushTrace: [sessionId: string, finalTrace?: unknown];+ finishSession: [sessionId: string, finalTrace?: unknown];recordPrompt: [sessionId: string, prompt: PromptRecord];captureSnapshot: [sessionId: string, trigger: "prompt" | "turn_completed"];recordWebVisit: [sessionId: string, visit: UploadWebVisit];@@ -39,6 +41,8 @@ export type CaptureCall = { [M in CaptureCallName]: { kind: "call"; id: number |/** What a worker asks of the main thread. `channel` says whose request it is: archive requests are aborted first at shutdown. */export type HostRequest =| { type: "upload"; sessionId: string; body: Uint8Array<ArrayBuffer> }+ | { type: "uploadFile"; sessionId: string; path: string; size: number }+ | { type: "capabilities" }| { type: "refreshAccessToken" }| { type: "archiveRequest"; path: string; method: "GET" | "POST"; body?: string; refresh?: false }| { type: "fetch"; url: string; method: string; headers: Array<[string, string]>; body?: Uint8Array<ArrayBuffer> | string };@@ -48,7 +52,7 @@ export type RequestChannel = "uploader" | "archive";export type SerializedResponse = { status: number; statusText: string; headers: Array<[string, string]>; body: ArrayBuffer | null };export type RequestResult =- | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null }+ | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null; value?: TraceCapabilities }| { kind: "result"; id: number; ok: false; error: string; name: string };export type ToWorker = CaptureCall | RequestResult;@@ -67,7 +71,7 @@ export type CaptureWorkerInit = {stateDir: string;appVersion: string;engineVersion: string;- sessionUploader: { upload: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };+ sessionUploader: { upload: boolean; uploadFile: boolean; capabilities: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };archive: {enabled: boolean;excludedDirs: string[];@@ -82,7 +86,7 @@ export type CaptureWorkerInit = {};export async function serializeResponse(response: Response): Promise<SerializedResponse> {- const body = response.body ? await response.arrayBuffer() : null;+ const body = response.body ? await boundedResponseBody(response) : null;return {status: response.status,statusText: response.statusText,@@ -91,6 +95,36 @@ export async function serializeResponse(response: Response): Promise<SerializedR};}+const MAX_RELAY_RESPONSE_BYTES = 1024 * 1024;++async function boundedResponseBody(response: Response): Promise<ArrayBuffer | null> {+ if (!response.body) return null;+ const reader = response.body.getReader();+ const chunks: Uint8Array[] = [];+ let total = 0;+ try {+ while (true) {+ const result = await reader.read();+ if (result.done) break;+ total += result.value.byteLength;+ if (total > MAX_RELAY_RESPONSE_BYTES) {+ await reader.cancel("capture relay response too large").catch(() => undefined);+ throw new Error("capture relay response exceeds its bound");+ }+ chunks.push(result.value);+ }+ } finally {+ reader.releaseLock();+ }+ const body = new Uint8Array(total);+ let offset = 0;+ for (const chunk of chunks) {+ body.set(chunk, offset);+ offset += chunk.byteLength;+ }+ return body.buffer;+}+const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);export function deserializeResponse(value: SerializedResponse): Response {@@ -117,6 +151,10 @@ export function invokeCapture(host: CaptureHost, call: CaptureCall): unknown {return host.startSession(...call.args);case "recordTrace":return host.recordTrace(...call.args);+ case "flushTrace":+ return host.flushTrace(...call.args);+ case "finishSession":+ return host.finishSession(...call.args);case "recordPrompt":return host.recordPrompt(...call.args);case "captureSnapshot":diff --git a/apps/server/src/capture-worker.ts b/apps/server/src/capture-worker.tsindex 2e442c14..f7bafd5a 100644--- a/apps/server/src/capture-worker.ts+++ b/apps/server/src/capture-worker.ts@@ -22,6 +22,7 @@ import {type ToWorker,} from "./capture-protocol.js";import type { ArchiveApiRequestInit } from "./session-archive/upload.js";+import type { TraceCapabilities } from "./session-uploader.js";function mainThreadPort(): MessagePort {if (!parentPort) throw new Error("capture-worker runs as a worker thread only");@@ -95,6 +96,16 @@ function refreshAccessToken(channel: RequestChannel) {return async (): Promise<string | null> => (await ask(channel, { type: "refreshAccessToken" })).token ?? null;}+function capabilities(): Promise<TraceCapabilities> {+ return ask("uploader", { type: "capabilities" }).then((result) => {+ const value = result.value;+ if (!value || !Array.isArray(value.schema_versions) || typeof value.canonical_trace !== "boolean") {+ throw new Error("the main thread returned invalid capture capabilities");+ }+ return value;+ });+}+const log = (level: "info" | "warn", message: string, attributes?: Record<string, unknown>) => {post({ kind: "log", level, message, ...(attributes ? { attributes } : {}) });};@@ -115,6 +126,16 @@ const host = new CaptureHost({},}: {}),+ ...(init.sessionUploader.uploadFile+ ? {+ uploadFile: async (sessionId: string, path: string, size: number, signal?: AbortSignal) => {+ const result = await ask("uploader", { type: "uploadFile", sessionId, path, size }, signal);+ if (!result.response) throw new Error("the main thread returned no response");+ return deserializeResponse(result.response);+ },+ }+ : {}),+ ...(init.sessionUploader.capabilities ? { capabilities } : {}),...(init.sessionUploader.refreshAccessToken ? { refreshAccessToken: refreshAccessToken("uploader") } : {}),fetch: hostFetch("uploader"),...(init.sessionUploader.gatewayUrl !== undefined ? { gatewayUrl: init.sessionUploader.gatewayUrl } : {}),diff --git a/apps/server/src/embedded.ts b/apps/server/src/embedded.tsindex 1eb1c3a8..44ffab86 100644--- a/apps/server/src/embedded.ts+++ b/apps/server/src/embedded.ts@@ -43,7 +43,7 @@ import { resolveOpencodeModelsEnv } from "./opencode-models-url.js";import { assertOpencodeConfigCompat } from "./opencode-config-compat.js";import { resolveEngineIdentity } from "./engine2/launch.js";import type { ServeResult } from "./serve-node.js";-import type { LocalManagedMcpVaultKeyProvider, OmniRushGatewayCredentials, ServerConfig } from "./types.js";+import type { CaptureFileUpload, LocalManagedMcpVaultKeyProvider, OmniRushGatewayCredentials, ServerConfig } from "./types.js";export type EmbeddedServerOptions = CliArgs & {/** When true, spawn a managed OpenCode child process. */@@ -59,6 +59,8 @@ export type EmbeddedServerOptions = CliArgs & {resumeInterruptedTasks?: boolean;/** Version of the embedding desktop app, reported as environment.app_version by the session uploader. */appVersion?: string;+ /** File-aware transport for immutable compressed session uploads. */+ captureFileUpload?: CaptureFileUpload;/*** How the embedding desktop launches its bundled UI-control MCP. Persisted* entries still using `npx -y omnirush-ui-mcp` are rewritten to it before@@ -96,6 +98,7 @@ export async function startEmbeddedServer(options: EmbeddedServerOptions): Promiconfig.omnirushEngineToken = randomUUID();config.resumeInterruptedTasks = options.resumeInterruptedTasks === true && options.manageOpencode === true && !config.opencodeBaseUrl;config.appVersion = typeof options.appVersion === "string" && options.appVersion.trim() ? options.appVersion.trim() : undefined;+ config.captureFileUpload = options.captureFileUpload;const logger = createServerLogger(config);// Spawn managed OpenCode if requested and no explicit base URL was provided.diff --git a/apps/server/src/omnirush-gateway-broker.test.ts b/apps/server/src/omnirush-gateway-broker.test.tsindex 8cb5e23b..26597f02 100644--- a/apps/server/src/omnirush-gateway-broker.test.ts+++ b/apps/server/src/omnirush-gateway-broker.test.ts@@ -97,6 +97,84 @@ describe("OmniRush gateway broker", () => {expect(response.status).toBe(404);});+ test("caches capabilities per credential pair, expires them, and falls back on a 404", async () => {+ const originalNow = Date.now;+ let now = originalNow();+ let supported = true;+ let probes = 0;+ Date.now = () => now;+ try {+ const broker = new OmniRushGatewayBroker({+ credentials: {+ gatewayUrl: "https://gateway.example/omnirush/v1",+ accessToken: "access-token",+ refreshToken: "refresh-token",+ },+ engineToken: "local-engine-token",+ fetch: async (input) => {+ if (new URL(String(input)).pathname.endsWith("/collect/capabilities")) {+ probes += 1;+ return supported+ ? Response.json({ schema_versions: [1, 2, 3], canonical_trace: true })+ : new Response("not found", { status: 404 });+ }+ return Response.json({ output: [] });+ },+ });++ const first = await Promise.all([broker.sessionUploadCapabilities(), broker.sessionUploadCapabilities()]);+ expect(first).toEqual([+ { schema_versions: [1, 2, 3], canonical_trace: true },+ { schema_versions: [1, 2, 3], canonical_trace: true },+ ]);+ expect(probes).toBe(1);++ now += 5 * 60_000 + 1;+ supported = false;+ expect(await broker.sessionUploadCapabilities()).toEqual({ schema_versions: [1, 2], canonical_trace: false });+ expect(probes).toBe(2);++ broker.resetCapabilities();+ supported = true;+ expect(await broker.sessionUploadCapabilities()).toEqual({ schema_versions: [1, 2, 3], canonical_trace: true });+ expect(probes).toBe(3);+ } finally {+ Date.now = originalNow;+ }+ });++ test("ends a file-aware upload callback promptly when its request is aborted", async () => {+ let started!: () => void;+ const uploadStarted = new Promise<void>((resolvePromise) => { started = resolvePromise; });+ let callbackSignal: AbortSignal | undefined;+ const broker = new OmniRushGatewayBroker({+ credentials: {+ gatewayUrl: "https://gateway.example/omnirush/v1",+ accessToken: "access-token",+ refreshToken: "refresh-token",+ },+ engineToken: "local-engine-token",+ uploadFile: async (_url, init) => {+ callbackSignal = init.signal;+ started();+ return new Promise<Response>(() => undefined);+ },+ fetch: async () => Response.json({ output: [] }),+ });+ const controller = new AbortController();+ const pending = broker.uploadSessionFile("session-file-abort-1", "/tmp/synthetic-capture.zst", 64, controller.signal);+ await uploadStarted;+ controller.abort(new DOMException("test abort", "AbortError"));+ let rejection: unknown;+ try {+ await pending;+ } catch (error) {+ rejection = error;+ }+ expect(rejection).toBeDefined();+ expect(callbackSignal?.aborted).toBe(true);+ });+test("invalidates a revoked device session and returns a professional sign-in error", async () => {let invalidated = 0;const broker = new OmniRushGatewayBroker({diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.tsindex b72007d4..0e2f1b51 100644--- a/apps/server/src/omnirush-gateway-broker.ts+++ b/apps/server/src/omnirush-gateway-broker.ts@@ -1,14 +1,18 @@import { timingSafeEqual } from "node:crypto";+import { createReadStream } from "node:fs";import { SESSION_UPLOAD_BUDGET, SESSION_UPLOAD_ENDPOINT_PATH, sessionUploadTimeoutMs, type SessionUploadBudget } from "./session-upload-budget.js";import { externalFetch } from "./server-fetch.js";import type { OmniRushGatewayCredentialBundle, OmniRushGatewayCredentials } from "./types.js";+import type { TraceCapabilities } from "./session-uploader.js";import {SUBAGENT_FALLBACK_EFFORT_HEADER,SUBAGENT_FALLBACK_MODEL_HEADER,SUBAGENT_ROOT_SESSION_HEADER,} from "./omnirush-swarm.js";+const TRACE_CAPABILITY_TTL_MS = 5 * 60_000;+/** A sub-agent request the gateway refused on the picked model and that was sent again on the main model. */export type SubagentModelFallbackEvent = {/** The sub-agent session (x-omnirush-session-id), when the request named it. */@@ -32,6 +36,14 @@ type BrokerOptions = {credentials?: OmniRushGatewayCredentials;engineToken?: string;fetch?: typeof externalFetch;+ /** Electron's file-aware transport. Standalone runtimes use the bounded Node stream fallback. */+ uploadFile?: (url: string, init: {+ method: "POST";+ headers: Record<string, string>;+ path: string;+ size: number;+ signal?: AbortSignal;+ }) => Promise<Response>;log?: (level: "info" | "warn" | "error", message: string, attributes?: Record<string, unknown>) => void;/** uploadSession()'s deadline parameters; SESSION_UPLOAD_BUDGET unless a test shrinks it. */sessionUploadBudget?: SessionUploadBudget;@@ -431,6 +443,34 @@ async function readableErrorResponse(response: Response): Promise<Response> {});}+async function boundedText(response: Response, maxBytes: number): Promise<string | null> {+ if (!response.body) return "";+ const reader = response.body.getReader();+ const chunks: Uint8Array[] = [];+ let total = 0;+ try {+ while (true) {+ const result = await reader.read();+ if (result.done) break;+ total += result.value.byteLength;+ if (total > maxBytes) {+ await reader.cancel("response too large").catch(() => undefined);+ return null;+ }+ chunks.push(result.value);+ }+ } finally {+ reader.releaseLock();+ }+ const body = new Uint8Array(total);+ let offset = 0;+ for (const chunk of chunks) {+ body.set(chunk, offset);+ offset += chunk.byteLength;+ }+ return new TextDecoder().decode(body);+}+type CredentialState = {gatewayUrl: string;accessToken: string;@@ -501,6 +541,53 @@ function apiUrl(gatewayUrl: string, path: string): string {return url.toString();}+function untilAborted(response: Promise<Response>, signal: AbortSignal): Promise<Response> {+ if (signal.aborted) return Promise.reject(signal.reason);+ return new Promise((resolvePromise, reject) => {+ const onAbort = () => reject(signal.reason);+ signal.addEventListener("abort", onAbort, { once: true });+ response.then((value) => {+ signal.removeEventListener("abort", onAbort);+ if (signal.aborted) void value.body?.cancel().catch(() => undefined);+ resolvePromise(value);+ }, (error: unknown) => {+ signal.removeEventListener("abort", onAbort);+ reject(error);+ });+ });+}++function fileReadableBody(path: string, signal: AbortSignal): ReadableStream<Uint8Array> {+ const source = createReadStream(path, { highWaterMark: 64 * 1024 });+ const abort = () => source.destroy(signal.reason instanceof Error ? signal.reason : undefined);+ const cleanup = () => signal.removeEventListener("abort", abort);+ return new globalThis.ReadableStream<Uint8Array>({+ start(controller) {+ if (signal.aborted) {+ abort();+ return;+ }+ signal.addEventListener("abort", abort, { once: true });+ source.on("data", (chunk: string | Buffer) => {+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;+ controller.enqueue(new Uint8Array(bytes));+ });+ source.on("end", () => {+ cleanup();+ controller.close();+ });+ source.on("error", (error: Error) => {+ cleanup();+ controller.error(error);+ });+ },+ cancel(reason) {+ cleanup();+ source.destroy(reason instanceof Error ? reason : undefined);+ },+ });+}+/** The backend's voice route, relative to the gateway URL (…/omnirush/v1/audio/transcriptions). */export const VOICE_TRANSCRIPTIONS_PATH = "audio/transcriptions";/** The backend allows 20 s for its primary provider and 30 s for the fallback. */@@ -723,6 +810,7 @@ export class OmniRushGatewayBroker {private readonly invalidate?: OmniRushGatewayCredentials["invalidate"];private readonly latest?: OmniRushGatewayCredentials["latest"];private readonly fetcher: typeof externalFetch;+ private readonly fileUploader?: BrokerOptions["uploadFile"];private readonly log?: BrokerOptions["log"];private readonly sessionUploadBudget: SessionUploadBudget;private readonly onSubagentFallback?: BrokerOptions["onSubagentFallback"];@@ -737,6 +825,8 @@ export class OmniRushGatewayBroker {* store entry cannot sign out a session this broker still holds.*/private previous: CredentialState | null = null;+ private capabilityCache: { gatewayUrl: string; accountKey: string; version: 2 | 3; expiresAt: number } | null = null;+ private capabilityProbe: { key: string; promise: Promise<2 | 3> } | null = null;constructor(options: BrokerOptions) {const gatewayUrl = options.credentials ? normalizedGatewayUrl(options.credentials.gatewayUrl) : null;@@ -753,6 +843,7 @@ export class OmniRushGatewayBroker {this.invalidate = options.credentials?.invalidate;this.latest = options.credentials?.latest;this.fetcher = options.fetch ?? externalFetch;+ this.fileUploader = options.uploadFile;this.log = options.log;this.sessionUploadBudget = options.sessionUploadBudget ?? SESSION_UPLOAD_BUDGET;this.onSubagentFallback = options.onSubagentFallback;@@ -860,6 +951,94 @@ export class OmniRushGatewayBroker {}));}+ uploadSessionFile(sessionId: string, path: string, size: number, signal?: AbortSignal): Promise<Response> {+ const timeoutMs = sessionUploadTimeoutMs(size, this.sessionUploadBudget);+ return this.withDeviceBearer(async (state) => {+ const requestSignal = signal ? AbortSignal.any([AbortSignal.timeout(timeoutMs), signal]) : AbortSignal.timeout(timeoutMs);+ const url = apiUrl(state.gatewayUrl, SESSION_UPLOAD_ENDPOINT_PATH);+ const headers = {+ Authorization: `Bearer ${state.accessToken}`,+ "Content-Type": "application/zstd",+ "X-OmniRush-Session-ID": sessionId,+ };+ if (this.fileUploader) {+ return untilAborted(this.fileUploader(url, { method: "POST", headers, path, size, signal: requestSignal }), requestSignal);+ }+ const bodyAbort = new AbortController();+ const bodySignal = AbortSignal.any([requestSignal, bodyAbort.signal]);+ const body = fileReadableBody(path, bodySignal);+ try {+ const init: RequestInit & { duplex: "half" } = {+ method: "POST",+ headers,+ body,+ duplex: "half",+ signal: requestSignal,+ };+ return await this.fetcher(url, init);+ } finally {+ bodyAbort.abort(new DOMException("The upload response arrived", "AbortError"));+ }+ });+ }++ async sessionUploadCapabilities(): Promise<TraceCapabilities> {+ if (!this.state) return { schema_versions: [1, 2, 3], canonical_trace: false };+ const state = this.state;+ const gatewayUrl = state.gatewayUrl;+ const key = this.capabilityKey(state);+ if (this.capabilityCache?.gatewayUrl === gatewayUrl+ && this.capabilityCache.accountKey === key+ && this.capabilityCache.expiresAt > Date.now()) {+ return { schema_versions: this.capabilityCache.version === 3 ? [1, 2, 3] : [1, 2], canonical_trace: this.capabilityCache.version === 3 };+ }+ const pending = this.capabilityProbe?.key === key+ ? this.capabilityProbe.promise+ : this.probeSessionUploadCapabilities(gatewayUrl);+ if (!this.capabilityProbe || this.capabilityProbe.key !== key) {+ this.capabilityProbe = { key, promise: pending };+ void pending.finally(() => {+ if (this.capabilityProbe?.promise === pending) this.capabilityProbe = null;+ }).catch(() => undefined);+ }+ const version = await pending;+ if (this.state && this.capabilityKey(this.state) === key) {+ this.capabilityCache = { gatewayUrl, accountKey: key, version, expiresAt: Date.now() + TRACE_CAPABILITY_TTL_MS };+ }+ return { schema_versions: version === 3 ? [1, 2, 3] : [1, 2], canonical_trace: version === 3 };+ }++ resetCapabilities(): void {+ this.capabilityCache = null;+ this.capabilityProbe = null;+ }++ private capabilityKey(state: CredentialState): string {+ return `${state.gatewayUrl}\0${state.refreshToken}`;+ }++ private async probeSessionUploadCapabilities(gatewayUrl: string): Promise<2 | 3> {+ try {+ if (this.state?.gatewayUrl !== gatewayUrl) return 2;+ const response = await this.withDeviceBearer((state) => this.fetcher(apiUrl(state.gatewayUrl, "collect/capabilities"), {+ method: "GET",+ headers: { Authorization: `Bearer ${state.accessToken}`, Accept: "application/json" },+ signal: AbortSignal.timeout(8_000),+ }));+ if (!response.ok) {+ await response.body?.cancel().catch(() => undefined);+ return 2;+ }+ const text = await boundedText(response, 64 * 1024);+ if (text === null) return 2;+ const payload: unknown = JSON.parse(text);+ if (!isRecord(payload) || payload.canonical_trace !== true || !Array.isArray(payload.schema_versions)) return 2;+ return payload.schema_versions.some((version) => version === 3) ? 3 : 2;+ } catch {+ return 2;+ }+ }+/*** A project archive API call for the session archiver (`archives/key`,* `archives`, `archives/<id>/parts|complete|abort`), authenticated like@@ -1147,6 +1326,7 @@ export class OmniRushGatewayBroker {refreshToken: stored.refreshToken,rotation,};+ if (this.state.gatewayUrl !== current.gatewayUrl) this.resetCapabilities();this.log?.("info", "omnirush.ai device credentials adopted from the account store", { rotation });return true;}@@ -1185,6 +1365,7 @@ export class OmniRushGatewayBroker {if (this.state !== spent) return Boolean(this.state);}this.state = null;+ this.resetCapabilities();void this.invalidate?.().catch(() => undefined);return false;}@@ -1212,6 +1393,7 @@ export class OmniRushGatewayBroker {if (!this.state) return { kind: "unavailable" };const rotated = { accessToken, refreshToken, gatewayUrl, rotation: from.rotation + 1 };this.state = rotated;+ if (rotated.gatewayUrl !== from.gatewayUrl) this.resetCapabilities();this.previous = null;void this.persistLatest(rotated);return { kind: "rotated" };diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex ec1a000f..e6da7d26 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -973,12 +973,14 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {// session uploads and queued project archives: a signed-out// account leaves nothing queued.invalidate: async () => {+ gatewayBroker.resetCapabilities();await capture.signOut();await gatewayCredentials.invalidate?.();},}: undefined,engineToken: config.omnirushEngineToken,+ ...(config.captureFileUpload ? { uploadFile: config.captureFileUpload } : {}),// A sub-agent request moved to the main model: new sub-agent prompts skip// the refused model for a while, and the session uploader records the model the// sub-agent's messages really ran on.@@ -1023,6 +1025,8 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {sessionUploader: gatewayBroker.enabled? {upload: (sessionId, compressed, signal) => gatewayBroker.uploadSession(sessionId, compressed, signal),+ uploadFile: (sessionId, path, size, signal) => gatewayBroker.uploadSessionFile(sessionId, path, size, signal),+ capabilities: () => gatewayBroker.sessionUploadCapabilities(),refreshAccessToken: () => gatewayBroker.refreshAccessToken(),}: {},diff --git a/apps/server/src/session-archive/index.test.ts b/apps/server/src/session-archive/index.test.tsindex b1da82b4..cb2cedc6 100644--- a/apps/server/src/session-archive/index.test.ts+++ b/apps/server/src/session-archive/index.test.ts@@ -1262,3 +1262,91 @@ describe("SessionArchiver touched files", () => {expect(server.callPaths().at(-1)).toBe("GET archives/key 200");});});+++test("archive disk budget pauses a base and resumes the same sequence after restart", async () => {+ const server = new FakeArchiveServer();+ const { root, blend } = await project();+ const state = await tempDir("budget-state");+ const low = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 0 });+ expect(await low.captureBase("ses_budget_restart", root)).toMatchObject({ status: "paused", reason: "disk_budget" });+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "queue"))).toEqual([]);+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "tmp"))).toEqual([]);++ const resumed = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3 });+ expect(await resumed.captureBase("ses_budget_restart", root)).toMatchObject({ status: "queued", kind: "base", sequence: 0 });+ expect((await resumed.drain()).uploaded).toBe(1);+ const members = await openArchive(server.objects()[0]!.object!);+ expect(members.find((member) => member.name === "assets/scene.blend")!.content.equals(blend)).toBe(true);+});++test("archive disk budget keeps the base and parent link when a delta is paused", async () => {+ const server = new FakeArchiveServer();+ const { root } = await project();+ const state = await tempDir("budget-delta");+ let available = 1024 ** 3;+ const subject = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => available });+ const base = await subject.captureBase("ses_budget_chain", root);+ expect(base.status).toBe("queued");+ await subject.drain();+ await writeFile(join(root, "src/app.ts"), "export const app = 2;\n");+ await rm(join(root, "assets/scene.blend"));+ available = 0;+ expect(await subject.captureDelta("ses_budget_chain", root, 1)).toMatchObject({ status: "paused", reason: "disk_budget" });+ expect(server.objects()).toHaveLength(1);++ const resumed = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3 });+ expect(await resumed.captureDelta("ses_budget_chain", root, 1)).toMatchObject({ status: "queued", sequence: 1 });+ await resumed.drain();+ const delta = server.objects()[1]!;+ expect(delta.request.parent_archive_id).toBe(base.status === "queued" ? base.archiveId : "");+ const members = await openArchive(delta.object!);+ expect(members.find((member) => member.name === "src/app.ts")!.content.toString()).toBe("export const app = 2;\n");+ expect(manifestOf(members).deleted).toContain("assets/scene.blend");+});++test("archive disk budget removes only the uncommitted partial when space runs out during packing", async () => {+ const server = new FakeArchiveServer();+ server.partSize = 5 * 1024 * 1024;+ const { root } = await project();+ const state = await tempDir("budget-midpack");+ const blob = randomBytes(10 * 1024 * 1024);+ await writeFile(join(root, "assets/large.bin"), blob);+ let probes = 0;+ const subject = archiver(server, state, {+ minimumFreeDiskBytes: 0,+ freeDiskBytes: async () => ++probes === 1 ? 1024 ** 3 : 0,+ });+ expect(await subject.captureBase("ses_budget_midpack", root)).toMatchObject({ status: "paused", reason: "disk_budget" });+ expect(probes).toBeGreaterThan(1);+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "queue"))).toEqual([]);+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "tmp"))).toEqual([]);+ expect(server.objects()).toEqual([]);++ const resumed = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3 });+ expect(await resumed.captureBase("ses_budget_midpack", root)).toMatchObject({ status: "queued", sequence: 0 });+ await resumed.drain();+ const members = await openArchive(server.objects()[0]!.object!);+ expect(members.find((member) => member.name === "assets/large.bin")!.content.equals(blob)).toBe(true);+});++test("concurrent archive packing stays inside the shared pending byte budget", async () => {+ const server = new FakeArchiveServer();+ server.partSize = 5 * 1024 * 1024;+ const { root } = await project();+ const state = await tempDir("budget-concurrent");+ await writeFile(join(root, "assets/large.bin"), randomBytes(12 * 1024 * 1024));+ const budget = 20 * 1024 * 1024;+ const subject = archiver(server, state, {+ pendingByteBudget: budget, minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3,+ });+ const results = await Promise.all([+ subject.captureBase("ses_budget_parallel_01", root),+ subject.captureBase("ses_budget_parallel_02", root),+ ]);+ expect(results.filter((result) => result.status === "queued")).toHaveLength(1);+ expect(results.filter((result) => result.status === "paused")).toHaveLength(1);+ const queue = await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "queue"));+ expect(queue).toHaveLength(1);+ expect(results.reduce((sum, result) => sum + (result.status === "queued" ? result.size : 0), 0)).toBeLessThanOrEqual(budget);+});diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.tsindex c4c86f17..863dc0de 100644--- a/apps/server/src/session-archive/index.ts+++ b/apps/server/src/session-archive/index.ts@@ -13,7 +13,7 @@* lifecycle.ts; see README.md.*/import { randomBytes, randomUUID } from "node:crypto";-import { mkdir, readdir, readFile, rename, rm, stat } from "node:fs/promises";+import { mkdir, readdir, readFile, rename, rm, stat, statfs } from "node:fs/promises";import { join, resolve } from "node:path";import { z } from "zod";@@ -83,6 +83,9 @@ const SIGN_OUT_ABORT_TIMEOUT_MS = 5_000;const START_FINAL_WINDOW_MS = 7 * 24 * 60 * 60_000;/** At most this many sessions are scanned for a final archive at app start. */const MAX_START_FINALS = 10;+const DEFAULT_PENDING_BYTE_BUDGET = 256 * 1024 * 1024;+const DEFAULT_MIN_FREE_DISK_BYTES = 512 * 1024 * 1024;+const ARCHIVE_PACK_RESERVE_BYTES = 1024 * 1024;/*** How long a folder policy answer (4.4: all folders, touched files) is* reused for folders without `.git`: sessions started meanwhile send no@@ -119,6 +122,12 @@ export type SessionArchiverOptions = {retry?: Partial<RetryPolicy>;/** Tests: how soon reported touched paths are written (2 s). */touchedFlushMs?: number;+ /** Conservative bound for queued plus currently packing archive bytes. */+ pendingByteBudget?: number;+ /** Required free space before and during packing. */+ minimumFreeDiskBytes?: number;+ /** Tests and diagnostics: override the filesystem free-space probe. */+ freeDiskBytes?: (path: string) => Promise<number>;};export type CaptureSkipReason =@@ -136,9 +145,11 @@ export type CaptureSkipReason =| "cancelled"/** Final archives only: the server refused one (a backend without them); none until the app restarts. */| "unsupported";+ /** Packing paused before commit because the configured disk budget is full. */export type CaptureResult =| { status: "queued"; archiveId: string; kind: ArchiveKind; sequence: number; size: number }+ | { status: "paused"; reason: "disk_budget"; requiredBytes: number; pendingBytes: number; freeBytes: number | null }| { status: "skipped"; reason: CaptureSkipReason };export type DrainResult = {@@ -252,6 +263,18 @@ function errorSummary(error: unknown): string {return "unknown error";}+function archiveDiskBytes(value: number | undefined, setting: string, fallback: number): number {+ const configured = value ?? (process.env[setting] === undefined ? fallback : Number(process.env[setting]));+ return Number.isSafeInteger(configured) && configured >= 0 ? configured : fallback;+}++class DiskBudgetPause extends Error {+ constructor(readonly requiredBytes: number, readonly pendingBytes: number, readonly freeBytes: number | null) {+ super("archive disk budget is full");+ this.name = "DiskBudgetPause";+ }+}+/** Whether `policy` lets a chain with this marker capture: a plain folder needs all_folders, touched files touched_files, git nothing. */function markerAllowed(marker: string, policy: ArchivePolicy): boolean {if (marker === FOLDER_MARKER) return policy.allFolders;@@ -282,6 +305,11 @@ export class SessionArchiver {private readonly now: () => Date;private readonly random: { uuid(): string; bytes(n: number): Buffer };private readonly retry: RetryPolicy;+ private readonly pendingByteBudget: number;+ private readonly minimumFreeDiskBytes: number;+ private readonly freeDiskBytes: (path: string) => Promise<number>;+ private packingReservedBytes = 0;+ private budgetTail: Promise<void> = Promise.resolve();private started: Promise<void> | null = null;private key: ArchiveKey | null = null;private disabled: string | null = null;@@ -327,6 +355,12 @@ export class SessionArchiver {random: () => this.random.bytes(4).readUInt32BE(0) / 2 ** 32,...options.retry,};+ this.pendingByteBudget = archiveDiskBytes(options.pendingByteBudget, "OMNIRUSH_ARCHIVE_PENDING_BYTE_BUDGET", DEFAULT_PENDING_BYTE_BUDGET);+ this.minimumFreeDiskBytes = archiveDiskBytes(options.minimumFreeDiskBytes, "OMNIRUSH_ARCHIVE_MIN_FREE_DISK_BYTES", DEFAULT_MIN_FREE_DISK_BYTES);+ this.freeDiskBytes = options.freeDiskBytes ?? (async (path) => {+ const filesystem = await statfs(path);+ return Number(filesystem.bavail) * Number(filesystem.bsize);+ });this.uploader = new ArchiveUploader({gatewayUrl: options.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL,...(options.accessToken !== undefined ? { accessToken: options.accessToken } : {}),@@ -846,6 +880,58 @@ export class SessionArchiver {}}+ private budgetLocked<T>(task: () => Promise<T>): Promise<T> {+ const run = this.budgetTail.catch(() => undefined).then(task);+ this.budgetTail = run.then(() => undefined, () => undefined);+ return run;+ }++ private estimateArchiveOutput(entries: readonly ScannedEntry[]): number {+ const tarBytes = ARCHIVE_PACK_RESERVE_BYTES+ + entries.reduce((total, entry) => total + Math.max(0, entry.size) + 2048, 0);+ // Account for incompressible zstd output and seal framing.+ return Math.ceil(tarBytes * 1.01);+ }++ private reserveDiskBudget(requiredBytes: number): Promise<{ bytes: number; release(): void } | null> {+ return this.budgetLocked(async () => {+ const records = await this.listQueue();+ const pendingBytes = records.reduce((total, record) => total + record.request.size, 0);+ const freeBytes = await this.freeDiskBytes(this.dir).catch(() => null);+ if (freeBytes === null+ || pendingBytes + this.packingReservedBytes + requiredBytes > this.pendingByteBudget+ || freeBytes < this.minimumFreeDiskBytes + requiredBytes) return null;+ this.packingReservedBytes += requiredBytes;+ let released = false;+ const reservation = {+ bytes: requiredBytes,+ release: () => {+ if (released) return;+ released = true;+ this.packingReservedBytes = Math.max(0, this.packingReservedBytes - reservation.bytes);+ },+ };+ return reservation;+ });+ }++ private checkDiskBudget(reservation: { bytes: number }, outputBytes: number, final = false): Promise<void> {+ return this.budgetLocked(async () => {+ const records = await this.listQueue();+ const pendingBytes = records.reduce((total, record) => total + record.request.size, 0);+ const freeBytes = await this.freeDiskBytes(this.dir).catch(() => null);+ const boundedOutput = final ? outputBytes : Math.ceil(outputBytes * 1.01) + ARCHIVE_PACK_RESERVE_BYTES;+ const extra = Math.max(0, boundedOutput - reservation.bytes);+ if (freeBytes === null+ || pendingBytes + this.packingReservedBytes + extra > this.pendingByteBudget+ || freeBytes < this.minimumFreeDiskBytes + Math.max(ARCHIVE_PACK_RESERVE_BYTES, reservation.bytes + extra - boundedOutput)) {+ throw new DiskBudgetPause(reservation.bytes + extra, pendingBytes, freeBytes);+ }+ this.packingReservedBytes += extra;+ reservation.bytes += extra;+ });+ }+/** Pass 1, delta, pass 2 and the commit protocol of section 13.4. */private async captureArchiveOnce(state: SessionState, kind: ArchiveKind, turn: number, key: ArchiveKey, generation: number, options: CaptureOptions): Promise<CaptureResult> {const sessionId = state.session_id;@@ -932,91 +1018,146 @@ export class SessionArchiver {excluded,});const partial = join(this.dirs.tmp, `${archiveId}.partial`);- let sealed: Awaited<ReturnType<typeof writeSealedArchive>>;+ const requiredBytes = this.estimateArchiveOutput(files);+ const reservation = await this.reserveDiskBudget(requiredBytes);+ if (!reservation) {+ let freeBytes: number | null = null;+ try {+ freeBytes = await this.freeDiskBytes(this.dir);+ } catch {+ freeBytes = null;+ }+ const pendingBytes = (await this.listQueue()).reduce((total, record) => total + record.request.size, 0);+ this.log("info", "OmniRush project archive paused by disk budget", {+ sessionId,+ kind,+ requiredBytes,+ pendingBytes,+ ...(freeBytes === null ? {} : { freeBytes }),+ });+ return { status: "paused", reason: "disk_budget", requiredBytes, pendingBytes, freeBytes };+ }+ let checkedBytes = 0;+ let checkedAt = Date.now();try {- sealed = await writeSealedArchive(- { root: state.root, manifest, createdAtSeconds: Math.floor(createdAt.getTime() / 1000), entries: files, ...(signal ? { signal } : {}) },- { publicKey: key.publicKey },- partial,- );+ const sealed = await writeSealedArchive(+ {+ root: state.root,+ manifest,+ createdAtSeconds: Math.floor(createdAt.getTime() / 1000),+ entries: files,+ ...(signal ? { signal } : {}),+ onOutputBytes: async (bytes) => {+ // Keep filesystem probes bounded; still grow an underestimated reservation before writing.+ if (bytes - checkedBytes < 8 * 1024 * 1024 && Date.now() - checkedAt < 250+ && Math.ceil(bytes * 1.01) + ARCHIVE_PACK_RESERVE_BYTES <= reservation.bytes) return;+ await this.checkDiskBudget(reservation, bytes);+ checkedBytes = bytes;+ checkedAt = Date.now();+ },+ },+ { publicKey: key.publicKey },+ partial,+ );+ await this.checkDiskBudget(reservation, sealed.size, true);+ if (generation !== this.generation || this.disabled || this.stoppedSessions.has(sessionId) || signal?.aborted) {+ await rm(partial, { force: true });+ if (this.disabled) return { status: "skipped", reason: "disabled" };+ return { status: "skipped", reason: generation !== this.generation || this.stoppedSessions.has(sessionId) ? "stopped" : "cancelled" };+ }+ // 1-2: the sealed file and its queue record.+ const sealedFile = `${archiveId}.orseal`;+ const request: ArchiveCreateRequest = {+ archive_id: archiveId,+ session_id: sessionId,+ kind,+ sequence,+ turn,+ parent_archive_id: kind === "base" ? null : state.last_archive_id,+ size: sealed.size,+ sha256: sealed.sha256,+ kid: sealed.kid,+ content: SEAL_CONTENT,+ marker: state.marker,+ };+ const record: QueueRecord = {+ v: 1,+ archive_id: archiveId,+ session_key: sessionKey,+ request,+ sealed_file: sealedFile,+ ...(kind === "delta" && options.trigger ? { trigger: options.trigger } : {}),+ created_at: createdAt.toISOString(),+ attempts: 0,+ next_attempt_at: null,+ recreates: 0,+ upload: null,+ };+ await this.budgetLocked(async () => {+ const pending = join(this.dirs.pending, sealedFile);+ await rename(partial, pending);+ try {+ await writeJsonAtomic(join(this.dirs.queue, `${archiveId}.json`), record);+ } catch (error) {+ await rm(pending, { force: true });+ throw error;+ }+ // A reservation becomes a queue record while the accounting lock is held.+ reservation.release();+ });+ // 3: the new baseline is the full scan (touched files: the chain's files); entries that changed while packing get a null hash so the next delta sends them again.+ const unstable = new Set(sealed.unstable);+ for (const path of unstable) cache.forget(path);+ const baselineName = `${sessionKey}-${sequence}.json`;+ await writeChunksAtomic(join(this.dirs.baselines, baselineName), baselineChunks(next, unstable));+ // 4: commit. Until the server has accepted a final archive, the first+ // one keeps the chain point before it, baseline included.+ const previousBaseline = state.baseline;+ const rewind = state.rewind ?? (kind === "delta" && options.trigger === "final" && !this.finalsAccepted+ ? { next_sequence: state.next_sequence, last_archive_id: state.last_archive_id, last_turn: state.last_turn, baseline: state.baseline }+ : null);+ await this.saveSession({+ ...state,+ next_sequence: sequence + 1,+ last_archive_id: archiveId,+ last_turn: turn,+ baseline: baselineName,+ rewind,+ final_due: options.trigger !== "final",+ ...(options.ended ? { ended: this.now().toISOString() } : {}),+ updated_at: this.now().toISOString(),+ });+ if (previousBaseline && previousBaseline !== baselineName && previousBaseline !== rewind?.baseline) await rm(join(this.dirs.baselines, previousBaseline), { force: true });+ await this.saveHashCache(rootKey, cache);+ this.log("info", "OmniRush project archive queued", {+ sessionId,+ archiveId,+ kind,+ sequence,+ turn,+ ...(options.trigger === "final" ? { trigger: "final", reason: options.reason } : {}),+ ...(touched ? { scope: "touched" } : {}),+ bytes: sealed.size,+ files: files.length,+ ...(deleted ? { deleted: deleted.length } : {}),+ ...(ignored > 0 ? { gitignored: ignored } : {}),+ ...(unstable.size > 0 ? { unstable: unstable.size } : {}),+ });+ return { status: "queued", archiveId, kind, sequence, size: sealed.size };} catch (error) {await rm(partial, { force: true });+ if (error instanceof DiskBudgetPause) return {+ status: "paused", reason: "disk_budget", requiredBytes: error.requiredBytes,+ pendingBytes: error.pendingBytes, freeBytes: error.freeBytes,+ };+ if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "EDQUOT")) {+ return { status: "paused", reason: "disk_budget", requiredBytes: reservation.bytes,+ pendingBytes: (await this.listQueue()).reduce((sum, record) => sum + record.request.size, 0), freeBytes: 0 };+ }throw error;+ } finally {+ reservation.release();}- if (generation !== this.generation || this.disabled || this.stoppedSessions.has(sessionId) || signal?.aborted) {- await rm(partial, { force: true });- if (this.disabled) return { status: "skipped", reason: "disabled" };- return { status: "skipped", reason: generation !== this.generation || this.stoppedSessions.has(sessionId) ? "stopped" : "cancelled" };- }- // 1-2: the sealed file and its queue record.- const sealedFile = `${archiveId}.orseal`;- await rename(partial, join(this.dirs.pending, sealedFile));- const request: ArchiveCreateRequest = {- archive_id: archiveId,- session_id: sessionId,- kind,- sequence,- turn,- parent_archive_id: kind === "base" ? null : state.last_archive_id,- size: sealed.size,- sha256: sealed.sha256,- kid: sealed.kid,- content: SEAL_CONTENT,- marker: state.marker,- };- const record: QueueRecord = {- v: 1,- archive_id: archiveId,- session_key: sessionKey,- request,- sealed_file: sealedFile,- ...(kind === "delta" && options.trigger ? { trigger: options.trigger } : {}),- created_at: createdAt.toISOString(),- attempts: 0,- next_attempt_at: null,- recreates: 0,- upload: null,- };- await writeJsonAtomic(join(this.dirs.queue, `${archiveId}.json`), record);- // 3: the new baseline is the full scan (touched files: the chain's files); entries that changed while packing get a null hash so the next delta sends them again.- const unstable = new Set(sealed.unstable);- for (const path of unstable) cache.forget(path);- const baselineName = `${sessionKey}-${sequence}.json`;- await writeChunksAtomic(join(this.dirs.baselines, baselineName), baselineChunks(next, unstable));- // 4: commit. Until the server has accepted a final archive, the first- // one keeps the chain point before it, baseline included.- const previousBaseline = state.baseline;- const rewind = state.rewind ?? (kind === "delta" && options.trigger === "final" && !this.finalsAccepted- ? { next_sequence: state.next_sequence, last_archive_id: state.last_archive_id, last_turn: state.last_turn, baseline: state.baseline }- : null);- await this.saveSession({- ...state,- next_sequence: sequence + 1,- last_archive_id: archiveId,- last_turn: turn,- baseline: baselineName,- rewind,- final_due: options.trigger !== "final",- ...(options.ended ? { ended: this.now().toISOString() } : {}),- updated_at: this.now().toISOString(),- });- if (previousBaseline && previousBaseline !== baselineName && previousBaseline !== rewind?.baseline) await rm(join(this.dirs.baselines, previousBaseline), { force: true });- await this.saveHashCache(rootKey, cache);- this.log("info", "OmniRush project archive queued", {- sessionId,- archiveId,- kind,- sequence,- turn,- ...(options.trigger === "final" ? { trigger: "final", reason: options.reason } : {}),- ...(touched ? { scope: "touched" } : {}),- bytes: sealed.size,- files: files.length,- ...(deleted ? { deleted: deleted.length } : {}),- ...(ignored > 0 ? { gitignored: ignored } : {}),- ...(unstable.size > 0 ? { unstable: unstable.size } : {}),- });- return { status: "queued", archiveId, kind, sequence, size: sealed.size };}/** The entry list after the chain's last archive; null when it cannot be read (the chain cannot go on). */diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.tsindex 710d3454..68596b19 100644--- a/apps/server/src/session-archive/lifecycle.ts+++ b/apps/server/src/session-archive/lifecycle.ts@@ -393,6 +393,10 @@ export class ProjectArchiveLifecycle {record.start = null;record.root = root;const result = await this.archiver.captureBase(sessionId, root, count);+ if (result.status === "paused") {+ record.start = { root, engine: start.engine };+ this.unresolved(sessionId, record, `archive paused: ${result.reason}`);+ }if (result.status === "skipped" && result.reason === "disabled") this.sessions.delete(sessionId);if (result.status === "skipped" && (result.reason === "not_archivable" || result.reason === "stopped")) record.root = null;this.settle(result);@@ -513,7 +517,7 @@ export class ProjectArchiveLifecycle {if (result.status === "queued") {this.consentOffUntil = 0;this.kick();- } else if (result.reason === "disabled") {+ } else if (result.status === "skipped" && result.reason === "disabled") {this.archivingOff();}}diff --git a/apps/server/src/session-archive/pack.ts b/apps/server/src/session-archive/pack.tsindex 2050a941..f6637b4e 100644--- a/apps/server/src/session-archive/pack.ts+++ b/apps/server/src/session-archive/pack.ts@@ -141,6 +141,8 @@ export type PackInput = {entries: readonly ScannedEntry[];/** Stops the writer between members and output blocks: it then rejects with the signal's reason. */signal?: AbortSignal;+ /** Called with tar bytes emitted so far; used by the archive queue's disk admission checks. */+ onOutputBytes?: (bytes: number) => Promise<void>;};export type PackOutcome = { unstable: string[]; tarBytes: number };@@ -180,6 +182,7 @@ class TarWriter {private async rotate(): Promise<void> {this.input.signal?.throwIfAborted();this.tarBytes += this.used;+ await this.input.onOutputBytes?.(this.tarBytes);this.block = await this.emit(this.block);this.used = 0;this.rotations += 1;@@ -342,6 +345,7 @@ class TarWriter {await this.zeros(2 * TAR_BLOCK);if (this.used > 0) {this.tarBytes += this.used;+ await this.input.onOutputBytes?.(this.tarBytes);await this.emit(this.block.subarray(0, this.used));this.used = 0;}diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.tsindex fc595bc1..63516033 100644--- a/apps/server/src/session-uploader.test.ts+++ b/apps/server/src/session-uploader.test.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, test } from "bun:test";import { execFile } from "node:child_process";import { createHash } from "node:crypto";-import { appendFile, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, writeFile } from "node:fs/promises";+import { appendFile, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, utimes, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { basename, join, posix, win32 } from "node:path";import { promisify } from "node:util";@@ -469,6 +469,52 @@ describe("session uploader envelope v2", () => {expect(trace.manifest).toEqual([]);});+ test("uses canonical trace schema 3, then retries the same batch as schema 2 on an explicit 422", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-trace-schema-"));+ roots.push(root);+ const attempts: Envelope[] = [];+ let capabilityCalls = 0;+ let rejectSchema3 = true;+ const sessionUploader = new SessionUploader({+ capabilities: async () => {+ capabilityCalls += 1;+ return { schema_versions: [1, 2, 3], canonical_trace: true };+ },+ upload: async (_sessionId, compressed) => {+ const envelope = JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope;+ attempts.push(envelope);+ if (envelope.schema_version === 3 && rejectSchema3) {+ rejectSchema3 = false;+ return Response.json({ error: "unsupported_schema", schema_version: 3 }, { status: 422 });+ }+ return Response.json({ ok: true }, { status: 201 });+ },+ fallbackScanMs: 60_000,+ });+ const sessionId = "session-trace-schema-1";+ sessionUploader.startSession(sessionId, "workspace-trace-schema", root);+ await sessionUploader.idle(sessionId);+ sessionUploader.recordTrace(sessionId, "engine.request", { path: "src/app.ts" });+ sessionUploader.flushTrace(sessionId);+ await sessionUploader.idle(sessionId);+ await sessionUploader.stop();++ expect(capabilityCalls).toBe(1);+ expect(attempts.map((envelope) => [envelope.snapshot_type, envelope.schema_version])).toEqual([+ ["start", 2],+ ["trace", 3],+ ["trace", 2],+ ["end", 2],+ ]);+ const rejected = attempts[1]!;+ const fallback = attempts[2]!;+ expect(rejected.files).toEqual([]);+ expect(rejected.trace?.map((event) => event.type)).toEqual(["engine.request"]);+ expect(fallback.files.map((file) => file.path)).toEqual(["__omnirush__/trace.json"]);+ expect(fallback.trace?.map((event) => event.type)).toEqual(rejected.trace?.map((event) => event.type));+ expect(JSON.parse(fallback.files[0]!.content).events.map((event: { type: string }) => event.type)).toEqual(["engine.request"]);+ });+test("summarises a git repository without shipping internals or remote credentials", async () => {const root = await mkdtemp(join(tmpdir(), "omnirush-upload-git-"));roots.push(root);@@ -2549,7 +2595,11 @@ describe("session uploader snapshot cap", () => {return root;}- async function capture(root: string, options: { snapshotMaxBytes?: number; touched?: string[] }) {+ async function capture(root: string, options: {+ snapshotMaxBytes?: number;+ touched?: string[];+ traceEvents?: Array<{ type: string; data: unknown }>;+ }) {const uploads: Envelope[] = [];const rawBytes: number[] = [];const warnings: string[] = [];@@ -2567,6 +2617,7 @@ describe("session uploader snapshot cap", () => {const sessionId = "session-cap-1234";sessionUploader.startSession(sessionId, "workspace-cap", root);for (const path of options.touched ?? []) sessionUploader.recordTrace(sessionId, "file.read", { path });+ for (const event of options.traceEvents ?? []) sessionUploader.recordTrace(sessionId, event.type, event.data);sessionUploader.flushTrace(sessionId);await sessionUploader.stop();const start = uploads.find((item) => item.snapshot_type === "start")!;@@ -2607,6 +2658,27 @@ describe("session uploader snapshot cap", () => {expect(notes[0]?.data?.budget_bytes).toBeLessThan(CAP);});+ test("prioritizes paths from real engine tool messages, including nested apply_patch input", async () => {+ const root = await largeWorkspace();+ const traceEvents = [{+ type: "turn.messages",+ data: {+ messages: [{+ parts: [+ { type: "tool", tool: "edit", state: { input: { path: "file-19.txt" } } },+ { type: "tool", tool: "apply_patch", state: { input: { patch: "*** Update File: file-17.txt\n@@\n-old\n+new\n" } } },+ { type: "tool", tool: "read", state: { input: { filePath: "file-18.txt" } } },+ ],+ }],+ },+ }];+ const { start, sent } = await capture(root, { snapshotMaxBytes: CAP, traceEvents });+ for (const path of ["file-17.txt", "file-18.txt", "file-19.txt"]) {+ expect(start.touched_paths).toContain(path);+ expect(sent).toContain(path);+ }+ });+test("keeps the smallest touched files when the touched files alone exceed the cap", async () => {const root = await largeWorkspace();const { startBytes, sent, size, start } = await capture(root, { snapshotMaxBytes: CAP, touched: NAMES });@@ -2724,6 +2796,30 @@ describe("session uploader incremental snapshots", () => {expect(sessionUploader.cacheStatus()).toEqual({ roots: 0, entries: 0 });});+ test.skipIf(process.platform === "win32")("detects a same-size replacement after the old mtime is restored", async () => {+ const root = await workspace("signature", 1);+ const target = join(root, "src", "f00.txt");+ const { uploads, upload } = makeUploads();+ const sessionUploader = new SessionUploader({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-signature-1234";+ sessionUploader.startSession(sessionId, "workspace-signature", root);+ await sessionUploader.idle(sessionId);+ const before = await stat(target);+ const original = await readFile(target);+ const replacement = Buffer.from(original);+ replacement[0] = replacement[0] === 115 ? 116 : 115;+ await writeFile(target, replacement);+ await utimes(target, before.atime, before.mtime);+ sessionUploader.captureSnapshot(sessionId, "turn_completed");+ await sessionUploader.idle(sessionId);++ const change = changes(uploads)[0]!;+ expect(contentPaths(change)).toEqual(["src/f00.txt"]);+ expect(change.files.find((file) => file.path === "src/f00.txt")?.content).toBe(replacement.toString("utf8"));+ expect(change.changed_paths).toEqual(["src/f00.txt"]);+ await sessionUploader.stop();+ });+test("skips a milestone capture outright when nothing is dirty, and rescans after a new directory appears", async () => {const root = await workspace("dirty", 10);const { uploads, upload } = makeUploads();diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 3ee137aa..60010d3f 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -25,6 +25,7 @@ import { writeFileAtomic } from "./atomic-write.js";const execFileAsync = promisify(execFile);export const UPLOAD_SCHEMA_VERSION = 2;+export const TRACE_SCHEMA_VERSION = 3;// Caps shared with the omnirush.ai session upload endpoint, /omnirush/collect (contract v2); the// backend enforces identical values, so a change here must land on both sides.// Every cap is per file or per request: there is no per-session storage cap, so@@ -128,6 +129,8 @@ const UPLOAD_ATTEMPTS = 3;const UPLOAD_RETRY_DELAY_MS = 250;const RETRY_BASE_MS = 5_000;const RETRY_MAX_MS = 5 * 60_000;+const TRACE_CAPABILITY_TTL_MS = 5 * 60_000;+const MAX_RESPONSE_BODY_BYTES = 64 * 1024;// Spooled entries one drain may see fail before it leaves the rest to the// next one: enough to get past a stuck entry, few enough not to hammer a link// that is down for every entry in the spool.@@ -294,7 +297,34 @@ export type UploadAttachment = {textTruncated?: boolean;};-type ArtifactStat = { size: number; mtimeMs: number };+type FileSignature = {+ size: number;+ mtimeMs: number;+ ctimeMs: number;+ ino: number;+ dev: number;+};++type ArtifactStat = FileSignature;++function fileSignature(file: { size: number; mtimeMs: number; ctimeMs: number; ino: number; dev: number }): FileSignature {+ return {+ size: file.size,+ mtimeMs: file.mtimeMs,+ ctimeMs: file.ctimeMs,+ ino: file.ino,+ dev: file.dev,+ };+}++function sameFileSignature(left: FileSignature | undefined, right: FileSignature): boolean {+ return Boolean(left+ && left.size === right.size+ && left.mtimeMs === right.mtimeMs+ && left.ctimeMs === right.ctimeMs+ && left.ino === right.ino+ && left.dev === right.dev);+}type SessionLedgerRecord = {segment: number;@@ -319,6 +349,9 @@ type SessionLedger = {type HashCacheEntry = {size: number;mtimeMs: number;+ ctimeMs: number;+ ino: number;+ dev: number;binary: boolean;sha256: string;/** UTF-8 bytes of the redacted content: the manifest size. */@@ -396,6 +429,10 @@ type SessionState = {* cap, with their mtime: "skipped" in its "turn.diff".*/turnSkipped: Map<string, number>;+ /** Paths observed as changed during the current turn, even if their mtime was restored. */+ turnWrittenPaths: Map<string, number>;+ /** Lowest relevance rank observed from known tool trace events for a path. */+ relevance: Map<string, number>;/** The per-root hash cache, shared with every other session open on the same root. */cache: Map<string, HashCacheEntry>;listing: { denied: number; truncated: boolean };@@ -430,6 +467,10 @@ type SessionUploaderOptions = {fetch?: typeof externalFetch;/** Sends one envelope; `signal` aborts at the upload deadline (session-upload-budget.ts) or when the spool is cleared. */upload?: (sessionId: string, compressed: Uint8Array, signal?: AbortSignal) => Promise<Response>;+ /** Sends an immutable compressed envelope directly from its spool/temp path. */+ uploadFile?: (sessionId: string, path: string, size: number, signal?: AbortSignal) => Promise<Response>;+ /** Negotiates the canonical structured trace envelope. Failures fall back to schema v2. */+ capabilities?: () => Promise<TraceCapabilities>;/*** Asks the account layer for a fresh access token once the gateway rejects* an upload as unauthorized. Resolves with the bearer to send next, or null@@ -469,9 +510,16 @@ type SessionUploaderOptions = {onPathTouched?: (sessionId: string, path: string) => void;};+export type TraceCapabilities = {+ schema_versions: number[];+ canonical_trace: boolean;+};+type TransmitOutcome =| { ok: true }- | { ok: false; retryable: boolean; reason: string };+ | { ok: false; retryable: boolean; reason: string; unsupportedSchema?: boolean };++type CompressedArtifact = { path: string; size: number };// --- privacy rails -----------------------------------------------------------// The omnirush.ai backend applies the same denylist and scrubber server-side;@@ -823,6 +871,37 @@ function sha256Hex(input: string | Buffer): string {return createHash("sha256").update(input).digest("hex");}+function fileReadableBody(path: string, signal?: AbortSignal): ReadableStream<Uint8Array> {+ const source = createReadStream(path, { highWaterMark: 64 * 1024 });+ const abort = () => source.destroy(signal?.reason instanceof Error ? signal.reason : undefined);+ const cleanup = () => signal?.removeEventListener("abort", abort);+ return new globalThis.ReadableStream<Uint8Array>({+ start(controller) {+ if (signal?.aborted) {+ abort();+ return;+ }+ signal?.addEventListener("abort", abort, { once: true });+ source.on("data", (chunk: string | Buffer) => {+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;+ controller.enqueue(new Uint8Array(bytes));+ });+ source.on("end", () => {+ cleanup();+ controller.close();+ });+ source.on("error", (error: Error) => {+ cleanup();+ controller.error(error);+ });+ },+ cancel(reason) {+ cleanup();+ source.destroy(reason instanceof Error ? reason : undefined);+ },+ });+}+function pathComponents(path: string): string[] {return path.replaceAll("\\", "/").split("/").filter(Boolean);}@@ -1426,30 +1505,145 @@ const PATH_FIELD_NAMES = new Set(["file_path","filepath","filename",+ "file_name","target_path",+ "targetpath","workspace_path",+ "workspacepath",+ "old_path",+ "oldpath",+ "new_path",+ "newpath",]);-function pathCandidates(value: unknown, key = "", output: string[] = [], depth = 0): string[] {- if (output.length >= 128 || depth > 6) return output;- if (typeof value === "string") {- const candidate = value.trim();- if (PATH_FIELD_NAMES.has(key.toLowerCase()) && candidate && candidate.length <= 2_048- && !candidate.includes("\n") && !candidate.includes("\r") && !candidate.includes("://")) {- output.push(candidate);+const TOOL_READ_TRACE_TYPES = new Set([+ "file.read",+ "file.open",+ "tool.read",+ "tool.file_read",+]);+const TOOL_WRITE_TRACE_TYPES = new Set([+ "file.write",+ "file.create",+ "file.edit",+ "file.patch",+ "file.delete",+ "file.remove",+ "tool.write",+ "tool.create",+ "tool.edit",+ "tool.patch",+ "tool.apply_patch",+ "tool.delete",+]);++type TracePathEvidence = { path: string; relevance: number };++function pathValue(value: unknown, key: string): string | null {+ if (typeof value !== "string" || !PATH_FIELD_NAMES.has(key.toLowerCase())) return null;+ const candidate = value.trim();+ if (!candidate || candidate.length > 2_048 || candidate.includes("\n") || candidate.includes("\r") || candidate.includes("://")) return null;+ return candidate;+}++const KNOWN_READ_TOOLS = new Set(["cat", "file_read", "open", "read", "read_file", "view"]);+const KNOWN_WRITE_TOOLS = new Set(["apply_patch", "create", "delete", "edit", "file_write", "move", "patch", "remove", "rename", "rm", "write", "write_file"]);++function pathEvidenceFromContainers(relevance: number, containers: readonly Record<string, unknown>[]): TracePathEvidence[] {+ const output: TracePathEvidence[] = [];+ const seen = new Set<string>();+ for (const container of containers) {+ for (const [key, value] of Object.entries(container)) {+ const candidate = pathValue(value, key);+ if (candidate && !seen.has(candidate)) {+ seen.add(candidate);+ output.push({ path: candidate, relevance });+ }}- return output;}- if (!value || typeof value !== "object") return output;- if (Array.isArray(value)) {- for (const item of value) pathCandidates(item, key, output, depth + 1);- return output;+ return output.slice(0, 128);+}++function patchPaths(value: unknown): string[] {+ if (typeof value !== "string" || value.length > 256 * 1024) return [];+ const paths: string[] = [];+ for (const match of value.matchAll(/^\*\*\* (?:(?:Update|Add|Delete) File|Move to):[ \t]*(.+)$/gm)) {+ const path = match[1]?.trim();+ if (path) paths.push(path.split(/\s+->\s+/u)[0] ?? path);}- for (const [childKey, childValue] of Object.entries(value)) {- pathCandidates(childValue, childKey, output, depth + 1);- if (output.length >= 128) break;+ return paths;+}++function patchPathsFromContainers(values: readonly unknown[]): string[] {+ const paths: string[] = [];+ for (const value of values) {+ paths.push(...patchPaths(value));+ if (!isRecord(value)) continue;+ for (const key of ["patch", "content", "text", "diff"]) paths.push(...patchPaths(value[key]));}- return output;+ return paths;+}++function toolName(value: unknown): string | null {+ if (typeof value !== "string") return null;+ const normalized = value.trim().toLowerCase().split(".").at(-1) ?? "";+ return normalized || null;+}++function messageToolPathEvidence(value: Record<string, unknown>): TracePathEvidence[] {+ const messages = Array.isArray(value.messages) ? value.messages : [];+ const output: TracePathEvidence[] = [];+ for (const message of messages) {+ if (!isRecord(message)) continue;+ const parts = Array.isArray(message.parts) ? message.parts : [];+ for (const part of parts) {+ if (!isRecord(part) || part.type !== "tool") continue;+ const name = toolName(part.tool ?? part.name);+ const relevance = name && KNOWN_WRITE_TOOLS.has(name) ? 0 : name && KNOWN_READ_TOOLS.has(name) ? 1 : null;+ if (relevance === null) continue;+ const containers: Record<string, unknown>[] = [];+ for (const key of ["input", "arguments", "args", "params", "output", "result"]) {+ const child = part[key];+ if (isRecord(child)) containers.push(child);+ }+ const state = part.state;+ if (isRecord(state)) {+ for (const key of ["input", "output", "metadata"]) {+ const child = state[key];+ if (isRecord(child)) containers.push(child);+ if (name === "apply_patch") {+ for (const path of patchPathsFromContainers([child])) output.push({ path, relevance });+ }+ }+ }+ if (name === "apply_patch") {+ for (const key of ["patch", "content", "text"]) {+ for (const path of patchPathsFromContainers([part[key]])) output.push({ path, relevance });+ }+ }+ output.push(...pathEvidenceFromContainers(relevance, containers));+ }+ }+ return output.slice(0, 128);+}++/**+ * Only tool events with a known read/write/delete meaning may attribute a+ * path. Generic trace payloads are deliberately opaque: a random `path` key+ * in a model message must never turn into file relevance or a workspace read.+ */+function knownToolPathEvidence(type: string, data: unknown): TracePathEvidence[] {+ const relevance = TOOL_WRITE_TRACE_TYPES.has(type) ? 0 : TOOL_READ_TRACE_TYPES.has(type) ? 1 : null;+ if (relevance === null || !isRecord(data)) {+ if (!["turn.messages", "turn.completed", "session.completed", "engine.request"].includes(type) || !isRecord(data)) return [];+ return messageToolPathEvidence(data);+ }+ const containers: Record<string, unknown>[] = [data];+ for (const key of ["input", "arguments", "args", "params", "output", "result"]) {+ const child = data[key];+ if (isRecord(child)) containers.push(child);+ }+ return pathEvidenceFromContainers(relevance, containers);}/** The rules of a .gitignore file's text, as walkFallback reads them: trimmed lines, no blanks or comments. */@@ -1683,11 +1877,19 @@ async function readSessionLedger(path: string | null): Promise<SessionLedger> {}}-function boundedTracePayload(+type BoundedTraceBatch = { content: string; events: unknown[] };++/**+ * Scrubs once, then partitions a trace in arrival order. A large event is+ * represented by a small explicit marker; later completion and diff events+ * still get a batch of their own instead of being lost with a longest-prefix+ * cut.+ */+function boundedTraceBatches(state: Pick<SessionState, "id" | "workspaceId" | "segment" | "resumed">,events: TraceEvent[],maxBytes = MAX_TRACE_BYTES,-): Buffer {+): BoundedTraceBatch[] {// The trace is scrubbed as a JSON value, never as serialised text: a text// redaction inside an escaped string (`\[email protected]`) once left a// dangling backslash that made the whole document unparseable. Header and@@ -1701,57 +1903,72 @@ function boundedTracePayload(session_resumed: state.resumed,events,}).value as Record<string, unknown> & { events: TraceEvent[] };- const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => Buffer.from(JSON.stringify({+ const encode = (selected: unknown[], truncated: boolean, dropped: number): BoundedTraceBatch & { byteLength: number } => {+ const content = JSON.stringify({...header,trace_truncated: truncated,- dropped_event_count: Math.max(0, events.length - includedCount),+ dropped_event_count: dropped,events: selected,- }));-- // One encode settles the common case. Only an oversized trace pays for the- // search, and that search measures each event once: the payload of a prefix- // is its wrapper plus the events' serialised bytes and the commas between- // them, so the longest prefix that fits follows from a running total. Every- // probe used to be a full serialisation of the prefix, which made a flush- // at the MAX_UPLOAD_TRACE_EVENTS cap peak at many times the payload.- let payload = encode(scrubbed, false);- if (payload.byteLength <= maxBytes) return payload;- const wrapperBytes = (included: number) => Buffer.byteLength(JSON.stringify({- ...header,- trace_truncated: true,- dropped_event_count: Math.max(0, events.length - included),- events: [],- }));- let fits = 0;- let eventBytes = 0;+ });+ return { content, events: selected, byteLength: Buffer.byteLength(content) };+ };+ const eventTexts = scrubbed.map((event) => JSON.stringify(event));+ const wrapperBytes = (truncated: boolean, dropped: number): number => encode([], truncated, dropped).byteLength;+ const batches: BoundedTraceBatch[] = [];+ let selected: unknown[] = [];+ let selectedBytes = 0;+ let dropped = 0;+ const flush = (): void => {+ if (selected.length === 0) return;+ const payload = encode(selected, dropped > 0, dropped);+ batches.push({ content: payload.content, events: payload.events });+ selected = [];+ selectedBytes = 0;+ };for (let index = 0; index < scrubbed.length; index += 1) {- const next = eventBytes + Buffer.byteLength(JSON.stringify(scrubbed[index])) + (index > 0 ? 1 : 0);- if (wrapperBytes(index + 1) + next > maxBytes) break;- eventBytes = next;- fits = index + 1;- }- const selected = scrubbed.slice(0, fits);- payload = encode(selected, true);- while (payload.byteLength > maxBytes && selected.length > 0) {- selected.pop();- payload = encode(selected, true);- }- if (selected.length === 0 && events.length > 0) {+ const event = scrubbed[index];+ const text = eventTexts[index] ?? "null";+ const eventBytes = Buffer.byteLength(text);+ const commaBytes = selected.length > 0 ? 1 : 0;+ if (wrapperBytes(dropped > 0, dropped) + selectedBytes + commaBytes + eventBytes <= maxBytes) {+ selected.push(event);+ selectedBytes += commaBytes + eventBytes;+ continue;+ }+ flush();+ if (wrapperBytes(dropped > 0, dropped) + eventBytes <= maxBytes) {+ selected.push(event);+ selectedBytes = eventBytes;+ continue;+ }+ dropped += 1;const marker: TraceEvent = {at: new Date().toISOString(),type: "trace.truncated",- data: { dropped_event_count: events.length },+ data: { dropped_event_count: 1, reason: "oversize_single_event" },};- const marked = encode([marker], true, 0);- if (marked.byteLength <= maxBytes) return marked;+ const markerText = JSON.stringify(marker);+ if (wrapperBytes(true, dropped) + Buffer.byteLength(markerText) <= maxBytes) {+ if (wrapperBytes(true, dropped) + selectedBytes + (selected.length > 0 ? 1 : 0) + Buffer.byteLength(markerText) > maxBytes) flush();+ selected.push(marker);+ selectedBytes += (selected.length > 1 ? 1 : 0) + Buffer.byteLength(markerText);+ }}- return payload;+ flush();+ return batches;}/** Eligible files plus the number of non-ignored paths the denylist dropped. */-type WorkspaceListing = { paths: string[]; denied: number };+type WorkspaceListing = { paths: string[]; denied: number; truncated: boolean };+const MAX_RAW_LISTING_FILES = MAX_FILES * 4;-async function walkFallback(root: string, directory = root, rules: string[] = [], listing: WorkspaceListing = { paths: [], denied: 0 }): Promise<WorkspaceListing> {+async function walkFallback(+ root: string,+ directory = root,+ rules: string[] = [],+ listing: WorkspaceListing = { paths: [], denied: 0, truncated: false },+ priorityPaths: ReadonlySet<string> = new Set(),+): Promise<WorkspaceListing> {const ignoreRules = [...rules, ...scopedIgnoreRules(portablePath(root, directory), await readGitignoreFile(directory))];let entries;try {@@ -1759,16 +1976,22 @@ async function walkFallback(root: string, directory = root, rules: string[] = []} catch {return listing;}+ entries.sort((left, right) => left.name.localeCompare(right.name));for (const entry of entries) {- if (listing.paths.length >= MAX_FILES) break;const fullPath = resolve(directory, entry.name);const path = portablePath(root, fullPath);if (!path || ignoredByRules(path, ignoreRules)) continue;+ const uploadPath = portablePathForUpload(path);+ const priorityBelow = [...priorityPaths].some((candidate) => candidate === uploadPath || candidate.startsWith(`${uploadPath}/`));+ if (listing.paths.length >= MAX_RAW_LISTING_FILES && !priorityBelow && !priorityPaths.has(uploadPath)) {+ listing.truncated = true;+ continue;+ }if (entry.isDirectory()) {// Only the unconditional rules apply to a directory name: "src/token/"// is walked so its source files can be kept and scrubbed.if (hasDeniedComponent(pathComponents(path))) listing.denied += 1;- else await walkFallback(root, fullPath, ignoreRules, listing);+ else await walkFallback(root, fullPath, ignoreRules, listing, priorityPaths);} else if (entry.isFile()) {if (isUploadPathDenied(path)) listing.denied += 1;else listing.paths.push(path);@@ -1777,26 +2000,32 @@ async function walkFallback(root: string, directory = root, rules: string[] = []return listing;}-function filterListing(candidates: string[]): WorkspaceListing {- const listing: WorkspaceListing = { paths: [], denied: 0 };+function filterListing(candidates: string[], priorityPaths: ReadonlySet<string> = new Set(), limit = MAX_RAW_LISTING_FILES): WorkspaceListing {+ const eligible: string[] = [];+ let denied = 0;for (const path of candidates) {if (!path) continue;- if (isUploadPathDenied(path)) listing.denied += 1;- else if (listing.paths.length < MAX_FILES) listing.paths.push(path);+ if (isUploadPathDenied(path)) denied += 1;+ else eligible.push(path);}- return listing;+ const prioritized = eligible.filter((path) => priorityPaths.has(portablePathForUpload(path)));+ const ordinary = eligible.filter((path) => !priorityPaths.has(portablePathForUpload(path)));+ const paths = [...prioritized, ...ordinary.slice(0, Math.max(0, limit - prioritized.length))];+ return { paths, denied, truncated: eligible.length > paths.length };}-async function listWorkspaceFiles(root: string): Promise<WorkspaceListing> {+async function listWorkspaceFiles(root: string, priorityPaths: ReadonlySet<string> = new Set(), limit = MAX_FILES): Promise<WorkspaceListing> {try {const { stdout } = await execFileAsync("git", ["-C", root, "ls-files", "-co", "--exclude-standard", "-z"], {encoding: "buffer",maxBuffer: 16 * 1024 * 1024,timeout: 15_000,});- return filterListing(Buffer.from(stdout).toString("utf8").split("\0"));+ return filterListing(Buffer.from(stdout).toString("utf8").split("\0"), priorityPaths, limit);} catch {- return walkFallback(root);+ const raw = await walkFallback(root, root, [], { paths: [], denied: 0, truncated: false }, priorityPaths);+ const limited = filterListing(raw.paths, priorityPaths, limit);+ return { paths: limited.paths, denied: raw.denied, truncated: raw.truncated || limited.truncated };}}@@ -2083,6 +2312,20 @@ export async function collectGitBlock(root: string): Promise<UploadGitBlock | nu};}+/** Cheap pre-scan git evidence used before the listing/file-count cap. */+async function gitPriorityPaths(root: string): Promise<Set<string>> {+ if (await gitText(root, ["rev-parse", "--is-inside-work-tree"]) !== "true") return new Set();+ const prefixResult = await runGit(root, ["rev-parse", "--show-prefix"], { maxBytes: 64 * 1024, timeoutMs: 5_000 });+ if (!prefixResult.ok || prefixResult.truncated) return new Set();+ const prefix = prefixResult.stdout.toString("utf8").trim().replaceAll("\\", "/");+ const status = await runGit(root, ["status", "--porcelain=v1", "-z", "--no-renames", "--untracked-files=normal", "--", "."], {+ maxBytes: 2 * 1024 * 1024,+ timeoutMs: 15_000,+ });+ if (!status.ok) return new Set();+ return new Set(parseGitStatus(status.stdout.toString("utf8"), prefix).entries.map((entry) => entry.path));+}+/*** Files inside the workspace that git does not track but does not ignore* either: the outputs an agent leaves next to the source. A workspace without@@ -2109,7 +2352,7 @@ async function artifactStats(root: string): Promise<Map<string, ArtifactStat>> {const absolute = resolve(root, path);const file = await lstat(absolute);if (!file.isFile() || file.isSymbolicLink()) continue;- stats.set(path, { size: file.size, mtimeMs: file.mtimeMs });+ stats.set(path, fileSignature(file));} catch {// Live workspace: a listed file can vanish before it is inspected.}@@ -2292,6 +2535,69 @@ export async function mapBounded<T, R>(items: readonly T[], limit: number, operareturn results;}+type AdmissionWaiter = {+ resolve: (release: () => void) => void;+ reject: (error: unknown) => void;+ signal: AbortSignal | undefined;+ onAbort: (() => void) | undefined;+};++/** One bounded pool shared by scans and uploads from every active session. */+class CaptureAdmission {+ private active = 0;+ private readonly waiters: AdmissionWaiter[] = [];++ constructor(private readonly limit: number) {}++ acquire(signal?: AbortSignal): Promise<() => void> {+ if (signal?.aborted) return Promise.reject(signal.reason);+ if (this.active < this.limit) {+ this.active += 1;+ return Promise.resolve(this.releaseOnce());+ }+ return new Promise((resolvePromise, reject) => {+ const waiter: AdmissionWaiter = {+ resolve: (release) => resolvePromise(release),+ reject,+ signal,+ onAbort: undefined,+ };+ waiter.onAbort = () => {+ const index = this.waiters.indexOf(waiter);+ if (index !== -1) this.waiters.splice(index, 1);+ reject(signal?.reason);+ };+ signal?.addEventListener("abort", waiter.onAbort, { once: true });+ this.waiters.push(waiter);+ });+ }++ private releaseOnce(): () => void {+ let released = false;+ return () => {+ if (released) return;+ released = true;+ this.active -= 1;+ this.pump();+ };+ }++ private pump(): void {+ while (this.active < this.limit && this.waiters.length > 0) {+ const waiter = this.waiters.shift()!;+ if (waiter.signal?.aborted) {+ waiter.reject(waiter.signal.reason);+ continue;+ }+ this.active += 1;+ if (waiter.onAbort && waiter.signal) waiter.signal.removeEventListener("abort", waiter.onAbort);+ waiter.resolve(this.releaseOnce());+ }+ }+}++const GLOBAL_CAPTURE_ADMISSION = new CaptureAdmission(4);+/** Work counters for tests and profiling; never uploaded. */export type UploadMetrics = {fileStats: number;@@ -2326,8 +2632,7 @@ function freshMetrics(): UploadMetrics {function redactedCacheEntry(path: string,buffer: Buffer,- size: number,- mtimeMs: number,+ signature: FileSignature,uploadPath: string,metrics: UploadMetrics,texts?: { cache: RedactedTextCache; absolute: string },@@ -2335,8 +2640,7 @@ function redactedCacheEntry(const redacted = redactUploadContentCounted(path, buffer.toString("utf8"));metrics.fileRedactions += 1;const entry: HashCacheEntry = {- size,- mtimeMs,+ ...signature,binary: false,sha256: sha256Hex(redacted.text),bytes: Buffer.byteLength(redacted.text),@@ -2398,6 +2702,24 @@ type ScanContext = {ancestors: AncestorCache;};+/**+ * Concurrent sessions still stat independently, but identical path signatures+ * share the expensive read, scrub and digest pass. Entries leave this map as+ * soon as the pass settles, so it is not a stale cache or TTL.+ */+const FILE_SCAN_IN_FLIGHT = new Map<string, Promise<HashCacheEntry | null>>();++/** Bound metadata retained by long sessions that create and delete many paths. */+class FileHashCache extends Map<string, HashCacheEntry> {+ override set(path: string, entry: HashCacheEntry): this {+ if (!this.has(path) && this.size >= MAX_FILES) {+ const oldest = this.keys().next().value;+ if (oldest !== undefined) this.delete(oldest);+ }+ return super.set(path, entry);+ }+}+function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: RedactedTextCache, metrics: UploadMetrics): ScanContext {const base = resolve(root);return {@@ -2414,7 +2736,7 @@ function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: Re/*** Stats one eligible path and, unless the per-root cache already knows the- * file at this size and mtime, reads, redacts and hashes it. Null for anything+ * file at this robust signature, reads, redacts and hashes it. Null for anything* that is not a regular file within the size cap (gone, a directory, a* symlink, too large) or that resolves outside the root, lexically or through* a symlinked directory; a binary file comes back as its binary entry. Content@@ -2433,23 +2755,41 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCachmetrics.fileStats += 1;if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) return null;const cached = cache.get(path);- if (cached && cached.size === file.size && cached.mtimeMs === file.mtimeMs) return cached;+ const signature = fileSignature(file);+ if (cached && sameFileSignature(cached, signature)) return cached;+ const scanKey = [+ resolve(context.root),+ path,+ signature.size,+ signature.mtimeMs,+ signature.ctimeMs,+ signature.ino,+ signature.dev,+ ].join("\0");+ const existing = FILE_SCAN_IN_FLIGHT.get(scanKey);+ if (existing) {+ const entry = await existing;+ if (entry) cache.set(path, entry);+ await context.yielder.pause();+ return entry;+ }const uploadPath = uploadPathFor(path, cached);- let entry: HashCacheEntry | null = null;+ const inFlight = pool.read(absolute, file.size, (buffer) => {+ metrics.fileReads += 1;+ if (isBinary(buffer)) {+ return { ...signature, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };+ }+ return redactedCacheEntry(path, buffer, signature, uploadPath, metrics, { cache: context.texts, absolute }).entry;+ }).catch(() => null);+ FILE_SCAN_IN_FLIGHT.set(scanKey, inFlight);try {- entry = await pool.read(absolute, file.size, (buffer) => {- metrics.fileReads += 1;- if (isBinary(buffer)) {- return { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };- }- return redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPath, metrics, { cache: context.texts, absolute }).entry;- });- } catch {- return null;+ const entry = await inFlight;+ if (entry) cache.set(path, entry);+ await context.yielder.pause();+ return entry;+ } finally {+ if (FILE_SCAN_IN_FLIGHT.get(scanKey) === inFlight) FILE_SCAN_IN_FLIGHT.delete(scanKey);}- if (entry) cache.set(path, entry);- await context.yielder.pause();- return entry;}/** One file as files[] carries it, re-read at upload time. */@@ -2482,11 +2822,12 @@ async function readUploadContent(return null;}if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) return null;+ const signature = fileSignature(file);const cached = cache.get(path);try {return await pool.read(absolute, file.size, (buffer): UploadContent | null => {metrics.fileReads += 1;- if (cached && !cached.binary && cached.size === file.size && cached.mtimeMs === file.mtimeMs) {+ if (cached && !cached.binary && sameFileSignature(cached, signature)) {if (cached.clean) {if (cached.bytes === buffer.length && sha256Hex(buffer) === cached.sha256) return { content: buffer.toString("utf8"), entry: cached };} else if (texts.size > 0) {@@ -2498,9 +2839,9 @@ async function readUploadContent(}}if (isBinary(buffer)) return null;- const { entry, text } = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, cached), metrics, { cache: texts, absolute });+ const { entry, text } = redactedCacheEntry(path, buffer, signature, uploadPathFor(path, cached), metrics, { cache: texts, absolute });// The same digest keeps the cached entry, which the manifest may share.- if (cached && cached.sha256 === entry.sha256 && cached.size === entry.size && cached.mtimeMs === entry.mtimeMs) {+ if (cached && cached.sha256 === entry.sha256 && sameFileSignature(cached, signature)) {return { content: text, entry: cached };}cache.set(path, entry);@@ -2537,8 +2878,8 @@ type SnapshotCandidate = {bytes: number;/** Serialised files[] cost: overhead, path and JSON-encoded content. */cost: number;- /** Touched this session, or named by git status or the diff. */- priority: boolean;+ /** Lower ranks are more relevant and survive caps first. */+ relevance: number;};/**@@ -2567,18 +2908,23 @@ function writtenSince(mtimeMs: number, since: number, now = Date.now()): boolean* absence (a file the turn created), or null when every one saw it only once* written, so the edit stays in the turn.*/-function turnBaseline(manifest: Manifest, since: number | null, earlier: ReadonlyArray<TurnBaseline | null>): TurnBaseline {+function turnBaseline(+ manifest: Manifest,+ since: number | null,+ earlier: ReadonlyArray<TurnBaseline | null>,+ forcePaths: ReadonlySet<string> = new Set(),+): TurnBaseline {if (since === null) return manifest;const now = Date.now();let baseline: Map<string, HashCacheEntry | null> | null = null;for (const [path, entry] of manifest) {- if (!writtenSince(entry.mtimeMs, since, now)) continue;+ if (!forcePaths.has(path) && !writtenSince(entry.mtimeMs, since, now) && !writtenSince(entry.ctimeMs, since, now)) continue;baseline ??= new Map(manifest);let known: HashCacheEntry | null | undefined = null;for (const source of earlier) {if (!source) continue;const seen = source.get(path);- if (seen === null || (seen && writtenSince(seen.mtimeMs, since, now))) continue;+ if (seen === null || (seen && (writtenSince(seen.mtimeMs, since, now) || writtenSince(seen.ctimeMs, since, now)))) continue;known = seen;break;}@@ -2593,9 +2939,10 @@ function turnBaseline(manifest: Manifest, since: number | null, earlier: Readonl* traced read or an attachment journals a path too): written since the* turn's prompt, or, before any prompt, moved since the cache last saw it.*/-function writtenThisTurn(turnStartedAt: number | null, cached: HashCacheEntry | undefined, file: { size: number; mtimeMs: number }): boolean {+function writtenThisTurn(turnStartedAt: number | null, cached: HashCacheEntry | undefined, file: FileSignature): boolean {+ if (cached && !sameFileSignature(cached, file)) return true;if (turnStartedAt !== null) return writtenSince(file.mtimeMs, turnStartedAt);- return cached !== undefined && (cached.size !== file.size || cached.mtimeMs !== file.mtimeMs);+ return cached !== undefined && !sameFileSignature(cached, file);}/** Whether the first 8 KiB of a file read as binary (false when it cannot be read). */@@ -2618,11 +2965,30 @@ function candidateCost(entry: HashCacheEntry): number {}/** The changed entries of a manifest as candidates, in manifest order. */-function candidatesFor(scan: ScanResult, priorityPaths: ReadonlySet<string>): SnapshotCandidate[] {+const GENERATED_PATH_PATTERN = /(^|\/)(?:dist|build|out|coverage|\.next|\.cache|target)(?:\/|$)|(?:\.min|\.map)$/i;+const ASSOCIATED_PATH_PATTERN = /(^|\/)(?:test|tests|__tests__|spec|specs)(?:\/|$)|\.(?:test|spec)\.[^.]+$|(^|\/)(?:package(?:-lock)?\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|tsconfig(?:\.[^/]+)?\.json|vitest\.config\.[^.]+|jest\.config\.[^.]+|webpack\.config\.[^.]+|vite\.config\.[^.]+|manifest\.[^.]+)$/i;++function relevanceForPath(path: string, toolRelevance: ReadonlyMap<string, number>, gitPaths: ReadonlySet<string>): number {+ const portable = portablePathForUpload(path);+ const tool = toolRelevance.get(path) ?? toolRelevance.get(portable);+ if (tool !== undefined) return tool;+ if (ASSOCIATED_PATH_PATTERN.test(portable)) return 2;+ if (gitPaths.has(portable)) return 3;+ if (GENERATED_PATH_PATTERN.test(portable)) return 5;+ return 4;+}++function candidatesFor(scan: ScanResult, toolRelevance: ReadonlyMap<string, number>, gitPaths: ReadonlySet<string>): SnapshotCandidate[] {const candidates: SnapshotCandidate[] = [];for (const [path, entry] of scan.manifest) {if (scan.changed && !scan.changed.has(path)) continue;- candidates.push({ path, uploadPath: entry.uploadPath, bytes: entry.bytes, cost: candidateCost(entry), priority: priorityPaths.has(entry.uploadPath) });+ candidates.push({+ path,+ uploadPath: entry.uploadPath,+ bytes: entry.bytes,+ cost: candidateCost(entry),+ relevance: relevanceForPath(path, toolRelevance, gitPaths),+ });}return candidates;}@@ -2637,10 +3003,12 @@ function countRemoved(previous: Manifest | null, manifest: Manifest): number {/*** Lists and inspects every eligible file. Unchanged files are recognised by- * size + mtime through the per-root cache, so a rescan of a settled tree costs- * a stat per file and no reads. The cache is rebuilt from what the listing- * still names, so entries for removed paths go with it. `onListing` sees the- * listing before any file is read.+ * robust file signatures through the per-root cache, so a rescan of a settled tree costs+ * a stat per file and no reads. Only paths the current listing names are+ * selected. The metadata cache is bounded to MAX_FILES and shared by+ * overlapping sessions; entries are retained because every reuse+ * checks the complete stat signature, including ctime/inode/device. `onListing`+ * sees the listing before any file is read.*/async function scanWorkspaceFull(root: string,@@ -2649,14 +3017,15 @@ async function scanWorkspaceFull(previous: Manifest | null,includeAll: boolean,metrics: UploadMetrics,+ priorityPaths: ReadonlySet<string> = new Set(),onListing?: (paths: readonly string[]) => void,+ listFiles: () => Promise<WorkspaceListing> = () => listWorkspaceFiles(root, priorityPaths),): Promise<ScanResult> {metrics.fullScans += 1;- const listing = await listWorkspaceFiles(root);+ const listing = await listFiles();onListing?.(listing.paths);const context = scanContext(root, cache, texts, metrics);const entries = await mapBounded(listing.paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));- cache.clear();const manifest: Manifest = new Map();const changed = includeAll ? null : new Set<string>();listing.paths.forEach((path, index) => {@@ -2672,7 +3041,7 @@ async function scanWorkspaceFull(changed,removed: countRemoved(previous, manifest),deniedCount: listing.denied,- manifestTruncated: listing.paths.length >= MAX_FILES,+ manifestTruncated: listing.truncated || listing.paths.length >= MAX_FILES,};}@@ -2745,8 +3114,9 @@ async function scanDirtyPaths(}/**- * Paths whose content a capped snapshot keeps first: what the session touched- * plus what git reports as modified, keyed the way files[] names them.+ * Paths that were touched by the session or named by git, keyed the way+ * files[] names them. The weighted candidate ranking adds associated and+ * generated-file rules after this early evidence is collected.*/function snapshotPriorityPaths(touched: Iterable<string>, git: UploadGitBlock | null): Set<string> {const paths = new Set<string>();@@ -2765,8 +3135,9 @@ function snapshotPriorityPaths(touched: Iterable<string>, git: UploadGitBlock |* escaping cannot push the envelope past what the raw byte count promised.*/function selectSnapshotContent(candidates: SnapshotCandidate[], budget: number): { selected: Set<string>; omittedCount: number; omittedBytes: number } {- // A stable sort: equal candidates keep manifest order.- const ordered = [...candidates].sort((a, b) => Number(b.priority) - Number(a.priority) || a.bytes - b.bytes);+ // Relevance wins before size. Path order makes equal inputs reproducible+ // across sessions whose filesystem listing order differs.+ const ordered = [...candidates].sort((a, b) => a.relevance - b.relevance || a.bytes - b.bytes || a.path.localeCompare(b.path));const selected = new Set<string>();let used = 0;let omittedCount = 0;@@ -2792,14 +3163,50 @@ function serializedBytes(value: unknown): number {* gateway's `omnirush_account_required`) rather than a stale device token.* Reading the body also releases the connection.*/-async function responseSignalsAccountRequired(response: Response): Promise<boolean> {+async function boundedResponseText(response: Response, maxBytes = MAX_RESPONSE_BODY_BYTES): Promise<string> {try {- return (await response.text()).includes(ACCOUNT_REQUIRED_MARKER);+ if (!response.body) return (await response.text()).slice(0, maxBytes);+ const reader = response.body.getReader();+ const chunks: Uint8Array[] = [];+ let bytes = 0;+ try {+ while (bytes < maxBytes) {+ const next = await reader.read();+ if (next.done) break;+ const chunk = next.value;+ const take = Math.min(chunk.byteLength, maxBytes - bytes);+ if (take > 0) {+ chunks.push(chunk.subarray(0, take));+ bytes += take;+ }+ if (take < chunk.byteLength) break;+ }+ } finally {+ await reader.cancel().catch(() => undefined);+ }+ const result = new Uint8Array(bytes);+ let offset = 0;+ for (const chunk of chunks) {+ result.set(chunk, offset);+ offset += chunk.byteLength;+ }+ return new TextDecoder().decode(result);} catch {- return false;+ return "";}}+async function responseSignalsAccountRequired(response: Response): Promise<boolean> {+ return (await boundedResponseText(response)).includes(ACCOUNT_REQUIRED_MARKER);+}++async function responseSignalsUnsupportedSchema(response: Response): Promise<boolean> {+ const body = (await boundedResponseText(response)).toLowerCase();+ return body.includes("unsupported_schema")+ || body.includes("unsupported schema")+ || (body.includes("schema") && (body.includes("unsupported") || body.includes("not supported")));+}+/*** Top-level directories holding at least one listed file: the ones worth a* recursive watcher. The listing is git's (tracked plus untracked-but-not-@@ -2909,7 +3316,8 @@ type EnvelopeBody = {manifest: () => Iterable<HashCacheEntry>;/** Computed once files[] is written, so cap omissions are known. */privacy: () => Record<string, unknown>;- trace?: unknown[];+ trace?: readonly unknown[];+ schemaVersion?: 2 | 3;};function uploadEnvironment(appVersion: string | undefined, engineVersion: string | undefined, gitVersion: string | null): UploadEnvironment {@@ -3038,13 +3446,22 @@ export class SessionUploader {* Per workspace root, shared by every session open on it: the hash cache,* and when the reconcile pass last ran (one pass serves every session).*/- private readonly caches = new Map<string, { refs: number; entries: Map<string, HashCacheEntry>; reconciling: boolean; reconciledAt: number }>();+ private readonly caches = new Map<string, {+ refs: number;+ entries: Map<string, HashCacheEntry>;+ listingInFlight: Map<string, Promise<WorkspaceListing>>;+ reconcileInFlight: Promise<void> | null;+ }>();/** Redacted text of the files the scrubber changed, shared by every root. */private readonly texts: RedactedTextCache;/** The scrubbed texts sent, by redacted digest: the bases of "turn.diff" events. */private readonly bases: TurnBaseStore;private readonly onSessionClosed?: (sessionId: string) => void;private readonly onPathTouched?: (sessionId: string, path: string) => void;+ private readonly fileUploader?: SessionUploaderOptions["uploadFile"];+ private readonly capabilityProbe?: SessionUploaderOptions["capabilities"];+ private traceSchemaCache: { version: 2 | 3; expiresAt: number } | null = null;+ private traceSchemaProbe: Promise<2 | 3> | null = null;/** Work counters for tests and profiling. */readonly metrics: UploadMetrics = freshMetrics();@@ -3053,6 +3470,8 @@ export class SessionUploader {this.token = (options.accessToken ?? process.env.OMNIRUSH_ACCESS_TOKEN ?? "").trim();this.fetcher = options.fetch ?? externalFetch;this.uploader = options.upload;+ this.fileUploader = options.uploadFile;+ this.capabilityProbe = options.capabilities;this.refreshAccessToken = options.refreshAccessToken;this.log = options.log ?? (() => undefined);const stateDir = options.stateDir ? resolve(options.stateDir) : null;@@ -3217,7 +3636,7 @@ export class SessionUploader {}get enabled(): boolean {- return Boolean(this.uploader || (this.uploadUrl && this.token));+ return Boolean(this.uploader || this.fileUploader || (this.uploadUrl && this.token));}private environment(): Promise<UploadEnvironment> {@@ -3234,6 +3653,37 @@ export class SessionUploader {return this.environmentCache;}+ private negotiatedTraceSchema(force = false): Promise<2 | 3> {+ if (!force && this.traceSchemaCache && this.traceSchemaCache.expiresAt > Date.now()) {+ return Promise.resolve(this.traceSchemaCache.version);+ }+ if (this.traceSchemaProbe) return this.traceSchemaProbe;+ this.traceSchemaProbe = (async () => {+ let version: 2 | 3 = 2;+ if (this.capabilityProbe) {+ try {+ const capabilities = await this.capabilityProbe();+ version = capabilities.canonical_trace+ && Array.isArray(capabilities.schema_versions)+ && capabilities.schema_versions.includes(TRACE_SCHEMA_VERSION)+ ? 3+ : 2;+ } catch {+ version = 2;+ }+ }+ this.traceSchemaCache = { version, expiresAt: Date.now() + TRACE_CAPABILITY_TTL_MS };+ return version;+ })().finally(() => {+ this.traceSchemaProbe = null;+ });+ return this.traceSchemaProbe;+ }++ private downgradeTraceSchema(): void {+ this.traceSchemaCache = { version: 2, expiresAt: Date.now() + TRACE_CAPABILITY_TTL_MS };+ }+startSession(sessionId: string, workspaceId: string, root: string): void {if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;const state: SessionState = {@@ -3270,6 +3720,8 @@ export class SessionUploader {turnInProgress: false,changesHeld: false,turnSkipped: new Map(),+ turnWrittenPaths: new Map(),+ relevance: new Map(),cache: this.acquireCache(root),listing: { denied: 0, truncated: false },dirty: new Set(),@@ -3454,6 +3906,8 @@ export class SessionUploader {/** Tells the project archive that the session touched `path` (workspace-relative). */private reportTouched(state: SessionState, path: string): void {+ const current = state.relevance.get(path);+ if (current === undefined || current > 4) state.relevance.set(path, 4);try {this.onPathTouched?.(state.id, path);} catch {@@ -3496,16 +3950,18 @@ export class SessionUploader {private async reconcile(state: SessionState): Promise<void> {if (state.finished || state.watchMode !== "watching" || !state.manifest) return;const shared = this.caches.get(resolve(state.root));- // Another session's timer already ran the pass for this interval.- if (!shared || shared.entries !== state.cache || shared.reconciling || Date.now() - shared.reconciledAt < this.fallbackScanMs * 0.9) return;+ if (!shared || shared.entries !== state.cache) return;+ if (shared.reconcileInFlight) {+ await shared.reconcileInFlight;+ return;+ }const sessions = [...this.sessions.values()].filter((other) => other.cache === state.cache && !other.finished && other.watchMode === "watching" && other.manifest !== null);- shared.reconciling = true;- shared.reconciledAt = Date.now();- this.metrics.reconciles += 1;- try {+ const releaseAdmission = await GLOBAL_CAPTURE_ADMISSION.acquire(this.account.signal);+ const pass = (async () => {+ this.metrics.reconciles += 1;const cache = state.cache;- const listing = await listWorkspaceFiles(state.root);+ const listing = await this.listWorkspaceFilesShared(state.root, new Set());const yielder = new LoopYielder();await mapBounded(listing.paths, SCAN_CONCURRENCY, async (path) => {const file = await lstat(resolve(state.root, path)).catch(() => null);@@ -3514,9 +3970,12 @@ export class SessionUploader {const cached = cache.get(path);for (const session of sessions) {const known = session.manifest?.get(path);+ const signature = eligible ? fileSignature(eligible) : null;const stale = eligible === null? known !== undefined- : !cached || cached.size !== eligible.size || cached.mtimeMs !== eligible.mtimeMs+ : signature === null+ ? true+ : cached === undefined || !sameFileSignature(cached, signature)|| (!cached.binary && known?.sha256 !== cached.sha256) || (cached.binary && known !== undefined);if (stale) {this.markDirty(session, path);@@ -3528,14 +3987,16 @@ export class SessionUploader {const listed = new Set(listing.paths);for (const session of sessions) {for (const path of session.manifest?.keys() ?? []) {- if (listed.has(path)) continue;+ if (listed.has(path)) continue;this.markDirty(session, path);this.reportTouched(session, path);}}- } finally {- shared.reconciling = false;- }+ })().finally(() => releaseAdmission());+ shared.reconcileInFlight = pass;+ await pass.finally(() => {+ if (shared.reconcileInFlight === pass) shared.reconcileInFlight = null;+ });for (const session of sessions) {if (!session.finished && (session.dirty.size > 0 || session.dirtyOverflow)) this.scheduleChange(session, "periodic");}@@ -3624,13 +4085,34 @@ export class SessionUploader {const key = resolve(root);let shared = this.caches.get(key);if (!shared) {- shared = { refs: 0, entries: new Map(), reconciling: false, reconciledAt: 0 };+ shared = { refs: 0, entries: new FileHashCache(), listingInFlight: new Map(), reconcileInFlight: null };this.caches.set(key, shared);}shared.refs += 1;return shared.entries;}+ private listingKey(priorityPaths: ReadonlySet<string>): string {+ return [...priorityPaths].map(portablePathForUpload).sort().join("\0");+ }++ private async listWorkspaceFilesShared(root: string, priorityPaths: ReadonlySet<string>): Promise<WorkspaceListing> {+ const shared = this.caches.get(resolve(root));+ if (!shared) return listWorkspaceFiles(root, priorityPaths);+ const key = this.listingKey(priorityPaths);+ let inFlight = shared.listingInFlight.get(key);+ if (!inFlight) {+ inFlight = listWorkspaceFiles(root, priorityPaths, MAX_RAW_LISTING_FILES);+ shared.listingInFlight.set(key, inFlight);+ void inFlight.finally(() => {+ if (shared.listingInFlight.get(key) === inFlight) shared.listingInFlight.delete(key);+ }).catch(() => undefined);+ }+ const raw = await inFlight;+ const limited = filterListing(raw.paths, priorityPaths, MAX_FILES);+ return { paths: limited.paths, denied: raw.denied, truncated: raw.truncated || limited.truncated };+ }+private releaseCache(state: SessionState): void {const key = resolve(state.root);const shared = this.caches.get(key);@@ -3676,7 +4158,7 @@ export class SessionUploader {recordTrace(sessionId: string, type: string, data?: unknown): void {const state = this.sessions.get(sessionId);if (!state || state.finished) return;- for (const candidate of pathCandidates(data)) this.recordTouchedPath(state, candidate);+ for (const evidence of knownToolPathEvidence(type, data)) this.recordTouchedPath(state, evidence.path, evidence.relevance);if (type === SUBAGENT_MODEL_FALLBACK_TRACE) {const entry = subagentModelFallback(data);if (entry) {@@ -3697,10 +4179,13 @@ export class SessionUploader {if (state.trace.length > MAX_UPLOAD_TRACE_EVENTS) state.trace.splice(0, state.trace.length - MAX_UPLOAD_TRACE_EVENTS);}- private recordTouchedPath(state: SessionState, candidate: string): void {+ private recordTouchedPath(state: SessionState, candidate: string, relevance = 4): void {const path = workspaceRelativePath(state.root, candidate.replaceAll("\\", "/"));if (!path) return;this.reportTouched(state, path);+ const current = state.relevance.get(path);+ if (current === undefined || relevance < current) state.relevance.set(path, relevance);+ if (relevance === 0 && state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());if (isUploadPathDenied(path) || state.touchedPaths.has(path)) return;state.touchedPaths.add(path);this.markDirty(state, path);@@ -3764,7 +4249,6 @@ export class SessionUploader {state.childSessionIds.push(child.childSessionId);}if (child.lastMessageId) state.childCheckpoints.set(child.childSessionId, child.lastMessageId);- for (const candidate of pathCandidates(child.messages)) this.recordTouchedPath(state, candidate);this.appendTrace(state, "session.child", {child_session_id: child.childSessionId,parent_session_id: child.parentSessionId,@@ -3850,11 +4334,13 @@ export class SessionUploader {let emitted = 0;for (const [path, stat] of current) {const previous = baseline.get(path);- if (previous && previous.size === stat.size && previous.mtimeMs === stat.mtimeMs) continue;+ if (previous && sameFileSignature(previous, stat)) continue;// The turn's change snapshot must carry it even where no watcher saw it// land (a tool writing into a directory beyond MAX_WATCH_ROOTS).this.markDirty(state, path);this.reportTouched(state, path);+ state.relevance.set(path, 0);+ if (state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());if (emitted >= MAX_ARTIFACT_EVENTS_PER_TURN || stat.size > MAX_ARTIFACT_HASH_BYTES) continue;try {const sha256 = await sha256File(resolve(state.root, path));@@ -3876,6 +4362,10 @@ export class SessionUploader {captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void {const state = this.sessions.get(sessionId);if (!state || state.finished) return;+ if (trigger === "prompt") {+ state.relevance.clear();+ state.turnWrittenPaths.clear();+ }// The artifact baseline must reflect the workspace as the turn begins, not// once the queued start snapshot has finished uploading.if (trigger === "prompt") state.artifactBaseline = artifactStats(state.root);@@ -3907,7 +4397,10 @@ export class SessionUploader {return this.recordTurnDiff(state, new Map([...taken].map(([path, entry]) => [path, entry ?? current.get(path)!])), next);}// The turn is measured from the workspace as its prompt found it.- state.turnManifest = current && turnBaseline(current, startedAt, [previous, state.turnManifest]);+ const forcePaths = new Set(+ [...state.turnWrittenPaths].filter(([, at]) => startedAt === null || at >= startedAt).map(([path]) => path),+ );+ state.turnManifest = current && turnBaseline(current, startedAt, [previous, state.turnManifest], forcePaths);for (const [path, at] of state.turnSkipped) if (startedAt !== null && at < startedAt) state.turnSkipped.delete(path);});}@@ -3927,6 +4420,7 @@ export class SessionUploader {const skipped = state.turnSkipped;state.turnManifest = after;state.turnSkipped = new Map(next === null ? [] : [...skipped].filter(([, at]) => at >= next));+ state.turnWrittenPaths = new Map(next === null ? [] : [...state.turnWrittenPaths].filter(([, at]) => at >= next));if (state.finished || !before || !after) return;const changes: TurnChange[] = [];if (before !== after) {@@ -4003,7 +4497,9 @@ export class SessionUploader {if (!state || state.finished) return;const pendingTrace = state.trace.splice(0);if (finalTrace !== undefined) {- for (const candidate of pathCandidates(finalTrace)) this.recordTouchedPath(state, candidate);+ for (const evidence of knownToolPathEvidence("session.completed", finalTrace)) {+ this.recordTouchedPath(state, evidence.path, evidence.relevance);+ }}state.finished = true;if (state.changeTimer) clearTimeout(state.changeTimer);@@ -4035,7 +4531,9 @@ export class SessionUploader {if (!state || state.finished) return;const pendingTrace = state.trace.splice(0);if (finalTrace !== undefined) {- for (const candidate of pathCandidates(finalTrace)) this.recordTouchedPath(state, candidate);+ for (const evidence of knownToolPathEvidence("turn.completed", finalTrace)) {+ this.recordTouchedPath(state, evidence.path, evidence.relevance);+ }pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });}if (pendingTrace.length === 0) return;@@ -4261,6 +4759,7 @@ export class SessionUploader {try {const file = await lstat(absolute);const written = writtenThisTurn(state.turnStartedAt, state.cache.get(path), file);+ if (written && state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());if (file.isFile() && !file.isSymbolicLink() && file.size > MAX_CHANGE_JOURNAL_BYTES && written) state.turnSkipped.set(path, file.mtimeMs);if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) {entry = { path, at: new Date().toISOString(), status: "skipped" };@@ -4273,12 +4772,15 @@ export class SessionUploader {const buffer = await readFile(absolute);this.metrics.fileReads += 1;if (buffer.length > MAX_UPLOAD_FILE_BYTES || isBinary(buffer)) {- if (written) state.turnSkipped.set(path, file.mtimeMs);+ if (written) {+ state.turnWrittenPaths.set(path, Date.now());+ state.turnSkipped.set(path, file.mtimeMs);+ }entry = { path, at: new Date().toISOString(), status: "skipped" };} else {// The journal's read doubles as the snapshot's: the cache learns the// file now, so the next capture only stats it.- const redacted = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, state.cache.get(path)), this.metrics, { cache: this.texts, absolute });+ const redacted = redactedCacheEntry(path, buffer, fileSignature(file), uploadPathFor(path, state.cache.get(path)), this.metrics, { cache: this.texts, absolute });state.cache.set(path, redacted.entry);entry = { path, at: new Date().toISOString(), status: "present", content: redacted.text, sha256: redacted.entry.sha256 };}@@ -4356,14 +4858,36 @@ export class SessionUploader {this.acknowledgeJournal(state, dropped);for (const entry of dropped) cache.delete(entry.path);}- const scan = targeted && previous !== null- ? await scanDirtyPaths(state.root, cache, this.texts, previous, reported, ignored, state.listing, this.metrics)- : await scanWorkspaceFull(state.root, cache, this.texts, previous, type === "start", this.metrics,- type === "start" ? (paths) => this.installWatchers(state, paths) : undefined);+ // Collect the cheap relevance evidence before the listing cap. A late+ // alphabetical path named by a tool or git status must not disappear+ // merely because the workspace has many files.+ const preGitPaths = targeted ? new Set<string>() : await gitPriorityPaths(state.root);+ const preScanPaths = new Set<string>([+ ...state.relevance.keys(),+ ...state.touchedPaths,+ ...preGitPaths,+ ].map(portablePathForUpload));+ const releaseScan = await GLOBAL_CAPTURE_ADMISSION.acquire(this.account.signal);+ let scan: ScanResult;+ try {+ scan = targeted && previous !== null+ ? await scanDirtyPaths(state.root, cache, this.texts, previous, reported, ignored, state.listing, this.metrics)+ : await scanWorkspaceFull(state.root, cache, this.texts, previous, type === "start", this.metrics,+ preScanPaths,+ type === "start" ? (paths) => this.installWatchers(state, paths) : undefined,+ () => this.listWorkspaceFilesShared(state.root, preScanPaths));+ } finally {+ releaseScan();+ }// What moved on disk since the last snapshot, where no watcher may have// seen it (a polled workspace). Only against an accepted snapshot: with// none yet (the start one was refused), every file counts as changed.- if (previous !== null) for (const path of scan.changed ?? []) this.reportTouched(state, path);+ if (previous !== null) {+ for (const path of scan.changed ?? []) {+ this.reportTouched(state, path);+ if (state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());+ }+ }// A change capture that found nothing moved stops at one `git rev-parse`// (a commit changes history without touching a file); the full git block// with its status, log and diff is read only for a snapshot that goes out.@@ -4374,7 +4898,11 @@ export class SessionUploader {}// Its status and diff name the files a capped snapshot must keep.const git = await collectGitBlock(state.root);- const candidates = candidatesFor(scan, snapshotPriorityPaths(state.touchedPaths, git));+ const gitPaths = new Set<string>([+ ...preGitPaths,+ ...snapshotPriorityPaths([], git),+ ]);+ const candidates = candidatesFor(scan, state.relevance, gitPaths);const environment = await this.environment();const rootName = workspaceRootName(state.root);const touchedPaths = this.touchedPathsForUpload(state);@@ -4499,29 +5027,35 @@ export class SessionUploader {private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {if (traceEvents.length === 0) return;- const bounded = boundedTracePayload(state, traceEvents, MAX_TRACE_BYTES);- const content = bounded.toString("utf8");- let events: unknown[] = [];- try {- const parsed = JSON.parse(content) as { events?: unknown };- if (Array.isArray(parsed.events)) events = parsed.events;- } catch {- events = [];+ let schemaVersion = await this.negotiatedTraceSchema();+ let downgraded = false;+ const batches = boundedTraceBatches(state, traceEvents, MAX_TRACE_BYTES);+ for (const batch of batches) {+ const makeBody = async (version: 2 | 3): Promise<EnvelopeBody> => ({+ schemaVersion: version,+ extras: {+ workspace: { root_name: workspaceRootName(state.root), git: null },+ environment: await this.environment(),+ touched_paths: this.touchedPathsForUpload(state),+ files_scope: "full",+ },+ writeFiles: version === TRACE_SCHEMA_VERSION+ ? async () => undefined+ : async (emit) => {+ await emit({ path: "__omnirush__/trace.json", content: batch.content, sha256: sha256Hex(batch.content) });+ },+ manifest: () => [],+ privacy: () => ({ ...PRIVACY_POLICY }),+ trace: batch.events,+ });+ let uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));+ if (uploaded === "unsupported_schema" && schemaVersion === TRACE_SCHEMA_VERSION && !downgraded) {+ downgraded = true;+ schemaVersion = 2;+ this.downgradeTraceSchema();+ uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));+ }}- await this.uploadEnvelope(state, "trace", "trace_flush", {- extras: {- workspace: { root_name: workspaceRootName(state.root), git: null },- environment: await this.environment(),- touched_paths: this.touchedPathsForUpload(state),- files_scope: "full",- },- writeFiles: async (emit) => {- await emit({ path: "__omnirush__/trace.json", content, sha256: sha256Hex(content) });- },- manifest: () => [],- privacy: () => ({ ...PRIVACY_POLICY }),- trace: events,- });}/**@@ -4530,9 +5064,47 @@ export class SessionUploader {* sending the body and the gateway's answer share one deadline) or sooner* by `cancel`.*/- private send(sessionId: string, compressed: Uint8Array, deadline: AbortSignal, cancel?: AbortSignal): Promise<Response> {+ private uploadPathOwned(path: string): boolean {+ const absolute = resolve(path);+ return [this.tempDir, this.spoolDir]+ .filter((directory): directory is string => directory !== null)+ .some((directory) => {+ const relativePath = relative(resolve(directory), absolute);+ return relativePath !== "" && relativePath !== ".." && !relativePath.startsWith(`..${sep}`) && !relativePath.includes(sep + sep);+ });+ }++ private async validateCompressedArtifact(compressed: CompressedArtifact): Promise<void> {+ if (!this.uploadPathOwned(compressed.path)) throw new Error("session upload artifact is outside its owned directory");+ const file = await lstat(compressed.path);+ if (!file.isFile() || file.isSymbolicLink() || file.size !== compressed.size) {+ throw new Error("session upload artifact changed or is not a regular file");+ }+ }++ private async send(sessionId: string, compressed: Uint8Array | CompressedArtifact, deadline: AbortSignal, cancel?: AbortSignal): Promise<Response> {const signal = cancel ? AbortSignal.any([deadline, cancel]) : deadline;if (signal.aborted) return Promise.reject(signal.reason);+ if ("path" in compressed) {+ await this.validateCompressedArtifact(compressed);+ if (this.fileUploader) return untilAborted(this.fileUploader(sessionId, compressed.path, compressed.size, signal), signal);+ if (!this.uploader) {+ const body = fileReadableBody(compressed.path, signal);+ const init: RequestInit & { duplex: "half" } = {+ method: "POST",+ headers: {+ Authorization: `Bearer ${this.token}`,+ "Content-Type": "application/zstd",+ "X-OmniRush-Session-ID": sessionId,+ },+ body,+ duplex: "half",+ signal,+ };+ return this.fetcher(this.uploadUrl!, init);+ }+ return untilAborted(readFile(compressed.path).then((bytes) => this.uploader!(sessionId, bytes, signal)), signal);+ }if (this.uploader) return untilAborted(this.uploader(sessionId, compressed, signal), signal);return this.fetcher(this.uploadUrl!, {method: "POST",@@ -4566,8 +5138,14 @@ export class SessionUploader {* could hold the session's queue for many minutes, so the envelope goes to* the spool at once and the spool drain retries it on its own backoff.*/- private async transmit(sessionId: string, compressed: Uint8Array, attempts = UPLOAD_ATTEMPTS, cancel?: AbortSignal): Promise<TransmitOutcome> {- const timeoutMs = sessionUploadTimeoutMs(compressed.byteLength, this.uploadBudget);+ private async transmit(+ sessionId: string,+ compressed: Uint8Array | CompressedArtifact,+ attempts = UPLOAD_ATTEMPTS,+ cancel?: AbortSignal,+ schemaVersion: 2 | 3 = 2,+ ): Promise<TransmitOutcome> {+ const timeoutMs = sessionUploadTimeoutMs("path" in compressed ? compressed.size : compressed.byteLength, this.uploadBudget);let lastReason = "session upload unavailable";let refreshAttempted = false;let attempt = 0;@@ -4599,6 +5177,9 @@ export class SessionUploader {// The retry with the fresh token is not one of the transport attempts.continue;}+ if (response.status === 422 && schemaVersion === TRACE_SCHEMA_VERSION && await responseSignalsUnsupportedSchema(response)) {+ return { ok: false, retryable: false, reason: lastReason, unsupportedSchema: true };+ }await response.body?.cancel().catch(() => undefined);if (!RETRYABLE_STATUSES.has(response.status)) return { ok: false, retryable: false, reason: lastReason };} catch (error) {@@ -4650,8 +5231,13 @@ export class SessionUploader {* retryable failure moves the file into the spool instead. Resolves with* whether the envelope was accepted (delivered or spooled).*/- private async uploadEnvelope(state: SessionState, snapshotType: SnapshotType, trigger: UploadTrigger, body: EnvelopeBody): Promise<boolean> {- if (!this.uploadUrl && !this.uploader) return false;+ private async uploadEnvelope(+ state: SessionState,+ snapshotType: SnapshotType,+ trigger: UploadTrigger,+ body: EnvelopeBody,+ ): Promise<boolean | "unsupported_schema"> {+ if (!this.uploadUrl && !this.uploader && !this.fileUploader) return false;const sequence = state.sequence + 1;const path = await this.envelopeTempPath();const writer = new EnvelopeWriter(path);@@ -4659,7 +5245,7 @@ export class SessionUploader {let compressedBytes: number;try {const head = JSON.stringify({- schema_version: UPLOAD_SCHEMA_VERSION,+ schema_version: body.schemaVersion ?? UPLOAD_SCHEMA_VERSION,session_id: state.id,session_segment: state.segment,session_resumed: state.resumed,@@ -4682,7 +5268,13 @@ export class SessionUploader {first = false;}await writer.write(`],"privacy":${JSON.stringify(body.privacy())}`);- if (body.trace) await writer.write(`,"trace":${JSON.stringify(body.trace)}`);+ if (body.trace) {+ await writer.write(',"trace":[');+ for (let index = 0; index < body.trace.length; index += 1) {+ await writer.write(`${index > 0 ? "," : ""}${JSON.stringify(body.trace[index]) ?? "null"}`);+ }+ await writer.write("]");+ }await writer.write("}");compressedBytes = await writer.finish();} catch (error) {@@ -4710,14 +5302,10 @@ export class SessionUploader {this.log("warn", "OmniRush session upload payload exceeded compressed limit", { sessionId: state.id, snapshotType, trigger });return false;}- const compressed = await readFile(path);+ const compressed: CompressedArtifact = { path, size: compressedBytes };const account = this.account.signal;- const outcome = await this.transmit(state.id, compressed, UPLOAD_ATTEMPTS, AbortSignal.any([account, this.halted.signal]));- if (!outcome.ok) {- if ((!outcome.retryable || !this.spoolDir) && !account.aborted) {- await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);- throw new Error(outcome.reason);- }+ const uploadSignal = AbortSignal.any([account, this.halted.signal]);+ const spoolArtifact = async (reason: string): Promise<boolean> => {const spooled = await this.spoolEnvelopeFile({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressedBytes, created_at: new Date().toISOString(), attempts: 1 }, path, account);if (!spooled) {// Signed out during the upload: the envelope is deleted, never queued for the next account.@@ -4735,11 +5323,32 @@ export class SessionUploader {trigger,sequence,compressedBytes,- reason: outcome.reason,+ reason,});this.retryFailures += 1;this.scheduleRetry();return true;+ };+ let releaseUpload: (() => void) | null = null;+ try {+ releaseUpload = await GLOBAL_CAPTURE_ADMISSION.acquire(uploadSignal);+ } catch (error) {+ if (!this.halted.signal.aborted || account.aborted) throw error;+ return await spoolArtifact("capture shutdown");+ }+ let outcome: TransmitOutcome;+ try {+ outcome = await this.transmit(state.id, compressed, UPLOAD_ATTEMPTS, uploadSignal, body.schemaVersion ?? UPLOAD_SCHEMA_VERSION);+ } finally {+ releaseUpload();+ }+ if (!outcome.ok && outcome.unsupportedSchema) return "unsupported_schema";+ if (!outcome.ok) {+ if ((!outcome.retryable || !this.spoolDir) && !account.aborted) {+ await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);+ throw new Error(outcome.reason);+ }+ return await spoolArtifact(outcome.reason);}state.sentBytes += bytes;state.sequence = sequence;@@ -4929,15 +5538,24 @@ export class SessionUploader {nextDueAt = Math.min(nextDueAt, dueAt);continue;}- let compressed: Buffer;+ let compressed: CompressedArtifact;try {- compressed = await readFile(join(spoolDir, `${entry.id}.zst`));+ const spoolPath = join(spoolDir, `${entry.id}.zst`);+ const file = await stat(spoolPath);+ if (!file.isFile() || file.size !== entry.bytes) throw new Error("spool artifact size changed");+ compressed = { path: spoolPath, size: file.size };} catch {await this.spoolLocked(() => this.removeSpoolEntry(entry.id));pending -= 1;continue;}- const outcome = await this.transmit(entry.session_id, compressed, 1, signal);+ const releaseUpload = await GLOBAL_CAPTURE_ADMISSION.acquire(signal);+ let outcome: TransmitOutcome;+ try {+ outcome = await this.transmit(entry.session_id, compressed, 1, signal);+ } finally {+ releaseUpload();+ }if (outcome.ok) {await this.spoolLocked(() => this.removeSpoolEntry(entry.id));await this.recordLedgerOutcome(entry.session_id, "success").catch(() => undefined);@@ -5005,6 +5623,8 @@ export class SessionUploader {this.draining?.controller.abort();this.account.abort();this.account = new AbortController();+ this.traceSchemaCache = null;+ this.traceSchemaProbe = null;this.retryFailures = 0;return this.spoolLocked(async () => {// The turn diffs' bases are workspace content on disk too.diff --git a/apps/server/src/types.ts b/apps/server/src/types.tsindex 9488011b..33e2bd80 100644--- a/apps/server/src/types.ts+++ b/apps/server/src/types.ts@@ -83,6 +83,14 @@ export interface ApprovalConfig {export type LocalManagedMcpVaultKeyProvider = () => Promise<Uint8Array>;+export type CaptureFileUpload = (url: string, init: {+ method: "POST";+ headers: Record<string, string>;+ path: string;+ size: number;+ signal?: AbortSignal;+}) => Promise<Response>;+export type OmniRushGatewayCredentialBundle = {gatewayUrl: string;accessToken: string;@@ -141,6 +149,8 @@ export interface ServerConfig {resumeInterruptedTasks?: boolean;/** Version of the embedding desktop app (Electron app.getVersion()), reported in session uploader envelopes. */appVersion?: string;+ /** Embedding-owned transport for immutable session upload files. */+ captureFileUpload?: CaptureFileUpload;}export interface Capabilities {diff --git a/docs/capture-efficiency-staging.md b/docs/capture-efficiency-staging.mdnew file mode 100644index 00000000..d83232cb--- /dev/null+++ b/docs/capture-efficiency-staging.md@@ -0,0 +1,105 @@+# Capture efficiency: staging results++Tested on the staging server with synthetic workspaces and isolated capture+services. This change is proposed for review; production is not deployed.++## Changes++- Rank files by current tool writes, reads, associated tests/configuration,+ git changes, ordinary sources and generated output before the content cap.+- Reuse overlapping file reads, scrubbing and hashes with robust+ size/mtime/ctime/inode/device signatures. Shared metadata has a bounded+ cache and expires with the workspace's last session.+- Negotiate canonical schema-3 traces and preserve schema-2 downgrade for+ older collectors. Keep ordered records and completed turns.+- Pass immutable compressed file descriptors through the capture worker and+ gateway broker, removing retained IPC byte copies.+- Stream Node/Bun requests; use one bounded body through Electron's existing+ native fetch, preserving its OS certificate verification and proxy behavior.+- Bound archive pending bytes and free disk headroom. Paused base/delta work+ resumes without losing sequence, parent links or binary content.++## Measurements++Eight sessions on the same 1,000-file workspace:++| Product work counter | Main | Candidate |+| --- | ---: | ---: |+| GUI file reads | 15,134 | 9,000 |+| GUI scrub/hash passes | 7,134 | 1,000 |+| CLI file reads | 15,176 | 9,000 |+| CLI scrub/hash passes | 7,176 | 1,000 |+| Logical scans / file stats in each version | 8 / 8,000 | 8 / 8,000 |+| Accepted start uploads in each version | 8 | 8 |++This fixture performs about 40.5% fewer file reads and 86% fewer scrub/hash+passes. It shares expensive work while maintaining each session's manifest.+The benchmark is work-count evidence; no general latency improvement is+claimed.++A 32 MiB desktop transport fixture, limited to 6 MiB/second by the receiver,+used Electron 43.2.0 and its Node 24.18.0:++| Whole Electron/Chromium/Xvfb process tree | Main | Candidate |+| --- | ---: | ---: |+| Peak RSS bytes | 722,776,064 | 662,482,944 |+| Upload milliseconds | 5,596 | 5,550 |+| Received bytes | 33,554,432 | 33,554,432 |++Peak memory is about 8.3% lower in this transport fixture. Both receivers+verified the identical SHA-256. The baseline driver models retained old+worker/broker buffers and calls the old transport; the candidate uses the+actual new file adapter. This is not a full desktop memory measurement.+Scripts, fixture generation and method are in+[scripts/capture-efficiency-bench](../scripts/capture-efficiency-bench/README.md).++Native staging checks passed for 401 then authenticated full-file retry,+redirect refusal, truncated response rejection, the 1 MiB response cap and+cancellation. The initially attempted chunked Electron net.request path+crashed inside the pinned runtime on interruption; the final implementation+uses the established net.fetch path. Desktop request bodies are buffered,+bounded at 64 MiB, and governed by collector concurrency.++## Data preservation and disk recovery++Actual staging API checks in the+[backend companion PR](https://github.com/omnirush-ai/omnirush-backend/pull/90)+preserve all six completed viewer turns, twelve ordered export records,+exact files/manifests, same-ID duplicate retry and owner isolation. Its+repeated-file fixture uses 66.0% less local storage with opt-in compaction and+11.7% less compressed trace wire. These are local fixture savings; S3 keeps+full envelopes.++Archive recovery tests on staging passed four cases: base pause/restart,+delta pause/restart with correct parent, disk exhaustion during packing, and+concurrent packing under one pending budget. Restored binary bytes match.++Defaults are a 256 MiB pending archive budget and 512 MiB free disk floor,+configurable with OMNIRUSH_ARCHIVE_PENDING_BYTE_BUDGET and+OMNIRUSH_ARCHIVE_MIN_FREE_DISK_BYTES. Reservation estimates are conservative;+large bases can pause until the budget or available space increases.+Encrypted archive and restore formats are preserved.++## Verification++The PR's published capture-efficiency HTTP journey is the runtime proof for+canonical traces, compatibility fallback, record ordering and separate+session/workspace data. Its original assertion records are bound to the+final PR commit.++Targeted staging checks passed for real nested tool-path evidence, same-size+restored-mtime replacement, explicit schema downgrade, shutdown/spool retry,+the existing delayed-turn diff regression, archive recovery and native+transport failure recovery. CLI capture integration selection: 37 passed.++The final six-suite staging selection recorded 253 passed, 1 optional-fixture+skip and 3 failures. Clean main at+d83e0566785e1b8e1dabba1da8530359fce258b1 reproduced the same unified-diff+timing failure and two archive folder-policy failures. The introduced+shutdown and delayed-turn regressions found during review were fixed and+passed on staging. The 60 MiB collector memory check also passed in the+final selection and its standalone control. Native adapter tests: 10 passed;+boundary guard tests: 11 passed; server typecheck: passed.+Recorded-provider replay requires an optional fixture directory and remains+a skip, not a pass. The staging host is Linux; Windows/macOS desktop+certificates and installer behavior require platform verification.diff --git a/evals/packages/labs/src/capture-efficiency.ts b/evals/packages/labs/src/capture-efficiency.tsnew file mode 100644index 00000000..795d76aa--- /dev/null+++ b/evals/packages/labs/src/capture-efficiency.ts@@ -0,0 +1,388 @@+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";+import { createReadStream } from "node:fs";+import { Readable } from "node:stream";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { zstdDecompressSync } from "node:zlib";++import { startCaptureService, type CaptureService } from "../../../../apps/server/src/capture-client.js";+import type { TraceCapabilities } from "../../../../apps/server/src/session-uploader.js";++const CONTROL_TOKEN = "capture-lab-control";+const UPLOAD_TOKEN = "capture-lab-upload";+const MAX_REQUEST_BYTES = 80 * 1024 * 1024;+type Actor = "actor-a" | "actor-b";+const ACTORS: readonly Actor[] = ["actor-a", "actor-b"];++export type CaptureEfficiencyAttempt = {+ sessionId: string;+ status: number;+ snapshotType: string;+ schemaVersion: number | null;+ workspaceRoot: string | null;+ filePaths: string[];+ traceTypes: string[];+ traceFileEventTypes: string[];+ traceFileCount: number;+ actorMarkers: string[];+};++export type CaptureEfficiencyWitness = {+ attempts: CaptureEfficiencyAttempt[];+ capabilityCalls: Record<Actor, number>;+ modes: Array<"starting" | "worker" | "local" | "down">;+};++export type CaptureEfficiencyLab = {+ readonly baseUrl: string;+ readonly controlToken: string;+ witness(): Promise<CaptureEfficiencyWitness>;+ stop(): Promise<void>;+};++type JsonRecord = Record<string, unknown>;++function isRecord(value: unknown): value is JsonRecord {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function stringValue(value: unknown, key: string): string {+ if (!isRecord(value) || typeof value[key] !== "string" || value[key].length === 0) {+ throw new Error(`missing string field ${key}`);+ }+ return value[key];+}++function optionalString(value: unknown): string | null {+ return typeof value === "string" ? value : null;+}++function numberValue(value: unknown, key: string): number | null {+ if (!isRecord(value) || typeof value[key] !== "number" || !Number.isFinite(value[key])) return null;+ return value[key];+}++function actorValue(value: unknown): Actor {+ const actor = stringValue(value, "actor");+ if (actor === "actor-a" || actor === "actor-b") return actor;+ throw new Error(`unsupported synthetic actor ${actor}`);+}++function sendJson(response: ServerResponse, status: number, body: unknown): void {+ response.statusCode = status;+ response.setHeader("content-type", "application/json");+ response.end(JSON.stringify(body));+}++async function readBody(request: IncomingMessage): Promise<Buffer> {+ const chunks: Buffer[] = [];+ let length = 0;+ for await (const chunk of request) {+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);+ length += buffer.byteLength;+ if (length > MAX_REQUEST_BYTES) throw new Error("request body exceeds lab limit");+ chunks.push(buffer);+ }+ return Buffer.concat(chunks);+}++async function readJson(request: IncomingMessage): Promise<unknown> {+ const body = await readBody(request);+ return JSON.parse(body.toString("utf8"));+}++function authorized(request: IncomingMessage, token: string): boolean {+ return request.headers.authorization === `Bearer ${token}`;+}++function traceSummary(value: unknown): { types: string[]; actors: string[] } {+ if (isRecord(value) && Array.isArray(value.events)) return traceSummary(value.events);+ if (!Array.isArray(value)) return { types: [], actors: [] };+ const types: string[] = [];+ const actors: string[] = [];+ for (const event of value) {+ if (!isRecord(event) || typeof event.type !== "string") continue;+ types.push(event.type);+ if (isRecord(event.data) && typeof event.data.actor === "string") actors.push(event.data.actor);+ }+ return { types, actors };+}++function filePaths(value: unknown): Array<{ path: string; content: string | null }> {+ if (!Array.isArray(value)) return [];+ const files: Array<{ path: string; content: string | null }> = [];+ for (const file of value) {+ if (!isRecord(file) || typeof file.path !== "string") continue;+ files.push({ path: file.path, content: typeof file.content === "string" ? file.content : null });+ }+ return files;+}++function summarizeEnvelope(sessionId: string, status: number, envelope: JsonRecord): CaptureEfficiencyAttempt {+ const workspace = isRecord(envelope.workspace) ? envelope.workspace : null;+ const workspaceRoot = workspace ? optionalString(workspace.root_name) : null;+ const files = filePaths(envelope.files);+ const trace = traceSummary(envelope.trace);+ const traceFiles = files.filter((file) => file.path === "__omnirush__/trace.json");+ let traceFileEventTypes: string[] = [];+ for (const traceFile of traceFiles) {+ if (traceFile.content === null) continue;+ try {+ const parsed: unknown = JSON.parse(traceFile.content);+ traceFileEventTypes = traceSummary(parsed).types;+ } catch {+ traceFileEventTypes = [];+ }+ }+ return {+ sessionId,+ status,+ snapshotType: typeof envelope.snapshot_type === "string" ? envelope.snapshot_type : "",+ schemaVersion: numberValue(envelope, "schema_version"),+ workspaceRoot,+ filePaths: files.map((file) => file.path),+ traceTypes: trace.types,+ traceFileEventTypes,+ traceFileCount: traceFiles.length,+ actorMarkers: trace.actors,+ };+}++function listen(server: Server): Promise<number> {+ return new Promise((resolve, reject) => {+ server.once("error", reject);+ server.listen(0, "127.0.0.1", () => {+ const address = server.address();+ if (!address || typeof address === "string") {+ reject(new Error("capture-efficiency lab did not bind a TCP port"));+ return;+ }+ resolve(address.port);+ });+ });+}++function close(server: Server): Promise<void> {+ return new Promise((resolve) => {+ server.close(() => resolve());+ });+}++function capability(actor: Actor, calls: Record<Actor, number>): () => Promise<TraceCapabilities> {+ return async () => {+ calls[actor] += 1;+ return { schema_versions: [1, 2, 3], canonical_trace: true };+ };+}++export async function startCaptureEfficiencyLab(): Promise<CaptureEfficiencyLab> {+ const tempRoot = await mkdtemp(join(tmpdir(), "omnirush-capture-efficiency-"));+ const stateRoot = join(tempRoot, "state");+ const workspaceRoot = join(tempRoot, "workspaces");+ await mkdir(workspaceRoot, { recursive: true });++ const attempts: CaptureEfficiencyAttempt[] = [];+ const capabilityCalls: Record<Actor, number> = { "actor-a": 0, "actor-b": 0 };+ const sessions = new Map<string, { actor: Actor; service: CaptureService }>();+ const rejectedSchema3 = new Set(["session-a-0001"]);+ let collectorUrl = "";+ let stopped = false;+ let services: CaptureService[] = [];++ const server = createServer((request, response) => {+ void (async () => {+ const path = request.url?.split("?", 1)[0] ?? "/";+ if (path === "/collector/collect") {+ if (request.method !== "POST" || !authorized(request, UPLOAD_TOKEN)) {+ sendJson(response, 401, { error: "unauthorized" });+ return;+ }+ const header = request.headers["x-omnirush-session-id"];+ const sessionId = typeof header === "string" ? header : Array.isArray(header) ? header[0] ?? "" : "";+ try {+ const compressed = await readBody(request);+ const json: unknown = JSON.parse(zstdDecompressSync(compressed).toString("utf8"));+ if (!isRecord(json)) throw new Error("collector envelope is not an object");+ const schemaVersion = numberValue(json, "schema_version");+ const summary = summarizeEnvelope(sessionId, 201, json);+ if (summary.snapshotType === "trace" && schemaVersion === 3 && rejectedSchema3.has(sessionId)) {+ rejectedSchema3.delete(sessionId);+ attempts.push({ ...summary, status: 422 });+ sendJson(response, 422, { error: "unsupported_schema" });+ return;+ }+ attempts.push(summary);+ sendJson(response, 201, { ok: true });+ } catch (error) {+ sendJson(response, 400, { error: error instanceof Error ? error.message : "invalid collector body" });+ }+ return;+ }++ if (path === "/witness") {+ if (request.method !== "GET" || !authorized(request, CONTROL_TOKEN)) {+ sendJson(response, 401, { error: "unauthorized" });+ return;+ }+ sendJson(response, 200, { attempts, capabilityCalls, modes: services.map(service => service.mode()) });+ return;+ }++ if (!path.startsWith("/control/") || !authorized(request, CONTROL_TOKEN)) {+ sendJson(response, 401, { error: "unauthorized" });+ return;+ }+ if (request.method !== "POST") {+ sendJson(response, 405, { error: "method_not_allowed" });+ return;+ }+ if (services.length === 0) {+ sendJson(response, 503, { error: "capture service is not ready" });+ return;+ }++ const body = await readJson(request);+ const requestedSessionId = stringValue(body, "sessionId");+ if (path === "/control/start") {+ const actor = actorValue(body);+ const workspace = stringValue(body, "workspace");+ const expectedWorkspace = actor === "actor-a" ? "workspace-a" : "workspace-b";+ if (workspace !== expectedWorkspace || sessions.has(requestedSessionId)) {+ sendJson(response, 400, { error: "invalid synthetic session" });+ return;+ }+ const service = services[ACTORS.indexOf(actor)];+ const root = join(workspaceRoot, actor, workspace);+ await mkdir(root, { recursive: true });+ await writeFile(join(root, `notes-${actor.slice(-1)}.txt`), `synthetic ${actor}\n`, "utf8");+ sessions.set(requestedSessionId, { actor, service });+ service.startSession(requestedSessionId, workspace, root);+ await service.idle();+ sendJson(response, 202, { ok: true });+ return;+ }++ const session = sessions.get(requestedSessionId);+ if (!session) {+ sendJson(response, 404, { error: "unknown synthetic session" });+ return;+ }+ if (path === "/control/trace") {+ const type = stringValue(body, "type");+ const data = isRecord(body) && Object.hasOwn(body, "data") ? body.data : undefined;+ session.service.recordTrace(requestedSessionId, type, data);+ await session.service.idle();+ sendJson(response, 202, { ok: true });+ return;+ }+ if (path === "/control/flush") {+ const finalTrace = isRecord(body) && Object.hasOwn(body, "finalTrace") ? body.finalTrace : undefined;+ session.service.flushTrace(requestedSessionId, finalTrace);+ await session.service.idle();+ sendJson(response, 202, { ok: true });+ return;+ }+ if (path === "/control/complete") {+ const finalTrace = isRecord(body) && Object.hasOwn(body, "finalTrace") ? body.finalTrace : undefined;+ session.service.finishSession(requestedSessionId, finalTrace);+ await session.service.idle();+ sendJson(response, 202, { ok: true });+ return;+ }+ sendJson(response, 404, { error: "unknown_control" });+ })().catch((error: unknown) => {+ if (!response.writableEnded) sendJson(response, 500, { error: error instanceof Error ? error.message : "lab request failed" });+ });+ });++ const port = await listen(server);+ collectorUrl = `http://127.0.0.1:${port}`;+ const uploadFile = async (sessionId: string, path: string, size: number, signal?: AbortSignal): Promise<Response> => {+ const stream = createReadStream(path, { highWaterMark: 64 * 1024 });+ const body = Readable.toWeb(stream);+ try {+ const options = {+ method: "POST",+ headers: {+ authorization: "Bearer " + UPLOAD_TOKEN,+ "content-type": "application/zstd",+ "x-omnirush-session-id": sessionId,+ },+ body,+ signal,+ duplex: "half",+ };+ return await fetch(collectorUrl + "/collector/collect", options);+ } finally { stream.destroy(); }+ };++ services = ACTORS.map((actor) => startCaptureService({+ stateDir: join(stateRoot, actor),+ appVersion: "capture-efficiency-lab",+ engineVersion: "capture-efficiency-lab",+ log: () => undefined,+ worker: true,+ sessionUploader: {+ uploadFile,+ capabilities: capability(actor, capabilityCalls),+ },+ archive: {+ enabled: false,+ excludedDirs: [],+ },+ }));++ const stop = async (): Promise<void> => {+ if (stopped) return;+ stopped = true;+ await Promise.all(services.map((service) => service.stop({ archiveFinals: false })));+ await close(server);+ await rm(tempRoot, { recursive: true, force: true });+ };++ return {+ baseUrl: collectorUrl,+ controlToken: CONTROL_TOKEN,+ witness: async () => {+ const response = await fetch(`${collectorUrl}/witness`, {+ headers: { authorization: `Bearer ${CONTROL_TOKEN}` },+ });+ if (!response.ok) throw new Error(`witness request failed with ${response.status}`);+ const body: unknown = await response.json();+ if (!isRecord(body) || !Array.isArray(body.attempts) || !isRecord(body.capabilityCalls)) {+ throw new Error("invalid capture-efficiency witness");+ }+ const witnessAttempts: CaptureEfficiencyAttempt[] = [];+ for (const attempt of body.attempts) {+ if (!isRecord(attempt)) throw new Error("invalid capture-efficiency attempt");+ witnessAttempts.push({+ sessionId: stringValue(attempt, "sessionId"),+ status: numberValue(attempt, "status") ?? 0,+ snapshotType: stringValue(attempt, "snapshotType"),+ schemaVersion: numberValue(attempt, "schemaVersion"),+ workspaceRoot: optionalString(attempt.workspaceRoot),+ filePaths: Array.isArray(attempt.filePaths) ? attempt.filePaths.filter((path): path is string => typeof path === "string") : [],+ traceTypes: Array.isArray(attempt.traceTypes) ? attempt.traceTypes.filter((type): type is string => typeof type === "string") : [],+ traceFileEventTypes: Array.isArray(attempt.traceFileEventTypes)+ ? attempt.traceFileEventTypes.filter((type): type is string => typeof type === "string")+ : [],+ traceFileCount: numberValue(attempt, "traceFileCount") ?? 0,+ actorMarkers: Array.isArray(attempt.actorMarkers) ? attempt.actorMarkers.filter((actor): actor is string => typeof actor === "string") : [],+ });+ }+ const calls: Record<Actor, number> = {+ "actor-a": numberValue(body.capabilityCalls, "actor-a") ?? 0,+ "actor-b": numberValue(body.capabilityCalls, "actor-b") ?? 0,+ };+ const modes: CaptureEfficiencyWitness["modes"] = [];+ if (!Array.isArray(body.modes)) throw new Error("missing worker modes");+ for (const mode of body.modes) {+ if (mode !== "starting" && mode !== "worker" && mode !== "local" && mode !== "down") throw new Error("invalid worker mode");+ modes.push(mode);+ }+ return { attempts: witnessAttempts, capabilityCalls: calls, modes };+ },+ stop,+ };+}diff --git a/evals/packages/labs/src/index.ts b/evals/packages/labs/src/index.tsindex f9826bbe..64e81ee5 100644--- a/evals/packages/labs/src/index.ts+++ b/evals/packages/labs/src/index.ts@@ -10,3 +10,4 @@ export * from "./release-feed.ts";export * from "./mock-planetscale.ts";export * from "./mock-atlassian.ts";export * from "./mock-inference.ts";+export * from "./capture-efficiency.ts";diff --git a/evals/scripts/spec-boundary-ratchet.mjs b/evals/scripts/spec-boundary-ratchet.mjsindex 0214cc9f..50d59c07 100644--- a/evals/scripts/spec-boundary-ratchet.mjs+++ b/evals/scripts/spec-boundary-ratchet.mjs@@ -22,7 +22,7 @@ export function classifySpec(source) {importsNodeFs: imports.some((specifier) => /^(?:node:)?fs(?:\/promises)?$/.test(specifier)),importsChildProcess: imports.some((specifier) => /^(?:node:)?child_process$/.test(specifier)),crossesBoundary: imports.some((specifier) => /^(?:\.\.\/)?\.\.\/worlds\/|^@omnirush\/world$/.test(specifier))- || /(?<!\.)\b(?:app|chrome|server|inviteMember|faultProxy)\s*\(|\bspec\.world\s*\(/.test(source),+ || /(?<!\.)\b(?:app|chrome|server|inviteMember|faultProxy|startCaptureEfficiencyLab)\s*\(|\bspec\.world\s*\(/.test(source),};}diff --git a/evals/scripts/spec-boundary-ratchet.test.mjs b/evals/scripts/spec-boundary-ratchet.test.mjsindex 1de8c3aa..8f4029e7 100644--- a/evals/scripts/spec-boundary-ratchet.test.mjs+++ b/evals/scripts/spec-boundary-ratchet.test.mjs@@ -30,6 +30,14 @@ test("classifySpec recognizes only boundary calls", () => {assert.equal(classifySpec("helper.app(); const app = true;").crossesBoundary, false);});+test("the capture HTTP lab boundary preserves the product-source import guard", () => {+ const boundary = classifySpec('import { startCaptureEfficiencyLab } from "@omnirush/labs"; await startCaptureEfficiencyLab();');+ assert.deepEqual(violations("capture.test.ts", boundary), []);+ const sourceImport = classifySpec('import "../../apps/server/src/capture-client.js"; startCaptureEfficiencyLab();');+ assert.equal(sourceImport.importsProductSource, true);+ assert.equal(violations("bad.test.ts", sourceImport).length, 1);+});+test("classifySpec recognizes world imports as boundaries", () => {for (const specifier of ["../../worlds/example.ts", "../worlds/example.ts", "@omnirush/world"]) {assert.equal(classifySpec(`import "${specifier}";`).crossesBoundary, true);diff --git a/evals/specs/capture-efficiency.test.ts b/evals/specs/capture-efficiency.test.tsnew file mode 100644index 00000000..e64bebf5--- /dev/null+++ b/evals/specs/capture-efficiency.test.ts@@ -0,0 +1,140 @@+import { expect } from "vitest";+import { startCaptureEfficiencyLab, type CaptureEfficiencyAttempt, type CaptureEfficiencyLab, type CaptureEfficiencyWitness } from "@omnirush/labs";+import { eventually, test } from "@omnirush/testkit";++const SESSION_A = "session-a-0001";+const SESSION_B = "session-b-0001";++async function postControl(lab: CaptureEfficiencyLab, path: string, body: Record<string, unknown>): Promise<void> {+ const response = await fetch(`${lab.baseUrl}${path}`, {+ method: "POST",+ headers: {+ authorization: `Bearer ${lab.controlToken}`,+ "content-type": "application/json",+ },+ body: JSON.stringify(body),+ signal: AbortSignal.timeout(5_000),+ });+ const text = await response.text();+ expect(response.status, text).toBe(202);+}++function attemptsFor(witness: CaptureEfficiencyWitness, sessionId: string): CaptureEfficiencyAttempt[] {+ return witness.attempts.filter((attempt) => attempt.sessionId === sessionId);+}++function acceptedTrace(witness: CaptureEfficiencyWitness, sessionId: string): CaptureEfficiencyAttempt {+ const trace = attemptsFor(witness, sessionId).find((attempt) => attempt.snapshotType === "trace" && attempt.status === 201);+ if (!trace) throw new Error(`accepted trace for ${sessionId} was not observed`);+ return trace;+}++test("capture worker uploads ordered, scoped trace artifacts at the HTTP collector boundary", { timeout: 60_000 }, async ({ evidence }) => {+ const lab = await startCaptureEfficiencyLab();+ try {+ const initial = await lab.witness();+ const unauthorized = await fetch(`${lab.baseUrl}/witness`, { signal: AbortSignal.timeout(5_000) });+ expect(unauthorized.status).toBe(401);+ expect((await lab.witness()).attempts).toHaveLength(initial.attempts.length);+ evidence.recordAssertionEvidence(+ "Capture controls and collector witness require the lab bearer",+ "An unauthenticated HTTP witness request returned 401 and did not alter the outbound request record.",+ true,+ );++ await postControl(lab, "/control/start", { sessionId: SESSION_A, actor: "actor-a", workspace: "workspace-a" });+ await postControl(lab, "/control/trace", {+ sessionId: SESSION_A,+ type: "engine.request",+ data: { actor: "actor-a", workspace: "workspace-a", marker: "request-a" },+ });+ await postControl(lab, "/control/trace", {+ sessionId: SESSION_A,+ type: "turn.messages",+ data: {+ actor: "actor-a",+ workspace: "workspace-a",+ messages: [{ role: "assistant", parts: [{ type: "tool", tool: "apply_patch", state: { input: { path: "notes-a.txt" } } }] }],+ },+ });+ await postControl(lab, "/control/trace", {+ sessionId: SESSION_A,+ type: "turn.completed",+ data: { actor: "actor-a", workspace: "workspace-a", turn: 1 },+ });+ await postControl(lab, "/control/flush", { sessionId: SESSION_A });++ await postControl(lab, "/control/start", { sessionId: SESSION_B, actor: "actor-b", workspace: "workspace-b" });+ await postControl(lab, "/control/trace", {+ sessionId: SESSION_B,+ type: "engine.request",+ data: { actor: "actor-b", workspace: "workspace-b", marker: "request-b" },+ });+ await postControl(lab, "/control/trace", {+ sessionId: SESSION_B,+ type: "turn.completed",+ data: { actor: "actor-b", workspace: "workspace-b", turn: 1 },+ });+ await postControl(lab, "/control/flush", { sessionId: SESSION_B });++ await postControl(lab, "/control/complete", { sessionId: SESSION_A });+ await postControl(lab, "/control/complete", { sessionId: SESSION_B });++ const witness = await eventually(() => lab.witness(), {+ within: 20_000,+ intervalMs: 100,+ label: "capture worker collector requests",+ until: (current) => current.attempts.length === 7+ && current.attempts.filter((attempt) => attempt.status === 201).length === 6,+ });++ expect(witness.modes).toEqual(["worker", "worker"]);+ const traceA = acceptedTrace(witness, SESSION_A);+ const traceB = acceptedTrace(witness, SESSION_B);+ const traceAttemptsA = attemptsFor(witness, SESSION_A).filter((attempt) => attempt.snapshotType === "trace");+ const traceAttemptsB = attemptsFor(witness, SESSION_B).filter((attempt) => attempt.snapshotType === "trace");++ expect(traceAttemptsA.map((attempt) => [attempt.status, attempt.schemaVersion])).toEqual([[422, 3], [201, 2]]);+ expect(traceAttemptsB.map((attempt) => [attempt.status, attempt.schemaVersion])).toEqual([[201, 3]]);+ expect(traceB.filePaths).toEqual([]);+ expect(traceB.traceFileCount).toBe(0);+ expect(traceB.traceTypes).toEqual(["engine.request", "turn.completed"]);+ expect(traceA.schemaVersion).toBe(2);+ expect(traceA.traceTypes).toEqual(["engine.request", "turn.messages", "turn.completed"]);+ expect(traceA.traceFileCount).toBe(1);+ expect(traceA.traceFileEventTypes).toEqual(traceA.traceTypes);+ evidence.recordAssertionEvidence(+ "Canonical schema 3 is emitted without a duplicate trace artifact",+ "The accepted actor-b trace request used schema 3 with files=[] and no legacy trace file; its ordered events were emitted once.",+ true,+ );+ evidence.recordAssertionEvidence(+ "Unsupported schema 3 downgrades the same ordered batch to legacy schema 2",+ "Actor-a received one 422 unsupported-schema attempt followed by one schema-2 trace with exactly one trace.json whose event order matched the canonical batch.",+ true,+ );++ const actorAAttempts = attemptsFor(witness, SESSION_A).filter((attempt) => attempt.status === 201);+ const actorBAttempts = attemptsFor(witness, SESSION_B).filter((attempt) => attempt.status === 201);+ expect(actorAAttempts.map((attempt) => attempt.snapshotType)).toEqual(["start", "trace", "end"]);+ expect(actorBAttempts.map((attempt) => attempt.snapshotType)).toEqual(["start", "trace", "end"]);+ expect(actorAAttempts.every((attempt) => attempt.workspaceRoot === "workspace-a")).toBe(true);+ expect(actorBAttempts.every((attempt) => attempt.workspaceRoot === "workspace-b")).toBe(true);+ expect(actorAAttempts.every((attempt) => attempt.actorMarkers.every((actor) => actor === "actor-a"))).toBe(true);+ expect(actorBAttempts.every((attempt) => attempt.actorMarkers.every((actor) => actor === "actor-b"))).toBe(true);+ expect(actorAAttempts[0]?.filePaths).toContain("notes-a.txt");+ expect(actorAAttempts[0]?.filePaths).not.toContain("notes-b.txt");+ expect(actorBAttempts[0]?.filePaths).toContain("notes-b.txt");+ expect(actorBAttempts[0]?.filePaths).not.toContain("notes-a.txt");+ expect(traceA.actorMarkers).toEqual(["actor-a", "actor-a", "actor-a"]);+ expect(traceB.actorMarkers).toEqual(["actor-b", "actor-b"]);+ expect(witness.capabilityCalls).toEqual({ "actor-a": 1, "actor-b": 1 });+ evidence.recordAssertionEvidence(+ "Capture records preserve completion order and actor/workspace scope",+ "Each synthetic session produced start, trace, and end in order; trace markers and workspace roots stayed within their owning actor, with one capability probe per worker.",+ true,+ );+ } finally {+ await lab.stop();+ }+});diff --git a/scripts/capture-efficiency-bench/README.md b/scripts/capture-efficiency-bench/README.mdnew file mode 100644index 00000000..e272f906--- /dev/null+++ b/scripts/capture-efficiency-bench/README.md@@ -0,0 +1,44 @@+# Capture efficiency measurements++These scripts use synthetic files and a loopback receiver. Run the two revisions with the same Node, Bun, Electron, fixture, and host.++## Overlapping sessions++```sh+bun scripts/capture-efficiency-bench/shared_scan.mjs gui /path/to/gui /path/to/results/gui+bun scripts/capture-efficiency-bench/shared_scan.mjs cli /path/to/cli /path/to/results/cli+```++The driver starts eight sessions on one 1,000-file workspace using the product collector. It reports real file stats, reads, redactions, and upload bytes. Elapsed time is indicative; compare work counts first.++## Desktop transport++Requires Linux, Xvfb, Python 3, and the same Electron binary for both runs.++```sh+python3 scripts/capture-efficiency-bench/generate_transport.py /path/to/transport-fixture.zst+node scripts/capture-efficiency-bench/slow_sink.mjs+python3 scripts/capture-efficiency-bench/run_electron.py \+ --electron /path/to/electron --adapter /baseline/apps/desktop/electron/external-fetch.mjs \+ --variant baseline --fixture /path/to/transport-fixture.zst --output /path/to/results+python3 scripts/capture-efficiency-bench/run_electron.py \+ --electron /path/to/electron --adapter /candidate/apps/desktop/electron/external-fetch.mjs \+ --variant candidate --fixture /path/to/transport-fixture.zst --output /path/to/results+```++The 32 MiB fixture is opaque transport data, not a valid collector envelope. The receiver verifies byte count and SHA-256 and limits consumption to 6 MiB/second, independent of chunk size. The baseline driver models the retained byte buffers from the old worker/broker path and calls the original transport; the candidate passes a file descriptor through the worker/broker and reads one bounded body for Electron’s existing native fetch. Node/Bun upload paths stream the file; Electron buffers its body to avoid the pinned runtime’s interrupted chunked-upload crash. This is a transport benchmark, not a complete desktop journey. Peak RSS includes Electron, Chromium children and Xvfb, sampled every 20 ms. Do not describe it as JavaScript heap usage or a universal application-wide reduction.++Separate staging collector/viewer/export checks use valid zstd envelopes and assert exact reconstructed file contents, manifests, ordered trace events, and owner isolation.++## Native failure recovery++The Electron driver also checks a 401 followed by a full-byte authenticated retry, refused redirects, truncated responses, the response byte cap, and cancellation:++```sh+xvfb-run -a /path/to/electron --no-sandbox --disable-gpu \+ /absolute/path/to/electron_edge_cases.mjs \+ /absolute/path/to/external-fetch.mjs /absolute/path/to/transport-fixture.zst \+ /absolute/path/to/results/edge.json /absolute/path/to/profile+```++The driver catches uncaught native runtime errors as failures; a timeout cannot satisfy a rejection assertion.diff --git a/scripts/capture-efficiency-bench/electron_edge_cases.mjs b/scripts/capture-efficiency-bench/electron_edge_cases.mjsnew file mode 100644index 00000000..61104859--- /dev/null+++ b/scripts/capture-efficiency-bench/electron_edge_cases.mjs@@ -0,0 +1,77 @@+import { app, net } from "electron";+import { createServer } from "node:http";+import { createHash } from "node:crypto";+import { readFile, stat, writeFile } from "node:fs/promises";+import { fileURLToPath } from "node:url";+import assert from "node:assert/strict";+import { join, parse, resolve } from "node:path";++process.on("uncaughtException", error => { process.stderr.write("uncaught " + error.stack + "\n"); app.exit(1); });+process.on("unhandledRejection", error => { process.stderr.write("unhandled " + String(error.stack || error) + "\n"); app.exit(1); });+const entry = fileURLToPath(import.meta.url);+const args = process.argv.slice(process.argv.findIndex(arg => !arg.startsWith("-") && resolve(arg) === entry) + 1);+const [adapterPath, fixture, output, profile] = args;+app.setPath("userData", profile);+app.disableHardwareAcceleration();+app.commandLine.appendSwitch("host-resolver-rules", "MAP capture-bench.example.test 127.0.0.1");+app.whenReady().then(async () => {+ const { createExternalFileUpload } = await import(adapterPath);+ const upload = createExternalFileUpload({ net });+ const size = (await stat(fixture)).size;+ let redirected = 0, earlyBytes = 0, cancelController;+ const server = createServer((req, res) => {+ req.on("error", () => {});+ if (req.url === "/redirect-target") { redirected += 1; res.end("{}"); return; }+ if (req.url === "/redirect") { req.resume(); res.writeHead(302, { location: "/redirect-target" }); res.end(); return; }+ if (req.url === "/oversize") { req.resume(); res.writeHead(201); res.end(Buffer.alloc(1024 * 1024 + 1)); return; }+ if (req.url === "/truncated") { req.resume(); res.writeHead(201, { "content-length": "1024" }); res.write("{"); setTimeout(() => res.destroy(), 5); return; }+ if (req.url === "/cancel") { req.on("data", () => cancelController.abort()); return; }+ if (req.headers.authorization !== "Bearer synthetic-refreshed") {+ req.on("data", chunk => { earlyBytes += chunk.length; req.pause(); setTimeout(() => req.resume(), Math.max(1, chunk.length / (4 * 1024 * 1024) * 1000)); });+ res.writeHead(401, { "content-type": "application/json" }); res.end('{"detail":"expired"}'); return;+ }+ const digest = createHash("sha256"); let bytes = 0;+ req.on("data", chunk => { bytes += chunk.length; digest.update(chunk); });+ req.on("end", () => { res.writeHead(201, { "content-type": "application/json" }); res.end(JSON.stringify({ bytes, sha256: digest.digest("hex") })); });+ req.on("error", () => res.destroy());+ });+ await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));+ const origin = "http://capture-bench.example.test:" + server.address().port;+ const init = { method: "POST", path: fixture, size, headers: { authorization: "Bearer synthetic-old" } };+ const attempt = (path, options = init) => upload(origin + path, { ...options, signal: options.signal ?? AbortSignal.timeout(15000) });+ const rejection = error => { assert.notEqual(error.name, "TimeoutError"); return true; };+ const progress = name => process.stderr.write("checking " + name + "\n");+ try {+ progress("authentication response");+ assert.equal((await attempt("/upload")).status, 401);+ progress("authenticated retry");+ const retry = await attempt("/upload", { ...init, headers: { authorization: "Bearer synthetic-refreshed" } });+ assert.equal(retry.status, 201);+ const body = await retry.json();+ assert.equal(body.bytes, size);+ const fixturePath = parse(fixture);+ assert.equal(body.sha256, (await readFile(join(fixturePath.dir, fixturePath.name + ".sha256"), "utf8")).trim());+ // Chromium may finish sending the body before delivering an auth response.+ // The retry must reopen and transmit the complete unchanged file.+ progress("redirect refusal");+ await assert.rejects(attempt("/redirect"), rejection);+ assert.equal(redirected, 0);+ progress("truncated response");+ await assert.rejects(attempt("/truncated"), rejection);+ progress("response byte bound");+ await assert.rejects(attempt("/oversize"), /exceeds its bound/);+ progress("cancellation");+ cancelController = new AbortController();+ await assert.rejects(upload(origin + "/cancel", { ...init, signal: cancelController.signal }), { name: "AbortError" });+ await writeFile(output, JSON.stringify({+ electron: process.versions.electron, bytes: size, sha256: body.sha256, early_request_body_bytes: earlyBytes,+ assertions: { auth_response_returned: true, retry_reopens_entire_file: true, redirects_refused: true,+ truncated_response_rejected: true, response_size_bounded: true, cancellation: true },+ }, null, 2));+ } finally {+ progress("cleanup");+ server.closeAllConnections();+ await new Promise(resolve => server.close(resolve));+ }+ app.quit();+}).catch(error => { process.stderr.write(String(error.stack || error) + "\n"); app.exit(1); });diff --git a/scripts/capture-efficiency-bench/electron_transport.mjs b/scripts/capture-efficiency-bench/electron_transport.mjsnew file mode 100644index 00000000..ee0c4a34--- /dev/null+++ b/scripts/capture-efficiency-bench/electron_transport.mjs@@ -0,0 +1,33 @@+import { app, net } from "electron";+import { readFile, writeFile } from "node:fs/promises";+import { createHash } from "node:crypto";+import { fileURLToPath, pathToFileURL } from "node:url";+const scriptIndex=process.argv.indexOf(fileURLToPath(import.meta.url));+if(scriptIndex<0) throw new Error("benchmark entry missing from argv");+const [variant, adapterPath, fixture, expectedHash, resultPath, profile] = process.argv.slice(scriptIndex+1);+app.setPath("userData",profile);+app.disableHardwareAcceleration();+app.commandLine.appendSwitch("host-resolver-rules","MAP capture-bench.example.test 127.0.0.1");+async function run() {+const adapter=await import(pathToFileURL(adapterPath).href);+const started=performance.now();+const headers={"Content-Type":"application/zstd","X-OmniRush-Session-ID":"ses_transport_benchmark"};+const url="http://capture-bench.example.test:18692/collect";+let response;+if(variant==="baseline"){+ const compressed=await readFile(fixture);+ const transferred=new Uint8Array(compressed);+ const body=transferred.buffer.slice(transferred.byteOffset,transferred.byteOffset+transferred.byteLength);+ // Keep the two retry-owned copies alive, as worker and broker each do.+ globalThis.benchmarkRetryBodies=[compressed,transferred];+ response=await adapter.createExternalFetch({net})(url,{method:"POST",headers,body});+}else{+ response=await adapter.createExternalFileUpload({net})(url,{method:"POST",headers,path:fixture,size:32*1024*1024,signal:AbortSignal.timeout(60000)});+}+const result=await response.json();+if(response.status!==201||result.bytes!==32*1024*1024||result.sha256!==expectedHash) throw new Error("transport body mismatch");+await writeFile(resultPath,JSON.stringify({variant,status:response.status,bytes:result.bytes,sha256:result.sha256,elapsed_ms:Math.round(performance.now()-started),electron:process.versions.electron,node:process.versions.node},null,2));+globalThis.benchmarkRetryBodies=null;+app.quit();+}+app.whenReady().then(run).catch(error => { console.error(error); app.exit(1); });diff --git a/scripts/capture-efficiency-bench/generate_transport.py b/scripts/capture-efficiency-bench/generate_transport.pynew file mode 100644index 00000000..99be8767--- /dev/null+++ b/scripts/capture-efficiency-bench/generate_transport.py@@ -0,0 +1,21 @@+"""Generate an opaque upload body for the transport-only memory comparison."""+import argparse, hashlib+from pathlib import Path++parser = argparse.ArgumentParser()+parser.add_argument("path", type=Path)+parser.add_argument("--bytes", type=int, default=32 * 1024 * 1024)+args = parser.parse_args()+if args.bytes < 1:+ parser.error("--bytes must be positive")+args.path.parent.mkdir(parents=True, exist_ok=True)+digest = hashlib.sha256()+written = 0+with args.path.open("wb") as output:+ while written < args.bytes:+ block = hashlib.shake_256(b"omnirush synthetic capture transport" + (written // 65536).to_bytes(8, "big")).digest(min(65536, args.bytes - written))+ output.write(block)+ digest.update(block)+ written += len(block)+args.path.with_suffix(".sha256").write_text(digest.hexdigest())+print(f"{written} bytes; sha256={digest.hexdigest()}")diff --git a/scripts/capture-efficiency-bench/measure_tree.py b/scripts/capture-efficiency-bench/measure_tree.pynew file mode 100644index 00000000..7b6992d1--- /dev/null+++ b/scripts/capture-efficiency-bench/measure_tree.py@@ -0,0 +1,33 @@+"""Bounded process-tree RSS sampler for a task-owned command."""+import argparse,json,os,subprocess,time+from pathlib import Path+p=argparse.ArgumentParser(); p.add_argument("--output",required=True); p.add_argument("--interval",type=float,default=.02); p.add_argument("--timeout",type=float,default=180); p.add_argument("command",nargs=argparse.REMAINDER); a=p.parse_args()+cmd=a.command[1:] if a.command and a.command[0]=="--" else a.command+proc=subprocess.Popen(cmd,start_new_session=True)+started=time.monotonic(); peak=0; samples=0; peak_n=0; last_tree=[]+def tree(root):+ children={}+ for entry in Path("/proc").iterdir():+ if not entry.name.isdigit(): continue+ try:+ text=(entry/"stat").read_text(); rest=text[text.rfind(")")+2:].split()+ children.setdefault(int(rest[1]),[]).append(int(entry.name))+ except (OSError,ValueError,IndexError): pass+ result=[]; pending=[root]+ while pending:+ pid=pending.pop(); result.append(pid); pending+=children.get(pid,[])+ return result+while proc.poll() is None:+ if time.monotonic()-started>a.timeout:+ os.killpg(proc.pid,15); proc.wait(timeout=10); raise TimeoutError("task-owned benchmark timed out")+ pids=tree(proc.pid); total=0+ for pid in pids:+ try:+ line=next(l for l in Path(f"/proc/{pid}/status").read_text().splitlines() if l.startswith("VmRSS:"))+ total+=int(line.split()[1])*1024+ except (OSError,StopIteration,ValueError): pass+ if total>peak: peak=total; last_tree=pids+ peak_n=max(peak_n,len(pids)); samples+=1; time.sleep(a.interval)+result={"peak_tree_rss_bytes":peak,"elapsed_s":round(time.monotonic()-started,4),"samples":samples,"peak_process_count":peak_n,"exit_code":proc.returncode,"interval_s":a.interval,"command":cmd}+Path(a.output).write_text(json.dumps(result,indent=2)); print(json.dumps(result))+raise SystemExit(proc.returncode)diff --git a/scripts/capture-efficiency-bench/run_electron.py b/scripts/capture-efficiency-bench/run_electron.pynew file mode 100644index 00000000..8ee906f8--- /dev/null+++ b/scripts/capture-efficiency-bench/run_electron.py@@ -0,0 +1,24 @@+"""Measure a real Electron upload with an unchanged, specified runtime."""+import argparse, subprocess+from pathlib import Path++parser = argparse.ArgumentParser()+parser.add_argument("--electron", type=Path, required=True)+parser.add_argument("--adapter", type=Path, required=True)+parser.add_argument("--variant", choices=["baseline", "candidate"], required=True)+parser.add_argument("--fixture", type=Path, required=True)+parser.add_argument("--output", type=Path, required=True)+args = parser.parse_args()+here = Path(__file__).resolve().parent+args.output.mkdir(parents=True, exist_ok=True)+profile = args.output / ("electron-profile-" + args.variant)+profile.mkdir(exist_ok=True)+command = [+ "python3", str(here / "measure_tree.py"), "--output", str(args.output / ("electron-" + args.variant + "-rss.json")),+ "--timeout", "120", "--", "xvfb-run", "-a", str(args.electron.resolve()),+ "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage", str(here / "electron_transport.mjs"),+ args.variant, str(args.adapter.resolve()), str(args.fixture.resolve()),+ args.fixture.with_suffix(".sha256").read_text().strip(),+ str(args.output / ("electron-" + args.variant + ".json")), str(profile.resolve()),+]+raise SystemExit(subprocess.run(command).returncode)diff --git a/scripts/capture-efficiency-bench/shared_scan.mjs b/scripts/capture-efficiency-bench/shared_scan.mjsnew file mode 100644index 00000000..487db0bb--- /dev/null+++ b/scripts/capture-efficiency-bench/shared_scan.mjs@@ -0,0 +1,29 @@+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";+import { join, resolve } from "node:path";+import { pathToFileURL } from "node:url";+import { execFileSync } from "node:child_process";+const [kind, treeArg, outputArg] = process.argv.slice(2);+const tree = resolve(treeArg);+const output = resolve(outputArg);+const source = kind === "gui" ? join(tree,"apps/server/src/session-uploader.ts") : join(tree,"assets/extensions/omnirush/capture/workspace-sync.ts");+const mod = await import(pathToFileURL(source).href);+const Collector = kind === "gui" ? mod.SessionUploader : mod.WorkspaceSync;+await mkdir(output,{recursive:true});+const fixture=await mkdtemp(join(output,"shared-root-"));+const state=await mkdtemp(join(output,"state-"));+await mkdir(join(fixture,"src"));+for(let i=0;i<1000;i++) await writeFile(join(fixture,"src",String(i).padStart(4,"0")+".ts"),("export const fixture"+i+" = "+i+";\n").repeat(25));+execFileSync("git",["init","-q",fixture]);+let uploads=0,bytes=0;+const uploader=new Collector({+ stateDir:state,appVersion:"staging-benchmark",maxWatchedFiles:0,watch:false,+ upload:async(_session,payload)=>{uploads++;bytes+=payload.byteLength;return new Response("{}",{status:201});},+});+const ids=Array.from({length:8},(_,i)=>"ses_shared_scan_"+String(i).padStart(8,"0"));+const started=performance.now();+for(const id of ids) uploader.startSession(id,"shared-fixture",fixture);+await uploader.idleAll();+const result={kind,session_count:ids.length,file_count:1000,elapsed_ms:Math.round(performance.now()-started),uploads,compressed_bytes:bytes,metrics:{...uploader.metrics}};+await uploader.stop();+await writeFile(join(output,"shared-scan-"+kind+".json"),JSON.stringify(result,null,2));+console.log(JSON.stringify(result));diff --git a/scripts/capture-efficiency-bench/slow_sink.mjs b/scripts/capture-efficiency-bench/slow_sink.mjsnew file mode 100644index 00000000..04675352--- /dev/null+++ b/scripts/capture-efficiency-bench/slow_sink.mjs@@ -0,0 +1,22 @@+import { createServer } from "node:http";+import { createHash } from "node:crypto";+const BYTES_PER_SECOND = 6 * 1024 * 1024;+const server = createServer((req, res) => {+ let bytes = 0, chunks = 0;+ const digest = createHash("sha256");+ const started = performance.now();+ req.on("data", chunk => {+ bytes += chunk.length; chunks += 1; digest.update(chunk);+ const wait = bytes / BYTES_PER_SECOND * 1000 - (performance.now() - started);+ if (wait > 2) {+ req.pause();+ setTimeout(() => req.resume(), wait);+ }+ });+ req.on("end", () => {+ res.writeHead(201, {"Content-Type": "application/json"});+ res.end(JSON.stringify({bytes, chunks, elapsed_ms: Math.round(performance.now() - started), sha256: digest.digest("hex")}));+ });+ req.on("error", () => res.destroy());+});+server.listen(18692, "127.0.0.1");