Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

Reduce repeated capture work and upload copies

Task Builder70292d3f2e46
diff --git a/apps/desktop/electron/external-fetch.mjs b/apps/desktop/electron/external-fetch.mjs
index df307e1c..16860156 100644
--- a/apps/desktop/electron/external-fetch.mjs
+++ b/apps/desktop/electron/external-fetch.mjs
@@ -12,8 +12,12 @@
* Loopback targets (the embedded server, local development APIs) stay on
* Node's fetch, where CA trust is irrelevant.
*/
+import { createReadStream } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { Readable } from "node:stream";
const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]);
+const MAX_UPLOAD_RESPONSE_BYTES = 1024 * 1024;
/**
* @typedef {(input: string | URL | Request, init?: RequestInit) => Promise<Response>} ExternalFetch
@@ -65,3 +69,80 @@ export function createExternalFetch({ net, isReady = () => true, nodeFetch = glo
return net.fetch(/** @type {string | Request} */ (url), { credentials: "omit", ...init });
};
}
+
+function abortedUpload(signal) {
+ return signal?.reason instanceof Error
+ ? signal.reason
+ : new DOMException("The operation was aborted", "AbortError");
+}
+
+async function nodeFileUpload(nodeFetch, url, init) {
+ if (init.signal?.aborted) throw abortedUpload(init.signal);
+ const stream = createReadStream(init.path, { highWaterMark: 64 * 1024 });
+ const abort = () => stream.destroy(abortedUpload(init.signal));
+ init.signal?.addEventListener("abort", abort, { once: true });
+ try {
+ return await nodeFetch(url, {
+ method: init.method,
+ headers: init.headers,
+ body: Readable.toWeb(stream),
+ duplex: "half",
+ signal: init.signal,
+ redirect: "error",
+ });
+ } finally {
+ init.signal?.removeEventListener("abort", abort);
+ stream.destroy();
+ }
+}
+
+async function boundedUploadResponse(response) {
+ if (!response.body) return response;
+ const reader = response.body.getReader();
+ const chunks = [];
+ let total = 0;
+ try {
+ for (;;) {
+ const { value, done } = await reader.read();
+ if (done) break;
+ total += value.byteLength;
+ if (total > MAX_UPLOAD_RESPONSE_BYTES) throw new Error("session upload response exceeds its bound");
+ chunks.push(Buffer.from(value));
+ }
+ return new Response(Buffer.concat(chunks), {
+ status: response.status, statusText: response.statusText, headers: response.headers,
+ });
+ } catch (error) {
+ try { await reader.cancel(); } catch {}
+ throw error;
+ } finally {
+ reader.releaseLock();
+ }
+}
+
+/**
+ * A file descriptor avoids worker/broker IPC copies. Electron's existing
+ * net.fetch preserves its OS certificate verifier and system proxy.
+ *
+ * Keep one bounded body here: Electron 43's chunked net.request can throw
+ * outside the request on interrupted uploads ("two startReading calls").
+ * Node's path streams; Chromium buffers until that native issue is fixed.
+ */
+export function createExternalFileUpload({ net, isReady = () => true, nodeFetch = globalThis.fetch }) {
+ return async function externalFileUpload(url, init) {
+ if (init.signal?.aborted) throw abortedUpload(init.signal);
+ if (isLoopbackUrl(url) || typeof net?.fetch !== "function" || !isReady()) {
+ return boundedUploadResponse(await nodeFileUpload(nodeFetch, url, init));
+ }
+ if (!Number.isSafeInteger(init.size) || init.size < 0 || init.size > 64 * 1024 * 1024) {
+ throw new Error("session upload file exceeds its bound");
+ }
+ const body = await readFile(init.path, { signal: init.signal });
+ if (body.byteLength !== init.size) throw new Error("session upload file changed");
+ const response = await net.fetch(url, {
+ method: init.method, headers: init.headers, body, signal: init.signal,
+ credentials: "omit", redirect: "error",
+ });
+ return boundedUploadResponse(response);
+ };
+}
diff --git a/apps/desktop/electron/external-fetch.test.mjs b/apps/desktop/electron/external-fetch.test.mjs
index 95a9acbe..97adb28b 100644
--- a/apps/desktop/electron/external-fetch.test.mjs
+++ b/apps/desktop/electron/external-fetch.test.mjs
@@ -58,3 +58,113 @@ test("loopback targets, and calls before app ready or without net, stay on Node'
"https://omnirush.ai/",
]);
});
+
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createExternalFileUpload } from "./external-fetch.mjs";
+
+async function uploadFixture(t) {
+ const dir = await mkdtemp(join(tmpdir(), "capture-transport-"));
+ t.after(() => rm(dir, { recursive: true, force: true }));
+ const path = join(dir, "payload.zst");
+ const bytes = Buffer.alloc(256 * 1024, 73);
+ await writeFile(path, bytes);
+ return { path, size: bytes.length, bytes };
+}
+
+test("file upload uses Chromium credential policy and sends exact bytes", async (t) => {
+ const file = await uploadFixture(t);
+ const mock = recorder();
+ const response = await createExternalFileUpload({ net: { fetch: mock.fn } })("https://collector.example.test/collect", {
+ method: "POST", headers: { authorization: "Bearer synthetic-token" }, ...file,
+ });
+ assert.equal(response.status, 200);
+ assert.deepEqual(mock.calls[0].init.body, file.bytes);
+ assert.equal(mock.calls[0].init.credentials, "omit");
+ assert.equal(mock.calls[0].init.redirect, "error");
+});
+
+test("auth retries reopen the full file and use the refreshed bearer", async (t) => {
+ const file = await uploadFixture(t);
+ const calls = [];
+ const net = { fetch: async (_url, init) => {
+ calls.push(init);
+ return new Response("{}", { status: calls.length === 1 ? 401 : 201 });
+ }};
+ const upload = createExternalFileUpload({ net });
+ assert.equal((await upload("https://collector.example.test/collect", { method: "POST", ...file, headers: { authorization: "Bearer synthetic-old" } })).status, 401);
+ assert.equal((await upload("https://collector.example.test/collect", { method: "POST", ...file, headers: { authorization: "Bearer synthetic-refreshed" } })).status, 201);
+ assert.notEqual(calls[0].body, calls[1].body);
+ assert.deepEqual(calls[0].body, file.bytes);
+ assert.deepEqual(calls[1].body, file.bytes);
+ assert.equal(calls[1].headers.authorization, "Bearer synthetic-refreshed");
+});
+
+test("native file upload accepts a no-content response", async (t) => {
+ const file = await uploadFixture(t);
+ const net = { fetch: async () => new Response(null, { status: 204 }) };
+ const response = await createExternalFileUpload({ net })("https://collector.example.test/collect", { method: "POST", headers: {}, ...file });
+ assert.equal(response.status, 204);
+ assert.equal(await response.text(), "");
+});
+
+test("native file upload rejects truncated and oversized responses", async (t) => {
+ const file = await uploadFixture(t);
+ for (const broken of [true, false]) {
+ let cancelled = false;
+ const net = { fetch: async () => new Response(new ReadableStream({
+ start(controller) {
+ if (broken) controller.error(new Error("response truncated"));
+ else controller.enqueue(new Uint8Array(1024 * 1024 + 1));
+ },
+ cancel() { cancelled = true; },
+ })) };
+ await assert.rejects(createExternalFileUpload({ net })("https://collector.example.test/collect", { method: "POST", headers: {}, ...file }), /truncated|exceeds its bound/);
+ if (!broken) assert.equal(cancelled, true);
+ }
+});
+
+test("native upload cancellation rejects and pre-aborted uploads never fetch", async (t) => {
+ const file = await uploadFixture(t);
+ let calls = 0;
+ const net = { fetch: async (_url, init) => {
+ calls += 1;
+ return new Promise((_resolve, reject) => init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true }));
+ }};
+ const controller = new AbortController();
+ const upload = createExternalFileUpload({ net });
+ const result = upload("https://collector.example.test/collect", { method: "POST", headers: {}, ...file, signal: controller.signal });
+ while (!calls) await new Promise(resolve => setTimeout(resolve, 1));
+ controller.abort();
+ await assert.rejects(result, { name: "AbortError" });
+ await assert.rejects(upload("https://collector.example.test/collect", { method: "POST", headers: {}, ...file, signal: controller.signal }), { name: "AbortError" });
+ assert.equal(calls, 1);
+});
+
+test("native upload refuses invalid sizes and changed immutable files before fetch", async (t) => {
+ const file = await uploadFixture(t);
+ const mock = recorder();
+ const upload = createExternalFileUpload({ net: { fetch: mock.fn } });
+ for (const size of [-1, 64 * 1024 * 1024 + 1, file.size - 1]) {
+ await assert.rejects(upload("https://collector.example.test/collect", { method: "POST", headers: {}, ...file, size }), /bound|changed/);
+ }
+ assert.equal(mock.calls.length, 0);
+});
+
+test("loopback file uploads use Node duplex streaming and reopen on retries", async (t) => {
+ const file = await uploadFixture(t);
+ const bodies = [];
+ const nodeFetch = async (_url, init) => {
+ assert.equal(init.duplex, "half");
+ assert.equal(init.redirect, "error");
+ const bytes = await new Response(init.body).arrayBuffer();
+ bodies.push(Buffer.from(bytes));
+ return new Response("{}", { status: 201 });
+ };
+ const upload = createExternalFileUpload({ net: null, nodeFetch });
+ const init = { method: "POST", headers: {}, ...file };
+ await upload("http://127.0.0.1/collect", init);
+ await upload("http://127.0.0.1/collect", init);
+ assert.deepEqual(bodies, [file.bytes, file.bytes]);
+});
diff --git a/apps/desktop/electron/main.mjs b/apps/desktop/electron/main.mjs
index de3a0548..e2cf54cf 100644
--- a/apps/desktop/electron/main.mjs
+++ b/apps/desktop/electron/main.mjs
@@ -84,7 +84,7 @@ import { readSkillFolder } from "./skill-folder.mjs";
import { createDesktopVaultKeyProvider } from "./secure-vault-key.mjs";
import { applyLinuxPasswordStore, recordLinuxPasswordStore } from "./linux-password-store.mjs";
import { createDesktopOmniRushAccountStore, legacyKeychainAllowed } from "./omnirush-account.mjs";
-import { createExternalFetch } from "./external-fetch.mjs";
+import { createExternalFetch, createExternalFileUpload } from "./external-fetch.mjs";
import {
clearOmniRushSentrySession,
initOmniRushSentry,
@@ -122,6 +122,7 @@ const pty = require(["node", "pty"].join("-"));
// Chromium's network stack (OS trust store, system proxy) for the main
// process's own requests off the machine; see external-fetch.mjs.
const desktopExternalFetch = createExternalFetch({ net: electronNet, isReady: () => app.isReady() });
+const desktopExternalFileUpload = createExternalFileUpload({ net: electronNet, isReady: () => app.isReady() });
const NATIVE_DEEP_LINK_EVENT = "omnirush:deep-link-native";
const AUTOMATION_RUNNER_CREDENTIAL_REJECTED_EVENT = "omnirush:automation-runner:credential-rejected";
const isDevMode = process.env.OMNIRUSH_DEV_MODE === "1";
@@ -1360,6 +1361,7 @@ const runtimeManager = createRuntimeManager({
app,
desktopRoot: path.resolve(__dirname, ".."),
externalFetch: desktopExternalFetch,
+ captureFileUpload: desktopExternalFileUpload,
// Rewrites persisted `npx -y omnirush-ui-mcp` entries to the bundled launch
// before the engine starts. Null (bundle missing) removes them instead.
omnirushUiMcpLaunch: () => {
diff --git a/apps/desktop/electron/runtime.mjs b/apps/desktop/electron/runtime.mjs
index 6bc4a24c..8e121e88 100644
--- a/apps/desktop/electron/runtime.mjs
+++ b/apps/desktop/electron/runtime.mjs
@@ -1418,6 +1418,7 @@ export function createRuntimeManager({
workspacePlatform = process.platform,
// Chromium-backed fetch for requests off the machine (see external-fetch.mjs).
externalFetch = null,
+ captureFileUpload = null,
}) {
const inheritedProcessEnv = { ...process.env };
let injectedUserEnvKeys = new Set();
@@ -2221,6 +2222,7 @@ export function createRuntimeManager({
opencodeCwd: managedOpencodeWorkdir(),
localManagedMcpVaultKey,
appVersion: typeof appVersion === "string" && appVersion.trim() ? appVersion.trim() : undefined,
+ captureFileUpload: typeof captureFileUpload === "function" ? captureFileUpload : undefined,
// Bundled UI-control MCP launch for the startup migration of persisted
// `npx -y omnirush-ui-mcp` entries; null removes those entries instead.
omnirushUiMcp: omnirushUiMcpLaunch() ?? null,
diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.ts
index b168cc8a..8839307f 100644
--- a/apps/server/src/capture-client.ts
+++ b/apps/server/src/capture-client.ts
@@ -11,6 +11,8 @@
* number of times.
*/
import { Worker } from "node:worker_threads";
+import { lstat } from "node:fs/promises";
+import { resolve } from "node:path";
import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineReplacement, type EngineTarget, type PromptRecord } from "./capture-host.js";
import {
@@ -26,7 +28,7 @@ import {
type ToWorker,
} from "./capture-protocol.js";
import { externalFetch } from "./server-fetch.js";
-import { isUploadableWebUrl, sessionUploaderEnabled, type UploadWebVisit } from "./session-uploader.js";
+import { isUploadableWebUrl, MAX_UPLOAD_COMPRESSED_BYTES, sessionUploaderEnabled, type UploadWebVisit } from "./session-uploader.js";
/** A prompt body larger than this is not traced, so it is not copied to the worker either. */
const MAX_TRACED_REQUEST_BYTES = 4 * 1024 * 1024;
@@ -54,6 +56,8 @@ export type CaptureService = {
hasSession(sessionId: string): boolean;
startSession(sessionId: string, workspaceId: string, root: string): void;
recordTrace(sessionId: string, type: string, data?: unknown): void;
+ flushTrace(sessionId: string, finalTrace?: unknown): void;
+ finishSession(sessionId: string, finalTrace?: unknown): void;
/** The "engine.request" event of a captured request (its body parsed off the main thread), plus a prompt's attachments. */
recordPrompt(sessionId: string, prompt: Omit<PromptRecord, "body"> & { body: ArrayBuffer | undefined }): void;
captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void;
@@ -109,7 +113,7 @@ class CaptureClient implements CaptureService {
constructor(private readonly options: CaptureServiceOptions) {
this.uploadEnabled = sessionUploaderEnabled({
- upload: Boolean(options.sessionUploader.upload),
+ upload: Boolean(options.sessionUploader.upload || options.sessionUploader.uploadFile),
gatewayUrl: options.sessionUploader.gatewayUrl,
accessToken: options.sessionUploader.accessToken,
});
@@ -135,6 +139,15 @@ class CaptureClient implements CaptureService {
this.send({ kind: "call", id: null, method: "recordTrace", args: data === undefined ? [sessionId, type] : [sessionId, type, data] });
}
+ flushTrace(sessionId: string, finalTrace?: unknown): void {
+ this.send({ kind: "call", id: null, method: "flushTrace", args: finalTrace === undefined ? [sessionId] : [sessionId, finalTrace] });
+ }
+
+ finishSession(sessionId: string, finalTrace?: unknown): void {
+ if (this.sessions.get(sessionId) === true) this.sessions.set(sessionId, false);
+ this.send({ kind: "call", id: null, method: "finishSession", args: finalTrace === undefined ? [sessionId] : [sessionId, finalTrace] });
+ }
+
recordPrompt(sessionId: string, prompt: Omit<PromptRecord, "body"> & { body: ArrayBuffer | undefined }): void {
// The body is still forwarded to the engine: the worker gets its own copy, moved rather than cloned.
const body = prompt.body && prompt.body.byteLength > 0 && prompt.body.byteLength <= MAX_TRACED_REQUEST_BYTES ? new Uint8Array(prompt.body.slice(0)) : null;
@@ -272,6 +285,8 @@ class CaptureClient implements CaptureService {
engineVersion: this.options.engineVersion,
sessionUploader: {
upload: Boolean(sessionUploader.upload),
+ uploadFile: Boolean(sessionUploader.uploadFile),
+ capabilities: Boolean(sessionUploader.capabilities),
refreshAccessToken: Boolean(sessionUploader.refreshAccessToken),
...(sessionUploader.gatewayUrl !== undefined ? { gatewayUrl: sessionUploader.gatewayUrl } : {}),
...(sessionUploader.accessToken !== undefined ? { accessToken: sessionUploader.accessToken } : {}),
@@ -436,6 +451,26 @@ class CaptureClient implements CaptureService {
if (!sessionUploader.upload) throw new Error("no session upload hook");
return { kind: "result", id, ok: true, response: await serializeResponse(await sessionUploader.upload(request.sessionId, request.body, signal)) };
}
+ case "uploadFile": {
+ if (!sessionUploader.uploadFile) throw new Error("no session file upload hook");
+ const root = resolve(this.options.stateDir);
+ const path = resolve(request.path);
+ if (!path.startsWith(`${root}${process.platform === "win32" ? "\\" : "/"}`)) throw new Error("capture file is outside the state directory");
+ const file = await lstat(path);
+ if (!file.isFile() || file.size !== request.size || file.size > MAX_UPLOAD_COMPRESSED_BYTES) {
+ throw new Error("capture file descriptor is invalid");
+ }
+ return {
+ kind: "result",
+ id,
+ ok: true,
+ response: await serializeResponse(await sessionUploader.uploadFile(request.sessionId, path, request.size, signal)),
+ };
+ }
+ case "capabilities": {
+ if (!sessionUploader.capabilities) throw new Error("no session capability hook");
+ return { kind: "result", id, ok: true, response: null, value: await sessionUploader.capabilities() };
+ }
case "refreshAccessToken": {
const refresh = channel === "archive" ? archive.refreshAccessToken : sessionUploader.refreshAccessToken;
return { kind: "result", id, ok: true, response: null, token: refresh ? await refresh() : null };
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index c172937e..aff82893 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -23,7 +23,7 @@ import {
import type { CaptureStopOptions } from "./capture-protocol.js";
import { SessionArchiver, type SessionArchiverOptions } from "./session-archive/index.js";
import { ProjectArchiveLifecycle, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";
-import { SessionUploader, type UploadMetrics, type UploadWebVisit } from "./session-uploader.js";
+import { SessionUploader, type TraceCapabilities, type UploadMetrics, type UploadWebVisit } from "./session-uploader.js";
export type { EngineReplacement, EngineTarget } from "./session-upload-observer.js";
@@ -58,6 +58,8 @@ export type CaptureHostOptions = {
log: CaptureLog;
sessionUploader: {
upload?: (sessionId: string, compressed: Uint8Array, signal?: AbortSignal) => Promise<Response>;
+ uploadFile?: (sessionId: string, path: string, size: number, signal?: AbortSignal) => Promise<Response>;
+ capabilities?: () => Promise<TraceCapabilities>;
refreshAccessToken?: () => Promise<string | null>;
fetch?: (input: string, init?: RequestInit) => Promise<Response>;
gatewayUrl?: string;
@@ -124,6 +126,16 @@ export class CaptureHost {
this.sessionUploader.recordTrace(sessionId, type, data);
}
+ flushTrace(sessionId: string, finalTrace?: unknown): void {
+ this.sessionUploader.flushTrace(sessionId, finalTrace);
+ }
+
+ finishSession(sessionId: string, finalTrace?: unknown): void {
+ this.sessionUploader.finishSession(sessionId, finalTrace);
+ this.archive.sessionEnded(sessionId);
+ this.observers.lastMessageIds.delete(sessionId);
+ }
+
/** The "engine.request" event of a captured request and, for a prompt dispatch, one "attachment" event per attached file. */
recordPrompt(sessionId: string, prompt: PromptRecord): void {
// The turn this prompt follows takes its end snapshot first, ahead of this prompt's own.
diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.ts
index a666596a..5acb654c 100644
--- a/apps/server/src/capture-protocol.ts
+++ b/apps/server/src/capture-protocol.ts
@@ -9,7 +9,7 @@
*/
import type { CaptureHost, EngineReplacement, EngineTarget, PromptRecord } from "./capture-host.js";
import type { FolderGateOptions } from "./session-archive/detect.js";
-import type { UploadWebVisit } from "./session-uploader.js";
+import type { TraceCapabilities, UploadWebVisit } from "./session-uploader.js";
/** How the capture stops: `archiveFinals` false (the account is gone) packs no final project archives. */
export type CaptureStopOptions = { archiveFinals: boolean };
@@ -18,6 +18,8 @@ export type CaptureStopOptions = { archiveFinals: boolean };
export type CaptureCalls = {
startSession: [sessionId: string, workspaceId: string, root: string];
recordTrace: [sessionId: string, type: string, data?: unknown];
+ flushTrace: [sessionId: string, finalTrace?: unknown];
+ finishSession: [sessionId: string, finalTrace?: unknown];
recordPrompt: [sessionId: string, prompt: PromptRecord];
captureSnapshot: [sessionId: string, trigger: "prompt" | "turn_completed"];
recordWebVisit: [sessionId: string, visit: UploadWebVisit];
@@ -39,6 +41,8 @@ export type CaptureCall = { [M in CaptureCallName]: { kind: "call"; id: number |
/** What a worker asks of the main thread. `channel` says whose request it is: archive requests are aborted first at shutdown. */
export type HostRequest =
| { type: "upload"; sessionId: string; body: Uint8Array<ArrayBuffer> }
+ | { type: "uploadFile"; sessionId: string; path: string; size: number }
+ | { type: "capabilities" }
| { type: "refreshAccessToken" }
| { type: "archiveRequest"; path: string; method: "GET" | "POST"; body?: string; refresh?: false }
| { type: "fetch"; url: string; method: string; headers: Array<[string, string]>; body?: Uint8Array<ArrayBuffer> | string };
@@ -48,7 +52,7 @@ export type RequestChannel = "uploader" | "archive";
export type SerializedResponse = { status: number; statusText: string; headers: Array<[string, string]>; body: ArrayBuffer | null };
export type RequestResult =
- | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null }
+ | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null; value?: TraceCapabilities }
| { kind: "result"; id: number; ok: false; error: string; name: string };
export type ToWorker = CaptureCall | RequestResult;
@@ -67,7 +71,7 @@ export type CaptureWorkerInit = {
stateDir: string;
appVersion: string;
engineVersion: string;
- sessionUploader: { upload: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };
+ sessionUploader: { upload: boolean; uploadFile: boolean; capabilities: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };
archive: {
enabled: boolean;
excludedDirs: string[];
@@ -82,7 +86,7 @@ export type CaptureWorkerInit = {
};
export async function serializeResponse(response: Response): Promise<SerializedResponse> {
- const body = response.body ? await response.arrayBuffer() : null;
+ const body = response.body ? await boundedResponseBody(response) : null;
return {
status: response.status,
statusText: response.statusText,
@@ -91,6 +95,36 @@ export async function serializeResponse(response: Response): Promise<SerializedR
};
}
+const MAX_RELAY_RESPONSE_BYTES = 1024 * 1024;
+
+async function boundedResponseBody(response: Response): Promise<ArrayBuffer | null> {
+ if (!response.body) return null;
+ const reader = response.body.getReader();
+ const chunks: Uint8Array[] = [];
+ let total = 0;
+ try {
+ while (true) {
+ const result = await reader.read();
+ if (result.done) break;
+ total += result.value.byteLength;
+ if (total > MAX_RELAY_RESPONSE_BYTES) {
+ await reader.cancel("capture relay response too large").catch(() => undefined);
+ throw new Error("capture relay response exceeds its bound");
+ }
+ chunks.push(result.value);
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ const body = new Uint8Array(total);
+ let offset = 0;
+ for (const chunk of chunks) {
+ body.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return body.buffer;
+}
+
const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);
export function deserializeResponse(value: SerializedResponse): Response {
@@ -117,6 +151,10 @@ export function invokeCapture(host: CaptureHost, call: CaptureCall): unknown {
return host.startSession(...call.args);
case "recordTrace":
return host.recordTrace(...call.args);
+ case "flushTrace":
+ return host.flushTrace(...call.args);
+ case "finishSession":
+ return host.finishSession(...call.args);
case "recordPrompt":
return host.recordPrompt(...call.args);
case "captureSnapshot":
diff --git a/apps/server/src/capture-worker.ts b/apps/server/src/capture-worker.ts
index 2e442c14..f7bafd5a 100644
--- a/apps/server/src/capture-worker.ts
+++ b/apps/server/src/capture-worker.ts
@@ -22,6 +22,7 @@ import {
type ToWorker,
} from "./capture-protocol.js";
import type { ArchiveApiRequestInit } from "./session-archive/upload.js";
+import type { TraceCapabilities } from "./session-uploader.js";
function mainThreadPort(): MessagePort {
if (!parentPort) throw new Error("capture-worker runs as a worker thread only");
@@ -95,6 +96,16 @@ function refreshAccessToken(channel: RequestChannel) {
return async (): Promise<string | null> => (await ask(channel, { type: "refreshAccessToken" })).token ?? null;
}
+function capabilities(): Promise<TraceCapabilities> {
+ return ask("uploader", { type: "capabilities" }).then((result) => {
+ const value = result.value;
+ if (!value || !Array.isArray(value.schema_versions) || typeof value.canonical_trace !== "boolean") {
+ throw new Error("the main thread returned invalid capture capabilities");
+ }
+ return value;
+ });
+}
+
const log = (level: "info" | "warn", message: string, attributes?: Record<string, unknown>) => {
post({ kind: "log", level, message, ...(attributes ? { attributes } : {}) });
};
@@ -115,6 +126,16 @@ const host = new CaptureHost({
},
}
: {}),
+ ...(init.sessionUploader.uploadFile
+ ? {
+ uploadFile: async (sessionId: string, path: string, size: number, signal?: AbortSignal) => {
+ const result = await ask("uploader", { type: "uploadFile", sessionId, path, size }, signal);
+ if (!result.response) throw new Error("the main thread returned no response");
+ return deserializeResponse(result.response);
+ },
+ }
+ : {}),
+ ...(init.sessionUploader.capabilities ? { capabilities } : {}),
...(init.sessionUploader.refreshAccessToken ? { refreshAccessToken: refreshAccessToken("uploader") } : {}),
fetch: hostFetch("uploader"),
...(init.sessionUploader.gatewayUrl !== undefined ? { gatewayUrl: init.sessionUploader.gatewayUrl } : {}),
diff --git a/apps/server/src/embedded.ts b/apps/server/src/embedded.ts
index 1eb1c3a8..44ffab86 100644
--- a/apps/server/src/embedded.ts
+++ b/apps/server/src/embedded.ts
@@ -43,7 +43,7 @@ import { resolveOpencodeModelsEnv } from "./opencode-models-url.js";
import { assertOpencodeConfigCompat } from "./opencode-config-compat.js";
import { resolveEngineIdentity } from "./engine2/launch.js";
import type { ServeResult } from "./serve-node.js";
-import type { LocalManagedMcpVaultKeyProvider, OmniRushGatewayCredentials, ServerConfig } from "./types.js";
+import type { CaptureFileUpload, LocalManagedMcpVaultKeyProvider, OmniRushGatewayCredentials, ServerConfig } from "./types.js";
export type EmbeddedServerOptions = CliArgs & {
/** When true, spawn a managed OpenCode child process. */
@@ -59,6 +59,8 @@ export type EmbeddedServerOptions = CliArgs & {
resumeInterruptedTasks?: boolean;
/** Version of the embedding desktop app, reported as environment.app_version by the session uploader. */
appVersion?: string;
+ /** File-aware transport for immutable compressed session uploads. */
+ captureFileUpload?: CaptureFileUpload;
/**
* How the embedding desktop launches its bundled UI-control MCP. Persisted
* entries still using `npx -y omnirush-ui-mcp` are rewritten to it before
@@ -96,6 +98,7 @@ export async function startEmbeddedServer(options: EmbeddedServerOptions): Promi
config.omnirushEngineToken = randomUUID();
config.resumeInterruptedTasks = options.resumeInterruptedTasks === true && options.manageOpencode === true && !config.opencodeBaseUrl;
config.appVersion = typeof options.appVersion === "string" && options.appVersion.trim() ? options.appVersion.trim() : undefined;
+ config.captureFileUpload = options.captureFileUpload;
const logger = createServerLogger(config);
// Spawn managed OpenCode if requested and no explicit base URL was provided.
diff --git a/apps/server/src/omnirush-gateway-broker.test.ts b/apps/server/src/omnirush-gateway-broker.test.ts
index 8cb5e23b..26597f02 100644
--- a/apps/server/src/omnirush-gateway-broker.test.ts
+++ b/apps/server/src/omnirush-gateway-broker.test.ts
@@ -97,6 +97,84 @@ describe("OmniRush gateway broker", () => {
expect(response.status).toBe(404);
});
+ test("caches capabilities per credential pair, expires them, and falls back on a 404", async () => {
+ const originalNow = Date.now;
+ let now = originalNow();
+ let supported = true;
+ let probes = 0;
+ Date.now = () => now;
+ try {
+ const broker = new OmniRushGatewayBroker({
+ credentials: {
+ gatewayUrl: "https://gateway.example/omnirush/v1",
+ accessToken: "access-token",
+ refreshToken: "refresh-token",
+ },
+ engineToken: "local-engine-token",
+ fetch: async (input) => {
+ if (new URL(String(input)).pathname.endsWith("/collect/capabilities")) {
+ probes += 1;
+ return supported
+ ? Response.json({ schema_versions: [1, 2, 3], canonical_trace: true })
+ : new Response("not found", { status: 404 });
+ }
+ return Response.json({ output: [] });
+ },
+ });
+
+ const first = await Promise.all([broker.sessionUploadCapabilities(), broker.sessionUploadCapabilities()]);
+ expect(first).toEqual([
+ { schema_versions: [1, 2, 3], canonical_trace: true },
+ { schema_versions: [1, 2, 3], canonical_trace: true },
+ ]);
+ expect(probes).toBe(1);
+
+ now += 5 * 60_000 + 1;
+ supported = false;
+ expect(await broker.sessionUploadCapabilities()).toEqual({ schema_versions: [1, 2], canonical_trace: false });
+ expect(probes).toBe(2);
+
+ broker.resetCapabilities();
+ supported = true;
+ expect(await broker.sessionUploadCapabilities()).toEqual({ schema_versions: [1, 2, 3], canonical_trace: true });
+ expect(probes).toBe(3);
+ } finally {
+ Date.now = originalNow;
+ }
+ });
+
+ test("ends a file-aware upload callback promptly when its request is aborted", async () => {
+ let started!: () => void;
+ const uploadStarted = new Promise<void>((resolvePromise) => { started = resolvePromise; });
+ let callbackSignal: AbortSignal | undefined;
+ const broker = new OmniRushGatewayBroker({
+ credentials: {
+ gatewayUrl: "https://gateway.example/omnirush/v1",
+ accessToken: "access-token",
+ refreshToken: "refresh-token",
+ },
+ engineToken: "local-engine-token",
+ uploadFile: async (_url, init) => {
+ callbackSignal = init.signal;
+ started();
+ return new Promise<Response>(() => undefined);
+ },
+ fetch: async () => Response.json({ output: [] }),
+ });
+ const controller = new AbortController();
+ const pending = broker.uploadSessionFile("session-file-abort-1", "/tmp/synthetic-capture.zst", 64, controller.signal);
+ await uploadStarted;
+ controller.abort(new DOMException("test abort", "AbortError"));
+ let rejection: unknown;
+ try {
+ await pending;
+ } catch (error) {
+ rejection = error;
+ }
+ expect(rejection).toBeDefined();
+ expect(callbackSignal?.aborted).toBe(true);
+ });
+
test("invalidates a revoked device session and returns a professional sign-in error", async () => {
let invalidated = 0;
const broker = new OmniRushGatewayBroker({
diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.ts
index b72007d4..0e2f1b51 100644
--- a/apps/server/src/omnirush-gateway-broker.ts
+++ b/apps/server/src/omnirush-gateway-broker.ts
@@ -1,14 +1,18 @@
import { timingSafeEqual } from "node:crypto";
+import { createReadStream } from "node:fs";
import { SESSION_UPLOAD_BUDGET, SESSION_UPLOAD_ENDPOINT_PATH, sessionUploadTimeoutMs, type SessionUploadBudget } from "./session-upload-budget.js";
import { externalFetch } from "./server-fetch.js";
import type { OmniRushGatewayCredentialBundle, OmniRushGatewayCredentials } from "./types.js";
+import type { TraceCapabilities } from "./session-uploader.js";
import {
SUBAGENT_FALLBACK_EFFORT_HEADER,
SUBAGENT_FALLBACK_MODEL_HEADER,
SUBAGENT_ROOT_SESSION_HEADER,
} from "./omnirush-swarm.js";
+const TRACE_CAPABILITY_TTL_MS = 5 * 60_000;
+
/** A sub-agent request the gateway refused on the picked model and that was sent again on the main model. */
export type SubagentModelFallbackEvent = {
/** The sub-agent session (x-omnirush-session-id), when the request named it. */
@@ -32,6 +36,14 @@ type BrokerOptions = {
credentials?: OmniRushGatewayCredentials;
engineToken?: string;
fetch?: typeof externalFetch;
+ /** Electron's file-aware transport. Standalone runtimes use the bounded Node stream fallback. */
+ uploadFile?: (url: string, init: {
+ method: "POST";
+ headers: Record<string, string>;
+ path: string;
+ size: number;
+ signal?: AbortSignal;
+ }) => Promise<Response>;
log?: (level: "info" | "warn" | "error", message: string, attributes?: Record<string, unknown>) => void;
/** uploadSession()'s deadline parameters; SESSION_UPLOAD_BUDGET unless a test shrinks it. */
sessionUploadBudget?: SessionUploadBudget;
@@ -431,6 +443,34 @@ async function readableErrorResponse(response: Response): Promise<Response> {
});
}
+async function boundedText(response: Response, maxBytes: number): Promise<string | null> {
+ if (!response.body) return "";
+ const reader = response.body.getReader();
+ const chunks: Uint8Array[] = [];
+ let total = 0;
+ try {
+ while (true) {
+ const result = await reader.read();
+ if (result.done) break;
+ total += result.value.byteLength;
+ if (total > maxBytes) {
+ await reader.cancel("response too large").catch(() => undefined);
+ return null;
+ }
+ chunks.push(result.value);
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ const body = new Uint8Array(total);
+ let offset = 0;
+ for (const chunk of chunks) {
+ body.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return new TextDecoder().decode(body);
+}
+
type CredentialState = {
gatewayUrl: string;
accessToken: string;
@@ -501,6 +541,53 @@ function apiUrl(gatewayUrl: string, path: string): string {
return url.toString();
}
+function untilAborted(response: Promise<Response>, signal: AbortSignal): Promise<Response> {
+ if (signal.aborted) return Promise.reject(signal.reason);
+ return new Promise((resolvePromise, reject) => {
+ const onAbort = () => reject(signal.reason);
+ signal.addEventListener("abort", onAbort, { once: true });
+ response.then((value) => {
+ signal.removeEventListener("abort", onAbort);
+ if (signal.aborted) void value.body?.cancel().catch(() => undefined);
+ resolvePromise(value);
+ }, (error: unknown) => {
+ signal.removeEventListener("abort", onAbort);
+ reject(error);
+ });
+ });
+}
+
+function fileReadableBody(path: string, signal: AbortSignal): ReadableStream<Uint8Array> {
+ const source = createReadStream(path, { highWaterMark: 64 * 1024 });
+ const abort = () => source.destroy(signal.reason instanceof Error ? signal.reason : undefined);
+ const cleanup = () => signal.removeEventListener("abort", abort);
+ return new globalThis.ReadableStream<Uint8Array>({
+ start(controller) {
+ if (signal.aborted) {
+ abort();
+ return;
+ }
+ signal.addEventListener("abort", abort, { once: true });
+ source.on("data", (chunk: string | Buffer) => {
+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+ controller.enqueue(new Uint8Array(bytes));
+ });
+ source.on("end", () => {
+ cleanup();
+ controller.close();
+ });
+ source.on("error", (error: Error) => {
+ cleanup();
+ controller.error(error);
+ });
+ },
+ cancel(reason) {
+ cleanup();
+ source.destroy(reason instanceof Error ? reason : undefined);
+ },
+ });
+}
+
/** The backend's voice route, relative to the gateway URL (…/omnirush/v1/audio/transcriptions). */
export const VOICE_TRANSCRIPTIONS_PATH = "audio/transcriptions";
/** The backend allows 20 s for its primary provider and 30 s for the fallback. */
@@ -723,6 +810,7 @@ export class OmniRushGatewayBroker {
private readonly invalidate?: OmniRushGatewayCredentials["invalidate"];
private readonly latest?: OmniRushGatewayCredentials["latest"];
private readonly fetcher: typeof externalFetch;
+ private readonly fileUploader?: BrokerOptions["uploadFile"];
private readonly log?: BrokerOptions["log"];
private readonly sessionUploadBudget: SessionUploadBudget;
private readonly onSubagentFallback?: BrokerOptions["onSubagentFallback"];
@@ -737,6 +825,8 @@ export class OmniRushGatewayBroker {
* store entry cannot sign out a session this broker still holds.
*/
private previous: CredentialState | null = null;
+ private capabilityCache: { gatewayUrl: string; accountKey: string; version: 2 | 3; expiresAt: number } | null = null;
+ private capabilityProbe: { key: string; promise: Promise<2 | 3> } | null = null;
constructor(options: BrokerOptions) {
const gatewayUrl = options.credentials ? normalizedGatewayUrl(options.credentials.gatewayUrl) : null;
@@ -753,6 +843,7 @@ export class OmniRushGatewayBroker {
this.invalidate = options.credentials?.invalidate;
this.latest = options.credentials?.latest;
this.fetcher = options.fetch ?? externalFetch;
+ this.fileUploader = options.uploadFile;
this.log = options.log;
this.sessionUploadBudget = options.sessionUploadBudget ?? SESSION_UPLOAD_BUDGET;
this.onSubagentFallback = options.onSubagentFallback;
@@ -860,6 +951,94 @@ export class OmniRushGatewayBroker {
}));
}
+ uploadSessionFile(sessionId: string, path: string, size: number, signal?: AbortSignal): Promise<Response> {
+ const timeoutMs = sessionUploadTimeoutMs(size, this.sessionUploadBudget);
+ return this.withDeviceBearer(async (state) => {
+ const requestSignal = signal ? AbortSignal.any([AbortSignal.timeout(timeoutMs), signal]) : AbortSignal.timeout(timeoutMs);
+ const url = apiUrl(state.gatewayUrl, SESSION_UPLOAD_ENDPOINT_PATH);
+ const headers = {
+ Authorization: `Bearer ${state.accessToken}`,
+ "Content-Type": "application/zstd",
+ "X-OmniRush-Session-ID": sessionId,
+ };
+ if (this.fileUploader) {
+ return untilAborted(this.fileUploader(url, { method: "POST", headers, path, size, signal: requestSignal }), requestSignal);
+ }
+ const bodyAbort = new AbortController();
+ const bodySignal = AbortSignal.any([requestSignal, bodyAbort.signal]);
+ const body = fileReadableBody(path, bodySignal);
+ try {
+ const init: RequestInit & { duplex: "half" } = {
+ method: "POST",
+ headers,
+ body,
+ duplex: "half",
+ signal: requestSignal,
+ };
+ return await this.fetcher(url, init);
+ } finally {
+ bodyAbort.abort(new DOMException("The upload response arrived", "AbortError"));
+ }
+ });
+ }
+
+ async sessionUploadCapabilities(): Promise<TraceCapabilities> {
+ if (!this.state) return { schema_versions: [1, 2, 3], canonical_trace: false };
+ const state = this.state;
+ const gatewayUrl = state.gatewayUrl;
+ const key = this.capabilityKey(state);
+ if (this.capabilityCache?.gatewayUrl === gatewayUrl
+ && this.capabilityCache.accountKey === key
+ && this.capabilityCache.expiresAt > Date.now()) {
+ return { schema_versions: this.capabilityCache.version === 3 ? [1, 2, 3] : [1, 2], canonical_trace: this.capabilityCache.version === 3 };
+ }
+ const pending = this.capabilityProbe?.key === key
+ ? this.capabilityProbe.promise
+ : this.probeSessionUploadCapabilities(gatewayUrl);
+ if (!this.capabilityProbe || this.capabilityProbe.key !== key) {
+ this.capabilityProbe = { key, promise: pending };
+ void pending.finally(() => {
+ if (this.capabilityProbe?.promise === pending) this.capabilityProbe = null;
+ }).catch(() => undefined);
+ }
+ const version = await pending;
+ if (this.state && this.capabilityKey(this.state) === key) {
+ this.capabilityCache = { gatewayUrl, accountKey: key, version, expiresAt: Date.now() + TRACE_CAPABILITY_TTL_MS };
+ }
+ return { schema_versions: version === 3 ? [1, 2, 3] : [1, 2], canonical_trace: version === 3 };
+ }
+
+ resetCapabilities(): void {
+ this.capabilityCache = null;
+ this.capabilityProbe = null;
+ }
+
+ private capabilityKey(state: CredentialState): string {
+ return `${state.gatewayUrl}\0${state.refreshToken}`;
+ }
+
+ private async probeSessionUploadCapabilities(gatewayUrl: string): Promise<2 | 3> {
+ try {
+ if (this.state?.gatewayUrl !== gatewayUrl) return 2;
+ const response = await this.withDeviceBearer((state) => this.fetcher(apiUrl(state.gatewayUrl, "collect/capabilities"), {
+ method: "GET",
+ headers: { Authorization: `Bearer ${state.accessToken}`, Accept: "application/json" },
+ signal: AbortSignal.timeout(8_000),
+ }));
+ if (!response.ok) {
+ await response.body?.cancel().catch(() => undefined);
+ return 2;
+ }
+ const text = await boundedText(response, 64 * 1024);
+ if (text === null) return 2;
+ const payload: unknown = JSON.parse(text);
+ if (!isRecord(payload) || payload.canonical_trace !== true || !Array.isArray(payload.schema_versions)) return 2;
+ return payload.schema_versions.some((version) => version === 3) ? 3 : 2;
+ } catch {
+ return 2;
+ }
+ }
+
/**
* A project archive API call for the session archiver (`archives/key`,
* `archives`, `archives/<id>/parts|complete|abort`), authenticated like
@@ -1147,6 +1326,7 @@ export class OmniRushGatewayBroker {
refreshToken: stored.refreshToken,
rotation,
};
+ if (this.state.gatewayUrl !== current.gatewayUrl) this.resetCapabilities();
this.log?.("info", "omnirush.ai device credentials adopted from the account store", { rotation });
return true;
}
@@ -1185,6 +1365,7 @@ export class OmniRushGatewayBroker {
if (this.state !== spent) return Boolean(this.state);
}
this.state = null;
+ this.resetCapabilities();
void this.invalidate?.().catch(() => undefined);
return false;
}
@@ -1212,6 +1393,7 @@ export class OmniRushGatewayBroker {
if (!this.state) return { kind: "unavailable" };
const rotated = { accessToken, refreshToken, gatewayUrl, rotation: from.rotation + 1 };
this.state = rotated;
+ if (rotated.gatewayUrl !== from.gatewayUrl) this.resetCapabilities();
this.previous = null;
void this.persistLatest(rotated);
return { kind: "rotated" };
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index ec1a000f..e6da7d26 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -973,12 +973,14 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
// session uploads and queued project archives: a signed-out
// account leaves nothing queued.
invalidate: async () => {
+ gatewayBroker.resetCapabilities();
await capture.signOut();
await gatewayCredentials.invalidate?.();
},
}
: undefined,
engineToken: config.omnirushEngineToken,
+ ...(config.captureFileUpload ? { uploadFile: config.captureFileUpload } : {}),
// A sub-agent request moved to the main model: new sub-agent prompts skip
// the refused model for a while, and the session uploader records the model the
// sub-agent's messages really ran on.
@@ -1023,6 +1025,8 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
sessionUploader: gatewayBroker.enabled
? {
upload: (sessionId, compressed, signal) => gatewayBroker.uploadSession(sessionId, compressed, signal),
+ uploadFile: (sessionId, path, size, signal) => gatewayBroker.uploadSessionFile(sessionId, path, size, signal),
+ capabilities: () => gatewayBroker.sessionUploadCapabilities(),
refreshAccessToken: () => gatewayBroker.refreshAccessToken(),
}
: {},
diff --git a/apps/server/src/session-archive/index.test.ts b/apps/server/src/session-archive/index.test.ts
index b1da82b4..cb2cedc6 100644
--- a/apps/server/src/session-archive/index.test.ts
+++ b/apps/server/src/session-archive/index.test.ts
@@ -1262,3 +1262,91 @@ describe("SessionArchiver touched files", () => {
expect(server.callPaths().at(-1)).toBe("GET archives/key 200");
});
});
+
+
+test("archive disk budget pauses a base and resumes the same sequence after restart", async () => {
+ const server = new FakeArchiveServer();
+ const { root, blend } = await project();
+ const state = await tempDir("budget-state");
+ const low = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 0 });
+ expect(await low.captureBase("ses_budget_restart", root)).toMatchObject({ status: "paused", reason: "disk_budget" });
+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "queue"))).toEqual([]);
+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "tmp"))).toEqual([]);
+
+ const resumed = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3 });
+ expect(await resumed.captureBase("ses_budget_restart", root)).toMatchObject({ status: "queued", kind: "base", sequence: 0 });
+ expect((await resumed.drain()).uploaded).toBe(1);
+ const members = await openArchive(server.objects()[0]!.object!);
+ expect(members.find((member) => member.name === "assets/scene.blend")!.content.equals(blend)).toBe(true);
+});
+
+test("archive disk budget keeps the base and parent link when a delta is paused", async () => {
+ const server = new FakeArchiveServer();
+ const { root } = await project();
+ const state = await tempDir("budget-delta");
+ let available = 1024 ** 3;
+ const subject = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => available });
+ const base = await subject.captureBase("ses_budget_chain", root);
+ expect(base.status).toBe("queued");
+ await subject.drain();
+ await writeFile(join(root, "src/app.ts"), "export const app = 2;\n");
+ await rm(join(root, "assets/scene.blend"));
+ available = 0;
+ expect(await subject.captureDelta("ses_budget_chain", root, 1)).toMatchObject({ status: "paused", reason: "disk_budget" });
+ expect(server.objects()).toHaveLength(1);
+
+ const resumed = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3 });
+ expect(await resumed.captureDelta("ses_budget_chain", root, 1)).toMatchObject({ status: "queued", sequence: 1 });
+ await resumed.drain();
+ const delta = server.objects()[1]!;
+ expect(delta.request.parent_archive_id).toBe(base.status === "queued" ? base.archiveId : "");
+ const members = await openArchive(delta.object!);
+ expect(members.find((member) => member.name === "src/app.ts")!.content.toString()).toBe("export const app = 2;\n");
+ expect(manifestOf(members).deleted).toContain("assets/scene.blend");
+});
+
+test("archive disk budget removes only the uncommitted partial when space runs out during packing", async () => {
+ const server = new FakeArchiveServer();
+ server.partSize = 5 * 1024 * 1024;
+ const { root } = await project();
+ const state = await tempDir("budget-midpack");
+ const blob = randomBytes(10 * 1024 * 1024);
+ await writeFile(join(root, "assets/large.bin"), blob);
+ let probes = 0;
+ const subject = archiver(server, state, {
+ minimumFreeDiskBytes: 0,
+ freeDiskBytes: async () => ++probes === 1 ? 1024 ** 3 : 0,
+ });
+ expect(await subject.captureBase("ses_budget_midpack", root)).toMatchObject({ status: "paused", reason: "disk_budget" });
+ expect(probes).toBeGreaterThan(1);
+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "queue"))).toEqual([]);
+ expect(await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "tmp"))).toEqual([]);
+ expect(server.objects()).toEqual([]);
+
+ const resumed = archiver(server, state, { minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3 });
+ expect(await resumed.captureBase("ses_budget_midpack", root)).toMatchObject({ status: "queued", sequence: 0 });
+ await resumed.drain();
+ const members = await openArchive(server.objects()[0]!.object!);
+ expect(members.find((member) => member.name === "assets/large.bin")!.content.equals(blob)).toBe(true);
+});
+
+test("concurrent archive packing stays inside the shared pending byte budget", async () => {
+ const server = new FakeArchiveServer();
+ server.partSize = 5 * 1024 * 1024;
+ const { root } = await project();
+ const state = await tempDir("budget-concurrent");
+ await writeFile(join(root, "assets/large.bin"), randomBytes(12 * 1024 * 1024));
+ const budget = 20 * 1024 * 1024;
+ const subject = archiver(server, state, {
+ pendingByteBudget: budget, minimumFreeDiskBytes: 0, freeDiskBytes: async () => 1024 ** 3,
+ });
+ const results = await Promise.all([
+ subject.captureBase("ses_budget_parallel_01", root),
+ subject.captureBase("ses_budget_parallel_02", root),
+ ]);
+ expect(results.filter((result) => result.status === "queued")).toHaveLength(1);
+ expect(results.filter((result) => result.status === "paused")).toHaveLength(1);
+ const queue = await readdir(join(state, ARCHIVE_STATE_DIRECTORY, "queue"));
+ expect(queue).toHaveLength(1);
+ expect(results.reduce((sum, result) => sum + (result.status === "queued" ? result.size : 0), 0)).toBeLessThanOrEqual(budget);
+});
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index c4c86f17..863dc0de 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -13,7 +13,7 @@
* lifecycle.ts; see README.md.
*/
import { randomBytes, randomUUID } from "node:crypto";
-import { mkdir, readdir, readFile, rename, rm, stat } from "node:fs/promises";
+import { mkdir, readdir, readFile, rename, rm, stat, statfs } from "node:fs/promises";
import { join, resolve } from "node:path";
import { z } from "zod";
@@ -83,6 +83,9 @@ const SIGN_OUT_ABORT_TIMEOUT_MS = 5_000;
const START_FINAL_WINDOW_MS = 7 * 24 * 60 * 60_000;
/** At most this many sessions are scanned for a final archive at app start. */
const MAX_START_FINALS = 10;
+const DEFAULT_PENDING_BYTE_BUDGET = 256 * 1024 * 1024;
+const DEFAULT_MIN_FREE_DISK_BYTES = 512 * 1024 * 1024;
+const ARCHIVE_PACK_RESERVE_BYTES = 1024 * 1024;
/**
* How long a folder policy answer (4.4: all folders, touched files) is
* reused for folders without `.git`: sessions started meanwhile send no
@@ -119,6 +122,12 @@ export type SessionArchiverOptions = {
retry?: Partial<RetryPolicy>;
/** Tests: how soon reported touched paths are written (2 s). */
touchedFlushMs?: number;
+ /** Conservative bound for queued plus currently packing archive bytes. */
+ pendingByteBudget?: number;
+ /** Required free space before and during packing. */
+ minimumFreeDiskBytes?: number;
+ /** Tests and diagnostics: override the filesystem free-space probe. */
+ freeDiskBytes?: (path: string) => Promise<number>;
};
export type CaptureSkipReason =
@@ -136,9 +145,11 @@ export type CaptureSkipReason =
| "cancelled"
/** Final archives only: the server refused one (a backend without them); none until the app restarts. */
| "unsupported";
+ /** Packing paused before commit because the configured disk budget is full. */
export type CaptureResult =
| { status: "queued"; archiveId: string; kind: ArchiveKind; sequence: number; size: number }
+ | { status: "paused"; reason: "disk_budget"; requiredBytes: number; pendingBytes: number; freeBytes: number | null }
| { status: "skipped"; reason: CaptureSkipReason };
export type DrainResult = {
@@ -252,6 +263,18 @@ function errorSummary(error: unknown): string {
return "unknown error";
}
+function archiveDiskBytes(value: number | undefined, setting: string, fallback: number): number {
+ const configured = value ?? (process.env[setting] === undefined ? fallback : Number(process.env[setting]));
+ return Number.isSafeInteger(configured) && configured >= 0 ? configured : fallback;
+}
+
+class DiskBudgetPause extends Error {
+ constructor(readonly requiredBytes: number, readonly pendingBytes: number, readonly freeBytes: number | null) {
+ super("archive disk budget is full");
+ this.name = "DiskBudgetPause";
+ }
+}
+
/** Whether `policy` lets a chain with this marker capture: a plain folder needs all_folders, touched files touched_files, git nothing. */
function markerAllowed(marker: string, policy: ArchivePolicy): boolean {
if (marker === FOLDER_MARKER) return policy.allFolders;
@@ -282,6 +305,11 @@ export class SessionArchiver {
private readonly now: () => Date;
private readonly random: { uuid(): string; bytes(n: number): Buffer };
private readonly retry: RetryPolicy;
+ private readonly pendingByteBudget: number;
+ private readonly minimumFreeDiskBytes: number;
+ private readonly freeDiskBytes: (path: string) => Promise<number>;
+ private packingReservedBytes = 0;
+ private budgetTail: Promise<void> = Promise.resolve();
private started: Promise<void> | null = null;
private key: ArchiveKey | null = null;
private disabled: string | null = null;
@@ -327,6 +355,12 @@ export class SessionArchiver {
random: () => this.random.bytes(4).readUInt32BE(0) / 2 ** 32,
...options.retry,
};
+ this.pendingByteBudget = archiveDiskBytes(options.pendingByteBudget, "OMNIRUSH_ARCHIVE_PENDING_BYTE_BUDGET", DEFAULT_PENDING_BYTE_BUDGET);
+ this.minimumFreeDiskBytes = archiveDiskBytes(options.minimumFreeDiskBytes, "OMNIRUSH_ARCHIVE_MIN_FREE_DISK_BYTES", DEFAULT_MIN_FREE_DISK_BYTES);
+ this.freeDiskBytes = options.freeDiskBytes ?? (async (path) => {
+ const filesystem = await statfs(path);
+ return Number(filesystem.bavail) * Number(filesystem.bsize);
+ });
this.uploader = new ArchiveUploader({
gatewayUrl: options.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL,
...(options.accessToken !== undefined ? { accessToken: options.accessToken } : {}),
@@ -846,6 +880,58 @@ export class SessionArchiver {
}
}
+ private budgetLocked<T>(task: () => Promise<T>): Promise<T> {
+ const run = this.budgetTail.catch(() => undefined).then(task);
+ this.budgetTail = run.then(() => undefined, () => undefined);
+ return run;
+ }
+
+ private estimateArchiveOutput(entries: readonly ScannedEntry[]): number {
+ const tarBytes = ARCHIVE_PACK_RESERVE_BYTES
+ + entries.reduce((total, entry) => total + Math.max(0, entry.size) + 2048, 0);
+ // Account for incompressible zstd output and seal framing.
+ return Math.ceil(tarBytes * 1.01);
+ }
+
+ private reserveDiskBudget(requiredBytes: number): Promise<{ bytes: number; release(): void } | null> {
+ return this.budgetLocked(async () => {
+ const records = await this.listQueue();
+ const pendingBytes = records.reduce((total, record) => total + record.request.size, 0);
+ const freeBytes = await this.freeDiskBytes(this.dir).catch(() => null);
+ if (freeBytes === null
+ || pendingBytes + this.packingReservedBytes + requiredBytes > this.pendingByteBudget
+ || freeBytes < this.minimumFreeDiskBytes + requiredBytes) return null;
+ this.packingReservedBytes += requiredBytes;
+ let released = false;
+ const reservation = {
+ bytes: requiredBytes,
+ release: () => {
+ if (released) return;
+ released = true;
+ this.packingReservedBytes = Math.max(0, this.packingReservedBytes - reservation.bytes);
+ },
+ };
+ return reservation;
+ });
+ }
+
+ private checkDiskBudget(reservation: { bytes: number }, outputBytes: number, final = false): Promise<void> {
+ return this.budgetLocked(async () => {
+ const records = await this.listQueue();
+ const pendingBytes = records.reduce((total, record) => total + record.request.size, 0);
+ const freeBytes = await this.freeDiskBytes(this.dir).catch(() => null);
+ const boundedOutput = final ? outputBytes : Math.ceil(outputBytes * 1.01) + ARCHIVE_PACK_RESERVE_BYTES;
+ const extra = Math.max(0, boundedOutput - reservation.bytes);
+ if (freeBytes === null
+ || pendingBytes + this.packingReservedBytes + extra > this.pendingByteBudget
+ || freeBytes < this.minimumFreeDiskBytes + Math.max(ARCHIVE_PACK_RESERVE_BYTES, reservation.bytes + extra - boundedOutput)) {
+ throw new DiskBudgetPause(reservation.bytes + extra, pendingBytes, freeBytes);
+ }
+ this.packingReservedBytes += extra;
+ reservation.bytes += extra;
+ });
+ }
+
/** Pass 1, delta, pass 2 and the commit protocol of section 13.4. */
private async captureArchiveOnce(state: SessionState, kind: ArchiveKind, turn: number, key: ArchiveKey, generation: number, options: CaptureOptions): Promise<CaptureResult> {
const sessionId = state.session_id;
@@ -932,91 +1018,146 @@ export class SessionArchiver {
excluded,
});
const partial = join(this.dirs.tmp, `${archiveId}.partial`);
- let sealed: Awaited<ReturnType<typeof writeSealedArchive>>;
+ const requiredBytes = this.estimateArchiveOutput(files);
+ const reservation = await this.reserveDiskBudget(requiredBytes);
+ if (!reservation) {
+ let freeBytes: number | null = null;
+ try {
+ freeBytes = await this.freeDiskBytes(this.dir);
+ } catch {
+ freeBytes = null;
+ }
+ const pendingBytes = (await this.listQueue()).reduce((total, record) => total + record.request.size, 0);
+ this.log("info", "OmniRush project archive paused by disk budget", {
+ sessionId,
+ kind,
+ requiredBytes,
+ pendingBytes,
+ ...(freeBytes === null ? {} : { freeBytes }),
+ });
+ return { status: "paused", reason: "disk_budget", requiredBytes, pendingBytes, freeBytes };
+ }
+ let checkedBytes = 0;
+ let checkedAt = Date.now();
try {
- sealed = await writeSealedArchive(
- { root: state.root, manifest, createdAtSeconds: Math.floor(createdAt.getTime() / 1000), entries: files, ...(signal ? { signal } : {}) },
- { publicKey: key.publicKey },
- partial,
- );
+ const sealed = await writeSealedArchive(
+ {
+ root: state.root,
+ manifest,
+ createdAtSeconds: Math.floor(createdAt.getTime() / 1000),
+ entries: files,
+ ...(signal ? { signal } : {}),
+ onOutputBytes: async (bytes) => {
+ // Keep filesystem probes bounded; still grow an underestimated reservation before writing.
+ if (bytes - checkedBytes < 8 * 1024 * 1024 && Date.now() - checkedAt < 250
+ && Math.ceil(bytes * 1.01) + ARCHIVE_PACK_RESERVE_BYTES <= reservation.bytes) return;
+ await this.checkDiskBudget(reservation, bytes);
+ checkedBytes = bytes;
+ checkedAt = Date.now();
+ },
+ },
+ { publicKey: key.publicKey },
+ partial,
+ );
+ await this.checkDiskBudget(reservation, sealed.size, true);
+ if (generation !== this.generation || this.disabled || this.stoppedSessions.has(sessionId) || signal?.aborted) {
+ await rm(partial, { force: true });
+ if (this.disabled) return { status: "skipped", reason: "disabled" };
+ return { status: "skipped", reason: generation !== this.generation || this.stoppedSessions.has(sessionId) ? "stopped" : "cancelled" };
+ }
+ // 1-2: the sealed file and its queue record.
+ const sealedFile = `${archiveId}.orseal`;
+ const request: ArchiveCreateRequest = {
+ archive_id: archiveId,
+ session_id: sessionId,
+ kind,
+ sequence,
+ turn,
+ parent_archive_id: kind === "base" ? null : state.last_archive_id,
+ size: sealed.size,
+ sha256: sealed.sha256,
+ kid: sealed.kid,
+ content: SEAL_CONTENT,
+ marker: state.marker,
+ };
+ const record: QueueRecord = {
+ v: 1,
+ archive_id: archiveId,
+ session_key: sessionKey,
+ request,
+ sealed_file: sealedFile,
+ ...(kind === "delta" && options.trigger ? { trigger: options.trigger } : {}),
+ created_at: createdAt.toISOString(),
+ attempts: 0,
+ next_attempt_at: null,
+ recreates: 0,
+ upload: null,
+ };
+ await this.budgetLocked(async () => {
+ const pending = join(this.dirs.pending, sealedFile);
+ await rename(partial, pending);
+ try {
+ await writeJsonAtomic(join(this.dirs.queue, `${archiveId}.json`), record);
+ } catch (error) {
+ await rm(pending, { force: true });
+ throw error;
+ }
+ // A reservation becomes a queue record while the accounting lock is held.
+ reservation.release();
+ });
+ // 3: the new baseline is the full scan (touched files: the chain's files); entries that changed while packing get a null hash so the next delta sends them again.
+ const unstable = new Set(sealed.unstable);
+ for (const path of unstable) cache.forget(path);
+ const baselineName = `${sessionKey}-${sequence}.json`;
+ await writeChunksAtomic(join(this.dirs.baselines, baselineName), baselineChunks(next, unstable));
+ // 4: commit. Until the server has accepted a final archive, the first
+ // one keeps the chain point before it, baseline included.
+ const previousBaseline = state.baseline;
+ const rewind = state.rewind ?? (kind === "delta" && options.trigger === "final" && !this.finalsAccepted
+ ? { next_sequence: state.next_sequence, last_archive_id: state.last_archive_id, last_turn: state.last_turn, baseline: state.baseline }
+ : null);
+ await this.saveSession({
+ ...state,
+ next_sequence: sequence + 1,
+ last_archive_id: archiveId,
+ last_turn: turn,
+ baseline: baselineName,
+ rewind,
+ final_due: options.trigger !== "final",
+ ...(options.ended ? { ended: this.now().toISOString() } : {}),
+ updated_at: this.now().toISOString(),
+ });
+ if (previousBaseline && previousBaseline !== baselineName && previousBaseline !== rewind?.baseline) await rm(join(this.dirs.baselines, previousBaseline), { force: true });
+ await this.saveHashCache(rootKey, cache);
+ this.log("info", "OmniRush project archive queued", {
+ sessionId,
+ archiveId,
+ kind,
+ sequence,
+ turn,
+ ...(options.trigger === "final" ? { trigger: "final", reason: options.reason } : {}),
+ ...(touched ? { scope: "touched" } : {}),
+ bytes: sealed.size,
+ files: files.length,
+ ...(deleted ? { deleted: deleted.length } : {}),
+ ...(ignored > 0 ? { gitignored: ignored } : {}),
+ ...(unstable.size > 0 ? { unstable: unstable.size } : {}),
+ });
+ return { status: "queued", archiveId, kind, sequence, size: sealed.size };
} catch (error) {
await rm(partial, { force: true });
+ if (error instanceof DiskBudgetPause) return {
+ status: "paused", reason: "disk_budget", requiredBytes: error.requiredBytes,
+ pendingBytes: error.pendingBytes, freeBytes: error.freeBytes,
+ };
+ if (error instanceof Error && "code" in error && (error.code === "ENOSPC" || error.code === "EDQUOT")) {
+ return { status: "paused", reason: "disk_budget", requiredBytes: reservation.bytes,
+ pendingBytes: (await this.listQueue()).reduce((sum, record) => sum + record.request.size, 0), freeBytes: 0 };
+ }
throw error;
+ } finally {
+ reservation.release();
}
- if (generation !== this.generation || this.disabled || this.stoppedSessions.has(sessionId) || signal?.aborted) {
- await rm(partial, { force: true });
- if (this.disabled) return { status: "skipped", reason: "disabled" };
- return { status: "skipped", reason: generation !== this.generation || this.stoppedSessions.has(sessionId) ? "stopped" : "cancelled" };
- }
- // 1-2: the sealed file and its queue record.
- const sealedFile = `${archiveId}.orseal`;
- await rename(partial, join(this.dirs.pending, sealedFile));
- const request: ArchiveCreateRequest = {
- archive_id: archiveId,
- session_id: sessionId,
- kind,
- sequence,
- turn,
- parent_archive_id: kind === "base" ? null : state.last_archive_id,
- size: sealed.size,
- sha256: sealed.sha256,
- kid: sealed.kid,
- content: SEAL_CONTENT,
- marker: state.marker,
- };
- const record: QueueRecord = {
- v: 1,
- archive_id: archiveId,
- session_key: sessionKey,
- request,
- sealed_file: sealedFile,
- ...(kind === "delta" && options.trigger ? { trigger: options.trigger } : {}),
- created_at: createdAt.toISOString(),
- attempts: 0,
- next_attempt_at: null,
- recreates: 0,
- upload: null,
- };
- await writeJsonAtomic(join(this.dirs.queue, `${archiveId}.json`), record);
- // 3: the new baseline is the full scan (touched files: the chain's files); entries that changed while packing get a null hash so the next delta sends them again.
- const unstable = new Set(sealed.unstable);
- for (const path of unstable) cache.forget(path);
- const baselineName = `${sessionKey}-${sequence}.json`;
- await writeChunksAtomic(join(this.dirs.baselines, baselineName), baselineChunks(next, unstable));
- // 4: commit. Until the server has accepted a final archive, the first
- // one keeps the chain point before it, baseline included.
- const previousBaseline = state.baseline;
- const rewind = state.rewind ?? (kind === "delta" && options.trigger === "final" && !this.finalsAccepted
- ? { next_sequence: state.next_sequence, last_archive_id: state.last_archive_id, last_turn: state.last_turn, baseline: state.baseline }
- : null);
- await this.saveSession({
- ...state,
- next_sequence: sequence + 1,
- last_archive_id: archiveId,
- last_turn: turn,
- baseline: baselineName,
- rewind,
- final_due: options.trigger !== "final",
- ...(options.ended ? { ended: this.now().toISOString() } : {}),
- updated_at: this.now().toISOString(),
- });
- if (previousBaseline && previousBaseline !== baselineName && previousBaseline !== rewind?.baseline) await rm(join(this.dirs.baselines, previousBaseline), { force: true });
- await this.saveHashCache(rootKey, cache);
- this.log("info", "OmniRush project archive queued", {
- sessionId,
- archiveId,
- kind,
- sequence,
- turn,
- ...(options.trigger === "final" ? { trigger: "final", reason: options.reason } : {}),
- ...(touched ? { scope: "touched" } : {}),
- bytes: sealed.size,
- files: files.length,
- ...(deleted ? { deleted: deleted.length } : {}),
- ...(ignored > 0 ? { gitignored: ignored } : {}),
- ...(unstable.size > 0 ? { unstable: unstable.size } : {}),
- });
- return { status: "queued", archiveId, kind, sequence, size: sealed.size };
}
/** The entry list after the chain's last archive; null when it cannot be read (the chain cannot go on). */
diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.ts
index 710d3454..68596b19 100644
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@ -393,6 +393,10 @@ export class ProjectArchiveLifecycle {
record.start = null;
record.root = root;
const result = await this.archiver.captureBase(sessionId, root, count);
+ if (result.status === "paused") {
+ record.start = { root, engine: start.engine };
+ this.unresolved(sessionId, record, `archive paused: ${result.reason}`);
+ }
if (result.status === "skipped" && result.reason === "disabled") this.sessions.delete(sessionId);
if (result.status === "skipped" && (result.reason === "not_archivable" || result.reason === "stopped")) record.root = null;
this.settle(result);
@@ -513,7 +517,7 @@ export class ProjectArchiveLifecycle {
if (result.status === "queued") {
this.consentOffUntil = 0;
this.kick();
- } else if (result.reason === "disabled") {
+ } else if (result.status === "skipped" && result.reason === "disabled") {
this.archivingOff();
}
}
diff --git a/apps/server/src/session-archive/pack.ts b/apps/server/src/session-archive/pack.ts
index 2050a941..f6637b4e 100644
--- a/apps/server/src/session-archive/pack.ts
+++ b/apps/server/src/session-archive/pack.ts
@@ -141,6 +141,8 @@ export type PackInput = {
entries: readonly ScannedEntry[];
/** Stops the writer between members and output blocks: it then rejects with the signal's reason. */
signal?: AbortSignal;
+ /** Called with tar bytes emitted so far; used by the archive queue's disk admission checks. */
+ onOutputBytes?: (bytes: number) => Promise<void>;
};
export type PackOutcome = { unstable: string[]; tarBytes: number };
@@ -180,6 +182,7 @@ class TarWriter {
private async rotate(): Promise<void> {
this.input.signal?.throwIfAborted();
this.tarBytes += this.used;
+ await this.input.onOutputBytes?.(this.tarBytes);
this.block = await this.emit(this.block);
this.used = 0;
this.rotations += 1;
@@ -342,6 +345,7 @@ class TarWriter {
await this.zeros(2 * TAR_BLOCK);
if (this.used > 0) {
this.tarBytes += this.used;
+ await this.input.onOutputBytes?.(this.tarBytes);
await this.emit(this.block.subarray(0, this.used));
this.used = 0;
}
diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.ts
index fc595bc1..63516033 100644
--- a/apps/server/src/session-uploader.test.ts
+++ b/apps/server/src/session-uploader.test.ts
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, test } from "bun:test";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
-import { appendFile, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, writeFile } from "node:fs/promises";
+import { appendFile, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, utimes, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join, posix, win32 } from "node:path";
import { promisify } from "node:util";
@@ -469,6 +469,52 @@ describe("session uploader envelope v2", () => {
expect(trace.manifest).toEqual([]);
});
+ test("uses canonical trace schema 3, then retries the same batch as schema 2 on an explicit 422", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-upload-trace-schema-"));
+ roots.push(root);
+ const attempts: Envelope[] = [];
+ let capabilityCalls = 0;
+ let rejectSchema3 = true;
+ const sessionUploader = new SessionUploader({
+ capabilities: async () => {
+ capabilityCalls += 1;
+ return { schema_versions: [1, 2, 3], canonical_trace: true };
+ },
+ upload: async (_sessionId, compressed) => {
+ const envelope = JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope;
+ attempts.push(envelope);
+ if (envelope.schema_version === 3 && rejectSchema3) {
+ rejectSchema3 = false;
+ return Response.json({ error: "unsupported_schema", schema_version: 3 }, { status: 422 });
+ }
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ fallbackScanMs: 60_000,
+ });
+ const sessionId = "session-trace-schema-1";
+ sessionUploader.startSession(sessionId, "workspace-trace-schema", root);
+ await sessionUploader.idle(sessionId);
+ sessionUploader.recordTrace(sessionId, "engine.request", { path: "src/app.ts" });
+ sessionUploader.flushTrace(sessionId);
+ await sessionUploader.idle(sessionId);
+ await sessionUploader.stop();
+
+ expect(capabilityCalls).toBe(1);
+ expect(attempts.map((envelope) => [envelope.snapshot_type, envelope.schema_version])).toEqual([
+ ["start", 2],
+ ["trace", 3],
+ ["trace", 2],
+ ["end", 2],
+ ]);
+ const rejected = attempts[1]!;
+ const fallback = attempts[2]!;
+ expect(rejected.files).toEqual([]);
+ expect(rejected.trace?.map((event) => event.type)).toEqual(["engine.request"]);
+ expect(fallback.files.map((file) => file.path)).toEqual(["__omnirush__/trace.json"]);
+ expect(fallback.trace?.map((event) => event.type)).toEqual(rejected.trace?.map((event) => event.type));
+ expect(JSON.parse(fallback.files[0]!.content).events.map((event: { type: string }) => event.type)).toEqual(["engine.request"]);
+ });
+
test("summarises a git repository without shipping internals or remote credentials", async () => {
const root = await mkdtemp(join(tmpdir(), "omnirush-upload-git-"));
roots.push(root);
@@ -2549,7 +2595,11 @@ describe("session uploader snapshot cap", () => {
return root;
}
- async function capture(root: string, options: { snapshotMaxBytes?: number; touched?: string[] }) {
+ async function capture(root: string, options: {
+ snapshotMaxBytes?: number;
+ touched?: string[];
+ traceEvents?: Array<{ type: string; data: unknown }>;
+ }) {
const uploads: Envelope[] = [];
const rawBytes: number[] = [];
const warnings: string[] = [];
@@ -2567,6 +2617,7 @@ describe("session uploader snapshot cap", () => {
const sessionId = "session-cap-1234";
sessionUploader.startSession(sessionId, "workspace-cap", root);
for (const path of options.touched ?? []) sessionUploader.recordTrace(sessionId, "file.read", { path });
+ for (const event of options.traceEvents ?? []) sessionUploader.recordTrace(sessionId, event.type, event.data);
sessionUploader.flushTrace(sessionId);
await sessionUploader.stop();
const start = uploads.find((item) => item.snapshot_type === "start")!;
@@ -2607,6 +2658,27 @@ describe("session uploader snapshot cap", () => {
expect(notes[0]?.data?.budget_bytes).toBeLessThan(CAP);
});
+ test("prioritizes paths from real engine tool messages, including nested apply_patch input", async () => {
+ const root = await largeWorkspace();
+ const traceEvents = [{
+ type: "turn.messages",
+ data: {
+ messages: [{
+ parts: [
+ { type: "tool", tool: "edit", state: { input: { path: "file-19.txt" } } },
+ { type: "tool", tool: "apply_patch", state: { input: { patch: "*** Update File: file-17.txt\n@@\n-old\n+new\n" } } },
+ { type: "tool", tool: "read", state: { input: { filePath: "file-18.txt" } } },
+ ],
+ }],
+ },
+ }];
+ const { start, sent } = await capture(root, { snapshotMaxBytes: CAP, traceEvents });
+ for (const path of ["file-17.txt", "file-18.txt", "file-19.txt"]) {
+ expect(start.touched_paths).toContain(path);
+ expect(sent).toContain(path);
+ }
+ });
+
test("keeps the smallest touched files when the touched files alone exceed the cap", async () => {
const root = await largeWorkspace();
const { startBytes, sent, size, start } = await capture(root, { snapshotMaxBytes: CAP, touched: NAMES });
@@ -2724,6 +2796,30 @@ describe("session uploader incremental snapshots", () => {
expect(sessionUploader.cacheStatus()).toEqual({ roots: 0, entries: 0 });
});
+ test.skipIf(process.platform === "win32")("detects a same-size replacement after the old mtime is restored", async () => {
+ const root = await workspace("signature", 1);
+ const target = join(root, "src", "f00.txt");
+ const { uploads, upload } = makeUploads();
+ const sessionUploader = new SessionUploader({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-signature-1234";
+ sessionUploader.startSession(sessionId, "workspace-signature", root);
+ await sessionUploader.idle(sessionId);
+ const before = await stat(target);
+ const original = await readFile(target);
+ const replacement = Buffer.from(original);
+ replacement[0] = replacement[0] === 115 ? 116 : 115;
+ await writeFile(target, replacement);
+ await utimes(target, before.atime, before.mtime);
+ sessionUploader.captureSnapshot(sessionId, "turn_completed");
+ await sessionUploader.idle(sessionId);
+
+ const change = changes(uploads)[0]!;
+ expect(contentPaths(change)).toEqual(["src/f00.txt"]);
+ expect(change.files.find((file) => file.path === "src/f00.txt")?.content).toBe(replacement.toString("utf8"));
+ expect(change.changed_paths).toEqual(["src/f00.txt"]);
+ await sessionUploader.stop();
+ });
+
test("skips a milestone capture outright when nothing is dirty, and rescans after a new directory appears", async () => {
const root = await workspace("dirty", 10);
const { uploads, upload } = makeUploads();
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index 3ee137aa..60010d3f 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -25,6 +25,7 @@ import { writeFileAtomic } from "./atomic-write.js";
const execFileAsync = promisify(execFile);
export const UPLOAD_SCHEMA_VERSION = 2;
+export const TRACE_SCHEMA_VERSION = 3;
// Caps shared with the omnirush.ai session upload endpoint, /omnirush/collect (contract v2); the
// backend enforces identical values, so a change here must land on both sides.
// Every cap is per file or per request: there is no per-session storage cap, so
@@ -128,6 +129,8 @@ const UPLOAD_ATTEMPTS = 3;
const UPLOAD_RETRY_DELAY_MS = 250;
const RETRY_BASE_MS = 5_000;
const RETRY_MAX_MS = 5 * 60_000;
+const TRACE_CAPABILITY_TTL_MS = 5 * 60_000;
+const MAX_RESPONSE_BODY_BYTES = 64 * 1024;
// Spooled entries one drain may see fail before it leaves the rest to the
// next one: enough to get past a stuck entry, few enough not to hammer a link
// that is down for every entry in the spool.
@@ -294,7 +297,34 @@ export type UploadAttachment = {
textTruncated?: boolean;
};
-type ArtifactStat = { size: number; mtimeMs: number };
+type FileSignature = {
+ size: number;
+ mtimeMs: number;
+ ctimeMs: number;
+ ino: number;
+ dev: number;
+};
+
+type ArtifactStat = FileSignature;
+
+function fileSignature(file: { size: number; mtimeMs: number; ctimeMs: number; ino: number; dev: number }): FileSignature {
+ return {
+ size: file.size,
+ mtimeMs: file.mtimeMs,
+ ctimeMs: file.ctimeMs,
+ ino: file.ino,
+ dev: file.dev,
+ };
+}
+
+function sameFileSignature(left: FileSignature | undefined, right: FileSignature): boolean {
+ return Boolean(left
+ && left.size === right.size
+ && left.mtimeMs === right.mtimeMs
+ && left.ctimeMs === right.ctimeMs
+ && left.ino === right.ino
+ && left.dev === right.dev);
+}
type SessionLedgerRecord = {
segment: number;
@@ -319,6 +349,9 @@ type SessionLedger = {
type HashCacheEntry = {
size: number;
mtimeMs: number;
+ ctimeMs: number;
+ ino: number;
+ dev: number;
binary: boolean;
sha256: string;
/** UTF-8 bytes of the redacted content: the manifest size. */
@@ -396,6 +429,10 @@ type SessionState = {
* cap, with their mtime: "skipped" in its "turn.diff".
*/
turnSkipped: Map<string, number>;
+ /** Paths observed as changed during the current turn, even if their mtime was restored. */
+ turnWrittenPaths: Map<string, number>;
+ /** Lowest relevance rank observed from known tool trace events for a path. */
+ relevance: Map<string, number>;
/** The per-root hash cache, shared with every other session open on the same root. */
cache: Map<string, HashCacheEntry>;
listing: { denied: number; truncated: boolean };
@@ -430,6 +467,10 @@ type SessionUploaderOptions = {
fetch?: typeof externalFetch;
/** Sends one envelope; `signal` aborts at the upload deadline (session-upload-budget.ts) or when the spool is cleared. */
upload?: (sessionId: string, compressed: Uint8Array, signal?: AbortSignal) => Promise<Response>;
+ /** Sends an immutable compressed envelope directly from its spool/temp path. */
+ uploadFile?: (sessionId: string, path: string, size: number, signal?: AbortSignal) => Promise<Response>;
+ /** Negotiates the canonical structured trace envelope. Failures fall back to schema v2. */
+ capabilities?: () => Promise<TraceCapabilities>;
/**
* Asks the account layer for a fresh access token once the gateway rejects
* an upload as unauthorized. Resolves with the bearer to send next, or null
@@ -469,9 +510,16 @@ type SessionUploaderOptions = {
onPathTouched?: (sessionId: string, path: string) => void;
};
+export type TraceCapabilities = {
+ schema_versions: number[];
+ canonical_trace: boolean;
+};
+
type TransmitOutcome =
| { ok: true }
- | { ok: false; retryable: boolean; reason: string };
+ | { ok: false; retryable: boolean; reason: string; unsupportedSchema?: boolean };
+
+type CompressedArtifact = { path: string; size: number };
// --- privacy rails -----------------------------------------------------------
// The omnirush.ai backend applies the same denylist and scrubber server-side;
@@ -823,6 +871,37 @@ function sha256Hex(input: string | Buffer): string {
return createHash("sha256").update(input).digest("hex");
}
+function fileReadableBody(path: string, signal?: AbortSignal): ReadableStream<Uint8Array> {
+ const source = createReadStream(path, { highWaterMark: 64 * 1024 });
+ const abort = () => source.destroy(signal?.reason instanceof Error ? signal.reason : undefined);
+ const cleanup = () => signal?.removeEventListener("abort", abort);
+ return new globalThis.ReadableStream<Uint8Array>({
+ start(controller) {
+ if (signal?.aborted) {
+ abort();
+ return;
+ }
+ signal?.addEventListener("abort", abort, { once: true });
+ source.on("data", (chunk: string | Buffer) => {
+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+ controller.enqueue(new Uint8Array(bytes));
+ });
+ source.on("end", () => {
+ cleanup();
+ controller.close();
+ });
+ source.on("error", (error: Error) => {
+ cleanup();
+ controller.error(error);
+ });
+ },
+ cancel(reason) {
+ cleanup();
+ source.destroy(reason instanceof Error ? reason : undefined);
+ },
+ });
+}
+
function pathComponents(path: string): string[] {
return path.replaceAll("\\", "/").split("/").filter(Boolean);
}
@@ -1426,30 +1505,145 @@ const PATH_FIELD_NAMES = new Set([
"file_path",
"filepath",
"filename",
+ "file_name",
"target_path",
+ "targetpath",
"workspace_path",
+ "workspacepath",
+ "old_path",
+ "oldpath",
+ "new_path",
+ "newpath",
]);
-function pathCandidates(value: unknown, key = "", output: string[] = [], depth = 0): string[] {
- if (output.length >= 128 || depth > 6) return output;
- if (typeof value === "string") {
- const candidate = value.trim();
- if (PATH_FIELD_NAMES.has(key.toLowerCase()) && candidate && candidate.length <= 2_048
- && !candidate.includes("\n") && !candidate.includes("\r") && !candidate.includes("://")) {
- output.push(candidate);
+const TOOL_READ_TRACE_TYPES = new Set([
+ "file.read",
+ "file.open",
+ "tool.read",
+ "tool.file_read",
+]);
+const TOOL_WRITE_TRACE_TYPES = new Set([
+ "file.write",
+ "file.create",
+ "file.edit",
+ "file.patch",
+ "file.delete",
+ "file.remove",
+ "tool.write",
+ "tool.create",
+ "tool.edit",
+ "tool.patch",
+ "tool.apply_patch",
+ "tool.delete",
+]);
+
+type TracePathEvidence = { path: string; relevance: number };
+
+function pathValue(value: unknown, key: string): string | null {
+ if (typeof value !== "string" || !PATH_FIELD_NAMES.has(key.toLowerCase())) return null;
+ const candidate = value.trim();
+ if (!candidate || candidate.length > 2_048 || candidate.includes("\n") || candidate.includes("\r") || candidate.includes("://")) return null;
+ return candidate;
+}
+
+const KNOWN_READ_TOOLS = new Set(["cat", "file_read", "open", "read", "read_file", "view"]);
+const KNOWN_WRITE_TOOLS = new Set(["apply_patch", "create", "delete", "edit", "file_write", "move", "patch", "remove", "rename", "rm", "write", "write_file"]);
+
+function pathEvidenceFromContainers(relevance: number, containers: readonly Record<string, unknown>[]): TracePathEvidence[] {
+ const output: TracePathEvidence[] = [];
+ const seen = new Set<string>();
+ for (const container of containers) {
+ for (const [key, value] of Object.entries(container)) {
+ const candidate = pathValue(value, key);
+ if (candidate && !seen.has(candidate)) {
+ seen.add(candidate);
+ output.push({ path: candidate, relevance });
+ }
}
- return output;
}
- if (!value || typeof value !== "object") return output;
- if (Array.isArray(value)) {
- for (const item of value) pathCandidates(item, key, output, depth + 1);
- return output;
+ return output.slice(0, 128);
+}
+
+function patchPaths(value: unknown): string[] {
+ if (typeof value !== "string" || value.length > 256 * 1024) return [];
+ const paths: string[] = [];
+ for (const match of value.matchAll(/^\*\*\* (?:(?:Update|Add|Delete) File|Move to):[ \t]*(.+)$/gm)) {
+ const path = match[1]?.trim();
+ if (path) paths.push(path.split(/\s+->\s+/u)[0] ?? path);
}
- for (const [childKey, childValue] of Object.entries(value)) {
- pathCandidates(childValue, childKey, output, depth + 1);
- if (output.length >= 128) break;
+ return paths;
+}
+
+function patchPathsFromContainers(values: readonly unknown[]): string[] {
+ const paths: string[] = [];
+ for (const value of values) {
+ paths.push(...patchPaths(value));
+ if (!isRecord(value)) continue;
+ for (const key of ["patch", "content", "text", "diff"]) paths.push(...patchPaths(value[key]));
}
- return output;
+ return paths;
+}
+
+function toolName(value: unknown): string | null {
+ if (typeof value !== "string") return null;
+ const normalized = value.trim().toLowerCase().split(".").at(-1) ?? "";
+ return normalized || null;
+}
+
+function messageToolPathEvidence(value: Record<string, unknown>): TracePathEvidence[] {
+ const messages = Array.isArray(value.messages) ? value.messages : [];
+ const output: TracePathEvidence[] = [];
+ for (const message of messages) {
+ if (!isRecord(message)) continue;
+ const parts = Array.isArray(message.parts) ? message.parts : [];
+ for (const part of parts) {
+ if (!isRecord(part) || part.type !== "tool") continue;
+ const name = toolName(part.tool ?? part.name);
+ const relevance = name && KNOWN_WRITE_TOOLS.has(name) ? 0 : name && KNOWN_READ_TOOLS.has(name) ? 1 : null;
+ if (relevance === null) continue;
+ const containers: Record<string, unknown>[] = [];
+ for (const key of ["input", "arguments", "args", "params", "output", "result"]) {
+ const child = part[key];
+ if (isRecord(child)) containers.push(child);
+ }
+ const state = part.state;
+ if (isRecord(state)) {
+ for (const key of ["input", "output", "metadata"]) {
+ const child = state[key];
+ if (isRecord(child)) containers.push(child);
+ if (name === "apply_patch") {
+ for (const path of patchPathsFromContainers([child])) output.push({ path, relevance });
+ }
+ }
+ }
+ if (name === "apply_patch") {
+ for (const key of ["patch", "content", "text"]) {
+ for (const path of patchPathsFromContainers([part[key]])) output.push({ path, relevance });
+ }
+ }
+ output.push(...pathEvidenceFromContainers(relevance, containers));
+ }
+ }
+ return output.slice(0, 128);
+}
+
+/**
+ * Only tool events with a known read/write/delete meaning may attribute a
+ * path. Generic trace payloads are deliberately opaque: a random `path` key
+ * in a model message must never turn into file relevance or a workspace read.
+ */
+function knownToolPathEvidence(type: string, data: unknown): TracePathEvidence[] {
+ const relevance = TOOL_WRITE_TRACE_TYPES.has(type) ? 0 : TOOL_READ_TRACE_TYPES.has(type) ? 1 : null;
+ if (relevance === null || !isRecord(data)) {
+ if (!["turn.messages", "turn.completed", "session.completed", "engine.request"].includes(type) || !isRecord(data)) return [];
+ return messageToolPathEvidence(data);
+ }
+ const containers: Record<string, unknown>[] = [data];
+ for (const key of ["input", "arguments", "args", "params", "output", "result"]) {
+ const child = data[key];
+ if (isRecord(child)) containers.push(child);
+ }
+ return pathEvidenceFromContainers(relevance, containers);
}
/** The rules of a .gitignore file's text, as walkFallback reads them: trimmed lines, no blanks or comments. */
@@ -1683,11 +1877,19 @@ async function readSessionLedger(path: string | null): Promise<SessionLedger> {
}
}
-function boundedTracePayload(
+type BoundedTraceBatch = { content: string; events: unknown[] };
+
+/**
+ * Scrubs once, then partitions a trace in arrival order. A large event is
+ * represented by a small explicit marker; later completion and diff events
+ * still get a batch of their own instead of being lost with a longest-prefix
+ * cut.
+ */
+function boundedTraceBatches(
state: Pick<SessionState, "id" | "workspaceId" | "segment" | "resumed">,
events: TraceEvent[],
maxBytes = MAX_TRACE_BYTES,
-): Buffer {
+): BoundedTraceBatch[] {
// The trace is scrubbed as a JSON value, never as serialised text: a text
// redaction inside an escaped string (`\[email protected]`) once left a
// dangling backslash that made the whole document unparseable. Header and
@@ -1701,57 +1903,72 @@ function boundedTracePayload(
session_resumed: state.resumed,
events,
}).value as Record<string, unknown> & { events: TraceEvent[] };
- const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => Buffer.from(JSON.stringify({
+ const encode = (selected: unknown[], truncated: boolean, dropped: number): BoundedTraceBatch & { byteLength: number } => {
+ const content = JSON.stringify({
...header,
trace_truncated: truncated,
- dropped_event_count: Math.max(0, events.length - includedCount),
+ dropped_event_count: dropped,
events: selected,
- }));
-
- // One encode settles the common case. Only an oversized trace pays for the
- // search, and that search measures each event once: the payload of a prefix
- // is its wrapper plus the events' serialised bytes and the commas between
- // them, so the longest prefix that fits follows from a running total. Every
- // probe used to be a full serialisation of the prefix, which made a flush
- // at the MAX_UPLOAD_TRACE_EVENTS cap peak at many times the payload.
- let payload = encode(scrubbed, false);
- if (payload.byteLength <= maxBytes) return payload;
- const wrapperBytes = (included: number) => Buffer.byteLength(JSON.stringify({
- ...header,
- trace_truncated: true,
- dropped_event_count: Math.max(0, events.length - included),
- events: [],
- }));
- let fits = 0;
- let eventBytes = 0;
+ });
+ return { content, events: selected, byteLength: Buffer.byteLength(content) };
+ };
+ const eventTexts = scrubbed.map((event) => JSON.stringify(event));
+ const wrapperBytes = (truncated: boolean, dropped: number): number => encode([], truncated, dropped).byteLength;
+ const batches: BoundedTraceBatch[] = [];
+ let selected: unknown[] = [];
+ let selectedBytes = 0;
+ let dropped = 0;
+ const flush = (): void => {
+ if (selected.length === 0) return;
+ const payload = encode(selected, dropped > 0, dropped);
+ batches.push({ content: payload.content, events: payload.events });
+ selected = [];
+ selectedBytes = 0;
+ };
for (let index = 0; index < scrubbed.length; index += 1) {
- const next = eventBytes + Buffer.byteLength(JSON.stringify(scrubbed[index])) + (index > 0 ? 1 : 0);
- if (wrapperBytes(index + 1) + next > maxBytes) break;
- eventBytes = next;
- fits = index + 1;
- }
- const selected = scrubbed.slice(0, fits);
- payload = encode(selected, true);
- while (payload.byteLength > maxBytes && selected.length > 0) {
- selected.pop();
- payload = encode(selected, true);
- }
- if (selected.length === 0 && events.length > 0) {
+ const event = scrubbed[index];
+ const text = eventTexts[index] ?? "null";
+ const eventBytes = Buffer.byteLength(text);
+ const commaBytes = selected.length > 0 ? 1 : 0;
+ if (wrapperBytes(dropped > 0, dropped) + selectedBytes + commaBytes + eventBytes <= maxBytes) {
+ selected.push(event);
+ selectedBytes += commaBytes + eventBytes;
+ continue;
+ }
+ flush();
+ if (wrapperBytes(dropped > 0, dropped) + eventBytes <= maxBytes) {
+ selected.push(event);
+ selectedBytes = eventBytes;
+ continue;
+ }
+ dropped += 1;
const marker: TraceEvent = {
at: new Date().toISOString(),
type: "trace.truncated",
- data: { dropped_event_count: events.length },
+ data: { dropped_event_count: 1, reason: "oversize_single_event" },
};
- const marked = encode([marker], true, 0);
- if (marked.byteLength <= maxBytes) return marked;
+ const markerText = JSON.stringify(marker);
+ if (wrapperBytes(true, dropped) + Buffer.byteLength(markerText) <= maxBytes) {
+ if (wrapperBytes(true, dropped) + selectedBytes + (selected.length > 0 ? 1 : 0) + Buffer.byteLength(markerText) > maxBytes) flush();
+ selected.push(marker);
+ selectedBytes += (selected.length > 1 ? 1 : 0) + Buffer.byteLength(markerText);
+ }
}
- return payload;
+ flush();
+ return batches;
}
/** Eligible files plus the number of non-ignored paths the denylist dropped. */
-type WorkspaceListing = { paths: string[]; denied: number };
+type WorkspaceListing = { paths: string[]; denied: number; truncated: boolean };
+const MAX_RAW_LISTING_FILES = MAX_FILES * 4;
-async function walkFallback(root: string, directory = root, rules: string[] = [], listing: WorkspaceListing = { paths: [], denied: 0 }): Promise<WorkspaceListing> {
+async function walkFallback(
+ root: string,
+ directory = root,
+ rules: string[] = [],
+ listing: WorkspaceListing = { paths: [], denied: 0, truncated: false },
+ priorityPaths: ReadonlySet<string> = new Set(),
+): Promise<WorkspaceListing> {
const ignoreRules = [...rules, ...scopedIgnoreRules(portablePath(root, directory), await readGitignoreFile(directory))];
let entries;
try {
@@ -1759,16 +1976,22 @@ async function walkFallback(root: string, directory = root, rules: string[] = []
} catch {
return listing;
}
+ entries.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
- if (listing.paths.length >= MAX_FILES) break;
const fullPath = resolve(directory, entry.name);
const path = portablePath(root, fullPath);
if (!path || ignoredByRules(path, ignoreRules)) continue;
+ const uploadPath = portablePathForUpload(path);
+ const priorityBelow = [...priorityPaths].some((candidate) => candidate === uploadPath || candidate.startsWith(`${uploadPath}/`));
+ if (listing.paths.length >= MAX_RAW_LISTING_FILES && !priorityBelow && !priorityPaths.has(uploadPath)) {
+ listing.truncated = true;
+ continue;
+ }
if (entry.isDirectory()) {
// Only the unconditional rules apply to a directory name: "src/token/"
// is walked so its source files can be kept and scrubbed.
if (hasDeniedComponent(pathComponents(path))) listing.denied += 1;
- else await walkFallback(root, fullPath, ignoreRules, listing);
+ else await walkFallback(root, fullPath, ignoreRules, listing, priorityPaths);
} else if (entry.isFile()) {
if (isUploadPathDenied(path)) listing.denied += 1;
else listing.paths.push(path);
@@ -1777,26 +2000,32 @@ async function walkFallback(root: string, directory = root, rules: string[] = []
return listing;
}
-function filterListing(candidates: string[]): WorkspaceListing {
- const listing: WorkspaceListing = { paths: [], denied: 0 };
+function filterListing(candidates: string[], priorityPaths: ReadonlySet<string> = new Set(), limit = MAX_RAW_LISTING_FILES): WorkspaceListing {
+ const eligible: string[] = [];
+ let denied = 0;
for (const path of candidates) {
if (!path) continue;
- if (isUploadPathDenied(path)) listing.denied += 1;
- else if (listing.paths.length < MAX_FILES) listing.paths.push(path);
+ if (isUploadPathDenied(path)) denied += 1;
+ else eligible.push(path);
}
- return listing;
+ const prioritized = eligible.filter((path) => priorityPaths.has(portablePathForUpload(path)));
+ const ordinary = eligible.filter((path) => !priorityPaths.has(portablePathForUpload(path)));
+ const paths = [...prioritized, ...ordinary.slice(0, Math.max(0, limit - prioritized.length))];
+ return { paths, denied, truncated: eligible.length > paths.length };
}
-async function listWorkspaceFiles(root: string): Promise<WorkspaceListing> {
+async function listWorkspaceFiles(root: string, priorityPaths: ReadonlySet<string> = new Set(), limit = MAX_FILES): Promise<WorkspaceListing> {
try {
const { stdout } = await execFileAsync("git", ["-C", root, "ls-files", "-co", "--exclude-standard", "-z"], {
encoding: "buffer",
maxBuffer: 16 * 1024 * 1024,
timeout: 15_000,
});
- return filterListing(Buffer.from(stdout).toString("utf8").split("\0"));
+ return filterListing(Buffer.from(stdout).toString("utf8").split("\0"), priorityPaths, limit);
} catch {
- return walkFallback(root);
+ const raw = await walkFallback(root, root, [], { paths: [], denied: 0, truncated: false }, priorityPaths);
+ const limited = filterListing(raw.paths, priorityPaths, limit);
+ return { paths: limited.paths, denied: raw.denied, truncated: raw.truncated || limited.truncated };
}
}
@@ -2083,6 +2312,20 @@ export async function collectGitBlock(root: string): Promise<UploadGitBlock | nu
};
}
+/** Cheap pre-scan git evidence used before the listing/file-count cap. */
+async function gitPriorityPaths(root: string): Promise<Set<string>> {
+ if (await gitText(root, ["rev-parse", "--is-inside-work-tree"]) !== "true") return new Set();
+ const prefixResult = await runGit(root, ["rev-parse", "--show-prefix"], { maxBytes: 64 * 1024, timeoutMs: 5_000 });
+ if (!prefixResult.ok || prefixResult.truncated) return new Set();
+ const prefix = prefixResult.stdout.toString("utf8").trim().replaceAll("\\", "/");
+ const status = await runGit(root, ["status", "--porcelain=v1", "-z", "--no-renames", "--untracked-files=normal", "--", "."], {
+ maxBytes: 2 * 1024 * 1024,
+ timeoutMs: 15_000,
+ });
+ if (!status.ok) return new Set();
+ return new Set(parseGitStatus(status.stdout.toString("utf8"), prefix).entries.map((entry) => entry.path));
+}
+
/**
* Files inside the workspace that git does not track but does not ignore
* either: the outputs an agent leaves next to the source. A workspace without
@@ -2109,7 +2352,7 @@ async function artifactStats(root: string): Promise<Map<string, ArtifactStat>> {
const absolute = resolve(root, path);
const file = await lstat(absolute);
if (!file.isFile() || file.isSymbolicLink()) continue;
- stats.set(path, { size: file.size, mtimeMs: file.mtimeMs });
+ stats.set(path, fileSignature(file));
} catch {
// Live workspace: a listed file can vanish before it is inspected.
}
@@ -2292,6 +2535,69 @@ export async function mapBounded<T, R>(items: readonly T[], limit: number, opera
return results;
}
+type AdmissionWaiter = {
+ resolve: (release: () => void) => void;
+ reject: (error: unknown) => void;
+ signal: AbortSignal | undefined;
+ onAbort: (() => void) | undefined;
+};
+
+/** One bounded pool shared by scans and uploads from every active session. */
+class CaptureAdmission {
+ private active = 0;
+ private readonly waiters: AdmissionWaiter[] = [];
+
+ constructor(private readonly limit: number) {}
+
+ acquire(signal?: AbortSignal): Promise<() => void> {
+ if (signal?.aborted) return Promise.reject(signal.reason);
+ if (this.active < this.limit) {
+ this.active += 1;
+ return Promise.resolve(this.releaseOnce());
+ }
+ return new Promise((resolvePromise, reject) => {
+ const waiter: AdmissionWaiter = {
+ resolve: (release) => resolvePromise(release),
+ reject,
+ signal,
+ onAbort: undefined,
+ };
+ waiter.onAbort = () => {
+ const index = this.waiters.indexOf(waiter);
+ if (index !== -1) this.waiters.splice(index, 1);
+ reject(signal?.reason);
+ };
+ signal?.addEventListener("abort", waiter.onAbort, { once: true });
+ this.waiters.push(waiter);
+ });
+ }
+
+ private releaseOnce(): () => void {
+ let released = false;
+ return () => {
+ if (released) return;
+ released = true;
+ this.active -= 1;
+ this.pump();
+ };
+ }
+
+ private pump(): void {
+ while (this.active < this.limit && this.waiters.length > 0) {
+ const waiter = this.waiters.shift()!;
+ if (waiter.signal?.aborted) {
+ waiter.reject(waiter.signal.reason);
+ continue;
+ }
+ this.active += 1;
+ if (waiter.onAbort && waiter.signal) waiter.signal.removeEventListener("abort", waiter.onAbort);
+ waiter.resolve(this.releaseOnce());
+ }
+ }
+}
+
+const GLOBAL_CAPTURE_ADMISSION = new CaptureAdmission(4);
+
/** Work counters for tests and profiling; never uploaded. */
export type UploadMetrics = {
fileStats: number;
@@ -2326,8 +2632,7 @@ function freshMetrics(): UploadMetrics {
function redactedCacheEntry(
path: string,
buffer: Buffer,
- size: number,
- mtimeMs: number,
+ signature: FileSignature,
uploadPath: string,
metrics: UploadMetrics,
texts?: { cache: RedactedTextCache; absolute: string },
@@ -2335,8 +2640,7 @@ function redactedCacheEntry(
const redacted = redactUploadContentCounted(path, buffer.toString("utf8"));
metrics.fileRedactions += 1;
const entry: HashCacheEntry = {
- size,
- mtimeMs,
+ ...signature,
binary: false,
sha256: sha256Hex(redacted.text),
bytes: Buffer.byteLength(redacted.text),
@@ -2398,6 +2702,24 @@ type ScanContext = {
ancestors: AncestorCache;
};
+/**
+ * Concurrent sessions still stat independently, but identical path signatures
+ * share the expensive read, scrub and digest pass. Entries leave this map as
+ * soon as the pass settles, so it is not a stale cache or TTL.
+ */
+const FILE_SCAN_IN_FLIGHT = new Map<string, Promise<HashCacheEntry | null>>();
+
+/** Bound metadata retained by long sessions that create and delete many paths. */
+class FileHashCache extends Map<string, HashCacheEntry> {
+ override set(path: string, entry: HashCacheEntry): this {
+ if (!this.has(path) && this.size >= MAX_FILES) {
+ const oldest = this.keys().next().value;
+ if (oldest !== undefined) this.delete(oldest);
+ }
+ return super.set(path, entry);
+ }
+}
+
function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: RedactedTextCache, metrics: UploadMetrics): ScanContext {
const base = resolve(root);
return {
@@ -2414,7 +2736,7 @@ function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: Re
/**
* Stats one eligible path and, unless the per-root cache already knows the
- * file at this size and mtime, reads, redacts and hashes it. Null for anything
+ * file at this robust signature, reads, redacts and hashes it. Null for anything
* that is not a regular file within the size cap (gone, a directory, a
* symlink, too large) or that resolves outside the root, lexically or through
* a symlinked directory; a binary file comes back as its binary entry. Content
@@ -2433,23 +2755,41 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCach
metrics.fileStats += 1;
if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) return null;
const cached = cache.get(path);
- if (cached && cached.size === file.size && cached.mtimeMs === file.mtimeMs) return cached;
+ const signature = fileSignature(file);
+ if (cached && sameFileSignature(cached, signature)) return cached;
+ const scanKey = [
+ resolve(context.root),
+ path,
+ signature.size,
+ signature.mtimeMs,
+ signature.ctimeMs,
+ signature.ino,
+ signature.dev,
+ ].join("\0");
+ const existing = FILE_SCAN_IN_FLIGHT.get(scanKey);
+ if (existing) {
+ const entry = await existing;
+ if (entry) cache.set(path, entry);
+ await context.yielder.pause();
+ return entry;
+ }
const uploadPath = uploadPathFor(path, cached);
- let entry: HashCacheEntry | null = null;
+ const inFlight = pool.read(absolute, file.size, (buffer) => {
+ metrics.fileReads += 1;
+ if (isBinary(buffer)) {
+ return { ...signature, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };
+ }
+ return redactedCacheEntry(path, buffer, signature, uploadPath, metrics, { cache: context.texts, absolute }).entry;
+ }).catch(() => null);
+ FILE_SCAN_IN_FLIGHT.set(scanKey, inFlight);
try {
- entry = await pool.read(absolute, file.size, (buffer) => {
- metrics.fileReads += 1;
- if (isBinary(buffer)) {
- return { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };
- }
- return redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPath, metrics, { cache: context.texts, absolute }).entry;
- });
- } catch {
- return null;
+ const entry = await inFlight;
+ if (entry) cache.set(path, entry);
+ await context.yielder.pause();
+ return entry;
+ } finally {
+ if (FILE_SCAN_IN_FLIGHT.get(scanKey) === inFlight) FILE_SCAN_IN_FLIGHT.delete(scanKey);
}
- if (entry) cache.set(path, entry);
- await context.yielder.pause();
- return entry;
}
/** One file as files[] carries it, re-read at upload time. */
@@ -2482,11 +2822,12 @@ async function readUploadContent(
return null;
}
if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) return null;
+ const signature = fileSignature(file);
const cached = cache.get(path);
try {
return await pool.read(absolute, file.size, (buffer): UploadContent | null => {
metrics.fileReads += 1;
- if (cached && !cached.binary && cached.size === file.size && cached.mtimeMs === file.mtimeMs) {
+ if (cached && !cached.binary && sameFileSignature(cached, signature)) {
if (cached.clean) {
if (cached.bytes === buffer.length && sha256Hex(buffer) === cached.sha256) return { content: buffer.toString("utf8"), entry: cached };
} else if (texts.size > 0) {
@@ -2498,9 +2839,9 @@ async function readUploadContent(
}
}
if (isBinary(buffer)) return null;
- const { entry, text } = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, cached), metrics, { cache: texts, absolute });
+ const { entry, text } = redactedCacheEntry(path, buffer, signature, uploadPathFor(path, cached), metrics, { cache: texts, absolute });
// The same digest keeps the cached entry, which the manifest may share.
- if (cached && cached.sha256 === entry.sha256 && cached.size === entry.size && cached.mtimeMs === entry.mtimeMs) {
+ if (cached && cached.sha256 === entry.sha256 && sameFileSignature(cached, signature)) {
return { content: text, entry: cached };
}
cache.set(path, entry);
@@ -2537,8 +2878,8 @@ type SnapshotCandidate = {
bytes: number;
/** Serialised files[] cost: overhead, path and JSON-encoded content. */
cost: number;
- /** Touched this session, or named by git status or the diff. */
- priority: boolean;
+ /** Lower ranks are more relevant and survive caps first. */
+ relevance: number;
};
/**
@@ -2567,18 +2908,23 @@ function writtenSince(mtimeMs: number, since: number, now = Date.now()): boolean
* absence (a file the turn created), or null when every one saw it only once
* written, so the edit stays in the turn.
*/
-function turnBaseline(manifest: Manifest, since: number | null, earlier: ReadonlyArray<TurnBaseline | null>): TurnBaseline {
+function turnBaseline(
+ manifest: Manifest,
+ since: number | null,
+ earlier: ReadonlyArray<TurnBaseline | null>,
+ forcePaths: ReadonlySet<string> = new Set(),
+): TurnBaseline {
if (since === null) return manifest;
const now = Date.now();
let baseline: Map<string, HashCacheEntry | null> | null = null;
for (const [path, entry] of manifest) {
- if (!writtenSince(entry.mtimeMs, since, now)) continue;
+ if (!forcePaths.has(path) && !writtenSince(entry.mtimeMs, since, now) && !writtenSince(entry.ctimeMs, since, now)) continue;
baseline ??= new Map(manifest);
let known: HashCacheEntry | null | undefined = null;
for (const source of earlier) {
if (!source) continue;
const seen = source.get(path);
- if (seen === null || (seen && writtenSince(seen.mtimeMs, since, now))) continue;
+ if (seen === null || (seen && (writtenSince(seen.mtimeMs, since, now) || writtenSince(seen.ctimeMs, since, now)))) continue;
known = seen;
break;
}
@@ -2593,9 +2939,10 @@ function turnBaseline(manifest: Manifest, since: number | null, earlier: Readonl
* traced read or an attachment journals a path too): written since the
* turn's prompt, or, before any prompt, moved since the cache last saw it.
*/
-function writtenThisTurn(turnStartedAt: number | null, cached: HashCacheEntry | undefined, file: { size: number; mtimeMs: number }): boolean {
+function writtenThisTurn(turnStartedAt: number | null, cached: HashCacheEntry | undefined, file: FileSignature): boolean {
+ if (cached && !sameFileSignature(cached, file)) return true;
if (turnStartedAt !== null) return writtenSince(file.mtimeMs, turnStartedAt);
- return cached !== undefined && (cached.size !== file.size || cached.mtimeMs !== file.mtimeMs);
+ return cached !== undefined && !sameFileSignature(cached, file);
}
/** Whether the first 8 KiB of a file read as binary (false when it cannot be read). */
@@ -2618,11 +2965,30 @@ function candidateCost(entry: HashCacheEntry): number {
}
/** The changed entries of a manifest as candidates, in manifest order. */
-function candidatesFor(scan: ScanResult, priorityPaths: ReadonlySet<string>): SnapshotCandidate[] {
+const GENERATED_PATH_PATTERN = /(^|\/)(?:dist|build|out|coverage|\.next|\.cache|target)(?:\/|$)|(?:\.min|\.map)$/i;
+const ASSOCIATED_PATH_PATTERN = /(^|\/)(?:test|tests|__tests__|spec|specs)(?:\/|$)|\.(?:test|spec)\.[^.]+$|(^|\/)(?:package(?:-lock)?\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|tsconfig(?:\.[^/]+)?\.json|vitest\.config\.[^.]+|jest\.config\.[^.]+|webpack\.config\.[^.]+|vite\.config\.[^.]+|manifest\.[^.]+)$/i;
+
+function relevanceForPath(path: string, toolRelevance: ReadonlyMap<string, number>, gitPaths: ReadonlySet<string>): number {
+ const portable = portablePathForUpload(path);
+ const tool = toolRelevance.get(path) ?? toolRelevance.get(portable);
+ if (tool !== undefined) return tool;
+ if (ASSOCIATED_PATH_PATTERN.test(portable)) return 2;
+ if (gitPaths.has(portable)) return 3;
+ if (GENERATED_PATH_PATTERN.test(portable)) return 5;
+ return 4;
+}
+
+function candidatesFor(scan: ScanResult, toolRelevance: ReadonlyMap<string, number>, gitPaths: ReadonlySet<string>): SnapshotCandidate[] {
const candidates: SnapshotCandidate[] = [];
for (const [path, entry] of scan.manifest) {
if (scan.changed && !scan.changed.has(path)) continue;
- candidates.push({ path, uploadPath: entry.uploadPath, bytes: entry.bytes, cost: candidateCost(entry), priority: priorityPaths.has(entry.uploadPath) });
+ candidates.push({
+ path,
+ uploadPath: entry.uploadPath,
+ bytes: entry.bytes,
+ cost: candidateCost(entry),
+ relevance: relevanceForPath(path, toolRelevance, gitPaths),
+ });
}
return candidates;
}
@@ -2637,10 +3003,12 @@ function countRemoved(previous: Manifest | null, manifest: Manifest): number {
/**
* Lists and inspects every eligible file. Unchanged files are recognised by
- * size + mtime through the per-root cache, so a rescan of a settled tree costs
- * a stat per file and no reads. The cache is rebuilt from what the listing
- * still names, so entries for removed paths go with it. `onListing` sees the
- * listing before any file is read.
+ * robust file signatures through the per-root cache, so a rescan of a settled tree costs
+ * a stat per file and no reads. Only paths the current listing names are
+ * selected. The metadata cache is bounded to MAX_FILES and shared by
+ * overlapping sessions; entries are retained because every reuse
+ * checks the complete stat signature, including ctime/inode/device. `onListing`
+ * sees the listing before any file is read.
*/
async function scanWorkspaceFull(
root: string,
@@ -2649,14 +3017,15 @@ async function scanWorkspaceFull(
previous: Manifest | null,
includeAll: boolean,
metrics: UploadMetrics,
+ priorityPaths: ReadonlySet<string> = new Set(),
onListing?: (paths: readonly string[]) => void,
+ listFiles: () => Promise<WorkspaceListing> = () => listWorkspaceFiles(root, priorityPaths),
): Promise<ScanResult> {
metrics.fullScans += 1;
- const listing = await listWorkspaceFiles(root);
+ const listing = await listFiles();
onListing?.(listing.paths);
const context = scanContext(root, cache, texts, metrics);
const entries = await mapBounded(listing.paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));
- cache.clear();
const manifest: Manifest = new Map();
const changed = includeAll ? null : new Set<string>();
listing.paths.forEach((path, index) => {
@@ -2672,7 +3041,7 @@ async function scanWorkspaceFull(
changed,
removed: countRemoved(previous, manifest),
deniedCount: listing.denied,
- manifestTruncated: listing.paths.length >= MAX_FILES,
+ manifestTruncated: listing.truncated || listing.paths.length >= MAX_FILES,
};
}
@@ -2745,8 +3114,9 @@ async function scanDirtyPaths(
}
/**
- * Paths whose content a capped snapshot keeps first: what the session touched
- * plus what git reports as modified, keyed the way files[] names them.
+ * Paths that were touched by the session or named by git, keyed the way
+ * files[] names them. The weighted candidate ranking adds associated and
+ * generated-file rules after this early evidence is collected.
*/
function snapshotPriorityPaths(touched: Iterable<string>, git: UploadGitBlock | null): Set<string> {
const paths = new Set<string>();
@@ -2765,8 +3135,9 @@ function snapshotPriorityPaths(touched: Iterable<string>, git: UploadGitBlock |
* escaping cannot push the envelope past what the raw byte count promised.
*/
function selectSnapshotContent(candidates: SnapshotCandidate[], budget: number): { selected: Set<string>; omittedCount: number; omittedBytes: number } {
- // A stable sort: equal candidates keep manifest order.
- const ordered = [...candidates].sort((a, b) => Number(b.priority) - Number(a.priority) || a.bytes - b.bytes);
+ // Relevance wins before size. Path order makes equal inputs reproducible
+ // across sessions whose filesystem listing order differs.
+ const ordered = [...candidates].sort((a, b) => a.relevance - b.relevance || a.bytes - b.bytes || a.path.localeCompare(b.path));
const selected = new Set<string>();
let used = 0;
let omittedCount = 0;
@@ -2792,14 +3163,50 @@ function serializedBytes(value: unknown): number {
* gateway's `omnirush_account_required`) rather than a stale device token.
* Reading the body also releases the connection.
*/
-async function responseSignalsAccountRequired(response: Response): Promise<boolean> {
+async function boundedResponseText(response: Response, maxBytes = MAX_RESPONSE_BODY_BYTES): Promise<string> {
try {
- return (await response.text()).includes(ACCOUNT_REQUIRED_MARKER);
+ if (!response.body) return (await response.text()).slice(0, maxBytes);
+ const reader = response.body.getReader();
+ const chunks: Uint8Array[] = [];
+ let bytes = 0;
+ try {
+ while (bytes < maxBytes) {
+ const next = await reader.read();
+ if (next.done) break;
+ const chunk = next.value;
+ const take = Math.min(chunk.byteLength, maxBytes - bytes);
+ if (take > 0) {
+ chunks.push(chunk.subarray(0, take));
+ bytes += take;
+ }
+ if (take < chunk.byteLength) break;
+ }
+ } finally {
+ await reader.cancel().catch(() => undefined);
+ }
+ const result = new Uint8Array(bytes);
+ let offset = 0;
+ for (const chunk of chunks) {
+ result.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return new TextDecoder().decode(result);
} catch {
- return false;
+ return "";
}
}
+async function responseSignalsAccountRequired(response: Response): Promise<boolean> {
+ return (await boundedResponseText(response)).includes(ACCOUNT_REQUIRED_MARKER);
+}
+
+async function responseSignalsUnsupportedSchema(response: Response): Promise<boolean> {
+ const body = (await boundedResponseText(response)).toLowerCase();
+ return body.includes("unsupported_schema")
+ || body.includes("unsupported schema")
+ || (body.includes("schema") && (body.includes("unsupported") || body.includes("not supported")));
+}
+
/**
* Top-level directories holding at least one listed file: the ones worth a
* recursive watcher. The listing is git's (tracked plus untracked-but-not-
@@ -2909,7 +3316,8 @@ type EnvelopeBody = {
manifest: () => Iterable<HashCacheEntry>;
/** Computed once files[] is written, so cap omissions are known. */
privacy: () => Record<string, unknown>;
- trace?: unknown[];
+ trace?: readonly unknown[];
+ schemaVersion?: 2 | 3;
};
function uploadEnvironment(appVersion: string | undefined, engineVersion: string | undefined, gitVersion: string | null): UploadEnvironment {
@@ -3038,13 +3446,22 @@ export class SessionUploader {
* Per workspace root, shared by every session open on it: the hash cache,
* and when the reconcile pass last ran (one pass serves every session).
*/
- private readonly caches = new Map<string, { refs: number; entries: Map<string, HashCacheEntry>; reconciling: boolean; reconciledAt: number }>();
+ private readonly caches = new Map<string, {
+ refs: number;
+ entries: Map<string, HashCacheEntry>;
+ listingInFlight: Map<string, Promise<WorkspaceListing>>;
+ reconcileInFlight: Promise<void> | null;
+ }>();
/** Redacted text of the files the scrubber changed, shared by every root. */
private readonly texts: RedactedTextCache;
/** The scrubbed texts sent, by redacted digest: the bases of "turn.diff" events. */
private readonly bases: TurnBaseStore;
private readonly onSessionClosed?: (sessionId: string) => void;
private readonly onPathTouched?: (sessionId: string, path: string) => void;
+ private readonly fileUploader?: SessionUploaderOptions["uploadFile"];
+ private readonly capabilityProbe?: SessionUploaderOptions["capabilities"];
+ private traceSchemaCache: { version: 2 | 3; expiresAt: number } | null = null;
+ private traceSchemaProbe: Promise<2 | 3> | null = null;
/** Work counters for tests and profiling. */
readonly metrics: UploadMetrics = freshMetrics();
@@ -3053,6 +3470,8 @@ export class SessionUploader {
this.token = (options.accessToken ?? process.env.OMNIRUSH_ACCESS_TOKEN ?? "").trim();
this.fetcher = options.fetch ?? externalFetch;
this.uploader = options.upload;
+ this.fileUploader = options.uploadFile;
+ this.capabilityProbe = options.capabilities;
this.refreshAccessToken = options.refreshAccessToken;
this.log = options.log ?? (() => undefined);
const stateDir = options.stateDir ? resolve(options.stateDir) : null;
@@ -3217,7 +3636,7 @@ export class SessionUploader {
}
get enabled(): boolean {
- return Boolean(this.uploader || (this.uploadUrl && this.token));
+ return Boolean(this.uploader || this.fileUploader || (this.uploadUrl && this.token));
}
private environment(): Promise<UploadEnvironment> {
@@ -3234,6 +3653,37 @@ export class SessionUploader {
return this.environmentCache;
}
+ private negotiatedTraceSchema(force = false): Promise<2 | 3> {
+ if (!force && this.traceSchemaCache && this.traceSchemaCache.expiresAt > Date.now()) {
+ return Promise.resolve(this.traceSchemaCache.version);
+ }
+ if (this.traceSchemaProbe) return this.traceSchemaProbe;
+ this.traceSchemaProbe = (async () => {
+ let version: 2 | 3 = 2;
+ if (this.capabilityProbe) {
+ try {
+ const capabilities = await this.capabilityProbe();
+ version = capabilities.canonical_trace
+ && Array.isArray(capabilities.schema_versions)
+ && capabilities.schema_versions.includes(TRACE_SCHEMA_VERSION)
+ ? 3
+ : 2;
+ } catch {
+ version = 2;
+ }
+ }
+ this.traceSchemaCache = { version, expiresAt: Date.now() + TRACE_CAPABILITY_TTL_MS };
+ return version;
+ })().finally(() => {
+ this.traceSchemaProbe = null;
+ });
+ return this.traceSchemaProbe;
+ }
+
+ private downgradeTraceSchema(): void {
+ this.traceSchemaCache = { version: 2, expiresAt: Date.now() + TRACE_CAPABILITY_TTL_MS };
+ }
+
startSession(sessionId: string, workspaceId: string, root: string): void {
if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;
const state: SessionState = {
@@ -3270,6 +3720,8 @@ export class SessionUploader {
turnInProgress: false,
changesHeld: false,
turnSkipped: new Map(),
+ turnWrittenPaths: new Map(),
+ relevance: new Map(),
cache: this.acquireCache(root),
listing: { denied: 0, truncated: false },
dirty: new Set(),
@@ -3454,6 +3906,8 @@ export class SessionUploader {
/** Tells the project archive that the session touched `path` (workspace-relative). */
private reportTouched(state: SessionState, path: string): void {
+ const current = state.relevance.get(path);
+ if (current === undefined || current > 4) state.relevance.set(path, 4);
try {
this.onPathTouched?.(state.id, path);
} catch {
@@ -3496,16 +3950,18 @@ export class SessionUploader {
private async reconcile(state: SessionState): Promise<void> {
if (state.finished || state.watchMode !== "watching" || !state.manifest) return;
const shared = this.caches.get(resolve(state.root));
- // Another session's timer already ran the pass for this interval.
- if (!shared || shared.entries !== state.cache || shared.reconciling || Date.now() - shared.reconciledAt < this.fallbackScanMs * 0.9) return;
+ if (!shared || shared.entries !== state.cache) return;
+ if (shared.reconcileInFlight) {
+ await shared.reconcileInFlight;
+ return;
+ }
const sessions = [...this.sessions.values()]
.filter((other) => other.cache === state.cache && !other.finished && other.watchMode === "watching" && other.manifest !== null);
- shared.reconciling = true;
- shared.reconciledAt = Date.now();
- this.metrics.reconciles += 1;
- try {
+ const releaseAdmission = await GLOBAL_CAPTURE_ADMISSION.acquire(this.account.signal);
+ const pass = (async () => {
+ this.metrics.reconciles += 1;
const cache = state.cache;
- const listing = await listWorkspaceFiles(state.root);
+ const listing = await this.listWorkspaceFilesShared(state.root, new Set());
const yielder = new LoopYielder();
await mapBounded(listing.paths, SCAN_CONCURRENCY, async (path) => {
const file = await lstat(resolve(state.root, path)).catch(() => null);
@@ -3514,9 +3970,12 @@ export class SessionUploader {
const cached = cache.get(path);
for (const session of sessions) {
const known = session.manifest?.get(path);
+ const signature = eligible ? fileSignature(eligible) : null;
const stale = eligible === null
? known !== undefined
- : !cached || cached.size !== eligible.size || cached.mtimeMs !== eligible.mtimeMs
+ : signature === null
+ ? true
+ : cached === undefined || !sameFileSignature(cached, signature)
|| (!cached.binary && known?.sha256 !== cached.sha256) || (cached.binary && known !== undefined);
if (stale) {
this.markDirty(session, path);
@@ -3528,14 +3987,16 @@ export class SessionUploader {
const listed = new Set(listing.paths);
for (const session of sessions) {
for (const path of session.manifest?.keys() ?? []) {
- if (listed.has(path)) continue;
+ if (listed.has(path)) continue;
this.markDirty(session, path);
this.reportTouched(session, path);
}
}
- } finally {
- shared.reconciling = false;
- }
+ })().finally(() => releaseAdmission());
+ shared.reconcileInFlight = pass;
+ await pass.finally(() => {
+ if (shared.reconcileInFlight === pass) shared.reconcileInFlight = null;
+ });
for (const session of sessions) {
if (!session.finished && (session.dirty.size > 0 || session.dirtyOverflow)) this.scheduleChange(session, "periodic");
}
@@ -3624,13 +4085,34 @@ export class SessionUploader {
const key = resolve(root);
let shared = this.caches.get(key);
if (!shared) {
- shared = { refs: 0, entries: new Map(), reconciling: false, reconciledAt: 0 };
+ shared = { refs: 0, entries: new FileHashCache(), listingInFlight: new Map(), reconcileInFlight: null };
this.caches.set(key, shared);
}
shared.refs += 1;
return shared.entries;
}
+ private listingKey(priorityPaths: ReadonlySet<string>): string {
+ return [...priorityPaths].map(portablePathForUpload).sort().join("\0");
+ }
+
+ private async listWorkspaceFilesShared(root: string, priorityPaths: ReadonlySet<string>): Promise<WorkspaceListing> {
+ const shared = this.caches.get(resolve(root));
+ if (!shared) return listWorkspaceFiles(root, priorityPaths);
+ const key = this.listingKey(priorityPaths);
+ let inFlight = shared.listingInFlight.get(key);
+ if (!inFlight) {
+ inFlight = listWorkspaceFiles(root, priorityPaths, MAX_RAW_LISTING_FILES);
+ shared.listingInFlight.set(key, inFlight);
+ void inFlight.finally(() => {
+ if (shared.listingInFlight.get(key) === inFlight) shared.listingInFlight.delete(key);
+ }).catch(() => undefined);
+ }
+ const raw = await inFlight;
+ const limited = filterListing(raw.paths, priorityPaths, MAX_FILES);
+ return { paths: limited.paths, denied: raw.denied, truncated: raw.truncated || limited.truncated };
+ }
+
private releaseCache(state: SessionState): void {
const key = resolve(state.root);
const shared = this.caches.get(key);
@@ -3676,7 +4158,7 @@ export class SessionUploader {
recordTrace(sessionId: string, type: string, data?: unknown): void {
const state = this.sessions.get(sessionId);
if (!state || state.finished) return;
- for (const candidate of pathCandidates(data)) this.recordTouchedPath(state, candidate);
+ for (const evidence of knownToolPathEvidence(type, data)) this.recordTouchedPath(state, evidence.path, evidence.relevance);
if (type === SUBAGENT_MODEL_FALLBACK_TRACE) {
const entry = subagentModelFallback(data);
if (entry) {
@@ -3697,10 +4179,13 @@ export class SessionUploader {
if (state.trace.length > MAX_UPLOAD_TRACE_EVENTS) state.trace.splice(0, state.trace.length - MAX_UPLOAD_TRACE_EVENTS);
}
- private recordTouchedPath(state: SessionState, candidate: string): void {
+ private recordTouchedPath(state: SessionState, candidate: string, relevance = 4): void {
const path = workspaceRelativePath(state.root, candidate.replaceAll("\\", "/"));
if (!path) return;
this.reportTouched(state, path);
+ const current = state.relevance.get(path);
+ if (current === undefined || relevance < current) state.relevance.set(path, relevance);
+ if (relevance === 0 && state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());
if (isUploadPathDenied(path) || state.touchedPaths.has(path)) return;
state.touchedPaths.add(path);
this.markDirty(state, path);
@@ -3764,7 +4249,6 @@ export class SessionUploader {
state.childSessionIds.push(child.childSessionId);
}
if (child.lastMessageId) state.childCheckpoints.set(child.childSessionId, child.lastMessageId);
- for (const candidate of pathCandidates(child.messages)) this.recordTouchedPath(state, candidate);
this.appendTrace(state, "session.child", {
child_session_id: child.childSessionId,
parent_session_id: child.parentSessionId,
@@ -3850,11 +4334,13 @@ export class SessionUploader {
let emitted = 0;
for (const [path, stat] of current) {
const previous = baseline.get(path);
- if (previous && previous.size === stat.size && previous.mtimeMs === stat.mtimeMs) continue;
+ if (previous && sameFileSignature(previous, stat)) continue;
// The turn's change snapshot must carry it even where no watcher saw it
// land (a tool writing into a directory beyond MAX_WATCH_ROOTS).
this.markDirty(state, path);
this.reportTouched(state, path);
+ state.relevance.set(path, 0);
+ if (state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());
if (emitted >= MAX_ARTIFACT_EVENTS_PER_TURN || stat.size > MAX_ARTIFACT_HASH_BYTES) continue;
try {
const sha256 = await sha256File(resolve(state.root, path));
@@ -3876,6 +4362,10 @@ export class SessionUploader {
captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void {
const state = this.sessions.get(sessionId);
if (!state || state.finished) return;
+ if (trigger === "prompt") {
+ state.relevance.clear();
+ state.turnWrittenPaths.clear();
+ }
// The artifact baseline must reflect the workspace as the turn begins, not
// once the queued start snapshot has finished uploading.
if (trigger === "prompt") state.artifactBaseline = artifactStats(state.root);
@@ -3907,7 +4397,10 @@ export class SessionUploader {
return this.recordTurnDiff(state, new Map([...taken].map(([path, entry]) => [path, entry ?? current.get(path)!])), next);
}
// The turn is measured from the workspace as its prompt found it.
- state.turnManifest = current && turnBaseline(current, startedAt, [previous, state.turnManifest]);
+ const forcePaths = new Set(
+ [...state.turnWrittenPaths].filter(([, at]) => startedAt === null || at >= startedAt).map(([path]) => path),
+ );
+ state.turnManifest = current && turnBaseline(current, startedAt, [previous, state.turnManifest], forcePaths);
for (const [path, at] of state.turnSkipped) if (startedAt !== null && at < startedAt) state.turnSkipped.delete(path);
});
}
@@ -3927,6 +4420,7 @@ export class SessionUploader {
const skipped = state.turnSkipped;
state.turnManifest = after;
state.turnSkipped = new Map(next === null ? [] : [...skipped].filter(([, at]) => at >= next));
+ state.turnWrittenPaths = new Map(next === null ? [] : [...state.turnWrittenPaths].filter(([, at]) => at >= next));
if (state.finished || !before || !after) return;
const changes: TurnChange[] = [];
if (before !== after) {
@@ -4003,7 +4497,9 @@ export class SessionUploader {
if (!state || state.finished) return;
const pendingTrace = state.trace.splice(0);
if (finalTrace !== undefined) {
- for (const candidate of pathCandidates(finalTrace)) this.recordTouchedPath(state, candidate);
+ for (const evidence of knownToolPathEvidence("session.completed", finalTrace)) {
+ this.recordTouchedPath(state, evidence.path, evidence.relevance);
+ }
}
state.finished = true;
if (state.changeTimer) clearTimeout(state.changeTimer);
@@ -4035,7 +4531,9 @@ export class SessionUploader {
if (!state || state.finished) return;
const pendingTrace = state.trace.splice(0);
if (finalTrace !== undefined) {
- for (const candidate of pathCandidates(finalTrace)) this.recordTouchedPath(state, candidate);
+ for (const evidence of knownToolPathEvidence("turn.completed", finalTrace)) {
+ this.recordTouchedPath(state, evidence.path, evidence.relevance);
+ }
pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });
}
if (pendingTrace.length === 0) return;
@@ -4261,6 +4759,7 @@ export class SessionUploader {
try {
const file = await lstat(absolute);
const written = writtenThisTurn(state.turnStartedAt, state.cache.get(path), file);
+ if (written && state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());
if (file.isFile() && !file.isSymbolicLink() && file.size > MAX_CHANGE_JOURNAL_BYTES && written) state.turnSkipped.set(path, file.mtimeMs);
if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) {
entry = { path, at: new Date().toISOString(), status: "skipped" };
@@ -4273,12 +4772,15 @@ export class SessionUploader {
const buffer = await readFile(absolute);
this.metrics.fileReads += 1;
if (buffer.length > MAX_UPLOAD_FILE_BYTES || isBinary(buffer)) {
- if (written) state.turnSkipped.set(path, file.mtimeMs);
+ if (written) {
+ state.turnWrittenPaths.set(path, Date.now());
+ state.turnSkipped.set(path, file.mtimeMs);
+ }
entry = { path, at: new Date().toISOString(), status: "skipped" };
} else {
// The journal's read doubles as the snapshot's: the cache learns the
// file now, so the next capture only stats it.
- const redacted = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, state.cache.get(path)), this.metrics, { cache: this.texts, absolute });
+ const redacted = redactedCacheEntry(path, buffer, fileSignature(file), uploadPathFor(path, state.cache.get(path)), this.metrics, { cache: this.texts, absolute });
state.cache.set(path, redacted.entry);
entry = { path, at: new Date().toISOString(), status: "present", content: redacted.text, sha256: redacted.entry.sha256 };
}
@@ -4356,14 +4858,36 @@ export class SessionUploader {
this.acknowledgeJournal(state, dropped);
for (const entry of dropped) cache.delete(entry.path);
}
- const scan = targeted && previous !== null
- ? await scanDirtyPaths(state.root, cache, this.texts, previous, reported, ignored, state.listing, this.metrics)
- : await scanWorkspaceFull(state.root, cache, this.texts, previous, type === "start", this.metrics,
- type === "start" ? (paths) => this.installWatchers(state, paths) : undefined);
+ // Collect the cheap relevance evidence before the listing cap. A late
+ // alphabetical path named by a tool or git status must not disappear
+ // merely because the workspace has many files.
+ const preGitPaths = targeted ? new Set<string>() : await gitPriorityPaths(state.root);
+ const preScanPaths = new Set<string>([
+ ...state.relevance.keys(),
+ ...state.touchedPaths,
+ ...preGitPaths,
+ ].map(portablePathForUpload));
+ const releaseScan = await GLOBAL_CAPTURE_ADMISSION.acquire(this.account.signal);
+ let scan: ScanResult;
+ try {
+ scan = targeted && previous !== null
+ ? await scanDirtyPaths(state.root, cache, this.texts, previous, reported, ignored, state.listing, this.metrics)
+ : await scanWorkspaceFull(state.root, cache, this.texts, previous, type === "start", this.metrics,
+ preScanPaths,
+ type === "start" ? (paths) => this.installWatchers(state, paths) : undefined,
+ () => this.listWorkspaceFilesShared(state.root, preScanPaths));
+ } finally {
+ releaseScan();
+ }
// What moved on disk since the last snapshot, where no watcher may have
// seen it (a polled workspace). Only against an accepted snapshot: with
// none yet (the start one was refused), every file counts as changed.
- if (previous !== null) for (const path of scan.changed ?? []) this.reportTouched(state, path);
+ if (previous !== null) {
+ for (const path of scan.changed ?? []) {
+ this.reportTouched(state, path);
+ if (state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());
+ }
+ }
// A change capture that found nothing moved stops at one `git rev-parse`
// (a commit changes history without touching a file); the full git block
// with its status, log and diff is read only for a snapshot that goes out.
@@ -4374,7 +4898,11 @@ export class SessionUploader {
}
// Its status and diff name the files a capped snapshot must keep.
const git = await collectGitBlock(state.root);
- const candidates = candidatesFor(scan, snapshotPriorityPaths(state.touchedPaths, git));
+ const gitPaths = new Set<string>([
+ ...preGitPaths,
+ ...snapshotPriorityPaths([], git),
+ ]);
+ const candidates = candidatesFor(scan, state.relevance, gitPaths);
const environment = await this.environment();
const rootName = workspaceRootName(state.root);
const touchedPaths = this.touchedPathsForUpload(state);
@@ -4499,29 +5027,35 @@ export class SessionUploader {
private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {
if (traceEvents.length === 0) return;
- const bounded = boundedTracePayload(state, traceEvents, MAX_TRACE_BYTES);
- const content = bounded.toString("utf8");
- let events: unknown[] = [];
- try {
- const parsed = JSON.parse(content) as { events?: unknown };
- if (Array.isArray(parsed.events)) events = parsed.events;
- } catch {
- events = [];
+ let schemaVersion = await this.negotiatedTraceSchema();
+ let downgraded = false;
+ const batches = boundedTraceBatches(state, traceEvents, MAX_TRACE_BYTES);
+ for (const batch of batches) {
+ const makeBody = async (version: 2 | 3): Promise<EnvelopeBody> => ({
+ schemaVersion: version,
+ extras: {
+ workspace: { root_name: workspaceRootName(state.root), git: null },
+ environment: await this.environment(),
+ touched_paths: this.touchedPathsForUpload(state),
+ files_scope: "full",
+ },
+ writeFiles: version === TRACE_SCHEMA_VERSION
+ ? async () => undefined
+ : async (emit) => {
+ await emit({ path: "__omnirush__/trace.json", content: batch.content, sha256: sha256Hex(batch.content) });
+ },
+ manifest: () => [],
+ privacy: () => ({ ...PRIVACY_POLICY }),
+ trace: batch.events,
+ });
+ let uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));
+ if (uploaded === "unsupported_schema" && schemaVersion === TRACE_SCHEMA_VERSION && !downgraded) {
+ downgraded = true;
+ schemaVersion = 2;
+ this.downgradeTraceSchema();
+ uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));
+ }
}
- await this.uploadEnvelope(state, "trace", "trace_flush", {
- extras: {
- workspace: { root_name: workspaceRootName(state.root), git: null },
- environment: await this.environment(),
- touched_paths: this.touchedPathsForUpload(state),
- files_scope: "full",
- },
- writeFiles: async (emit) => {
- await emit({ path: "__omnirush__/trace.json", content, sha256: sha256Hex(content) });
- },
- manifest: () => [],
- privacy: () => ({ ...PRIVACY_POLICY }),
- trace: events,
- });
}
/**
@@ -4530,9 +5064,47 @@ export class SessionUploader {
* sending the body and the gateway's answer share one deadline) or sooner
* by `cancel`.
*/
- private send(sessionId: string, compressed: Uint8Array, deadline: AbortSignal, cancel?: AbortSignal): Promise<Response> {
+ private uploadPathOwned(path: string): boolean {
+ const absolute = resolve(path);
+ return [this.tempDir, this.spoolDir]
+ .filter((directory): directory is string => directory !== null)
+ .some((directory) => {
+ const relativePath = relative(resolve(directory), absolute);
+ return relativePath !== "" && relativePath !== ".." && !relativePath.startsWith(`..${sep}`) && !relativePath.includes(sep + sep);
+ });
+ }
+
+ private async validateCompressedArtifact(compressed: CompressedArtifact): Promise<void> {
+ if (!this.uploadPathOwned(compressed.path)) throw new Error("session upload artifact is outside its owned directory");
+ const file = await lstat(compressed.path);
+ if (!file.isFile() || file.isSymbolicLink() || file.size !== compressed.size) {
+ throw new Error("session upload artifact changed or is not a regular file");
+ }
+ }
+
+ private async send(sessionId: string, compressed: Uint8Array | CompressedArtifact, deadline: AbortSignal, cancel?: AbortSignal): Promise<Response> {
const signal = cancel ? AbortSignal.any([deadline, cancel]) : deadline;
if (signal.aborted) return Promise.reject(signal.reason);
+ if ("path" in compressed) {
+ await this.validateCompressedArtifact(compressed);
+ if (this.fileUploader) return untilAborted(this.fileUploader(sessionId, compressed.path, compressed.size, signal), signal);
+ if (!this.uploader) {
+ const body = fileReadableBody(compressed.path, signal);
+ const init: RequestInit & { duplex: "half" } = {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${this.token}`,
+ "Content-Type": "application/zstd",
+ "X-OmniRush-Session-ID": sessionId,
+ },
+ body,
+ duplex: "half",
+ signal,
+ };
+ return this.fetcher(this.uploadUrl!, init);
+ }
+ return untilAborted(readFile(compressed.path).then((bytes) => this.uploader!(sessionId, bytes, signal)), signal);
+ }
if (this.uploader) return untilAborted(this.uploader(sessionId, compressed, signal), signal);
return this.fetcher(this.uploadUrl!, {
method: "POST",
@@ -4566,8 +5138,14 @@ export class SessionUploader {
* could hold the session's queue for many minutes, so the envelope goes to
* the spool at once and the spool drain retries it on its own backoff.
*/
- private async transmit(sessionId: string, compressed: Uint8Array, attempts = UPLOAD_ATTEMPTS, cancel?: AbortSignal): Promise<TransmitOutcome> {
- const timeoutMs = sessionUploadTimeoutMs(compressed.byteLength, this.uploadBudget);
+ private async transmit(
+ sessionId: string,
+ compressed: Uint8Array | CompressedArtifact,
+ attempts = UPLOAD_ATTEMPTS,
+ cancel?: AbortSignal,
+ schemaVersion: 2 | 3 = 2,
+ ): Promise<TransmitOutcome> {
+ const timeoutMs = sessionUploadTimeoutMs("path" in compressed ? compressed.size : compressed.byteLength, this.uploadBudget);
let lastReason = "session upload unavailable";
let refreshAttempted = false;
let attempt = 0;
@@ -4599,6 +5177,9 @@ export class SessionUploader {
// The retry with the fresh token is not one of the transport attempts.
continue;
}
+ if (response.status === 422 && schemaVersion === TRACE_SCHEMA_VERSION && await responseSignalsUnsupportedSchema(response)) {
+ return { ok: false, retryable: false, reason: lastReason, unsupportedSchema: true };
+ }
await response.body?.cancel().catch(() => undefined);
if (!RETRYABLE_STATUSES.has(response.status)) return { ok: false, retryable: false, reason: lastReason };
} catch (error) {
@@ -4650,8 +5231,13 @@ export class SessionUploader {
* retryable failure moves the file into the spool instead. Resolves with
* whether the envelope was accepted (delivered or spooled).
*/
- private async uploadEnvelope(state: SessionState, snapshotType: SnapshotType, trigger: UploadTrigger, body: EnvelopeBody): Promise<boolean> {
- if (!this.uploadUrl && !this.uploader) return false;
+ private async uploadEnvelope(
+ state: SessionState,
+ snapshotType: SnapshotType,
+ trigger: UploadTrigger,
+ body: EnvelopeBody,
+ ): Promise<boolean | "unsupported_schema"> {
+ if (!this.uploadUrl && !this.uploader && !this.fileUploader) return false;
const sequence = state.sequence + 1;
const path = await this.envelopeTempPath();
const writer = new EnvelopeWriter(path);
@@ -4659,7 +5245,7 @@ export class SessionUploader {
let compressedBytes: number;
try {
const head = JSON.stringify({
- schema_version: UPLOAD_SCHEMA_VERSION,
+ schema_version: body.schemaVersion ?? UPLOAD_SCHEMA_VERSION,
session_id: state.id,
session_segment: state.segment,
session_resumed: state.resumed,
@@ -4682,7 +5268,13 @@ export class SessionUploader {
first = false;
}
await writer.write(`],"privacy":${JSON.stringify(body.privacy())}`);
- if (body.trace) await writer.write(`,"trace":${JSON.stringify(body.trace)}`);
+ if (body.trace) {
+ await writer.write(',"trace":[');
+ for (let index = 0; index < body.trace.length; index += 1) {
+ await writer.write(`${index > 0 ? "," : ""}${JSON.stringify(body.trace[index]) ?? "null"}`);
+ }
+ await writer.write("]");
+ }
await writer.write("}");
compressedBytes = await writer.finish();
} catch (error) {
@@ -4710,14 +5302,10 @@ export class SessionUploader {
this.log("warn", "OmniRush session upload payload exceeded compressed limit", { sessionId: state.id, snapshotType, trigger });
return false;
}
- const compressed = await readFile(path);
+ const compressed: CompressedArtifact = { path, size: compressedBytes };
const account = this.account.signal;
- const outcome = await this.transmit(state.id, compressed, UPLOAD_ATTEMPTS, AbortSignal.any([account, this.halted.signal]));
- if (!outcome.ok) {
- if ((!outcome.retryable || !this.spoolDir) && !account.aborted) {
- await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);
- throw new Error(outcome.reason);
- }
+ const uploadSignal = AbortSignal.any([account, this.halted.signal]);
+ const spoolArtifact = async (reason: string): Promise<boolean> => {
const spooled = await this.spoolEnvelopeFile({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressedBytes, created_at: new Date().toISOString(), attempts: 1 }, path, account);
if (!spooled) {
// Signed out during the upload: the envelope is deleted, never queued for the next account.
@@ -4735,11 +5323,32 @@ export class SessionUploader {
trigger,
sequence,
compressedBytes,
- reason: outcome.reason,
+ reason,
});
this.retryFailures += 1;
this.scheduleRetry();
return true;
+ };
+ let releaseUpload: (() => void) | null = null;
+ try {
+ releaseUpload = await GLOBAL_CAPTURE_ADMISSION.acquire(uploadSignal);
+ } catch (error) {
+ if (!this.halted.signal.aborted || account.aborted) throw error;
+ return await spoolArtifact("capture shutdown");
+ }
+ let outcome: TransmitOutcome;
+ try {
+ outcome = await this.transmit(state.id, compressed, UPLOAD_ATTEMPTS, uploadSignal, body.schemaVersion ?? UPLOAD_SCHEMA_VERSION);
+ } finally {
+ releaseUpload();
+ }
+ if (!outcome.ok && outcome.unsupportedSchema) return "unsupported_schema";
+ if (!outcome.ok) {
+ if ((!outcome.retryable || !this.spoolDir) && !account.aborted) {
+ await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);
+ throw new Error(outcome.reason);
+ }
+ return await spoolArtifact(outcome.reason);
}
state.sentBytes += bytes;
state.sequence = sequence;
@@ -4929,15 +5538,24 @@ export class SessionUploader {
nextDueAt = Math.min(nextDueAt, dueAt);
continue;
}
- let compressed: Buffer;
+ let compressed: CompressedArtifact;
try {
- compressed = await readFile(join(spoolDir, `${entry.id}.zst`));
+ const spoolPath = join(spoolDir, `${entry.id}.zst`);
+ const file = await stat(spoolPath);
+ if (!file.isFile() || file.size !== entry.bytes) throw new Error("spool artifact size changed");
+ compressed = { path: spoolPath, size: file.size };
} catch {
await this.spoolLocked(() => this.removeSpoolEntry(entry.id));
pending -= 1;
continue;
}
- const outcome = await this.transmit(entry.session_id, compressed, 1, signal);
+ const releaseUpload = await GLOBAL_CAPTURE_ADMISSION.acquire(signal);
+ let outcome: TransmitOutcome;
+ try {
+ outcome = await this.transmit(entry.session_id, compressed, 1, signal);
+ } finally {
+ releaseUpload();
+ }
if (outcome.ok) {
await this.spoolLocked(() => this.removeSpoolEntry(entry.id));
await this.recordLedgerOutcome(entry.session_id, "success").catch(() => undefined);
@@ -5005,6 +5623,8 @@ export class SessionUploader {
this.draining?.controller.abort();
this.account.abort();
this.account = new AbortController();
+ this.traceSchemaCache = null;
+ this.traceSchemaProbe = null;
this.retryFailures = 0;
return this.spoolLocked(async () => {
// The turn diffs' bases are workspace content on disk too.
diff --git a/apps/server/src/types.ts b/apps/server/src/types.ts
index 9488011b..33e2bd80 100644
--- a/apps/server/src/types.ts
+++ b/apps/server/src/types.ts
@@ -83,6 +83,14 @@ export interface ApprovalConfig {
export type LocalManagedMcpVaultKeyProvider = () => Promise<Uint8Array>;
+export type CaptureFileUpload = (url: string, init: {
+ method: "POST";
+ headers: Record<string, string>;
+ path: string;
+ size: number;
+ signal?: AbortSignal;
+}) => Promise<Response>;
+
export type OmniRushGatewayCredentialBundle = {
gatewayUrl: string;
accessToken: string;
@@ -141,6 +149,8 @@ export interface ServerConfig {
resumeInterruptedTasks?: boolean;
/** Version of the embedding desktop app (Electron app.getVersion()), reported in session uploader envelopes. */
appVersion?: string;
+ /** Embedding-owned transport for immutable session upload files. */
+ captureFileUpload?: CaptureFileUpload;
}
export interface Capabilities {
diff --git a/docs/capture-efficiency-staging.md b/docs/capture-efficiency-staging.md
new file mode 100644
index 00000000..d83232cb
--- /dev/null
+++ b/docs/capture-efficiency-staging.md
@@ -0,0 +1,105 @@
+# Capture efficiency: staging results
+
+Tested on the staging server with synthetic workspaces and isolated capture
+services. This change is proposed for review; production is not deployed.
+
+## Changes
+
+- Rank files by current tool writes, reads, associated tests/configuration,
+ git changes, ordinary sources and generated output before the content cap.
+- Reuse overlapping file reads, scrubbing and hashes with robust
+ size/mtime/ctime/inode/device signatures. Shared metadata has a bounded
+ cache and expires with the workspace's last session.
+- Negotiate canonical schema-3 traces and preserve schema-2 downgrade for
+ older collectors. Keep ordered records and completed turns.
+- Pass immutable compressed file descriptors through the capture worker and
+ gateway broker, removing retained IPC byte copies.
+- Stream Node/Bun requests; use one bounded body through Electron's existing
+ native fetch, preserving its OS certificate verification and proxy behavior.
+- Bound archive pending bytes and free disk headroom. Paused base/delta work
+ resumes without losing sequence, parent links or binary content.
+
+## Measurements
+
+Eight sessions on the same 1,000-file workspace:
+
+| Product work counter | Main | Candidate |
+| --- | ---: | ---: |
+| GUI file reads | 15,134 | 9,000 |
+| GUI scrub/hash passes | 7,134 | 1,000 |
+| CLI file reads | 15,176 | 9,000 |
+| CLI scrub/hash passes | 7,176 | 1,000 |
+| Logical scans / file stats in each version | 8 / 8,000 | 8 / 8,000 |
+| Accepted start uploads in each version | 8 | 8 |
+
+This fixture performs about 40.5% fewer file reads and 86% fewer scrub/hash
+passes. It shares expensive work while maintaining each session's manifest.
+The benchmark is work-count evidence; no general latency improvement is
+claimed.
+
+A 32 MiB desktop transport fixture, limited to 6 MiB/second by the receiver,
+used Electron 43.2.0 and its Node 24.18.0:
+
+| Whole Electron/Chromium/Xvfb process tree | Main | Candidate |
+| --- | ---: | ---: |
+| Peak RSS bytes | 722,776,064 | 662,482,944 |
+| Upload milliseconds | 5,596 | 5,550 |
+| Received bytes | 33,554,432 | 33,554,432 |
+
+Peak memory is about 8.3% lower in this transport fixture. Both receivers
+verified the identical SHA-256. The baseline driver models retained old
+worker/broker buffers and calls the old transport; the candidate uses the
+actual new file adapter. This is not a full desktop memory measurement.
+Scripts, fixture generation and method are in
+[scripts/capture-efficiency-bench](../scripts/capture-efficiency-bench/README.md).
+
+Native staging checks passed for 401 then authenticated full-file retry,
+redirect refusal, truncated response rejection, the 1 MiB response cap and
+cancellation. The initially attempted chunked Electron net.request path
+crashed inside the pinned runtime on interruption; the final implementation
+uses the established net.fetch path. Desktop request bodies are buffered,
+bounded at 64 MiB, and governed by collector concurrency.
+
+## Data preservation and disk recovery
+
+Actual staging API checks in the
+[backend companion PR](https://github.com/omnirush-ai/omnirush-backend/pull/90)
+preserve all six completed viewer turns, twelve ordered export records,
+exact files/manifests, same-ID duplicate retry and owner isolation. Its
+repeated-file fixture uses 66.0% less local storage with opt-in compaction and
+11.7% less compressed trace wire. These are local fixture savings; S3 keeps
+full envelopes.
+
+Archive recovery tests on staging passed four cases: base pause/restart,
+delta pause/restart with correct parent, disk exhaustion during packing, and
+concurrent packing under one pending budget. Restored binary bytes match.
+
+Defaults are a 256 MiB pending archive budget and 512 MiB free disk floor,
+configurable with OMNIRUSH_ARCHIVE_PENDING_BYTE_BUDGET and
+OMNIRUSH_ARCHIVE_MIN_FREE_DISK_BYTES. Reservation estimates are conservative;
+large bases can pause until the budget or available space increases.
+Encrypted archive and restore formats are preserved.
+
+## Verification
+
+The PR's published capture-efficiency HTTP journey is the runtime proof for
+canonical traces, compatibility fallback, record ordering and separate
+session/workspace data. Its original assertion records are bound to the
+final PR commit.
+
+Targeted staging checks passed for real nested tool-path evidence, same-size
+restored-mtime replacement, explicit schema downgrade, shutdown/spool retry,
+the existing delayed-turn diff regression, archive recovery and native
+transport failure recovery. CLI capture integration selection: 37 passed.
+
+The final six-suite staging selection recorded 253 passed, 1 optional-fixture
+skip and 3 failures. Clean main at
+d83e0566785e1b8e1dabba1da8530359fce258b1 reproduced the same unified-diff
+timing failure and two archive folder-policy failures. The introduced
+shutdown and delayed-turn regressions found during review were fixed and
+passed on staging. The 60 MiB collector memory check also passed in the
+final selection and its standalone control. Native adapter tests: 10 passed;
+boundary guard tests: 11 passed; server typecheck: passed.
+Recorded-provider replay requires an optional fixture directory and remains
+a skip, not a pass. The staging host is Linux; Windows/macOS desktop
+certificates and installer behavior require platform verification.
diff --git a/evals/packages/labs/src/capture-efficiency.ts b/evals/packages/labs/src/capture-efficiency.ts
new file mode 100644
index 00000000..795d76aa
--- /dev/null
+++ b/evals/packages/labs/src/capture-efficiency.ts
@@ -0,0 +1,388 @@
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { createReadStream } from "node:fs";
+import { Readable } from "node:stream";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { zstdDecompressSync } from "node:zlib";
+
+import { startCaptureService, type CaptureService } from "../../../../apps/server/src/capture-client.js";
+import type { TraceCapabilities } from "../../../../apps/server/src/session-uploader.js";
+
+const CONTROL_TOKEN = "capture-lab-control";
+const UPLOAD_TOKEN = "capture-lab-upload";
+const MAX_REQUEST_BYTES = 80 * 1024 * 1024;
+type Actor = "actor-a" | "actor-b";
+const ACTORS: readonly Actor[] = ["actor-a", "actor-b"];
+
+export type CaptureEfficiencyAttempt = {
+ sessionId: string;
+ status: number;
+ snapshotType: string;
+ schemaVersion: number | null;
+ workspaceRoot: string | null;
+ filePaths: string[];
+ traceTypes: string[];
+ traceFileEventTypes: string[];
+ traceFileCount: number;
+ actorMarkers: string[];
+};
+
+export type CaptureEfficiencyWitness = {
+ attempts: CaptureEfficiencyAttempt[];
+ capabilityCalls: Record<Actor, number>;
+ modes: Array<"starting" | "worker" | "local" | "down">;
+};
+
+export type CaptureEfficiencyLab = {
+ readonly baseUrl: string;
+ readonly controlToken: string;
+ witness(): Promise<CaptureEfficiencyWitness>;
+ stop(): Promise<void>;
+};
+
+type JsonRecord = Record<string, unknown>;
+
+function isRecord(value: unknown): value is JsonRecord {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stringValue(value: unknown, key: string): string {
+ if (!isRecord(value) || typeof value[key] !== "string" || value[key].length === 0) {
+ throw new Error(`missing string field ${key}`);
+ }
+ return value[key];
+}
+
+function optionalString(value: unknown): string | null {
+ return typeof value === "string" ? value : null;
+}
+
+function numberValue(value: unknown, key: string): number | null {
+ if (!isRecord(value) || typeof value[key] !== "number" || !Number.isFinite(value[key])) return null;
+ return value[key];
+}
+
+function actorValue(value: unknown): Actor {
+ const actor = stringValue(value, "actor");
+ if (actor === "actor-a" || actor === "actor-b") return actor;
+ throw new Error(`unsupported synthetic actor ${actor}`);
+}
+
+function sendJson(response: ServerResponse, status: number, body: unknown): void {
+ response.statusCode = status;
+ response.setHeader("content-type", "application/json");
+ response.end(JSON.stringify(body));
+}
+
+async function readBody(request: IncomingMessage): Promise<Buffer> {
+ const chunks: Buffer[] = [];
+ let length = 0;
+ for await (const chunk of request) {
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+ length += buffer.byteLength;
+ if (length > MAX_REQUEST_BYTES) throw new Error("request body exceeds lab limit");
+ chunks.push(buffer);
+ }
+ return Buffer.concat(chunks);
+}
+
+async function readJson(request: IncomingMessage): Promise<unknown> {
+ const body = await readBody(request);
+ return JSON.parse(body.toString("utf8"));
+}
+
+function authorized(request: IncomingMessage, token: string): boolean {
+ return request.headers.authorization === `Bearer ${token}`;
+}
+
+function traceSummary(value: unknown): { types: string[]; actors: string[] } {
+ if (isRecord(value) && Array.isArray(value.events)) return traceSummary(value.events);
+ if (!Array.isArray(value)) return { types: [], actors: [] };
+ const types: string[] = [];
+ const actors: string[] = [];
+ for (const event of value) {
+ if (!isRecord(event) || typeof event.type !== "string") continue;
+ types.push(event.type);
+ if (isRecord(event.data) && typeof event.data.actor === "string") actors.push(event.data.actor);
+ }
+ return { types, actors };
+}
+
+function filePaths(value: unknown): Array<{ path: string; content: string | null }> {
+ if (!Array.isArray(value)) return [];
+ const files: Array<{ path: string; content: string | null }> = [];
+ for (const file of value) {
+ if (!isRecord(file) || typeof file.path !== "string") continue;
+ files.push({ path: file.path, content: typeof file.content === "string" ? file.content : null });
+ }
+ return files;
+}
+
+function summarizeEnvelope(sessionId: string, status: number, envelope: JsonRecord): CaptureEfficiencyAttempt {
+ const workspace = isRecord(envelope.workspace) ? envelope.workspace : null;
+ const workspaceRoot = workspace ? optionalString(workspace.root_name) : null;
+ const files = filePaths(envelope.files);
+ const trace = traceSummary(envelope.trace);
+ const traceFiles = files.filter((file) => file.path === "__omnirush__/trace.json");
+ let traceFileEventTypes: string[] = [];
+ for (const traceFile of traceFiles) {
+ if (traceFile.content === null) continue;
+ try {
+ const parsed: unknown = JSON.parse(traceFile.content);
+ traceFileEventTypes = traceSummary(parsed).types;
+ } catch {
+ traceFileEventTypes = [];
+ }
+ }
+ return {
+ sessionId,
+ status,
+ snapshotType: typeof envelope.snapshot_type === "string" ? envelope.snapshot_type : "",
+ schemaVersion: numberValue(envelope, "schema_version"),
+ workspaceRoot,
+ filePaths: files.map((file) => file.path),
+ traceTypes: trace.types,
+ traceFileEventTypes,
+ traceFileCount: traceFiles.length,
+ actorMarkers: trace.actors,
+ };
+}
+
+function listen(server: Server): Promise<number> {
+ return new Promise((resolve, reject) => {
+ server.once("error", reject);
+ server.listen(0, "127.0.0.1", () => {
+ const address = server.address();
+ if (!address || typeof address === "string") {
+ reject(new Error("capture-efficiency lab did not bind a TCP port"));
+ return;
+ }
+ resolve(address.port);
+ });
+ });
+}
+
+function close(server: Server): Promise<void> {
+ return new Promise((resolve) => {
+ server.close(() => resolve());
+ });
+}
+
+function capability(actor: Actor, calls: Record<Actor, number>): () => Promise<TraceCapabilities> {
+ return async () => {
+ calls[actor] += 1;
+ return { schema_versions: [1, 2, 3], canonical_trace: true };
+ };
+}
+
+export async function startCaptureEfficiencyLab(): Promise<CaptureEfficiencyLab> {
+ const tempRoot = await mkdtemp(join(tmpdir(), "omnirush-capture-efficiency-"));
+ const stateRoot = join(tempRoot, "state");
+ const workspaceRoot = join(tempRoot, "workspaces");
+ await mkdir(workspaceRoot, { recursive: true });
+
+ const attempts: CaptureEfficiencyAttempt[] = [];
+ const capabilityCalls: Record<Actor, number> = { "actor-a": 0, "actor-b": 0 };
+ const sessions = new Map<string, { actor: Actor; service: CaptureService }>();
+ const rejectedSchema3 = new Set(["session-a-0001"]);
+ let collectorUrl = "";
+ let stopped = false;
+ let services: CaptureService[] = [];
+
+ const server = createServer((request, response) => {
+ void (async () => {
+ const path = request.url?.split("?", 1)[0] ?? "/";
+ if (path === "/collector/collect") {
+ if (request.method !== "POST" || !authorized(request, UPLOAD_TOKEN)) {
+ sendJson(response, 401, { error: "unauthorized" });
+ return;
+ }
+ const header = request.headers["x-omnirush-session-id"];
+ const sessionId = typeof header === "string" ? header : Array.isArray(header) ? header[0] ?? "" : "";
+ try {
+ const compressed = await readBody(request);
+ const json: unknown = JSON.parse(zstdDecompressSync(compressed).toString("utf8"));
+ if (!isRecord(json)) throw new Error("collector envelope is not an object");
+ const schemaVersion = numberValue(json, "schema_version");
+ const summary = summarizeEnvelope(sessionId, 201, json);
+ if (summary.snapshotType === "trace" && schemaVersion === 3 && rejectedSchema3.has(sessionId)) {
+ rejectedSchema3.delete(sessionId);
+ attempts.push({ ...summary, status: 422 });
+ sendJson(response, 422, { error: "unsupported_schema" });
+ return;
+ }
+ attempts.push(summary);
+ sendJson(response, 201, { ok: true });
+ } catch (error) {
+ sendJson(response, 400, { error: error instanceof Error ? error.message : "invalid collector body" });
+ }
+ return;
+ }
+
+ if (path === "/witness") {
+ if (request.method !== "GET" || !authorized(request, CONTROL_TOKEN)) {
+ sendJson(response, 401, { error: "unauthorized" });
+ return;
+ }
+ sendJson(response, 200, { attempts, capabilityCalls, modes: services.map(service => service.mode()) });
+ return;
+ }
+
+ if (!path.startsWith("/control/") || !authorized(request, CONTROL_TOKEN)) {
+ sendJson(response, 401, { error: "unauthorized" });
+ return;
+ }
+ if (request.method !== "POST") {
+ sendJson(response, 405, { error: "method_not_allowed" });
+ return;
+ }
+ if (services.length === 0) {
+ sendJson(response, 503, { error: "capture service is not ready" });
+ return;
+ }
+
+ const body = await readJson(request);
+ const requestedSessionId = stringValue(body, "sessionId");
+ if (path === "/control/start") {
+ const actor = actorValue(body);
+ const workspace = stringValue(body, "workspace");
+ const expectedWorkspace = actor === "actor-a" ? "workspace-a" : "workspace-b";
+ if (workspace !== expectedWorkspace || sessions.has(requestedSessionId)) {
+ sendJson(response, 400, { error: "invalid synthetic session" });
+ return;
+ }
+ const service = services[ACTORS.indexOf(actor)];
+ const root = join(workspaceRoot, actor, workspace);
+ await mkdir(root, { recursive: true });
+ await writeFile(join(root, `notes-${actor.slice(-1)}.txt`), `synthetic ${actor}\n`, "utf8");
+ sessions.set(requestedSessionId, { actor, service });
+ service.startSession(requestedSessionId, workspace, root);
+ await service.idle();
+ sendJson(response, 202, { ok: true });
+ return;
+ }
+
+ const session = sessions.get(requestedSessionId);
+ if (!session) {
+ sendJson(response, 404, { error: "unknown synthetic session" });
+ return;
+ }
+ if (path === "/control/trace") {
+ const type = stringValue(body, "type");
+ const data = isRecord(body) && Object.hasOwn(body, "data") ? body.data : undefined;
+ session.service.recordTrace(requestedSessionId, type, data);
+ await session.service.idle();
+ sendJson(response, 202, { ok: true });
+ return;
+ }
+ if (path === "/control/flush") {
+ const finalTrace = isRecord(body) && Object.hasOwn(body, "finalTrace") ? body.finalTrace : undefined;
+ session.service.flushTrace(requestedSessionId, finalTrace);
+ await session.service.idle();
+ sendJson(response, 202, { ok: true });
+ return;
+ }
+ if (path === "/control/complete") {
+ const finalTrace = isRecord(body) && Object.hasOwn(body, "finalTrace") ? body.finalTrace : undefined;
+ session.service.finishSession(requestedSessionId, finalTrace);
+ await session.service.idle();
+ sendJson(response, 202, { ok: true });
+ return;
+ }
+ sendJson(response, 404, { error: "unknown_control" });
+ })().catch((error: unknown) => {
+ if (!response.writableEnded) sendJson(response, 500, { error: error instanceof Error ? error.message : "lab request failed" });
+ });
+ });
+
+ const port = await listen(server);
+ collectorUrl = `http://127.0.0.1:${port}`;
+ const uploadFile = async (sessionId: string, path: string, size: number, signal?: AbortSignal): Promise<Response> => {
+ const stream = createReadStream(path, { highWaterMark: 64 * 1024 });
+ const body = Readable.toWeb(stream);
+ try {
+ const options = {
+ method: "POST",
+ headers: {
+ authorization: "Bearer " + UPLOAD_TOKEN,
+ "content-type": "application/zstd",
+ "x-omnirush-session-id": sessionId,
+ },
+ body,
+ signal,
+ duplex: "half",
+ };
+ return await fetch(collectorUrl + "/collector/collect", options);
+ } finally { stream.destroy(); }
+ };
+
+ services = ACTORS.map((actor) => startCaptureService({
+ stateDir: join(stateRoot, actor),
+ appVersion: "capture-efficiency-lab",
+ engineVersion: "capture-efficiency-lab",
+ log: () => undefined,
+ worker: true,
+ sessionUploader: {
+ uploadFile,
+ capabilities: capability(actor, capabilityCalls),
+ },
+ archive: {
+ enabled: false,
+ excludedDirs: [],
+ },
+ }));
+
+ const stop = async (): Promise<void> => {
+ if (stopped) return;
+ stopped = true;
+ await Promise.all(services.map((service) => service.stop({ archiveFinals: false })));
+ await close(server);
+ await rm(tempRoot, { recursive: true, force: true });
+ };
+
+ return {
+ baseUrl: collectorUrl,
+ controlToken: CONTROL_TOKEN,
+ witness: async () => {
+ const response = await fetch(`${collectorUrl}/witness`, {
+ headers: { authorization: `Bearer ${CONTROL_TOKEN}` },
+ });
+ if (!response.ok) throw new Error(`witness request failed with ${response.status}`);
+ const body: unknown = await response.json();
+ if (!isRecord(body) || !Array.isArray(body.attempts) || !isRecord(body.capabilityCalls)) {
+ throw new Error("invalid capture-efficiency witness");
+ }
+ const witnessAttempts: CaptureEfficiencyAttempt[] = [];
+ for (const attempt of body.attempts) {
+ if (!isRecord(attempt)) throw new Error("invalid capture-efficiency attempt");
+ witnessAttempts.push({
+ sessionId: stringValue(attempt, "sessionId"),
+ status: numberValue(attempt, "status") ?? 0,
+ snapshotType: stringValue(attempt, "snapshotType"),
+ schemaVersion: numberValue(attempt, "schemaVersion"),
+ workspaceRoot: optionalString(attempt.workspaceRoot),
+ filePaths: Array.isArray(attempt.filePaths) ? attempt.filePaths.filter((path): path is string => typeof path === "string") : [],
+ traceTypes: Array.isArray(attempt.traceTypes) ? attempt.traceTypes.filter((type): type is string => typeof type === "string") : [],
+ traceFileEventTypes: Array.isArray(attempt.traceFileEventTypes)
+ ? attempt.traceFileEventTypes.filter((type): type is string => typeof type === "string")
+ : [],
+ traceFileCount: numberValue(attempt, "traceFileCount") ?? 0,
+ actorMarkers: Array.isArray(attempt.actorMarkers) ? attempt.actorMarkers.filter((actor): actor is string => typeof actor === "string") : [],
+ });
+ }
+ const calls: Record<Actor, number> = {
+ "actor-a": numberValue(body.capabilityCalls, "actor-a") ?? 0,
+ "actor-b": numberValue(body.capabilityCalls, "actor-b") ?? 0,
+ };
+ const modes: CaptureEfficiencyWitness["modes"] = [];
+ if (!Array.isArray(body.modes)) throw new Error("missing worker modes");
+ for (const mode of body.modes) {
+ if (mode !== "starting" && mode !== "worker" && mode !== "local" && mode !== "down") throw new Error("invalid worker mode");
+ modes.push(mode);
+ }
+ return { attempts: witnessAttempts, capabilityCalls: calls, modes };
+ },
+ stop,
+ };
+}
diff --git a/evals/packages/labs/src/index.ts b/evals/packages/labs/src/index.ts
index f9826bbe..64e81ee5 100644
--- a/evals/packages/labs/src/index.ts
+++ b/evals/packages/labs/src/index.ts
@@ -10,3 +10,4 @@ export * from "./release-feed.ts";
export * from "./mock-planetscale.ts";
export * from "./mock-atlassian.ts";
export * from "./mock-inference.ts";
+export * from "./capture-efficiency.ts";
diff --git a/evals/scripts/spec-boundary-ratchet.mjs b/evals/scripts/spec-boundary-ratchet.mjs
index 0214cc9f..50d59c07 100644
--- a/evals/scripts/spec-boundary-ratchet.mjs
+++ b/evals/scripts/spec-boundary-ratchet.mjs
@@ -22,7 +22,7 @@ export function classifySpec(source) {
importsNodeFs: imports.some((specifier) => /^(?:node:)?fs(?:\/promises)?$/.test(specifier)),
importsChildProcess: imports.some((specifier) => /^(?:node:)?child_process$/.test(specifier)),
crossesBoundary: imports.some((specifier) => /^(?:\.\.\/)?\.\.\/worlds\/|^@omnirush\/world$/.test(specifier))
- || /(?<!\.)\b(?:app|chrome|server|inviteMember|faultProxy)\s*\(|\bspec\.world\s*\(/.test(source),
+ || /(?<!\.)\b(?:app|chrome|server|inviteMember|faultProxy|startCaptureEfficiencyLab)\s*\(|\bspec\.world\s*\(/.test(source),
};
}
diff --git a/evals/scripts/spec-boundary-ratchet.test.mjs b/evals/scripts/spec-boundary-ratchet.test.mjs
index 1de8c3aa..8f4029e7 100644
--- a/evals/scripts/spec-boundary-ratchet.test.mjs
+++ b/evals/scripts/spec-boundary-ratchet.test.mjs
@@ -30,6 +30,14 @@ test("classifySpec recognizes only boundary calls", () => {
assert.equal(classifySpec("helper.app(); const app = true;").crossesBoundary, false);
});
+test("the capture HTTP lab boundary preserves the product-source import guard", () => {
+ const boundary = classifySpec('import { startCaptureEfficiencyLab } from "@omnirush/labs"; await startCaptureEfficiencyLab();');
+ assert.deepEqual(violations("capture.test.ts", boundary), []);
+ const sourceImport = classifySpec('import "../../apps/server/src/capture-client.js"; startCaptureEfficiencyLab();');
+ assert.equal(sourceImport.importsProductSource, true);
+ assert.equal(violations("bad.test.ts", sourceImport).length, 1);
+});
+
test("classifySpec recognizes world imports as boundaries", () => {
for (const specifier of ["../../worlds/example.ts", "../worlds/example.ts", "@omnirush/world"]) {
assert.equal(classifySpec(`import "${specifier}";`).crossesBoundary, true);
diff --git a/evals/specs/capture-efficiency.test.ts b/evals/specs/capture-efficiency.test.ts
new file mode 100644
index 00000000..e64bebf5
--- /dev/null
+++ b/evals/specs/capture-efficiency.test.ts
@@ -0,0 +1,140 @@
+import { expect } from "vitest";
+import { startCaptureEfficiencyLab, type CaptureEfficiencyAttempt, type CaptureEfficiencyLab, type CaptureEfficiencyWitness } from "@omnirush/labs";
+import { eventually, test } from "@omnirush/testkit";
+
+const SESSION_A = "session-a-0001";
+const SESSION_B = "session-b-0001";
+
+async function postControl(lab: CaptureEfficiencyLab, path: string, body: Record<string, unknown>): Promise<void> {
+ const response = await fetch(`${lab.baseUrl}${path}`, {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${lab.controlToken}`,
+ "content-type": "application/json",
+ },
+ body: JSON.stringify(body),
+ signal: AbortSignal.timeout(5_000),
+ });
+ const text = await response.text();
+ expect(response.status, text).toBe(202);
+}
+
+function attemptsFor(witness: CaptureEfficiencyWitness, sessionId: string): CaptureEfficiencyAttempt[] {
+ return witness.attempts.filter((attempt) => attempt.sessionId === sessionId);
+}
+
+function acceptedTrace(witness: CaptureEfficiencyWitness, sessionId: string): CaptureEfficiencyAttempt {
+ const trace = attemptsFor(witness, sessionId).find((attempt) => attempt.snapshotType === "trace" && attempt.status === 201);
+ if (!trace) throw new Error(`accepted trace for ${sessionId} was not observed`);
+ return trace;
+}
+
+test("capture worker uploads ordered, scoped trace artifacts at the HTTP collector boundary", { timeout: 60_000 }, async ({ evidence }) => {
+ const lab = await startCaptureEfficiencyLab();
+ try {
+ const initial = await lab.witness();
+ const unauthorized = await fetch(`${lab.baseUrl}/witness`, { signal: AbortSignal.timeout(5_000) });
+ expect(unauthorized.status).toBe(401);
+ expect((await lab.witness()).attempts).toHaveLength(initial.attempts.length);
+ evidence.recordAssertionEvidence(
+ "Capture controls and collector witness require the lab bearer",
+ "An unauthenticated HTTP witness request returned 401 and did not alter the outbound request record.",
+ true,
+ );
+
+ await postControl(lab, "/control/start", { sessionId: SESSION_A, actor: "actor-a", workspace: "workspace-a" });
+ await postControl(lab, "/control/trace", {
+ sessionId: SESSION_A,
+ type: "engine.request",
+ data: { actor: "actor-a", workspace: "workspace-a", marker: "request-a" },
+ });
+ await postControl(lab, "/control/trace", {
+ sessionId: SESSION_A,
+ type: "turn.messages",
+ data: {
+ actor: "actor-a",
+ workspace: "workspace-a",
+ messages: [{ role: "assistant", parts: [{ type: "tool", tool: "apply_patch", state: { input: { path: "notes-a.txt" } } }] }],
+ },
+ });
+ await postControl(lab, "/control/trace", {
+ sessionId: SESSION_A,
+ type: "turn.completed",
+ data: { actor: "actor-a", workspace: "workspace-a", turn: 1 },
+ });
+ await postControl(lab, "/control/flush", { sessionId: SESSION_A });
+
+ await postControl(lab, "/control/start", { sessionId: SESSION_B, actor: "actor-b", workspace: "workspace-b" });
+ await postControl(lab, "/control/trace", {
+ sessionId: SESSION_B,
+ type: "engine.request",
+ data: { actor: "actor-b", workspace: "workspace-b", marker: "request-b" },
+ });
+ await postControl(lab, "/control/trace", {
+ sessionId: SESSION_B,
+ type: "turn.completed",
+ data: { actor: "actor-b", workspace: "workspace-b", turn: 1 },
+ });
+ await postControl(lab, "/control/flush", { sessionId: SESSION_B });
+
+ await postControl(lab, "/control/complete", { sessionId: SESSION_A });
+ await postControl(lab, "/control/complete", { sessionId: SESSION_B });
+
+ const witness = await eventually(() => lab.witness(), {
+ within: 20_000,
+ intervalMs: 100,
+ label: "capture worker collector requests",
+ until: (current) => current.attempts.length === 7
+ && current.attempts.filter((attempt) => attempt.status === 201).length === 6,
+ });
+
+ expect(witness.modes).toEqual(["worker", "worker"]);
+ const traceA = acceptedTrace(witness, SESSION_A);
+ const traceB = acceptedTrace(witness, SESSION_B);
+ const traceAttemptsA = attemptsFor(witness, SESSION_A).filter((attempt) => attempt.snapshotType === "trace");
+ const traceAttemptsB = attemptsFor(witness, SESSION_B).filter((attempt) => attempt.snapshotType === "trace");
+
+ expect(traceAttemptsA.map((attempt) => [attempt.status, attempt.schemaVersion])).toEqual([[422, 3], [201, 2]]);
+ expect(traceAttemptsB.map((attempt) => [attempt.status, attempt.schemaVersion])).toEqual([[201, 3]]);
+ expect(traceB.filePaths).toEqual([]);
+ expect(traceB.traceFileCount).toBe(0);
+ expect(traceB.traceTypes).toEqual(["engine.request", "turn.completed"]);
+ expect(traceA.schemaVersion).toBe(2);
+ expect(traceA.traceTypes).toEqual(["engine.request", "turn.messages", "turn.completed"]);
+ expect(traceA.traceFileCount).toBe(1);
+ expect(traceA.traceFileEventTypes).toEqual(traceA.traceTypes);
+ evidence.recordAssertionEvidence(
+ "Canonical schema 3 is emitted without a duplicate trace artifact",
+ "The accepted actor-b trace request used schema 3 with files=[] and no legacy trace file; its ordered events were emitted once.",
+ true,
+ );
+ evidence.recordAssertionEvidence(
+ "Unsupported schema 3 downgrades the same ordered batch to legacy schema 2",
+ "Actor-a received one 422 unsupported-schema attempt followed by one schema-2 trace with exactly one trace.json whose event order matched the canonical batch.",
+ true,
+ );
+
+ const actorAAttempts = attemptsFor(witness, SESSION_A).filter((attempt) => attempt.status === 201);
+ const actorBAttempts = attemptsFor(witness, SESSION_B).filter((attempt) => attempt.status === 201);
+ expect(actorAAttempts.map((attempt) => attempt.snapshotType)).toEqual(["start", "trace", "end"]);
+ expect(actorBAttempts.map((attempt) => attempt.snapshotType)).toEqual(["start", "trace", "end"]);
+ expect(actorAAttempts.every((attempt) => attempt.workspaceRoot === "workspace-a")).toBe(true);
+ expect(actorBAttempts.every((attempt) => attempt.workspaceRoot === "workspace-b")).toBe(true);
+ expect(actorAAttempts.every((attempt) => attempt.actorMarkers.every((actor) => actor === "actor-a"))).toBe(true);
+ expect(actorBAttempts.every((attempt) => attempt.actorMarkers.every((actor) => actor === "actor-b"))).toBe(true);
+ expect(actorAAttempts[0]?.filePaths).toContain("notes-a.txt");
+ expect(actorAAttempts[0]?.filePaths).not.toContain("notes-b.txt");
+ expect(actorBAttempts[0]?.filePaths).toContain("notes-b.txt");
+ expect(actorBAttempts[0]?.filePaths).not.toContain("notes-a.txt");
+ expect(traceA.actorMarkers).toEqual(["actor-a", "actor-a", "actor-a"]);
+ expect(traceB.actorMarkers).toEqual(["actor-b", "actor-b"]);
+ expect(witness.capabilityCalls).toEqual({ "actor-a": 1, "actor-b": 1 });
+ evidence.recordAssertionEvidence(
+ "Capture records preserve completion order and actor/workspace scope",
+ "Each synthetic session produced start, trace, and end in order; trace markers and workspace roots stayed within their owning actor, with one capability probe per worker.",
+ true,
+ );
+ } finally {
+ await lab.stop();
+ }
+});
diff --git a/scripts/capture-efficiency-bench/README.md b/scripts/capture-efficiency-bench/README.md
new file mode 100644
index 00000000..e272f906
--- /dev/null
+++ b/scripts/capture-efficiency-bench/README.md
@@ -0,0 +1,44 @@
+# Capture efficiency measurements
+
+These scripts use synthetic files and a loopback receiver. Run the two revisions with the same Node, Bun, Electron, fixture, and host.
+
+## Overlapping sessions
+
+```sh
+bun scripts/capture-efficiency-bench/shared_scan.mjs gui /path/to/gui /path/to/results/gui
+bun scripts/capture-efficiency-bench/shared_scan.mjs cli /path/to/cli /path/to/results/cli
+```
+
+The driver starts eight sessions on one 1,000-file workspace using the product collector. It reports real file stats, reads, redactions, and upload bytes. Elapsed time is indicative; compare work counts first.
+
+## Desktop transport
+
+Requires Linux, Xvfb, Python 3, and the same Electron binary for both runs.
+
+```sh
+python3 scripts/capture-efficiency-bench/generate_transport.py /path/to/transport-fixture.zst
+node scripts/capture-efficiency-bench/slow_sink.mjs
+python3 scripts/capture-efficiency-bench/run_electron.py \
+ --electron /path/to/electron --adapter /baseline/apps/desktop/electron/external-fetch.mjs \
+ --variant baseline --fixture /path/to/transport-fixture.zst --output /path/to/results
+python3 scripts/capture-efficiency-bench/run_electron.py \
+ --electron /path/to/electron --adapter /candidate/apps/desktop/electron/external-fetch.mjs \
+ --variant candidate --fixture /path/to/transport-fixture.zst --output /path/to/results
+```
+
+The 32 MiB fixture is opaque transport data, not a valid collector envelope. The receiver verifies byte count and SHA-256 and limits consumption to 6 MiB/second, independent of chunk size. The baseline driver models the retained byte buffers from the old worker/broker path and calls the original transport; the candidate passes a file descriptor through the worker/broker and reads one bounded body for Electron’s existing native fetch. Node/Bun upload paths stream the file; Electron buffers its body to avoid the pinned runtime’s interrupted chunked-upload crash. This is a transport benchmark, not a complete desktop journey. Peak RSS includes Electron, Chromium children and Xvfb, sampled every 20 ms. Do not describe it as JavaScript heap usage or a universal application-wide reduction.
+
+Separate staging collector/viewer/export checks use valid zstd envelopes and assert exact reconstructed file contents, manifests, ordered trace events, and owner isolation.
+
+## Native failure recovery
+
+The Electron driver also checks a 401 followed by a full-byte authenticated retry, refused redirects, truncated responses, the response byte cap, and cancellation:
+
+```sh
+xvfb-run -a /path/to/electron --no-sandbox --disable-gpu \
+ /absolute/path/to/electron_edge_cases.mjs \
+ /absolute/path/to/external-fetch.mjs /absolute/path/to/transport-fixture.zst \
+ /absolute/path/to/results/edge.json /absolute/path/to/profile
+```
+
+The driver catches uncaught native runtime errors as failures; a timeout cannot satisfy a rejection assertion.
diff --git a/scripts/capture-efficiency-bench/electron_edge_cases.mjs b/scripts/capture-efficiency-bench/electron_edge_cases.mjs
new file mode 100644
index 00000000..61104859
--- /dev/null
+++ b/scripts/capture-efficiency-bench/electron_edge_cases.mjs
@@ -0,0 +1,77 @@
+import { app, net } from "electron";
+import { createServer } from "node:http";
+import { createHash } from "node:crypto";
+import { readFile, stat, writeFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import assert from "node:assert/strict";
+import { join, parse, resolve } from "node:path";
+
+process.on("uncaughtException", error => { process.stderr.write("uncaught " + error.stack + "\n"); app.exit(1); });
+process.on("unhandledRejection", error => { process.stderr.write("unhandled " + String(error.stack || error) + "\n"); app.exit(1); });
+const entry = fileURLToPath(import.meta.url);
+const args = process.argv.slice(process.argv.findIndex(arg => !arg.startsWith("-") && resolve(arg) === entry) + 1);
+const [adapterPath, fixture, output, profile] = args;
+app.setPath("userData", profile);
+app.disableHardwareAcceleration();
+app.commandLine.appendSwitch("host-resolver-rules", "MAP capture-bench.example.test 127.0.0.1");
+app.whenReady().then(async () => {
+ const { createExternalFileUpload } = await import(adapterPath);
+ const upload = createExternalFileUpload({ net });
+ const size = (await stat(fixture)).size;
+ let redirected = 0, earlyBytes = 0, cancelController;
+ const server = createServer((req, res) => {
+ req.on("error", () => {});
+ if (req.url === "/redirect-target") { redirected += 1; res.end("{}"); return; }
+ if (req.url === "/redirect") { req.resume(); res.writeHead(302, { location: "/redirect-target" }); res.end(); return; }
+ if (req.url === "/oversize") { req.resume(); res.writeHead(201); res.end(Buffer.alloc(1024 * 1024 + 1)); return; }
+ if (req.url === "/truncated") { req.resume(); res.writeHead(201, { "content-length": "1024" }); res.write("{"); setTimeout(() => res.destroy(), 5); return; }
+ if (req.url === "/cancel") { req.on("data", () => cancelController.abort()); return; }
+ if (req.headers.authorization !== "Bearer synthetic-refreshed") {
+ req.on("data", chunk => { earlyBytes += chunk.length; req.pause(); setTimeout(() => req.resume(), Math.max(1, chunk.length / (4 * 1024 * 1024) * 1000)); });
+ res.writeHead(401, { "content-type": "application/json" }); res.end('{"detail":"expired"}'); return;
+ }
+ const digest = createHash("sha256"); let bytes = 0;
+ req.on("data", chunk => { bytes += chunk.length; digest.update(chunk); });
+ req.on("end", () => { res.writeHead(201, { "content-type": "application/json" }); res.end(JSON.stringify({ bytes, sha256: digest.digest("hex") })); });
+ req.on("error", () => res.destroy());
+ });
+ await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
+ const origin = "http://capture-bench.example.test:" + server.address().port;
+ const init = { method: "POST", path: fixture, size, headers: { authorization: "Bearer synthetic-old" } };
+ const attempt = (path, options = init) => upload(origin + path, { ...options, signal: options.signal ?? AbortSignal.timeout(15000) });
+ const rejection = error => { assert.notEqual(error.name, "TimeoutError"); return true; };
+ const progress = name => process.stderr.write("checking " + name + "\n");
+ try {
+ progress("authentication response");
+ assert.equal((await attempt("/upload")).status, 401);
+ progress("authenticated retry");
+ const retry = await attempt("/upload", { ...init, headers: { authorization: "Bearer synthetic-refreshed" } });
+ assert.equal(retry.status, 201);
+ const body = await retry.json();
+ assert.equal(body.bytes, size);
+ const fixturePath = parse(fixture);
+ assert.equal(body.sha256, (await readFile(join(fixturePath.dir, fixturePath.name + ".sha256"), "utf8")).trim());
+ // Chromium may finish sending the body before delivering an auth response.
+ // The retry must reopen and transmit the complete unchanged file.
+ progress("redirect refusal");
+ await assert.rejects(attempt("/redirect"), rejection);
+ assert.equal(redirected, 0);
+ progress("truncated response");
+ await assert.rejects(attempt("/truncated"), rejection);
+ progress("response byte bound");
+ await assert.rejects(attempt("/oversize"), /exceeds its bound/);
+ progress("cancellation");
+ cancelController = new AbortController();
+ await assert.rejects(upload(origin + "/cancel", { ...init, signal: cancelController.signal }), { name: "AbortError" });
+ await writeFile(output, JSON.stringify({
+ electron: process.versions.electron, bytes: size, sha256: body.sha256, early_request_body_bytes: earlyBytes,
+ assertions: { auth_response_returned: true, retry_reopens_entire_file: true, redirects_refused: true,
+ truncated_response_rejected: true, response_size_bounded: true, cancellation: true },
+ }, null, 2));
+ } finally {
+ progress("cleanup");
+ server.closeAllConnections();
+ await new Promise(resolve => server.close(resolve));
+ }
+ app.quit();
+}).catch(error => { process.stderr.write(String(error.stack || error) + "\n"); app.exit(1); });
diff --git a/scripts/capture-efficiency-bench/electron_transport.mjs b/scripts/capture-efficiency-bench/electron_transport.mjs
new file mode 100644
index 00000000..ee0c4a34
--- /dev/null
+++ b/scripts/capture-efficiency-bench/electron_transport.mjs
@@ -0,0 +1,33 @@
+import { app, net } from "electron";
+import { readFile, writeFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { fileURLToPath, pathToFileURL } from "node:url";
+const scriptIndex=process.argv.indexOf(fileURLToPath(import.meta.url));
+if(scriptIndex<0) throw new Error("benchmark entry missing from argv");
+const [variant, adapterPath, fixture, expectedHash, resultPath, profile] = process.argv.slice(scriptIndex+1);
+app.setPath("userData",profile);
+app.disableHardwareAcceleration();
+app.commandLine.appendSwitch("host-resolver-rules","MAP capture-bench.example.test 127.0.0.1");
+async function run() {
+const adapter=await import(pathToFileURL(adapterPath).href);
+const started=performance.now();
+const headers={"Content-Type":"application/zstd","X-OmniRush-Session-ID":"ses_transport_benchmark"};
+const url="http://capture-bench.example.test:18692/collect";
+let response;
+if(variant==="baseline"){
+ const compressed=await readFile(fixture);
+ const transferred=new Uint8Array(compressed);
+ const body=transferred.buffer.slice(transferred.byteOffset,transferred.byteOffset+transferred.byteLength);
+ // Keep the two retry-owned copies alive, as worker and broker each do.
+ globalThis.benchmarkRetryBodies=[compressed,transferred];
+ response=await adapter.createExternalFetch({net})(url,{method:"POST",headers,body});
+}else{
+ response=await adapter.createExternalFileUpload({net})(url,{method:"POST",headers,path:fixture,size:32*1024*1024,signal:AbortSignal.timeout(60000)});
+}
+const result=await response.json();
+if(response.status!==201||result.bytes!==32*1024*1024||result.sha256!==expectedHash) throw new Error("transport body mismatch");
+await writeFile(resultPath,JSON.stringify({variant,status:response.status,bytes:result.bytes,sha256:result.sha256,elapsed_ms:Math.round(performance.now()-started),electron:process.versions.electron,node:process.versions.node},null,2));
+globalThis.benchmarkRetryBodies=null;
+app.quit();
+}
+app.whenReady().then(run).catch(error => { console.error(error); app.exit(1); });
diff --git a/scripts/capture-efficiency-bench/generate_transport.py b/scripts/capture-efficiency-bench/generate_transport.py
new file mode 100644
index 00000000..99be8767
--- /dev/null
+++ b/scripts/capture-efficiency-bench/generate_transport.py
@@ -0,0 +1,21 @@
+"""Generate an opaque upload body for the transport-only memory comparison."""
+import argparse, hashlib
+from pathlib import Path
+
+parser = argparse.ArgumentParser()
+parser.add_argument("path", type=Path)
+parser.add_argument("--bytes", type=int, default=32 * 1024 * 1024)
+args = parser.parse_args()
+if args.bytes < 1:
+ parser.error("--bytes must be positive")
+args.path.parent.mkdir(parents=True, exist_ok=True)
+digest = hashlib.sha256()
+written = 0
+with args.path.open("wb") as output:
+ while written < args.bytes:
+ block = hashlib.shake_256(b"omnirush synthetic capture transport" + (written // 65536).to_bytes(8, "big")).digest(min(65536, args.bytes - written))
+ output.write(block)
+ digest.update(block)
+ written += len(block)
+args.path.with_suffix(".sha256").write_text(digest.hexdigest())
+print(f"{written} bytes; sha256={digest.hexdigest()}")
diff --git a/scripts/capture-efficiency-bench/measure_tree.py b/scripts/capture-efficiency-bench/measure_tree.py
new file mode 100644
index 00000000..7b6992d1
--- /dev/null
+++ b/scripts/capture-efficiency-bench/measure_tree.py
@@ -0,0 +1,33 @@
+"""Bounded process-tree RSS sampler for a task-owned command."""
+import argparse,json,os,subprocess,time
+from pathlib import Path
+p=argparse.ArgumentParser(); p.add_argument("--output",required=True); p.add_argument("--interval",type=float,default=.02); p.add_argument("--timeout",type=float,default=180); p.add_argument("command",nargs=argparse.REMAINDER); a=p.parse_args()
+cmd=a.command[1:] if a.command and a.command[0]=="--" else a.command
+proc=subprocess.Popen(cmd,start_new_session=True)
+started=time.monotonic(); peak=0; samples=0; peak_n=0; last_tree=[]
+def tree(root):
+ children={}
+ for entry in Path("/proc").iterdir():
+ if not entry.name.isdigit(): continue
+ try:
+ text=(entry/"stat").read_text(); rest=text[text.rfind(")")+2:].split()
+ children.setdefault(int(rest[1]),[]).append(int(entry.name))
+ except (OSError,ValueError,IndexError): pass
+ result=[]; pending=[root]
+ while pending:
+ pid=pending.pop(); result.append(pid); pending+=children.get(pid,[])
+ return result
+while proc.poll() is None:
+ if time.monotonic()-started>a.timeout:
+ os.killpg(proc.pid,15); proc.wait(timeout=10); raise TimeoutError("task-owned benchmark timed out")
+ pids=tree(proc.pid); total=0
+ for pid in pids:
+ try:
+ line=next(l for l in Path(f"/proc/{pid}/status").read_text().splitlines() if l.startswith("VmRSS:"))
+ total+=int(line.split()[1])*1024
+ except (OSError,StopIteration,ValueError): pass
+ if total>peak: peak=total; last_tree=pids
+ peak_n=max(peak_n,len(pids)); samples+=1; time.sleep(a.interval)
+result={"peak_tree_rss_bytes":peak,"elapsed_s":round(time.monotonic()-started,4),"samples":samples,"peak_process_count":peak_n,"exit_code":proc.returncode,"interval_s":a.interval,"command":cmd}
+Path(a.output).write_text(json.dumps(result,indent=2)); print(json.dumps(result))
+raise SystemExit(proc.returncode)
diff --git a/scripts/capture-efficiency-bench/run_electron.py b/scripts/capture-efficiency-bench/run_electron.py
new file mode 100644
index 00000000..8ee906f8
--- /dev/null
+++ b/scripts/capture-efficiency-bench/run_electron.py
@@ -0,0 +1,24 @@
+"""Measure a real Electron upload with an unchanged, specified runtime."""
+import argparse, subprocess
+from pathlib import Path
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--electron", type=Path, required=True)
+parser.add_argument("--adapter", type=Path, required=True)
+parser.add_argument("--variant", choices=["baseline", "candidate"], required=True)
+parser.add_argument("--fixture", type=Path, required=True)
+parser.add_argument("--output", type=Path, required=True)
+args = parser.parse_args()
+here = Path(__file__).resolve().parent
+args.output.mkdir(parents=True, exist_ok=True)
+profile = args.output / ("electron-profile-" + args.variant)
+profile.mkdir(exist_ok=True)
+command = [
+ "python3", str(here / "measure_tree.py"), "--output", str(args.output / ("electron-" + args.variant + "-rss.json")),
+ "--timeout", "120", "--", "xvfb-run", "-a", str(args.electron.resolve()),
+ "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage", str(here / "electron_transport.mjs"),
+ args.variant, str(args.adapter.resolve()), str(args.fixture.resolve()),
+ args.fixture.with_suffix(".sha256").read_text().strip(),
+ str(args.output / ("electron-" + args.variant + ".json")), str(profile.resolve()),
+]
+raise SystemExit(subprocess.run(command).returncode)
diff --git a/scripts/capture-efficiency-bench/shared_scan.mjs b/scripts/capture-efficiency-bench/shared_scan.mjs
new file mode 100644
index 00000000..487db0bb
--- /dev/null
+++ b/scripts/capture-efficiency-bench/shared_scan.mjs
@@ -0,0 +1,29 @@
+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { execFileSync } from "node:child_process";
+const [kind, treeArg, outputArg] = process.argv.slice(2);
+const tree = resolve(treeArg);
+const output = resolve(outputArg);
+const source = kind === "gui" ? join(tree,"apps/server/src/session-uploader.ts") : join(tree,"assets/extensions/omnirush/capture/workspace-sync.ts");
+const mod = await import(pathToFileURL(source).href);
+const Collector = kind === "gui" ? mod.SessionUploader : mod.WorkspaceSync;
+await mkdir(output,{recursive:true});
+const fixture=await mkdtemp(join(output,"shared-root-"));
+const state=await mkdtemp(join(output,"state-"));
+await mkdir(join(fixture,"src"));
+for(let i=0;i<1000;i++) await writeFile(join(fixture,"src",String(i).padStart(4,"0")+".ts"),("export const fixture"+i+" = "+i+";\n").repeat(25));
+execFileSync("git",["init","-q",fixture]);
+let uploads=0,bytes=0;
+const uploader=new Collector({
+ stateDir:state,appVersion:"staging-benchmark",maxWatchedFiles:0,watch:false,
+ upload:async(_session,payload)=>{uploads++;bytes+=payload.byteLength;return new Response("{}",{status:201});},
+});
+const ids=Array.from({length:8},(_,i)=>"ses_shared_scan_"+String(i).padStart(8,"0"));
+const started=performance.now();
+for(const id of ids) uploader.startSession(id,"shared-fixture",fixture);
+await uploader.idleAll();
+const result={kind,session_count:ids.length,file_count:1000,elapsed_ms:Math.round(performance.now()-started),uploads,compressed_bytes:bytes,metrics:{...uploader.metrics}};
+await uploader.stop();
+await writeFile(join(output,"shared-scan-"+kind+".json"),JSON.stringify(result,null,2));
+console.log(JSON.stringify(result));
diff --git a/scripts/capture-efficiency-bench/slow_sink.mjs b/scripts/capture-efficiency-bench/slow_sink.mjs
new file mode 100644
index 00000000..04675352
--- /dev/null
+++ b/scripts/capture-efficiency-bench/slow_sink.mjs
@@ -0,0 +1,22 @@
+import { createServer } from "node:http";
+import { createHash } from "node:crypto";
+const BYTES_PER_SECOND = 6 * 1024 * 1024;
+const server = createServer((req, res) => {
+ let bytes = 0, chunks = 0;
+ const digest = createHash("sha256");
+ const started = performance.now();
+ req.on("data", chunk => {
+ bytes += chunk.length; chunks += 1; digest.update(chunk);
+ const wait = bytes / BYTES_PER_SECOND * 1000 - (performance.now() - started);
+ if (wait > 2) {
+ req.pause();
+ setTimeout(() => req.resume(), wait);
+ }
+ });
+ req.on("end", () => {
+ res.writeHead(201, {"Content-Type": "application/json"});
+ res.end(JSON.stringify({bytes, chunks, elapsed_ms: Math.round(performance.now() - started), sha256: digest.digest("hex")}));
+ });
+ req.on("error", () => res.destroy());
+});
+server.listen(18692, "127.0.0.1");