A desktop coding agent with free access to frontier models.
TypeScript
perf(capture): run capture on a worker thread so session switches never wait on it
ogsamrat78cce81ee80b
diff --git a/apps/server/src/capture-client.test.ts b/apps/server/src/capture-client.test.tsnew file mode 100644index 00000000..e151918d--- /dev/null+++ b/apps/server/src/capture-client.test.ts@@ -0,0 +1,248 @@+import { afterEach, describe, expect, test } from "bun:test";+import { execFile } from "node:child_process";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { promisify } from "node:util";+import { zstdDecompressSync } from "node:zlib";++import { startCaptureService, type CaptureService, type CaptureServiceOptions } from "./capture-client.js";+import { FakeArchiveServer, slowPartTwo } from "./session-archive/fake-archive-server.js";++/**+ * The capture worker: the collector's and the archiver's work runs off the+ * main event loop, a resumed session starts with the whole workspace, and+ * shutdown with work in flight settles in bounded time with archive uploads+ * aborted first.+ */++const execFileAsync = promisify(execFile);+const cleanups: Array<() => unknown> = [];++afterEach(async () => {+ while (cleanups.length) await cleanups.pop()?.();+});++type Envelope = {+ session_id: string;+ snapshot_type: string;+ trigger: string;+ files_scope: string;+ changed_paths?: string[];+ files: Array<{ path: string; content: string }>;+ manifest: Array<{ path: string; sha256: string }>;+};++async function tempDir(prefix: string): Promise<string> {+ const dir = await mkdtemp(join(tmpdir(), `omnirush-capture-${prefix}-`));+ cleanups.push(() => rm(dir, { recursive: true, force: true }));+ return dir;+}++async function git(root: string, ...args: string[]): Promise<void> {+ await execFileAsync("git", ["-C", root, "-c", "commit.gpgsign=false", "-c", "user.name=Dev", "-c", "[email protected]", ...args]);+}++/**+ * A git workspace of `count` source files of about 5 KiB each, every one with+ * an e-mail address and a secret-named assignment, so the scrubber rewrites+ * all of them: a scan reads, hashes and scrubs every byte.+ */+async function syntheticWorkspace(count: number): Promise<string> {+ const root = await tempDir("workspace");+ await git(root, "init", "-q");+ const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"];+ const files = Array.from({ length: count }, (_, index) => index);+ for (let start = 0; start < files.length; start += 256) {+ await Promise.all(files.slice(start, start + 256).map(async (index) => {+ const dir = join(root, "src", `m${index % 64}`);+ await mkdir(dir, { recursive: true });+ const lines = [`// module ${index}`, `export const owner = "dev${index}@example.com";`, `const password = "Pa55-word-${index}-x";`];+ for (let line = 0; line < 90; line += 1) lines.push(`export const v${line} = "${words[(index + line) % words.length]} ${index * 31 + line}";`);+ await writeFile(join(dir, `file${index}.ts`), `${lines.join("\n")}\n`);+ }));+ }+ return root;+}++function uploadSink() {+ const compressed: Array<{ sessionId: string; bytes: Uint8Array }> = [];+ return {+ compressed,+ // Only kept here: decoding a large envelope on this thread would itself stall the loop being measured.+ upload: async (sessionId: string, bytes: Uint8Array) => {+ compressed.push({ sessionId, bytes });+ return Response.json({ ok: true }, { status: 201 });+ },+ envelopes: (): Envelope[] => compressed.map((item) => JSON.parse(zstdDecompressSync(item.bytes).toString("utf8"))),+ };+}++function service(options: Partial<CaptureServiceOptions> & Pick<CaptureServiceOptions, "stateDir" | "collector">): CaptureService {+ const capture = startCaptureService({+ appVersion: "0.0.0-test",+ engineVersion: "0.0.0-test",+ log: () => undefined,+ archive: { enabled: false, excludedDirs: [] },+ worker: true,+ ...options,+ });+ cleanups.push(() => capture.stop());+ return capture;+}++/** The longest the main event loop went without running a 5 ms timer, less the 5 ms. */+function loopMonitor() {+ let last = performance.now();+ let worst = 0;+ const timer = setInterval(() => {+ const now = performance.now();+ worst = Math.max(worst, now - last - 5);+ last = now;+ }, 5);+ return { stop: () => {+ clearInterval(timer);+ return worst;+ } };+}++async function until(check: () => boolean | Promise<boolean>, timeoutMs: number, what: string): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (!(await check())) {+ if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);+ await new Promise((resolvePromise) => setTimeout(resolvePromise, 20));+ }+}++describe("capture worker", () => {+ test("a 6,000-file start snapshot is read, scrubbed, compressed and uploaded without a main-loop stall over 100 ms", async () => {+ const root = await syntheticWorkspace(6_000);+ const sink = uploadSink();+ const capture = service({ stateDir: await tempDir("state"), collector: { upload: sink.upload } });+ const monitor = loopMonitor();+ const started = performance.now();+ capture.startSession("session-offload-0001", "workspace-offload", root);+ await until(() => sink.compressed.length > 0, 90_000, "the start snapshot");+ const elapsedMs = performance.now() - started;+ const worstStallMs = monitor.stop();+ expect(capture.mode()).toBe("worker");++ const [start] = sink.envelopes();+ expect(start).toMatchObject({ snapshot_type: "start", trigger: "session_start", files_scope: "full" });+ const sources = start!.files.filter((file) => file.path.startsWith("src/"));+ expect(sources).toHaveLength(6_000);+ // The privacy rails ran on the worker exactly as in-process.+ expect(sources.every((file) => !file.content.includes("@example.com") && file.content.includes("[REDACTED_PII]"))).toBe(true);+ expect(sources.every((file) => file.content.includes('const password = "[REDACTED]";'))).toBe(true);+ const diagnostics = await capture.diagnostics();+ expect(diagnostics?.metrics.fileRedactions).toBeGreaterThanOrEqual(6_000);+ // Seconds of reading, hashing and scrubbing, and the loop kept running throughout.+ expect(elapsedMs).toBeGreaterThan(200);+ expect(worstStallMs).toBeLessThan(100);+ await capture.stop();+ }, 120_000);++ test("a resumed session's start snapshot through the worker carries the whole workspace, with what changed while the app was closed", async () => {+ const root = await syntheticWorkspace(50);+ const stateDir = await tempDir("state");+ const sessionId = "session-resume-0001";+ const firstSink = uploadSink();+ const first = service({ stateDir, collector: { upload: firstSink.upload } });+ first.startSession(sessionId, "workspace-resume", root);+ await first.stop();+ expect(firstSink.envelopes().map((item) => [item.snapshot_type, item.files_scope])).toEqual([["start", "full"], ["end", "changed"]]);++ await writeFile(join(root, "src", "m7", "file7.ts"), "export const edited = true;\n");+ const secondSink = uploadSink();+ const second = service({ stateDir, collector: { upload: secondSink.upload } });+ second.startSession(sessionId, "workspace-resume", root);+ await second.idle();+ const [resumed] = secondSink.envelopes();+ expect(resumed).toMatchObject({ snapshot_type: "start", trigger: "resume", files_scope: "full" });+ expect(resumed!.changed_paths).toBeUndefined();+ const sources = resumed!.files.filter((file) => file.path.startsWith("src/"));+ expect(sources).toHaveLength(50);+ expect(sources.find((file) => file.path === "src/m7/file7.ts")?.content).toBe("export const edited = true;\n");+ expect(resumed!.manifest).toHaveLength(50);+ await second.stop();+ }, 60_000);++ test("stopping with a start snapshot in flight finishes it and the session's end snapshot, then ends the worker", async () => {+ const root = await syntheticWorkspace(3_000);+ const sink = uploadSink();+ const capture = service({ stateDir: await tempDir("state"), collector: { upload: sink.upload } });+ capture.startSession("session-stop-00001", "workspace-stop", root);+ await capture.diagnostics();+ const stopping = performance.now();+ await capture.stop();+ expect(performance.now() - stopping).toBeLessThan(15_000);+ expect(capture.mode()).toBe("down");+ expect(sink.envelopes().map((item) => item.snapshot_type)).toEqual(["start", "end"]);+ // Nothing runs after stop: calls are dropped, queries answer at once.+ capture.startSession("session-stop-00002", "workspace-stop", root);+ expect(await capture.diagnostics()).toBeNull();+ }, 60_000);++ test("stopping aborts an archive part upload in flight on the main thread within a second", async () => {+ const root = await tempDir("project");+ await writeFile(join(root, "README.md"), "# project\n".repeat(200));+ await git(root, "init", "-q");+ await git(root, "add", "README.md");+ await git(root, "commit", "-q", "-m", "initial");+ const archive = new FakeArchiveServer();+ archive.partSize = 256;+ const slow = slowPartTwo(archive);+ const engine = Bun.serve({+ hostname: "127.0.0.1",+ port: 0,+ fetch: (request) => (new URL(request.url).pathname.endsWith("/message") ? Response.json([]) : Response.json({ id: "ses_archive_0001" })),+ });+ cleanups.push(() => engine.stop(true));+ const sink = uploadSink();+ const capture = service({+ stateDir: await tempDir("state"),+ collector: { upload: sink.upload },+ archive: {+ enabled: true,+ excludedDirs: [],+ request: (path, init) => archive.respond(`https://api.omnirush.test/omnirush/${path}`, {+ method: init.method,+ headers: { authorization: `Bearer ${archive.token}` },+ ...(init.body === undefined ? {} : { body: init.body }),+ }),+ refreshAccessToken: async () => null,+ fetch: slow.fetch,+ baseIdleMs: 0,+ },+ });+ capture.startSession("ses_archive_0001", "workspace-archive", root);+ capture.archiveSessionStarted("ses_archive_0001", root, { baseUrl: `http://127.0.0.1:${engine.port}`, headers: [], search: "", engine: "v1" });+ await slow.started;++ slow.markCommand();+ await capture.stop();+ expect(slow.puts).toEqual([{ part: 2, bytes: 256, abortedAfterMs: expect.any(Number), completed: false }]);+ expect(slow.puts[0]!.abortedAfterMs!).toBeLessThan(1_000);+ // The collector still delivered the session's end snapshot on the way out.+ expect(sink.envelopes().map((item) => item.snapshot_type)).toContain("end");+ }, 60_000);++ test("OMNIRUSH_CAPTURE_WORKER=0 captures in-process with the same envelopes", async () => {+ const root = await syntheticWorkspace(20);+ const results: Array<Array<[string, string, number]>> = [];+ for (const worker of [true, false]) {+ const sink = uploadSink();+ const capture = service({ stateDir: await tempDir("state"), collector: { upload: sink.upload }, worker });+ expect(capture.mode()).toBe(worker ? "starting" : "local");+ capture.startSession("session-modes-0001", "workspace-modes", root);+ expect(capture.hasSession("session-modes-0001")).toBe(true);+ expect(capture.recordWebVisit("session-modes-0001", { url: "https://example.com/docs", title: "Docs", text: "mail [email protected]" })).toBe(true);+ expect(capture.recordWebVisit("session-modes-0001", { url: "http://localhost:3000/" })).toBe(false);+ await capture.stop();+ expect(capture.mode()).toBe("down");+ results.push(sink.envelopes().map((item) => [item.snapshot_type, item.files_scope, item.files.length]));+ }+ expect(results[0]).toEqual([["start", "full", 21], ["trace", "full", 1], ["end", "changed", 1]]);+ expect(results[1]).toEqual(results[0]!);+ }, 60_000);+});diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.tsnew file mode 100644index 00000000..2243ada2--- /dev/null+++ b/apps/server/src/capture-client.ts@@ -0,0 +1,440 @@+/**+ * The server's handle on session capture (the workspace collector, the+ * project archive and the turn observers, see capture-host.ts). By default+ * the capture runs on a worker thread (capture-worker.ts): the main event+ * loop, which serves the app (in the desktop app, the Electron main process),+ * only posts calls and answers the worker's requests for the gateway broker+ * and external egress. Opening or switching sessions never waits on a scan,+ * a snapshot or an archive. If the worker cannot start (a runtime without+ * worker support for this module, or OMNIRUSH_CAPTURE_WORKER=0) the same host+ * runs in-process; a worker that dies after starting is replaced, a bounded+ * number of times.+ */+import { Worker } from "node:worker_threads";++import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineTarget, type PromptRecord } from "./capture-host.js";+import {+ invokeCapture,+ serializeResponse,+ transferables,+ type CaptureCall,+ type CaptureWorkerInit,+ type FromWorker,+ type HostRequest,+ type RequestChannel,+ type RequestResult,+ type ToWorker,+} from "./capture-protocol.js";+import { externalFetch } from "./server-fetch.js";+import { isCollectableWebUrl, workspaceCollectorEnabled, type CollectorWebVisit } from "./workspace-collector.js";++/** A prompt body larger than this is not traced, so it is not copied to the worker either. */+const MAX_TRACED_REQUEST_BYTES = 4 * 1024 * 1024;+/** Shutdown waits this long for the last traces and end snapshots before the worker is terminated. */+const STOP_TIMEOUT_MS = 20_000;+/** A worker that exits unexpectedly is replaced at most this many times per server. */+const MAX_WORKER_RESTARTS = 3;+const SESSION_ID_PATTERN = /^[A-Za-z0-9._:-]{8,128}$/;++export type CaptureServiceOptions = Omit<CaptureHostOptions, "onSessionClosed"> & {+ /** Run capture on a worker thread; default on unless OMNIRUSH_CAPTURE_WORKER is 0, false, no or off. */+ worker?: boolean;+};++export type CaptureService = {+ /** The collector has an account to upload to (the sign-in gate's question). */+ readonly collectorEnabled: boolean;+ /** Where capture runs right now. */+ mode(): "starting" | "worker" | "local" | "down";+ /** Whether the collector is tracking this session (started and not finished). */+ hasSession(sessionId: string): boolean;+ startSession(sessionId: string, workspaceId: string, root: string): void;+ recordTrace(sessionId: string, type: string, data?: unknown): void;+ /** The "engine.request" event of a collected request (its body parsed off the main thread), plus a prompt's attachments. */+ recordPrompt(sessionId: string, prompt: Omit<PromptRecord, "body"> & { body: ArrayBuffer | undefined }): void;+ captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void;+ /** Whether the visit is recorded (a tracked session and a collectable URL). */+ recordWebVisit(sessionId: string, visit: CollectorWebVisit): boolean;+ archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void;+ observeSession(sessionId: string, target: EngineTarget): void;+ sessionDeleted(sessionId: string): void;+ signOut(): Promise<void>;+ stop(): Promise<void>;+ /** Every queued capture, upload and archive step settled (tests, profiling). */+ idle(): Promise<void>;+ diagnostics(): Promise<CaptureDiagnostics | null>;+};++export function captureWorkerEnabled(env: NodeJS.ProcessEnv = process.env): boolean {+ return !["0", "false", "no", "off"].includes((env.OMNIRUSH_CAPTURE_WORKER ?? "").trim().toLowerCase());+}++function workerUrl(): URL {+ // Tests and development run the TypeScript sources; builds run the compiled modules.+ return new URL(import.meta.url.endsWith(".ts") ? "./capture-worker.ts" : "./capture-worker.js", import.meta.url);+}++function errorSummary(error: unknown): string {+ return error instanceof Error ? `${error.name}: ${error.message}` : String(error);+}++type QueuedCall = { call: CaptureCall; transfer: ArrayBuffer[] };++class CaptureClient implements CaptureService {+ readonly collectorEnabled: boolean;+ private current: "starting" | "worker" | "local" | "down" = "starting";+ private worker: Worker | null = null;+ private local: CaptureHost | null = null;+ private queue: QueuedCall[] = [];+ private callSeq = 0;+ /** Calls waiting for the worker's reply. */+ private readonly replies = new Map<number, (value: unknown) => void>();+ /** The worker's requests being served on this thread. */+ private readonly served = new Map<number, { controller: AbortController; channel: RequestChannel }>();+ /** Sessions the worker's collector tracks: true while live, false once finishing. */+ private readonly sessions = new Map<string, boolean>();+ private restarts = 0;+ private stopping: Promise<void> | null = null;++ constructor(private readonly options: CaptureServiceOptions) {+ this.collectorEnabled = workspaceCollectorEnabled({+ upload: Boolean(options.collector.upload),+ gatewayUrl: options.collector.gatewayUrl,+ accessToken: options.collector.accessToken,+ });+ if (options.worker ?? captureWorkerEnabled()) this.spawn();+ else this.runLocally(null);+ }++ mode(): "starting" | "worker" | "local" | "down" {+ return this.current;+ }++ hasSession(sessionId: string): boolean {+ if (this.local) return this.local.collector.hasSession(sessionId);+ return this.sessions.get(sessionId) === true;+ }++ startSession(sessionId: string, workspaceId: string, root: string): void {+ if (this.collectorEnabled && SESSION_ID_PATTERN.test(sessionId) && !this.sessions.has(sessionId)) this.sessions.set(sessionId, true);+ this.send({ kind: "call", id: null, method: "startSession", args: [sessionId, workspaceId, root] });+ }++ recordTrace(sessionId: string, type: string, data?: unknown): void {+ this.send({ kind: "call", id: null, method: "recordTrace", args: data === undefined ? [sessionId, type] : [sessionId, type, data] });+ }++ recordPrompt(sessionId: string, prompt: Omit<PromptRecord, "body"> & { body: ArrayBuffer | undefined }): void {+ // The body is still forwarded to the engine: the worker gets its own copy, moved rather than cloned.+ const body = prompt.body && prompt.body.byteLength > 0 && prompt.body.byteLength <= MAX_TRACED_REQUEST_BYTES ? new Uint8Array(prompt.body.slice(0)) : null;+ this.send({ kind: "call", id: null, method: "recordPrompt", args: [sessionId, { ...prompt, body }] }, body ? [body.buffer] : []);+ }++ captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void {+ this.send({ kind: "call", id: null, method: "captureSnapshot", args: [sessionId, trigger] });+ }++ recordWebVisit(sessionId: string, visit: CollectorWebVisit): boolean {+ if (this.local) return this.local.recordWebVisit(sessionId, visit);+ if (!this.hasSession(sessionId) || typeof visit.url !== "string" || !isCollectableWebUrl(visit.url)) return false;+ this.send({ kind: "call", id: null, method: "recordWebVisit", args: [sessionId, visit] });+ return true;+ }++ archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void {+ this.send({ kind: "call", id: null, method: "archiveSessionStarted", args: [sessionId, root, target] });+ }++ observeSession(sessionId: string, target: EngineTarget): void {+ this.send({ kind: "call", id: null, method: "observeSession", args: [sessionId, target] });+ }++ sessionDeleted(sessionId: string): void {+ if (this.sessions.get(sessionId) === true) this.sessions.set(sessionId, false);+ this.send({ kind: "call", id: null, method: "sessionDeleted", args: [sessionId] });+ }++ async signOut(): Promise<void> {+ await this.query({ kind: "call", id: null, method: "signOut", args: [] });+ }++ async idle(): Promise<void> {+ await this.query({ kind: "call", id: null, method: "idle", args: [] });+ }++ async diagnostics(): Promise<CaptureDiagnostics | null> {+ const value = await this.query({ kind: "call", id: null, method: "diagnostics", args: [] });+ return isDiagnostics(value) ? value : null;+ }++ stop(): Promise<void> {+ this.stopping ??= this.shutdown();+ return this.stopping;+ }++ private async shutdown(): Promise<void> {+ // First, and synchronously: archive uploads this thread makes for the worker are aborted now.+ for (const { controller, channel } of this.served.values()) if (channel === "archive") controller.abort();+ const done = this.dispatch({ kind: "call", id: null, method: "stop", args: [] }, [], true);+ let timer: ReturnType<typeof setTimeout> | undefined;+ const timedOut = new Promise<void>((resolvePromise) => {+ timer = setTimeout(() => {+ this.options.log("warn", "OmniRush capture did not stop in time; the capture worker is terminated", { timeoutMs: STOP_TIMEOUT_MS });+ resolvePromise();+ }, STOP_TIMEOUT_MS);+ timer.unref?.();+ });+ await Promise.race([done, timedOut]);+ clearTimeout(timer);+ const worker = this.worker;+ this.worker = null;+ this.current = "down";+ if (worker) await worker.terminate().catch(() => undefined);+ this.settleOutstanding();+ }++ // --- calls ------------------------------------------------------------------++ private send(call: CaptureCall, transfer: ArrayBuffer[] = []): void {+ if (this.stopping) return;+ void this.dispatch(call, transfer, false);+ }++ private query(call: CaptureCall): Promise<unknown> {+ if (this.stopping) return Promise.resolve(null);+ return this.dispatch(call, [], true);+ }++ /** Runs or posts one call; resolves with its result when `awaited`, else at once. */+ private dispatch(call: CaptureCall, transfer: ArrayBuffer[], awaited: boolean): Promise<unknown> {+ if (this.local) {+ try {+ return Promise.resolve(invokeCapture(this.local, call)).catch(() => null);+ } catch (error) {+ this.options.log("warn", "OmniRush capture call failed", { method: call.method, error: errorSummary(error) });+ return Promise.resolve(null);+ }+ }+ if (this.current === "down") return Promise.resolve(null);+ let reply: Promise<unknown> = Promise.resolve(null);+ if (awaited) {+ const id = ++this.callSeq;+ call.id = id;+ reply = new Promise((resolvePromise) => this.replies.set(id, resolvePromise));+ }+ if (this.current === "worker" && this.worker) this.post(this.worker, call, transfer);+ else this.queue.push({ call, transfer });+ return reply;+ }++ /** Posts a call; one whose arguments cannot be cloned is dropped with a warn line, never thrown into the request. */+ private post(worker: Worker, call: CaptureCall, transfer: ArrayBuffer[]): void {+ try {+ worker.postMessage(call satisfies ToWorker, transfer);+ } catch (error) {+ this.options.log("warn", "OmniRush capture call could not be sent to the capture worker", { method: call.method, error: errorSummary(error) });+ if (call.id === null) return;+ const resolvePromise = this.replies.get(call.id);+ this.replies.delete(call.id);+ resolvePromise?.(null);+ }+ }++ // --- the worker -----------------------------------------------------------------++ private workerInit(): CaptureWorkerInit {+ const { collector, archive } = this.options;+ return {+ stateDir: this.options.stateDir,+ appVersion: this.options.appVersion,+ engineVersion: this.options.engineVersion,+ collector: {+ upload: Boolean(collector.upload),+ refreshAccessToken: Boolean(collector.refreshAccessToken),+ ...(collector.gatewayUrl !== undefined ? { gatewayUrl: collector.gatewayUrl } : {}),+ ...(collector.accessToken !== undefined ? { accessToken: collector.accessToken } : {}),+ },+ archive: {+ enabled: archive.enabled,+ excludedDirs: archive.excludedDirs,+ request: Boolean(archive.request),+ refreshAccessToken: Boolean(archive.refreshAccessToken),+ ...(archive.gatewayUrl !== undefined ? { gatewayUrl: archive.gatewayUrl } : {}),+ ...(archive.accessToken !== undefined ? { accessToken: archive.accessToken } : {}),+ ...(archive.baseIdleMs !== undefined ? { baseIdleMs: archive.baseIdleMs } : {}),+ ...(archive.baseMaxDeferMs !== undefined ? { baseMaxDeferMs: archive.baseMaxDeferMs } : {}),+ },+ };+ }++ private spawn(): void {+ this.current = "starting";+ let worker: Worker;+ try {+ worker = new Worker(workerUrl(), { workerData: this.workerInit() });+ } catch (error) {+ this.runLocally(error);+ return;+ }+ // The server's listener keeps the process alive; the worker never holds it open on its own.+ worker.unref();+ this.worker = worker;+ worker.on("message", (message: FromWorker) => this.onMessage(worker, message));+ worker.on("error", (error: unknown) => {+ if (worker === this.worker) this.options.log("warn", "OmniRush capture worker failed", { error: errorSummary(error) });+ });+ worker.on("exit", (code: number) => this.onExit(worker, code));+ }++ private onMessage(worker: Worker, message: FromWorker): void {+ if (worker !== this.worker) return;+ switch (message.kind) {+ case "ready": {+ this.current = "worker";+ const queued = this.queue;+ this.queue = [];+ for (const { call, transfer } of queued) this.post(worker, call, transfer);+ return;+ }+ case "log":+ this.options.log(message.level, message.message, message.attributes);+ return;+ case "closed":+ if (this.sessions.get(message.sessionId) === false) this.sessions.delete(message.sessionId);+ return;+ case "reply": {+ const resolvePromise = this.replies.get(message.id);+ this.replies.delete(message.id);+ resolvePromise?.(message.ok ? message.value : null);+ return;+ }+ case "abort":+ this.served.get(message.id)?.controller.abort();+ return;+ case "request":+ void this.serve(worker, message.id, message.channel, message.request);+ return;+ }+ }++ private onExit(worker: Worker, code: number): void {+ if (worker !== this.worker) return;+ this.worker = null;+ for (const { controller } of this.served.values()) controller.abort();+ this.settleOutstanding();+ if (this.stopping) return;+ if (this.current === "starting") {+ this.runLocally(new Error(`the capture worker exited with code ${code} before it was ready`));+ return;+ }+ // Its sessions are gone with it; each resumes on its next prompt.+ this.sessions.clear();+ if (this.restarts < MAX_WORKER_RESTARTS) {+ this.restarts += 1;+ this.options.log("warn", "OmniRush capture worker exited; starting a new one", { code, restarts: this.restarts });+ this.spawn();+ return;+ }+ this.current = "down";+ this.options.log("warn", "OmniRush capture worker exited too often; capture is off until the app restarts", { code });+ }++ /** Every caller waiting on the worker gets an empty result. */+ private settleOutstanding(): void {+ for (const resolvePromise of this.replies.values()) resolvePromise(null);+ this.replies.clear();+ this.served.clear();+ }++ /** No worker: the host runs on this thread, starting with the calls queued so far. */+ private runLocally(reason: unknown): void {+ if (reason !== null) {+ this.options.log("warn", "OmniRush capture worker could not start; capturing in-process", { error: errorSummary(reason) });+ }+ const { worker: _worker, ...hostOptions } = this.options;+ try {+ this.local = new CaptureHost(hostOptions);+ } catch (error) {+ // Only reached after a worker failed the same way: capture is off rather than the server down.+ if (reason === null) throw error;+ this.options.log("warn", "OmniRush capture could not start in-process either; capture is off until the app restarts", { error: errorSummary(error) });+ this.current = "down";+ this.queue = [];+ this.settleOutstanding();+ return;+ }+ this.current = "local";+ const queued = this.queue;+ this.queue = [];+ for (const { call } of queued) {+ const id = call.id;+ let value: unknown = null;+ try {+ value = invokeCapture(this.local, call);+ } catch (error) {+ this.options.log("warn", "OmniRush capture call failed", { method: call.method, error: errorSummary(error) });+ }+ if (id === null) continue;+ const resolvePromise = this.replies.get(id);+ this.replies.delete(id);+ Promise.resolve(value).then((settled) => resolvePromise?.(settled), () => resolvePromise?.(null));+ }+ }++ // --- the worker's requests ------------------------------------------------------------++ private async serve(worker: Worker, id: number, channel: RequestChannel, request: HostRequest): Promise<void> {+ const controller = new AbortController();+ // Shutting down: archive uploads were aborted, and none starts again.+ if (this.stopping && channel === "archive") controller.abort();+ this.served.set(id, { controller, channel });+ let result: RequestResult;+ try {+ if (controller.signal.aborted) throw new DOMException("The server is stopping", "AbortError");+ result = await this.perform(channel, request, controller.signal, id);+ } catch (error) {+ result = { kind: "result", id, ok: false, error: error instanceof Error ? error.message : String(error), name: error instanceof Error ? error.name : "Error" };+ } finally {+ this.served.delete(id);+ }+ if (worker !== this.worker) return;+ worker.postMessage(result satisfies ToWorker, result.ok ? transferables(result.response) : []);+ }++ private async perform(channel: RequestChannel, request: HostRequest, signal: AbortSignal, id: number): Promise<RequestResult> {+ const { collector, archive } = this.options;+ switch (request.type) {+ case "collect": {+ if (!collector.upload) throw new Error("no collector upload hook");+ return { kind: "result", id, ok: true, response: await serializeResponse(await collector.upload(request.sessionId, request.body)) };+ }+ case "refreshAccessToken": {+ const refresh = channel === "archive" ? archive.refreshAccessToken : collector.refreshAccessToken;+ return { kind: "result", id, ok: true, response: null, token: refresh ? await refresh() : null };+ }+ case "archiveRequest": {+ if (!archive.request) throw new Error("no archive request hook");+ const response = await archive.request(request.path, { method: request.method, ...(request.body !== undefined ? { body: request.body } : {}), signal });+ return { kind: "result", id, ok: true, response: await serializeResponse(response) };+ }+ case "fetch": {+ const egress = (channel === "archive" ? archive.fetch : collector.fetch) ?? externalFetch;+ const response = await egress(request.url, {+ method: request.method,+ headers: request.headers,+ ...(request.body !== undefined ? { body: request.body } : {}),+ signal,+ });+ return { kind: "result", id, ok: true, response: await serializeResponse(response) };+ }+ }+ }+}++function isDiagnostics(value: unknown): value is CaptureDiagnostics {+ return typeof value === "object" && value !== null && "metrics" in value && "cache" in value;+}++export function startCaptureService(options: CaptureServiceOptions): CaptureService {+ return new CaptureClient(options);+}diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.tsnew file mode 100644index 00000000..584696e9--- /dev/null+++ b/apps/server/src/capture-host.ts@@ -0,0 +1,178 @@+/**+ * Everything the embedded server captures about sessions, under one owner:+ * the workspace collector, the project archive and the turn observers. The+ * server runs it on a worker thread (capture-worker.ts, driven through+ * capture-client.ts), so reading, hashing and scrubbing files, building JSON+ * and tar streams, compressing and sealing never hold the main event loop+ * that serves the app; where no worker can start it runs in-process with the+ * same behaviour. Every call takes and returns plain data, so the same calls+ * cross the thread boundary unchanged.+ */+import { collectPromptAttachments, promptBodyForTrace, v2PromptBodyForTrace } from "./collector-attachments.js";+import {+ createSessionObservers,+ observeCollectedSession,+ projectArchiveEngineReads,+ type EngineTarget,+ type SessionObservers,+} from "./collector-observer.js";+import { SessionArchiver, type SessionArchiverOptions } from "./session-archive/index.js";+import { ProjectArchiveLifecycle, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";+import { WorkspaceCollector, type CollectorMetrics, type CollectorWebVisit } from "./workspace-collector.js";++export type { EngineTarget } from "./collector-observer.js";++export type CaptureLog = ArchiveLifecycleLog;++/** Prompt bodies larger than this are not parsed for the trace (nor for attachments). */+const MAX_TRACED_REQUEST_BYTES = 4 * 1024 * 1024;++/** A collected engine request, as the "engine.request" trace event and the prompt's attachments are built from it. */+export type PromptRecord = {+ method: string;+ path: string;+ /** The request body; null when there is none or it is larger than a traced body may be. */+ body: Uint8Array | null;+ engine: "v1" | "v2";+ /** The workspace root, where attached files copied into the workspace are read from. */+ root: string;+ /** A prompt dispatch: its file parts become "attachment" events. */+ attachments: boolean;+};++export type CaptureHostOptions = {+ /** The collector state dir (the archive keeps its files under it too). */+ stateDir: string;+ appVersion: string;+ engineVersion: string;+ log: CaptureLog;+ collector: {+ upload?: (sessionId: string, compressed: Uint8Array) => Promise<Response>;+ refreshAccessToken?: () => Promise<string | null>;+ fetch?: (input: string, init?: RequestInit) => Promise<Response>;+ gatewayUrl?: string;+ accessToken?: string;+ };+ archive: Pick<SessionArchiverOptions, "request" | "refreshAccessToken" | "fetch" | "gatewayUrl" | "accessToken"> & {+ /** Archiving is on for this device (OMNIRUSH_ARCHIVE_ENABLED) and an account is connected. */+ enabled: boolean;+ /** App data, config and cache directories: pruned under a project root, never archived as one. */+ excludedDirs: string[];+ baseIdleMs?: number;+ baseMaxDeferMs?: number;+ };+ /** A finished session's last upload settled (the in-process `hasSession` mirror of a worker follows it). */+ onSessionClosed?: (sessionId: string) => void;+};++export type CaptureDiagnostics = {+ metrics: CollectorMetrics;+ cache: { roots: number; entries: number };+};++function requestPayload(body: Uint8Array | null): unknown {+ if (!body || body.byteLength > MAX_TRACED_REQUEST_BYTES) return undefined;+ try {+ return JSON.parse(new TextDecoder().decode(body));+ } catch {+ return undefined;+ }+}++export class CaptureHost {+ readonly collector: WorkspaceCollector;+ readonly archive: ProjectArchiveLifecycle;+ private readonly observers: SessionObservers = createSessionObservers();++ constructor(options: CaptureHostOptions) {+ this.collector = new WorkspaceCollector({+ ...options.collector,+ stateDir: options.stateDir,+ appVersion: options.appVersion,+ engineVersion: options.engineVersion,+ log: options.log,+ ...(options.onSessionClosed ? { onSessionClosed: options.onSessionClosed } : {}),+ });+ const { enabled, excludedDirs, baseIdleMs, baseMaxDeferMs, ...auth } = options.archive;+ this.archive = new ProjectArchiveLifecycle({+ archiver: new SessionArchiver({ stateDir: options.stateDir, excludedDirs, log: options.log, ...auth }),+ enabled: enabled && this.collector.enabled,+ log: options.log,+ ...(baseIdleMs !== undefined ? { baseIdleMs } : {}),+ ...(baseMaxDeferMs !== undefined ? { baseMaxDeferMs } : {}),+ });+ this.archive.start();+ }++ startSession(sessionId: string, workspaceId: string, root: string): void {+ this.collector.startSession(sessionId, workspaceId, root);+ }++ recordTrace(sessionId: string, type: string, data?: unknown): void {+ this.collector.recordTrace(sessionId, type, data);+ }++ /** The "engine.request" event of a collected request and, for a prompt dispatch, one "attachment" event per attached file. */+ recordPrompt(sessionId: string, prompt: PromptRecord): void {+ const payload = requestPayload(prompt.body);+ this.collector.recordTrace(sessionId, "engine.request", {+ method: prompt.method,+ path: prompt.path,+ body: prompt.engine === "v2" ? v2PromptBodyForTrace(payload) : promptBodyForTrace(payload),+ });+ if (!prompt.attachments) return;+ void collectPromptAttachments(payload, prompt.root).then((attachments) => {+ for (const attachment of attachments) this.collector.recordAttachment(sessionId, attachment);+ }).catch(() => undefined);+ }++ captureSnapshot(sessionId: string, trigger: "prompt" | "turn_completed"): void {+ this.collector.captureSnapshot(sessionId, trigger);+ }++ recordWebVisit(sessionId: string, visit: CollectorWebVisit): boolean {+ return this.collector.recordWebVisit(sessionId, visit);+ }++ /** A prompt was dispatched: the project archive's session start (a base once per session, see lifecycle.ts). */+ archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void {+ this.archive.sessionStarted({ sessionId, root, engine: projectArchiveEngineReads(target, sessionId) });+ }++ /** The engine accepted a collected request: follow the session until its turn settles. */+ observeSession(sessionId: string, target: EngineTarget): void {+ observeCollectedSession({ collector: this.collector, archive: this.archive, observers: this.observers, sessionId, target });+ }++ /** The session was deleted in the engine. */+ sessionDeleted(sessionId: string): void {+ this.collector.recordTrace(sessionId, "session.deleted");+ this.collector.finishSession(sessionId);+ this.archive.sessionEnded(sessionId);+ this.observers.lastMessageIds.delete(sessionId);+ }++ /** The account is gone: queued archives and spooled uploads are deleted, nothing more is archived. */+ async signOut(): Promise<void> {+ await this.archive.signOut();+ await this.collector.clearSpool().catch(() => undefined);+ }++ /** Server shutdown: in-flight archive uploads abort first, then every session's trace and end snapshot go out. */+ async stop(): Promise<void> {+ const archiveStopped = this.archive.stop();+ this.observers.controller.abort();+ await this.collector.stop().catch(() => undefined);+ await archiveStopped;+ }++ /** Resolves once every queued capture, upload and archive step has settled (tests, profiling). */+ async idle(): Promise<void> {+ await this.collector.idleAll();+ await this.archive.settled();+ }++ diagnostics(): CaptureDiagnostics {+ return { metrics: { ...this.collector.metrics }, cache: this.collector.cacheStatus() };+ }+}diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.tsnew file mode 100644index 00000000..0db2ec01--- /dev/null+++ b/apps/server/src/capture-protocol.ts@@ -0,0 +1,135 @@+/**+ * The messages between the server's main thread and its capture worker+ * (capture-client.ts, capture-worker.ts). The main thread calls the+ * CaptureHost methods named in CaptureCalls; the worker asks the main thread+ * for what only it can do (the gateway broker's authenticated requests, and+ * external egress through the embedding runtime's network stack) and reports+ * logs and closed sessions. Everything is plain data; request and response+ * bodies travel as transferred buffers.+ */+import type { CaptureHost, EngineTarget, PromptRecord } from "./capture-host.js";+import type { CollectorWebVisit } from "./workspace-collector.js";++/** CaptureHost methods callable from the main thread, with their arguments. */+export type CaptureCalls = {+ startSession: [sessionId: string, workspaceId: string, root: string];+ recordTrace: [sessionId: string, type: string, data?: unknown];+ recordPrompt: [sessionId: string, prompt: PromptRecord];+ captureSnapshot: [sessionId: string, trigger: "prompt" | "turn_completed"];+ recordWebVisit: [sessionId: string, visit: CollectorWebVisit];+ archiveSessionStarted: [sessionId: string, root: string, target: EngineTarget];+ observeSession: [sessionId: string, target: EngineTarget];+ sessionDeleted: [sessionId: string];+ signOut: [];+ stop: [];+ idle: [];+ diagnostics: [];+};++export type CaptureCallName = keyof CaptureCalls;++/** One call; `id` is set when the caller waits for its result. */+export type CaptureCall = { [M in CaptureCallName]: { kind: "call"; id: number | null; method: M; args: CaptureCalls[M] } }[CaptureCallName];++/** What a worker asks of the main thread. `channel` says whose request it is: archive requests are aborted first at shutdown. */+export type HostRequest =+ | { type: "collect"; sessionId: string; body: Uint8Array<ArrayBuffer> }+ | { type: "refreshAccessToken" }+ | { type: "archiveRequest"; path: string; method: "GET" | "POST"; body?: string }+ | { type: "fetch"; url: string; method: string; headers: Array<[string, string]>; body?: Uint8Array<ArrayBuffer> | string };++export type RequestChannel = "collector" | "archive";++export type SerializedResponse = { status: number; statusText: string; headers: Array<[string, string]>; body: ArrayBuffer | null };++export type RequestResult =+ | { kind: "result"; id: number; ok: true; response: SerializedResponse | null; token?: string | null }+ | { kind: "result"; id: number; ok: false; error: string; name: string };++export type ToWorker = CaptureCall | RequestResult;++export type FromWorker =+ | { kind: "ready" }+ | { kind: "log"; level: "info" | "warn"; message: string; attributes?: Record<string, unknown> }+ | { kind: "closed"; sessionId: string }+ | { kind: "reply"; id: number; ok: true; value: unknown }+ | { kind: "reply"; id: number; ok: false; error: string }+ | { kind: "request"; id: number; channel: RequestChannel; request: HostRequest }+ | { kind: "abort"; id: number };++/** What the worker is built with (its workerData): the main thread's hooks become requests. */+export type CaptureWorkerInit = {+ stateDir: string;+ appVersion: string;+ engineVersion: string;+ collector: { upload: boolean; refreshAccessToken: boolean; gatewayUrl?: string; accessToken?: string };+ archive: {+ enabled: boolean;+ excludedDirs: string[];+ request: boolean;+ refreshAccessToken: boolean;+ gatewayUrl?: string;+ accessToken?: string;+ baseIdleMs?: number;+ baseMaxDeferMs?: number;+ };+};++export async function serializeResponse(response: Response): Promise<SerializedResponse> {+ const body = response.body ? await response.arrayBuffer() : null;+ return {+ status: response.status,+ statusText: response.statusText,+ headers: [...response.headers],+ body: body && body.byteLength > 0 ? body : null,+ };+}++const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);++export function deserializeResponse(value: SerializedResponse): Response {+ return new Response(NULL_BODY_STATUSES.has(value.status) ? null : value.body, {+ status: value.status,+ statusText: value.statusText,+ headers: value.headers,+ });+}++/** The buffers of a request or result that can be transferred instead of copied. */+export function transferables(value: HostRequest | SerializedResponse | null): ArrayBuffer[] {+ if (!value) return [];+ const body = "body" in value ? value.body : undefined;+ if (body instanceof ArrayBuffer) return [body];+ if (body instanceof Uint8Array && body.byteOffset === 0 && body.byteLength === body.buffer.byteLength) return [body.buffer];+ return [];+}++/** Runs one call on a host (the worker's, or the in-process fallback). */+export function invokeCapture(host: CaptureHost, call: CaptureCall): unknown {+ switch (call.method) {+ case "startSession":+ return host.startSession(...call.args);+ case "recordTrace":+ return host.recordTrace(...call.args);+ case "recordPrompt":+ return host.recordPrompt(...call.args);+ case "captureSnapshot":+ return host.captureSnapshot(...call.args);+ case "recordWebVisit":+ return host.recordWebVisit(...call.args);+ case "archiveSessionStarted":+ return host.archiveSessionStarted(...call.args);+ case "observeSession":+ return host.observeSession(...call.args);+ case "sessionDeleted":+ return host.sessionDeleted(...call.args);+ case "signOut":+ return host.signOut();+ case "stop":+ return host.stop();+ case "idle":+ return host.idle();+ case "diagnostics":+ return host.diagnostics();+ }+}diff --git a/apps/server/src/capture-worker.ts b/apps/server/src/capture-worker.tsnew file mode 100644index 00000000..077a4c0f--- /dev/null+++ b/apps/server/src/capture-worker.ts@@ -0,0 +1,174 @@+/**+ * The capture worker: a worker thread running the server's CaptureHost (the+ * workspace collector, the project archive and the turn observers), so their+ * CPU work never holds the main event loop. Started and driven by+ * capture-client.ts; the messages are defined in capture-protocol.ts. The+ * gateway broker's requests and external egress are made by the main thread+ * on this worker's behalf; everything else (file reads, git, the scrubber,+ * zstd, sealing, the loopback engine reads) happens here.+ */+import { parentPort, workerData, type MessagePort } from "node:worker_threads";++import { CaptureHost } from "./capture-host.js";+import {+ deserializeResponse,+ invokeCapture,+ transferables,+ type CaptureWorkerInit,+ type FromWorker,+ type HostRequest,+ type RequestChannel,+ type RequestResult,+ type ToWorker,+} from "./capture-protocol.js";++function mainThreadPort(): MessagePort {+ if (!parentPort) throw new Error("capture-worker runs as a worker thread only");+ return parentPort;+}++const port = mainThreadPort();+const init: CaptureWorkerInit = workerData;++function post(message: FromWorker, transfer: ArrayBuffer[] = []): void {+ port.postMessage(message, transfer);+}++let requestSeq = 0;+const pending = new Map<number, (result: RequestResult) => void>();++function abortError(signal: AbortSignal): Error {+ return signal.reason instanceof Error ? signal.reason : new DOMException("This operation was aborted", "AbortError");+}++/** Asks the main thread to perform `request`; an abort of `signal` cancels it there too. */+function ask(channel: RequestChannel, request: HostRequest, signal?: AbortSignal | null): Promise<RequestResult & { ok: true }> {+ if (signal?.aborted) return Promise.reject(abortError(signal));+ const id = ++requestSeq;+ return new Promise((resolvePromise, reject) => {+ const onAbort = () => {+ pending.delete(id);+ post({ kind: "abort", id });+ if (signal) reject(abortError(signal));+ };+ signal?.addEventListener("abort", onAbort, { once: true });+ pending.set(id, (result) => {+ signal?.removeEventListener("abort", onAbort);+ if (result.ok) resolvePromise(result);+ else {+ const error = new Error(result.error);+ error.name = result.name;+ reject(error);+ }+ });+ post({ kind: "request", id, channel, request }, transferables(request));+ });+}++/** A private copy of a body, so the caller's buffer stays usable (a retry sends it again) while the copy is transferred. */+function ownedBody(body: RequestInit["body"]): Uint8Array<ArrayBuffer> | string | undefined {+ if (body === undefined || body === null) return undefined;+ if (typeof body === "string") return body;+ if (body instanceof ArrayBuffer) return new Uint8Array(body.slice(0));+ if (ArrayBuffer.isView(body)) return new Uint8Array(body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength));+ throw new TypeError("capture worker requests carry string or byte bodies only");+}++/** External egress through the main thread's runtime (Electron's network stack in the desktop app). */+function hostFetch(channel: RequestChannel) {+ return async (url: string, requestInit: RequestInit = {}): Promise<Response> => {+ const body = ownedBody(requestInit.body);+ const result = await ask(channel, {+ type: "fetch",+ url,+ method: requestInit.method ?? "GET",+ headers: [...new Headers(requestInit.headers)],+ ...(body !== undefined ? { body } : {}),+ }, requestInit.signal);+ if (!result.response) throw new Error("the main thread returned no response");+ return deserializeResponse(result.response);+ };+}++function refreshAccessToken(channel: RequestChannel) {+ return async (): Promise<string | null> => (await ask(channel, { type: "refreshAccessToken" })).token ?? null;+}++const log = (level: "info" | "warn", message: string, attributes?: Record<string, unknown>) => {+ post({ kind: "log", level, message, ...(attributes ? { attributes } : {}) });+};++const host = new CaptureHost({+ stateDir: init.stateDir,+ appVersion: init.appVersion,+ engineVersion: init.engineVersion,+ log,+ collector: {+ ...(init.collector.upload+ ? {+ upload: async (sessionId: string, compressed: Uint8Array) => {+ const body = new Uint8Array(compressed);+ const result = await ask("collector", { type: "collect", sessionId, body });+ if (!result.response) throw new Error("the main thread returned no response");+ return deserializeResponse(result.response);+ },+ }+ : {}),+ ...(init.collector.refreshAccessToken ? { refreshAccessToken: refreshAccessToken("collector") } : {}),+ fetch: hostFetch("collector"),+ ...(init.collector.gatewayUrl !== undefined ? { gatewayUrl: init.collector.gatewayUrl } : {}),+ ...(init.collector.accessToken !== undefined ? { accessToken: init.collector.accessToken } : {}),+ },+ archive: {+ enabled: init.archive.enabled,+ excludedDirs: init.archive.excludedDirs,+ ...(init.archive.request+ ? {+ request: async (path: string, requestInit: { method: "GET" | "POST"; body?: string; signal?: AbortSignal }) => {+ const result = await ask("archive", {+ type: "archiveRequest",+ path,+ method: requestInit.method,+ ...(requestInit.body !== undefined ? { body: requestInit.body } : {}),+ }, requestInit.signal);+ if (!result.response) throw new Error("the main thread returned no response");+ return deserializeResponse(result.response);+ },+ }+ : {}),+ ...(init.archive.refreshAccessToken ? { refreshAccessToken: refreshAccessToken("archive") } : {}),+ fetch: hostFetch("archive"),+ ...(init.archive.gatewayUrl !== undefined ? { gatewayUrl: init.archive.gatewayUrl } : {}),+ ...(init.archive.accessToken !== undefined ? { accessToken: init.archive.accessToken } : {}),+ ...(init.archive.baseIdleMs !== undefined ? { baseIdleMs: init.archive.baseIdleMs } : {}),+ ...(init.archive.baseMaxDeferMs !== undefined ? { baseMaxDeferMs: init.archive.baseMaxDeferMs } : {}),+ },+ onSessionClosed: (sessionId) => post({ kind: "closed", sessionId }),+});++port.on("message", (message: ToWorker) => {+ if (message.kind === "result") {+ const settle = pending.get(message.id);+ pending.delete(message.id);+ settle?.(message);+ return;+ }+ const { id } = message;+ let value: unknown;+ try {+ value = invokeCapture(host, message);+ } catch (error) {+ if (id !== null) post({ kind: "reply", id, ok: false, error: error instanceof Error ? error.message : String(error) });+ return;+ }+ if (id === null) {+ if (value instanceof Promise) value.catch(() => undefined);+ return;+ }+ Promise.resolve(value).then(+ (settled) => post({ kind: "reply", id, ok: true, value: settled ?? null }),+ (error: unknown) => post({ kind: "reply", id, ok: false, error: error instanceof Error ? error.message : String(error) }),+ );+});++post({ kind: "ready" });diff --git a/apps/server/src/collector-attachments.ts b/apps/server/src/collector-attachments.tsindex 8bf601c4..50ba7798 100644--- a/apps/server/src/collector-attachments.ts+++ b/apps/server/src/collector-attachments.ts@@ -244,3 +244,22 @@ export function promptBodyForTrace(payload: unknown): unknown {}return payload;}++/**+ * A v2 prompt body as recorded in the "engine.request" trace event. Inline+ * file attachments (`files[].uri` data URLs, at the top level or under+ * `prompt`) are replaced by a marker, as promptBodyForTrace does for v1 parts.+ */+export function v2PromptBodyForTrace(payload: unknown): unknown {+ const stripFiles = (value: unknown): unknown => {+ if (!isRecord(value) || !Array.isArray(value.files)) return value;+ const files = value.files.map((file) => {+ if (!isRecord(file) || typeof file.uri !== "string" || !file.uri.startsWith("data:")) return file;+ const mime = (file.uri.slice(5).split(",")[0] ?? "").split(";")[0] || "application/octet-stream";+ return { ...file, uri: `data:${mime};omitted`, omitted_uri_chars: file.uri.length };+ });+ return { ...value, files };+ };+ const stripped = stripFiles(promptBodyForTrace(payload));+ return isRecord(stripped) && isRecord(stripped.prompt) ? { ...stripped, prompt: stripFiles(stripped.prompt) } : stripped;+}diff --git a/apps/server/src/collector-observer.ts b/apps/server/src/collector-observer.tsnew file mode 100644index 00000000..2f51ddc7--- /dev/null+++ b/apps/server/src/collector-observer.ts@@ -0,0 +1,344 @@+/**+ * The engine reads behind collection: the turn observer that waits for a+ * collected session to settle and hands its messages, model, subagents and+ * turn milestone to the collector and the project archive, and the reads a+ * project archive session start makes. They run beside the collector (on+ * the capture worker when there is one, see capture-host.ts), so parsing a+ * long transcript never holds the server's main event loop. An engine is+ * named by plain data (EngineTarget) so the target can cross threads.+ */+import { loopbackFetch } from "./server-fetch.js";+import { isFinishedAssistantMessage, type ArchiveEngineReads, type ProjectArchiveLifecycle } from "./session-archive/lifecycle.js";+import { MAX_COLLECTOR_CHILD_SESSION_DEPTH, type CollectorSessionModel, type WorkspaceCollector } from "./workspace-collector.js";++/** The engine a collected request went to: base URL, request headers (the engine's auth), query and API generation. */+export type EngineTarget = {+ baseUrl: string;+ headers: Array<[string, string]>;+ search: string;+ /** The v2 daemon reports activity and context through its own routes, wrapped in `data`. */+ engine: "v1" | "v2";+};++/** Turn observers of one server: the sessions being observed, their last seen message, and the stop signal. */+export type SessionObservers = {+ sessions: Set<string>;+ lastMessageIds: Map<string, string>;+ controller: AbortController;+};++export function createSessionObservers(): SessionObservers {+ return { sessions: new Set(), lastMessageIds: new Map(), controller: new AbortController() };+}++/** An engine target from a proxied request's headers, without the body headers of that request. */+export function engineTarget(baseUrl: string, headers: Headers, search: string, engine: "v1" | "v2" = "v1"): EngineTarget {+ const copy = new Headers(headers);+ copy.delete("content-length");+ copy.delete("content-type");+ return { baseUrl, headers: [...copy], search, engine };+}++export function buildOpencodeProxyUrl(baseUrl: string, path: string, search: string) {+ const target = new URL(baseUrl);+ const trimmedPath = path.replace(/^\/opencode/, "");+ target.pathname = trimmedPath.startsWith("/") ? trimmedPath : `/${trimmedPath}`;+ target.search = search;+ return target.toString();+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function optionalTraceString(value: unknown): string | null {+ return typeof value === "string" && value.trim() ? value.trim() : null;+}++/**+ * The model the turn ran on, read from the turn's first assistant message+ * (providerID / modelID / mode or agent). The user message that opened the+ * turn supplies the variant and agent when the assistant record lacks them.+ */+function turnModelFromMessages(messages: unknown): CollectorSessionModel | null {+ if (!Array.isArray(messages)) return null;+ let userVariant: string | null = null;+ let userAgent: string | null = null;+ for (const message of messages) {+ if (!isRecord(message)) continue;+ const info = isRecord(message.info) ? message.info : message;+ const model = isRecord(info.model) ? info.model : {};+ // v2 context messages carry the role as `type` and the model as a ModelRef.+ const role = info.role ?? info.type;+ if (role === "user") {+ userVariant = optionalTraceString(info.variant) ?? optionalTraceString(model.variant) ?? userVariant;+ userAgent = optionalTraceString(info.agent) ?? userAgent;+ continue;+ }+ if (role !== "assistant") continue;+ const providerId = optionalTraceString(info.providerID) ?? optionalTraceString(model.providerID);+ const modelId = optionalTraceString(info.modelID) ?? optionalTraceString(model.modelID) ?? optionalTraceString(model.id);+ if (!providerId && !modelId) continue;+ return {+ provider_id: providerId,+ model_id: modelId,+ variant: optionalTraceString(info.variant) ?? optionalTraceString(model.variant) ?? userVariant,+ agent: optionalTraceString(info.agent) ?? optionalTraceString(info.mode) ?? userAgent,+ };+ }+ return null;+}++/**+ * Walks the task-tool subagent tree below a settled root session: one+ * "session.child" event per child carrying only the messages that are new+ * since that child's checkpoint, recursively for grandchildren.+ */+async function captureChildSessions(input: {+ collector: WorkspaceCollector;+ rootSessionId: string;+ parentSessionId: string;+ depth: number;+ fetchJson: (path: string, maxBytes: number) => Promise<unknown>;+ checkpoints: Record<string, string>;+ known: Set<string>;+ visited: Set<string>;+}): Promise<void> {+ if (input.depth > MAX_COLLECTOR_CHILD_SESSION_DEPTH) return;+ const children = await input.fetchJson(`/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024);+ if (!Array.isArray(children)) return;+ for (const child of children.slice(0, 200)) {+ const childId = isRecord(child) && typeof child.id === "string" && child.id ? child.id : null;+ if (!childId || childId === input.rootSessionId || input.visited.has(childId)) continue;+ input.visited.add(childId);+ const messages = await input.fetchJson(`/session/${encodeURIComponent(childId)}/message`, 8 * 1024 * 1024);+ const list = Array.isArray(messages) ? messages : [];+ const delta = newTraceMessages(list, input.checkpoints[childId]);+ const newMessages = Array.isArray(delta) ? delta : [];+ if (newMessages.length > 0 || !input.known.has(childId)) {+ input.collector.recordChildSession(input.rootSessionId, {+ childSessionId: childId,+ parentSessionId: input.parentSessionId,+ title: isRecord(child) ? optionalTraceString(child.title) : null,+ agent: turnModelFromMessages(list)?.agent ?? null,+ messages: newMessages,+ lastMessageId: traceMessageId(list.at(-1)),+ });+ }+ await captureChildSessions({ ...input, parentSessionId: childId, depth: input.depth + 1 });+ }+}++function collectorDelay(ms: number, signal: AbortSignal): Promise<void> {+ return new Promise((resolvePromise, reject) => {+ if (signal.aborted) {+ reject(signal.reason);+ return;+ }+ const timer = setTimeout(resolvePromise, ms);+ timer.unref?.();+ signal.addEventListener("abort", () => {+ clearTimeout(timer);+ reject(signal.reason);+ }, { once: true });+ });+}++async function readCollectorResponse(response: Response, maxBytes = 8 * 1024 * 1024): Promise<unknown> {+ const reader = response.body?.getReader();+ if (!reader) return null;+ const chunks: Uint8Array[] = [];+ let total = 0;+ try {+ while (true) {+ const { done, value } = await reader.read();+ if (done) break;+ total += value.byteLength;+ if (total > maxBytes) throw new Error("trace response exceeded local limit");+ chunks.push(value);+ }+ } finally {+ reader.releaseLock();+ }+ const bytes = new Uint8Array(total);+ let offset = 0;+ for (const chunk of chunks) {+ bytes.set(chunk, offset);+ offset += chunk.byteLength;+ }+ return JSON.parse(new TextDecoder().decode(bytes));+}++function traceHasTerminalAssistant(messages: unknown): boolean {+ if (!Array.isArray(messages)) return false;+ for (let index = messages.length - 1; index >= 0; index -= 1) {+ const message = messages[index];+ if (!isRecord(message)) continue;+ const info = isRecord(message.info) ? message.info : message;+ if ((info.role ?? info.type) !== "assistant") continue;+ return isFinishedAssistantMessage(message);+ }+ return false;+}++function traceMessageId(message: unknown): string | null {+ if (!isRecord(message)) return null;+ const info = isRecord(message.info) ? message.info : message;+ return typeof info.id === "string" && info.id ? info.id : null;+}++function newTraceMessages(messages: unknown, previousId: string | undefined): unknown {+ if (!Array.isArray(messages) || !previousId) return messages;+ const index = messages.findIndex((message) => traceMessageId(message) === previousId);+ return index >= 0 ? messages.slice(index + 1) : messages;+}++/**+ * The engine reads behind a project archive session start (is it a child+ * session, how many turns has it completed), made like the collector+ * observer's: same engine, headers and query, never from the request path.+ */+export function projectArchiveEngineReads(target: EngineTarget, sessionId: string): ArchiveEngineReads {+ const v2 = target.engine === "v2";+ const read = async (path: string, maxBytes: number): Promise<unknown> => {+ const response = await loopbackFetch(buildOpencodeProxyUrl(target.baseUrl, path, target.search), {+ headers: new Headers(target.headers),+ signal: AbortSignal.timeout(20_000),+ });+ if (!response.ok) {+ await response.body?.cancel().catch(() => undefined);+ return null;+ }+ const payload = await readCollectorResponse(response, maxBytes);+ return v2 && isRecord(payload) && "data" in payload ? payload.data : payload;+ };+ const session = `${v2 ? "/api/session" : "/session"}/${encodeURIComponent(sessionId)}`;+ return {+ session: () => read(session, 1024 * 1024),+ // The same message list the observer counts turns from at turn end.+ messages: () => read(v2 ? `${session}/context` : `${session}/message`, 8 * 1024 * 1024),+ };+}++/**+ * Follows a collected session until its turn settles, then records the+ * turn's model, subagents and messages, takes the turn's change snapshot,+ * flushes the trace and hands the engine's messages to the project archive.+ */+export function observeCollectedSession(input: {+ collector: WorkspaceCollector;+ archive: Pick<ProjectArchiveLifecycle, "turnCompleted">;+ observers: SessionObservers;+ sessionId: string;+ target: EngineTarget;+}) {+ if (!input.collector.enabled) return;+ const observer = input.observers;+ if (observer.sessions.has(input.sessionId)) return;+ observer.sessions.add(input.sessionId);+ const { baseUrl, search } = input.target;+ const v2 = input.target.engine === "v2";+ const headers = new Headers(input.target.headers);+ const statusUrl = buildOpencodeProxyUrl(baseUrl, v2 ? "/api/session/active" : "/session/status", search);+ const messagesUrl = buildOpencodeProxyUrl(+ baseUrl,+ v2 ? `/api/session/${encodeURIComponent(input.sessionId)}/context` : `/session/${encodeURIComponent(input.sessionId)}/message`,+ search,+ );+ const enginePayload = (payload: unknown) => (v2 && isRecord(payload) && "data" in payload ? payload.data : payload);+ void (async () => {+ let observedBusy = false;+ let consecutiveSettled = 0;+ const checkpoint = await input.collector.sessionCheckpoint(input.sessionId);+ if (checkpoint.lastMessageId) observer.lastMessageIds.set(input.sessionId, checkpoint.lastMessageId);+ for (let attempt = 0; attempt < 3_600; attempt += 1) {+ await collectorDelay(1_000, observer.controller.signal);+ const response = await loopbackFetch(statusUrl, {+ headers,+ signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(10_000)]),+ });+ if (!response.ok) {+ await response.body?.cancel().catch(() => undefined);+ consecutiveSettled = 0;+ continue;+ }+ const statuses = enginePayload(await readCollectorResponse(response, 1024 * 1024));+ const session = isRecord(statuses) ? statuses[input.sessionId] : undefined;+ const statusType = isRecord(session) && typeof session.type === "string" ? session.type : "idle";+ if (statusType !== "idle") {+ observedBusy = true;+ consecutiveSettled = 0;+ continue;+ }+ const messagesResponse = await loopbackFetch(messagesUrl, {+ headers,+ signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]),+ });+ const messages = messagesResponse.ok+ ? enginePayload(await readCollectorResponse(messagesResponse))+ : { status: messagesResponse.status, unavailable: true };+ const terminal = traceHasTerminalAssistant(messages);+ consecutiveSettled += 1;+ if ((!observedBusy && !terminal) || consecutiveSettled < 2 || attempt < 3) continue;+ const delta = newTraceMessages(messages, observer.lastMessageIds.get(input.sessionId));+ if (Array.isArray(messages)) {+ const lastId = traceMessageId(messages.at(-1));+ if (lastId) {+ observer.lastMessageIds.set(input.sessionId, lastId);+ void input.collector.setSessionCheckpoint(input.sessionId, lastId);+ }+ }+ const model = turnModelFromMessages(delta);+ if (model) input.collector.recordSessionModel(input.sessionId, model);+ // The v2 daemon has no subagent children route; only its root turn is captured.+ if (!v2) {+ try {+ await captureChildSessions({+ collector: input.collector,+ rootSessionId: input.sessionId,+ parentSessionId: input.sessionId,+ depth: 1,+ fetchJson: async (path, maxBytes) => {+ const response = await loopbackFetch(buildOpencodeProxyUrl(baseUrl, path, search), {+ headers,+ signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]),+ });+ if (!response.ok) {+ await response.body?.cancel().catch(() => undefined);+ return null;+ }+ return readCollectorResponse(response, maxBytes);+ },+ checkpoints: await input.collector.childCheckpoints(input.sessionId),+ known: new Set(await input.collector.childSessionIds(input.sessionId)),+ visited: new Set([input.sessionId]),+ });+ } catch (error) {+ if (observer.controller.signal.aborted) throw error;+ input.collector.recordTrace(input.sessionId, "session.children_failed", {+ error: error instanceof Error ? error.message : "unknown",+ });+ }+ }+ input.collector.recordTrace(input.sessionId, "session.idle", { status: statusType });+ // The turn snapshot runs first so the artifacts it discovers are part of+ // the trace flushed right behind it.+ input.collector.captureSnapshot(input.sessionId, "turn_completed");+ input.collector.flushTrace(input.sessionId, { messages: delta });+ // The delta's turn number is the engine's completed-turn count, which survives app restarts.+ input.archive.turnCompleted(input.sessionId, messages);+ return;+ }+ input.collector.recordTrace(input.sessionId, "session.observer_timeout");+ input.collector.flushTrace(input.sessionId);+ })().catch((error: unknown) => {+ if (!observer.controller.signal.aborted) {+ input.collector.recordTrace(input.sessionId, "session.observer_failed", {+ error: error instanceof Error ? error.message : "unknown",+ });+ input.collector.flushTrace(input.sessionId);+ }+ }).finally(() => {+ observer.sessions.delete(input.sessionId);+ });+}diff --git a/apps/server/src/project-archive.ts b/apps/server/src/project-archive.tsindex f7c04e8d..bbd17ad5 100644--- a/apps/server/src/project-archive.ts+++ b/apps/server/src/project-archive.ts@@ -14,6 +14,16 @@ import { SessionArchiver, type SessionArchiverOptions } from "./session-archive/import { ProjectArchiveLifecycle, projectArchiveEnabled, type ArchiveLifecycleLog } from "./session-archive/lifecycle.js";import type { ServerConfig } from "./types.js";+/**+ * A base archive is packed once no prompt has been dispatched on any session+ * for this long, and at the latest PROJECT_ARCHIVE_BASE_MAX_DEFER_MS after+ * its own prompt: chats opened and prompted one after another are packed once+ * the burst is over, and a lone chat's base still shows the folder as its+ * first turn began (session-archive/lifecycle.ts).+ */+export const PROJECT_ARCHIVE_BASE_IDLE_MS = 2_000;+export const PROJECT_ARCHIVE_BASE_MAX_DEFER_MS = 10_000;+/** App data, config and cache directories: pruned when they sit under a project root, and never archived as a root. */function appDirectories(config: ServerConfig): string[] {return [@@ -25,37 +35,65 @@ function appDirectories(config: ServerConfig): string[] {];}-export function createProjectArchive(input: {+type ProjectArchiveInput = {config: ServerConfig;gatewayBroker: Pick<OmniRushGatewayBroker, "enabled" | "archiveRequest" | "refreshAccessToken">;/** The workspace collector has an account to upload to. */collectorEnabled: boolean;- log: ArchiveLifecycleLog;env?: NodeJS.ProcessEnv;+};++/**+ * Where and how a server's archiver works: its state dir, the directories it+ * prunes, whether it is on, and how it authenticates. Disabled (signed out,+ * or turned off on this device) it gets no bearer, so clearing what a+ * previous run left never reaches the network.+ */+export function projectArchiveSettings(input: ProjectArchiveInput): {+ stateDir: string;+ excludedDirs: string[];+ enabled: boolean;+ auth: "broker" | "environment" | "none";+ gatewayUrl?: string;+ accessToken?: string;+} {+ const env = input.env ?? process.env;+ const enabled = projectArchiveEnabled(env) && input.collectorEnabled;+ const base = { stateDir: runtimeStorageDir(input.config), excludedDirs: appDirectories(input.config), enabled };+ if (!enabled) return { ...base, auth: "none" };+ if (input.gatewayBroker.enabled) return { ...base, auth: "broker" };+ return {+ ...base,+ auth: "environment",+ ...(env.OMNIRUSH_GATEWAY_URL !== undefined ? { gatewayUrl: env.OMNIRUSH_GATEWAY_URL } : {}),+ ...(env.OMNIRUSH_ACCESS_TOKEN !== undefined ? { accessToken: env.OMNIRUSH_ACCESS_TOKEN } : {}),+ };+}++/** The archive lifecycle in this thread (the server itself runs it on its capture worker, see capture-host.ts). */+export function createProjectArchive(input: ProjectArchiveInput & {+ log: ArchiveLifecycleLog;/** Tests: S3 part uploads (and API calls in the environment-token mode). */fetch?: SessionArchiverOptions["fetch"];}): ProjectArchiveLifecycle {- const env = input.env ?? process.env;- const enabled = projectArchiveEnabled(env) && input.collectorEnabled;- // Disabled (signed out, or turned off on this device): no bearer, so- // clearing what a previous run left never reaches the network.- const auth: Partial<SessionArchiverOptions> = !enabled- ? {}- : input.gatewayBroker.enabled- ? {- request: (path, init) => input.gatewayBroker.archiveRequest(path, init),- refreshAccessToken: () => input.gatewayBroker.refreshAccessToken(),- }- : { gatewayUrl: env.OMNIRUSH_GATEWAY_URL, accessToken: env.OMNIRUSH_ACCESS_TOKEN };+ const settings = projectArchiveSettings(input);+ const auth: Partial<SessionArchiverOptions> = settings.auth === "broker"+ ? {+ request: (path, init) => input.gatewayBroker.archiveRequest(path, init),+ refreshAccessToken: () => input.gatewayBroker.refreshAccessToken(),+ }+ : settings.auth === "environment"+ ? { gatewayUrl: settings.gatewayUrl, accessToken: settings.accessToken }+ : {};return new ProjectArchiveLifecycle({archiver: new SessionArchiver({- stateDir: runtimeStorageDir(input.config),- excludedDirs: appDirectories(input.config),+ stateDir: settings.stateDir,+ excludedDirs: settings.excludedDirs,log: input.log,...auth,...(input.fetch ? { fetch: input.fetch } : {}),}),- enabled,+ enabled: settings.enabled,log: input.log,});}diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex fe67088d..94da9dbc 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -157,11 +157,11 @@ import { findManagedEngineWorkspace } from "./workspaces.js";import { startThreadApprovalReplayer, type ThreadApprovalReplayer } from "./thread-approvals.js";import { CloudProviderSync, parseCloudProviderDenSession } from "./cloud-provider-sync.js";import { createEngineV2Preview, type EngineV2Preview } from "./engine-v2-preview.js";-import { MAX_COLLECTOR_CHILD_SESSION_DEPTH, WorkspaceCollector, type CollectorSessionModel } from "./workspace-collector.js";-import { collectPromptAttachments, promptBodyForTrace } from "./collector-attachments.js";+import { workspaceCollectorEnabled } from "./workspace-collector.js";+import { startCaptureService, type CaptureService } from "./capture-client.js";+import { buildOpencodeProxyUrl, engineTarget } from "./collector-observer.js";import { OmniRushGatewayBroker } from "./omnirush-gateway-broker.js";-import { createProjectArchive } from "./project-archive.js";-import { isFinishedAssistantMessage, type ArchiveEngineReads, type ProjectArchiveLifecycle } from "./session-archive/lifecycle.js";+import { PROJECT_ARCHIVE_BASE_IDLE_MS, PROJECT_ARCHIVE_BASE_MAX_DEFER_MS, projectArchiveSettings } from "./project-archive.js";import { runtimeStorageDir } from "./runtime-db.js";import pkg from "../package.json" with { type: "json" };import constants from "../../../constants.json" with { type: "json" };@@ -180,13 +180,7 @@ let desktopCloudSyncQueue: Promise<void> = Promise.resolve();const agentDiagnosticsLastRunByServer = new WeakMap<ServerConfig, Map<string, number>>();const agentDiagnosticsInFlightByServer = new WeakMap<ServerConfig, Set<string>>();const commandAdmissionsByServer = new WeakMap<ServerConfig, Map<string, { fingerprint: string; admittedAt: number }>>();-const workspaceCollectorsByServer = new WeakMap<ServerConfig, WorkspaceCollector>();-const projectArchivesByServer = new WeakMap<ServerConfig, ProjectArchiveLifecycle>();-const collectorObserversByServer = new WeakMap<ServerConfig, {- sessions: Set<string>;- lastMessageIds: Map<string, string>;- controller: AbortController;-}>();+const captureServicesByServer = new WeakMap<ServerConfig, CaptureService>();const AGENT_DIAGNOSTICS_RATE_LIMIT_CAPACITY = 1_000;const AGENT_DIAGNOSTICS_MAX_IN_FLIGHT_PER_SERVER = 16;const AGENT_DIAGNOSTICS_MAX_REQUEST_BYTES = 256 * 1024;@@ -766,7 +760,7 @@ function collectorAccountRequiredResponse(): Response {*/function collectorDispatchRefused(config: ServerConfig, workspace: WorkspaceInfo | undefined): boolean {if (!workspace || workspace.workspaceType === "remote") return false;- return !workspaceCollectorsByServer.get(config)?.enabled && !collectorGateBypassed();+ return !captureServicesByServer.get(config)?.collectorEnabled && !collectorGateBypassed();}/** The gate as an ApiError, for dispatches that are not proxied HTTP requests. */@@ -781,25 +775,6 @@ function isCollectorV2PromptDispatch(method: string, routePath: string): booleanreturn method === "POST" && /^\/api\/session\/[^/]+\/(?:prompt|prompt_async|command|generate)$/.test(routePath.replace(/\/+$/, ""));}-/**- * A v2 prompt body as recorded in the "engine.request" trace event. Inline- * file attachments (`files[].uri` data URLs, at the top level or under- * `prompt`) are replaced by a marker, as promptBodyForTrace does for v1 parts.- */-function v2PromptBodyForTrace(payload: unknown): unknown {- const stripFiles = (value: unknown): unknown => {- if (!isRecord(value) || !Array.isArray(value.files)) return value;- const files = value.files.map((file) => {- if (!isRecord(file) || typeof file.uri !== "string" || !file.uri.startsWith("data:")) return file;- const mime = (file.uri.slice(5).split(",")[0] ?? "").split(";")[0] || "application/octet-stream";- return { ...file, uri: `data:${mime};omitted`, omitted_uri_chars: file.uri.length };- });- return { ...value, files };- };- const stripped = stripFiles(promptBodyForTrace(payload));- return isRecord(stripped) && isRecord(stripped.prompt) ? { ...stripped, prompt: stripFiles(stripped.prompt) } : stripped;-}-/*** Workflow steps are prompt dispatches like any other: they enter the same* engine proxy as the app's requests, so the gate, the collector's session@@ -822,84 +797,6 @@ export function createLocalWorkflowPromptDispatcher(config: ServerConfig): Local};}-function optionalTraceString(value: unknown): string | null {- return typeof value === "string" && value.trim() ? value.trim() : null;-}--/**- * The model the turn ran on, read from the turn's first assistant message- * (providerID / modelID / mode or agent). The user message that opened the- * turn supplies the variant and agent when the assistant record lacks them.- */-function turnModelFromMessages(messages: unknown): CollectorSessionModel | null {- if (!Array.isArray(messages)) return null;- let userVariant: string | null = null;- let userAgent: string | null = null;- for (const message of messages) {- if (!isRecord(message)) continue;- const info = isRecord(message.info) ? message.info : message;- const model = isRecord(info.model) ? info.model : {};- // v2 context messages carry the role as `type` and the model as a ModelRef.- const role = info.role ?? info.type;- if (role === "user") {- userVariant = optionalTraceString(info.variant) ?? optionalTraceString(model.variant) ?? userVariant;- userAgent = optionalTraceString(info.agent) ?? userAgent;- continue;- }- if (role !== "assistant") continue;- const providerId = optionalTraceString(info.providerID) ?? optionalTraceString(model.providerID);- const modelId = optionalTraceString(info.modelID) ?? optionalTraceString(model.modelID) ?? optionalTraceString(model.id);- if (!providerId && !modelId) continue;- return {- provider_id: providerId,- model_id: modelId,- variant: optionalTraceString(info.variant) ?? optionalTraceString(model.variant) ?? userVariant,- agent: optionalTraceString(info.agent) ?? optionalTraceString(info.mode) ?? userAgent,- };- }- return null;-}--/**- * Walks the task-tool subagent tree below a settled root session: one- * "session.child" event per child carrying only the messages that are new- * since that child's checkpoint, recursively for grandchildren.- */-async function captureChildSessions(input: {- collector: WorkspaceCollector;- rootSessionId: string;- parentSessionId: string;- depth: number;- fetchJson: (path: string, maxBytes: number) => Promise<unknown>;- checkpoints: Record<string, string>;- known: Set<string>;- visited: Set<string>;-}): Promise<void> {- if (input.depth > MAX_COLLECTOR_CHILD_SESSION_DEPTH) return;- const children = await input.fetchJson(`/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024);- if (!Array.isArray(children)) return;- for (const child of children.slice(0, 200)) {- const childId = isRecord(child) && typeof child.id === "string" && child.id ? child.id : null;- if (!childId || childId === input.rootSessionId || input.visited.has(childId)) continue;- input.visited.add(childId);- const messages = await input.fetchJson(`/session/${encodeURIComponent(childId)}/message`, 8 * 1024 * 1024);- const list = Array.isArray(messages) ? messages : [];- const delta = newTraceMessages(list, input.checkpoints[childId]);- const newMessages = Array.isArray(delta) ? delta : [];- if (newMessages.length > 0 || !input.known.has(childId)) {- input.collector.recordChildSession(input.rootSessionId, {- childSessionId: childId,- parentSessionId: input.parentSessionId,- title: isRecord(child) ? optionalTraceString(child.title) : null,- agent: turnModelFromMessages(list)?.agent ?? null,- messages: newMessages,- lastMessageId: traceMessageId(list.at(-1)),- });- }- await captureChildSessions({ ...input, parentSessionId: childId, depth: input.depth + 1 });- }-}-// The session a collected engine request belongs to, read from the decoded// route path exactly as the engine reads its `:id` parameter. Null when the// path is not a session request or its identifier is malformed.@@ -914,237 +811,6 @@ function collectorDeletedSessionId(method: string, proxyPath: string): string |return match?.[1] ? decodeEngineRouteParam(match[1]) : null;}-function collectorRequestPayload(body: ArrayBuffer | undefined): unknown {- if (!body || body.byteLength > 4 * 1024 * 1024) return undefined;- try {- return JSON.parse(new TextDecoder().decode(body));- } catch {- return undefined;- }-}--function collectorDelay(ms: number, signal: AbortSignal): Promise<void> {- return new Promise((resolvePromise, reject) => {- if (signal.aborted) {- reject(signal.reason);- return;- }- const timer = setTimeout(resolvePromise, ms);- timer.unref?.();- signal.addEventListener("abort", () => {- clearTimeout(timer);- reject(signal.reason);- }, { once: true });- });-}--async function readCollectorResponse(response: Response, maxBytes = 8 * 1024 * 1024): Promise<unknown> {- const reader = response.body?.getReader();- if (!reader) return null;- const chunks: Uint8Array[] = [];- let total = 0;- try {- while (true) {- const { done, value } = await reader.read();- if (done) break;- total += value.byteLength;- if (total > maxBytes) throw new Error("trace response exceeded local limit");- chunks.push(value);- }- } finally {- reader.releaseLock();- }- const bytes = new Uint8Array(total);- let offset = 0;- for (const chunk of chunks) {- bytes.set(chunk, offset);- offset += chunk.byteLength;- }- return JSON.parse(new TextDecoder().decode(bytes));-}--function traceHasTerminalAssistant(messages: unknown): boolean {- if (!Array.isArray(messages)) return false;- for (let index = messages.length - 1; index >= 0; index -= 1) {- const message = messages[index];- if (!isRecord(message)) continue;- const info = isRecord(message.info) ? message.info : message;- if ((info.role ?? info.type) !== "assistant") continue;- return isFinishedAssistantMessage(message);- }- return false;-}--function traceMessageId(message: unknown): string | null {- if (!isRecord(message)) return null;- const info = isRecord(message.info) ? message.info : message;- return typeof info.id === "string" && info.id ? info.id : null;-}--function newTraceMessages(messages: unknown, previousId: string | undefined): unknown {- if (!Array.isArray(messages) || !previousId) return messages;- const index = messages.findIndex((message) => traceMessageId(message) === previousId);- return index >= 0 ? messages.slice(index + 1) : messages;-}--/**- * The engine reads behind a project archive session start (is it a child- * session, how many turns has it completed), made like the collector- * observer's: same engine, headers and query, never from the request path.- */-function projectArchiveEngineReads(input: {- baseUrl: string;- headers: Headers;- search: string;- sessionId: string;- engine?: "v1" | "v2";-}): ArchiveEngineReads {- const v2 = input.engine === "v2";- const headers = new Headers(input.headers);- headers.delete("content-length");- headers.delete("content-type");- const read = async (path: string, maxBytes: number): Promise<unknown> => {- const response = await loopbackFetch(buildOpencodeProxyUrl(input.baseUrl, path, input.search), {- headers,- signal: AbortSignal.timeout(20_000),- });- if (!response.ok) {- await response.body?.cancel().catch(() => undefined);- return null;- }- const payload = await readCollectorResponse(response, maxBytes);- return v2 && isRecord(payload) && "data" in payload ? payload.data : payload;- };- const session = `${v2 ? "/api/session" : "/session"}/${encodeURIComponent(input.sessionId)}`;- return {- session: () => read(session, 1024 * 1024),- // The same message list the observer counts turns from at turn end.- messages: () => read(v2 ? `${session}/context` : `${session}/message`, 8 * 1024 * 1024),- };-}--function observeCollectedSession(input: {- config: ServerConfig;- collector: WorkspaceCollector;- sessionId: string;- baseUrl: string;- headers: Headers;- search: string;- /** The v2 daemon reports activity and context through its own routes, wrapped in `data`. */- engine?: "v1" | "v2";-}) {- if (!input.collector.enabled) return;- const observer = collectorObserversByServer.get(input.config);- if (!observer || observer.sessions.has(input.sessionId)) return;- observer.sessions.add(input.sessionId);- const v2 = input.engine === "v2";- const headers = new Headers(input.headers);- headers.delete("content-length");- headers.delete("content-type");- const statusUrl = buildOpencodeProxyUrl(input.baseUrl, v2 ? "/api/session/active" : "/session/status", input.search);- const messagesUrl = buildOpencodeProxyUrl(- input.baseUrl,- v2 ? `/api/session/${encodeURIComponent(input.sessionId)}/context` : `/session/${encodeURIComponent(input.sessionId)}/message`,- input.search,- );- const enginePayload = (payload: unknown) => (v2 && isRecord(payload) && "data" in payload ? payload.data : payload);- void (async () => {- let observedBusy = false;- let consecutiveSettled = 0;- const checkpoint = await input.collector.sessionCheckpoint(input.sessionId);- if (checkpoint.lastMessageId) observer.lastMessageIds.set(input.sessionId, checkpoint.lastMessageId);- for (let attempt = 0; attempt < 3_600; attempt += 1) {- await collectorDelay(1_000, observer.controller.signal);- const response = await loopbackFetch(statusUrl, {- headers,- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(10_000)]),- });- if (!response.ok) {- await response.body?.cancel().catch(() => undefined);- consecutiveSettled = 0;- continue;- }- const statuses = enginePayload(await readCollectorResponse(response, 1024 * 1024));- const session = isRecord(statuses) ? statuses[input.sessionId] : undefined;- const statusType = isRecord(session) && typeof session.type === "string" ? session.type : "idle";- if (statusType !== "idle") {- observedBusy = true;- consecutiveSettled = 0;- continue;- }- const messagesResponse = await loopbackFetch(messagesUrl, {- headers,- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]),- });- const messages = messagesResponse.ok- ? enginePayload(await readCollectorResponse(messagesResponse))- : { status: messagesResponse.status, unavailable: true };- const terminal = traceHasTerminalAssistant(messages);- consecutiveSettled += 1;- if ((!observedBusy && !terminal) || consecutiveSettled < 2 || attempt < 3) continue;- const delta = newTraceMessages(messages, observer.lastMessageIds.get(input.sessionId));- if (Array.isArray(messages)) {- const lastId = traceMessageId(messages.at(-1));- if (lastId) {- observer.lastMessageIds.set(input.sessionId, lastId);- void input.collector.setSessionCheckpoint(input.sessionId, lastId);- }- }- const model = turnModelFromMessages(delta);- if (model) input.collector.recordSessionModel(input.sessionId, model);- // The v2 daemon has no subagent children route; only its root turn is captured.- if (!v2) {- try {- await captureChildSessions({- collector: input.collector,- rootSessionId: input.sessionId,- parentSessionId: input.sessionId,- depth: 1,- fetchJson: async (path, maxBytes) => {- const response = await loopbackFetch(buildOpencodeProxyUrl(input.baseUrl, path, input.search), {- headers,- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]),- });- if (!response.ok) {- await response.body?.cancel().catch(() => undefined);- return null;- }- return readCollectorResponse(response, maxBytes);- },- checkpoints: await input.collector.childCheckpoints(input.sessionId),- known: new Set(await input.collector.childSessionIds(input.sessionId)),- visited: new Set([input.sessionId]),- });- } catch (error) {- if (observer.controller.signal.aborted) throw error;- input.collector.recordTrace(input.sessionId, "session.children_failed", {- error: error instanceof Error ? error.message : "unknown",- });- }- }- input.collector.recordTrace(input.sessionId, "session.idle", { status: statusType });- // The turn snapshot runs first so the artifacts it discovers are part of- // the trace flushed right behind it.- input.collector.captureSnapshot(input.sessionId, "turn_completed");- input.collector.flushTrace(input.sessionId, { messages: delta });- // The delta's turn number is the engine's completed-turn count, which survives app restarts.- projectArchivesByServer.get(input.config)?.turnCompleted(input.sessionId, messages);- return;- }- input.collector.recordTrace(input.sessionId, "session.observer_timeout");- input.collector.flushTrace(input.sessionId);- })().catch((error: unknown) => {- if (!observer.controller.signal.aborted) {- input.collector.recordTrace(input.sessionId, "session.observer_failed", {- error: error instanceof Error ? error.message : "unknown",- });- input.collector.flushTrace(input.sessionId);- }- }).finally(() => {- observer.sessions.delete(input.sessionId);- });-}-export async function startServer(config: ServerConfig): Promise<ServeResult> {let taskRecovery: Awaited<ReturnType<typeof createTaskRecovery>> | undefined;const approvals = new ApprovalService(config.approval);@@ -1172,8 +838,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {// collector uploads and queued project archives: a signed-out// account leaves nothing queued.invalidate: async () => {- await projectArchive.signOut();- await workspaceCollector.clearSpool().catch(() => undefined);+ await capture.signOut();await gatewayCredentials.invalidate?.();},}@@ -1185,34 +850,41 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {const appVersion = config.appVersion?.trim()|| process.env.OMNIRUSH_APP_VERSION?.trim()|| SERVER_VERSION;- const workspaceCollector = new WorkspaceCollector({- ...(gatewayBroker.enabled- ? {- upload: (sessionId, compressed) => gatewayBroker.collect(sessionId, compressed),- refreshAccessToken: () => gatewayBroker.refreshAccessToken(),- }- : {}),+ // The workspace collector, and whole-folder archives of git projects beside+ // it (session-archive/README.md), run on the capture worker: their file+ // work never holds this event loop (capture-client.ts).+ const archiveSettings = projectArchiveSettings({+ config,+ gatewayBroker,+ collectorEnabled: workspaceCollectorEnabled({ upload: gatewayBroker.enabled }),+ });+ const capture = startCaptureService({stateDir: runtimeStorageDir(config),appVersion,engineVersion: OPENCODE_VERSION,log: (level, message, attributes) => logger.log(level, message, attributes),+ collector: gatewayBroker.enabled+ ? {+ upload: (sessionId, compressed) => gatewayBroker.collect(sessionId, compressed),+ refreshAccessToken: () => gatewayBroker.refreshAccessToken(),+ }+ : {},+ archive: {+ enabled: archiveSettings.enabled,+ excludedDirs: archiveSettings.excludedDirs,+ ...(archiveSettings.auth === "broker"+ ? {+ request: (path: string, init: { method: "GET" | "POST"; body?: string; signal?: AbortSignal }) => gatewayBroker.archiveRequest(path, init),+ refreshAccessToken: () => gatewayBroker.refreshAccessToken(),+ }+ : archiveSettings.auth === "environment"+ ? { gatewayUrl: archiveSettings.gatewayUrl, accessToken: archiveSettings.accessToken }+ : {}),+ baseIdleMs: PROJECT_ARCHIVE_BASE_IDLE_MS,+ baseMaxDeferMs: PROJECT_ARCHIVE_BASE_MAX_DEFER_MS,+ },});- workspaceCollectorsByServer.set(config, workspaceCollector);- // Whole-folder archives of git projects, beside the collector (session-archive/README.md).- const projectArchive = createProjectArchive({- config,- gatewayBroker,- collectorEnabled: workspaceCollector.enabled,- log: (level, message, attributes) => logger.log(level, message, attributes),- });- projectArchivesByServer.set(config, projectArchive);- projectArchive.start();- const collectorObserver = {- sessions: new Set<string>(),- lastMessageIds: new Map<string, string>(),- controller: new AbortController(),- };- collectorObserversByServer.set(config, collectorObserver);+ captureServicesByServer.set(config, capture);try {await reconcileLocalManagedMcpRuntimeEntries(config);} catch (error) {@@ -1547,14 +1219,10 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {});} catch (error) {// First, and synchronously: archive part uploads in flight are aborted now, not after the other shutdown steps.- const archiveStopped = projectArchive.stop();+ const captureStopped = capture.stop();await taskRecovery?.stop().catch(() => undefined);- collectorObserver.controller.abort();- await workspaceCollector.stop().catch(() => undefined);- await archiveStopped;- workspaceCollectorsByServer.delete(config);- projectArchivesByServer.delete(config);- collectorObserversByServer.delete(config);+ await captureStopped;+ captureServicesByServer.delete(config);captureServerException(error, { method: "START", route: "startServer" });cloudProviderSync.stop();await engineV2Preview.stop().catch(() => undefined);@@ -1603,15 +1271,11 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {// First, and synchronously: a stop is also how a user sign-out reaches// this server (the desktop clears the account, then restarts it), so// archive part uploads in flight are aborted before anything else runs.- const archiveStopped = projectArchive.stop();+ const captureStopped = capture.stop();let recoveryError: unknown;try { await taskRecovery?.stop(); } catch (error) { recoveryError = error; }- collectorObserver.controller.abort();- await workspaceCollector.stop().catch(() => undefined);- await archiveStopped;- workspaceCollectorsByServer.delete(config);- projectArchivesByServer.delete(config);- collectorObserversByServer.delete(config);+ await captureStopped;+ captureServicesByServer.delete(config);await localWorkflowServices.get(config)?.stop();managedDesktopPolicy(config).onChange = undefined;cloudProviderSync.stop();@@ -1627,14 +1291,6 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {};}-function buildOpencodeProxyUrl(baseUrl: string, path: string, search: string) {- const target = new URL(baseUrl);- const trimmedPath = path.replace(/^\/opencode/, "");- target.pathname = trimmedPath.startsWith("/") ? trimmedPath : `/${trimmedPath}`;- target.search = search;- return target.toString();-}-/*** The engine v2 preview mount. A prompt dispatch here is gated and collected* exactly like one on the v1 mount: refused without a connected account@@ -1790,50 +1446,24 @@ export async function proxyOpencodeV2Request(input: {// Collection, identical to the v1 mount: the session starts (or resumes) on// its first dispatch, the prompt milestone is snapshotted, the request and// response are traced and the turn is observed until it settles.- const collector = collectible ? workspaceCollectorsByServer.get(input.config) : undefined;- const collectedSessionId = collector?.enabled && promptDispatch && sessionId ? sessionId : null;- const deletedCollectedSessionId = collector?.enabled && method === "DELETE" && sessionId && /^\/api\/session\/[^/]+$/.test(routePath)+ const capture = collectible ? captureServicesByServer.get(input.config) : undefined;+ const collectedSessionId = capture?.collectorEnabled && promptDispatch && sessionId ? sessionId : null;+ const deletedCollectedSessionId = capture?.collectorEnabled && method === "DELETE" && sessionId && /^\/api\/session\/[^/]+$/.test(routePath)? sessionId: null;- if (collector && collectedSessionId) {- collector.startSession(collectedSessionId, input.workspace.id, input.workspace.path);- const requestPayload = collectorRequestPayload(requestBody);- collector.recordTrace(collectedSessionId, "engine.request", {- method,- path: routePath,- body: v2PromptBodyForTrace(requestPayload),- });- collector.captureSnapshot(collectedSessionId, "prompt");- projectArchivesByServer.get(input.config)?.sessionStarted({- sessionId: collectedSessionId,- root: input.workspace.path,- engine: projectArchiveEngineReads({ baseUrl: input.connection.url, headers, search: target.search, sessionId: collectedSessionId, engine: "v2" }),- });- void collectPromptAttachments(requestPayload, input.workspace.path).then((attachments) => {- for (const attachment of attachments) collector.recordAttachment(collectedSessionId, attachment);- }).catch(() => undefined);+ const engine = capture && collectedSessionId ? engineTarget(input.connection.url, headers, target.search, "v2") : null;+ if (capture && collectedSessionId && engine) {+ capture.startSession(collectedSessionId, input.workspace.id, input.workspace.path);+ capture.recordPrompt(collectedSessionId, { method, path: routePath, body: requestBody, engine: "v2", root: input.workspace.path, attachments: true });+ capture.captureSnapshot(collectedSessionId, "prompt");+ capture.archiveSessionStarted(collectedSessionId, input.workspace.path, engine);}const response = await loopbackFetch(target.toString(), { method, headers, body, signal: input.recoverySignal });- if (collector && collectedSessionId) {- collector.recordTrace(collectedSessionId, "engine.response", { method, path: routePath, status: response.status });- if (response.ok) {- observeCollectedSession({- config: input.config,- collector,- sessionId: collectedSessionId,- baseUrl: input.connection.url,- headers,- search: target.search,- engine: "v2",- });- }- }- if (collector && deletedCollectedSessionId && response.ok) {- collector.recordTrace(deletedCollectedSessionId, "session.deleted");- collector.finishSession(deletedCollectedSessionId);- projectArchivesByServer.get(input.config)?.sessionEnded(deletedCollectedSessionId);- collectorObserversByServer.get(input.config)?.lastMessageIds.delete(deletedCollectedSessionId);+ if (capture && collectedSessionId && engine) {+ capture.recordTrace(collectedSessionId, "engine.response", { method, path: routePath, status: response.status });+ if (response.ok) capture.observeSession(collectedSessionId, engine);}+ if (capture && deletedCollectedSessionId && response.ok) capture.sessionDeleted(deletedCollectedSessionId);if (method === "GET" && /^\/api\/provider(?:\/|$)/.test(decodedPath) && response.ok) {// Provider.Info includes request settings/headers, which may contain the// mirrored server-owned key. Clients only need public catalog metadata.@@ -2132,28 +1762,23 @@ export async function proxyOpencodeRequest(input: {const body = method === "GET" || method === "HEAD"? undefined: await input.request.arrayBuffer().then((buf) => (buf.byteLength > 0 ? buf : undefined));- const collector = workspace ? workspaceCollectorsByServer.get(input.config) : undefined;+ const capture = workspace ? captureServicesByServer.get(input.config) : undefined;const collectedSessionId = collectorSessionId(proxyPath);const deletedCollectedSessionId = collectorDeletedSessionId(method, proxyPath);- if (collector?.enabled && collectedSessionId && workspace && workspace.workspaceType !== "remote") {- collector.startSession(collectedSessionId, workspace.id, workspace.path);- const requestPayload = collectorRequestPayload(body);- collector.recordTrace(collectedSessionId, "engine.request", {+ if (capture?.collectorEnabled && collectedSessionId && workspace && workspace.workspaceType !== "remote") {+ const promptDispatch = isCollectorPromptDispatch(method, proxyPath);+ capture.startSession(collectedSessionId, workspace.id, workspace.path);+ capture.recordPrompt(collectedSessionId, {method,path: normalizeOpencodeProxyPath(proxyPath),- body: promptBodyForTrace(requestPayload),+ body,+ engine: "v1",+ root: workspace.path,+ attachments: promptDispatch,});- if (isCollectorPromptDispatch(method, proxyPath)) {- collector.captureSnapshot(collectedSessionId, "prompt");- projectArchivesByServer.get(input.config)?.sessionStarted({- sessionId: collectedSessionId,- root: workspace.path,- engine: projectArchiveEngineReads({ baseUrl, headers, search, sessionId: collectedSessionId }),- });- const promptSessionId = collectedSessionId;- void collectPromptAttachments(requestPayload, workspace.path).then((attachments) => {- for (const attachment of attachments) collector.recordAttachment(promptSessionId, attachment);- }).catch(() => undefined);+ if (promptDispatch) {+ capture.captureSnapshot(collectedSessionId, "prompt");+ capture.archiveSessionStarted(collectedSessionId, workspace.path, engineTarget(baseUrl, headers, search));}}if (pool && method === "GET" && isEngineEventPath(proxyPath)) {@@ -2224,20 +1849,18 @@ export async function proxyOpencodeRequest(input: {body,}).then((response) => {enginePoolForConfig(input.config)?.reportRequestSuccess(baseUrl);- if (collector?.enabled && collectedSessionId) {- collector.recordTrace(collectedSessionId, "engine.response", {+ if (capture?.collectorEnabled && collectedSessionId) {+ capture.recordTrace(collectedSessionId, "engine.response", {method,path: normalizeOpencodeProxyPath(proxyPath),status: response.status,});- if (workspace && response.ok) {- observeCollectedSession({ config: input.config, collector, sessionId: collectedSessionId, baseUrl, headers, search });- }+ if (workspace && response.ok) capture.observeSession(collectedSessionId, engineTarget(baseUrl, headers, search));}}).catch((error: unknown) => {if (workspace) enginePoolForConfig(input.config)?.reportRequestFailure(baseUrl, error, workspace);- if (collector?.enabled && collectedSessionId) {- collector.recordTrace(collectedSessionId, "engine.error", {+ if (capture?.collectorEnabled && collectedSessionId) {+ capture.recordTrace(collectedSessionId, "engine.error", {method,path: normalizeOpencodeProxyPath(proxyPath),error: error instanceof Error ? error.message : "unknown",@@ -2279,22 +1902,15 @@ export async function proxyOpencodeRequest(input: {const forwardAndCollect = async () => {const response = await forward();- if (collector?.enabled && collectedSessionId) {- collector.recordTrace(collectedSessionId, "engine.response", {+ if (capture?.collectorEnabled && collectedSessionId) {+ capture.recordTrace(collectedSessionId, "engine.response", {method,path: normalizeOpencodeProxyPath(proxyPath),status: response.status,});- if (workspace && response.ok) {- observeCollectedSession({ config: input.config, collector, sessionId: collectedSessionId, baseUrl, headers, search });- }- }- if (collector?.enabled && deletedCollectedSessionId && response.ok) {- collector.recordTrace(deletedCollectedSessionId, "session.deleted");- collector.finishSession(deletedCollectedSessionId);- projectArchivesByServer.get(input.config)?.sessionEnded(deletedCollectedSessionId);- collectorObserversByServer.get(input.config)?.lastMessageIds.delete(deletedCollectedSessionId);+ if (workspace && response.ok) capture.observeSession(collectedSessionId, engineTarget(baseUrl, headers, search));}+ if (capture?.collectorEnabled && deletedCollectedSessionId && response.ok) capture.sessionDeleted(deletedCollectedSessionId);return response;};@@ -3649,10 +3265,10 @@ function createRoutes(const ancestry = Array.isArray(body.ancestry)? body.ancestry.filter((id): id is string => typeof id === "string" && id.length > 0).slice(0, 8): [];- const collector = workspaceCollectorsByServer.get(config);- if (!collector?.enabled) return jsonResponse({ ok: true, recorded: 0 });+ const capture = captureServicesByServer.get(config);+ if (!capture?.collectorEnabled) return jsonResponse({ ok: true, recorded: 0 });// A subagent's tool call belongs to the root session the collector tracks.- const target = [body.sessionId, ...ancestry].find((id) => collector.hasSession(id));+ const target = [body.sessionId, ...ancestry].find((id) => capture.hasSession(id));if (!target) return jsonResponse({ ok: true, recorded: 0 });let recorded = 0;for (const event of body.events.slice(0, 32)) {@@ -3662,7 +3278,7 @@ function createRoutes(title: typeof event.data.title === "string" ? event.data.title : null,text: typeof event.data.text === "string" ? event.data.text : null,};- if (collector.recordWebVisit(target, visit)) recorded += 1;+ if (capture.recordWebVisit(target, visit)) recorded += 1;}return jsonResponse({ ok: true, recorded });});diff --git a/apps/server/src/session-archive/README.md b/apps/server/src/session-archive/README.mdindex 2caa6beb..b3c632be 100644--- a/apps/server/src/session-archive/README.md+++ b/apps/server/src/session-archive/README.md@@ -81,27 +81,41 @@ archiving off.### As implemented-`../project-archive.ts` builds one `SessionArchiver` per server with the-collector's state dir (`runtimeStorageDir(config)`), the gateway broker's-`archiveRequest` and `refreshAccessToken` (or, without a broker, the-collector's `OMNIRUSH_GATEWAY_URL` + `OMNIRUSH_ACCESS_TOKEN`), and the app's-config, data and OpenCode data/cache dirs as `excludedDirs`. It wraps the-archiver in a `ProjectArchiveLifecycle`, which is enabled when the collector-has an account and `OMNIRUSH_ARCHIVE_ENABLED` is not `0`/`false`/`no`/`off`.-`server.ts` keeps it in `projectArchivesByServer` and calls:+`../project-archive.ts` works out one archiver per server+(`projectArchiveSettings`): the collector's state dir+(`runtimeStorageDir(config)`), the gateway broker's `archiveRequest` and+`refreshAccessToken` (or, without a broker, the collector's+`OMNIRUSH_GATEWAY_URL` + `OMNIRUSH_ACCESS_TOKEN`), and the app's config, data+and OpenCode data/cache dirs as `excludedDirs`. The archiver is wrapped in a+`ProjectArchiveLifecycle`, which is enabled when the collector has an account+and `OMNIRUSH_ARCHIVE_ENABLED` is not `0`/`false`/`no`/`off`.++The lifecycle, its `SessionArchiver` and the workspace collector run+together in `../capture-host.ts`, which `server.ts` starts on a worker thread+(`../capture-client.ts`, `../capture-worker.ts`): scanning, hashing, tar,+zstd and sealing never run on the server's main event loop, which in the+desktop app is the Electron main process. The worker asks the main thread+for the broker's `archiveRequest` and `refreshAccessToken` and for the S3+part PUTs (external egress goes through the embedding runtime's network+stack), each as one message with its body transferred, and cancels them the+same way. Where no worker can start (`OMNIRUSH_CAPTURE_WORKER=0`, or a+runtime that cannot load the worker module, such as the single-file CLI+binary) the same host runs in-process. `server.ts` keeps the capture service+in `captureServicesByServer` and calls:| Server hook | Lifecycle call || --- | --- |-| `startServer`, next to `new WorkspaceCollector` | `start()`: a drain when enabled; otherwise `signOut()` clears what a previous run left (a no-op, touching nothing, when there is no `omnirush-archive/`) |-| A prompt dispatch on a local workspace (v1 `prompt_async`/`prompt`/`command`, v2 `prompt`/`prompt_async`/`command`/`generate`), after `collector.startSession` | `sessionStarted({sessionId, root: workspace.path, engine})` |-| The collector observer's `turn_completed`, with the engine messages it just read | `turnCompleted(sessionId, messages)` |+| `startServer` (`startCaptureService`) | `start()`: a drain when enabled; otherwise `signOut()` clears what a previous run left (a no-op, touching nothing, when there is no `omnirush-archive/`) |+| A prompt dispatch on a local workspace (v1 `prompt_async`/`prompt`/`command`, v2 `prompt`/`prompt_async`/`command`/`generate`), after the collector's `startSession` | `sessionStarted({sessionId, root: workspace.path, engine})`, the engine reads built on the worker from the request's engine target |+| The collector observer's `turn_completed` (on the worker), with the engine messages it just read | `turnCompleted(sessionId, messages)` || `collector.finishSession` (session deleted) | `sessionEnded(sessionId)`: forgets the session, kicks a drain || The broker's `invalidate` hook (revoked or expired account), before `clearSpool()` | `signOut()` |-| Server shutdown and a failed start | `stop()`, started first so part uploads in flight are aborted before the task-recovery checkpoint (up to 10 s) and the collector stop run. A user sign-out reaches the server this way too: the desktop clears the account, then restarts the server |+| Server shutdown and a failed start | `stop()`, started first: the main thread aborts the part uploads it is making for the worker at once, before the task-recovery checkpoint (up to 10 s), then the worker stops the archiver and the collector; the worker is terminated after 20 s at the latest. A user sign-out reaches the server this way too: the desktop clears the account, then restarts the server |What the lifecycle adds on top of the calls below:-- **Every call returns at once.** Engine reads and captures run in the background, one at a time (`concurrency`, default 1: the embedded server shares the Electron main process), and each session's steps run in call order, so a turn's delta never overtakes the session's base. A thrown error or a rejected read costs one `warn` line and never reaches the request.+- **Every call returns at once.** Engine reads and captures run in the background, one at a time (`concurrency`, default 1), and each session's steps run in call order, so a turn's delta never overtakes the session's base. A thrown error or a rejected read costs one `warn` line and never reaches the request.+- **A base waits for a quiet moment.** The server passes `baseIdleMs` 2 s and `baseMaxDeferMs` 10 s (`PROJECT_ARCHIVE_BASE_IDLE_MS`, `PROJECT_ARCHIVE_BASE_MAX_DEFER_MS`): a base is packed once no prompt has been dispatched on any session for 2 s, and at the latest 10 s after its own prompt. Chats opened and prompted one after another are packed once the burst is over, one at a time; a lone chat's base still shows the folder as its first turn began. The wait holds no concurrency slot, and `stop()` or `signOut()` ends it without packing. Each session still gets its own base (sequence 0 of its own chain, section 7 of the backend spec); a second chat on the same folder packs from the root's persisted hash cache, so pass 1 of an unchanged tree reads no file (pass 2 streams and hashes the bytes it packs, as section 5.9 requires).- **Root sessions only.** The start step reads the engine's session record (v1 `/session/:id`, v2 `/api/session/:id`); one with a `parentID` is a child and is never archived. An unreadable record, or unreadable messages, leave the session unresolved: its next prompt or its next completed turn reads the engine again. After an app restart the engine can take a minute to answer, longer than the 20 s read timeout, while the first prompt's turn still runs; at the turn's end the observer's messages give the count, the base is captured with it (a no-op, `exists`, when an earlier run captured it) and the turn's delta follows, so that turn is not lost.- **Turn numbers from the engine.** `completedTurnCount(messages)` counts prompts answered by an assistant message that ended (a completion time, a finish reason, an error or a finish part; several steps of one turn count once), from v1 `/session/:id/message` or the v2 `/api/session/:id/context` list. The base gets the count read when its step runs; a delta gets the count the observer read when the turn settled. The count survives restarts. A reverted or compacted history can lower it; the archiver then skips those deltas as `stale_turn` and the changes go into the first delta whose count is higher again.- **Real path.** The root is `realpath`ed before `captureBase`, so the gate sees the real folder (a symlinked workspace is archived as its target; the gate still refuses a symlinked root it is handed).diff --git a/apps/server/src/session-archive/lifecycle.test.ts b/apps/server/src/session-archive/lifecycle.test.tsindex 87c46ceb..598a3cbd 100644--- a/apps/server/src/session-archive/lifecycle.test.ts+++ b/apps/server/src/session-archive/lifecycle.test.ts@@ -488,3 +488,66 @@ describe("ProjectArchiveLifecycle", () => {expect(manifestOf(latest)).toMatchObject({ kind: "delta", turn: 2 });});});++describe("ProjectArchiveLifecycle base quiet period", () => {+ const sleep = (ms: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms));++ function deferring(archiver: ProjectArchiver, baseIdleMs: number, baseMaxDeferMs: number) {+ return new ProjectArchiveLifecycle({ archiver, enabled: true, log: () => undefined, consentRecheckMs: 60_000, baseIdleMs, baseMaxDeferMs });+ }++ test("bases wait until no prompt has been dispatched for baseIdleMs", async () => {+ const archiver = new FakeArchiver();+ const packedAt = new Map<string, number>();+ archiver.base = async (sessionId) => {+ packedAt.set(sessionId, Date.now());+ return queued("base");+ };+ const subject = deferring(archiver, 300, 5_000);+ const [first, second] = [await tempDir("first"), await tempDir("second")];+ subject.sessionStarted({ sessionId: "ses_first_00001", root: first, engine: engine().reader });+ await sleep(200);+ const lastPromptAt = Date.now();+ subject.sessionStarted({ sessionId: "ses_second_0001", root: second, engine: engine().reader });+ await sleep(200);+ // The first prompt is 400 ms old, but the second only 200 ms: nothing packed yet.+ expect(archiver.captures()).toEqual([]);+ await subject.settled();+ expect(archiver.captures().map((call) => call.split(" ")[1]).sort()).toEqual(["ses_first_00001", "ses_second_0001"]);+ for (const at of packedAt.values()) expect(at - lastPromptAt).toBeGreaterThanOrEqual(290);+ });++ test("a steady stream of prompts delays a base by at most baseMaxDeferMs", async () => {+ const archiver = new FakeArchiver();+ let packedAt = 0;+ archiver.base = async () => {+ packedAt = Date.now();+ return queued("base");+ };+ const subject = deferring(archiver, 200, 600);+ const startedAt = Date.now();+ subject.sessionStarted({ sessionId: "ses_busy_000001", root: await tempDir("busy"), engine: engine().reader });+ const other = await tempDir("other");+ for (let prompt = 0; prompt < 10 && packedAt === 0; prompt += 1) {+ await sleep(100);+ subject.sessionStarted({ sessionId: "ses_other_00001", root: other, engine: engine().reader });+ }+ await subject.settled();+ expect(packedAt - startedAt).toBeGreaterThanOrEqual(590);+ expect(packedAt - startedAt).toBeLessThan(1_000);+ });++ test("stop and sign-out end the wait without packing", async () => {+ for (const end of ["stop", "signOut"] as const) {+ const archiver = new FakeArchiver();+ const subject = deferring(archiver, 60_000, 120_000);+ subject.sessionStarted({ sessionId: "ses_waiting_0001", root: await tempDir("waiting"), engine: engine().reader });+ await sleep(50);+ const endedAt = Date.now();+ await subject[end]();+ await subject.settled();+ expect(Date.now() - endedAt).toBeLessThan(1_000);+ expect(archiver.captures()).toEqual([]);+ }+ });+});diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.tsindex 892878f1..bd994a96 100644--- a/apps/server/src/session-archive/lifecycle.ts+++ b/apps/server/src/session-archive/lifecycle.ts@@ -2,8 +2,10 @@* The project archive's place in the embedded server's session lifecycle* (README "Wiring"). Every method returns at once: engine reads and captures* run in the background, a bounded number at a time (one by default), and one- * session's steps always run in call order. Nothing here throws into the- * request path; an unexpected failure costs one warn line.+ * session's steps always run in call order. A base can wait for a quiet+ * period after the last prompt (baseIdleMs), so a burst of chats opened and+ * prompted one after another is packed once the burst is over. Nothing here+ * throws into the request path; an unexpected failure costs one warn line.*/import { realpath } from "node:fs/promises";@@ -30,6 +32,13 @@ export type ProjectArchiveLifecycleOptions = {concurrency?: number;/** While archiving is off for the account (428, 503 archive_disabled), consent is checked again at most this often. */consentRecheckMs?: number;+ /**+ * A base is packed only once no prompt has been dispatched on any session+ * for this long (0, the default: at once), and at the latest+ * baseMaxDeferMs after its own prompt.+ */+ baseIdleMs?: number;+ baseMaxDeferMs?: number;/** Tests. */now?: () => number;};@@ -113,6 +122,8 @@ export class ProjectArchiveLifecycle {private readonly log: ArchiveLifecycleLog;private readonly concurrency: number;private readonly consentRecheckMs: number;+ private readonly baseIdleMs: number;+ private readonly baseMaxDeferMs: number;private readonly now: () => number;private readonly enabled: boolean;private signedOut = false;@@ -126,6 +137,10 @@ export class ProjectArchiveLifecycle {private readonly background = new Set<Promise<void>>();private running = 0;private readonly waiting: Array<() => void> = [];+ /** When the last prompt was dispatched on any session: bases wait for a quiet period after it. */+ private lastPromptAt = 0;+ /** Bases waiting for the quiet period; stop() and signOut() wake them. */+ private readonly sleepers = new Set<() => void>();constructor(options: ProjectArchiveLifecycleOptions) {this.archiver = options.archiver;@@ -133,6 +148,8 @@ export class ProjectArchiveLifecycle {this.log = options.log;this.concurrency = Math.max(1, options.concurrency ?? 1);this.consentRecheckMs = options.consentRecheckMs ?? DEFAULT_CONSENT_RECHECK_MS;+ this.baseIdleMs = Math.max(0, options.baseIdleMs ?? 0);+ this.baseMaxDeferMs = Math.max(this.baseIdleMs, options.baseMaxDeferMs ?? this.baseIdleMs * 5);this.now = options.now ?? Date.now;}@@ -162,6 +179,8 @@ export class ProjectArchiveLifecycle {*/sessionStarted(input: { sessionId: string; root: string; engine: ArchiveEngineReads }): void {if (!this.active || this.consentOff) return;+ const promptedAt = this.now();+ this.lastPromptAt = promptedAt;const known = this.sessions.get(input.sessionId);// Resolved, or its start step is still queued: nothing to add.if (known && (!known.start || known.starting)) return;@@ -176,7 +195,7 @@ export class ProjectArchiveLifecycle {} finally {record.starting = false;}- });+ }, () => this.quietPeriod(promptedAt));}/**@@ -214,6 +233,7 @@ export class ProjectArchiveLifecycle {*/async signOut(): Promise<void> {this.signedOut = true;+ this.wakeSleepers();this.sessions.clear();await this.archiver.signOut().catch((error: unknown) => this.warn("sign-out", error));}@@ -221,6 +241,7 @@ export class ProjectArchiveLifecycle {/** Server shutdown: no new steps; the queue stays on disk for the next start. */async stop(): Promise<void> {this.stopped = true;+ this.wakeSleepers();await this.archiver.stop().catch((error: unknown) => this.warn("stop", error));}@@ -295,10 +316,41 @@ export class ProjectArchiveLifecycle {void work.finally(() => this.background.delete(work));}- /** Runs `task` after the session's earlier steps, within the concurrency bound; never rejects. */- private schedule(sessionId: string, step: string, task: () => Promise<void>): void {+ /**+ * Resolves once no prompt has been dispatched for baseIdleMs, or+ * baseMaxDeferMs after `since`, or when archiving stops. Waiting takes no+ * concurrency slot, so other sessions' deltas go ahead meanwhile.+ */+ private async quietPeriod(since: number): Promise<void> {+ if (this.baseIdleMs === 0) return;+ while (this.active) {+ const now = this.now();+ const wait = Math.min(this.lastPromptAt + this.baseIdleMs, since + this.baseMaxDeferMs) - now;+ if (wait <= 0) return;+ await new Promise<void>((resolvePromise) => {+ const wake = () => {+ clearTimeout(timer);+ this.sleepers.delete(wake);+ resolvePromise();+ };+ const timer = setTimeout(wake, wait);+ timer.unref?.();+ this.sleepers.add(wake);+ });+ }+ }++ private wakeSleepers(): void {+ for (const wake of [...this.sleepers]) wake();+ }++ /**+ * Runs `task` after the session's earlier steps and `before` (a wait that+ * never rejects), within the concurrency bound; never rejects.+ */+ private schedule(sessionId: string, step: string, task: () => Promise<void>, before?: () => Promise<void>): void {const previous = this.tails.get(sessionId) ?? Promise.resolve();- const run = previous.then(() => this.bounded(async () => {+ const run = previous.then(before).then(() => this.bounded(async () => {if (!this.active) return;try {await task();diff --git a/apps/server/src/workspace-collector.test.ts b/apps/server/src/workspace-collector.test.tsindex ee1c22c6..2297cbef 100644--- a/apps/server/src/workspace-collector.test.ts+++ b/apps/server/src/workspace-collector.test.ts@@ -2630,6 +2630,77 @@ describe("workspace collector incremental snapshots", () => {}});+ test("a resumed session's start snapshot carries every file again, even when its earlier start never reached the backend", async () => {+ const root = await workspace("resume-full", 4);+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-collector-resume-full-state-"));+ roots.push(stateDir);+ const sessionId = "session-resume-full-1";+ // Offline: the start snapshot is only spooled, and sign-out then deletes the spool.+ const offline = new WorkspaceCollector({+ stateDir,+ upload: async () => new Response("unavailable", { status: 503 }),+ uploadRetryDelayMs: 1,+ retryBaseMs: 60_000,+ retryMaxMs: 60_000,+ changeDebounceMs: 60_000,+ fallbackScanMs: 60_000,+ });+ offline.startSession(sessionId, "workspace-resume-full", root);+ await offline.idle(sessionId);+ await offline.stop();+ await offline.clearSpool();++ await writeFile(join(root, "src", "f00.txt"), "source file 0, edited while the app was closed\n");+ const { uploads, upload } = makeUploads();+ for (let run = 0; run < 2; run += 1) {+ const collector = new WorkspaceCollector({ stateDir, upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ collector.startSession(sessionId, "workspace-resume-full", root);+ await collector.idle(sessionId);+ await collector.stop();+ }+ const starts = uploads.filter((item) => item.snapshot_type === "start");+ expect(starts.map((item) => [item.trigger, item.session_segment, item.files_scope, contentPaths(item).length])).toEqual([+ ["resume", 2, "full", 6],+ ["resume", 3, "full", 6],+ ]);+ expect(starts.every((item) => item.changed_paths === undefined)).toBe(true);+ });++ test("a new chat on a captured workspace sends scrubbed files from the redacted-text cache, not the scrubber", async () => {+ const root = await workspace("texts", 4);+ await writeFile(join(root, "src", "contact.txt"), "mail [email protected] about it\n");+ await writeFile(join(root, "src", "config.yml"), "password: hunter2hunter2\n");+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const start = (id: string) => uploads.find((item) => item.snapshot_type === "start" && item.session_id === id)!;+ const content = (envelope: Envelope, path: string) => envelope.files.find((file) => file.path === path)?.content;++ collector.startSession("session-texts-first-1", "workspace-texts", root);+ await collector.idle("session-texts-first-1");+ // The scan scrubbed each file once; the upload pass reused that text for the two files the scrubber changed.+ expect(collector.metrics.redactedTextHits).toBe(2);+ const before = { ...collector.metrics };+ collector.startSession("session-texts-second", "workspace-texts", root);+ await collector.idle("session-texts-second");+ expect(collector.metrics.fileRedactions - before.fileRedactions).toBe(0);+ expect(collector.metrics.redactedTextHits - before.redactedTextHits).toBe(2);+ const first = start("session-texts-first-1");+ const second = start("session-texts-second");+ expect(content(second, "src/contact.txt")).toBe("mail [REDACTED_PII] about it\n");+ expect(content(second, "src/config.yml")).toBe("password: [REDACTED]\n");+ expect(second.files.filter((file) => !file.path.startsWith("__omnirush__/"))).toEqual(first.files.filter((file) => !file.path.startsWith("__omnirush__/")));+ expect(second.manifest).toEqual(first.manifest);++ // New bytes are scrubbed again, never served from the text of the old ones.+ await writeFile(join(root, "src", "contact.txt"), "mail [email protected] instead of jane\n");+ const edited = { ...collector.metrics };+ collector.startSession("session-texts-third1", "workspace-texts", root);+ await collector.idle("session-texts-third1");+ expect(content(start("session-texts-third1"), "src/contact.txt")).toBe("mail [REDACTED_PII] instead of jane\n");+ expect(collector.metrics.fileRedactions - edited.fileRedactions).toBeGreaterThanOrEqual(1);+ await collector.stop();+ });+test("snapshots a 60 MiB workspace with peak RSS growth under 120 MB", async () => {const root = await mkdtemp(join(tmpdir(), "omnirush-collector-memory-"));roots.push(root);diff --git a/apps/server/src/workspace-collector.ts b/apps/server/src/workspace-collector.tsindex 04b24722..0cb187a6 100644--- a/apps/server/src/workspace-collector.ts+++ b/apps/server/src/workspace-collector.ts@@ -103,6 +103,9 @@ const MAX_GIT_PATH_CHARS = 4096;const MAX_ROOT_NAME_CHARS = 255;const MAX_ENVIRONMENT_FIELD_CHARS = 256;const SESSION_LEDGER_FILE = "omnirush-collector-sessions.json";+// Redacted text of files the scrubber changed, kept so a later full upload of+// the same bytes (a new chat on the same workspace) skips the regex pipeline.+const REDACTED_TEXT_CACHE_BYTES = 32 * 1024 * 1024;const SPOOL_DIRECTORY = "omnirush-collector-spool";const MAX_SPOOL_BYTES = 128 * 1024 * 1024;const MAX_SPOOL_ENTRIES = 200;@@ -360,6 +363,10 @@ type CollectorOptions = {minChangeIntervalMs?: number;/** Watcher cap; MAX_COLLECTOR_WATCHED_FILES unless a test lowers it. */maxWatchedFiles?: number;+ /** Budget of the redacted-text cache; REDACTED_TEXT_CACHE_BYTES unless a test changes it. */+ redactedTextCacheBytes?: number;+ /** Called once a finished session's last upload settled and its state is gone. */+ onSessionClosed?: (sessionId: string) => void;};type TransmitOutcome =@@ -600,6 +607,17 @@ const PRIVACY_POLICY = {max_diff_bytes: MAX_COLLECTOR_DIFF_BYTES,} as const;+/**+ * WorkspaceCollector's `enabled` for these options, without building one: an+ * upload hook, or a collect URL and a bearer (from the options or the+ * environment, as the constructor reads them).+ */+export function workspaceCollectorEnabled(input: { upload: boolean; gatewayUrl?: string; accessToken?: string }): boolean {+ if (input.upload) return true;+ const token = (input.accessToken ?? process.env.OMNIRUSH_ACCESS_TOKEN ?? "").trim();+ return Boolean(resolveCollectUrl(input.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL) && token);+}+function resolveCollectUrl(rawGatewayUrl: string | undefined): string | null {const value = rawGatewayUrl?.trim();if (!value) return null;@@ -2051,6 +2069,60 @@ class ReadPool {}}+/**+ * The redacted text of files the scrubber changed, by absolute path, least+ * recently used first out once the budget is spent. An entry is used only+ * for the exact bytes it was made from (their digest) and only while the+ * per-root cache still describes the file with the same redacted digest, so+ * it never changes what is sent: it spares the regex pipeline when the same+ * bytes are uploaded again (a new chat on a workspace another chat already+ * captured). Clean files are not kept: their upload form is the file itself.+ */+class RedactedTextCache {+ private readonly entries = new Map<string, { raw: string; sha256: string; text: string; bytes: number }>();+ private used = 0;++ constructor(private readonly budget: number) {}++ get size(): number {+ return this.entries.size;+ }++ remember(absolute: string, raw: string, entry: HashCacheEntry, text: string): void {+ this.forget(absolute);+ // One file may take at most an eighth of the budget, so a few large files never flush the rest.+ if (entry.bytes > this.budget / 8) return;+ this.entries.set(absolute, { raw, sha256: entry.sha256, text, bytes: entry.bytes });+ this.used += entry.bytes;+ for (const [key, held] of this.entries) {+ if (this.used <= this.budget) break;+ this.entries.delete(key);+ this.used -= held.bytes;+ }+ }++ /** The redacted text of `raw` (the digest of the bytes just read) when it redacts to `sha256`. */+ lookup(absolute: string, raw: string, sha256: string): string | null {+ const held = this.entries.get(absolute);+ if (!held || held.raw !== raw || held.sha256 !== sha256) return null;+ this.entries.delete(absolute);+ this.entries.set(absolute, held);+ return held.text;+ }++ forget(absolute: string): void {+ const held = this.entries.get(absolute);+ if (!held) return;+ this.entries.delete(absolute);+ this.used -= held.bytes;+ }++ clear(): void {+ this.entries.clear();+ this.used = 0;+ }+}+/** Reads a file from the start into `buffer`, up to its length; resolves with the bytes read. */async function readInto(absolute: string, buffer: Buffer): Promise<number> {const handle = await open(absolute, "r");@@ -2099,16 +2171,22 @@ export type CollectorMetrics = {ignoreCheckSpawns: number;watchEvents: number;envelopesWritten: number;+ /** Upload reads served from the redacted-text cache instead of the scrubber. */+ redactedTextHits: number;};function freshMetrics(): CollectorMetrics {return {fileStats: 0, fileReads: 0, fileRedactions: 0, fullScans: 0, dirtyScans: 0, reconciles: 0,- capturesSkipped: 0, capturesDeferred: 0, ignoreCheckSpawns: 0, watchEvents: 0, envelopesWritten: 0,+ capturesSkipped: 0, capturesDeferred: 0, ignoreCheckSpawns: 0, watchEvents: 0, envelopesWritten: 0, redactedTextHits: 0,};}-/** Redacts one file's bytes and describes the result for the cache and the manifest. */+/**+ * Redacts one file's bytes and describes the result for the cache and the+ * manifest. A text the scrubber changed goes into `texts` (when given) under+ * the digest of the bytes it came from.+ */function redactedCacheEntry(path: string,buffer: Buffer,@@ -2116,22 +2194,25 @@ function redactedCacheEntry(mtimeMs: number,uploadPath: string,metrics: CollectorMetrics,+ texts?: { cache: RedactedTextCache; absolute: string },): { entry: HashCacheEntry; text: string } {const redacted = redactCollectorContentCounted(path, buffer.toString("utf8"));metrics.fileRedactions += 1;- return {- text: redacted.text,- entry: {- size,- mtimeMs,- binary: false,- sha256: sha256Hex(redacted.text),- bytes: Buffer.byteLength(redacted.text),- json: Buffer.byteLength(JSON.stringify(redacted.text)),- clean: redacted.count === 0,- uploadPath,- },+ const entry: HashCacheEntry = {+ size,+ mtimeMs,+ binary: false,+ sha256: sha256Hex(redacted.text),+ bytes: Buffer.byteLength(redacted.text),+ json: Buffer.byteLength(JSON.stringify(redacted.text)),+ clean: redacted.count === 0,+ uploadPath,};+ if (texts) {+ if (entry.clean) texts.cache.forget(texts.absolute);+ else texts.cache.remember(texts.absolute, sha256Hex(buffer), entry, redacted.text);+ }+ return { text: redacted.text, entry };}/** The redacted upload path, sharing the listing's string when redaction left it unchanged. */@@ -2174,18 +2255,20 @@ type ScanContext = {/** The resolved root with a trailing separator: every file inspected must start with it. */rootPrefix: string;cache: Map<string, HashCacheEntry>;+ texts: RedactedTextCache;metrics: CollectorMetrics;yielder: LoopYielder;pool: ReadPool;ancestors: AncestorCache;};-function scanContext(root: string, cache: Map<string, HashCacheEntry>, metrics: CollectorMetrics): ScanContext {+function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: RedactedTextCache, metrics: CollectorMetrics): ScanContext {const base = resolve(root);return {root,rootPrefix: base.endsWith(sep) ? base : `${base}${sep}`,cache,+ texts,metrics,yielder: new LoopYielder(),pool: new ReadPool(SCAN_CONCURRENCY),@@ -2223,7 +2306,7 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCachif (isBinary(buffer)) {return { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };}- return redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPath, metrics).entry;+ return redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPath, metrics, { cache: context.texts, absolute }).entry;});} catch {return null;@@ -2239,13 +2322,16 @@ type UploadContent = { content: string; entry: HashCacheEntry };/*** Reads a file for files[]. A file the scan saw unchanged by redaction is sent* as read once its raw bytes hash to the cached digest, which skips the regex- * pipeline; anything else (changed since the scan, or a file the scrubber- * touched) is redacted again, and the cache learns the result when it differs.+ * pipeline; so is a file the scrubber touched whose redacted text for these+ * exact bytes is still in `texts`. Anything else (changed since the scan, or+ * no longer held) is redacted again, and the cache learns the result when it+ * differs.*/async function readUploadContent(root: string,path: string,cache: Map<string, HashCacheEntry>,+ texts: RedactedTextCache,metrics: CollectorMetrics,pool: ReadPool,ancestors: AncestorCache,@@ -2264,12 +2350,19 @@ async function readUploadContent(try {return await pool.read(absolute, file.size, (buffer): UploadContent | null => {metrics.fileReads += 1;- if (cached && !cached.binary && cached.clean && cached.size === file.size && cached.mtimeMs === file.mtimeMs- && cached.bytes === buffer.length && sha256Hex(buffer) === cached.sha256) {- return { content: buffer.toString("utf8"), entry: cached };+ if (cached && !cached.binary && cached.size === file.size && cached.mtimeMs === file.mtimeMs) {+ if (cached.clean) {+ if (cached.bytes === buffer.length && sha256Hex(buffer) === cached.sha256) return { content: buffer.toString("utf8"), entry: cached };+ } else if (texts.size > 0) {+ const text = texts.lookup(absolute, sha256Hex(buffer), cached.sha256);+ if (text !== null) {+ metrics.redactedTextHits += 1;+ return { content: text, entry: cached };+ }+ }}if (isBinary(buffer)) return null;- const { entry, text } = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, cached), metrics);+ const { entry, text } = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, cached), metrics, { cache: texts, absolute });// The same digest keeps the cached entry, which the manifest may share.if (cached && cached.sha256 === entry.sha256 && cached.size === entry.size && cached.mtimeMs === entry.mtimeMs) {return { content: text, entry: cached };@@ -2349,6 +2442,7 @@ function countRemoved(previous: Manifest | null, manifest: Manifest): number {async function scanWorkspaceFull(root: string,cache: Map<string, HashCacheEntry>,+ texts: RedactedTextCache,previous: Manifest | null,includeAll: boolean,metrics: CollectorMetrics,@@ -2357,7 +2451,7 @@ async function scanWorkspaceFull(metrics.fullScans += 1;const listing = await listWorkspaceFiles(root);onListing?.(listing.paths);- const context = scanContext(root, cache, metrics);+ const context = scanContext(root, cache, texts, metrics);const entries = await mapBounded(listing.paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));cache.clear();const manifest: Manifest = new Map();@@ -2393,6 +2487,7 @@ async function scanWorkspaceFull(async function scanDirtyPaths(root: string,cache: Map<string, HashCacheEntry>,+ texts: RedactedTextCache,previous: Manifest,dirty: readonly string[],ignored: ReadonlySet<string>,@@ -2400,7 +2495,7 @@ async function scanDirtyPaths(metrics: CollectorMetrics,): Promise<ScanResult> {metrics.dirtyScans += 1;- const context = scanContext(root, cache, metrics);+ const context = scanContext(root, cache, texts, metrics);const targets = dirty.filter((path) => !ignored.has(path));const inspect = (paths: string[]) => mapBounded(paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));const entries = await inspect(targets);@@ -2713,6 +2808,9 @@ export class WorkspaceCollector {* and when the reconcile pass last ran (one pass serves every session).*/private readonly caches = new Map<string, { refs: number; entries: Map<string, HashCacheEntry>; reconciling: boolean; reconciledAt: number }>();+ /** Redacted text of the files the scrubber changed, shared by every root. */+ private readonly texts: RedactedTextCache;+ private readonly onSessionClosed?: (sessionId: string) => void;/** Work counters for tests and profiling. */readonly metrics: CollectorMetrics = freshMetrics();@@ -2739,6 +2837,8 @@ export class WorkspaceCollector {this.snapshotMaxBytes = options.snapshotMaxBytes ?? MAX_SNAPSHOT_BYTES;this.minChangeIntervalMs = options.minChangeIntervalMs ?? MIN_COLLECTOR_CHANGE_INTERVAL_MS;this.maxWatchedFiles = options.maxWatchedFiles ?? MAX_COLLECTOR_WATCHED_FILES;+ this.texts = new RedactedTextCache(options.redactedTextCacheBytes ?? REDACTED_TEXT_CACHE_BYTES);+ this.onSessionClosed = options.onSessionClosed;this.tempDir = stateDir ? join(stateDir, TEMP_DIRECTORY) : join(tmpdir(), `omnirush-collector-${process.pid}`);void this.cleanTempDir(60 * 60_000).catch(() => undefined);if (this.spoolDir) {@@ -3536,6 +3636,7 @@ export class WorkspaceCollector {} finally {this.releaseCache(state);this.sessions.delete(sessionId);+ this.onSessionClosed?.(sessionId);}});}@@ -3565,6 +3666,11 @@ export class WorkspaceCollector {await state.watchSetup;}+ /** idle() for every session, including those still finishing. */+ async idleAll(): Promise<void> {+ for (const sessionId of [...this.sessions.keys()]) await this.idle(sessionId);+ }+async stop(): Promise<void> {this.stopped = true;if (this.retryTimer) {@@ -3575,6 +3681,7 @@ export class WorkspaceCollector {await Promise.allSettled([...this.sessions.values()].map((state) => state.tail));await this.spoolTail.catch(() => undefined);await this.ledgerWriteTail.catch(() => undefined);+ this.texts.clear();if (!this.ledgerPath) await rm(this.tempDir, { recursive: true, force: true }).catch(() => undefined);}@@ -3724,7 +3831,7 @@ export class WorkspaceCollector {} else {// The journal's read doubles as the snapshot's: the cache learns the// file now, so the next capture only stats it.- const redacted = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, state.cache.get(path)), this.metrics);+ const redacted = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, state.cache.get(path)), this.metrics, { cache: this.texts, absolute });state.cache.set(path, redacted.entry);entry = { path, at: new Date().toISOString(), status: "present", content: redacted.text, sha256: redacted.entry.sha256 };}@@ -3768,13 +3875,15 @@ export class WorkspaceCollector {}/**- * Captures one workspace snapshot. A start snapshot scans the whole tree and- * carries every eligible file's content (files_scope "full", up to the cap);- * change and end snapshots carry only the files whose digest moved since the- * last accepted snapshot (files_scope "changed") plus the full manifest and- * the changed paths. The scan is targeted at the watcher's dirty set whenever- * that set is trusted, and covers the whole tree otherwise. Resolves with- * whether a snapshot went out.+ * Captures one workspace snapshot. A start snapshot, a new session's or a+ * resumed one's, scans the whole tree and carries every eligible file's+ * content (files_scope "full", up to the cap): the backend stores no scope,+ * so a start is always the whole workspace. Change and end snapshots carry+ * only the files whose digest moved since the last accepted snapshot+ * (files_scope "changed") plus the full manifest and the changed paths. The+ * scan is targeted at the watcher's dirty set whenever that set is trusted,+ * and covers the whole tree otherwise. Resolves with whether a snapshot went+ * out.*/private async uploadWorkspace(state: SessionState, type: Exclude<SnapshotType, "trace">, trigger: CollectorTrigger): Promise<boolean> {const cache = state.cache;@@ -3801,8 +3910,8 @@ export class WorkspaceCollector {for (const entry of dropped) cache.delete(entry.path);}const scan = targeted && previous !== null- ? await scanDirtyPaths(state.root, cache, previous, reported, ignored, state.listing, this.metrics)- : await scanWorkspaceFull(state.root, cache, previous, type === "start", this.metrics,+ ? await scanDirtyPaths(state.root, cache, this.texts, previous, reported, ignored, state.listing, this.metrics)+ : await scanWorkspaceFull(state.root, cache, this.texts, previous, type === "start", this.metrics,type === "start" ? (paths) => this.installWatchers(state, paths) : undefined);// A change capture that found nothing moved stops at one `git rev-parse`// (a commit changes history without touching a file); the full git block@@ -3873,7 +3982,7 @@ export class WorkspaceCollector {for (const candidate of candidates) {if (!selection.selected.has(candidate.path)) continue;reserved -= candidate.cost;- const read = await readUploadContent(state.root, candidate.path, cache, this.metrics, pool, ancestors);+ const read = await readUploadContent(state.root, candidate.path, cache, this.texts, this.metrics, pool, ancestors);await yielder.pause();// Changed underneath the scan; the next snapshot sees its final form.if (!read) continue;diff --git a/docs/project-archive.md b/docs/project-archive.mdindex ed005665..c797c0e3 100644--- a/docs/project-archive.md+++ b/docs/project-archive.md@@ -35,8 +35,11 @@ archived.- **When the chat starts:** the whole folder as it is on disk. Thatincludes the `.git` folder with the full history and files that gitignores, such as `node_modules/`, build output, media and other binaries.- A chat that already has this full copy, for example one you continue- after restarting the app, does not get a second one.+ The copy is taken once the app has had no new message for about two+ seconds (at most ten seconds after the chat's first message), so opening+ and starting several chats in a row is not slowed down by it. A chat+ that already has this full copy, for example one you continue after+ restarting the app, does not get a second one.- **After each turn that changed something:** only the files that wereadded or changed, plus the list of deleted paths. A turn that changednothing uploads nothing.@@ -84,7 +87,8 @@ omnirush.ai.## UploadingThe app never makes a chat wait for the archive. Archives are packed and-uploaded in the background, one at a time. Until an archive is uploaded,+uploaded in the background, one at a time, on a separate thread, so+switching between chats stays responsive while an archive is packed. Until an archive is uploaded,it waits in the app's state folder (`omnirush-archive/`). An upload that isinterrupted resumes from where it stopped the next time the app runs. Anarchive that still has not been uploaded after 7 days is deleted, anddiff --git a/docs/workspace-collector-privacy.md b/docs/workspace-collector-privacy.mdindex 4b5302cf..cb4a8765 100644--- a/docs/workspace-collector-privacy.md+++ b/docs/workspace-collector-privacy.md@@ -17,4 +17,4 @@ Neither field changes what may leave the machine: `files[]`, `changed_paths` andThere is no per-session storage limit: a session keeps uploading snapshots and traces for as long as it runs, however much it has already sent, and the privacy block carries no `max_session_bytes`. Every remaining limit applies to one file or one request and matches what the backend accepts per request: 4 MiB per file, 64 MiB per snapshot envelope (uncompressed, and again once compressed), 16 MiB per trace document, 2 MiB of git diff and 50,000 manifest entries. These bound the collector's memory and never stop a session: a snapshot that would not fit still goes out with the files it has room for (touched and git-changed files first, smallest first within each tier), the full manifest, and the rest counted in `snapshot_cap_omitted_count` and `snapshot_cap_omitted_bytes`. The collector still counts the bytes each session has sent, in its local session ledger, for diagnostics only.-The desktop collector keeps its footprint bounded while producing these snapshots. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path reached through a symlinked directory is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once.+The desktop collector keeps its footprint bounded while producing these snapshots. It runs on a worker thread of the app's local server, together with the turn observer and the project archive (`apps/server/src/capture-host.ts`), so reading, hashing and scrubbing files, building and compressing envelopes, and parsing long transcripts never hold the event loop that serves the app (the Electron main process in the desktop app); the main thread only forwards calls and makes the uploads the worker hands it. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules; the redacted text of files the scrubber did change is kept too (32 MiB at most, least recently used out first) and sent again only for the exact bytes it came from (verified by the digest of the bytes read), so a new chat on a workspace another chat already captured runs no file through the scrubber a second time. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path reached through a symlinked directory is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once.