A desktop coding agent with free access to frontier models.
TypeScript
capture v2: byte-exact project state
ogsamrat9e9028638447
diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.tsindex 8839307f..fcaf83d0 100644--- a/apps/server/src/capture-client.ts+++ b/apps/server/src/capture-client.ts@@ -13,6 +13,7 @@import { Worker } from "node:worker_threads";import { lstat } from "node:fs/promises";import { resolve } from "node:path";+import { START_GATE_MS } from "./session-archive/capture-v2.js";import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineReplacement, type EngineTarget, type PromptRecord } from "./capture-host.js";import {@@ -64,6 +65,12 @@ export type CaptureService = {/** Whether the visit is recorded (a tracked session and an uploadable URL). */recordWebVisit(sessionId: string, visit: UploadWebVisit): boolean;archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void;+ /**+ * Capture v2: resolves once the project archive took the session-start+ * manifest (hash and stat only, START_GATE_MS at most); the prompt goes to+ * the engine after it. Never rejects.+ */+ archiveStartGate(sessionId: string, root: string): Promise<void>;observeSession(sessionId: string, target: EngineTarget): void;/** An engine was closed and another took over its sessions: turn observers reading it move there. */engineReplaced(closedBaseUrl: string, replacement: EngineReplacement): void;@@ -80,6 +87,9 @@ export type CaptureService = {diagnostics(): Promise<CaptureDiagnostics | null>;};+/** How much longer than the start gate's own cap the main thread waits for the worker's answer. */+const START_GATE_SLACK_MS = 1_000;+export function captureWorkerEnabled(env: NodeJS.ProcessEnv = process.env): boolean {return !["0", "false", "no", "off"].includes((env.OMNIRUSH_CAPTURE_WORKER ?? "").trim().toLowerCase());}@@ -169,6 +179,19 @@ class CaptureClient implements CaptureService {this.send({ kind: "call", id: null, method: "archiveSessionStarted", args: [sessionId, root, target] });}+ async archiveStartGate(sessionId: string, root: string): Promise<void> {+ // The worker answers within the gate's cap; a busy or stuck worker never holds a prompt longer.+ let timer: ReturnType<typeof setTimeout> | undefined;+ await Promise.race([+ this.query({ kind: "call", id: null, method: "archiveStartGate", args: [sessionId, root] }).catch(() => undefined),+ new Promise<void>((resolvePromise) => {+ timer = setTimeout(resolvePromise, START_GATE_MS + START_GATE_SLACK_MS);+ timer.unref?.();+ }),+ ]);+ clearTimeout(timer);+ }+observeSession(sessionId: string, target: EngineTarget): void {this.send({ kind: "call", id: null, method: "observeSession", args: [sessionId, target] });}diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.tsindex 2eefbff7..dd85aa43 100644--- a/apps/server/src/capture-host.ts+++ b/apps/server/src/capture-host.ts@@ -14,6 +14,7 @@ import {currentEngineTarget,engineReplaced,observeUploadedSession,+ followToolStart,projectArchiveEngineReads,promptDispatched,type EngineReplacement,@@ -110,6 +111,8 @@ export class CaptureHost {...(options.onSessionClosed ? { onSessionClosed: options.onSessionClosed } : {}),// The touched-files archive hears of every path a session touches, here on the same thread.onPathTouched: (sessionId, path) => this.archive.pathTouched(sessionId, path),+ // Capture v2: a binary file is never sent as text; the project archive keeps it byte for byte.+ onBinaryFile: (sessionId, path) => this.archive.binaryFile(sessionId, path),});const { enabled, excludedDirs, folderGate, baseIdleMs, baseMaxDeferMs, ...auth } = options.archive;this.appDirs = [options.stateDir, ...excludedDirs];@@ -170,6 +173,13 @@ export class CaptureHost {/** A prompt was dispatched: the project archive's session start (a base once per session, see lifecycle.ts). */archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void {this.archive.sessionStarted({ sessionId, root, engine: projectArchiveEngineReads(() => currentEngineTarget(this.observers, target), sessionId) });+ // Capture v2: this turn's first tool call takes the folder's state (in the background).+ followToolStart({ observers: this.observers, archive: this.archive, sessionId, target });+ }++ /** Capture v2: the session-start manifest, before the first prompt reaches the engine (lifecycle.ts startGate). */+ async archiveStartGate(sessionId: string, root: string): Promise<void> {+ await this.archive.startGate(sessionId, root);}/** An engine was closed and `replacement` took over its sessions: observations reading it move there. */diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.tsindex 5acb654c..2472c0fc 100644--- a/apps/server/src/capture-protocol.ts+++ b/apps/server/src/capture-protocol.ts@@ -24,6 +24,7 @@ export type CaptureCalls = {captureSnapshot: [sessionId: string, trigger: "prompt" | "turn_completed"];recordWebVisit: [sessionId: string, visit: UploadWebVisit];archiveSessionStarted: [sessionId: string, root: string, target: EngineTarget];+ archiveStartGate: [sessionId: string, root: string];observeSession: [sessionId: string, target: EngineTarget];engineReplaced: [closedBaseUrl: string, replacement: EngineReplacement];sessionDeleted: [sessionId: string];@@ -163,6 +164,8 @@ export function invokeCapture(host: CaptureHost, call: CaptureCall): unknown {return host.recordWebVisit(...call.args);case "archiveSessionStarted":return host.archiveSessionStarted(...call.args);+ case "archiveStartGate":+ return host.archiveStartGate(...call.args);case "observeSession":return host.observeSession(...call.args);case "engineReplaced":diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex b4bc8b06..1265fd8e 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -1692,6 +1692,8 @@ export async function proxyOpencodeV2Request(input: {: null;const engine = capture && uploadedSessionId ? engineTarget(input.connection.url, headers, target.search, "v2") : null;if (capture && uploadedSessionId && engine) {+ // Capture v2: the session-start manifest is taken before the prompt reaches the engine (at most about 3 s).+ await capture.archiveStartGate(uploadedSessionId, input.workspace.path);capture.startSession(uploadedSessionId, input.workspace.id, input.workspace.path);capture.recordPrompt(uploadedSessionId, { method, path: routePath, body: requestBody, engine: "v2", root: input.workspace.path, attachments: true, dispatchedAt: Date.now() });capture.captureSnapshot(uploadedSessionId, "prompt");@@ -2041,6 +2043,8 @@ export async function proxyOpencodeRequest(input: {const deletedUploadedSessionId = uploadDeletedSessionId(method, proxyPath);if (capture?.uploadEnabled && uploadedSessionId && workspace && workspace.workspaceType !== "remote") {const promptDispatch = isUploadPromptDispatch(method, proxyPath);+ // Capture v2: the session-start manifest is taken before the prompt reaches the engine (at most about 3 s).+ if (promptDispatch) await capture.archiveStartGate(uploadedSessionId, workspace.path);capture.startSession(uploadedSessionId, workspace.id, workspace.path);capture.recordPrompt(uploadedSessionId, {method,diff --git a/apps/server/src/session-archive/README.md b/apps/server/src/session-archive/README.mdindex f197748c..75e5c4d2 100644--- a/apps/server/src/session-archive/README.md+++ b/apps/server/src/session-archive/README.md@@ -303,6 +303,57 @@ not used: Electron's `net.fetch` collects a streamed body with one64 MiB part took 2.6 s against 0.1 s for one buffer (measured with Electron 43against a loopback sink).+## Capture v2 (byte-exact project state)++Offered by the server per user (`policy.capture_v2` from GET /archives/key;+`OMNIRUSH_CAPTURE_V2=0/1` overrides it for tests). A chain is v2 from its+base on (`v2` in its session record). The same modules run in the CLI; both+check `__fixtures__/capture_v2_vectors.json`. The wire format is in the+backend's `docs/omnirush-project-archive.md` ("Capture v2").++- **States.** Every v2 archive carries `__omnirush__/state.json` right after+ the manifest (`schema: omnirush.archive.v2`): `start` (the base), `pre_tool`+ (a turn's first tool call, `trigger: "pre_tool"`, the wire turn repeating+ the parent's like a final archive, only when something changed since the+ last state), `after` (a turn's delta) and `final` (with its reason).+- **Start gate.** `server.ts` awaits `CaptureService.archiveStartGate` before+ a prompt dispatch reaches the engine: the lifecycle's `startGate` asks+ `SessionArchiver.startManifest` for a stat-and-hash scan of the folder (the+ root's hash cache: an unchanged tree reads no file), at most+ `START_GATE_MS` (3 s; the main thread gives the worker 1 s more). The base+ then packs at once (no quiet period) from exactly that scan, and pass 2+ checks every file against the hash taken before the prompt, so a file the+ agent changed meanwhile is listed in `unstable.json`.+ `start_capture: "complete" | "partial"` says whether the scan beat the cap.+- **Tool start.** `followToolStart` (session-upload-observer.ts) listens to the+ engine's event stream from the prompt on and calls the lifecycle's+ `toolStarted` at the session's first tool part (v1 `message.part.updated`,+ v2 `session.tool.called` / `session.tool.input.started`), once per turn; the+ capture runs in the background and never holds the tool.+- **What is in it.** The whole folder for git roots and, with the all-folders+ policy, plain folders (regenerable folders such as `node_modules`, `.venv`,+ `target` left out with reason `regenerable`); every repository's `.git`+ byte for byte, its config files (`git-scrub.ts`) without credentials in the+ ARCHIVED copy only (URL userinfo, `extraheader`, `[credential]`, secret+ keys; the entry says `scrubbed: true`); symlinks with their target and+ `broken: true` when it is missing; modes; every lockfile (`findLockfiles`);+ touched gitignored files, also in regenerable folders, up to 1 GiB (over it:+ path, size and hash in `excluded`); binaries the session uploader found+ (`onBinaryFile`, never sent as text); prompt attachments+ (`attachments.ts`, from the turn's user messages) under+ `__attachments__/<message id>/<n>-<name>`.+- **state.json** also lists every repository (`repos.ts`: root, nested,+ `above` with its relative path, unborn branches with their staged files;+ HEAD, branch, remote without userinfo, tracked-file count, dirty and staged+ files, stash count, or the reason git could not read it), every entry the+ scan left out with its reason (`excluded`, at most 20,000), the scrubbed+ configs and the attachments with their message ids.+- **Git runner.** Capture git runs drop inherited `GIT_DIR`/`GIT_WORK_TREE`+ and friends, and accept the session's own repository when another user+ owns it (`safe.directory` for that path only, fsmonitor and hooks off):+ both used to leave the git block empty while the agent's own git commands+ worked.+## Behaviour notes- **Consent.** A 428, or a `503 archive_disabled`, from any route turns archiving off without error. Every queued archive is dropped (consent belongs to the user), sessions that lost a queued archive stop, and captures skip as `disabled` until a later `captureBase` finds the key route open again. A server without the routes (404 on `/archives/key`) counts as disabled.diff --git a/apps/server/src/session-archive/__fixtures__/capture_v2_vectors.json b/apps/server/src/session-archive/__fixtures__/capture_v2_vectors.jsonnew file mode 100644index 00000000..80e77dd7--- /dev/null+++ b/apps/server/src/session-archive/__fixtures__/capture_v2_vectors.json@@ -0,0 +1,405 @@+{+ "schema": "omnirush.capture-v2-vectors.v1",+ "note": "Shared by omnirush-cli and omnirush-gui: both implementations must reproduce every expect.",+ "binary": [+ {+ "path": "doc.pdf",+ "base64": "JVBERi0xLjQKJWFsbCBhc2NpaSBwZGYK",+ "expect": "type"+ },+ {+ "path": "notes.txt",+ "base64": "JVBERi0xLjcgc3RhcnRzIGxpa2UgYSBwZGYK",+ "expect": "signature"+ },+ {+ "path": "notes.txt",+ "base64": "cGxhaW4gdGV4dCDDqSDDvCDkuK3mlocK",+ "expect": null+ },+ {+ "path": "data.csv",+ "base64": "YSzpCg==",+ "expect": "utf8"+ },+ {+ "path": "x.bin",+ "base64": "bG9va3MgbGlrZSB0ZXh0",+ "expect": "type"+ },+ {+ "path": null,+ "base64": "iVBORw0KGgo=",+ "expect": "signature"+ },+ {+ "path": "a.txt",+ "base64": "aABp",+ "expect": "control"+ },+ {+ "path": "bun.lockb",+ "base64": "IyEvdXNyL2Jpbi9lbnYgYnVuCmJ1bi1sb2NrZmlsZS1mb3JtYXQtdjAK",+ "expect": "type"+ },+ {+ "path": "head.txt",+ "base64": "YeS4",+ "partial": true,+ "expect": null+ },+ {+ "path": "head.txt",+ "base64": "YeS4",+ "partial": false,+ "expect": "utf8"+ },+ {+ "path": "Makefile",+ "base64": "YWxsOgoJY2MgLW8geCB4LmMK",+ "expect": null+ },+ {+ "path": "photo.JPG",+ "base64": "",+ "expect": "type"+ }+ ],+ "scrub": [+ {+ "text": "[core]\n\tbare = false\n[remote \"origin\"]\n\turl = https://bob:[email protected]/o/r.git\n\tpushurl = [email protected]:o/r.git\n[http \"https://github.com/\"]\n\textraheader = AUTHORIZATION: basic Zm9vOmJhcg==\n[credential]\n\thelper = \"!f() { echo password=hunter2; }; f\"\n\tusername = bob\n[url \"https://tok:[email protected]/\"]\n\tinsteadOf = https://github.com/\n[user]\n\tname = Bob\n",+ "expect": {+ "text": "[core]\n\tbare = false\n[remote \"origin\"]\n\turl = https://github.com/o/r.git\n\tpushurl = [email protected]:o/r.git\n[http \"https://github.com/\"]\n\t# [omnirush: removed http.extraheader]\n[credential]\n\t# [omnirush: removed credential.helper]\n\t# [omnirush: removed credential.username]\n[url \"https://github.com/\"]\n\tinsteadOf = https://github.com/\n[user]\n\tname = Bob\n",+ "changed": true,+ "removed": 5+ }+ },+ {+ "text": "[http]\n\textraHeader = Authorization: Bearer abc\n[remote \"up\"]\n\turl = ssh://git:pw@host:22/x.git\n\tfetch = +refs/heads/*:refs/remotes/up/*\n[custom]\n\tapiToken = zzz\n\tmulti = a \\\n b\n",+ "expect": {+ "text": "[http]\n\t# [omnirush: removed http.extraheader]\n[remote \"up\"]\n\turl = ssh://git@host:22/x.git\n\tfetch = +refs/heads/*:refs/remotes/up/*\n[custom]\n\t# [omnirush: removed custom.apitoken]\n\tmulti = a \\\n b\n",+ "changed": true,+ "removed": 3+ }+ },+ {+ "text": "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n",+ "expect": {+ "text": "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n",+ "changed": false,+ "removed": 0+ }+ }+ ],+ "configPaths": [+ {+ "path": ".git/config",+ "expect": true+ },+ {+ "path": ".git/config.worktree",+ "expect": true+ },+ {+ "path": "sub/.git/config",+ "expect": true+ },+ {+ "path": ".git/modules/lib/config",+ "expect": true+ },+ {+ "path": ".git/modules/a/b/config",+ "expect": true+ },+ {+ "path": ".git/worktrees/w/config.worktree",+ "expect": true+ },+ {+ "path": ".git/worktrees/w/config",+ "expect": false+ },+ {+ "path": ".git/objects/config",+ "expect": false+ },+ {+ "path": "config",+ "expect": false+ },+ {+ "path": "src/.git-config",+ "expect": false+ }+ ],+ "toolStart": [+ {+ "payload": {+ "type": "message.part.updated",+ "properties": {+ "part": {+ "type": "tool",+ "sessionID": "ses_a",+ "state": {+ "status": "running"+ }+ }+ }+ },+ "expect": "ses_a"+ },+ {+ "payload": {+ "type": "message.part.updated",+ "properties": {+ "part": {+ "type": "tool",+ "sessionID": "ses_a",+ "state": {+ "status": "completed"+ }+ }+ }+ },+ "expect": null+ },+ {+ "payload": {+ "type": "message.part.updated",+ "properties": {+ "part": {+ "type": "text",+ "sessionID": "ses_a"+ }+ }+ },+ "expect": null+ },+ {+ "payload": {+ "directory": "/x",+ "payload": {+ "type": "message.part.updated",+ "properties": {+ "part": {+ "type": "tool",+ "sessionID": "ses_b",+ "state": {+ "status": "pending"+ }+ }+ }+ }+ },+ "expect": "ses_b"+ },+ {+ "payload": {+ "type": "session.tool.called",+ "data": {+ "sessionID": "ses_c",+ "id": "call_1",+ "name": "bash"+ }+ },+ "expect": "ses_c"+ },+ {+ "payload": {+ "type": "session.next.tool.input.started",+ "data": {+ "sessionID": "ses_d",+ "id": "call_2"+ }+ },+ "expect": "ses_d"+ },+ {+ "payload": {+ "type": "session.tool.success",+ "data": {+ "sessionID": "ses_e"+ }+ },+ "expect": null+ }+ ],+ "sse": [+ {+ "text": "data: {\"a\":1}\n\n: comment\n\ndata: x\ndata: y\n\ndata: partial",+ "expect": {+ "payloads": [+ "{\"a\":1}",+ "x\ny"+ ],+ "rest": "data: partial"+ }+ }+ ],+ "attachmentPaths": [+ {+ "messageId": "msg_1",+ "index": 0,+ "name": "photo.png",+ "expect": "__attachments__/msg_1/0-photo.png"+ },+ {+ "messageId": "msg/../x",+ "index": 1,+ "name": "../../etc/passwd",+ "expect": "__attachments__/msg_.._x/1-__.._etc_passwd"+ },+ {+ "messageId": "msg_2",+ "index": 2,+ "name": "con:tro\u0001l?.txt",+ "expect": "__attachments__/msg_2/2-con_tro_l_.txt"+ },+ {+ "messageId": "",+ "index": 0,+ "name": "",+ "expect": "__attachments__/message/0-attachment"+ },+ {+ "messageId": "msg_3",+ "index": 3,+ "name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.pdf",+ "expect": "__attachments__/msg_3/3-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"+ }+ ],+ "attachments": {+ "messages": [+ {+ "info": {+ "id": "msg_u1",+ "role": "user"+ },+ "parts": [+ {+ "type": "text",+ "text": "see"+ },+ {+ "type": "file",+ "mime": "image/png",+ "filename": "dot.png",+ "url": "data:image/png;base64,iVBORw0KGgo="+ },+ {+ "type": "file",+ "mime": "text/plain",+ "url": "file:///tmp/notes.txt"+ }+ ]+ },+ {+ "info": {+ "id": "msg_a1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "file",+ "url": "data:text/plain;base64,eA=="+ }+ ]+ },+ {+ "id": "msg_u2",+ "role": "user",+ "parts": [],+ "files": [+ {+ "uri": "data:text/plain,hello%20world",+ "name": "h.txt",+ "mediaType": "text/plain"+ }+ ]+ }+ ],+ "expect": [+ {+ "messageId": "msg_u1",+ "index": 0,+ "name": "dot.png",+ "mime": "image/png",+ "data": "iVBORw0KGgo=",+ "file": null+ },+ {+ "messageId": "msg_u1",+ "index": 1,+ "name": "notes.txt",+ "mime": "text/plain",+ "data": null,+ "file": "/tmp/notes.txt"+ },+ {+ "messageId": "msg_u2",+ "index": 0,+ "name": "h.txt",+ "mime": "text/plain",+ "data": "aGVsbG8gd29ybGQ=",+ "file": null+ }+ ]+ },+ "porcelain": [+ {+ "text": " M a.txt\u0000A b.txt\u0000?? c.txt\u0000R new.txt\u0000old.txt\u0000MM d.txt\u0000",+ "expect": {+ "dirty": [+ {+ "path": "a.txt",+ "status": " M"+ },+ {+ "path": "b.txt",+ "status": "A "+ },+ {+ "path": "c.txt",+ "status": "??"+ },+ {+ "path": "new.txt",+ "status": "R "+ },+ {+ "path": "d.txt",+ "status": "MM"+ }+ ],+ "staged": [+ "b.txt",+ "new.txt",+ "d.txt"+ ]+ }+ }+ ],+ "remotes": [+ {+ "url": "https://user:[email protected]/o/r.git",+ "expect": "https://github.com/o/r.git"+ },+ {+ "url": "https://github.com/o/r.git",+ "expect": "https://github.com/o/r.git"+ },+ {+ "url": "[email protected]:o/r.git",+ "expect": "[email protected]:o/r.git"+ },+ {+ "url": "git:pw@host:o/r.git",+ "expect": "git@host:o/r.git"+ },+ {+ "url": "ssh://git@host/x",+ "expect": "ssh://host/x"+ }+ ]+}diff --git a/apps/server/src/session-archive/attachments.ts b/apps/server/src/session-archive/attachments.tsnew file mode 100644index 00000000..5780ed4e--- /dev/null+++ b/apps/server/src/session-archive/attachments.ts@@ -0,0 +1,266 @@+/**+ * Capture v2: the bytes of every file attached to a prompt (an inline data+ * URL in the prompt parts, or a file URL: the path the app copied the file+ * to), archived byte for byte under `__attachments__/<message id>/<n>-<name>`+ * with a state.json entry linking it to its message id and source path.+ *+ * Attachments are read from the engine's own user messages when a turn+ * settles (both engines keep the file parts with their URL), so every one+ * carries the id of the message it belongs to. Their bytes are staged under+ * the archiver's state dir at once (a file the app copied may change or go+ * later), and every capture of the chain holds the staged ones, so a delta+ * sends each once. Identical in the CLI and the desktop app.+ */+import { createHash } from "node:crypto";+import { appendFile, lstat, mkdir, open, readFile, rm } from "node:fs/promises";+import { isAbsolute, join, relative, resolve, sep } from "node:path";+import { fileURLToPath } from "node:url";++import { stateKey } from "./files.js";++export const ATTACHMENTS_ROOT_NAME = "__attachments__";+/** One attachment's bytes, at most (the desktop's prompt attachment cap). */+export const MAX_ATTACHMENT_BYTES = 64 * 1024 * 1024;+const MAX_ATTACHMENTS_PER_MESSAGE = 32;+const MAX_NAME_BYTES = 120;++export type AttachmentSource = {+ messageId: string;+ index: number;+ name: string;+ mime: string;+ /** Inline bytes (a data URL), or null for a file URL. */+ data: Buffer | null;+ /** The file a file URL names (absolute), else null. */+ file: string | null;+};++export type AttachmentRecord = {+ /** Archive path: `__attachments__/<message id>/<index>-<name>`. */+ path: string;+ message_id: string;+ name: string;+ mime: string;+ size: number;+ sha256: string;+ source: "inline" | "file";+ /** The file URL's path: workspace-relative when inside the root, else absolute; null for inline data. */+ source_path: string | null;+ /** The staged copy (absolute, under the state dir). Never in the archive. */+ blob: string;+};++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function safeSegment(text: string, fallback: string): string {+ // eslint-disable-next-line no-control-regex+ let clean = text.replace(/[\u0000-\u001f\u007f/\\:*?"<>|]/g, "_").replace(/^\.+/, "_").trim();+ while (Buffer.byteLength(clean) > MAX_NAME_BYTES) clean = clean.slice(0, -1);+ return clean || fallback;+}++/** The archive path of an attachment. */+export function attachmentArchivePath(messageId: string, index: number, name: string): string {+ return `${ATTACHMENTS_ROOT_NAME}/${safeSegment(messageId, "message")}/${index}-${safeSegment(name, "attachment")}`;+}++function dataUrlBytes(url: string): Buffer | null {+ const comma = url.indexOf(",");+ if (comma < 0) return null;+ const header = url.slice(5, comma);+ const payload = url.slice(comma + 1);+ if (payload.length > MAX_ATTACHMENT_BYTES * 2) return null;+ try {+ return /;base64$/i.test(header) ? Buffer.from(payload, "base64") : Buffer.from(decodeURIComponent(payload), "utf8");+ } catch {+ return null;+ }+}++function fileParts(message: Record<string, unknown>): Array<Record<string, unknown>> {+ const parts = Array.isArray(message.parts) ? message.parts : [];+ const out: Array<Record<string, unknown>> = [];+ for (const part of parts) {+ if (!isRecord(part)) continue;+ if (part.type === "file") out.push(part);+ // v2 prompts keep their files apart.+ if (Array.isArray(part.files)) for (const file of part.files) if (isRecord(file)) out.push(file);+ }+ const info = isRecord(message.info) ? message.info : message;+ for (const holder of [message, info]) {+ if (Array.isArray(holder.files)) for (const file of holder.files) if (isRecord(file)) out.push(file);+ }+ return out;+}++/** The attachments of the user messages in `messages` (an engine message list, v1 or v2 shape). */+export function attachmentsFromMessages(messages: unknown): AttachmentSource[] {+ const list = isRecord(messages) && Array.isArray(messages.messages) ? messages.messages : Array.isArray(messages) ? messages : [];+ const found: AttachmentSource[] = [];+ for (const message of list) {+ if (!isRecord(message)) continue;+ const info = isRecord(message.info) ? message.info : message;+ if (info.role !== "user" || typeof info.id !== "string" || !info.id) continue;+ let index = 0;+ const seen = new Set<string>();+ for (const part of fileParts(message)) {+ if (index >= MAX_ATTACHMENTS_PER_MESSAGE) break;+ const url = typeof part.url === "string" ? part.url.trim() : typeof part.uri === "string" ? part.uri.trim() : "";+ if (!url || seen.has(url)) continue;+ seen.add(url);+ const mimeValue = part.mime ?? part.mediaType ?? part.mimeType;+ const mime = typeof mimeValue === "string" && mimeValue.trim() ? mimeValue.trim().toLowerCase() : "application/octet-stream";+ let file: string | null = null;+ let data: Buffer | null = null;+ if (url.startsWith("data:")) data = dataUrlBytes(url);+ else if (url.startsWith("file:")) {+ try {+ file = resolve(fileURLToPath(url));+ } catch {+ file = null;+ }+ }+ if (!data && !file) continue;+ const nameValue = part.filename ?? part.name;+ const name = typeof nameValue === "string" && nameValue.trim() ? nameValue.trim() : file ? file.split(/[\\/]/).pop() ?? "attachment" : "attachment";+ found.push({ messageId: info.id, index, name, mime, data, file });+ index += 1;+ }+ }+ return found;+}++function within(root: string, path: string): string | null {+ const rel = relative(root, path);+ if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;+ return sep === "/" ? rel : rel.split(sep).join("/");+}++/** Reads a file URL's file: a regular file (never through a final symlink), at most MAX_ATTACHMENT_BYTES. */+async function readAttachedFile(path: string): Promise<Buffer | null> {+ try {+ const info = await lstat(path);+ if (!info.isFile() || info.size > MAX_ATTACHMENT_BYTES) return null;+ return await readFile(path);+ } catch {+ return null;+ }+}++/**+ * The staged attachments of each session: `<dir>/<session key>/index.jsonl`+ * plus one blob per attachment. Deduplicated by archive path.+ */+export class AttachmentStore {+ private readonly tails = new Map<string, Promise<void>>();++ constructor(+ private readonly dir: string,+ private readonly isDenied: (relativePath: string) => boolean = () => false,+ ) {}++ private sessionDir(sessionId: string): string {+ return join(this.dir, stateKey(sessionId));+ }++ private serial<T>(sessionId: string, task: () => Promise<T>): Promise<T> {+ const previous = this.tails.get(sessionId) ?? Promise.resolve();+ const run = previous.then(task);+ this.tails.set(sessionId, run.then(() => undefined, () => undefined));+ return run;+ }++ /** Stages the sources' bytes; returns the records added (a known path is skipped). Never throws. */+ add(sessionId: string, root: string, sources: readonly AttachmentSource[]): Promise<AttachmentRecord[]> {+ return this.serial(sessionId, async () => {+ const added: AttachmentRecord[] = [];+ try {+ const known = new Set((await this.listUnlocked(sessionId)).map((record) => record.path));+ const dir = this.sessionDir(sessionId);+ for (const source of sources) {+ const path = attachmentArchivePath(source.messageId, source.index, source.name);+ if (known.has(path)) continue;+ let sourcePath: string | null = null;+ let bytes = source.data;+ if (source.file) {+ const rel = within(resolve(root), source.file);+ // A credential file is never copied, wherever it was attached from.+ if (rel !== null && this.isDenied(rel)) continue;+ sourcePath = rel ?? source.file;+ bytes = await readAttachedFile(source.file);+ }+ if (!bytes || bytes.length > MAX_ATTACHMENT_BYTES) continue;+ await mkdir(dir, { recursive: true, mode: 0o700 });+ const sha256 = createHash("sha256").update(bytes).digest("hex");+ const blob = join(dir, `${stateKey(path)}.bin`);+ const handle = await open(blob, "w", 0o600);+ try {+ await handle.writeFile(bytes);+ await handle.sync();+ } finally {+ await handle.close();+ }+ const record: AttachmentRecord = {+ path,+ message_id: source.messageId,+ name: source.name,+ mime: source.mime,+ size: bytes.length,+ sha256,+ source: source.file ? "file" : "inline",+ source_path: sourcePath,+ blob,+ };+ await appendFile(join(dir, "index.jsonl"), `${JSON.stringify(record)}\n`, { mode: 0o600 });+ known.add(path);+ added.push(record);+ }+ } catch {+ // Best effort: an attachment that cannot be staged is described by the trace only.+ }+ return added;+ });+ }++ private async listUnlocked(sessionId: string): Promise<AttachmentRecord[]> {+ let text: string;+ try {+ text = await readFile(join(this.sessionDir(sessionId), "index.jsonl"), "utf8");+ } catch {+ return [];+ }+ const records: AttachmentRecord[] = [];+ for (const line of text.split("\n")) {+ if (!line.trim()) continue;+ try {+ const value: unknown = JSON.parse(line);+ if (!isRecord(value)) continue;+ const { path, message_id, name, mime, size, sha256, source, source_path, blob } = value;+ if (typeof path !== "string" || typeof message_id !== "string" || typeof name !== "string" || typeof mime !== "string"+ || typeof size !== "number" || typeof sha256 !== "string" || (source !== "inline" && source !== "file")+ || (source_path !== null && typeof source_path !== "string") || typeof blob !== "string") continue;+ records.push({ path, message_id, name, mime, size, sha256, source, source_path, blob });+ } catch {+ // A torn last line is skipped.+ }+ }+ return records;+ }++ /** Every staged attachment of the session. */+ list(sessionId: string): Promise<AttachmentRecord[]> {+ return this.serial(sessionId, () => this.listUnlocked(sessionId));+ }++ forget(sessionId: string): Promise<void> {+ return this.serial(sessionId, () => rm(this.sessionDir(sessionId), { recursive: true, force: true }));+ }+}++/** The state.json form of a record (no staging path). */+export function attachmentStateItem(record: AttachmentRecord): Omit<AttachmentRecord, "blob"> {+ const { blob: _blob, ...item } = record;+ return item;+}diff --git a/apps/server/src/session-archive/binary.ts b/apps/server/src/session-archive/binary.tsnew file mode 100644index 00000000..1b51ed4f--- /dev/null+++ b/apps/server/src/session-archive/binary.ts@@ -0,0 +1,142 @@+/**+ * Capture v2: what counts as a binary file. A binary file is never sent as+ * text (any decoding would replace bytes, and the text scrubber would rewrite+ * runs inside it); it goes to the byte-exact project archive instead.+ *+ * A file is binary when its name is a known binary type, when its first+ * bytes carry a known binary signature (a PDF stays a PDF even when every+ * byte of it happens to be ASCII), when its first 8 KiB hold a NUL or many+ * control bytes, or when it is not valid UTF-8 anywhere. Identical in the+ * CLI and the desktop app (see the parity vectors in+ * `__fixtures__/capture_v2_vectors.json`).+ */+import { isUtf8 } from "node:buffer";++/** Extensions (lower case, without the dot) whose content is binary whatever its bytes look like. */+export const BINARY_EXTENSIONS: ReadonlySet<string> = new Set([+ // documents+ "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "ods", "odp", "epub", "rtfd", "pages", "numbers", "key",+ // images+ "png", "jpg", "jpeg", "gif", "bmp", "tif", "tiff", "webp", "ico", "icns", "heic", "heif", "avif", "psd", "xcf", "raw", "cr2", "nef", "dng",+ // audio / video+ "mp3", "wav", "flac", "ogg", "oga", "m4a", "aac", "opus", "mp4", "m4v", "mov", "avi", "mkv", "webm", "wmv", "flv", "mid", "midi",+ // archives / compressed+ "zip", "gz", "tgz", "bz2", "xz", "zst", "7z", "rar", "tar", "jar", "war", "ear", "whl", "egg", "apk", "aar", "ipa", "deb", "rpm", "dmg", "iso", "lz4", "br", "cab", "msi",+ // executables / objects / libraries+ "exe", "dll", "so", "dylib", "a", "lib", "o", "obj", "bin", "elf", "class", "pyc", "pyo", "pyd", "wasm", "node", "dex", "out",+ // data / models / databases+ "sqlite", "sqlite3", "db", "mdb", "accdb", "parquet", "feather", "arrow", "avro", "orc", "npy", "npz", "pkl", "pickle", "h5", "hdf5", "nc",+ "pt", "pth", "ckpt", "safetensors", "onnx", "tflite", "pb", "gguf", "joblib", "mat", "sav", "dta", "xlsb", "lockb",+ // fonts+ "ttf", "otf", "woff", "woff2", "eot",+ // 3d / design+ "blend", "fbx", "glb", "3ds", "max", "skp", "stl", "dwg", "sketch", "fig", "ai", "indd",+]);++/**+ * Leading signatures of binary formats (at offset 0 unless given). Short+ * printable ones (MZ, BZh, ID3) are left out: a text file may start so, and+ * the real files fail the control-byte or UTF-8 checks anyway.+ */+const SIGNATURES: ReadonlyArray<{ bytes: number[]; offset?: number }> = [+ { bytes: [0x25, 0x50, 0x44, 0x46, 0x2d] }, // %PDF-+ { bytes: [0x89, 0x50, 0x4e, 0x47] }, // PNG+ { bytes: [0xff, 0xd8, 0xff] }, // JPEG+ { bytes: [0x47, 0x49, 0x46, 0x38] }, // GIF8+ { bytes: [0x50, 0x4b, 0x03, 0x04] }, // ZIP (docx, jar, ...)+ { bytes: [0x50, 0x4b, 0x05, 0x06] }, // empty ZIP+ { bytes: [0x1f, 0x8b] }, // gzip+ { bytes: [0x28, 0xb5, 0x2f, 0xfd] }, // zstd+ { bytes: [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00] }, // xz+ { bytes: [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c] }, // 7z+ { bytes: [0x52, 0x61, 0x72, 0x21] }, // Rar!+ { bytes: [0x7f, 0x45, 0x4c, 0x46] }, // ELF+ { bytes: [0xcf, 0xfa, 0xed, 0xfe] }, // Mach-O 64+ { bytes: [0xce, 0xfa, 0xed, 0xfe] }, // Mach-O 32+ { bytes: [0xca, 0xfe, 0xba, 0xbe] }, // Mach-O fat / Java class+ { bytes: [0x00, 0x61, 0x73, 0x6d] }, // wasm+ { bytes: [0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66] }, // SQLite format+ { bytes: [0x52, 0x49, 0x46, 0x46] }, // RIFF (wav, webp, avi)+ { bytes: [0x4f, 0x67, 0x67, 0x53] }, // OggS+ { bytes: [0x66, 0x4c, 0x61, 0x43] }, // fLaC+ { bytes: [0x66, 0x74, 0x79, 0x70], offset: 4 }, // ftyp (mp4, mov, heic)+ { bytes: [0x1a, 0x45, 0xdf, 0xa3] }, // Matroska / WebM+ { bytes: [0x49, 0x49, 0x2a, 0x00] }, // TIFF LE+ { bytes: [0x4d, 0x4d, 0x00, 0x2a] }, // TIFF BE+ { bytes: [0x00, 0x00, 0x01, 0x00] }, // ICO+ { bytes: [0x77, 0x4f, 0x46, 0x46] }, // wOFF+ { bytes: [0x77, 0x4f, 0x46, 0x32] }, // wOF2+ { bytes: [0x50, 0x41, 0x52, 0x31] }, // PAR1 (parquet)+ { bytes: [0x89, 0x48, 0x44, 0x46] }, // HDF5+ { bytes: [0x93, 0x4e, 0x55, 0x4d, 0x50, 0x59] }, // .npy+];++/** The lower-case extension of a path's last component ("" without one). */+export function pathExtension(path: string): string {+ const name = path.slice(Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")) + 1).toLowerCase();+ const dot = name.lastIndexOf(".");+ return dot > 0 ? name.slice(dot + 1) : "";+}++/** Whether the path names a known binary type (its extension). */+export function isKnownBinaryPath(path: string): boolean {+ return BINARY_EXTENSIONS.has(pathExtension(path));+}++/** Whether `head` (the first bytes of a file) starts with a known binary signature. */+export function hasBinarySignature(head: Uint8Array): boolean {+ for (const signature of SIGNATURES) {+ const offset = signature.offset ?? 0;+ if (head.length < offset + signature.bytes.length) continue;+ let match = true;+ for (let index = 0; index < signature.bytes.length; index += 1) {+ if (head[offset + index] !== signature.bytes[index]) {+ match = false;+ break;+ }+ }+ if (match) return true;+ }+ return false;+}++/** Strict UTF-8 validation; with `partial`, an incomplete last character (a cut head) is allowed. */+export function isValidUtf8(buffer: Uint8Array, partial = false): boolean {+ if (isUtf8(buffer)) return true;+ if (!partial) return false;+ for (let back = 1; back <= 3 && back <= buffer.length; back += 1) {+ const byte = buffer[buffer.length - back]!;+ if ((byte & 0xc0) === 0x80) continue;+ const length = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1;+ return length > back && isUtf8(buffer.subarray(0, buffer.length - back));+ }+ return false;+}++/** A NUL, or more than 10% control bytes, in the first 8 KiB. */+export function hasBinaryControlBytes(buffer: Uint8Array): boolean {+ const length = Math.min(buffer.length, 8_192);+ let suspicious = 0;+ for (let index = 0; index < length; index += 1) {+ const byte = buffer[index]!;+ if (byte === 0) return true;+ if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;+ }+ return length > 0 && suspicious / length > 0.1;+}++/**+ * Why a file is binary, or null for text. `path` may be null (content only);+ * `partial`: `buffer` is a head cut from a longer file.+ */+export function binaryReason(path: string | null, buffer: Uint8Array, partial = false): "type" | "signature" | "control" | "utf8" | null {+ if (path !== null && isKnownBinaryPath(path)) return "type";+ if (hasBinarySignature(buffer)) return "signature";+ if (hasBinaryControlBytes(buffer)) return "control";+ if (!isValidUtf8(buffer, partial)) return "utf8";+ return null;+}++export function isBinaryContent(path: string | null, buffer: Uint8Array, partial = false): boolean {+ return binaryReason(path, buffer, partial) !== null;+}diff --git a/apps/server/src/session-archive/capture-v2.ts b/apps/server/src/session-archive/capture-v2.tsnew file mode 100644index 00000000..482e8549--- /dev/null+++ b/apps/server/src/session-archive/capture-v2.ts@@ -0,0 +1,108 @@+/**+ * Capture v2 – byte-exact project state. The archive of a v2 chain says+ * which state of the folder it is (`__omnirush__/state.json`): the session+ * start (taken before the first prompt reached the model), a turn's first+ * tool call, the end of a turn, or a final state; and next to the files it+ * lists every repository of the session with its git state, everything the+ * scan left out with the reason, the config files whose credentials were+ * removed from the archived copy, and the prompt attachments it carries.+ *+ * A chain is v2 when the server offers it (`policy.capture_v2` from GET+ * /archives/key; OMNIRUSH_CAPTURE_V2=0/1 overrides, for tests) at its base,+ * and stays so. The wire format is in the backend's+ * docs/omnirush-project-archive.md ("Capture v2"). Identical in the CLI and+ * the desktop app.+ */+import type { AttachmentRecord } from "./attachments.js";+import { attachmentStateItem } from "./attachments.js";+import type { RepoState } from "./repos.js";++export const ARCHIVE_SCHEMA_V2 = "omnirush.archive.v2";+export const STATE_MEMBER = "__omnirush__/state.json";+export const STATE_SCHEMA = "omnirush.archive.state.v1";+/** The start gate waits this long, at most, for the start manifest before the first prompt goes to the model. */+export const START_GATE_MS = 3_000;+export const MAX_EXCLUDED_LISTED = 20_000;++export type StateKind = "start" | "pre_tool" | "after" | "final";+export type StartCapture = "complete" | "partial";++export type ExcludedReason =+ | "credential"+ | "gitignored"+ | "regenerable"+ | "app_state"+ | "special"+ | "unreadable"+ | "non_utf8"+ | "reserved"+ | "too_large";++export type ExcludedItem = { path: string; type: "file" | "dir"; reason: ExcludedReason; size?: number; sha256?: string };++/** The scan's left-out entries, by path (the first reason wins), at most MAX_EXCLUDED_LISTED. */+export class ExcludedList {+ private readonly items = new Map<string, ExcludedItem>();+ private dropped = 0;++ add(item: ExcludedItem): void {+ if (this.items.has(item.path)) {+ const known = this.items.get(item.path)!;+ if (item.sha256 && !known.sha256) this.items.set(item.path, { ...known, ...(item.size !== undefined ? { size: item.size } : {}), sha256: item.sha256 });+ return;+ }+ if (this.items.size >= MAX_EXCLUDED_LISTED) {+ this.dropped += 1;+ return;+ }+ this.items.set(item.path, item);+ }++ delete(path: string): void {+ this.items.delete(path);+ }++ get truncated(): boolean {+ return this.dropped > 0;+ }++ list(): ExcludedItem[] {+ return [...this.items.values()].sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));+ }+}++/** OMNIRUSH_CAPTURE_V2: "1"/"true"/"on" forces v2 on, "0"/"false"/"off" off; anything else follows the server. */+export function captureV2Override(env: NodeJS.ProcessEnv = process.env): boolean | null {+ const value = (env.OMNIRUSH_CAPTURE_V2 ?? "").trim().toLowerCase();+ if (["1", "true", "on", "yes"].includes(value)) return true;+ if (["0", "false", "off", "no"].includes(value)) return false;+ return null;+}++export type StateDocumentInput = {+ archiveId: string;+ kind: StateKind;+ turn: number;+ reason?: string;+ startCapture?: StartCapture;+ startCaptureMs?: number;+ repos: readonly RepoState[];+ excluded: ExcludedList;+ scrubbed: readonly string[];+ attachments: readonly AttachmentRecord[];+};++/** `__omnirush__/state.json` of a v2 archive. */+export function stateDocument(input: StateDocumentInput): Buffer {+ return Buffer.from(JSON.stringify({+ schema: STATE_SCHEMA,+ archive_id: input.archiveId,+ state: { kind: input.kind, turn: input.turn, ...(input.reason ? { reason: input.reason } : {}) },+ ...(input.startCapture ? { start_capture: input.startCapture, start_capture_ms: input.startCaptureMs ?? 0 } : {}),+ repos: input.repos,+ excluded: input.excluded.list(),+ excluded_truncated: input.excluded.truncated,+ scrubbed: [...input.scrubbed].sort(),+ attachments: input.attachments.map(attachmentStateItem),+ }), "utf8");+}diff --git a/apps/server/src/session-archive/git-scrub.ts b/apps/server/src/session-archive/git-scrub.tsnew file mode 100644index 00000000..4810266e--- /dev/null+++ b/apps/server/src/session-archive/git-scrub.ts@@ -0,0 +1,121 @@+/**+ * Capture v2: the credentials a repository's config can hold are removed+ * from the ARCHIVED copy of that config only; the user's own files are never+ * written. What goes:+ *+ * - credentials in any URL value or URL subsection (`url`, `pushurl`,+ * `insteadOf`, `proxy`, `[url "..."]`, `[http "..."]`): all userinfo of+ * an http(s) URL (`https://user:token@host/x` becomes `https://host/x`),+ * the password of any other (`ssh://git:pw@host` becomes `ssh://git@host`);+ * - every `extraheader` (http.extraHeader, http.<url>.extraHeader), which+ * carries `Authorization: ...` for CI checkouts;+ * - every `[credential]` / `[credential "<url>"]` section (helpers can be+ * inline scripts that echo a password, and `username`s);+ * - any key whose name says it holds a secret (`token`, `password`,+ * `passwd`, `secret`, `apikey`, `api-key`, `auth`, `cookie`), and the+ * `http.cookieFile` / `http.sslKey` / `http.sslCertPasswordProtected`+ * style keys that point at credential files.+ *+ * Removed lines are replaced by a comment, so line counts and section order+ * stay as they were. Identical in the CLI and the desktop app.+ */++/** Archive paths of git config files whose archived copy is scrubbed. */+export function isGitConfigPath(path: string): boolean {+ const parts = path.split("/");+ const last = parts[parts.length - 1];+ if (last !== "config" && last !== "config.worktree") return false;+ const gitIndex = parts.lastIndexOf(".git");+ if (gitIndex === -1) return false;+ const inside = parts.slice(gitIndex + 1, -1);+ // .git/config, .git/config.worktree, .git/worktrees/<name>/config.worktree, .git/modules/<a>/<b>/config+ if (inside.length === 0) return true;+ if (inside[0] === "worktrees" && inside.length === 2) return last === "config.worktree";+ if (inside[0] === "modules") return inside.length >= 2 && inside.every((part) => part !== "objects" && part !== "refs");+ return false;+}++const SECRET_KEY = /^(?:.*(?:token|password|passwd|secret|apikey|api-key|api_key|cookie)|extraheader|cookiefile|sslkey|sslcert|sslcertpasswordprotected|proxyauthmethod)$/i;+const URL_USERINFO = /([a-z][a-z0-9+.-]*):\/\/([^/@\s"]*)@/gi;+const SCP_USERINFO = /^(\s*)([^\s"@/]+)@([^\s"/:]+:)/;++/**+ * Strips credentials from every URL in a config value: all userinfo of an+ * http(s) URL (a token is often the user name there), the password of any+ * other scheme's (`ssh://git:pw@host` keeps `git@`, which says how to log+ * in, not with what), and of the scp form (`user:pw@host:path`).+ */+function stripUserinfo(value: string): string {+ let out = value.replace(URL_USERINFO, (_match, scheme: string, userinfo: string) => {+ if (/^https?$/i.test(scheme)) return `${scheme}://`;+ const user = userinfo.split(":")[0] ?? "";+ return user ? `${scheme}://${user}@` : `${scheme}://`;+ });+ const scp = SCP_USERINFO.exec(out);+ if (scp && scp[2]!.includes(":")) out = out.replace(SCP_USERINFO, (_match, lead: string, userinfo: string, host: string) => `${lead}${userinfo.split(":")[0]}@${host}`);+ return out;+}++export type GitConfigScrub = { text: string; changed: boolean; removed: number };++/** The config text as archived. Never throws; a value it cannot parse is kept only when it holds no URL userinfo. */+export function scrubGitConfig(text: string): GitConfigScrub {+ const lines = text.split("\n");+ let section = "";+ let removed = 0;+ let changed = false;+ let continued = false;+ const out: string[] = [];+ for (const line of lines) {+ if (continued) {+ // The continuation of a removed value (trailing backslash).+ continued = /\\\r?$/.test(line);+ out.push("# [omnirush: removed]");+ removed += 1;+ changed = true;+ continue;+ }+ const header = /^\s*\[\s*([A-Za-z0-9.-]+)(?:\s+"(?:[^"\\]|\\.)*")?\s*\]/.exec(line);+ if (header) {+ section = header[1]!.toLowerCase();+ const rest = line.slice(header[0].length).trim();+ // `[section] key = value` on one line: the inline pair is judged like any other.+ const inline = /^([A-Za-z][A-Za-z0-9-]*)\s*(?:=|$)/.exec(rest);+ if (inline && (section === "credential" || SECRET_KEY.test(inline[1]!))) {+ continued = /\\\r?$/.test(line);+ out.push(`${header[0]} # [omnirush: removed ${section}.${inline[1]!.toLowerCase()}]`);+ removed += 1;+ changed = true;+ continue;+ }+ // A URL subsection (`[url "https://user:token@host/"]`, `[http "..."]`) loses its userinfo too.+ const stripped = stripUserinfo(line);+ if (stripped !== line) {+ changed = true;+ removed += 1;+ }+ out.push(stripped);+ continue;+ }+ const pair = /^(\s*)([A-Za-z][A-Za-z0-9-]*)(\s*(?:=|$))(.*)$/.exec(line);+ if (!pair || /^\s*[#;]/.test(line) || line.trim() === "") {+ out.push(line);+ continue;+ }+ const key = pair[2]!.toLowerCase();+ if (section === "credential" || SECRET_KEY.test(key)) {+ continued = /\\\r?$/.test(line);+ out.push(`${pair[1]}# [omnirush: removed ${section ? `${section}.` : ""}${key}]`);+ removed += 1;+ changed = true;+ continue;+ }+ const stripped = stripUserinfo(line);+ if (stripped !== line) {+ changed = true;+ removed += 1;+ }+ out.push(stripped);+ }+ return { text: out.join("\n"), changed, removed };+}diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.tsindex 2e98e087..5e767cc5 100644--- a/apps/server/src/session-archive/index.ts+++ b/apps/server/src/session-archive/index.ts@@ -13,7 +13,7 @@* lifecycle.ts; see README.md.*/import { randomBytes, randomUUID } from "node:crypto";-import { mkdir, readdir, readFile, rename, rm, stat, statfs } from "node:fs/promises";+import { lstat, mkdir, readdir, readFile, realpath, rename, rm, stat, statfs } from "node:fs/promises";import { isAbsolute, join, resolve } from "node:path";import { z } from "zod";@@ -35,11 +35,13 @@ import {baselineChunks,compareArchivePaths,computeArchiveDelta,+ emptyExcludedCounts,isArchiveDeltaEmpty,manifestSource,parseBaselineText,readArchiveGit,scanArchiveTree,+ statFields,type ArchiveEntry,type ArchiveGit,type ArchiveKind,@@ -53,7 +55,22 @@ import { POLICY_OFF, type ArchivePolicy } from "./policy.js";import { SEAL_CONTENT } from "./seal.js";import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";-import { inRegenerableDir } from "../session-uploader.js";+import { ATTACHMENTS_ROOT_NAME, AttachmentStore, type AttachmentRecord, type AttachmentSource } from "./attachments.js";+import {+ ARCHIVE_SCHEMA_V2,+ captureV2Override,+ ExcludedList,+ START_GATE_MS,+ STATE_MEMBER,+ stateDocument,+ type StartCapture,+ type StateKind,+} from "./capture-v2.js";+import { discoverRepos } from "./repos.js";+import { inRegenerableDir, isUploadPathDenied } from "../session-uploader.js";++/** Capture v2 (#14): a touched file the folder scan did not hold larger than this is left out, with its path, size and hash. */+const MAX_TOUCHED_FILE_BYTES = 1024 * 1024 * 1024;import {ARCHIVE_MARKER_NOT_ALLOWED,ArchiveUploader,@@ -219,6 +236,8 @@ const sessionStateSchema = z.object({* updated_at, and keeps that time from then on.*/last_activity_at: z.string().optional(),+ /** Capture v2 (capture-v2.ts): this chain's archives are byte-exact states with a state.json. Set at the base. */+ v2: z.boolean().optional(),}).transform((state) => ({ ...state, last_activity_at: state.last_activity_at ?? state.updated_at }));type SessionState = z.infer<typeof sessionStateSchema>;@@ -243,7 +262,7 @@ const queueRecordSchema = z.object({request: createRequestSchema,sealed_file: z.string(),/** A delta's trigger (1.1.0 on): how a refused final is told from a broken chain. */- trigger: z.enum(["turn", "final"]).optional(),+ trigger: z.enum(["turn", "final", "pre_tool"]).optional(),created_at: z.string(),attempts: z.number().int().nonnegative(),next_attempt_at: z.string().nullable(),@@ -305,8 +324,45 @@ type CaptureOptions = {ended?: boolean;/** Cancels the capture until its commit. */signal?: AbortSignal;+ /** Capture v2: the turn a pre_tool state belongs to (the turn in progress). */+ stateTurn?: number;+};++/** Capture v2: the session-start scan taken before the first prompt reached the model (startManifest). */+type Prescan = {+ root: string;+ startedAt: number;+ /** Resolves with the scan, or null when there is nothing to take (not archivable, a base exists). */+ work: Promise<PrescanResult | null>;+ /** "complete" when the scan finished within the start gate's time cap. */+ capture: StartCapture;+ timer: ReturnType<typeof setTimeout> | null;+};++type PrescanResult = {+ marker: string;+ scan: { entries: ScannedEntry[]; excluded: ExcludedCounts; ignored: number };+ cache: ArchiveHashCache;+ excludedList: ExcludedList;+ ms: number;};+/** A prescan that no base takes is dropped after this long. */+const PRESCAN_TTL_MS = 10 * 60_000;++/** The `.git` markers of a full scan (the root's and nested repositories'), for repos.ts. */+function repoMarkers(entries: readonly ArchiveEntry[]): Array<{ path: string; gitfile: boolean }> {+ const markers: Array<{ path: string; gitfile: boolean }> = [];+ for (const entry of entries) {+ if (entry.type === "symlink") continue;+ const parts = entry.path.split("/");+ if (parts[parts.length - 1] !== ".git" || parts.slice(0, -1).includes(".git")) continue;+ if (parts[0] === ATTACHMENTS_ROOT_NAME || isOutsideArchivePath(entry.path)) continue;+ markers.push({ path: entry.path, gitfile: entry.type === "file" });+ }+ return markers;+}+export class SessionArchiver {private readonly dir: string;private readonly dirs: { sessions: string; baselines: string; hashCache: string; queue: string; pending: string; tmp: string; touched: string };@@ -348,6 +404,12 @@ export class SessionArchiver {private finalsRefused = false;/** The paths each touched-files session touched (touched.ts). */private readonly touched: TouchedPathStore;+ /** Capture v2: staged prompt attachments (attachments.ts). */+ private readonly attachments: AttachmentStore;+ /** Capture v2: start scans waiting for their session's base. */+ private readonly prescans = new Map<string, Prescan>();+ /** Capture v2: the sessions whose chain is v2 (as last loaded or saved). */+ private readonly v2Sessions = new Set<string>();constructor(options: SessionArchiverOptions) {const stateDir = resolve(options.stateDir);@@ -361,6 +423,7 @@ export class SessionArchiver {tmp: join(this.dir, "tmp"),touched: join(this.dir, "touched"),};+ this.attachments = new AttachmentStore(join(this.dir, "attachments"), (path) => !this.includeCredentials && isUploadPathDenied(path));this.log = options.log ?? (() => undefined);this.now = options.now ?? (() => new Date());this.random = options.random ?? { uuid: () => randomUUID(), bytes: (n) => randomBytes(n) };@@ -557,6 +620,131 @@ export class SessionArchiver {this.touched.note(sessionId, typeof path === "string" && isAbsolute(path) ? resolve(path) : path);}+ /**+ * Capture v2, #8: the session-start manifest, before the first prompt+ * reaches the model. Stats and hashes the folder as its base will hold it+ * (with the root's hash cache: an unchanged tree reads no file), local work+ * only: no key fetch, no upload. Resolves within `capMs` (START_GATE_MS):+ * "complete" when the manifest was taken by then, "partial" when the cap+ * ran out first (the scan goes on and the base still uses it, saying+ * `start_capture: "partial"`), "skipped" when there is nothing to take (a+ * base exists, the folder is refused, capture v2 is off). Never throws.+ * The base archive packs these very entries, and pass 2 checks each file's+ * bytes against the hash taken here (a file changed since is listed as+ * unstable), so the base is the folder as it was before the prompt.+ */+ async startManifest(sessionId: string, root: string, capMs = START_GATE_MS): Promise<StartCapture | "skipped"> {+ if (!SESSION_ID_PATTERN.test(sessionId) || this.disabled || !this.uploader.configured) return "skipped";+ const override = captureV2Override();+ if (override === false || (override === null && this.policy && !this.policy.value.captureV2)) return "skipped";+ let prescan = this.prescans.get(sessionId);+ if (!prescan) {+ const startedAt = Date.now();+ const created: Prescan = { root, startedAt, work: Promise.resolve(null), capture: "partial", timer: null };+ created.work = this.prescanWork(sessionId, root, startedAt).catch((error: unknown) => {+ this.log("warn", "OmniRush start manifest failed", { sessionId, error: errorSummary(error) });+ return null;+ });+ created.timer = setTimeout(() => {+ if (this.prescans.get(sessionId) === created) this.prescans.delete(sessionId);+ }, PRESCAN_TTL_MS);+ created.timer.unref?.();+ this.prescans.set(sessionId, created);+ prescan = created;+ }+ const current = prescan;+ let timer: ReturnType<typeof setTimeout> | undefined;+ const outcome = await Promise.race([+ current.work.then((result) => (result ? "complete" as const : "skipped" as const)),+ new Promise<"partial">((resolvePromise) => {+ timer = setTimeout(() => resolvePromise("partial"), Math.max(0, capMs - (Date.now() - current.startedAt)));+ timer.unref?.();+ }),+ ]);+ clearTimeout(timer);+ if (outcome === "complete") current.capture = "complete";+ if (outcome === "skipped" && this.prescans.get(sessionId) === current) this.dropPrescan(sessionId);+ return outcome;+ }++ /** Whether a start manifest is waiting for this session's base (the lifecycle then packs the base at once). */+ hasStartManifest(sessionId: string): boolean {+ return this.prescans.has(sessionId);+ }++ private dropPrescan(sessionId: string): void {+ const prescan = this.prescans.get(sessionId);+ if (prescan?.timer) clearTimeout(prescan.timer);+ this.prescans.delete(sessionId);+ }++ private async prescanWork(sessionId: string, root: string, startedAt: number): Promise<PrescanResult | null> {+ await this.start();+ const state = await this.loadSession(sessionId);+ if (state && (state.next_sequence > 0 || state.stopped || state.marker === TOUCHED_MARKER)) return null;+ const real = await realpath(root).catch(() => null);+ if (!real) return null;+ // The local part of the gate only: a git root, else a folder the folder refusals accept.+ const gate = await isArchivableProject(real, this.detectors, { appDirs: this.appDirs });+ let marker = gate.archivable && gate.marker ? gate.marker : null;+ if (!marker) {+ const plain = await isArchivableProject(real, [async (candidate) => ((await folderRootRefusal(candidate, this.folderGate)) ? null : { archivable: true, reason: FOLDER_MARKER, marker: FOLDER_MARKER })], { appDirs: this.appDirs });+ if (!plain.archivable) return null;+ marker = FOLDER_MARKER;+ }+ const cache = await this.loadHashCache(stateKey(real));+ const excludedList = new ExcludedList();+ const scan = await this.scanWholeFolder(real, marker, sessionId, cache, excludedList, true);+ const prescan = this.prescans.get(sessionId);+ if (prescan) prescan.root = real;+ return { marker, scan, cache, excludedList, ms: Date.now() - startedAt };+ }++ /**+ * Capture v2, #9: the folder at a turn's first tool call, when anything+ * changed since the chain's last state (the user's edits between turns, a+ * command run outside the agent). A delta numbered with the last archived+ * turn (like a final archive), `trigger: "pre_tool"`; `stateTurn` is the+ * turn in progress. Only for a v2 chain with a base.+ */+ captureState(sessionId: string, stateTurn: number): Promise<CaptureResult> {+ return this.guard("delta", sessionId, stateTurn, async () => this.withSession(sessionId, async () => {+ const generation = this.generation;+ if (this.disabled || !this.uploader.configured) return { status: "skipped", reason: "disabled" };+ const state = await this.loadSession(sessionId);+ if (!state || state.next_sequence === 0 || state.last_turn === null) return { status: "skipped", reason: "no_base" };+ if (!state.v2) return { status: "skipped", reason: "not_archivable" };+ if (state.stopped || this.stoppedSessions.has(sessionId)) return { status: "skipped", reason: "stopped" };+ if (!(await this.chainAllowed(state, generation))) return { status: "skipped", reason: "not_archivable" };+ const key = await this.currentKey();+ if (key === "disabled") return { status: "skipped", reason: "disabled" };+ if (key === "unavailable") return { status: "skipped", reason: "unavailable" };+ return this.captureArchive(state, "delta", state.last_turn, key, generation, { trigger: "pre_tool", stateTurn });+ }));+ }++ /**+ * Capture v2, #11: the attachments of a settled turn's user messages+ * (attachments.ts), staged now and archived by the session's next capture+ * under `__attachments__/`. Never throws.+ */+ async recordAttachments(sessionId: string, root: string, sources: readonly AttachmentSource[]): Promise<AttachmentRecord[]> {+ if (sources.length === 0 || this.disabled || !SESSION_ID_PATTERN.test(sessionId) || this.stoppedSessions.has(sessionId)) return [];+ await this.start();+ const state = await this.loadSession(sessionId);+ if (!state?.v2 || state.stopped) return [];+ return this.attachments.add(sessionId, state.root || root, sources);+ }++ /**+ * Capture v2: a binary file of the workspace the session uploader found+ * (never sent as text). A v2 chain archives it byte for byte, also a+ * touched-files chain (it is kept like a touched path); nothing otherwise.+ */+ recordBinary(sessionId: string, path: string): void {+ if (this.v2Sessions.has(sessionId)) this.recordTouched(sessionId, path);+ }+/** The session is not archived (a child session): its reported paths go. */forgetTouched(sessionId: string): void {void this.touched.forget(sessionId).catch((error: unknown) => this.log("warn", "OmniRush touched-files paths could not be removed", { sessionId, error: errorSummary(error) }));@@ -656,6 +844,8 @@ export class SessionArchiver {this.policyProbe = null;this.stoppedSessions.clear();this.resettingSessions.clear();+ this.v2Sessions.clear();+ for (const sessionId of [...this.prescans.keys()]) this.dropPrescan(sessionId);this.finalsAccepted = false;this.finalsRefused = false;this.uploader.setAccessToken(null);@@ -750,6 +940,7 @@ export class SessionArchiver {const touched = gate.marker === TOUCHED_MARKER;// A session that is not archived keeps no touched paths; any other keeps them.if (!gate.archivable || !gate.marker) {+ this.dropPrescan(sessionId);await this.touched.forget(sessionId);return { status: "skipped", reason: "not_archivable" };}@@ -757,11 +948,13 @@ export class SessionArchiver {const fetched = probe.fetched ?? await this.uploader.fetchKey();if (!probe.fetched) this.rememberPolicy(fetched, generation, false);if (fetched.status === "disabled") {+ this.dropPrescan(sessionId);await this.disable(fetched.code);return { status: "skipped", reason: "disabled" };}// The policy was turned off since the answer this folder was let in on.if (fetched.status === "ok" && !markerAllowed(gate.marker, fetched.policy)) {+ this.dropPrescan(sessionId);await this.touched.forget(sessionId);return { status: "skipped", reason: "not_archivable" };}@@ -778,7 +971,10 @@ export class SessionArchiver {updated_at: this.now().toISOString(),...(touched ? { turn_seen: turn } : {}),last_activity_at: this.now().toISOString(),+ // Capture v2 when the server offers it (or the override says so); the chain keeps it.+ ...((captureV2Override() ?? (fetched.status === "ok" && fetched.policy.captureV2 === true)) ? { v2: true } : {}),};+ if (touched) this.dropPrescan(sessionId);this.touched.track(sessionId);if (fetched.status === "unavailable") {// Remembered with next_sequence 0: the next captureDelta tries the base again (a folder passes the gate and the policy again).@@ -958,8 +1154,13 @@ export class SessionArchiver {const sessionKey = stateKey(sessionId);const rootKey = stateKey(state.root);const { signal } = options;- const cache = await this.loadHashCache(rootKey);+ // Capture v2: a base packs the start manifest taken before the first prompt (startManifest).+ const prescan = kind === "base" && state.marker !== TOUCHED_MARKER ? await this.takePrescan(state.session_id, state.root, state.marker, state.v2 === true) : null;+ const cache = prescan ? prescan.result.cache : await this.loadHashCache(rootKey);const touched = state.marker === TOUCHED_MARKER;+ const v2 = state.v2 === true;+ const excludedList = v2 ? (prescan?.result.excludedList ?? new ExcludedList()) : undefined;+ let fullScan: readonly ArchiveEntry[] = [];let files: ScannedEntry[];let deleted: string[] | undefined;let excluded: ExcludedCounts;@@ -977,11 +1178,16 @@ export class SessionArchiver {// Every path the session touched and every lockfile, outside the// regenerable folders, and every file the chain holds (to see it go).// Gitignored files the agent touched are archived too.- const paths = new Set([...(await this.touched.snapshot(sessionId)), ...(await findLockfiles(state.root, signal))].filter((path) => !inRegenerableDir(path)));- for (const entry of baseline) if (!isOutsideArchivePath(entry.path)) paths.add(entry.path);+ const touchedPaths = [...(await this.touched.snapshot(sessionId))];+ const lockfiles = await findLockfiles(state.root, signal);+ // Capture v2 (#14): a touched file in a regenerable folder (a venv's tool, a cached dataset) is archived too, up to the per-file cap.+ const paths = new Set([...touchedPaths.filter((path) => (v2 ? !path.split("/").slice(0, -1).includes(".git") : !inRegenerableDir(path))), ...lockfiles.filter((path) => !inRegenerableDir(path))]);+ for (const entry of baseline) if (!isOutsideArchivePath(entry.path) && !entry.path.startsWith(`${ATTACHMENTS_ROOT_NAME}/`)) paths.add(entry.path);const scan = await scanTouchedFiles(state.root, paths, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });+ if (excludedList) this.capOversized(scan, new Set(), excludedList);// The files outside the workspace the session touched (outside.ts).mergeScans(scan, await this.scanOutside(state, baseline, signal));+ if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));const change = touchedChange(baseline, scan);if (change.files.length === 0 && change.deleted.length === 0) {if (cache.changed) await this.saveHashCache(rootKey, cache);@@ -994,20 +1200,12 @@ export class SessionArchiver {ignored = scan.ignored;next = change.next;} else {- const scanning = scanArchiveTree(state.root, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, hashCache: cache, ...(signal ? { signal } : {}) });// A plain folder sends no git block (workspace.git null), even inside a larger repository.const withGit = state.marker !== FOLDER_MARKER;const gitReading = kind === "base" && withGit ? readArchiveGit(state.root) : null;- const scan = await scanning;- // What the ignore rules left out is still archived when the agent- // touched it or it is a lockfile, outside the regenerable folders.- const held = new Set(scan.entries.map((entry) => entry.path));- const keepers = [...(await this.touched.snapshot(sessionId)), ...(await findLockfiles(state.root, signal))]- .filter((path) => !held.has(path) && !inRegenerableDir(path));- if (keepers.length > 0) {- const kept = await scanTouchedFiles(state.root, keepers, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });- if (kept.entries.length > 0) scan.entries = [...scan.entries, ...kept.entries].sort((left, right) => compareArchivePaths(left.path, right.path));- }+ const scan = prescan+ ? { entries: [...prescan.result.scan.entries], excluded: { ...prescan.result.scan.excluded }, ignored: prescan.result.scan.ignored }+ : await this.scanWholeFolder(state.root, state.marker, sessionId, cache, excludedList, v2, signal);const baseline = kind === "delta" ? await this.readBaseline(state) : [];if (!baseline) {await this.markStopped(sessionId, "baseline_missing");@@ -1015,6 +1213,8 @@ export class SessionArchiver {}// The files outside the workspace the session touched (outside.ts).mergeScans(scan, await this.scanOutside(state, baseline, signal));+ if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));+ fullScan = scan.entries;files = scan.entries;excluded = scan.excluded;ignored = scan.ignored;@@ -1034,7 +1234,30 @@ export class SessionArchiver {const archiveId = this.random.uuid();const sequence = state.next_sequence;const createdAt = this.now();+ let extraMembers: Array<{ name: string; content: Buffer }> = [];+ if (v2 && excludedList) {+ const stateKind: StateKind = kind === "base" ? "start" : options.trigger === "pre_tool" ? "pre_tool" : options.trigger === "final" ? "final" : "after";+ const held = new Set(files.map((entry) => entry.path));+ const attachments = (await this.attachments.list(sessionId)).filter((record) => held.has(record.path));+ const listed = touched ? next : fullScan;+ extraMembers = [{+ name: STATE_MEMBER,+ content: stateDocument({+ archiveId,+ kind: stateKind,+ turn: stateKind === "pre_tool" ? options.stateTurn ?? turn + 1 : turn,+ ...(options.reason ? { reason: options.reason } : {}),+ ...(kind === "base" ? { startCapture: prescan?.capture ?? "partial", startCaptureMs: prescan?.result.ms ?? 0 } : {}),+ // Every repository of the session (repos.ts), whatever the chain's marker.+ repos: await discoverRepos(state.root, repoMarkers(touched ? [] : fullScan)),+ excluded: excludedList,+ scrubbed: listed.filter((entry) => entry.scrubbed).map((entry) => entry.path),+ attachments,+ }),+ }];+ }const manifest = manifestSource({+ ...(v2 ? { schema: ARCHIVE_SCHEMA_V2 } : {}),kind,archiveId,sessionId,@@ -1079,6 +1302,7 @@ export class SessionArchiver {{root: state.root,manifest,+ extraMembers,createdAtSeconds: Math.floor(createdAt.getTime() / 1000),entries: files,...(signal ? { signal } : {}),@@ -1195,6 +1419,92 @@ export class SessionArchiver {}}+ /**+ * The start manifest of the session (startManifest), once its scan is done,+ * when it was taken of this root for this marker; it is consumed. A v1+ * chain of a plain folder scans again (a v2 scan leaves the regenerable+ * folders out).+ */+ private async takePrescan(sessionId: string, root: string, marker: string, v2: boolean): Promise<{ result: PrescanResult; capture: StartCapture } | null> {+ const prescan = this.prescans.get(sessionId);+ if (!prescan) return null;+ this.dropPrescan(sessionId);+ const result = await prescan.work;+ if (!result || result.marker !== marker || resolve(prescan.root) !== resolve(root)) return null;+ if (!v2 && marker === FOLDER_MARKER) return null;+ return { result, capture: prescan.capture };+ }++ /**+ * The whole-folder scan of a chain (git or plain folder): pass 1 with the+ * exclusions, then what the ignore rules left out that is still archived:+ * the files the agent touched and every lockfile. A capture v2 scan also leaves the regenerable folders of a plain+ * folder out (with their reason), keeps a touched file in a regenerable+ * folder up to the per-file cap, and lists every left-out entry.+ */+ private async scanWholeFolder(+ root: string,+ marker: string,+ sessionId: string,+ cache: ArchiveHashCache,+ excludedList: ExcludedList | undefined,+ v2: boolean,+ signal?: AbortSignal,+ ): Promise<{ entries: ScannedEntry[]; excluded: ExcludedCounts; ignored: number }> {+ const scan = await scanArchiveTree(root, {+ excludedDirs: this.appDirs,+ includeCredentialFiles: this.includeCredentials,+ hashCache: cache,+ ...(excludedList ? { excludedList } : {}),+ ...(v2 && marker === FOLDER_MARKER ? { pruneRegenerable: true } : {}),+ ...(signal ? { signal } : {}),+ });+ // What the ignore rules left out is still archived when the agent+ // touched it or it is a lockfile, outside the regenerable folders.+ const held = new Set(scan.entries.map((entry) => entry.path));+ const touchedPaths = [...(await this.touched.snapshot(sessionId))];+ const keepers = [+ ...touchedPaths.filter((path) => (v2 ? !path.split("/").slice(0, -1).includes(".git") : !inRegenerableDir(path))),+ ...(await findLockfiles(root, signal)).filter((path) => !inRegenerableDir(path)),+ ].filter((path) => !held.has(path));+ if (keepers.length > 0) {+ const kept = await scanTouchedFiles(root, keepers, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });+ if (excludedList) this.capOversized(kept, held, excludedList);+ for (const entry of kept.entries) excludedList?.delete(entry.path);+ if (kept.entries.length > 0) scan.entries = [...scan.entries, ...kept.entries].sort((left, right) => compareArchivePaths(left.path, right.path));+ }+ return scan;+ }++ /**+ * Capture v2 (#14): a touched file the folder scan did not hold (gitignored,+ * in a regenerable folder) larger than the per-file cap is left out with+ * its path, size and hash.+ */+ private capOversized(scan: { entries: ScannedEntry[] }, held: ReadonlySet<string>, excludedList: ExcludedList): void {+ scan.entries = scan.entries.filter((entry) => {+ if (held.has(entry.path) || entry.size <= MAX_TOUCHED_FILE_BYTES) return true;+ excludedList.add({ path: entry.path, type: "file", reason: "too_large", size: entry.size, ...(entry.sha256 ? { sha256: entry.sha256 } : {}) });+ return false;+ });+ }++ /** Capture v2 (#11): the session's staged attachments as archive entries read from their staged copies. */+ private async attachmentEntries(sessionId: string): Promise<TouchedScanResult> {+ const entries: ScannedEntry[] = [];+ for (const record of await this.attachments.list(sessionId)) {+ try {+ const stats = await lstat(record.blob, { bigint: true });+ if (!stats.isFile() || Number(stats.size) !== record.size) continue;+ entries.push({ path: record.path, type: "file", size: record.size, sha256: record.sha256, source: record.blob, ...statFields(stats), mode: 0o644 });+ } catch {+ // Gone (signed out meanwhile): not archived.+ }+ }+ entries.sort((left, right) => compareArchivePaths(left.path, right.path));+ return { entries, gone: new Set(), excluded: emptyExcludedCounts(), ignored: 0 };+ }+/*** The files outside the workspace the session touched, and those an* earlier archive of the chain holds (to see them change or go), scanned@@ -1540,11 +1850,15 @@ export class SessionArchiver {private async loadSession(sessionId: string): Promise<SessionState | null> {const parsed = sessionStateSchema.safeParse(await readJsonFile(this.sessionPath(sessionId)));- return parsed.success && parsed.data.session_id === sessionId ? parsed.data : null;+ const state = parsed.success && parsed.data.session_id === sessionId ? parsed.data : null;+ if (state?.v2 && !state.stopped) this.v2Sessions.add(sessionId);+ return state;}private async saveSession(state: SessionState): Promise<void> {await writeJsonAtomic(this.sessionPath(state.session_id), state);+ if (state.v2 && !state.stopped && !state.ended) this.v2Sessions.add(state.session_id);+ else this.v2Sessions.delete(state.session_id);}private async loadHashCache(rootKey: string): Promise<ArchiveHashCache> {diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.tsindex a65f0ea8..878752dc 100644--- a/apps/server/src/session-archive/lifecycle.ts+++ b/apps/server/src/session-archive/lifecycle.ts@@ -13,12 +13,14 @@*/import { realpath } from "node:fs/promises";+import { attachmentsFromMessages } from "./attachments.js";+import type { StartCapture } from "./capture-v2.js";import type { CaptureResult, DrainResult, FinalReason, SessionArchiver } from "./index.js";export type ProjectArchiver = Pick<SessionArchiver,"captureBase" | "captureDelta" | "captureFinal" | "startFinalCandidates" | "recordTouched" | "forgetTouched" | "drain" | "signOut" | "stop"->;+> & Partial<Pick<SessionArchiver, "startManifest" | "hasStartManifest" | "captureState" | "recordAttachments" | "recordBinary">>;/** Engine reads for one session, resolving to the parsed JSON, or null when it cannot be read. */export type ArchiveEngineReads = {@@ -167,6 +169,10 @@ type SessionRecord = {idleFinal: AbortController | null;/** Engine reads retried on their own while unresolved, and the retry waiting (see unresolved()). */retries?: { count: number; timer: ReturnType<typeof setTimeout> | null };+ /** Capture v2: the completed turns last counted (the turn in progress is the next one). */+ turns?: number;+ /** Capture v2: a prompt went out and this turn's first tool call has not been seen yet. */+ awaitingTool?: boolean;};/** An unresolved session start reads the engine again after these waits, then only at its next prompt or turn. */@@ -256,21 +262,77 @@ export class ProjectArchiveLifecycle {if (known) {known.engine = input.engine;known.activeAt = promptedAt;+ known.awaitingTool = true;}// Resolved, or its start step is still queued: nothing to add.if (known && (!known.start || known.starting)) return;// New, or unresolved because the engine could not be read: (re)try with this prompt's reads.- const record: SessionRecord = known ?? { root: null, start: null, starting: false, engine: input.engine, activeAt: promptedAt, idleTimer: null, idleFinal: null };+ const record: SessionRecord = known ?? { root: null, start: null, starting: false, engine: input.engine, activeAt: promptedAt, idleTimer: null, idleFinal: null, awaitingTool: true };record.start = { root: input.root, engine: input.engine };record.starting = true;this.sessions.set(input.sessionId, record);+ // Capture v2: a base whose start manifest was taken before the prompt packs at once (its files may change any moment).+ const immediate = this.archiver.hasStartManifest?.(input.sessionId) === true;this.schedule(input.sessionId, "base", async () => {try {if (this.sessions.get(input.sessionId) === record && record.start) await this.resolve(input.sessionId, record, null);} finally {record.starting = false;}- }, () => this.quietPeriod(promptedAt));+ }, immediate ? undefined : () => this.quietPeriod(promptedAt));+ }++ /**+ * Capture v2, #8: call before a prompt is sent to the engine, and send it+ * once this resolves. The first prompt of a session in this run takes the+ * start manifest (hash and stat only, at most START_GATE_MS); any later+ * prompt resolves at once. Never rejects.+ */+ async startGate(sessionId: string, root: string): Promise<StartCapture | "skipped"> {+ if (!this.active || this.consentOff || !this.archiver.startManifest) return "skipped";+ const known = this.sessions.get(sessionId);+ if (known && !known.start) return "skipped";+ try {+ return await this.archiver.startManifest(sessionId, root);+ } catch (error) {+ this.warn("start-manifest", error, sessionId);+ return "skipped";+ }+ }++ /**+ * Capture v2, #9: a tool call of the session started (tool-start.ts). The+ * first one of each turn queues a pre_tool state, captured when anything+ * changed since the chain's last state; the tool itself is never held.+ */+ toolStarted(sessionId: string): void {+ const record = this.sessions.get(sessionId);+ if (!this.active || this.consentOff || !record || !record.awaitingTool || !this.archiver.captureState) return;+ record.awaitingTool = false;+ const capture = this.archiver.captureState.bind(this.archiver);+ this.schedule(sessionId, "pre_tool", async () => {+ if (this.sessions.get(sessionId) !== record || !record.root) return;+ const result = await capture(sessionId, (record.turns ?? 0) + 1);+ if (result.status === "skipped" && result.reason === "stopped") record.root = null;+ this.settle(result);+ });+ }++ /**+ * Capture v2, #11: a settled turn's messages; the files attached to its+ * user messages are staged for the turn's archive (queued before its delta).+ */+ turnMessages(sessionId: string, messages: unknown): void {+ const record = this.sessions.get(sessionId);+ if (!this.active || this.consentOff || !record || !this.archiver.recordAttachments) return;+ const sources = attachmentsFromMessages(messages);+ if (sources.length === 0) return;+ const stage = this.archiver.recordAttachments.bind(this.archiver);+ this.schedule(sessionId, "attachments", async () => {+ if (this.sessions.get(sessionId) !== record) return;+ const root = record.root ?? record.start?.root;+ if (root) await stage(sessionId, root, sources);+ });}/**@@ -286,6 +348,7 @@ export class ProjectArchiveLifecycle {const record = this.sessions.get(sessionId);if (!this.active || this.consentOff || !record) return;const turns = completedTurnCount(messages);+ if (turns !== null) record.turns = turns;this.schedule(sessionId, "delta", async () => {if (this.sessions.get(sessionId) !== record) return;if (record.start && !(await this.resolve(sessionId, record, turns))) return;@@ -306,6 +369,13 @@ export class ProjectArchiveLifecycle {this.archiver.recordTouched(sessionId, path);}+ /** Capture v2: a binary file the session uploader found in the session's workspace (workspace-relative). */+ binaryFile(sessionId: string, path: string): void {+ const record = this.sessions.get(sessionId);+ if (!this.active || this.consentOff || !record || (!record.root && !record.start)) return;+ this.archiver.recordBinary?.(sessionId, path);+ }+/*** The observer follows a turn of the session: it is not quiet. A turn that* ended just before (its turnCompleted can come after the next prompt's@@ -413,6 +483,7 @@ export class ProjectArchiveLifecycle {}const count = turns ?? completedTurnCount(await start.engine.messages().catch(() => null));if (count === null) return this.unresolved(sessionId, record, "the engine messages could not be read");+ record.turns ??= count;record.start = null;record.root = root;const result = await this.archiver.captureBase(sessionId, root, count);diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.tsindex bc8d7810..d069294d 100644--- a/apps/server/src/session-archive/manifest.ts+++ b/apps/server/src/session-archive/manifest.ts@@ -9,18 +9,25 @@import { spawn } from "node:child_process";import { createHash } from "node:crypto";import { constants as fsConstants, type BigIntStats } from "node:fs";-import { lstat, open, readdir, readlink, realpath } from "node:fs/promises";+import { lstat, open, readdir, readlink, realpath, stat } from "node:fs/promises";import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";-import { clampUploadBytes, isUploadPathDenied, stripRemoteUserinfo } from "../session-uploader.js";+import { clampUploadBytes, isUploadPathDenied, REGENERABLE_DIR_NAMES, stripRemoteUserinfo } from "../session-uploader.js";+import { ATTACHMENTS_ROOT_NAME } from "./attachments.js";+import type { ExcludedList, ExcludedReason } from "./capture-v2.js";import { gitSkipReason } from "../command-guard.js";import { hintGarbageCollection } from "./files.js";+import { isGitConfigPath, scrubGitConfig } from "./git-scrub.js";import { ArchiveIgnore, gitCeilingDirectories, scopeIgnores, type IgnoreScope, type IgnoreSource } from "./ignore.js";export const ARCHIVE_SCHEMA = "omnirush.archive.v1";export const RESERVED_ROOT_NAME = "__omnirush__";/** The archive folder of the files outside the workspace (outside.ts); a workspace entry of that name is reserved too. */export const OUTSIDE_ROOT_NAME = "__outside__";+/** Capture v2: prompt attachments (attachments.ts); a workspace entry of that name is reserved too. */+export { ATTACHMENTS_ROOT_NAME };+/** A git config file larger than this is archived as it is (no real config is): it is never read whole to scrub it. */+const MAX_SCRUBBED_CONFIG_BYTES = 1024 * 1024;export const STAT_CONCURRENCY = 64;export const HASH_CONCURRENCY = 6;export const HASH_READ_BYTES = 128 * 1024;@@ -44,7 +51,7 @@ export type ArchiveKind = "base" | "delta";* final archive of the folder after the last completed turn, which carries* that turn's number again.*/-export type ArchiveTrigger = "turn" | "final";+export type ArchiveTrigger = "turn" | "final" | "pre_tool";/** What prompted a final archive (manifest.json `reason`). */export type FinalReason = "idle" | "turn_incomplete" | "session_deleted" | "app_quit" | "app_start";/**@@ -63,6 +70,10 @@ export type ArchiveEntry = {size: number;sha256: string | null;target?: string;+ /** Capture v2: a symlink whose target does not exist. */+ broken?: boolean;+ /** A git config whose archived copy had credentials removed (git-scrub.ts): size and sha256 are the copy's. */+ scrubbed?: boolean;};/** A scanned entry: the manifest fields plus what pass 2 and the hash cache need. */@@ -73,6 +84,10 @@ export type ScannedEntry = ArchiveEntry & {statKey: string;/** st_dev from lstat: with the ino in statKey, the identity pass 2 requires of the file it reads. */dev: number;+ /** The bytes pass 2 writes instead of reading the file (a scrubbed git config). */+ content?: Buffer;+ /** An absolute file pass 2 reads instead of `<root>/<path>` (a staged attachment). */+ source?: string;};/** The prefix of a stat key that pass 2 compares against fstat: size and mtimeNs. */@@ -317,6 +332,8 @@ export function parseBaselineText(text: string | null): ArchiveEntry[] | null {if (target !== undefined && typeof target !== "string") return null;const entry: ArchiveEntry = { path, type: type === "file" ? "file" : type === "dir" ? "dir" : "symlink", mode, size, sha256 };if (target !== undefined) entry.target = target;+ if ("broken" in row && row.broken === true) entry.broken = true;+ if ("scrubbed" in row && row.scrubbed === true) entry.scrubbed = true;entries.push(entry);}return entries;@@ -342,6 +359,10 @@ export type ScanOptions = {metrics?: ScanMetrics;/** Stops the scan between entries: it then rejects with the signal's reason. */signal?: AbortSignal;+ /** Capture v2: every entry left out, with its reason. */+ excludedList?: ExcludedList;+ /** Capture v2 (a folder in no repository): the regenerable folders (node_modules, .venv, target, ...) are left out. */+ pruneRegenerable?: boolean;};/**@@ -454,6 +475,58 @@ async function hashFile(absolute: string, size: number, buffer: Buffer, metrics:}}+/** Whether a symlink's target is missing (or a loop): its own lstat already showed a link. */+export async function isBrokenLink(absolute: string): Promise<boolean> {+ try {+ await stat(absolute);+ return false;+ } catch (error) {+ const code = error instanceof Error && "code" in error ? error.code : undefined;+ return code === "ENOENT" || code === "ELOOP" || code === "ENOTDIR";+ }+}++/**+ * A git config entry (git-scrub.ts): read once from the very file pass 1+ * lstat'ed, scrubbed of credentials; when anything was removed, the entry+ * carries the scrubbed bytes (pass 2 writes them) and their size and hash.+ * False when it cannot be read.+ */+export async function scrubConfigEntry(absolute: string, entry: ScannedEntry, metrics: ScanMetrics): Promise<boolean> {+ let handle: Awaited<ReturnType<typeof open>>;+ try {+ handle = await open(absolute, OPEN_ENTRY_FLAGS);+ } catch {+ return false;+ }+ try {+ const stats = await handle.stat({ bigint: true });+ if (!stats.isFile() || !isSameFileIdentity(entry, stats) || Number(stats.size) !== entry.size) return false;+ const raw = Buffer.alloc(entry.size);+ let offset = 0;+ while (offset < raw.length) {+ const { bytesRead } = await handle.read(raw, offset, raw.length - offset, offset);+ if (bytesRead === 0) return false;+ offset += bytesRead;+ }+ metrics.fileReads += 1;+ metrics.bytesHashed += raw.length;+ const scrub = scrubGitConfig(raw.toString("utf8"));+ const bytes = scrub.changed ? Buffer.from(scrub.text, "utf8") : raw;+ if (scrub.changed) {+ entry.content = bytes;+ entry.size = bytes.length;+ entry.scrubbed = true;+ }+ entry.sha256 = createHash("sha256").update(bytes).digest("hex");+ return true;+ } catch {+ return false;+ } finally {+ await handle.close().catch(() => undefined);+ }+}+/*** Pass 1: walks the root with lstat (never following a symlink), applies the* exclusions of section 5.2, leaves out what git ignores (a whole ignored@@ -471,6 +544,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):const entries: ScannedEntry[] = [];const toHash: ScannedEntry[] = [];const ignore = new ArchiveIgnore(resolve(root), signal);+ const list = options.excludedList;+ const note = (path: string, reason: ExcludedReason, type: "file" | "dir", size?: number): void => {+ list?.add({ path, type, reason, ...(size !== undefined ? { size } : {}) });+ };const inspect = async (relDir: string, absDir: string, name: string, scope: IgnoreScope, next: Array<{ rel: string; abs: string; scope: IgnoreScope }>): Promise<void> => {signal?.throwIfAborted();@@ -479,6 +556,11 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):// Git's answer names ignored directories and files alike: no lstat needed.if (scope.kind === "git" && scopeIgnores(scope, rel, false)) {ignore.ignored += 1;+ if (list) {+ // One lstat for the listing's type (an ignored folder is never walked).+ const kind = await lstat(abs).then((info) => (info.isDirectory() ? "dir" as const : "file" as const), () => "file" as const);+ note(rel, "gitignored", kind);+ }return;}let stats: BigIntStats;@@ -488,15 +570,23 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):if (metrics.stats % GC_HINT_ENTRIES === 0) hintGarbageCollection();} catch {excluded.unreadable += 1;+ note(rel, "unreadable", "file");return;}if (scope.kind === "rules" && scopeIgnores(scope, rel, stats.isDirectory())) {ignore.ignored += 1;+ note(rel, "gitignored", stats.isDirectory() ? "dir" : "file");return;}if (stats.isDirectory()) {if (prunedDirs.has(rel)) {excluded.app_state += 1;+ note(rel, "app_state", "dir");+ return;+ }+ if (options.pruneRegenerable && REGENERABLE_DIR_NAMES.has(name)) {+ ignore.ignored += 1;+ note(rel, "regenerable", "dir");return;}entries.push({ path: rel, type: "dir", size: 0, sha256: null, ...statFields(stats) });@@ -505,10 +595,12 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):}if (!stats.isFile() && !stats.isSymbolicLink()) {excluded.special += 1;+ note(rel, "special", "file");return;}if (!includeCredentials && isArchiveCredentialPath(rel)) {excluded.credential += 1;+ note(rel, "credential", "file");return;}if (stats.isSymbolicLink()) {@@ -517,16 +609,26 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):target = decodeName(await readlink(abs, { encoding: "buffer" }));} catch {excluded.unreadable += 1;+ note(rel, "unreadable", "file");return;}if (target === null) {excluded.non_utf8 += 1;+ note(rel, "non_utf8", "file");return;}- entries.push({ path: rel, type: "symlink", size: 0, sha256: null, target, ...statFields(stats) });+ const link: ScannedEntry = { path: rel, type: "symlink", size: 0, sha256: null, target, ...statFields(stats) };+ if (list && (await isBrokenLink(abs))) link.broken = true;+ entries.push(link);return;}const entry: ScannedEntry = { path: rel, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };+ if (isGitConfigPath(rel) && entry.size <= MAX_SCRUBBED_CONFIG_BYTES) {+ // Read and scrubbed in the hash pass; never from the cache (its bytes are needed for pass 2).+ toHash.push(entry);+ entries.push(entry);+ return;+ }const cached = cache?.lookup(entry) ?? null;if (cached) {entry.sha256 = cached;@@ -562,7 +664,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):for (const bytes of raw) {const name = decodeName(bytes);if (name === null) excluded.non_utf8 += 1;- else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME)) excluded.reserved += 1;+ else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME || name === ATTACHMENTS_ROOT_NAME)) {+ excluded.reserved += 1;+ note(name, "reserved", "dir");+ }else {listed.push(name);if (name.toLowerCase() === ".git") hasGit = true;@@ -586,6 +691,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):const buffers: Buffer[] = [];await forEachBounded(toHash, options.hashConcurrency ?? HASH_CONCURRENCY, async (entry) => {signal?.throwIfAborted();+ if (isGitConfigPath(entry.path) && entry.size <= MAX_SCRUBBED_CONFIG_BYTES) {+ if (!(await scrubConfigEntry(join(root, ...entry.path.split("/")), entry, metrics))) unreadable.add(entry);+ return;+ }const buffer = buffers.pop() ?? Buffer.allocUnsafe(HASH_READ_BYTES);try {const sha256 = await hashFile(join(root, ...entry.path.split("/")), entry.size, buffer, metrics, signal);@@ -602,6 +711,7 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):});signal?.throwIfAborted();excluded.unreadable += unreadable.size;+ for (const entry of unreadable) note(entry.path, "unreadable", "file", entry.size);const kept = unreadable.size > 0 ? entries.filter((entry) => !unreadable.has(entry)) : entries;kept.sort((left, right) => compareArchivePaths(left.path, right.path));cache?.commitScan();@@ -669,10 +779,14 @@ export function isArchiveDeltaEmpty(delta: ArchiveDelta<ArchiveEntry>): booleanexport function archiveEntry(entry: ArchiveEntry): ArchiveEntry {const out: ArchiveEntry = { path: entry.path, type: entry.type, mode: entry.mode, size: entry.size, sha256: entry.sha256 };if (entry.type === "symlink") out.target = entry.target ?? "";+ if (entry.type === "symlink" && entry.broken) out.broken = true;+ if (entry.type === "file" && entry.scrubbed) out.scrubbed = true;return out;}export type ManifestInput = {+ /** Capture v2 chains: ARCHIVE_SCHEMA_V2 (capture-v2.ts). */+ schema?: string;kind: ArchiveKind;archiveId: string;sessionId: string;@@ -704,7 +818,7 @@ export type ManifestSource = { size: number; chunks: () => Iterable<Buffer> };*/export function manifestSource(input: ManifestInput): ManifestSource {const head = JSON.stringify({- schema: ARCHIVE_SCHEMA,+ schema: input.schema ?? ARCHIVE_SCHEMA,kind: input.kind,archive_id: input.archiveId,session_id: input.sessionId,diff --git a/apps/server/src/session-archive/pack.ts b/apps/server/src/session-archive/pack.tsindex 5b26ca3b..640a2358 100644--- a/apps/server/src/session-archive/pack.ts+++ b/apps/server/src/session-archive/pack.ts@@ -136,6 +136,8 @@ export type PackInput = {root: string;/** `__omnirush__/manifest.json`: its bytes, or a source that serialises it while it is written. */manifest: Buffer | ManifestSource;+ /** Capture v2: members written right after the manifest (`__omnirush__/state.json`). */+ extraMembers?: ReadonlyArray<{ name: string; content: Buffer }>;/** created_at in seconds: the mtime of the manifest and unstable members. */createdAtSeconds: number;/** Exactly the manifest's `files`, in the same order. */@@ -311,6 +313,7 @@ class TarWriter {async write(): Promise<void> {const { input, unstable } = this;await this.smallFile(MANIFEST_MEMBER, input.manifest);+ for (const member of input.extraMembers ?? []) await this.smallFile(member.name, member.content);let members = 0;for (const entry of input.entries) {input.signal?.throwIfAborted();@@ -320,7 +323,22 @@ class TarWriter {await this.put(tarMemberHeader({ name: `${entry.path}/`, mode: entry.mode, size: 0, mtime: entry.mtime, typeflag: "5" }));continue;}+ if (entry.type === "file" && entry.content) {+ // Bytes prepared in pass 1 (a scrubbed git config).+ await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.content.length, mtime: entry.mtime, typeflag: "0" }));+ await this.put(entry.content);+ await this.zeros(paddingFor(entry.content.length));+ continue;+ }const parts = entry.path.split("/");+ if (entry.type === "file" && entry.source) {+ // A staged file (an attachment) read from its own absolute path under the state dir.+ await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.size, mtime: entry.mtime, typeflag: "0" }));+ const stable = await this.fileContent(entry, entry.source, true);+ await this.zeros(paddingFor(entry.size));+ if (!stable) unstable.push(entry.path);+ continue;+ }// A file outside the workspace (`__outside__/...`) is read from its own absolute path.const outside = isOutsideArchivePath(entry.path);const source = outside ? outsideSourcePath(entry.path) : null;diff --git a/apps/server/src/session-archive/policy.ts b/apps/server/src/session-archive/policy.tsindex b9f8fc38..d36991d6 100644--- a/apps/server/src/session-archive/policy.ts+++ b/apps/server/src/session-archive/policy.ts@@ -8,14 +8,21 @@*/import { z } from "zod";-export type ArchivePolicy = { allFolders: boolean; touchedFiles: boolean };+export type ArchivePolicy = { allFolders: boolean; touchedFiles: boolean; captureV2?: boolean };/** Both off: what a failed probe, a missing policy or a disabled account means. */export const POLICY_OFF: ArchivePolicy = { allFolders: false, touchedFiles: false };const allFoldersSchema = z.object({ policy: z.object({ all_folders: z.literal(true) }) });const touchedFilesSchema = z.object({ policy: z.object({ touched_files: z.literal(true) }) });+/** Capture v2 (capture-v2.ts): the server takes the byte-exact state archives. */+const captureV2Schema = z.object({ policy: z.object({ capture_v2: z.literal(true) }) });export function parseArchivePolicy(body: unknown): ArchivePolicy {- return { allFolders: allFoldersSchema.safeParse(body).success, touchedFiles: touchedFilesSchema.safeParse(body).success };+ return {+ allFolders: allFoldersSchema.safeParse(body).success,+ touchedFiles: touchedFilesSchema.safeParse(body).success,+ // Present only when on: a policy without it reads exactly as before.+ ...(captureV2Schema.safeParse(body).success ? { captureV2: true } : {}),+ };}diff --git a/apps/server/src/session-archive/repos.ts b/apps/server/src/session-archive/repos.tsnew file mode 100644index 00000000..037617e9--- /dev/null+++ b/apps/server/src/session-archive/repos.ts@@ -0,0 +1,371 @@+/**+ * Capture v2: the git state of every repository a session works in.+ *+ * - the root's own repository (`position: "root"`);+ * - repositories in subfolders (`"nested"`): a folder holding `.git` (a+ * directory, or a gitfile: a submodule or a linked worktree) found by+ * the archive scan;+ * - a repository whose work tree starts above the root (`"above"`): the+ * root is a folder inside it; its relative position is recorded+ * (`path: "../.."`), its `.git` is never archived (it is outside the+ * project folder), only this metadata.+ *+ * Each gets HEAD, branch (also on an unborn branch: a repository without+ * commits), the remote without userinfo, the tracked-file count, the dirty+ * files with their status, the staged files and the stash count; or, when+ * git cannot read it, why (`error`).+ *+ * Why the old git blocks came back empty while the agent's own git commands+ * worked, and what this runner does about each:+ * - `dubious ownership` (a folder owned by another user: a container+ * running as root on a mounted checkout, a shared drive): git refuses+ * the repository. The capture retries once with `safe.directory` set to+ * that repository only, with fsmonitor and hooks off, and records it;+ * - git environment inherited from the launching process (`GIT_DIR`,+ * `GIT_WORK_TREE`, `GIT_INDEX_FILE`, ... when started from a git hook or+ * `git rebase -x`) pointing git somewhere else: removed;+ * - short timeouts under load (5 s for `rev-parse` while a full scan+ * hashes the tree): 20 s here, and a timeout is reported, not hidden;+ * - a repository created by the agent (`git init`, `git clone`) after the+ * session started, or in a subfolder: found again at every state;+ * - a plain-folder or touched-files chain never read git at all: every+ * v2 state reads the repositories, whatever the chain's marker.+ */+import { spawn } from "node:child_process";+import { realpathSync } from "node:fs";+import { lstat, readFile } from "node:fs/promises";+import { homedir } from "node:os";+import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";++import { gitSkipReason } from "../command-guard.js";++export type RepoPosition = "root" | "nested" | "above";+export type RepoError = "timeout" | "dubious_ownership" | "not_a_repository" | "git_missing" | "failed";++export type RepoState = {+ path: string;+ position: RepoPosition;+ git_dir: string | null;+ gitfile: boolean;+ head: string | null;+ branch: string | null;+ unborn: boolean;+ remote: string | null;+ tracked_files: number | null;+ dirty: boolean;+ dirty_files: Array<{ path: string; status: string }>;+ dirty_truncated: boolean;+ staged_files: string[];+ stash_count: number;+ archived: "byte_exact" | "metadata_only";+ safe_directory_override?: true;+ error: RepoError | null;+};++export const MAX_REPOS = 64;+const MAX_DIRTY_LISTED = 500;+/** One repository item of state.json stays far below the reader's 1 MiB item cap (backend spec 19.3). */+const MAX_LISTED_PATH_BYTES = 128 * 1024;++/** The first items of `items` whose paths fit `MAX_LISTED_PATH_BYTES` (and MAX_DIRTY_LISTED), and whether any was left out. */+function boundedList<T>(items: readonly T[], pathOf: (item: T) => string): { kept: T[]; truncated: boolean } {+ const kept: T[] = [];+ let bytes = 0;+ for (const item of items) {+ bytes += Buffer.byteLength(pathOf(item)) + 32;+ if (kept.length >= MAX_DIRTY_LISTED || bytes > MAX_LISTED_PATH_BYTES) return { kept, truncated: true };+ kept.push(item);+ }+ return { kept, truncated: false };+}+const GIT_TIMEOUT_MS = 20_000;+/** Variables that redirect git away from the folder it is run in. */+const REDIRECTING_GIT_ENV = [+ "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES",+ "GIT_COMMON_DIR", "GIT_NAMESPACE", "GIT_PREFIX", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT",+ "GIT_DISCOVERY_ACROSS_FILESYSTEM", "GIT_EXEC_PATH",+];++/** The environment capture git runs with: no redirecting variables, no prompts, no optional locks, C locale. */+export function captureGitEnv(base: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {+ const env: NodeJS.ProcessEnv = { ...base };+ for (const name of REDIRECTING_GIT_ENV) delete env[name];+ for (const name of Object.keys(env)) if (/^GIT_CONFIG_(?:KEY|VALUE)_\d+$/.test(name)) delete env[name];+ let home = homedir();+ try {+ home = realpathSync(home);+ } catch {+ // As given.+ }+ const ceiling = base.GIT_CEILING_DIRECTORIES;+ env.GIT_CEILING_DIRECTORIES = home ? (ceiling ? `${ceiling}${delimiter}${home}` : home) : ceiling ?? "";+ env.GIT_TERMINAL_PROMPT = "0";+ env.GIT_OPTIONAL_LOCKS = "0";+ env.GIT_PAGER = "cat";+ env.PAGER = "cat";+ env.LC_ALL = "C";+ return env;+}++export type RepoGitRun = { ok: boolean; stdout: Buffer; stderr: string; timedOut: boolean; missing: boolean; truncated: boolean };++/** One git run in `cwd`, read-only settings, bounded output; never throws. `onChunk` streams stdout instead of keeping it. */+export async function runRepoGit(+ cwd: string,+ args: string[],+ options: { maxBytes?: number; timeoutMs?: number; safeDirectory?: string; onChunk?: (chunk: Buffer) => void } = {},+): Promise<RepoGitRun> {+ // A Mac without the Command Line Tools: the git shim would open the installer (command-guard.ts).+ if (await gitSkipReason()) return { ok: false, stdout: Buffer.alloc(0), stderr: "", timedOut: false, missing: true, truncated: false };+ return spawnRepoGit(cwd, args, options);+}++function spawnRepoGit(+ cwd: string,+ args: string[],+ options: { maxBytes?: number; timeoutMs?: number; safeDirectory?: string; onChunk?: (chunk: Buffer) => void },+): Promise<RepoGitRun> {+ const maxBytes = options.maxBytes ?? 256 * 1024;+ return new Promise((resolvePromise) => {+ const out: Buffer[] = [];+ const err: Buffer[] = [];+ let total = 0;+ let errTotal = 0;+ let truncated = false;+ let timedOut = false;+ let settled = false;+ let child: ReturnType<typeof spawn> | null = null;+ const finish = (ok: boolean, missing = false) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ resolvePromise({ ok, stdout: Buffer.concat(out), stderr: Buffer.concat(err).toString("utf8"), timedOut, missing, truncated });+ };+ const timer = setTimeout(() => {+ timedOut = true;+ child?.kill("SIGKILL");+ finish(false);+ }, options.timeoutMs ?? GIT_TIMEOUT_MS);+ timer.unref?.();+ const config = [+ "-c", "core.fsmonitor=false", "-c", "core.quotePath=false", "-c", `core.hooksPath=${process.platform === "win32" ? "NUL" : "/dev/null"}`,+ ...(options.safeDirectory ? ["-c", `safe.directory=${options.safeDirectory}`] : []),+ ];+ try {+ child = spawn("git", ["-C", cwd, ...config, ...args], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, env: captureGitEnv() });+ } catch {+ finish(false, true);+ return;+ }+ child.stdout?.on("data", (chunk: Buffer) => {+ if (options.onChunk) {+ options.onChunk(chunk);+ return;+ }+ if (truncated) return;+ const room = maxBytes - total;+ if (chunk.length > room) {+ out.push(chunk.subarray(0, room));+ total = maxBytes;+ truncated = true;+ child?.kill("SIGKILL");+ return;+ }+ out.push(chunk);+ total += chunk.length;+ });+ child.stderr?.on("data", (chunk: Buffer) => {+ if (errTotal < 16 * 1024) {+ err.push(chunk);+ errTotal += chunk.length;+ }+ });+ child.on("error", (error: NodeJS.ErrnoException) => finish(false, error.code === "ENOENT"));+ child.on("close", (code) => finish(!timedOut && (truncated || code === 0)));+ });+}++function errorOf(run: RepoGitRun): RepoError {+ if (run.missing) return "git_missing";+ if (run.timedOut) return "timeout";+ if (/dubious ownership|safe\.directory/i.test(run.stderr)) return "dubious_ownership";+ if (/not a git repository/i.test(run.stderr)) return "not_a_repository";+ return "failed";+}++function portable(path: string): string {+ return sep === "/" ? path : path.split(sep).join("/");+}++/** Strips `user:password@` from a remote URL (scheme form) and the password from an scp form. */+export function stripRemoteCredentials(url: string): string {+ const trimmed = url.trim();+ const scheme = /^([a-z][a-z0-9+.-]*:\/\/)(?:[^/@]*@)?(.*)$/i.exec(trimmed);+ if (scheme) return `${scheme[1]}${scheme[2]}`;+ const scp = /^([^/@:\s]+)(?::[^@/]*)?@([^/:\s]+:.*)$/.exec(trimmed);+ if (scp) return `${scp[1]}@${scp[2]}`;+ return trimmed;+}++type Located = { top: string; gitDir: string; safeDirectory?: string } | { error: RepoError };++async function locate(dir: string): Promise<Located> {+ const args = ["rev-parse", "--show-toplevel", "--absolute-git-dir"];+ let run = await runRepoGit(dir, args, { maxBytes: 64 * 1024 });+ let safeDirectory: string | undefined;+ if (!run.ok && errorOf(run) === "dubious_ownership") {+ // Only this repository, never `*`: the user's own folder the agent works in.+ const match = /repository at '([^']+)'/.exec(run.stderr);+ safeDirectory = match?.[1] ?? dir;+ run = await runRepoGit(dir, args, { maxBytes: 64 * 1024, safeDirectory });+ }+ if (!run.ok) return { error: errorOf(run) };+ const [top, gitDir] = run.stdout.toString("utf8").split(/\r?\n/);+ if (!top || !gitDir) return { error: "not_a_repository" };+ return { top, gitDir, ...(safeDirectory ? { safeDirectory } : {}) };+}++function within(root: string, path: string): string | null {+ const rel = relative(root, path);+ if (rel === "") return "";+ if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;+ return portable(rel);+}++/** Parses `git status --porcelain=v1 -z` into dirty and staged files. */+export function parsePorcelainZ(text: string): { dirty: Array<{ path: string; status: string }>; staged: string[] } {+ const dirty: Array<{ path: string; status: string }> = [];+ const staged: string[] = [];+ const parts = text.split("\0");+ for (let index = 0; index < parts.length; index += 1) {+ const item = parts[index]!;+ if (item.length < 4) continue;+ const status = item.slice(0, 2);+ const path = item.slice(3);+ dirty.push({ path, status });+ if (status[0] !== " " && status[0] !== "?" && status[0] !== "!") staged.push(path);+ // A rename or copy is followed by its source path.+ if (status[0] === "R" || status[0] === "C") index += 1;+ }+ return { dirty, staged };+}++/** The state of the repository whose work tree is `top`, as seen from `root`. */+export async function readRepoState(root: string, located: { top: string; gitDir: string; safeDirectory?: string }, position: RepoPosition, gitfile: boolean): Promise<RepoState> {+ const { top, gitDir, safeDirectory } = located;+ const opt = (extra: { maxBytes?: number; timeoutMs?: number } = {}) => ({ ...extra, ...(safeDirectory ? { safeDirectory } : {}) });+ let tracked = 0;+ const [head, branch, remotes, status, lsFiles, stash] = await Promise.all([+ runRepoGit(top, ["rev-parse", "--verify", "-q", "HEAD"], opt()),+ runRepoGit(top, ["symbolic-ref", "--short", "-q", "HEAD"], opt()),+ runRepoGit(top, ["remote"], opt()),+ runRepoGit(top, ["status", "--porcelain=v1", "-z", "--no-renames", "--untracked-files=normal"], opt({ maxBytes: 4 * 1024 * 1024 })),+ runRepoGit(top, ["ls-files", "-z"], {+ ...opt(),+ onChunk: (chunk) => {+ for (const byte of chunk) if (byte === 0) tracked += 1;+ },+ }),+ runRepoGit(top, ["rev-list", "--walk-reflogs", "--count", "refs/stash"], opt()),+ ]);+ const headText = head.ok ? head.stdout.toString("utf8").trim() : "";+ const branchText = branch.ok ? branch.stdout.toString("utf8").trim() : "";+ const names = remotes.ok ? remotes.stdout.toString("utf8").split(/\r?\n/).map((name) => name.trim()).filter(Boolean) : [];+ const remoteName = names.includes("origin") ? "origin" : names[0];+ const remoteUrl = remoteName ? await runRepoGit(top, ["remote", "get-url", remoteName], opt()) : null;+ const parsed = status.ok ? parsePorcelainZ(status.stdout.toString("utf8")) : { dirty: [], staged: [] };+ const dirtyListed = boundedList(parsed.dirty, (item) => item.path);+ const stagedListed = boundedList(parsed.staged, (path) => path);+ const relTop = relative(root, top);+ const insideRoot = within(root, gitDir) !== null;+ const failed = [status, lsFiles].find((run) => !run.ok && !run.truncated);+ return {+ path: portable(relTop),+ position,+ git_dir: insideRoot ? within(root, gitDir) : null,+ gitfile,+ head: /^[0-9a-f]{40,64}$/.test(headText) ? headText : null,+ branch: branchText || null,+ unborn: !/^[0-9a-f]{40,64}$/.test(headText) && Boolean(branchText),+ remote: remoteUrl?.ok ? stripRemoteCredentials(remoteUrl.stdout.toString("utf8").trim()) || null : null,+ tracked_files: lsFiles.ok ? tracked : null,+ dirty: parsed.dirty.length > 0,+ dirty_files: dirtyListed.kept,+ dirty_truncated: dirtyListed.truncated || stagedListed.truncated || status.truncated,+ staged_files: stagedListed.kept,+ stash_count: stash.ok ? Number.parseInt(stash.stdout.toString("utf8").trim(), 10) || 0 : 0,+ archived: position !== "above" && insideRoot ? "byte_exact" : "metadata_only",+ ...(safeDirectory ? { safe_directory_override: true as const } : {}),+ error: failed ? errorOf(failed) : null,+ };+}++/** A repository marker git could not read: recorded with the reason. */+function unreadable(root: string, dir: string, position: RepoPosition, gitfile: boolean, error: RepoError): RepoState {+ return {+ path: portable(relative(root, dir)), position, git_dir: null, gitfile, head: null, branch: null, unborn: false, remote: null,+ tracked_files: null, dirty: false, dirty_files: [], dirty_truncated: false, staged_files: [], stash_count: 0, archived: "metadata_only", error,+ };+}++/**+ * Every repository of the session: `markers` are the root-relative paths of+ * the `.git` entries the archive scan found (`.git`, `sub/x/.git`), with+ * whether each is a gitfile. The root's enclosing repository (above the+ * root) is found by git itself, never above home.+ */+export async function discoverRepos(root: string, markers: ReadonlyArray<{ path: string; gitfile: boolean }>): Promise<RepoState[]> {+ const absRoot = resolve(root);+ const repos: RepoState[] = [];+ const seenTops = new Set<string>();+ const ordered = [...markers].sort((left, right) => left.path.split("/").length - right.path.split("/").length).slice(0, MAX_REPOS);+ const hasRootMarker = ordered.some((marker) => marker.path === ".git");+ if (!hasRootMarker) {+ // The root inside a larger repository (or none).+ const located = await locate(absRoot);+ if ("top" in located) {+ const rel = relative(absRoot, located.top);+ if (rel !== "" && (rel === ".." || rel.startsWith(`..${sep}`))) {+ seenTops.add(located.top);+ repos.push(await readRepoState(absRoot, located, "above", false));+ }+ } else if (located.error === "dubious_ownership" || located.error === "timeout" || located.error === "git_missing") {+ repos.push(unreadable(absRoot, absRoot, "above", false, located.error));+ }+ }+ for (const marker of ordered) {+ const dir = marker.path === ".git" ? absRoot : join(absRoot, ...marker.path.split("/").slice(0, -1));+ const position: RepoPosition = marker.path === ".git" ? "root" : "nested";+ const located = await locate(dir);+ if ("error" in located) {+ repos.push(unreadable(absRoot, dir, position, marker.gitfile, located.error));+ continue;+ }+ // `.git` folders of a repository's own internals or a broken marker resolving elsewhere are not repositories of their own.+ if (resolve(located.top) !== resolve(dir) && realpathOr(located.top) !== realpathOr(dir)) continue;+ if (seenTops.has(located.top)) continue;+ seenTops.add(located.top);+ repos.push(await readRepoState(absRoot, located, position, marker.gitfile));+ }+ return repos;+}++function realpathOr(path: string): string {+ try {+ return realpathSync(path);+ } catch {+ return resolve(path);+ }+}++/** Whether `path` (absolute) holds a `.git` gitfile, and where it points. */+export async function readGitfile(path: string): Promise<string | null> {+ try {+ const info = await lstat(path);+ if (!info.isFile() || info.size > 4096) return null;+ const match = /^gitdir:\s*(.+?)\s*$/m.exec(await readFile(path, "utf8"));+ return match ? resolve(dirname(path), match[1]!) : null;+ } catch {+ return null;+ }+}diff --git a/apps/server/src/session-archive/tool-start.ts b/apps/server/src/session-archive/tool-start.tsnew file mode 100644index 00000000..4ea14a78--- /dev/null+++ b/apps/server/src/session-archive/tool-start.ts@@ -0,0 +1,115 @@+/**+ * Capture v2: the moment a turn's first tool call starts. While the observer+ * follows a turn it listens to the engine's event stream (v1 `/event`, v2+ * `/api/event`) and calls `onToolStart` once, at the first tool part of that+ * session that is pending or running (v1 `message.part.updated` with a+ * `tool` part, v2 `session.tool.input.started` / `session.tool.called`).+ * The capture it triggers runs in the background: the tool is never held.+ * Identical in the CLI and the desktop app.+ */++export type ToolStartFetch = (url: string, init: { headers: Headers; signal: AbortSignal }) => Promise<Response>;++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++const V2_TOOL_EVENTS = /^session\.(?:next\.)?tool\.(?:input\.started|called)$/;++/** The session id of a tool-start event, or null when `payload` is not one. */+export function toolStartSession(payload: unknown): string | null {+ if (!isRecord(payload)) return null;+ // /global/event wraps each event as { directory, payload }.+ if (isRecord(payload.payload) && typeof payload.payload.type === "string") return toolStartSession(payload.payload);+ const type = typeof payload.type === "string" ? payload.type : "";+ if (type === "message.part.updated") {+ const properties = isRecord(payload.properties) ? payload.properties : null;+ const part = properties && isRecord(properties.part) ? properties.part : null;+ if (!part || part.type !== "tool") return null;+ const status = isRecord(part.state) ? part.state.status : undefined;+ if (status !== "pending" && status !== "running") return null;+ return typeof part.sessionID === "string" ? part.sessionID : null;+ }+ if (V2_TOOL_EVENTS.test(type)) {+ const data = isRecord(payload.data) ? payload.data : isRecord(payload.properties) ? payload.properties : null;+ return data && typeof data.sessionID === "string" ? data.sessionID : null;+ }+ return null;+}++/** Splits SSE text into the `data:` payloads of complete frames; returns the unfinished rest. */+export function sseFrames(buffer: string): { payloads: string[]; rest: string } {+ const payloads: string[] = [];+ let rest = buffer;+ for (;;) {+ const match = /\r?\n\r?\n/.exec(rest);+ if (!match) break;+ const frame = rest.slice(0, match.index);+ rest = rest.slice(match.index + match[0].length);+ const data = frame.split(/\r?\n/).filter((line) => line.startsWith("data:")).map((line) => line.slice(5).replace(/^ /, "")).join("\n");+ if (data) payloads.push(data);+ }+ return { payloads, rest };+}++const MAX_FRAME_CHARS = 4 * 1024 * 1024;++/**+ * Listens until `signal` aborts or the first tool start of `sessionId`+ * (then `onToolStart` runs once and the stream closes). Never throws; a+ * stream that cannot be opened is simply no hook (the turn's end capture+ * still runs).+ */+export async function watchToolStarts(input: {+ url: string;+ headers: Headers;+ sessionId: string;+ signal: AbortSignal;+ fetch: ToolStartFetch;+ onToolStart: () => void;+}): Promise<void> {+ const controller = new AbortController();+ const stop = () => controller.abort();+ if (input.signal.aborted) return;+ input.signal.addEventListener("abort", stop, { once: true });+ try {+ const headers = new Headers(input.headers);+ headers.set("accept", "text/event-stream");+ const response = await input.fetch(input.url, { headers, signal: controller.signal });+ if (!response.ok || !response.body) {+ await response.body?.cancel().catch(() => undefined);+ return;+ }+ const reader = response.body.getReader();+ const decoder = new TextDecoder();+ let buffer = "";+ for (;;) {+ const { done, value } = await reader.read();+ if (done) return;+ buffer += decoder.decode(value, { stream: true });+ const { payloads, rest } = sseFrames(buffer);+ buffer = rest.length > MAX_FRAME_CHARS ? "" : rest;+ for (const text of payloads) {+ let payload: unknown;+ try {+ payload = JSON.parse(text);+ } catch {+ continue;+ }+ if (toolStartSession(payload) === input.sessionId) {+ try {+ input.onToolStart();+ } finally {+ controller.abort();+ }+ await reader.cancel().catch(() => undefined);+ return;+ }+ }+ }+ } catch {+ // Aborted, or the engine went away: no hook for this turn.+ } finally {+ input.signal.removeEventListener("abort", stop);+ }+}diff --git a/apps/server/src/session-archive/touched.ts b/apps/server/src/session-archive/touched.tsindex 2867ca2f..69b8f284 100644--- a/apps/server/src/session-archive/touched.ts+++ b/apps/server/src/session-archive/touched.ts@@ -132,10 +132,13 @@ const LOCKFILE_NAMES = new Set(["Cargo.lock", "poetry.lock", "Pipfile.lock", "uv.lock", "pdm.lock", "pixi.lock", "conda-lock.yml","Gemfile.lock", "composer.lock", "go.sum", "go.work.sum", "mix.lock", "pubspec.lock", "Package.resolved","packages.lock.json", "gradle.lockfile", "flake.lock", "Podfile.lock", "Cartfile.resolved",+ // Capture v2 (#4): more ecosystems' pins.+ "renv.lock", "Manifest.toml", "paket.lock", "stack.yaml.lock", "cabal.project.freeze", "shard.lock",+ "requirements.lock", "requirements-dev.lock", "vcpkg-lock.json", "MODULE.bazel.lock",]);-const LOCKFILE_MAX_DEPTH = 6;-const LOCKFILE_MAX_DIRS = 5_000;-const LOCKFILE_MAX_FOUND = 200;+const LOCKFILE_MAX_DEPTH = 8;+const LOCKFILE_MAX_DIRS = 20_000;+const LOCKFILE_MAX_FOUND = 1_000;/*** The lockfiles under `root`, workspace-relative and portable: a bounded walkdiff --git a/apps/server/src/session-archive/turn-files.ts b/apps/server/src/session-archive/turn-files.tsindex bf9ba6cd..b489571f 100644--- a/apps/server/src/session-archive/turn-files.ts+++ b/apps/server/src/session-archive/turn-files.ts@@ -292,9 +292,16 @@ export async function recordTurnFiles(input: TurnFilesInput): Promise<TurnFilesRlet recordPath: string;if (rel !== null) {// Inside the workspace: the turn diff and the workspace artifacts carry what it wrote; reads are recorded.- if (access === "write") continue;if (!context.includeCredentialFiles && isArchiveCredentialPath(rel)) continue;if (context.appDirs.some((dir) => within(dir, absolute) !== null)) continue;+ // Capture v2 (#14): a file the turn read, wrote or named in a command counts as touched, so a+ // gitignored one (a dataset, a generated output, a venv's tool) is archived byte for byte.+ try {+ input.archive?.pathTouched(input.sessionId, rel);+ } catch {+ // The archive's bookkeeping never stops a capture.+ }+ if (access === "write") continue;recordPath = rel;} else {const archivePath = outsideArchivePath(absolute);diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.tsindex 25bbc9e8..5a33df45 100644--- a/apps/server/src/session-upload-observer.ts+++ b/apps/server/src/session-upload-observer.ts@@ -11,6 +11,7 @@ import { loopbackFetch } from "./server-fetch.js";import { isFinishedAssistantMessage, isPromptMessage, type ArchiveEngineReads, type ProjectArchiveLifecycle } from "./session-archive/lifecycle.js";import { MAX_UPLOAD_CHILD_SESSION_DEPTH, type UploadSessionModel, type SessionUploader } from "./session-uploader.js";import { recordTurnFiles, type TurnFilesInput } from "./session-archive/turn-files.js";+import { watchToolStarts } from "./session-archive/tool-start.js";/** The engine a captured request went to: base URL, request headers (the engine's auth), query and API generation. */export type EngineTarget = {@@ -751,7 +752,7 @@ export function projectArchiveEngineReads(target: EngineTarget | (() => EngineTa*/export function observeUploadedSession(input: {sessionUploader: ObservedUploader;- archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete">;+ archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete"> & Partial<Pick<ProjectArchiveLifecycle, "turnMessages">>;/*** The session's workspace: each settled turn reports the files outside it* that the turn touched to the archive, and records the files it read@@ -920,6 +921,8 @@ export function observeUploadedSession(input: {// survives app restarts; without the messages the archiver numbers it// right after the last archived turn. It also arms the idle final archive.turnArchived = true;+ // Capture v2: the turn's attachments are staged ahead of its delta.+ if (history) input.archive.turnMessages?.(sessionId, history.delta);input.archive.turnCompleted(sessionId, history ? history.outline : null);};@@ -1057,3 +1060,45 @@ export function observeUploadedSession(input: {});return session.done;}++/** Capture v2: the tool-start listener of each session's current turn (followToolStart). */+const TOOL_WATCHES = new WeakMap<SessionObservers, Map<string, AbortController>>();+/** A turn's listener stops after this long at most (the observer's own bound). */+const MAX_TOOL_WATCH_MS = MAX_OBSERVED_TURN_MS;++/**+ * Capture v2, #9: call when a prompt of the session goes to the engine.+ * Listens to the engine's event stream until the turn's first tool call+ * starts and tells the project archive (toolStarted), which captures the+ * folder's state in the background when it changed. A newer prompt of the+ * session replaces the listener; the observers' stop ends it.+ */+export function followToolStart(input: {+ observers: SessionObservers;+ archive: Pick<ProjectArchiveLifecycle, "toolStarted">;+ sessionId: string;+ target: EngineTarget;+}): void {+ let watches = TOOL_WATCHES.get(input.observers);+ if (!watches) {+ watches = new Map();+ TOOL_WATCHES.set(input.observers, watches);+ }+ watches.get(input.sessionId)?.abort();+ const controller = new AbortController();+ watches.set(input.sessionId, controller);+ const map = watches;+ const { baseUrl, headers, search, engine } = input.target;+ const url = buildOpencodeProxyUrl(baseUrl, engine === "v2" ? "/api/event" : "/event", search);+ const signal = AbortSignal.any([controller.signal, input.observers.controller.signal, AbortSignal.timeout(MAX_TOOL_WATCH_MS)]);+ void watchToolStarts({+ url,+ headers: new Headers(headers),+ sessionId: input.sessionId,+ signal,+ fetch: (target, init) => loopbackFetch(target, init),+ onToolStart: () => input.archive.toolStarted(input.sessionId),+ }).finally(() => {+ if (map.get(input.sessionId) === controller) map.delete(input.sessionId);+ });+}diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 5e5ec085..ed212587 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -1,7 +1,7 @@import { createHash, randomBytes } from "node:crypto";import { execFile, spawn } from "node:child_process";import { createReadStream, createWriteStream, watch, type FSWatcher, type WriteStream } from "node:fs";-import { lstat, mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";+import { lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";import { release as osRelease, tmpdir } from "node:os";import nodePath, { basename, dirname, extname, join, relative, resolve, sep, type PlatformPath } from "node:path";import { promisify } from "node:util";@@ -20,6 +20,8 @@ import {removeLegacyUploadContent,} from "./session-upload-state.js";import { TurnBaseStore, TurnDiffBuilder, type TurnDiffInput } from "./turn-diff.js";+import { binaryReason, isKnownBinaryPath } from "./session-archive/binary.js";+import { captureGitEnv } from "./session-archive/repos.js";import { SUBAGENT_MODEL_FALLBACK_TRACE } from "./omnirush-swarm.js";import { writeFileAtomic } from "./atomic-write.js";import { XCODE_CLT_MISSING, gitSkipReason } from "./command-guard.js";@@ -367,8 +369,27 @@ type HashCacheEntry = {/** Redaction changed nothing, so the raw bytes are the upload form (verified by digest on upload). */clean: boolean;uploadPath: string;+ /** Capture v2: the file's permission bits (the executable bit included). */+ mode?: number;};+/**+ * Capture v2: a manifest entry that carries no text: a binary file (its raw+ * size and SHA-256; its bytes go to the byte-exact project archive) or a+ * symlink (its target, also when broken; sha256 and size are those of the+ * target string).+ */+type ExtraManifestEntry = {+ uploadPath: string;+ kind: "binary" | "symlink";+ sha256: string;+ size: number;+ mode: number;+ target?: string;+ broken?: boolean;+};++type SpoolMeta = {id: string;session_id: string;@@ -465,6 +486,8 @@ type SessionState = {lastSuccessAt?: string;ready: Promise<void>;tail: Promise<void>;+ /** Capture v2: the binary files and symlinks of the workspace as last scanned (manifest entries without text). */+ extras: Map<string, ExtraManifestEntry>;};type SessionUploaderOptions = {@@ -516,6 +539,12 @@ type SessionUploaderOptions = {* touched-files mode keeps them; it applies its own exclusions.*/onPathTouched?: (sessionId: string, path: string) => void;+ /**+ * Capture v2: a binary file of the workspace within the per-file cap+ * (workspace-relative). It is never sent as text: the host hands it to the+ * byte-exact project archive.+ */+ onBinaryFile?: (sessionId: string, path: string) => void;};export type TraceCapabilities = {@@ -1775,39 +1804,6 @@ export function stripRemoteUserinfo(url: string): string {return trimmed;}-/**- * Whether `buffer` holds binary content: a NUL or many control bytes in its- * first 8 KiB, or bytes that are not valid UTF-8 anywhere (a PDF, an image).- * Text is uploaded as UTF-8, so a file that does not decode strictly would be- * stored altered (each invalid byte becoming U+FFFD). `partial` is set for a- * head cut from a longer file, which may end inside a character.- */-function isBinary(buffer: Buffer, partial = false): boolean {- const length = Math.min(buffer.length, 8_192);- let suspicious = 0;- // An indexed loop: iterating a Buffer is several times slower, once per file.- for (let index = 0; index < length; index += 1) {- const byte = buffer[index]!;- if (byte === 0) return true;- if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;- }- if (length > 0 && suspicious / length > 0.1) return true;- return !isValidUtf8(buffer, partial);-}--/** Strict UTF-8 validation (native); with `partial`, an incomplete last character is allowed. */-function isValidUtf8(buffer: Buffer, partial: boolean): boolean {- if (isUtf8(buffer)) return true;- if (!partial) return false;- for (let back = 1; back <= 3 && back <= buffer.length; back += 1) {- const byte = buffer[buffer.length - back]!;- if ((byte & 0xc0) === 0x80) continue;- const length = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1;- return length > back && isUtf8(buffer.subarray(0, buffer.length - back));- }- return false;-}-const PATH_FIELD_NAMES = new Set(["path","file_path",@@ -2314,7 +2310,8 @@ async function walkFallback(listing.denied += 1;if ((listing.deniedPaths ??= []).length < MAX_LISTED_EXCLUSIONS) listing.deniedPaths.push(`${path}/`);} else await walkFallback(root, fullPath, ignoreRules, listing, priorityPaths);- } else if (entry.isFile()) {+ } else if (entry.isFile() || entry.isSymbolicLink()) {+ // Capture v2: a symlink is listed too (never followed; the scan records its target).if (isUploadPathDenied(path)) {listing.denied += 1;if ((listing.deniedPaths ??= []).length < MAX_LISTED_EXCLUSIONS) listing.deniedPaths.push(path);@@ -2390,10 +2387,13 @@ function runGitNow(root: string, args: string[], options: { maxBytes: number; ti}, options.timeoutMs);timer.unref?.();try {- child = spawn("git", ["-C", root, "-c", "core.quotePath=false", ...args], {+ // Capture v2: no inherited GIT_DIR/GIT_WORK_TREE (a server started from a git hook), no fsmonitor, and+ // the session's own root accepted when another user owns it (git's dubious-ownership refusal left+ // the git block empty while the agent's own git commands worked). See session-archive/repos.ts.+ child = spawn("git", ["-C", root, "-c", "core.quotePath=false", "-c", "core.fsmonitor=false", "-c", `safe.directory=${resolve(root)}`, ...args], {stdio: ["ignore", "pipe", "ignore"],windowsHide: true,- env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0", GIT_PAGER: "cat", PAGER: "cat" },+ env: { ...captureGitEnv(), GIT_CEILING_DIRECTORIES: process.env.GIT_CEILING_DIRECTORIES ?? "", LC_ALL: process.env.LC_ALL ?? "" },});} catch {finish(false);@@ -2551,6 +2551,9 @@ export function filterUploadDiff(raw: string, inputTruncated = false): { diff: sconst path = diffHeaderPath(headerEnd === -1 ? section : section.slice(0, headerEnd));if (!path || isUploadPathDenied(path)) continue;if (/^(?:Binary files .* differ|GIT binary patch)/m.test(section)) continue;+ // Capture v2: a binary type is never diffed as text (its bytes go to the byte-exact archive), and+ // a section that was not valid UTF-8 (utf8DiffText) never reaches here.+ if (isKnownBinaryPath(path)) continue;// A diff is text even when the file is JSON: hunks are not parseable documents.// Each file's section under the mode its path selects, as the file's snapshot.const redacted = redactUploadText(section, { context: path, mode: redactModeForPath(path) }).text;@@ -2568,6 +2571,29 @@ export function filterUploadDiff(raw: string, inputTruncated = false): { diff: sreturn { diff: output || null, truncated };}+/**+ * Capture v2: git's diff output as text, without the file sections that are+ * not valid UTF-8. git diffs any file without a NUL in its first 8000 bytes+ * as text (a PDF, a latin-1 file); decoding those would replace their bytes+ * with U+FFFD, so they are dropped here (their bytes are in the byte-exact+ * archive). A truncated last section may end inside a character.+ */+export function utf8DiffText(raw: Buffer): string {+ if (isUtf8(raw)) return raw.toString("utf8");+ const marker = Buffer.from("\ndiff --git ");+ const sections: Buffer[] = [];+ let start = 0;+ for (;;) {+ const next = raw.indexOf(marker, start);+ const end = next === -1 ? raw.length : next + 1;+ sections.push(raw.subarray(start, end));+ if (next === -1) break;+ start = end;+ }+ return sections.filter((section, index) => isUtf8(section) || (index === sections.length - 1 && binaryReason(null, section, true) === null))+ .map((section) => section.toString("utf8")).join("");+}+/*** Combined staged + unstaged text diff of the workspace. `--relative` with the* `.` pathspec limits the diff to the workspace subtree and makes every header@@ -2581,7 +2607,7 @@ async function gitDiff(root: string, hasHead: boolean): Promise<{ diff: string |const options = { maxBytes: GIT_DIFF_READ_BYTES, timeoutMs: 20_000 };if (hasHead) {const result = await runGit(root, ["diff", "HEAD", ...common, ...scope], options);- if (result.ok) return filterUploadDiff(result.stdout.toString("utf8"), result.truncated);+ if (result.ok) return filterUploadDiff(utf8DiffText(result.stdout), result.truncated);}// An unborn branch has no HEAD to diff against: combine the index (against// the empty tree) with the working tree changes on top of it.@@ -2590,7 +2616,7 @@ async function gitDiff(root: string, hasHead: boolean): Promise<{ diff: string |runGit(root, ["diff", ...common, ...scope], options),]);if (!staged.ok && !unstaged.ok) return { diff: null, truncated: false };- const text = `${staged.ok ? staged.stdout.toString("utf8") : ""}${unstaged.ok ? unstaged.stdout.toString("utf8") : ""}`;+ const text = `${staged.ok ? utf8DiffText(staged.stdout) : ""}${unstaged.ok ? utf8DiffText(unstaged.stdout) : ""}`;return filterUploadDiff(text, staged.truncated || unstaged.truncated);}@@ -2602,13 +2628,14 @@ async function gitDiff(root: string, hasHead: boolean): Promise<{ diff: string |* (porcelain status and the combined text diff), all redacted.*/export async function collectGitBlock(root: string): Promise<UploadGitBlock | null> {- const inside = await gitText(root, ["rev-parse", "--is-inside-work-tree"]);+ // 20 s, not 5: under load (a full scan hashing the tree) rev-parse timed out and the block was dropped.+ const inside = await gitText(root, ["rev-parse", "--is-inside-work-tree"], 64 * 1024, 20_000);if (inside !== "true") return null;// Where the workspace sits inside the repository ("" at the top level,// "packages/app/" for a nested workspace). Status, log and diff below are// scoped to that subtree; without a reliable answer the block is skipped// rather than risk describing files outside the workspace root.- const prefixResult = await runGit(root, ["rev-parse", "--show-prefix"], { maxBytes: 64 * 1024, timeoutMs: 5_000 });+ const prefixResult = await runGit(root, ["rev-parse", "--show-prefix"], { maxBytes: 64 * 1024, timeoutMs: 20_000 });if (!prefixResult.ok || prefixResult.truncated) return null;const prefix = prefixResult.stdout.toString("utf8").trim().replaceAll("\\", "/");const scope = prefix ? ["--", "."] : [];@@ -3094,6 +3121,8 @@ type ScanContext = {yielder: LoopYielder;pool: ReadPool;ancestors: AncestorCache;+ /** Capture v2: the binary files and symlinks the scan met, by path. */+ extras: Map<string, ExtraManifestEntry>;};/**@@ -3125,6 +3154,7 @@ function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: Reyielder: new LoopYielder(),pool: new ReadPool(SCAN_CONCURRENCY),ancestors: new Map(),+ extras: new Map(),};}@@ -3147,10 +3177,19 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCachreturn null;}metrics.fileStats += 1;- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) return null;+ if (file.isSymbolicLink()) {+ // Capture v2: a symlink is listed with its target, also a broken one; never followed.+ const link = await symlinkExtra(absolute, uploadPathFor(path, undefined), file.mode);+ if (link) context.extras.set(path, link);+ return null;+ }+ if (!file.isFile() || file.size > MAX_UPLOAD_FILE_BYTES) return null;const cached = cache.get(path);const signature = fileSignature(file);- if (cached && sameFileSignature(cached, signature)) return cached;+ if (cached && sameFileSignature(cached, signature)) {+ if (cached.binary) context.extras.set(path, binaryExtra(cached));+ return cached;+ }const scanKey = [resolve(context.root),path,@@ -3168,17 +3207,20 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCachreturn entry;}const uploadPath = uploadPathFor(path, cached);- const inFlight = pool.read(absolute, file.size, (buffer) => {+ const mode = file.mode & 0o7777;+ const inFlight = pool.read(absolute, file.size, (buffer): HashCacheEntry => {metrics.fileReads += 1;- if (isBinary(buffer)) {- return { ...signature, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };+ // Capture v2: a known binary type or signature is binary too (a PDF that is all ASCII); its raw digest is kept.+ if (binaryReason(path, buffer) !== null) {+ return { ...signature, binary: true, sha256: sha256Hex(buffer), bytes: buffer.length, json: 0, clean: false, uploadPath, mode };}- return redactedCacheEntry(path, buffer, signature, uploadPath, metrics, { cache: context.texts, absolute }).entry;+ return { ...redactedCacheEntry(path, buffer, signature, uploadPath, metrics, { cache: context.texts, absolute }).entry, mode };}).catch(() => null);FILE_SCAN_IN_FLIGHT.set(scanKey, inFlight);try {const entry = await inFlight;if (entry) cache.set(path, entry);+ if (entry?.binary) context.extras.set(path, binaryExtra(entry));await context.yielder.pause();return entry;} finally {@@ -3186,6 +3228,37 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCach}}+/** Capture v2: the manifest entry of a binary file the scan read (its raw digest, size and mode). */+function binaryExtra(entry: HashCacheEntry): ExtraManifestEntry {+ return { uploadPath: entry.uploadPath, kind: "binary", sha256: entry.sha256, size: entry.bytes, mode: entry.mode ?? 0o644 };+}++/** Capture v2: the manifest entry of a symlink (its target as written, also when it is broken), or null when unreadable. */+async function symlinkExtra(absolute: string, uploadPath: string, mode: number): Promise<ExtraManifestEntry | null> {+ let target: string;+ try {+ target = await readlink(absolute);+ } catch {+ return null;+ }+ let broken = false;+ try {+ await stat(absolute);+ } catch {+ broken = true;+ }+ const clamped = clampUploadText(redactUploadText(target).text, 4_096);+ return { uploadPath, kind: "symlink", sha256: sha256Hex(Buffer.from(clamped, "utf8")), size: Buffer.byteLength(clamped), mode: mode & 0o7777, target: clamped, broken };+}++/** One manifest line of a binary file or symlink. */+function extraEntryJson(entry: ExtraManifestEntry): string {+ const head = `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.size},"mode":${entry.mode}`;+ if (entry.kind === "symlink") return `${head},"type":"symlink","target":${JSON.stringify(entry.target ?? "")},"broken":${entry.broken === true}}`;+ return `${head},"binary":true,"archived":"byte_exact"}`;+}++/** One file as files[] carries it, re-read at upload time. */type UploadContent = { content: string; entry: HashCacheEntry };@@ -3232,7 +3305,7 @@ async function readUploadContent(}}}- if (isBinary(buffer)) return null;+ if (binaryReason(path, buffer) !== null) return null;const { entry, text } = redactedCacheEntry(path, buffer, signature, uploadPathFor(path, cached), metrics, { cache: texts, absolute });// The same digest keeps the cached entry, which the manifest may share.if (cached && cached.sha256 === entry.sha256 && sameFileSignature(cached, signature)) {@@ -3267,6 +3340,9 @@ type ScanResult = {binaryPaths?: Map<string, number>;tooLargePaths?: Map<string, number>;cappedPaths?: string[];+ /** Capture v2: binary files and symlinks found; `extrasScope` says whether that is all of them (a full scan) or the inspected paths only. */+ extras: Map<string, ExtraManifestEntry>;+ extrasScope: "full" | ReadonlySet<string>;};/** A changed file competing for the snapshot's content budget. */@@ -3349,15 +3425,16 @@ async function startsBinary(absolute: string): Promise<boolean> {const head = Buffer.alloc(8_192);try {const read = await readInto(absolute, head);- return isBinary(head.subarray(0, read), read === head.length);+ return binaryReason(absolute, head.subarray(0, read), read === head.length) !== null;} catch {return false;}}/** The manifest entry as the envelope carries it: path (redacted), sha256, size. */-function manifestEntryJson(entry: HashCacheEntry): string {- return `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.bytes}}`;+function manifestEntryJson(entry: HashCacheEntry | ExtraManifestEntry): string {+ if ("kind" in entry) return extraEntryJson(entry);+ return `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.bytes}${entry.mode !== undefined ? `,"mode":${entry.mode}` : ""}}`;}function candidateCost(entry: HashCacheEntry): number {@@ -3465,6 +3542,8 @@ async function scanWorkspaceFull(binaryPaths,tooLargePaths,cappedPaths,+ extras: context.extras,+ extrasScope: "full",};}@@ -3533,6 +3612,8 @@ async function scanDirtyPaths(removed: countRemoved(previous, manifest),deniedCount: listing.denied,manifestTruncated: listing.truncated || manifest.size >= MAX_FILES,+ extras: context.extras,+ extrasScope: new Set(targets),};}@@ -3736,7 +3817,7 @@ type EnvelopeBody = {extras: Record<string, unknown>;/** Streams files[] one entry at a time through `emit`. */writeFiles: (emit: (file: UploadFile) => Promise<void>) => Promise<void>;- manifest: () => Iterable<HashCacheEntry>;+ manifest: () => Iterable<HashCacheEntry | ExtraManifestEntry>;/** Computed once files[] is written, so cap omissions are known. */privacy: () => Record<string, unknown>;/** Fields written after privacy, once files[] is out (excluded_files). */@@ -3897,6 +3978,7 @@ export class SessionUploader {private readonly bases: TurnBaseStore;private readonly onSessionClosed?: (sessionId: string) => void;private readonly onPathTouched?: (sessionId: string, path: string) => void;+ private readonly onBinaryFile?: (sessionId: string, path: string) => void;private readonly fileUploader?: SessionUploaderOptions["uploadFile"];private readonly capabilityProbe?: SessionUploaderOptions["capabilities"];private traceSchemaCache: { version: 2 | 3; expiresAt: number } | null = null;@@ -3945,6 +4027,7 @@ export class SessionUploader {});this.onSessionClosed = options.onSessionClosed;this.onPathTouched = options.onPathTouched;+ this.onBinaryFile = options.onBinaryFile;this.tempDir = stateDir ? join(stateDir, UPLOAD_TEMP_DIRECTORY) : join(tmpdir(), `omnirush-upload-${process.pid}`);this.bases = new TurnBaseStore(stateDir ? join(stateDir, UPLOAD_BASE_DIRECTORY) : join(this.tempDir, UPLOAD_BASE_DIRECTORY),@@ -4180,6 +4263,7 @@ export class SessionUploader {watchSetup: Promise.resolve(),trace: [],changeJournal: new Map(),+ extras: new Map(),touchedPaths: new Set(),changeJournalBytes: 0,changeCaptureTail: Promise.resolve(),@@ -5261,7 +5345,7 @@ export class SessionUploader {} else {const buffer = await readFile(absolute);this.metrics.fileReads += 1;- if (buffer.length > MAX_UPLOAD_FILE_BYTES || isBinary(buffer)) {+ if (buffer.length > MAX_UPLOAD_FILE_BYTES || binaryReason(path, buffer) !== null) {if (written) {state.turnWrittenPaths.set(path, Date.now());state.turnSkipped.set(path, file.mtimeMs);@@ -5369,6 +5453,15 @@ export class SessionUploader {} finally {releaseScan();}+ // Capture v2: the binary files and symlinks the manifest lists next to the text files.+ const extraEntries = scan.extrasScope === "full" ? new Map(scan.extras) : new Map(state.extras);+ if (scan.extrasScope !== "full") {+ for (const path of scan.extrasScope) extraEntries.delete(path);+ for (const path of ignored) extraEntries.delete(path);+ for (const [path, entry] of scan.extras) extraEntries.set(path, entry);+ }+ // A binary file goes to the byte-exact project archive (never as text).+ for (const [path, entry] of extraEntries) if (entry.kind === "binary") this.onBinaryFile?.(state.id, path);// What moved on disk since the last snapshot, where no watcher may have// seen it (a polled workspace). Only against an accepted snapshot: with// none yet (the start one was refused), every file counts as changed.@@ -5432,6 +5525,7 @@ export class SessionUploader {let fixedBytes = serializedBytes(extras) + SNAPSHOT_WRAPPER_MARGIN_BYTES;for (const file of leading) fixedBytes += serializedBytes(file) + 1;for (const entry of scan.manifest.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;+ for (const entry of extraEntries.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;const budget = Math.max(0, this.snapshotMaxBytes - fixedBytes);const selection = selectSnapshotContent(candidates, budget);const capOmitted: SnapshotCandidate[] = [];@@ -5477,7 +5571,10 @@ export class SessionUploader {await this.bases.writable();}},- manifest: () => manifest.values(),+ manifest: function* () {+ yield* manifest.values();+ yield* extraEntries.values();+ },privacy: () => ({...PRIVACY_POLICY,denied_file_count: scan.deniedCount,@@ -5518,6 +5615,7 @@ export class SessionUploader {// next change snapshot reports exactly what moved since this one.accepted = true;state.manifest = manifest;+ state.extras = extraEntries;state.turnManifest ??= turnBaseline(manifest, state.turnStartedAt, [previous]);state.listing = { denied: scan.deniedCount, truncated: scan.manifestTruncated };state.lastHead = git?.commit ?? null;