Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

capture v2: byte-exact project state

ogsamrat9e9028638447
diff --git a/apps/server/src/capture-client.ts b/apps/server/src/capture-client.ts
index 8839307f..fcaf83d0 100644
--- a/apps/server/src/capture-client.ts
+++ b/apps/server/src/capture-client.ts
@@ -13,6 +13,7 @@
import { Worker } from "node:worker_threads";
import { lstat } from "node:fs/promises";
import { resolve } from "node:path";
+import { START_GATE_MS } from "./session-archive/capture-v2.js";
import { CaptureHost, type CaptureDiagnostics, type CaptureHostOptions, type EngineReplacement, type EngineTarget, type PromptRecord } from "./capture-host.js";
import {
@@ -64,6 +65,12 @@ export type CaptureService = {
/** Whether the visit is recorded (a tracked session and an uploadable URL). */
recordWebVisit(sessionId: string, visit: UploadWebVisit): boolean;
archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void;
+ /**
+ * Capture v2: resolves once the project archive took the session-start
+ * manifest (hash and stat only, START_GATE_MS at most); the prompt goes to
+ * the engine after it. Never rejects.
+ */
+ archiveStartGate(sessionId: string, root: string): Promise<void>;
observeSession(sessionId: string, target: EngineTarget): void;
/** An engine was closed and another took over its sessions: turn observers reading it move there. */
engineReplaced(closedBaseUrl: string, replacement: EngineReplacement): void;
@@ -80,6 +87,9 @@ export type CaptureService = {
diagnostics(): Promise<CaptureDiagnostics | null>;
};
+/** How much longer than the start gate's own cap the main thread waits for the worker's answer. */
+const START_GATE_SLACK_MS = 1_000;
+
export function captureWorkerEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
return !["0", "false", "no", "off"].includes((env.OMNIRUSH_CAPTURE_WORKER ?? "").trim().toLowerCase());
}
@@ -169,6 +179,19 @@ class CaptureClient implements CaptureService {
this.send({ kind: "call", id: null, method: "archiveSessionStarted", args: [sessionId, root, target] });
}
+ async archiveStartGate(sessionId: string, root: string): Promise<void> {
+ // The worker answers within the gate's cap; a busy or stuck worker never holds a prompt longer.
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ await Promise.race([
+ this.query({ kind: "call", id: null, method: "archiveStartGate", args: [sessionId, root] }).catch(() => undefined),
+ new Promise<void>((resolvePromise) => {
+ timer = setTimeout(resolvePromise, START_GATE_MS + START_GATE_SLACK_MS);
+ timer.unref?.();
+ }),
+ ]);
+ clearTimeout(timer);
+ }
+
observeSession(sessionId: string, target: EngineTarget): void {
this.send({ kind: "call", id: null, method: "observeSession", args: [sessionId, target] });
}
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index 2eefbff7..dd85aa43 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -14,6 +14,7 @@ import {
currentEngineTarget,
engineReplaced,
observeUploadedSession,
+ followToolStart,
projectArchiveEngineReads,
promptDispatched,
type EngineReplacement,
@@ -110,6 +111,8 @@ export class CaptureHost {
...(options.onSessionClosed ? { onSessionClosed: options.onSessionClosed } : {}),
// The touched-files archive hears of every path a session touches, here on the same thread.
onPathTouched: (sessionId, path) => this.archive.pathTouched(sessionId, path),
+ // Capture v2: a binary file is never sent as text; the project archive keeps it byte for byte.
+ onBinaryFile: (sessionId, path) => this.archive.binaryFile(sessionId, path),
});
const { enabled, excludedDirs, folderGate, baseIdleMs, baseMaxDeferMs, ...auth } = options.archive;
this.appDirs = [options.stateDir, ...excludedDirs];
@@ -170,6 +173,13 @@ export class CaptureHost {
/** A prompt was dispatched: the project archive's session start (a base once per session, see lifecycle.ts). */
archiveSessionStarted(sessionId: string, root: string, target: EngineTarget): void {
this.archive.sessionStarted({ sessionId, root, engine: projectArchiveEngineReads(() => currentEngineTarget(this.observers, target), sessionId) });
+ // Capture v2: this turn's first tool call takes the folder's state (in the background).
+ followToolStart({ observers: this.observers, archive: this.archive, sessionId, target });
+ }
+
+ /** Capture v2: the session-start manifest, before the first prompt reaches the engine (lifecycle.ts startGate). */
+ async archiveStartGate(sessionId: string, root: string): Promise<void> {
+ await this.archive.startGate(sessionId, root);
}
/** An engine was closed and `replacement` took over its sessions: observations reading it move there. */
diff --git a/apps/server/src/capture-protocol.ts b/apps/server/src/capture-protocol.ts
index 5acb654c..2472c0fc 100644
--- a/apps/server/src/capture-protocol.ts
+++ b/apps/server/src/capture-protocol.ts
@@ -24,6 +24,7 @@ export type CaptureCalls = {
captureSnapshot: [sessionId: string, trigger: "prompt" | "turn_completed"];
recordWebVisit: [sessionId: string, visit: UploadWebVisit];
archiveSessionStarted: [sessionId: string, root: string, target: EngineTarget];
+ archiveStartGate: [sessionId: string, root: string];
observeSession: [sessionId: string, target: EngineTarget];
engineReplaced: [closedBaseUrl: string, replacement: EngineReplacement];
sessionDeleted: [sessionId: string];
@@ -163,6 +164,8 @@ export function invokeCapture(host: CaptureHost, call: CaptureCall): unknown {
return host.recordWebVisit(...call.args);
case "archiveSessionStarted":
return host.archiveSessionStarted(...call.args);
+ case "archiveStartGate":
+ return host.archiveStartGate(...call.args);
case "observeSession":
return host.observeSession(...call.args);
case "engineReplaced":
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index b4bc8b06..1265fd8e 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -1692,6 +1692,8 @@ export async function proxyOpencodeV2Request(input: {
: null;
const engine = capture && uploadedSessionId ? engineTarget(input.connection.url, headers, target.search, "v2") : null;
if (capture && uploadedSessionId && engine) {
+ // Capture v2: the session-start manifest is taken before the prompt reaches the engine (at most about 3 s).
+ await capture.archiveStartGate(uploadedSessionId, input.workspace.path);
capture.startSession(uploadedSessionId, input.workspace.id, input.workspace.path);
capture.recordPrompt(uploadedSessionId, { method, path: routePath, body: requestBody, engine: "v2", root: input.workspace.path, attachments: true, dispatchedAt: Date.now() });
capture.captureSnapshot(uploadedSessionId, "prompt");
@@ -2041,6 +2043,8 @@ export async function proxyOpencodeRequest(input: {
const deletedUploadedSessionId = uploadDeletedSessionId(method, proxyPath);
if (capture?.uploadEnabled && uploadedSessionId && workspace && workspace.workspaceType !== "remote") {
const promptDispatch = isUploadPromptDispatch(method, proxyPath);
+ // Capture v2: the session-start manifest is taken before the prompt reaches the engine (at most about 3 s).
+ if (promptDispatch) await capture.archiveStartGate(uploadedSessionId, workspace.path);
capture.startSession(uploadedSessionId, workspace.id, workspace.path);
capture.recordPrompt(uploadedSessionId, {
method,
diff --git a/apps/server/src/session-archive/README.md b/apps/server/src/session-archive/README.md
index f197748c..75e5c4d2 100644
--- a/apps/server/src/session-archive/README.md
+++ b/apps/server/src/session-archive/README.md
@@ -303,6 +303,57 @@ not used: Electron's `net.fetch` collects a streamed body with one
64 MiB part took 2.6 s against 0.1 s for one buffer (measured with Electron 43
against a loopback sink).
+## Capture v2 (byte-exact project state)
+
+Offered by the server per user (`policy.capture_v2` from GET /archives/key;
+`OMNIRUSH_CAPTURE_V2=0/1` overrides it for tests). A chain is v2 from its
+base on (`v2` in its session record). The same modules run in the CLI; both
+check `__fixtures__/capture_v2_vectors.json`. The wire format is in the
+backend's `docs/omnirush-project-archive.md` ("Capture v2").
+
+- **States.** Every v2 archive carries `__omnirush__/state.json` right after
+ the manifest (`schema: omnirush.archive.v2`): `start` (the base), `pre_tool`
+ (a turn's first tool call, `trigger: "pre_tool"`, the wire turn repeating
+ the parent's like a final archive, only when something changed since the
+ last state), `after` (a turn's delta) and `final` (with its reason).
+- **Start gate.** `server.ts` awaits `CaptureService.archiveStartGate` before
+ a prompt dispatch reaches the engine: the lifecycle's `startGate` asks
+ `SessionArchiver.startManifest` for a stat-and-hash scan of the folder (the
+ root's hash cache: an unchanged tree reads no file), at most
+ `START_GATE_MS` (3 s; the main thread gives the worker 1 s more). The base
+ then packs at once (no quiet period) from exactly that scan, and pass 2
+ checks every file against the hash taken before the prompt, so a file the
+ agent changed meanwhile is listed in `unstable.json`.
+ `start_capture: "complete" | "partial"` says whether the scan beat the cap.
+- **Tool start.** `followToolStart` (session-upload-observer.ts) listens to the
+ engine's event stream from the prompt on and calls the lifecycle's
+ `toolStarted` at the session's first tool part (v1 `message.part.updated`,
+ v2 `session.tool.called` / `session.tool.input.started`), once per turn; the
+ capture runs in the background and never holds the tool.
+- **What is in it.** The whole folder for git roots and, with the all-folders
+ policy, plain folders (regenerable folders such as `node_modules`, `.venv`,
+ `target` left out with reason `regenerable`); every repository's `.git`
+ byte for byte, its config files (`git-scrub.ts`) without credentials in the
+ ARCHIVED copy only (URL userinfo, `extraheader`, `[credential]`, secret
+ keys; the entry says `scrubbed: true`); symlinks with their target and
+ `broken: true` when it is missing; modes; every lockfile (`findLockfiles`);
+ touched gitignored files, also in regenerable folders, up to 1 GiB (over it:
+ path, size and hash in `excluded`); binaries the session uploader found
+ (`onBinaryFile`, never sent as text); prompt attachments
+ (`attachments.ts`, from the turn's user messages) under
+ `__attachments__/<message id>/<n>-<name>`.
+- **state.json** also lists every repository (`repos.ts`: root, nested,
+ `above` with its relative path, unborn branches with their staged files;
+ HEAD, branch, remote without userinfo, tracked-file count, dirty and staged
+ files, stash count, or the reason git could not read it), every entry the
+ scan left out with its reason (`excluded`, at most 20,000), the scrubbed
+ configs and the attachments with their message ids.
+- **Git runner.** Capture git runs drop inherited `GIT_DIR`/`GIT_WORK_TREE`
+ and friends, and accept the session's own repository when another user
+ owns it (`safe.directory` for that path only, fsmonitor and hooks off):
+ both used to leave the git block empty while the agent's own git commands
+ worked.
+
## Behaviour notes
- **Consent.** A 428, or a `503 archive_disabled`, from any route turns archiving off without error. Every queued archive is dropped (consent belongs to the user), sessions that lost a queued archive stop, and captures skip as `disabled` until a later `captureBase` finds the key route open again. A server without the routes (404 on `/archives/key`) counts as disabled.
diff --git a/apps/server/src/session-archive/__fixtures__/capture_v2_vectors.json b/apps/server/src/session-archive/__fixtures__/capture_v2_vectors.json
new file mode 100644
index 00000000..80e77dd7
--- /dev/null
+++ b/apps/server/src/session-archive/__fixtures__/capture_v2_vectors.json
@@ -0,0 +1,405 @@
+{
+ "schema": "omnirush.capture-v2-vectors.v1",
+ "note": "Shared by omnirush-cli and omnirush-gui: both implementations must reproduce every expect.",
+ "binary": [
+ {
+ "path": "doc.pdf",
+ "base64": "JVBERi0xLjQKJWFsbCBhc2NpaSBwZGYK",
+ "expect": "type"
+ },
+ {
+ "path": "notes.txt",
+ "base64": "JVBERi0xLjcgc3RhcnRzIGxpa2UgYSBwZGYK",
+ "expect": "signature"
+ },
+ {
+ "path": "notes.txt",
+ "base64": "cGxhaW4gdGV4dCDDqSDDvCDkuK3mlocK",
+ "expect": null
+ },
+ {
+ "path": "data.csv",
+ "base64": "YSzpCg==",
+ "expect": "utf8"
+ },
+ {
+ "path": "x.bin",
+ "base64": "bG9va3MgbGlrZSB0ZXh0",
+ "expect": "type"
+ },
+ {
+ "path": null,
+ "base64": "iVBORw0KGgo=",
+ "expect": "signature"
+ },
+ {
+ "path": "a.txt",
+ "base64": "aABp",
+ "expect": "control"
+ },
+ {
+ "path": "bun.lockb",
+ "base64": "IyEvdXNyL2Jpbi9lbnYgYnVuCmJ1bi1sb2NrZmlsZS1mb3JtYXQtdjAK",
+ "expect": "type"
+ },
+ {
+ "path": "head.txt",
+ "base64": "YeS4",
+ "partial": true,
+ "expect": null
+ },
+ {
+ "path": "head.txt",
+ "base64": "YeS4",
+ "partial": false,
+ "expect": "utf8"
+ },
+ {
+ "path": "Makefile",
+ "base64": "YWxsOgoJY2MgLW8geCB4LmMK",
+ "expect": null
+ },
+ {
+ "path": "photo.JPG",
+ "base64": "",
+ "expect": "type"
+ }
+ ],
+ "scrub": [
+ {
+ "text": "[core]\n\tbare = false\n[remote \"origin\"]\n\turl = https://bob:[email protected]/o/r.git\n\tpushurl = [email protected]:o/r.git\n[http \"https://github.com/\"]\n\textraheader = AUTHORIZATION: basic Zm9vOmJhcg==\n[credential]\n\thelper = \"!f() { echo password=hunter2; }; f\"\n\tusername = bob\n[url \"https://tok:[email protected]/\"]\n\tinsteadOf = https://github.com/\n[user]\n\tname = Bob\n",
+ "expect": {
+ "text": "[core]\n\tbare = false\n[remote \"origin\"]\n\turl = https://github.com/o/r.git\n\tpushurl = [email protected]:o/r.git\n[http \"https://github.com/\"]\n\t# [omnirush: removed http.extraheader]\n[credential]\n\t# [omnirush: removed credential.helper]\n\t# [omnirush: removed credential.username]\n[url \"https://github.com/\"]\n\tinsteadOf = https://github.com/\n[user]\n\tname = Bob\n",
+ "changed": true,
+ "removed": 5
+ }
+ },
+ {
+ "text": "[http]\n\textraHeader = Authorization: Bearer abc\n[remote \"up\"]\n\turl = ssh://git:pw@host:22/x.git\n\tfetch = +refs/heads/*:refs/remotes/up/*\n[custom]\n\tapiToken = zzz\n\tmulti = a \\\n b\n",
+ "expect": {
+ "text": "[http]\n\t# [omnirush: removed http.extraheader]\n[remote \"up\"]\n\turl = ssh://git@host:22/x.git\n\tfetch = +refs/heads/*:refs/remotes/up/*\n[custom]\n\t# [omnirush: removed custom.apitoken]\n\tmulti = a \\\n b\n",
+ "changed": true,
+ "removed": 3
+ }
+ },
+ {
+ "text": "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n",
+ "expect": {
+ "text": "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n",
+ "changed": false,
+ "removed": 0
+ }
+ }
+ ],
+ "configPaths": [
+ {
+ "path": ".git/config",
+ "expect": true
+ },
+ {
+ "path": ".git/config.worktree",
+ "expect": true
+ },
+ {
+ "path": "sub/.git/config",
+ "expect": true
+ },
+ {
+ "path": ".git/modules/lib/config",
+ "expect": true
+ },
+ {
+ "path": ".git/modules/a/b/config",
+ "expect": true
+ },
+ {
+ "path": ".git/worktrees/w/config.worktree",
+ "expect": true
+ },
+ {
+ "path": ".git/worktrees/w/config",
+ "expect": false
+ },
+ {
+ "path": ".git/objects/config",
+ "expect": false
+ },
+ {
+ "path": "config",
+ "expect": false
+ },
+ {
+ "path": "src/.git-config",
+ "expect": false
+ }
+ ],
+ "toolStart": [
+ {
+ "payload": {
+ "type": "message.part.updated",
+ "properties": {
+ "part": {
+ "type": "tool",
+ "sessionID": "ses_a",
+ "state": {
+ "status": "running"
+ }
+ }
+ }
+ },
+ "expect": "ses_a"
+ },
+ {
+ "payload": {
+ "type": "message.part.updated",
+ "properties": {
+ "part": {
+ "type": "tool",
+ "sessionID": "ses_a",
+ "state": {
+ "status": "completed"
+ }
+ }
+ }
+ },
+ "expect": null
+ },
+ {
+ "payload": {
+ "type": "message.part.updated",
+ "properties": {
+ "part": {
+ "type": "text",
+ "sessionID": "ses_a"
+ }
+ }
+ },
+ "expect": null
+ },
+ {
+ "payload": {
+ "directory": "/x",
+ "payload": {
+ "type": "message.part.updated",
+ "properties": {
+ "part": {
+ "type": "tool",
+ "sessionID": "ses_b",
+ "state": {
+ "status": "pending"
+ }
+ }
+ }
+ }
+ },
+ "expect": "ses_b"
+ },
+ {
+ "payload": {
+ "type": "session.tool.called",
+ "data": {
+ "sessionID": "ses_c",
+ "id": "call_1",
+ "name": "bash"
+ }
+ },
+ "expect": "ses_c"
+ },
+ {
+ "payload": {
+ "type": "session.next.tool.input.started",
+ "data": {
+ "sessionID": "ses_d",
+ "id": "call_2"
+ }
+ },
+ "expect": "ses_d"
+ },
+ {
+ "payload": {
+ "type": "session.tool.success",
+ "data": {
+ "sessionID": "ses_e"
+ }
+ },
+ "expect": null
+ }
+ ],
+ "sse": [
+ {
+ "text": "data: {\"a\":1}\n\n: comment\n\ndata: x\ndata: y\n\ndata: partial",
+ "expect": {
+ "payloads": [
+ "{\"a\":1}",
+ "x\ny"
+ ],
+ "rest": "data: partial"
+ }
+ }
+ ],
+ "attachmentPaths": [
+ {
+ "messageId": "msg_1",
+ "index": 0,
+ "name": "photo.png",
+ "expect": "__attachments__/msg_1/0-photo.png"
+ },
+ {
+ "messageId": "msg/../x",
+ "index": 1,
+ "name": "../../etc/passwd",
+ "expect": "__attachments__/msg_.._x/1-__.._etc_passwd"
+ },
+ {
+ "messageId": "msg_2",
+ "index": 2,
+ "name": "con:tro\u0001l?.txt",
+ "expect": "__attachments__/msg_2/2-con_tro_l_.txt"
+ },
+ {
+ "messageId": "",
+ "index": 0,
+ "name": "",
+ "expect": "__attachments__/message/0-attachment"
+ },
+ {
+ "messageId": "msg_3",
+ "index": 3,
+ "name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.pdf",
+ "expect": "__attachments__/msg_3/3-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+ }
+ ],
+ "attachments": {
+ "messages": [
+ {
+ "info": {
+ "id": "msg_u1",
+ "role": "user"
+ },
+ "parts": [
+ {
+ "type": "text",
+ "text": "see"
+ },
+ {
+ "type": "file",
+ "mime": "image/png",
+ "filename": "dot.png",
+ "url": "data:image/png;base64,iVBORw0KGgo="
+ },
+ {
+ "type": "file",
+ "mime": "text/plain",
+ "url": "file:///tmp/notes.txt"
+ }
+ ]
+ },
+ {
+ "info": {
+ "id": "msg_a1",
+ "role": "assistant"
+ },
+ "parts": [
+ {
+ "type": "file",
+ "url": "data:text/plain;base64,eA=="
+ }
+ ]
+ },
+ {
+ "id": "msg_u2",
+ "role": "user",
+ "parts": [],
+ "files": [
+ {
+ "uri": "data:text/plain,hello%20world",
+ "name": "h.txt",
+ "mediaType": "text/plain"
+ }
+ ]
+ }
+ ],
+ "expect": [
+ {
+ "messageId": "msg_u1",
+ "index": 0,
+ "name": "dot.png",
+ "mime": "image/png",
+ "data": "iVBORw0KGgo=",
+ "file": null
+ },
+ {
+ "messageId": "msg_u1",
+ "index": 1,
+ "name": "notes.txt",
+ "mime": "text/plain",
+ "data": null,
+ "file": "/tmp/notes.txt"
+ },
+ {
+ "messageId": "msg_u2",
+ "index": 0,
+ "name": "h.txt",
+ "mime": "text/plain",
+ "data": "aGVsbG8gd29ybGQ=",
+ "file": null
+ }
+ ]
+ },
+ "porcelain": [
+ {
+ "text": " M a.txt\u0000A b.txt\u0000?? c.txt\u0000R new.txt\u0000old.txt\u0000MM d.txt\u0000",
+ "expect": {
+ "dirty": [
+ {
+ "path": "a.txt",
+ "status": " M"
+ },
+ {
+ "path": "b.txt",
+ "status": "A "
+ },
+ {
+ "path": "c.txt",
+ "status": "??"
+ },
+ {
+ "path": "new.txt",
+ "status": "R "
+ },
+ {
+ "path": "d.txt",
+ "status": "MM"
+ }
+ ],
+ "staged": [
+ "b.txt",
+ "new.txt",
+ "d.txt"
+ ]
+ }
+ }
+ ],
+ "remotes": [
+ {
+ "url": "https://user:[email protected]/o/r.git",
+ "expect": "https://github.com/o/r.git"
+ },
+ {
+ "url": "https://github.com/o/r.git",
+ "expect": "https://github.com/o/r.git"
+ },
+ {
+ "url": "[email protected]:o/r.git",
+ "expect": "[email protected]:o/r.git"
+ },
+ {
+ "url": "git:pw@host:o/r.git",
+ "expect": "git@host:o/r.git"
+ },
+ {
+ "url": "ssh://git@host/x",
+ "expect": "ssh://host/x"
+ }
+ ]
+}
diff --git a/apps/server/src/session-archive/attachments.ts b/apps/server/src/session-archive/attachments.ts
new file mode 100644
index 00000000..5780ed4e
--- /dev/null
+++ b/apps/server/src/session-archive/attachments.ts
@@ -0,0 +1,266 @@
+/**
+ * Capture v2: the bytes of every file attached to a prompt (an inline data
+ * URL in the prompt parts, or a file URL: the path the app copied the file
+ * to), archived byte for byte under `__attachments__/<message id>/<n>-<name>`
+ * with a state.json entry linking it to its message id and source path.
+ *
+ * Attachments are read from the engine's own user messages when a turn
+ * settles (both engines keep the file parts with their URL), so every one
+ * carries the id of the message it belongs to. Their bytes are staged under
+ * the archiver's state dir at once (a file the app copied may change or go
+ * later), and every capture of the chain holds the staged ones, so a delta
+ * sends each once. Identical in the CLI and the desktop app.
+ */
+import { createHash } from "node:crypto";
+import { appendFile, lstat, mkdir, open, readFile, rm } from "node:fs/promises";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { stateKey } from "./files.js";
+
+export const ATTACHMENTS_ROOT_NAME = "__attachments__";
+/** One attachment's bytes, at most (the desktop's prompt attachment cap). */
+export const MAX_ATTACHMENT_BYTES = 64 * 1024 * 1024;
+const MAX_ATTACHMENTS_PER_MESSAGE = 32;
+const MAX_NAME_BYTES = 120;
+
+export type AttachmentSource = {
+ messageId: string;
+ index: number;
+ name: string;
+ mime: string;
+ /** Inline bytes (a data URL), or null for a file URL. */
+ data: Buffer | null;
+ /** The file a file URL names (absolute), else null. */
+ file: string | null;
+};
+
+export type AttachmentRecord = {
+ /** Archive path: `__attachments__/<message id>/<index>-<name>`. */
+ path: string;
+ message_id: string;
+ name: string;
+ mime: string;
+ size: number;
+ sha256: string;
+ source: "inline" | "file";
+ /** The file URL's path: workspace-relative when inside the root, else absolute; null for inline data. */
+ source_path: string | null;
+ /** The staged copy (absolute, under the state dir). Never in the archive. */
+ blob: string;
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function safeSegment(text: string, fallback: string): string {
+ // eslint-disable-next-line no-control-regex
+ let clean = text.replace(/[\u0000-\u001f\u007f/\\:*?"<>|]/g, "_").replace(/^\.+/, "_").trim();
+ while (Buffer.byteLength(clean) > MAX_NAME_BYTES) clean = clean.slice(0, -1);
+ return clean || fallback;
+}
+
+/** The archive path of an attachment. */
+export function attachmentArchivePath(messageId: string, index: number, name: string): string {
+ return `${ATTACHMENTS_ROOT_NAME}/${safeSegment(messageId, "message")}/${index}-${safeSegment(name, "attachment")}`;
+}
+
+function dataUrlBytes(url: string): Buffer | null {
+ const comma = url.indexOf(",");
+ if (comma < 0) return null;
+ const header = url.slice(5, comma);
+ const payload = url.slice(comma + 1);
+ if (payload.length > MAX_ATTACHMENT_BYTES * 2) return null;
+ try {
+ return /;base64$/i.test(header) ? Buffer.from(payload, "base64") : Buffer.from(decodeURIComponent(payload), "utf8");
+ } catch {
+ return null;
+ }
+}
+
+function fileParts(message: Record<string, unknown>): Array<Record<string, unknown>> {
+ const parts = Array.isArray(message.parts) ? message.parts : [];
+ const out: Array<Record<string, unknown>> = [];
+ for (const part of parts) {
+ if (!isRecord(part)) continue;
+ if (part.type === "file") out.push(part);
+ // v2 prompts keep their files apart.
+ if (Array.isArray(part.files)) for (const file of part.files) if (isRecord(file)) out.push(file);
+ }
+ const info = isRecord(message.info) ? message.info : message;
+ for (const holder of [message, info]) {
+ if (Array.isArray(holder.files)) for (const file of holder.files) if (isRecord(file)) out.push(file);
+ }
+ return out;
+}
+
+/** The attachments of the user messages in `messages` (an engine message list, v1 or v2 shape). */
+export function attachmentsFromMessages(messages: unknown): AttachmentSource[] {
+ const list = isRecord(messages) && Array.isArray(messages.messages) ? messages.messages : Array.isArray(messages) ? messages : [];
+ const found: AttachmentSource[] = [];
+ for (const message of list) {
+ if (!isRecord(message)) continue;
+ const info = isRecord(message.info) ? message.info : message;
+ if (info.role !== "user" || typeof info.id !== "string" || !info.id) continue;
+ let index = 0;
+ const seen = new Set<string>();
+ for (const part of fileParts(message)) {
+ if (index >= MAX_ATTACHMENTS_PER_MESSAGE) break;
+ const url = typeof part.url === "string" ? part.url.trim() : typeof part.uri === "string" ? part.uri.trim() : "";
+ if (!url || seen.has(url)) continue;
+ seen.add(url);
+ const mimeValue = part.mime ?? part.mediaType ?? part.mimeType;
+ const mime = typeof mimeValue === "string" && mimeValue.trim() ? mimeValue.trim().toLowerCase() : "application/octet-stream";
+ let file: string | null = null;
+ let data: Buffer | null = null;
+ if (url.startsWith("data:")) data = dataUrlBytes(url);
+ else if (url.startsWith("file:")) {
+ try {
+ file = resolve(fileURLToPath(url));
+ } catch {
+ file = null;
+ }
+ }
+ if (!data && !file) continue;
+ const nameValue = part.filename ?? part.name;
+ const name = typeof nameValue === "string" && nameValue.trim() ? nameValue.trim() : file ? file.split(/[\\/]/).pop() ?? "attachment" : "attachment";
+ found.push({ messageId: info.id, index, name, mime, data, file });
+ index += 1;
+ }
+ }
+ return found;
+}
+
+function within(root: string, path: string): string | null {
+ const rel = relative(root, path);
+ if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;
+ return sep === "/" ? rel : rel.split(sep).join("/");
+}
+
+/** Reads a file URL's file: a regular file (never through a final symlink), at most MAX_ATTACHMENT_BYTES. */
+async function readAttachedFile(path: string): Promise<Buffer | null> {
+ try {
+ const info = await lstat(path);
+ if (!info.isFile() || info.size > MAX_ATTACHMENT_BYTES) return null;
+ return await readFile(path);
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * The staged attachments of each session: `<dir>/<session key>/index.jsonl`
+ * plus one blob per attachment. Deduplicated by archive path.
+ */
+export class AttachmentStore {
+ private readonly tails = new Map<string, Promise<void>>();
+
+ constructor(
+ private readonly dir: string,
+ private readonly isDenied: (relativePath: string) => boolean = () => false,
+ ) {}
+
+ private sessionDir(sessionId: string): string {
+ return join(this.dir, stateKey(sessionId));
+ }
+
+ private serial<T>(sessionId: string, task: () => Promise<T>): Promise<T> {
+ const previous = this.tails.get(sessionId) ?? Promise.resolve();
+ const run = previous.then(task);
+ this.tails.set(sessionId, run.then(() => undefined, () => undefined));
+ return run;
+ }
+
+ /** Stages the sources' bytes; returns the records added (a known path is skipped). Never throws. */
+ add(sessionId: string, root: string, sources: readonly AttachmentSource[]): Promise<AttachmentRecord[]> {
+ return this.serial(sessionId, async () => {
+ const added: AttachmentRecord[] = [];
+ try {
+ const known = new Set((await this.listUnlocked(sessionId)).map((record) => record.path));
+ const dir = this.sessionDir(sessionId);
+ for (const source of sources) {
+ const path = attachmentArchivePath(source.messageId, source.index, source.name);
+ if (known.has(path)) continue;
+ let sourcePath: string | null = null;
+ let bytes = source.data;
+ if (source.file) {
+ const rel = within(resolve(root), source.file);
+ // A credential file is never copied, wherever it was attached from.
+ if (rel !== null && this.isDenied(rel)) continue;
+ sourcePath = rel ?? source.file;
+ bytes = await readAttachedFile(source.file);
+ }
+ if (!bytes || bytes.length > MAX_ATTACHMENT_BYTES) continue;
+ await mkdir(dir, { recursive: true, mode: 0o700 });
+ const sha256 = createHash("sha256").update(bytes).digest("hex");
+ const blob = join(dir, `${stateKey(path)}.bin`);
+ const handle = await open(blob, "w", 0o600);
+ try {
+ await handle.writeFile(bytes);
+ await handle.sync();
+ } finally {
+ await handle.close();
+ }
+ const record: AttachmentRecord = {
+ path,
+ message_id: source.messageId,
+ name: source.name,
+ mime: source.mime,
+ size: bytes.length,
+ sha256,
+ source: source.file ? "file" : "inline",
+ source_path: sourcePath,
+ blob,
+ };
+ await appendFile(join(dir, "index.jsonl"), `${JSON.stringify(record)}\n`, { mode: 0o600 });
+ known.add(path);
+ added.push(record);
+ }
+ } catch {
+ // Best effort: an attachment that cannot be staged is described by the trace only.
+ }
+ return added;
+ });
+ }
+
+ private async listUnlocked(sessionId: string): Promise<AttachmentRecord[]> {
+ let text: string;
+ try {
+ text = await readFile(join(this.sessionDir(sessionId), "index.jsonl"), "utf8");
+ } catch {
+ return [];
+ }
+ const records: AttachmentRecord[] = [];
+ for (const line of text.split("\n")) {
+ if (!line.trim()) continue;
+ try {
+ const value: unknown = JSON.parse(line);
+ if (!isRecord(value)) continue;
+ const { path, message_id, name, mime, size, sha256, source, source_path, blob } = value;
+ if (typeof path !== "string" || typeof message_id !== "string" || typeof name !== "string" || typeof mime !== "string"
+ || typeof size !== "number" || typeof sha256 !== "string" || (source !== "inline" && source !== "file")
+ || (source_path !== null && typeof source_path !== "string") || typeof blob !== "string") continue;
+ records.push({ path, message_id, name, mime, size, sha256, source, source_path, blob });
+ } catch {
+ // A torn last line is skipped.
+ }
+ }
+ return records;
+ }
+
+ /** Every staged attachment of the session. */
+ list(sessionId: string): Promise<AttachmentRecord[]> {
+ return this.serial(sessionId, () => this.listUnlocked(sessionId));
+ }
+
+ forget(sessionId: string): Promise<void> {
+ return this.serial(sessionId, () => rm(this.sessionDir(sessionId), { recursive: true, force: true }));
+ }
+}
+
+/** The state.json form of a record (no staging path). */
+export function attachmentStateItem(record: AttachmentRecord): Omit<AttachmentRecord, "blob"> {
+ const { blob: _blob, ...item } = record;
+ return item;
+}
diff --git a/apps/server/src/session-archive/binary.ts b/apps/server/src/session-archive/binary.ts
new file mode 100644
index 00000000..1b51ed4f
--- /dev/null
+++ b/apps/server/src/session-archive/binary.ts
@@ -0,0 +1,142 @@
+/**
+ * Capture v2: what counts as a binary file. A binary file is never sent as
+ * text (any decoding would replace bytes, and the text scrubber would rewrite
+ * runs inside it); it goes to the byte-exact project archive instead.
+ *
+ * A file is binary when its name is a known binary type, when its first
+ * bytes carry a known binary signature (a PDF stays a PDF even when every
+ * byte of it happens to be ASCII), when its first 8 KiB hold a NUL or many
+ * control bytes, or when it is not valid UTF-8 anywhere. Identical in the
+ * CLI and the desktop app (see the parity vectors in
+ * `__fixtures__/capture_v2_vectors.json`).
+ */
+import { isUtf8 } from "node:buffer";
+
+/** Extensions (lower case, without the dot) whose content is binary whatever its bytes look like. */
+export const BINARY_EXTENSIONS: ReadonlySet<string> = new Set([
+ // documents
+ "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "ods", "odp", "epub", "rtfd", "pages", "numbers", "key",
+ // images
+ "png", "jpg", "jpeg", "gif", "bmp", "tif", "tiff", "webp", "ico", "icns", "heic", "heif", "avif", "psd", "xcf", "raw", "cr2", "nef", "dng",
+ // audio / video
+ "mp3", "wav", "flac", "ogg", "oga", "m4a", "aac", "opus", "mp4", "m4v", "mov", "avi", "mkv", "webm", "wmv", "flv", "mid", "midi",
+ // archives / compressed
+ "zip", "gz", "tgz", "bz2", "xz", "zst", "7z", "rar", "tar", "jar", "war", "ear", "whl", "egg", "apk", "aar", "ipa", "deb", "rpm", "dmg", "iso", "lz4", "br", "cab", "msi",
+ // executables / objects / libraries
+ "exe", "dll", "so", "dylib", "a", "lib", "o", "obj", "bin", "elf", "class", "pyc", "pyo", "pyd", "wasm", "node", "dex", "out",
+ // data / models / databases
+ "sqlite", "sqlite3", "db", "mdb", "accdb", "parquet", "feather", "arrow", "avro", "orc", "npy", "npz", "pkl", "pickle", "h5", "hdf5", "nc",
+ "pt", "pth", "ckpt", "safetensors", "onnx", "tflite", "pb", "gguf", "joblib", "mat", "sav", "dta", "xlsb", "lockb",
+ // fonts
+ "ttf", "otf", "woff", "woff2", "eot",
+ // 3d / design
+ "blend", "fbx", "glb", "3ds", "max", "skp", "stl", "dwg", "sketch", "fig", "ai", "indd",
+]);
+
+/**
+ * Leading signatures of binary formats (at offset 0 unless given). Short
+ * printable ones (MZ, BZh, ID3) are left out: a text file may start so, and
+ * the real files fail the control-byte or UTF-8 checks anyway.
+ */
+const SIGNATURES: ReadonlyArray<{ bytes: number[]; offset?: number }> = [
+ { bytes: [0x25, 0x50, 0x44, 0x46, 0x2d] }, // %PDF-
+ { bytes: [0x89, 0x50, 0x4e, 0x47] }, // PNG
+ { bytes: [0xff, 0xd8, 0xff] }, // JPEG
+ { bytes: [0x47, 0x49, 0x46, 0x38] }, // GIF8
+ { bytes: [0x50, 0x4b, 0x03, 0x04] }, // ZIP (docx, jar, ...)
+ { bytes: [0x50, 0x4b, 0x05, 0x06] }, // empty ZIP
+ { bytes: [0x1f, 0x8b] }, // gzip
+ { bytes: [0x28, 0xb5, 0x2f, 0xfd] }, // zstd
+ { bytes: [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00] }, // xz
+ { bytes: [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c] }, // 7z
+ { bytes: [0x52, 0x61, 0x72, 0x21] }, // Rar!
+ { bytes: [0x7f, 0x45, 0x4c, 0x46] }, // ELF
+ { bytes: [0xcf, 0xfa, 0xed, 0xfe] }, // Mach-O 64
+ { bytes: [0xce, 0xfa, 0xed, 0xfe] }, // Mach-O 32
+ { bytes: [0xca, 0xfe, 0xba, 0xbe] }, // Mach-O fat / Java class
+ { bytes: [0x00, 0x61, 0x73, 0x6d] }, // wasm
+ { bytes: [0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66] }, // SQLite format
+ { bytes: [0x52, 0x49, 0x46, 0x46] }, // RIFF (wav, webp, avi)
+ { bytes: [0x4f, 0x67, 0x67, 0x53] }, // OggS
+ { bytes: [0x66, 0x4c, 0x61, 0x43] }, // fLaC
+ { bytes: [0x66, 0x74, 0x79, 0x70], offset: 4 }, // ftyp (mp4, mov, heic)
+ { bytes: [0x1a, 0x45, 0xdf, 0xa3] }, // Matroska / WebM
+ { bytes: [0x49, 0x49, 0x2a, 0x00] }, // TIFF LE
+ { bytes: [0x4d, 0x4d, 0x00, 0x2a] }, // TIFF BE
+ { bytes: [0x00, 0x00, 0x01, 0x00] }, // ICO
+ { bytes: [0x77, 0x4f, 0x46, 0x46] }, // wOFF
+ { bytes: [0x77, 0x4f, 0x46, 0x32] }, // wOF2
+ { bytes: [0x50, 0x41, 0x52, 0x31] }, // PAR1 (parquet)
+ { bytes: [0x89, 0x48, 0x44, 0x46] }, // HDF5
+ { bytes: [0x93, 0x4e, 0x55, 0x4d, 0x50, 0x59] }, // .npy
+];
+
+/** The lower-case extension of a path's last component ("" without one). */
+export function pathExtension(path: string): string {
+ const name = path.slice(Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")) + 1).toLowerCase();
+ const dot = name.lastIndexOf(".");
+ return dot > 0 ? name.slice(dot + 1) : "";
+}
+
+/** Whether the path names a known binary type (its extension). */
+export function isKnownBinaryPath(path: string): boolean {
+ return BINARY_EXTENSIONS.has(pathExtension(path));
+}
+
+/** Whether `head` (the first bytes of a file) starts with a known binary signature. */
+export function hasBinarySignature(head: Uint8Array): boolean {
+ for (const signature of SIGNATURES) {
+ const offset = signature.offset ?? 0;
+ if (head.length < offset + signature.bytes.length) continue;
+ let match = true;
+ for (let index = 0; index < signature.bytes.length; index += 1) {
+ if (head[offset + index] !== signature.bytes[index]) {
+ match = false;
+ break;
+ }
+ }
+ if (match) return true;
+ }
+ return false;
+}
+
+/** Strict UTF-8 validation; with `partial`, an incomplete last character (a cut head) is allowed. */
+export function isValidUtf8(buffer: Uint8Array, partial = false): boolean {
+ if (isUtf8(buffer)) return true;
+ if (!partial) return false;
+ for (let back = 1; back <= 3 && back <= buffer.length; back += 1) {
+ const byte = buffer[buffer.length - back]!;
+ if ((byte & 0xc0) === 0x80) continue;
+ const length = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1;
+ return length > back && isUtf8(buffer.subarray(0, buffer.length - back));
+ }
+ return false;
+}
+
+/** A NUL, or more than 10% control bytes, in the first 8 KiB. */
+export function hasBinaryControlBytes(buffer: Uint8Array): boolean {
+ const length = Math.min(buffer.length, 8_192);
+ let suspicious = 0;
+ for (let index = 0; index < length; index += 1) {
+ const byte = buffer[index]!;
+ if (byte === 0) return true;
+ if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;
+ }
+ return length > 0 && suspicious / length > 0.1;
+}
+
+/**
+ * Why a file is binary, or null for text. `path` may be null (content only);
+ * `partial`: `buffer` is a head cut from a longer file.
+ */
+export function binaryReason(path: string | null, buffer: Uint8Array, partial = false): "type" | "signature" | "control" | "utf8" | null {
+ if (path !== null && isKnownBinaryPath(path)) return "type";
+ if (hasBinarySignature(buffer)) return "signature";
+ if (hasBinaryControlBytes(buffer)) return "control";
+ if (!isValidUtf8(buffer, partial)) return "utf8";
+ return null;
+}
+
+export function isBinaryContent(path: string | null, buffer: Uint8Array, partial = false): boolean {
+ return binaryReason(path, buffer, partial) !== null;
+}
diff --git a/apps/server/src/session-archive/capture-v2.ts b/apps/server/src/session-archive/capture-v2.ts
new file mode 100644
index 00000000..482e8549
--- /dev/null
+++ b/apps/server/src/session-archive/capture-v2.ts
@@ -0,0 +1,108 @@
+/**
+ * Capture v2 – byte-exact project state. The archive of a v2 chain says
+ * which state of the folder it is (`__omnirush__/state.json`): the session
+ * start (taken before the first prompt reached the model), a turn's first
+ * tool call, the end of a turn, or a final state; and next to the files it
+ * lists every repository of the session with its git state, everything the
+ * scan left out with the reason, the config files whose credentials were
+ * removed from the archived copy, and the prompt attachments it carries.
+ *
+ * A chain is v2 when the server offers it (`policy.capture_v2` from GET
+ * /archives/key; OMNIRUSH_CAPTURE_V2=0/1 overrides, for tests) at its base,
+ * and stays so. The wire format is in the backend's
+ * docs/omnirush-project-archive.md ("Capture v2"). Identical in the CLI and
+ * the desktop app.
+ */
+import type { AttachmentRecord } from "./attachments.js";
+import { attachmentStateItem } from "./attachments.js";
+import type { RepoState } from "./repos.js";
+
+export const ARCHIVE_SCHEMA_V2 = "omnirush.archive.v2";
+export const STATE_MEMBER = "__omnirush__/state.json";
+export const STATE_SCHEMA = "omnirush.archive.state.v1";
+/** The start gate waits this long, at most, for the start manifest before the first prompt goes to the model. */
+export const START_GATE_MS = 3_000;
+export const MAX_EXCLUDED_LISTED = 20_000;
+
+export type StateKind = "start" | "pre_tool" | "after" | "final";
+export type StartCapture = "complete" | "partial";
+
+export type ExcludedReason =
+ | "credential"
+ | "gitignored"
+ | "regenerable"
+ | "app_state"
+ | "special"
+ | "unreadable"
+ | "non_utf8"
+ | "reserved"
+ | "too_large";
+
+export type ExcludedItem = { path: string; type: "file" | "dir"; reason: ExcludedReason; size?: number; sha256?: string };
+
+/** The scan's left-out entries, by path (the first reason wins), at most MAX_EXCLUDED_LISTED. */
+export class ExcludedList {
+ private readonly items = new Map<string, ExcludedItem>();
+ private dropped = 0;
+
+ add(item: ExcludedItem): void {
+ if (this.items.has(item.path)) {
+ const known = this.items.get(item.path)!;
+ if (item.sha256 && !known.sha256) this.items.set(item.path, { ...known, ...(item.size !== undefined ? { size: item.size } : {}), sha256: item.sha256 });
+ return;
+ }
+ if (this.items.size >= MAX_EXCLUDED_LISTED) {
+ this.dropped += 1;
+ return;
+ }
+ this.items.set(item.path, item);
+ }
+
+ delete(path: string): void {
+ this.items.delete(path);
+ }
+
+ get truncated(): boolean {
+ return this.dropped > 0;
+ }
+
+ list(): ExcludedItem[] {
+ return [...this.items.values()].sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
+ }
+}
+
+/** OMNIRUSH_CAPTURE_V2: "1"/"true"/"on" forces v2 on, "0"/"false"/"off" off; anything else follows the server. */
+export function captureV2Override(env: NodeJS.ProcessEnv = process.env): boolean | null {
+ const value = (env.OMNIRUSH_CAPTURE_V2 ?? "").trim().toLowerCase();
+ if (["1", "true", "on", "yes"].includes(value)) return true;
+ if (["0", "false", "off", "no"].includes(value)) return false;
+ return null;
+}
+
+export type StateDocumentInput = {
+ archiveId: string;
+ kind: StateKind;
+ turn: number;
+ reason?: string;
+ startCapture?: StartCapture;
+ startCaptureMs?: number;
+ repos: readonly RepoState[];
+ excluded: ExcludedList;
+ scrubbed: readonly string[];
+ attachments: readonly AttachmentRecord[];
+};
+
+/** `__omnirush__/state.json` of a v2 archive. */
+export function stateDocument(input: StateDocumentInput): Buffer {
+ return Buffer.from(JSON.stringify({
+ schema: STATE_SCHEMA,
+ archive_id: input.archiveId,
+ state: { kind: input.kind, turn: input.turn, ...(input.reason ? { reason: input.reason } : {}) },
+ ...(input.startCapture ? { start_capture: input.startCapture, start_capture_ms: input.startCaptureMs ?? 0 } : {}),
+ repos: input.repos,
+ excluded: input.excluded.list(),
+ excluded_truncated: input.excluded.truncated,
+ scrubbed: [...input.scrubbed].sort(),
+ attachments: input.attachments.map(attachmentStateItem),
+ }), "utf8");
+}
diff --git a/apps/server/src/session-archive/git-scrub.ts b/apps/server/src/session-archive/git-scrub.ts
new file mode 100644
index 00000000..4810266e
--- /dev/null
+++ b/apps/server/src/session-archive/git-scrub.ts
@@ -0,0 +1,121 @@
+/**
+ * Capture v2: the credentials a repository's config can hold are removed
+ * from the ARCHIVED copy of that config only; the user's own files are never
+ * written. What goes:
+ *
+ * - credentials in any URL value or URL subsection (`url`, `pushurl`,
+ * `insteadOf`, `proxy`, `[url "..."]`, `[http "..."]`): all userinfo of
+ * an http(s) URL (`https://user:token@host/x` becomes `https://host/x`),
+ * the password of any other (`ssh://git:pw@host` becomes `ssh://git@host`);
+ * - every `extraheader` (http.extraHeader, http.<url>.extraHeader), which
+ * carries `Authorization: ...` for CI checkouts;
+ * - every `[credential]` / `[credential "<url>"]` section (helpers can be
+ * inline scripts that echo a password, and `username`s);
+ * - any key whose name says it holds a secret (`token`, `password`,
+ * `passwd`, `secret`, `apikey`, `api-key`, `auth`, `cookie`), and the
+ * `http.cookieFile` / `http.sslKey` / `http.sslCertPasswordProtected`
+ * style keys that point at credential files.
+ *
+ * Removed lines are replaced by a comment, so line counts and section order
+ * stay as they were. Identical in the CLI and the desktop app.
+ */
+
+/** Archive paths of git config files whose archived copy is scrubbed. */
+export function isGitConfigPath(path: string): boolean {
+ const parts = path.split("/");
+ const last = parts[parts.length - 1];
+ if (last !== "config" && last !== "config.worktree") return false;
+ const gitIndex = parts.lastIndexOf(".git");
+ if (gitIndex === -1) return false;
+ const inside = parts.slice(gitIndex + 1, -1);
+ // .git/config, .git/config.worktree, .git/worktrees/<name>/config.worktree, .git/modules/<a>/<b>/config
+ if (inside.length === 0) return true;
+ if (inside[0] === "worktrees" && inside.length === 2) return last === "config.worktree";
+ if (inside[0] === "modules") return inside.length >= 2 && inside.every((part) => part !== "objects" && part !== "refs");
+ return false;
+}
+
+const SECRET_KEY = /^(?:.*(?:token|password|passwd|secret|apikey|api-key|api_key|cookie)|extraheader|cookiefile|sslkey|sslcert|sslcertpasswordprotected|proxyauthmethod)$/i;
+const URL_USERINFO = /([a-z][a-z0-9+.-]*):\/\/([^/@\s"]*)@/gi;
+const SCP_USERINFO = /^(\s*)([^\s"@/]+)@([^\s"/:]+:)/;
+
+/**
+ * Strips credentials from every URL in a config value: all userinfo of an
+ * http(s) URL (a token is often the user name there), the password of any
+ * other scheme's (`ssh://git:pw@host` keeps `git@`, which says how to log
+ * in, not with what), and of the scp form (`user:pw@host:path`).
+ */
+function stripUserinfo(value: string): string {
+ let out = value.replace(URL_USERINFO, (_match, scheme: string, userinfo: string) => {
+ if (/^https?$/i.test(scheme)) return `${scheme}://`;
+ const user = userinfo.split(":")[0] ?? "";
+ return user ? `${scheme}://${user}@` : `${scheme}://`;
+ });
+ const scp = SCP_USERINFO.exec(out);
+ if (scp && scp[2]!.includes(":")) out = out.replace(SCP_USERINFO, (_match, lead: string, userinfo: string, host: string) => `${lead}${userinfo.split(":")[0]}@${host}`);
+ return out;
+}
+
+export type GitConfigScrub = { text: string; changed: boolean; removed: number };
+
+/** The config text as archived. Never throws; a value it cannot parse is kept only when it holds no URL userinfo. */
+export function scrubGitConfig(text: string): GitConfigScrub {
+ const lines = text.split("\n");
+ let section = "";
+ let removed = 0;
+ let changed = false;
+ let continued = false;
+ const out: string[] = [];
+ for (const line of lines) {
+ if (continued) {
+ // The continuation of a removed value (trailing backslash).
+ continued = /\\\r?$/.test(line);
+ out.push("# [omnirush: removed]");
+ removed += 1;
+ changed = true;
+ continue;
+ }
+ const header = /^\s*\[\s*([A-Za-z0-9.-]+)(?:\s+"(?:[^"\\]|\\.)*")?\s*\]/.exec(line);
+ if (header) {
+ section = header[1]!.toLowerCase();
+ const rest = line.slice(header[0].length).trim();
+ // `[section] key = value` on one line: the inline pair is judged like any other.
+ const inline = /^([A-Za-z][A-Za-z0-9-]*)\s*(?:=|$)/.exec(rest);
+ if (inline && (section === "credential" || SECRET_KEY.test(inline[1]!))) {
+ continued = /\\\r?$/.test(line);
+ out.push(`${header[0]} # [omnirush: removed ${section}.${inline[1]!.toLowerCase()}]`);
+ removed += 1;
+ changed = true;
+ continue;
+ }
+ // A URL subsection (`[url "https://user:token@host/"]`, `[http "..."]`) loses its userinfo too.
+ const stripped = stripUserinfo(line);
+ if (stripped !== line) {
+ changed = true;
+ removed += 1;
+ }
+ out.push(stripped);
+ continue;
+ }
+ const pair = /^(\s*)([A-Za-z][A-Za-z0-9-]*)(\s*(?:=|$))(.*)$/.exec(line);
+ if (!pair || /^\s*[#;]/.test(line) || line.trim() === "") {
+ out.push(line);
+ continue;
+ }
+ const key = pair[2]!.toLowerCase();
+ if (section === "credential" || SECRET_KEY.test(key)) {
+ continued = /\\\r?$/.test(line);
+ out.push(`${pair[1]}# [omnirush: removed ${section ? `${section}.` : ""}${key}]`);
+ removed += 1;
+ changed = true;
+ continue;
+ }
+ const stripped = stripUserinfo(line);
+ if (stripped !== line) {
+ changed = true;
+ removed += 1;
+ }
+ out.push(stripped);
+ }
+ return { text: out.join("\n"), changed, removed };
+}
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index 2e98e087..5e767cc5 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -13,7 +13,7 @@
* lifecycle.ts; see README.md.
*/
import { randomBytes, randomUUID } from "node:crypto";
-import { mkdir, readdir, readFile, rename, rm, stat, statfs } from "node:fs/promises";
+import { lstat, mkdir, readdir, readFile, realpath, rename, rm, stat, statfs } from "node:fs/promises";
import { isAbsolute, join, resolve } from "node:path";
import { z } from "zod";
@@ -35,11 +35,13 @@ import {
baselineChunks,
compareArchivePaths,
computeArchiveDelta,
+ emptyExcludedCounts,
isArchiveDeltaEmpty,
manifestSource,
parseBaselineText,
readArchiveGit,
scanArchiveTree,
+ statFields,
type ArchiveEntry,
type ArchiveGit,
type ArchiveKind,
@@ -53,7 +55,22 @@ import { POLICY_OFF, type ArchivePolicy } from "./policy.js";
import { SEAL_CONTENT } from "./seal.js";
import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";
import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";
-import { inRegenerableDir } from "../session-uploader.js";
+import { ATTACHMENTS_ROOT_NAME, AttachmentStore, type AttachmentRecord, type AttachmentSource } from "./attachments.js";
+import {
+ ARCHIVE_SCHEMA_V2,
+ captureV2Override,
+ ExcludedList,
+ START_GATE_MS,
+ STATE_MEMBER,
+ stateDocument,
+ type StartCapture,
+ type StateKind,
+} from "./capture-v2.js";
+import { discoverRepos } from "./repos.js";
+import { inRegenerableDir, isUploadPathDenied } from "../session-uploader.js";
+
+/** Capture v2 (#14): a touched file the folder scan did not hold larger than this is left out, with its path, size and hash. */
+const MAX_TOUCHED_FILE_BYTES = 1024 * 1024 * 1024;
import {
ARCHIVE_MARKER_NOT_ALLOWED,
ArchiveUploader,
@@ -219,6 +236,8 @@ const sessionStateSchema = z.object({
* updated_at, and keeps that time from then on.
*/
last_activity_at: z.string().optional(),
+ /** Capture v2 (capture-v2.ts): this chain's archives are byte-exact states with a state.json. Set at the base. */
+ v2: z.boolean().optional(),
}).transform((state) => ({ ...state, last_activity_at: state.last_activity_at ?? state.updated_at }));
type SessionState = z.infer<typeof sessionStateSchema>;
@@ -243,7 +262,7 @@ const queueRecordSchema = z.object({
request: createRequestSchema,
sealed_file: z.string(),
/** A delta's trigger (1.1.0 on): how a refused final is told from a broken chain. */
- trigger: z.enum(["turn", "final"]).optional(),
+ trigger: z.enum(["turn", "final", "pre_tool"]).optional(),
created_at: z.string(),
attempts: z.number().int().nonnegative(),
next_attempt_at: z.string().nullable(),
@@ -305,8 +324,45 @@ type CaptureOptions = {
ended?: boolean;
/** Cancels the capture until its commit. */
signal?: AbortSignal;
+ /** Capture v2: the turn a pre_tool state belongs to (the turn in progress). */
+ stateTurn?: number;
+};
+
+/** Capture v2: the session-start scan taken before the first prompt reached the model (startManifest). */
+type Prescan = {
+ root: string;
+ startedAt: number;
+ /** Resolves with the scan, or null when there is nothing to take (not archivable, a base exists). */
+ work: Promise<PrescanResult | null>;
+ /** "complete" when the scan finished within the start gate's time cap. */
+ capture: StartCapture;
+ timer: ReturnType<typeof setTimeout> | null;
+};
+
+type PrescanResult = {
+ marker: string;
+ scan: { entries: ScannedEntry[]; excluded: ExcludedCounts; ignored: number };
+ cache: ArchiveHashCache;
+ excludedList: ExcludedList;
+ ms: number;
};
+/** A prescan that no base takes is dropped after this long. */
+const PRESCAN_TTL_MS = 10 * 60_000;
+
+/** The `.git` markers of a full scan (the root's and nested repositories'), for repos.ts. */
+function repoMarkers(entries: readonly ArchiveEntry[]): Array<{ path: string; gitfile: boolean }> {
+ const markers: Array<{ path: string; gitfile: boolean }> = [];
+ for (const entry of entries) {
+ if (entry.type === "symlink") continue;
+ const parts = entry.path.split("/");
+ if (parts[parts.length - 1] !== ".git" || parts.slice(0, -1).includes(".git")) continue;
+ if (parts[0] === ATTACHMENTS_ROOT_NAME || isOutsideArchivePath(entry.path)) continue;
+ markers.push({ path: entry.path, gitfile: entry.type === "file" });
+ }
+ return markers;
+}
+
export class SessionArchiver {
private readonly dir: string;
private readonly dirs: { sessions: string; baselines: string; hashCache: string; queue: string; pending: string; tmp: string; touched: string };
@@ -348,6 +404,12 @@ export class SessionArchiver {
private finalsRefused = false;
/** The paths each touched-files session touched (touched.ts). */
private readonly touched: TouchedPathStore;
+ /** Capture v2: staged prompt attachments (attachments.ts). */
+ private readonly attachments: AttachmentStore;
+ /** Capture v2: start scans waiting for their session's base. */
+ private readonly prescans = new Map<string, Prescan>();
+ /** Capture v2: the sessions whose chain is v2 (as last loaded or saved). */
+ private readonly v2Sessions = new Set<string>();
constructor(options: SessionArchiverOptions) {
const stateDir = resolve(options.stateDir);
@@ -361,6 +423,7 @@ export class SessionArchiver {
tmp: join(this.dir, "tmp"),
touched: join(this.dir, "touched"),
};
+ this.attachments = new AttachmentStore(join(this.dir, "attachments"), (path) => !this.includeCredentials && isUploadPathDenied(path));
this.log = options.log ?? (() => undefined);
this.now = options.now ?? (() => new Date());
this.random = options.random ?? { uuid: () => randomUUID(), bytes: (n) => randomBytes(n) };
@@ -557,6 +620,131 @@ export class SessionArchiver {
this.touched.note(sessionId, typeof path === "string" && isAbsolute(path) ? resolve(path) : path);
}
+ /**
+ * Capture v2, #8: the session-start manifest, before the first prompt
+ * reaches the model. Stats and hashes the folder as its base will hold it
+ * (with the root's hash cache: an unchanged tree reads no file), local work
+ * only: no key fetch, no upload. Resolves within `capMs` (START_GATE_MS):
+ * "complete" when the manifest was taken by then, "partial" when the cap
+ * ran out first (the scan goes on and the base still uses it, saying
+ * `start_capture: "partial"`), "skipped" when there is nothing to take (a
+ * base exists, the folder is refused, capture v2 is off). Never throws.
+ * The base archive packs these very entries, and pass 2 checks each file's
+ * bytes against the hash taken here (a file changed since is listed as
+ * unstable), so the base is the folder as it was before the prompt.
+ */
+ async startManifest(sessionId: string, root: string, capMs = START_GATE_MS): Promise<StartCapture | "skipped"> {
+ if (!SESSION_ID_PATTERN.test(sessionId) || this.disabled || !this.uploader.configured) return "skipped";
+ const override = captureV2Override();
+ if (override === false || (override === null && this.policy && !this.policy.value.captureV2)) return "skipped";
+ let prescan = this.prescans.get(sessionId);
+ if (!prescan) {
+ const startedAt = Date.now();
+ const created: Prescan = { root, startedAt, work: Promise.resolve(null), capture: "partial", timer: null };
+ created.work = this.prescanWork(sessionId, root, startedAt).catch((error: unknown) => {
+ this.log("warn", "OmniRush start manifest failed", { sessionId, error: errorSummary(error) });
+ return null;
+ });
+ created.timer = setTimeout(() => {
+ if (this.prescans.get(sessionId) === created) this.prescans.delete(sessionId);
+ }, PRESCAN_TTL_MS);
+ created.timer.unref?.();
+ this.prescans.set(sessionId, created);
+ prescan = created;
+ }
+ const current = prescan;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const outcome = await Promise.race([
+ current.work.then((result) => (result ? "complete" as const : "skipped" as const)),
+ new Promise<"partial">((resolvePromise) => {
+ timer = setTimeout(() => resolvePromise("partial"), Math.max(0, capMs - (Date.now() - current.startedAt)));
+ timer.unref?.();
+ }),
+ ]);
+ clearTimeout(timer);
+ if (outcome === "complete") current.capture = "complete";
+ if (outcome === "skipped" && this.prescans.get(sessionId) === current) this.dropPrescan(sessionId);
+ return outcome;
+ }
+
+ /** Whether a start manifest is waiting for this session's base (the lifecycle then packs the base at once). */
+ hasStartManifest(sessionId: string): boolean {
+ return this.prescans.has(sessionId);
+ }
+
+ private dropPrescan(sessionId: string): void {
+ const prescan = this.prescans.get(sessionId);
+ if (prescan?.timer) clearTimeout(prescan.timer);
+ this.prescans.delete(sessionId);
+ }
+
+ private async prescanWork(sessionId: string, root: string, startedAt: number): Promise<PrescanResult | null> {
+ await this.start();
+ const state = await this.loadSession(sessionId);
+ if (state && (state.next_sequence > 0 || state.stopped || state.marker === TOUCHED_MARKER)) return null;
+ const real = await realpath(root).catch(() => null);
+ if (!real) return null;
+ // The local part of the gate only: a git root, else a folder the folder refusals accept.
+ const gate = await isArchivableProject(real, this.detectors, { appDirs: this.appDirs });
+ let marker = gate.archivable && gate.marker ? gate.marker : null;
+ if (!marker) {
+ const plain = await isArchivableProject(real, [async (candidate) => ((await folderRootRefusal(candidate, this.folderGate)) ? null : { archivable: true, reason: FOLDER_MARKER, marker: FOLDER_MARKER })], { appDirs: this.appDirs });
+ if (!plain.archivable) return null;
+ marker = FOLDER_MARKER;
+ }
+ const cache = await this.loadHashCache(stateKey(real));
+ const excludedList = new ExcludedList();
+ const scan = await this.scanWholeFolder(real, marker, sessionId, cache, excludedList, true);
+ const prescan = this.prescans.get(sessionId);
+ if (prescan) prescan.root = real;
+ return { marker, scan, cache, excludedList, ms: Date.now() - startedAt };
+ }
+
+ /**
+ * Capture v2, #9: the folder at a turn's first tool call, when anything
+ * changed since the chain's last state (the user's edits between turns, a
+ * command run outside the agent). A delta numbered with the last archived
+ * turn (like a final archive), `trigger: "pre_tool"`; `stateTurn` is the
+ * turn in progress. Only for a v2 chain with a base.
+ */
+ captureState(sessionId: string, stateTurn: number): Promise<CaptureResult> {
+ return this.guard("delta", sessionId, stateTurn, async () => this.withSession(sessionId, async () => {
+ const generation = this.generation;
+ if (this.disabled || !this.uploader.configured) return { status: "skipped", reason: "disabled" };
+ const state = await this.loadSession(sessionId);
+ if (!state || state.next_sequence === 0 || state.last_turn === null) return { status: "skipped", reason: "no_base" };
+ if (!state.v2) return { status: "skipped", reason: "not_archivable" };
+ if (state.stopped || this.stoppedSessions.has(sessionId)) return { status: "skipped", reason: "stopped" };
+ if (!(await this.chainAllowed(state, generation))) return { status: "skipped", reason: "not_archivable" };
+ const key = await this.currentKey();
+ if (key === "disabled") return { status: "skipped", reason: "disabled" };
+ if (key === "unavailable") return { status: "skipped", reason: "unavailable" };
+ return this.captureArchive(state, "delta", state.last_turn, key, generation, { trigger: "pre_tool", stateTurn });
+ }));
+ }
+
+ /**
+ * Capture v2, #11: the attachments of a settled turn's user messages
+ * (attachments.ts), staged now and archived by the session's next capture
+ * under `__attachments__/`. Never throws.
+ */
+ async recordAttachments(sessionId: string, root: string, sources: readonly AttachmentSource[]): Promise<AttachmentRecord[]> {
+ if (sources.length === 0 || this.disabled || !SESSION_ID_PATTERN.test(sessionId) || this.stoppedSessions.has(sessionId)) return [];
+ await this.start();
+ const state = await this.loadSession(sessionId);
+ if (!state?.v2 || state.stopped) return [];
+ return this.attachments.add(sessionId, state.root || root, sources);
+ }
+
+ /**
+ * Capture v2: a binary file of the workspace the session uploader found
+ * (never sent as text). A v2 chain archives it byte for byte, also a
+ * touched-files chain (it is kept like a touched path); nothing otherwise.
+ */
+ recordBinary(sessionId: string, path: string): void {
+ if (this.v2Sessions.has(sessionId)) this.recordTouched(sessionId, path);
+ }
+
/** The session is not archived (a child session): its reported paths go. */
forgetTouched(sessionId: string): void {
void this.touched.forget(sessionId).catch((error: unknown) => this.log("warn", "OmniRush touched-files paths could not be removed", { sessionId, error: errorSummary(error) }));
@@ -656,6 +844,8 @@ export class SessionArchiver {
this.policyProbe = null;
this.stoppedSessions.clear();
this.resettingSessions.clear();
+ this.v2Sessions.clear();
+ for (const sessionId of [...this.prescans.keys()]) this.dropPrescan(sessionId);
this.finalsAccepted = false;
this.finalsRefused = false;
this.uploader.setAccessToken(null);
@@ -750,6 +940,7 @@ export class SessionArchiver {
const touched = gate.marker === TOUCHED_MARKER;
// A session that is not archived keeps no touched paths; any other keeps them.
if (!gate.archivable || !gate.marker) {
+ this.dropPrescan(sessionId);
await this.touched.forget(sessionId);
return { status: "skipped", reason: "not_archivable" };
}
@@ -757,11 +948,13 @@ export class SessionArchiver {
const fetched = probe.fetched ?? await this.uploader.fetchKey();
if (!probe.fetched) this.rememberPolicy(fetched, generation, false);
if (fetched.status === "disabled") {
+ this.dropPrescan(sessionId);
await this.disable(fetched.code);
return { status: "skipped", reason: "disabled" };
}
// The policy was turned off since the answer this folder was let in on.
if (fetched.status === "ok" && !markerAllowed(gate.marker, fetched.policy)) {
+ this.dropPrescan(sessionId);
await this.touched.forget(sessionId);
return { status: "skipped", reason: "not_archivable" };
}
@@ -778,7 +971,10 @@ export class SessionArchiver {
updated_at: this.now().toISOString(),
...(touched ? { turn_seen: turn } : {}),
last_activity_at: this.now().toISOString(),
+ // Capture v2 when the server offers it (or the override says so); the chain keeps it.
+ ...((captureV2Override() ?? (fetched.status === "ok" && fetched.policy.captureV2 === true)) ? { v2: true } : {}),
};
+ if (touched) this.dropPrescan(sessionId);
this.touched.track(sessionId);
if (fetched.status === "unavailable") {
// Remembered with next_sequence 0: the next captureDelta tries the base again (a folder passes the gate and the policy again).
@@ -958,8 +1154,13 @@ export class SessionArchiver {
const sessionKey = stateKey(sessionId);
const rootKey = stateKey(state.root);
const { signal } = options;
- const cache = await this.loadHashCache(rootKey);
+ // Capture v2: a base packs the start manifest taken before the first prompt (startManifest).
+ const prescan = kind === "base" && state.marker !== TOUCHED_MARKER ? await this.takePrescan(state.session_id, state.root, state.marker, state.v2 === true) : null;
+ const cache = prescan ? prescan.result.cache : await this.loadHashCache(rootKey);
const touched = state.marker === TOUCHED_MARKER;
+ const v2 = state.v2 === true;
+ const excludedList = v2 ? (prescan?.result.excludedList ?? new ExcludedList()) : undefined;
+ let fullScan: readonly ArchiveEntry[] = [];
let files: ScannedEntry[];
let deleted: string[] | undefined;
let excluded: ExcludedCounts;
@@ -977,11 +1178,16 @@ export class SessionArchiver {
// Every path the session touched and every lockfile, outside the
// regenerable folders, and every file the chain holds (to see it go).
// Gitignored files the agent touched are archived too.
- const paths = new Set([...(await this.touched.snapshot(sessionId)), ...(await findLockfiles(state.root, signal))].filter((path) => !inRegenerableDir(path)));
- for (const entry of baseline) if (!isOutsideArchivePath(entry.path)) paths.add(entry.path);
+ const touchedPaths = [...(await this.touched.snapshot(sessionId))];
+ const lockfiles = await findLockfiles(state.root, signal);
+ // Capture v2 (#14): a touched file in a regenerable folder (a venv's tool, a cached dataset) is archived too, up to the per-file cap.
+ const paths = new Set([...touchedPaths.filter((path) => (v2 ? !path.split("/").slice(0, -1).includes(".git") : !inRegenerableDir(path))), ...lockfiles.filter((path) => !inRegenerableDir(path))]);
+ for (const entry of baseline) if (!isOutsideArchivePath(entry.path) && !entry.path.startsWith(`${ATTACHMENTS_ROOT_NAME}/`)) paths.add(entry.path);
const scan = await scanTouchedFiles(state.root, paths, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });
+ if (excludedList) this.capOversized(scan, new Set(), excludedList);
// The files outside the workspace the session touched (outside.ts).
mergeScans(scan, await this.scanOutside(state, baseline, signal));
+ if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));
const change = touchedChange(baseline, scan);
if (change.files.length === 0 && change.deleted.length === 0) {
if (cache.changed) await this.saveHashCache(rootKey, cache);
@@ -994,20 +1200,12 @@ export class SessionArchiver {
ignored = scan.ignored;
next = change.next;
} else {
- const scanning = scanArchiveTree(state.root, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, hashCache: cache, ...(signal ? { signal } : {}) });
// A plain folder sends no git block (workspace.git null), even inside a larger repository.
const withGit = state.marker !== FOLDER_MARKER;
const gitReading = kind === "base" && withGit ? readArchiveGit(state.root) : null;
- const scan = await scanning;
- // What the ignore rules left out is still archived when the agent
- // touched it or it is a lockfile, outside the regenerable folders.
- const held = new Set(scan.entries.map((entry) => entry.path));
- const keepers = [...(await this.touched.snapshot(sessionId)), ...(await findLockfiles(state.root, signal))]
- .filter((path) => !held.has(path) && !inRegenerableDir(path));
- if (keepers.length > 0) {
- const kept = await scanTouchedFiles(state.root, keepers, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });
- if (kept.entries.length > 0) scan.entries = [...scan.entries, ...kept.entries].sort((left, right) => compareArchivePaths(left.path, right.path));
- }
+ const scan = prescan
+ ? { entries: [...prescan.result.scan.entries], excluded: { ...prescan.result.scan.excluded }, ignored: prescan.result.scan.ignored }
+ : await this.scanWholeFolder(state.root, state.marker, sessionId, cache, excludedList, v2, signal);
const baseline = kind === "delta" ? await this.readBaseline(state) : [];
if (!baseline) {
await this.markStopped(sessionId, "baseline_missing");
@@ -1015,6 +1213,8 @@ export class SessionArchiver {
}
// The files outside the workspace the session touched (outside.ts).
mergeScans(scan, await this.scanOutside(state, baseline, signal));
+ if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));
+ fullScan = scan.entries;
files = scan.entries;
excluded = scan.excluded;
ignored = scan.ignored;
@@ -1034,7 +1234,30 @@ export class SessionArchiver {
const archiveId = this.random.uuid();
const sequence = state.next_sequence;
const createdAt = this.now();
+ let extraMembers: Array<{ name: string; content: Buffer }> = [];
+ if (v2 && excludedList) {
+ const stateKind: StateKind = kind === "base" ? "start" : options.trigger === "pre_tool" ? "pre_tool" : options.trigger === "final" ? "final" : "after";
+ const held = new Set(files.map((entry) => entry.path));
+ const attachments = (await this.attachments.list(sessionId)).filter((record) => held.has(record.path));
+ const listed = touched ? next : fullScan;
+ extraMembers = [{
+ name: STATE_MEMBER,
+ content: stateDocument({
+ archiveId,
+ kind: stateKind,
+ turn: stateKind === "pre_tool" ? options.stateTurn ?? turn + 1 : turn,
+ ...(options.reason ? { reason: options.reason } : {}),
+ ...(kind === "base" ? { startCapture: prescan?.capture ?? "partial", startCaptureMs: prescan?.result.ms ?? 0 } : {}),
+ // Every repository of the session (repos.ts), whatever the chain's marker.
+ repos: await discoverRepos(state.root, repoMarkers(touched ? [] : fullScan)),
+ excluded: excludedList,
+ scrubbed: listed.filter((entry) => entry.scrubbed).map((entry) => entry.path),
+ attachments,
+ }),
+ }];
+ }
const manifest = manifestSource({
+ ...(v2 ? { schema: ARCHIVE_SCHEMA_V2 } : {}),
kind,
archiveId,
sessionId,
@@ -1079,6 +1302,7 @@ export class SessionArchiver {
{
root: state.root,
manifest,
+ extraMembers,
createdAtSeconds: Math.floor(createdAt.getTime() / 1000),
entries: files,
...(signal ? { signal } : {}),
@@ -1195,6 +1419,92 @@ export class SessionArchiver {
}
}
+ /**
+ * The start manifest of the session (startManifest), once its scan is done,
+ * when it was taken of this root for this marker; it is consumed. A v1
+ * chain of a plain folder scans again (a v2 scan leaves the regenerable
+ * folders out).
+ */
+ private async takePrescan(sessionId: string, root: string, marker: string, v2: boolean): Promise<{ result: PrescanResult; capture: StartCapture } | null> {
+ const prescan = this.prescans.get(sessionId);
+ if (!prescan) return null;
+ this.dropPrescan(sessionId);
+ const result = await prescan.work;
+ if (!result || result.marker !== marker || resolve(prescan.root) !== resolve(root)) return null;
+ if (!v2 && marker === FOLDER_MARKER) return null;
+ return { result, capture: prescan.capture };
+ }
+
+ /**
+ * The whole-folder scan of a chain (git or plain folder): pass 1 with the
+ * exclusions, then what the ignore rules left out that is still archived:
+ * the files the agent touched and every lockfile. A capture v2 scan also leaves the regenerable folders of a plain
+ * folder out (with their reason), keeps a touched file in a regenerable
+ * folder up to the per-file cap, and lists every left-out entry.
+ */
+ private async scanWholeFolder(
+ root: string,
+ marker: string,
+ sessionId: string,
+ cache: ArchiveHashCache,
+ excludedList: ExcludedList | undefined,
+ v2: boolean,
+ signal?: AbortSignal,
+ ): Promise<{ entries: ScannedEntry[]; excluded: ExcludedCounts; ignored: number }> {
+ const scan = await scanArchiveTree(root, {
+ excludedDirs: this.appDirs,
+ includeCredentialFiles: this.includeCredentials,
+ hashCache: cache,
+ ...(excludedList ? { excludedList } : {}),
+ ...(v2 && marker === FOLDER_MARKER ? { pruneRegenerable: true } : {}),
+ ...(signal ? { signal } : {}),
+ });
+ // What the ignore rules left out is still archived when the agent
+ // touched it or it is a lockfile, outside the regenerable folders.
+ const held = new Set(scan.entries.map((entry) => entry.path));
+ const touchedPaths = [...(await this.touched.snapshot(sessionId))];
+ const keepers = [
+ ...touchedPaths.filter((path) => (v2 ? !path.split("/").slice(0, -1).includes(".git") : !inRegenerableDir(path))),
+ ...(await findLockfiles(root, signal)).filter((path) => !inRegenerableDir(path)),
+ ].filter((path) => !held.has(path));
+ if (keepers.length > 0) {
+ const kept = await scanTouchedFiles(root, keepers, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });
+ if (excludedList) this.capOversized(kept, held, excludedList);
+ for (const entry of kept.entries) excludedList?.delete(entry.path);
+ if (kept.entries.length > 0) scan.entries = [...scan.entries, ...kept.entries].sort((left, right) => compareArchivePaths(left.path, right.path));
+ }
+ return scan;
+ }
+
+ /**
+ * Capture v2 (#14): a touched file the folder scan did not hold (gitignored,
+ * in a regenerable folder) larger than the per-file cap is left out with
+ * its path, size and hash.
+ */
+ private capOversized(scan: { entries: ScannedEntry[] }, held: ReadonlySet<string>, excludedList: ExcludedList): void {
+ scan.entries = scan.entries.filter((entry) => {
+ if (held.has(entry.path) || entry.size <= MAX_TOUCHED_FILE_BYTES) return true;
+ excludedList.add({ path: entry.path, type: "file", reason: "too_large", size: entry.size, ...(entry.sha256 ? { sha256: entry.sha256 } : {}) });
+ return false;
+ });
+ }
+
+ /** Capture v2 (#11): the session's staged attachments as archive entries read from their staged copies. */
+ private async attachmentEntries(sessionId: string): Promise<TouchedScanResult> {
+ const entries: ScannedEntry[] = [];
+ for (const record of await this.attachments.list(sessionId)) {
+ try {
+ const stats = await lstat(record.blob, { bigint: true });
+ if (!stats.isFile() || Number(stats.size) !== record.size) continue;
+ entries.push({ path: record.path, type: "file", size: record.size, sha256: record.sha256, source: record.blob, ...statFields(stats), mode: 0o644 });
+ } catch {
+ // Gone (signed out meanwhile): not archived.
+ }
+ }
+ entries.sort((left, right) => compareArchivePaths(left.path, right.path));
+ return { entries, gone: new Set(), excluded: emptyExcludedCounts(), ignored: 0 };
+ }
+
/**
* The files outside the workspace the session touched, and those an
* earlier archive of the chain holds (to see them change or go), scanned
@@ -1540,11 +1850,15 @@ export class SessionArchiver {
private async loadSession(sessionId: string): Promise<SessionState | null> {
const parsed = sessionStateSchema.safeParse(await readJsonFile(this.sessionPath(sessionId)));
- return parsed.success && parsed.data.session_id === sessionId ? parsed.data : null;
+ const state = parsed.success && parsed.data.session_id === sessionId ? parsed.data : null;
+ if (state?.v2 && !state.stopped) this.v2Sessions.add(sessionId);
+ return state;
}
private async saveSession(state: SessionState): Promise<void> {
await writeJsonAtomic(this.sessionPath(state.session_id), state);
+ if (state.v2 && !state.stopped && !state.ended) this.v2Sessions.add(state.session_id);
+ else this.v2Sessions.delete(state.session_id);
}
private async loadHashCache(rootKey: string): Promise<ArchiveHashCache> {
diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.ts
index a65f0ea8..878752dc 100644
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@ -13,12 +13,14 @@
*/
import { realpath } from "node:fs/promises";
+import { attachmentsFromMessages } from "./attachments.js";
+import type { StartCapture } from "./capture-v2.js";
import type { CaptureResult, DrainResult, FinalReason, SessionArchiver } from "./index.js";
export type ProjectArchiver = Pick<
SessionArchiver,
"captureBase" | "captureDelta" | "captureFinal" | "startFinalCandidates" | "recordTouched" | "forgetTouched" | "drain" | "signOut" | "stop"
->;
+> & Partial<Pick<SessionArchiver, "startManifest" | "hasStartManifest" | "captureState" | "recordAttachments" | "recordBinary">>;
/** Engine reads for one session, resolving to the parsed JSON, or null when it cannot be read. */
export type ArchiveEngineReads = {
@@ -167,6 +169,10 @@ type SessionRecord = {
idleFinal: AbortController | null;
/** Engine reads retried on their own while unresolved, and the retry waiting (see unresolved()). */
retries?: { count: number; timer: ReturnType<typeof setTimeout> | null };
+ /** Capture v2: the completed turns last counted (the turn in progress is the next one). */
+ turns?: number;
+ /** Capture v2: a prompt went out and this turn's first tool call has not been seen yet. */
+ awaitingTool?: boolean;
};
/** An unresolved session start reads the engine again after these waits, then only at its next prompt or turn. */
@@ -256,21 +262,77 @@ export class ProjectArchiveLifecycle {
if (known) {
known.engine = input.engine;
known.activeAt = promptedAt;
+ known.awaitingTool = true;
}
// Resolved, or its start step is still queued: nothing to add.
if (known && (!known.start || known.starting)) return;
// New, or unresolved because the engine could not be read: (re)try with this prompt's reads.
- const record: SessionRecord = known ?? { root: null, start: null, starting: false, engine: input.engine, activeAt: promptedAt, idleTimer: null, idleFinal: null };
+ const record: SessionRecord = known ?? { root: null, start: null, starting: false, engine: input.engine, activeAt: promptedAt, idleTimer: null, idleFinal: null, awaitingTool: true };
record.start = { root: input.root, engine: input.engine };
record.starting = true;
this.sessions.set(input.sessionId, record);
+ // Capture v2: a base whose start manifest was taken before the prompt packs at once (its files may change any moment).
+ const immediate = this.archiver.hasStartManifest?.(input.sessionId) === true;
this.schedule(input.sessionId, "base", async () => {
try {
if (this.sessions.get(input.sessionId) === record && record.start) await this.resolve(input.sessionId, record, null);
} finally {
record.starting = false;
}
- }, () => this.quietPeriod(promptedAt));
+ }, immediate ? undefined : () => this.quietPeriod(promptedAt));
+ }
+
+ /**
+ * Capture v2, #8: call before a prompt is sent to the engine, and send it
+ * once this resolves. The first prompt of a session in this run takes the
+ * start manifest (hash and stat only, at most START_GATE_MS); any later
+ * prompt resolves at once. Never rejects.
+ */
+ async startGate(sessionId: string, root: string): Promise<StartCapture | "skipped"> {
+ if (!this.active || this.consentOff || !this.archiver.startManifest) return "skipped";
+ const known = this.sessions.get(sessionId);
+ if (known && !known.start) return "skipped";
+ try {
+ return await this.archiver.startManifest(sessionId, root);
+ } catch (error) {
+ this.warn("start-manifest", error, sessionId);
+ return "skipped";
+ }
+ }
+
+ /**
+ * Capture v2, #9: a tool call of the session started (tool-start.ts). The
+ * first one of each turn queues a pre_tool state, captured when anything
+ * changed since the chain's last state; the tool itself is never held.
+ */
+ toolStarted(sessionId: string): void {
+ const record = this.sessions.get(sessionId);
+ if (!this.active || this.consentOff || !record || !record.awaitingTool || !this.archiver.captureState) return;
+ record.awaitingTool = false;
+ const capture = this.archiver.captureState.bind(this.archiver);
+ this.schedule(sessionId, "pre_tool", async () => {
+ if (this.sessions.get(sessionId) !== record || !record.root) return;
+ const result = await capture(sessionId, (record.turns ?? 0) + 1);
+ if (result.status === "skipped" && result.reason === "stopped") record.root = null;
+ this.settle(result);
+ });
+ }
+
+ /**
+ * Capture v2, #11: a settled turn's messages; the files attached to its
+ * user messages are staged for the turn's archive (queued before its delta).
+ */
+ turnMessages(sessionId: string, messages: unknown): void {
+ const record = this.sessions.get(sessionId);
+ if (!this.active || this.consentOff || !record || !this.archiver.recordAttachments) return;
+ const sources = attachmentsFromMessages(messages);
+ if (sources.length === 0) return;
+ const stage = this.archiver.recordAttachments.bind(this.archiver);
+ this.schedule(sessionId, "attachments", async () => {
+ if (this.sessions.get(sessionId) !== record) return;
+ const root = record.root ?? record.start?.root;
+ if (root) await stage(sessionId, root, sources);
+ });
}
/**
@@ -286,6 +348,7 @@ export class ProjectArchiveLifecycle {
const record = this.sessions.get(sessionId);
if (!this.active || this.consentOff || !record) return;
const turns = completedTurnCount(messages);
+ if (turns !== null) record.turns = turns;
this.schedule(sessionId, "delta", async () => {
if (this.sessions.get(sessionId) !== record) return;
if (record.start && !(await this.resolve(sessionId, record, turns))) return;
@@ -306,6 +369,13 @@ export class ProjectArchiveLifecycle {
this.archiver.recordTouched(sessionId, path);
}
+ /** Capture v2: a binary file the session uploader found in the session's workspace (workspace-relative). */
+ binaryFile(sessionId: string, path: string): void {
+ const record = this.sessions.get(sessionId);
+ if (!this.active || this.consentOff || !record || (!record.root && !record.start)) return;
+ this.archiver.recordBinary?.(sessionId, path);
+ }
+
/**
* The observer follows a turn of the session: it is not quiet. A turn that
* ended just before (its turnCompleted can come after the next prompt's
@@ -413,6 +483,7 @@ export class ProjectArchiveLifecycle {
}
const count = turns ?? completedTurnCount(await start.engine.messages().catch(() => null));
if (count === null) return this.unresolved(sessionId, record, "the engine messages could not be read");
+ record.turns ??= count;
record.start = null;
record.root = root;
const result = await this.archiver.captureBase(sessionId, root, count);
diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.ts
index bc8d7810..d069294d 100644
--- a/apps/server/src/session-archive/manifest.ts
+++ b/apps/server/src/session-archive/manifest.ts
@@ -9,18 +9,25 @@
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { constants as fsConstants, type BigIntStats } from "node:fs";
-import { lstat, open, readdir, readlink, realpath } from "node:fs/promises";
+import { lstat, open, readdir, readlink, realpath, stat } from "node:fs/promises";
import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";
-import { clampUploadBytes, isUploadPathDenied, stripRemoteUserinfo } from "../session-uploader.js";
+import { clampUploadBytes, isUploadPathDenied, REGENERABLE_DIR_NAMES, stripRemoteUserinfo } from "../session-uploader.js";
+import { ATTACHMENTS_ROOT_NAME } from "./attachments.js";
+import type { ExcludedList, ExcludedReason } from "./capture-v2.js";
import { gitSkipReason } from "../command-guard.js";
import { hintGarbageCollection } from "./files.js";
+import { isGitConfigPath, scrubGitConfig } from "./git-scrub.js";
import { ArchiveIgnore, gitCeilingDirectories, scopeIgnores, type IgnoreScope, type IgnoreSource } from "./ignore.js";
export const ARCHIVE_SCHEMA = "omnirush.archive.v1";
export const RESERVED_ROOT_NAME = "__omnirush__";
/** The archive folder of the files outside the workspace (outside.ts); a workspace entry of that name is reserved too. */
export const OUTSIDE_ROOT_NAME = "__outside__";
+/** Capture v2: prompt attachments (attachments.ts); a workspace entry of that name is reserved too. */
+export { ATTACHMENTS_ROOT_NAME };
+/** A git config file larger than this is archived as it is (no real config is): it is never read whole to scrub it. */
+const MAX_SCRUBBED_CONFIG_BYTES = 1024 * 1024;
export const STAT_CONCURRENCY = 64;
export const HASH_CONCURRENCY = 6;
export const HASH_READ_BYTES = 128 * 1024;
@@ -44,7 +51,7 @@ export type ArchiveKind = "base" | "delta";
* final archive of the folder after the last completed turn, which carries
* that turn's number again.
*/
-export type ArchiveTrigger = "turn" | "final";
+export type ArchiveTrigger = "turn" | "final" | "pre_tool";
/** What prompted a final archive (manifest.json `reason`). */
export type FinalReason = "idle" | "turn_incomplete" | "session_deleted" | "app_quit" | "app_start";
/**
@@ -63,6 +70,10 @@ export type ArchiveEntry = {
size: number;
sha256: string | null;
target?: string;
+ /** Capture v2: a symlink whose target does not exist. */
+ broken?: boolean;
+ /** A git config whose archived copy had credentials removed (git-scrub.ts): size and sha256 are the copy's. */
+ scrubbed?: boolean;
};
/** A scanned entry: the manifest fields plus what pass 2 and the hash cache need. */
@@ -73,6 +84,10 @@ export type ScannedEntry = ArchiveEntry & {
statKey: string;
/** st_dev from lstat: with the ino in statKey, the identity pass 2 requires of the file it reads. */
dev: number;
+ /** The bytes pass 2 writes instead of reading the file (a scrubbed git config). */
+ content?: Buffer;
+ /** An absolute file pass 2 reads instead of `<root>/<path>` (a staged attachment). */
+ source?: string;
};
/** The prefix of a stat key that pass 2 compares against fstat: size and mtimeNs. */
@@ -317,6 +332,8 @@ export function parseBaselineText(text: string | null): ArchiveEntry[] | null {
if (target !== undefined && typeof target !== "string") return null;
const entry: ArchiveEntry = { path, type: type === "file" ? "file" : type === "dir" ? "dir" : "symlink", mode, size, sha256 };
if (target !== undefined) entry.target = target;
+ if ("broken" in row && row.broken === true) entry.broken = true;
+ if ("scrubbed" in row && row.scrubbed === true) entry.scrubbed = true;
entries.push(entry);
}
return entries;
@@ -342,6 +359,10 @@ export type ScanOptions = {
metrics?: ScanMetrics;
/** Stops the scan between entries: it then rejects with the signal's reason. */
signal?: AbortSignal;
+ /** Capture v2: every entry left out, with its reason. */
+ excludedList?: ExcludedList;
+ /** Capture v2 (a folder in no repository): the regenerable folders (node_modules, .venv, target, ...) are left out. */
+ pruneRegenerable?: boolean;
};
/**
@@ -454,6 +475,58 @@ async function hashFile(absolute: string, size: number, buffer: Buffer, metrics:
}
}
+/** Whether a symlink's target is missing (or a loop): its own lstat already showed a link. */
+export async function isBrokenLink(absolute: string): Promise<boolean> {
+ try {
+ await stat(absolute);
+ return false;
+ } catch (error) {
+ const code = error instanceof Error && "code" in error ? error.code : undefined;
+ return code === "ENOENT" || code === "ELOOP" || code === "ENOTDIR";
+ }
+}
+
+/**
+ * A git config entry (git-scrub.ts): read once from the very file pass 1
+ * lstat'ed, scrubbed of credentials; when anything was removed, the entry
+ * carries the scrubbed bytes (pass 2 writes them) and their size and hash.
+ * False when it cannot be read.
+ */
+export async function scrubConfigEntry(absolute: string, entry: ScannedEntry, metrics: ScanMetrics): Promise<boolean> {
+ let handle: Awaited<ReturnType<typeof open>>;
+ try {
+ handle = await open(absolute, OPEN_ENTRY_FLAGS);
+ } catch {
+ return false;
+ }
+ try {
+ const stats = await handle.stat({ bigint: true });
+ if (!stats.isFile() || !isSameFileIdentity(entry, stats) || Number(stats.size) !== entry.size) return false;
+ const raw = Buffer.alloc(entry.size);
+ let offset = 0;
+ while (offset < raw.length) {
+ const { bytesRead } = await handle.read(raw, offset, raw.length - offset, offset);
+ if (bytesRead === 0) return false;
+ offset += bytesRead;
+ }
+ metrics.fileReads += 1;
+ metrics.bytesHashed += raw.length;
+ const scrub = scrubGitConfig(raw.toString("utf8"));
+ const bytes = scrub.changed ? Buffer.from(scrub.text, "utf8") : raw;
+ if (scrub.changed) {
+ entry.content = bytes;
+ entry.size = bytes.length;
+ entry.scrubbed = true;
+ }
+ entry.sha256 = createHash("sha256").update(bytes).digest("hex");
+ return true;
+ } catch {
+ return false;
+ } finally {
+ await handle.close().catch(() => undefined);
+ }
+}
+
/**
* Pass 1: walks the root with lstat (never following a symlink), applies the
* exclusions of section 5.2, leaves out what git ignores (a whole ignored
@@ -471,6 +544,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
const entries: ScannedEntry[] = [];
const toHash: ScannedEntry[] = [];
const ignore = new ArchiveIgnore(resolve(root), signal);
+ const list = options.excludedList;
+ const note = (path: string, reason: ExcludedReason, type: "file" | "dir", size?: number): void => {
+ list?.add({ path, type, reason, ...(size !== undefined ? { size } : {}) });
+ };
const inspect = async (relDir: string, absDir: string, name: string, scope: IgnoreScope, next: Array<{ rel: string; abs: string; scope: IgnoreScope }>): Promise<void> => {
signal?.throwIfAborted();
@@ -479,6 +556,11 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
// Git's answer names ignored directories and files alike: no lstat needed.
if (scope.kind === "git" && scopeIgnores(scope, rel, false)) {
ignore.ignored += 1;
+ if (list) {
+ // One lstat for the listing's type (an ignored folder is never walked).
+ const kind = await lstat(abs).then((info) => (info.isDirectory() ? "dir" as const : "file" as const), () => "file" as const);
+ note(rel, "gitignored", kind);
+ }
return;
}
let stats: BigIntStats;
@@ -488,15 +570,23 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
if (metrics.stats % GC_HINT_ENTRIES === 0) hintGarbageCollection();
} catch {
excluded.unreadable += 1;
+ note(rel, "unreadable", "file");
return;
}
if (scope.kind === "rules" && scopeIgnores(scope, rel, stats.isDirectory())) {
ignore.ignored += 1;
+ note(rel, "gitignored", stats.isDirectory() ? "dir" : "file");
return;
}
if (stats.isDirectory()) {
if (prunedDirs.has(rel)) {
excluded.app_state += 1;
+ note(rel, "app_state", "dir");
+ return;
+ }
+ if (options.pruneRegenerable && REGENERABLE_DIR_NAMES.has(name)) {
+ ignore.ignored += 1;
+ note(rel, "regenerable", "dir");
return;
}
entries.push({ path: rel, type: "dir", size: 0, sha256: null, ...statFields(stats) });
@@ -505,10 +595,12 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
}
if (!stats.isFile() && !stats.isSymbolicLink()) {
excluded.special += 1;
+ note(rel, "special", "file");
return;
}
if (!includeCredentials && isArchiveCredentialPath(rel)) {
excluded.credential += 1;
+ note(rel, "credential", "file");
return;
}
if (stats.isSymbolicLink()) {
@@ -517,16 +609,26 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
target = decodeName(await readlink(abs, { encoding: "buffer" }));
} catch {
excluded.unreadable += 1;
+ note(rel, "unreadable", "file");
return;
}
if (target === null) {
excluded.non_utf8 += 1;
+ note(rel, "non_utf8", "file");
return;
}
- entries.push({ path: rel, type: "symlink", size: 0, sha256: null, target, ...statFields(stats) });
+ const link: ScannedEntry = { path: rel, type: "symlink", size: 0, sha256: null, target, ...statFields(stats) };
+ if (list && (await isBrokenLink(abs))) link.broken = true;
+ entries.push(link);
return;
}
const entry: ScannedEntry = { path: rel, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };
+ if (isGitConfigPath(rel) && entry.size <= MAX_SCRUBBED_CONFIG_BYTES) {
+ // Read and scrubbed in the hash pass; never from the cache (its bytes are needed for pass 2).
+ toHash.push(entry);
+ entries.push(entry);
+ return;
+ }
const cached = cache?.lookup(entry) ?? null;
if (cached) {
entry.sha256 = cached;
@@ -562,7 +664,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
for (const bytes of raw) {
const name = decodeName(bytes);
if (name === null) excluded.non_utf8 += 1;
- else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME)) excluded.reserved += 1;
+ else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME || name === ATTACHMENTS_ROOT_NAME)) {
+ excluded.reserved += 1;
+ note(name, "reserved", "dir");
+ }
else {
listed.push(name);
if (name.toLowerCase() === ".git") hasGit = true;
@@ -586,6 +691,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
const buffers: Buffer[] = [];
await forEachBounded(toHash, options.hashConcurrency ?? HASH_CONCURRENCY, async (entry) => {
signal?.throwIfAborted();
+ if (isGitConfigPath(entry.path) && entry.size <= MAX_SCRUBBED_CONFIG_BYTES) {
+ if (!(await scrubConfigEntry(join(root, ...entry.path.split("/")), entry, metrics))) unreadable.add(entry);
+ return;
+ }
const buffer = buffers.pop() ?? Buffer.allocUnsafe(HASH_READ_BYTES);
try {
const sha256 = await hashFile(join(root, ...entry.path.split("/")), entry.size, buffer, metrics, signal);
@@ -602,6 +711,7 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
});
signal?.throwIfAborted();
excluded.unreadable += unreadable.size;
+ for (const entry of unreadable) note(entry.path, "unreadable", "file", entry.size);
const kept = unreadable.size > 0 ? entries.filter((entry) => !unreadable.has(entry)) : entries;
kept.sort((left, right) => compareArchivePaths(left.path, right.path));
cache?.commitScan();
@@ -669,10 +779,14 @@ export function isArchiveDeltaEmpty(delta: ArchiveDelta<ArchiveEntry>): boolean
export function archiveEntry(entry: ArchiveEntry): ArchiveEntry {
const out: ArchiveEntry = { path: entry.path, type: entry.type, mode: entry.mode, size: entry.size, sha256: entry.sha256 };
if (entry.type === "symlink") out.target = entry.target ?? "";
+ if (entry.type === "symlink" && entry.broken) out.broken = true;
+ if (entry.type === "file" && entry.scrubbed) out.scrubbed = true;
return out;
}
export type ManifestInput = {
+ /** Capture v2 chains: ARCHIVE_SCHEMA_V2 (capture-v2.ts). */
+ schema?: string;
kind: ArchiveKind;
archiveId: string;
sessionId: string;
@@ -704,7 +818,7 @@ export type ManifestSource = { size: number; chunks: () => Iterable<Buffer> };
*/
export function manifestSource(input: ManifestInput): ManifestSource {
const head = JSON.stringify({
- schema: ARCHIVE_SCHEMA,
+ schema: input.schema ?? ARCHIVE_SCHEMA,
kind: input.kind,
archive_id: input.archiveId,
session_id: input.sessionId,
diff --git a/apps/server/src/session-archive/pack.ts b/apps/server/src/session-archive/pack.ts
index 5b26ca3b..640a2358 100644
--- a/apps/server/src/session-archive/pack.ts
+++ b/apps/server/src/session-archive/pack.ts
@@ -136,6 +136,8 @@ export type PackInput = {
root: string;
/** `__omnirush__/manifest.json`: its bytes, or a source that serialises it while it is written. */
manifest: Buffer | ManifestSource;
+ /** Capture v2: members written right after the manifest (`__omnirush__/state.json`). */
+ extraMembers?: ReadonlyArray<{ name: string; content: Buffer }>;
/** created_at in seconds: the mtime of the manifest and unstable members. */
createdAtSeconds: number;
/** Exactly the manifest's `files`, in the same order. */
@@ -311,6 +313,7 @@ class TarWriter {
async write(): Promise<void> {
const { input, unstable } = this;
await this.smallFile(MANIFEST_MEMBER, input.manifest);
+ for (const member of input.extraMembers ?? []) await this.smallFile(member.name, member.content);
let members = 0;
for (const entry of input.entries) {
input.signal?.throwIfAborted();
@@ -320,7 +323,22 @@ class TarWriter {
await this.put(tarMemberHeader({ name: `${entry.path}/`, mode: entry.mode, size: 0, mtime: entry.mtime, typeflag: "5" }));
continue;
}
+ if (entry.type === "file" && entry.content) {
+ // Bytes prepared in pass 1 (a scrubbed git config).
+ await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.content.length, mtime: entry.mtime, typeflag: "0" }));
+ await this.put(entry.content);
+ await this.zeros(paddingFor(entry.content.length));
+ continue;
+ }
const parts = entry.path.split("/");
+ if (entry.type === "file" && entry.source) {
+ // A staged file (an attachment) read from its own absolute path under the state dir.
+ await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.size, mtime: entry.mtime, typeflag: "0" }));
+ const stable = await this.fileContent(entry, entry.source, true);
+ await this.zeros(paddingFor(entry.size));
+ if (!stable) unstable.push(entry.path);
+ continue;
+ }
// A file outside the workspace (`__outside__/...`) is read from its own absolute path.
const outside = isOutsideArchivePath(entry.path);
const source = outside ? outsideSourcePath(entry.path) : null;
diff --git a/apps/server/src/session-archive/policy.ts b/apps/server/src/session-archive/policy.ts
index b9f8fc38..d36991d6 100644
--- a/apps/server/src/session-archive/policy.ts
+++ b/apps/server/src/session-archive/policy.ts
@@ -8,14 +8,21 @@
*/
import { z } from "zod";
-export type ArchivePolicy = { allFolders: boolean; touchedFiles: boolean };
+export type ArchivePolicy = { allFolders: boolean; touchedFiles: boolean; captureV2?: boolean };
/** Both off: what a failed probe, a missing policy or a disabled account means. */
export const POLICY_OFF: ArchivePolicy = { allFolders: false, touchedFiles: false };
const allFoldersSchema = z.object({ policy: z.object({ all_folders: z.literal(true) }) });
const touchedFilesSchema = z.object({ policy: z.object({ touched_files: z.literal(true) }) });
+/** Capture v2 (capture-v2.ts): the server takes the byte-exact state archives. */
+const captureV2Schema = z.object({ policy: z.object({ capture_v2: z.literal(true) }) });
export function parseArchivePolicy(body: unknown): ArchivePolicy {
- return { allFolders: allFoldersSchema.safeParse(body).success, touchedFiles: touchedFilesSchema.safeParse(body).success };
+ return {
+ allFolders: allFoldersSchema.safeParse(body).success,
+ touchedFiles: touchedFilesSchema.safeParse(body).success,
+ // Present only when on: a policy without it reads exactly as before.
+ ...(captureV2Schema.safeParse(body).success ? { captureV2: true } : {}),
+ };
}
diff --git a/apps/server/src/session-archive/repos.ts b/apps/server/src/session-archive/repos.ts
new file mode 100644
index 00000000..037617e9
--- /dev/null
+++ b/apps/server/src/session-archive/repos.ts
@@ -0,0 +1,371 @@
+/**
+ * Capture v2: the git state of every repository a session works in.
+ *
+ * - the root's own repository (`position: "root"`);
+ * - repositories in subfolders (`"nested"`): a folder holding `.git` (a
+ * directory, or a gitfile: a submodule or a linked worktree) found by
+ * the archive scan;
+ * - a repository whose work tree starts above the root (`"above"`): the
+ * root is a folder inside it; its relative position is recorded
+ * (`path: "../.."`), its `.git` is never archived (it is outside the
+ * project folder), only this metadata.
+ *
+ * Each gets HEAD, branch (also on an unborn branch: a repository without
+ * commits), the remote without userinfo, the tracked-file count, the dirty
+ * files with their status, the staged files and the stash count; or, when
+ * git cannot read it, why (`error`).
+ *
+ * Why the old git blocks came back empty while the agent's own git commands
+ * worked, and what this runner does about each:
+ * - `dubious ownership` (a folder owned by another user: a container
+ * running as root on a mounted checkout, a shared drive): git refuses
+ * the repository. The capture retries once with `safe.directory` set to
+ * that repository only, with fsmonitor and hooks off, and records it;
+ * - git environment inherited from the launching process (`GIT_DIR`,
+ * `GIT_WORK_TREE`, `GIT_INDEX_FILE`, ... when started from a git hook or
+ * `git rebase -x`) pointing git somewhere else: removed;
+ * - short timeouts under load (5 s for `rev-parse` while a full scan
+ * hashes the tree): 20 s here, and a timeout is reported, not hidden;
+ * - a repository created by the agent (`git init`, `git clone`) after the
+ * session started, or in a subfolder: found again at every state;
+ * - a plain-folder or touched-files chain never read git at all: every
+ * v2 state reads the repositories, whatever the chain's marker.
+ */
+import { spawn } from "node:child_process";
+import { realpathSync } from "node:fs";
+import { lstat, readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+
+import { gitSkipReason } from "../command-guard.js";
+
+export type RepoPosition = "root" | "nested" | "above";
+export type RepoError = "timeout" | "dubious_ownership" | "not_a_repository" | "git_missing" | "failed";
+
+export type RepoState = {
+ path: string;
+ position: RepoPosition;
+ git_dir: string | null;
+ gitfile: boolean;
+ head: string | null;
+ branch: string | null;
+ unborn: boolean;
+ remote: string | null;
+ tracked_files: number | null;
+ dirty: boolean;
+ dirty_files: Array<{ path: string; status: string }>;
+ dirty_truncated: boolean;
+ staged_files: string[];
+ stash_count: number;
+ archived: "byte_exact" | "metadata_only";
+ safe_directory_override?: true;
+ error: RepoError | null;
+};
+
+export const MAX_REPOS = 64;
+const MAX_DIRTY_LISTED = 500;
+/** One repository item of state.json stays far below the reader's 1 MiB item cap (backend spec 19.3). */
+const MAX_LISTED_PATH_BYTES = 128 * 1024;
+
+/** The first items of `items` whose paths fit `MAX_LISTED_PATH_BYTES` (and MAX_DIRTY_LISTED), and whether any was left out. */
+function boundedList<T>(items: readonly T[], pathOf: (item: T) => string): { kept: T[]; truncated: boolean } {
+ const kept: T[] = [];
+ let bytes = 0;
+ for (const item of items) {
+ bytes += Buffer.byteLength(pathOf(item)) + 32;
+ if (kept.length >= MAX_DIRTY_LISTED || bytes > MAX_LISTED_PATH_BYTES) return { kept, truncated: true };
+ kept.push(item);
+ }
+ return { kept, truncated: false };
+}
+const GIT_TIMEOUT_MS = 20_000;
+/** Variables that redirect git away from the folder it is run in. */
+const REDIRECTING_GIT_ENV = [
+ "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES",
+ "GIT_COMMON_DIR", "GIT_NAMESPACE", "GIT_PREFIX", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT",
+ "GIT_DISCOVERY_ACROSS_FILESYSTEM", "GIT_EXEC_PATH",
+];
+
+/** The environment capture git runs with: no redirecting variables, no prompts, no optional locks, C locale. */
+export function captureGitEnv(base: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
+ const env: NodeJS.ProcessEnv = { ...base };
+ for (const name of REDIRECTING_GIT_ENV) delete env[name];
+ for (const name of Object.keys(env)) if (/^GIT_CONFIG_(?:KEY|VALUE)_\d+$/.test(name)) delete env[name];
+ let home = homedir();
+ try {
+ home = realpathSync(home);
+ } catch {
+ // As given.
+ }
+ const ceiling = base.GIT_CEILING_DIRECTORIES;
+ env.GIT_CEILING_DIRECTORIES = home ? (ceiling ? `${ceiling}${delimiter}${home}` : home) : ceiling ?? "";
+ env.GIT_TERMINAL_PROMPT = "0";
+ env.GIT_OPTIONAL_LOCKS = "0";
+ env.GIT_PAGER = "cat";
+ env.PAGER = "cat";
+ env.LC_ALL = "C";
+ return env;
+}
+
+export type RepoGitRun = { ok: boolean; stdout: Buffer; stderr: string; timedOut: boolean; missing: boolean; truncated: boolean };
+
+/** One git run in `cwd`, read-only settings, bounded output; never throws. `onChunk` streams stdout instead of keeping it. */
+export async function runRepoGit(
+ cwd: string,
+ args: string[],
+ options: { maxBytes?: number; timeoutMs?: number; safeDirectory?: string; onChunk?: (chunk: Buffer) => void } = {},
+): Promise<RepoGitRun> {
+ // A Mac without the Command Line Tools: the git shim would open the installer (command-guard.ts).
+ if (await gitSkipReason()) return { ok: false, stdout: Buffer.alloc(0), stderr: "", timedOut: false, missing: true, truncated: false };
+ return spawnRepoGit(cwd, args, options);
+}
+
+function spawnRepoGit(
+ cwd: string,
+ args: string[],
+ options: { maxBytes?: number; timeoutMs?: number; safeDirectory?: string; onChunk?: (chunk: Buffer) => void },
+): Promise<RepoGitRun> {
+ const maxBytes = options.maxBytes ?? 256 * 1024;
+ return new Promise((resolvePromise) => {
+ const out: Buffer[] = [];
+ const err: Buffer[] = [];
+ let total = 0;
+ let errTotal = 0;
+ let truncated = false;
+ let timedOut = false;
+ let settled = false;
+ let child: ReturnType<typeof spawn> | null = null;
+ const finish = (ok: boolean, missing = false) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ resolvePromise({ ok, stdout: Buffer.concat(out), stderr: Buffer.concat(err).toString("utf8"), timedOut, missing, truncated });
+ };
+ const timer = setTimeout(() => {
+ timedOut = true;
+ child?.kill("SIGKILL");
+ finish(false);
+ }, options.timeoutMs ?? GIT_TIMEOUT_MS);
+ timer.unref?.();
+ const config = [
+ "-c", "core.fsmonitor=false", "-c", "core.quotePath=false", "-c", `core.hooksPath=${process.platform === "win32" ? "NUL" : "/dev/null"}`,
+ ...(options.safeDirectory ? ["-c", `safe.directory=${options.safeDirectory}`] : []),
+ ];
+ try {
+ child = spawn("git", ["-C", cwd, ...config, ...args], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, env: captureGitEnv() });
+ } catch {
+ finish(false, true);
+ return;
+ }
+ child.stdout?.on("data", (chunk: Buffer) => {
+ if (options.onChunk) {
+ options.onChunk(chunk);
+ return;
+ }
+ if (truncated) return;
+ const room = maxBytes - total;
+ if (chunk.length > room) {
+ out.push(chunk.subarray(0, room));
+ total = maxBytes;
+ truncated = true;
+ child?.kill("SIGKILL");
+ return;
+ }
+ out.push(chunk);
+ total += chunk.length;
+ });
+ child.stderr?.on("data", (chunk: Buffer) => {
+ if (errTotal < 16 * 1024) {
+ err.push(chunk);
+ errTotal += chunk.length;
+ }
+ });
+ child.on("error", (error: NodeJS.ErrnoException) => finish(false, error.code === "ENOENT"));
+ child.on("close", (code) => finish(!timedOut && (truncated || code === 0)));
+ });
+}
+
+function errorOf(run: RepoGitRun): RepoError {
+ if (run.missing) return "git_missing";
+ if (run.timedOut) return "timeout";
+ if (/dubious ownership|safe\.directory/i.test(run.stderr)) return "dubious_ownership";
+ if (/not a git repository/i.test(run.stderr)) return "not_a_repository";
+ return "failed";
+}
+
+function portable(path: string): string {
+ return sep === "/" ? path : path.split(sep).join("/");
+}
+
+/** Strips `user:password@` from a remote URL (scheme form) and the password from an scp form. */
+export function stripRemoteCredentials(url: string): string {
+ const trimmed = url.trim();
+ const scheme = /^([a-z][a-z0-9+.-]*:\/\/)(?:[^/@]*@)?(.*)$/i.exec(trimmed);
+ if (scheme) return `${scheme[1]}${scheme[2]}`;
+ const scp = /^([^/@:\s]+)(?::[^@/]*)?@([^/:\s]+:.*)$/.exec(trimmed);
+ if (scp) return `${scp[1]}@${scp[2]}`;
+ return trimmed;
+}
+
+type Located = { top: string; gitDir: string; safeDirectory?: string } | { error: RepoError };
+
+async function locate(dir: string): Promise<Located> {
+ const args = ["rev-parse", "--show-toplevel", "--absolute-git-dir"];
+ let run = await runRepoGit(dir, args, { maxBytes: 64 * 1024 });
+ let safeDirectory: string | undefined;
+ if (!run.ok && errorOf(run) === "dubious_ownership") {
+ // Only this repository, never `*`: the user's own folder the agent works in.
+ const match = /repository at '([^']+)'/.exec(run.stderr);
+ safeDirectory = match?.[1] ?? dir;
+ run = await runRepoGit(dir, args, { maxBytes: 64 * 1024, safeDirectory });
+ }
+ if (!run.ok) return { error: errorOf(run) };
+ const [top, gitDir] = run.stdout.toString("utf8").split(/\r?\n/);
+ if (!top || !gitDir) return { error: "not_a_repository" };
+ return { top, gitDir, ...(safeDirectory ? { safeDirectory } : {}) };
+}
+
+function within(root: string, path: string): string | null {
+ const rel = relative(root, path);
+ if (rel === "") return "";
+ if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;
+ return portable(rel);
+}
+
+/** Parses `git status --porcelain=v1 -z` into dirty and staged files. */
+export function parsePorcelainZ(text: string): { dirty: Array<{ path: string; status: string }>; staged: string[] } {
+ const dirty: Array<{ path: string; status: string }> = [];
+ const staged: string[] = [];
+ const parts = text.split("\0");
+ for (let index = 0; index < parts.length; index += 1) {
+ const item = parts[index]!;
+ if (item.length < 4) continue;
+ const status = item.slice(0, 2);
+ const path = item.slice(3);
+ dirty.push({ path, status });
+ if (status[0] !== " " && status[0] !== "?" && status[0] !== "!") staged.push(path);
+ // A rename or copy is followed by its source path.
+ if (status[0] === "R" || status[0] === "C") index += 1;
+ }
+ return { dirty, staged };
+}
+
+/** The state of the repository whose work tree is `top`, as seen from `root`. */
+export async function readRepoState(root: string, located: { top: string; gitDir: string; safeDirectory?: string }, position: RepoPosition, gitfile: boolean): Promise<RepoState> {
+ const { top, gitDir, safeDirectory } = located;
+ const opt = (extra: { maxBytes?: number; timeoutMs?: number } = {}) => ({ ...extra, ...(safeDirectory ? { safeDirectory } : {}) });
+ let tracked = 0;
+ const [head, branch, remotes, status, lsFiles, stash] = await Promise.all([
+ runRepoGit(top, ["rev-parse", "--verify", "-q", "HEAD"], opt()),
+ runRepoGit(top, ["symbolic-ref", "--short", "-q", "HEAD"], opt()),
+ runRepoGit(top, ["remote"], opt()),
+ runRepoGit(top, ["status", "--porcelain=v1", "-z", "--no-renames", "--untracked-files=normal"], opt({ maxBytes: 4 * 1024 * 1024 })),
+ runRepoGit(top, ["ls-files", "-z"], {
+ ...opt(),
+ onChunk: (chunk) => {
+ for (const byte of chunk) if (byte === 0) tracked += 1;
+ },
+ }),
+ runRepoGit(top, ["rev-list", "--walk-reflogs", "--count", "refs/stash"], opt()),
+ ]);
+ const headText = head.ok ? head.stdout.toString("utf8").trim() : "";
+ const branchText = branch.ok ? branch.stdout.toString("utf8").trim() : "";
+ const names = remotes.ok ? remotes.stdout.toString("utf8").split(/\r?\n/).map((name) => name.trim()).filter(Boolean) : [];
+ const remoteName = names.includes("origin") ? "origin" : names[0];
+ const remoteUrl = remoteName ? await runRepoGit(top, ["remote", "get-url", remoteName], opt()) : null;
+ const parsed = status.ok ? parsePorcelainZ(status.stdout.toString("utf8")) : { dirty: [], staged: [] };
+ const dirtyListed = boundedList(parsed.dirty, (item) => item.path);
+ const stagedListed = boundedList(parsed.staged, (path) => path);
+ const relTop = relative(root, top);
+ const insideRoot = within(root, gitDir) !== null;
+ const failed = [status, lsFiles].find((run) => !run.ok && !run.truncated);
+ return {
+ path: portable(relTop),
+ position,
+ git_dir: insideRoot ? within(root, gitDir) : null,
+ gitfile,
+ head: /^[0-9a-f]{40,64}$/.test(headText) ? headText : null,
+ branch: branchText || null,
+ unborn: !/^[0-9a-f]{40,64}$/.test(headText) && Boolean(branchText),
+ remote: remoteUrl?.ok ? stripRemoteCredentials(remoteUrl.stdout.toString("utf8").trim()) || null : null,
+ tracked_files: lsFiles.ok ? tracked : null,
+ dirty: parsed.dirty.length > 0,
+ dirty_files: dirtyListed.kept,
+ dirty_truncated: dirtyListed.truncated || stagedListed.truncated || status.truncated,
+ staged_files: stagedListed.kept,
+ stash_count: stash.ok ? Number.parseInt(stash.stdout.toString("utf8").trim(), 10) || 0 : 0,
+ archived: position !== "above" && insideRoot ? "byte_exact" : "metadata_only",
+ ...(safeDirectory ? { safe_directory_override: true as const } : {}),
+ error: failed ? errorOf(failed) : null,
+ };
+}
+
+/** A repository marker git could not read: recorded with the reason. */
+function unreadable(root: string, dir: string, position: RepoPosition, gitfile: boolean, error: RepoError): RepoState {
+ return {
+ path: portable(relative(root, dir)), position, git_dir: null, gitfile, head: null, branch: null, unborn: false, remote: null,
+ tracked_files: null, dirty: false, dirty_files: [], dirty_truncated: false, staged_files: [], stash_count: 0, archived: "metadata_only", error,
+ };
+}
+
+/**
+ * Every repository of the session: `markers` are the root-relative paths of
+ * the `.git` entries the archive scan found (`.git`, `sub/x/.git`), with
+ * whether each is a gitfile. The root's enclosing repository (above the
+ * root) is found by git itself, never above home.
+ */
+export async function discoverRepos(root: string, markers: ReadonlyArray<{ path: string; gitfile: boolean }>): Promise<RepoState[]> {
+ const absRoot = resolve(root);
+ const repos: RepoState[] = [];
+ const seenTops = new Set<string>();
+ const ordered = [...markers].sort((left, right) => left.path.split("/").length - right.path.split("/").length).slice(0, MAX_REPOS);
+ const hasRootMarker = ordered.some((marker) => marker.path === ".git");
+ if (!hasRootMarker) {
+ // The root inside a larger repository (or none).
+ const located = await locate(absRoot);
+ if ("top" in located) {
+ const rel = relative(absRoot, located.top);
+ if (rel !== "" && (rel === ".." || rel.startsWith(`..${sep}`))) {
+ seenTops.add(located.top);
+ repos.push(await readRepoState(absRoot, located, "above", false));
+ }
+ } else if (located.error === "dubious_ownership" || located.error === "timeout" || located.error === "git_missing") {
+ repos.push(unreadable(absRoot, absRoot, "above", false, located.error));
+ }
+ }
+ for (const marker of ordered) {
+ const dir = marker.path === ".git" ? absRoot : join(absRoot, ...marker.path.split("/").slice(0, -1));
+ const position: RepoPosition = marker.path === ".git" ? "root" : "nested";
+ const located = await locate(dir);
+ if ("error" in located) {
+ repos.push(unreadable(absRoot, dir, position, marker.gitfile, located.error));
+ continue;
+ }
+ // `.git` folders of a repository's own internals or a broken marker resolving elsewhere are not repositories of their own.
+ if (resolve(located.top) !== resolve(dir) && realpathOr(located.top) !== realpathOr(dir)) continue;
+ if (seenTops.has(located.top)) continue;
+ seenTops.add(located.top);
+ repos.push(await readRepoState(absRoot, located, position, marker.gitfile));
+ }
+ return repos;
+}
+
+function realpathOr(path: string): string {
+ try {
+ return realpathSync(path);
+ } catch {
+ return resolve(path);
+ }
+}
+
+/** Whether `path` (absolute) holds a `.git` gitfile, and where it points. */
+export async function readGitfile(path: string): Promise<string | null> {
+ try {
+ const info = await lstat(path);
+ if (!info.isFile() || info.size > 4096) return null;
+ const match = /^gitdir:\s*(.+?)\s*$/m.exec(await readFile(path, "utf8"));
+ return match ? resolve(dirname(path), match[1]!) : null;
+ } catch {
+ return null;
+ }
+}
diff --git a/apps/server/src/session-archive/tool-start.ts b/apps/server/src/session-archive/tool-start.ts
new file mode 100644
index 00000000..4ea14a78
--- /dev/null
+++ b/apps/server/src/session-archive/tool-start.ts
@@ -0,0 +1,115 @@
+/**
+ * Capture v2: the moment a turn's first tool call starts. While the observer
+ * follows a turn it listens to the engine's event stream (v1 `/event`, v2
+ * `/api/event`) and calls `onToolStart` once, at the first tool part of that
+ * session that is pending or running (v1 `message.part.updated` with a
+ * `tool` part, v2 `session.tool.input.started` / `session.tool.called`).
+ * The capture it triggers runs in the background: the tool is never held.
+ * Identical in the CLI and the desktop app.
+ */
+
+export type ToolStartFetch = (url: string, init: { headers: Headers; signal: AbortSignal }) => Promise<Response>;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+const V2_TOOL_EVENTS = /^session\.(?:next\.)?tool\.(?:input\.started|called)$/;
+
+/** The session id of a tool-start event, or null when `payload` is not one. */
+export function toolStartSession(payload: unknown): string | null {
+ if (!isRecord(payload)) return null;
+ // /global/event wraps each event as { directory, payload }.
+ if (isRecord(payload.payload) && typeof payload.payload.type === "string") return toolStartSession(payload.payload);
+ const type = typeof payload.type === "string" ? payload.type : "";
+ if (type === "message.part.updated") {
+ const properties = isRecord(payload.properties) ? payload.properties : null;
+ const part = properties && isRecord(properties.part) ? properties.part : null;
+ if (!part || part.type !== "tool") return null;
+ const status = isRecord(part.state) ? part.state.status : undefined;
+ if (status !== "pending" && status !== "running") return null;
+ return typeof part.sessionID === "string" ? part.sessionID : null;
+ }
+ if (V2_TOOL_EVENTS.test(type)) {
+ const data = isRecord(payload.data) ? payload.data : isRecord(payload.properties) ? payload.properties : null;
+ return data && typeof data.sessionID === "string" ? data.sessionID : null;
+ }
+ return null;
+}
+
+/** Splits SSE text into the `data:` payloads of complete frames; returns the unfinished rest. */
+export function sseFrames(buffer: string): { payloads: string[]; rest: string } {
+ const payloads: string[] = [];
+ let rest = buffer;
+ for (;;) {
+ const match = /\r?\n\r?\n/.exec(rest);
+ if (!match) break;
+ const frame = rest.slice(0, match.index);
+ rest = rest.slice(match.index + match[0].length);
+ const data = frame.split(/\r?\n/).filter((line) => line.startsWith("data:")).map((line) => line.slice(5).replace(/^ /, "")).join("\n");
+ if (data) payloads.push(data);
+ }
+ return { payloads, rest };
+}
+
+const MAX_FRAME_CHARS = 4 * 1024 * 1024;
+
+/**
+ * Listens until `signal` aborts or the first tool start of `sessionId`
+ * (then `onToolStart` runs once and the stream closes). Never throws; a
+ * stream that cannot be opened is simply no hook (the turn's end capture
+ * still runs).
+ */
+export async function watchToolStarts(input: {
+ url: string;
+ headers: Headers;
+ sessionId: string;
+ signal: AbortSignal;
+ fetch: ToolStartFetch;
+ onToolStart: () => void;
+}): Promise<void> {
+ const controller = new AbortController();
+ const stop = () => controller.abort();
+ if (input.signal.aborted) return;
+ input.signal.addEventListener("abort", stop, { once: true });
+ try {
+ const headers = new Headers(input.headers);
+ headers.set("accept", "text/event-stream");
+ const response = await input.fetch(input.url, { headers, signal: controller.signal });
+ if (!response.ok || !response.body) {
+ await response.body?.cancel().catch(() => undefined);
+ return;
+ }
+ const reader = response.body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = "";
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) return;
+ buffer += decoder.decode(value, { stream: true });
+ const { payloads, rest } = sseFrames(buffer);
+ buffer = rest.length > MAX_FRAME_CHARS ? "" : rest;
+ for (const text of payloads) {
+ let payload: unknown;
+ try {
+ payload = JSON.parse(text);
+ } catch {
+ continue;
+ }
+ if (toolStartSession(payload) === input.sessionId) {
+ try {
+ input.onToolStart();
+ } finally {
+ controller.abort();
+ }
+ await reader.cancel().catch(() => undefined);
+ return;
+ }
+ }
+ }
+ } catch {
+ // Aborted, or the engine went away: no hook for this turn.
+ } finally {
+ input.signal.removeEventListener("abort", stop);
+ }
+}
diff --git a/apps/server/src/session-archive/touched.ts b/apps/server/src/session-archive/touched.ts
index 2867ca2f..69b8f284 100644
--- a/apps/server/src/session-archive/touched.ts
+++ b/apps/server/src/session-archive/touched.ts
@@ -132,10 +132,13 @@ const LOCKFILE_NAMES = new Set([
"Cargo.lock", "poetry.lock", "Pipfile.lock", "uv.lock", "pdm.lock", "pixi.lock", "conda-lock.yml",
"Gemfile.lock", "composer.lock", "go.sum", "go.work.sum", "mix.lock", "pubspec.lock", "Package.resolved",
"packages.lock.json", "gradle.lockfile", "flake.lock", "Podfile.lock", "Cartfile.resolved",
+ // Capture v2 (#4): more ecosystems' pins.
+ "renv.lock", "Manifest.toml", "paket.lock", "stack.yaml.lock", "cabal.project.freeze", "shard.lock",
+ "requirements.lock", "requirements-dev.lock", "vcpkg-lock.json", "MODULE.bazel.lock",
]);
-const LOCKFILE_MAX_DEPTH = 6;
-const LOCKFILE_MAX_DIRS = 5_000;
-const LOCKFILE_MAX_FOUND = 200;
+const LOCKFILE_MAX_DEPTH = 8;
+const LOCKFILE_MAX_DIRS = 20_000;
+const LOCKFILE_MAX_FOUND = 1_000;
/**
* The lockfiles under `root`, workspace-relative and portable: a bounded walk
diff --git a/apps/server/src/session-archive/turn-files.ts b/apps/server/src/session-archive/turn-files.ts
index bf9ba6cd..b489571f 100644
--- a/apps/server/src/session-archive/turn-files.ts
+++ b/apps/server/src/session-archive/turn-files.ts
@@ -292,9 +292,16 @@ export async function recordTurnFiles(input: TurnFilesInput): Promise<TurnFilesR
let recordPath: string;
if (rel !== null) {
// Inside the workspace: the turn diff and the workspace artifacts carry what it wrote; reads are recorded.
- if (access === "write") continue;
if (!context.includeCredentialFiles && isArchiveCredentialPath(rel)) continue;
if (context.appDirs.some((dir) => within(dir, absolute) !== null)) continue;
+ // Capture v2 (#14): a file the turn read, wrote or named in a command counts as touched, so a
+ // gitignored one (a dataset, a generated output, a venv's tool) is archived byte for byte.
+ try {
+ input.archive?.pathTouched(input.sessionId, rel);
+ } catch {
+ // The archive's bookkeeping never stops a capture.
+ }
+ if (access === "write") continue;
recordPath = rel;
} else {
const archivePath = outsideArchivePath(absolute);
diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.ts
index 25bbc9e8..5a33df45 100644
--- a/apps/server/src/session-upload-observer.ts
+++ b/apps/server/src/session-upload-observer.ts
@@ -11,6 +11,7 @@ import { loopbackFetch } from "./server-fetch.js";
import { isFinishedAssistantMessage, isPromptMessage, type ArchiveEngineReads, type ProjectArchiveLifecycle } from "./session-archive/lifecycle.js";
import { MAX_UPLOAD_CHILD_SESSION_DEPTH, type UploadSessionModel, type SessionUploader } from "./session-uploader.js";
import { recordTurnFiles, type TurnFilesInput } from "./session-archive/turn-files.js";
+import { watchToolStarts } from "./session-archive/tool-start.js";
/** The engine a captured request went to: base URL, request headers (the engine's auth), query and API generation. */
export type EngineTarget = {
@@ -751,7 +752,7 @@ export function projectArchiveEngineReads(target: EngineTarget | (() => EngineTa
*/
export function observeUploadedSession(input: {
sessionUploader: ObservedUploader;
- archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete">;
+ archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete"> & Partial<Pick<ProjectArchiveLifecycle, "turnMessages">>;
/**
* The session's workspace: each settled turn reports the files outside it
* that the turn touched to the archive, and records the files it read
@@ -920,6 +921,8 @@ export function observeUploadedSession(input: {
// survives app restarts; without the messages the archiver numbers it
// right after the last archived turn. It also arms the idle final archive.
turnArchived = true;
+ // Capture v2: the turn's attachments are staged ahead of its delta.
+ if (history) input.archive.turnMessages?.(sessionId, history.delta);
input.archive.turnCompleted(sessionId, history ? history.outline : null);
};
@@ -1057,3 +1060,45 @@ export function observeUploadedSession(input: {
});
return session.done;
}
+
+/** Capture v2: the tool-start listener of each session's current turn (followToolStart). */
+const TOOL_WATCHES = new WeakMap<SessionObservers, Map<string, AbortController>>();
+/** A turn's listener stops after this long at most (the observer's own bound). */
+const MAX_TOOL_WATCH_MS = MAX_OBSERVED_TURN_MS;
+
+/**
+ * Capture v2, #9: call when a prompt of the session goes to the engine.
+ * Listens to the engine's event stream until the turn's first tool call
+ * starts and tells the project archive (toolStarted), which captures the
+ * folder's state in the background when it changed. A newer prompt of the
+ * session replaces the listener; the observers' stop ends it.
+ */
+export function followToolStart(input: {
+ observers: SessionObservers;
+ archive: Pick<ProjectArchiveLifecycle, "toolStarted">;
+ sessionId: string;
+ target: EngineTarget;
+}): void {
+ let watches = TOOL_WATCHES.get(input.observers);
+ if (!watches) {
+ watches = new Map();
+ TOOL_WATCHES.set(input.observers, watches);
+ }
+ watches.get(input.sessionId)?.abort();
+ const controller = new AbortController();
+ watches.set(input.sessionId, controller);
+ const map = watches;
+ const { baseUrl, headers, search, engine } = input.target;
+ const url = buildOpencodeProxyUrl(baseUrl, engine === "v2" ? "/api/event" : "/event", search);
+ const signal = AbortSignal.any([controller.signal, input.observers.controller.signal, AbortSignal.timeout(MAX_TOOL_WATCH_MS)]);
+ void watchToolStarts({
+ url,
+ headers: new Headers(headers),
+ sessionId: input.sessionId,
+ signal,
+ fetch: (target, init) => loopbackFetch(target, init),
+ onToolStart: () => input.archive.toolStarted(input.sessionId),
+ }).finally(() => {
+ if (map.get(input.sessionId) === controller) map.delete(input.sessionId);
+ });
+}
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index 5e5ec085..ed212587 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -1,7 +1,7 @@
import { createHash, randomBytes } from "node:crypto";
import { execFile, spawn } from "node:child_process";
import { createReadStream, createWriteStream, watch, type FSWatcher, type WriteStream } from "node:fs";
-import { lstat, mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat } from "node:fs/promises";
import { release as osRelease, tmpdir } from "node:os";
import nodePath, { basename, dirname, extname, join, relative, resolve, sep, type PlatformPath } from "node:path";
import { promisify } from "node:util";
@@ -20,6 +20,8 @@ import {
removeLegacyUploadContent,
} from "./session-upload-state.js";
import { TurnBaseStore, TurnDiffBuilder, type TurnDiffInput } from "./turn-diff.js";
+import { binaryReason, isKnownBinaryPath } from "./session-archive/binary.js";
+import { captureGitEnv } from "./session-archive/repos.js";
import { SUBAGENT_MODEL_FALLBACK_TRACE } from "./omnirush-swarm.js";
import { writeFileAtomic } from "./atomic-write.js";
import { XCODE_CLT_MISSING, gitSkipReason } from "./command-guard.js";
@@ -367,8 +369,27 @@ type HashCacheEntry = {
/** Redaction changed nothing, so the raw bytes are the upload form (verified by digest on upload). */
clean: boolean;
uploadPath: string;
+ /** Capture v2: the file's permission bits (the executable bit included). */
+ mode?: number;
};
+/**
+ * Capture v2: a manifest entry that carries no text: a binary file (its raw
+ * size and SHA-256; its bytes go to the byte-exact project archive) or a
+ * symlink (its target, also when broken; sha256 and size are those of the
+ * target string).
+ */
+type ExtraManifestEntry = {
+ uploadPath: string;
+ kind: "binary" | "symlink";
+ sha256: string;
+ size: number;
+ mode: number;
+ target?: string;
+ broken?: boolean;
+};
+
+
type SpoolMeta = {
id: string;
session_id: string;
@@ -465,6 +486,8 @@ type SessionState = {
lastSuccessAt?: string;
ready: Promise<void>;
tail: Promise<void>;
+ /** Capture v2: the binary files and symlinks of the workspace as last scanned (manifest entries without text). */
+ extras: Map<string, ExtraManifestEntry>;
};
type SessionUploaderOptions = {
@@ -516,6 +539,12 @@ type SessionUploaderOptions = {
* touched-files mode keeps them; it applies its own exclusions.
*/
onPathTouched?: (sessionId: string, path: string) => void;
+ /**
+ * Capture v2: a binary file of the workspace within the per-file cap
+ * (workspace-relative). It is never sent as text: the host hands it to the
+ * byte-exact project archive.
+ */
+ onBinaryFile?: (sessionId: string, path: string) => void;
};
export type TraceCapabilities = {
@@ -1775,39 +1804,6 @@ export function stripRemoteUserinfo(url: string): string {
return trimmed;
}
-/**
- * Whether `buffer` holds binary content: a NUL or many control bytes in its
- * first 8 KiB, or bytes that are not valid UTF-8 anywhere (a PDF, an image).
- * Text is uploaded as UTF-8, so a file that does not decode strictly would be
- * stored altered (each invalid byte becoming U+FFFD). `partial` is set for a
- * head cut from a longer file, which may end inside a character.
- */
-function isBinary(buffer: Buffer, partial = false): boolean {
- const length = Math.min(buffer.length, 8_192);
- let suspicious = 0;
- // An indexed loop: iterating a Buffer is several times slower, once per file.
- for (let index = 0; index < length; index += 1) {
- const byte = buffer[index]!;
- if (byte === 0) return true;
- if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;
- }
- if (length > 0 && suspicious / length > 0.1) return true;
- return !isValidUtf8(buffer, partial);
-}
-
-/** Strict UTF-8 validation (native); with `partial`, an incomplete last character is allowed. */
-function isValidUtf8(buffer: Buffer, partial: boolean): boolean {
- if (isUtf8(buffer)) return true;
- if (!partial) return false;
- for (let back = 1; back <= 3 && back <= buffer.length; back += 1) {
- const byte = buffer[buffer.length - back]!;
- if ((byte & 0xc0) === 0x80) continue;
- const length = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1;
- return length > back && isUtf8(buffer.subarray(0, buffer.length - back));
- }
- return false;
-}
-
const PATH_FIELD_NAMES = new Set([
"path",
"file_path",
@@ -2314,7 +2310,8 @@ async function walkFallback(
listing.denied += 1;
if ((listing.deniedPaths ??= []).length < MAX_LISTED_EXCLUSIONS) listing.deniedPaths.push(`${path}/`);
} else await walkFallback(root, fullPath, ignoreRules, listing, priorityPaths);
- } else if (entry.isFile()) {
+ } else if (entry.isFile() || entry.isSymbolicLink()) {
+ // Capture v2: a symlink is listed too (never followed; the scan records its target).
if (isUploadPathDenied(path)) {
listing.denied += 1;
if ((listing.deniedPaths ??= []).length < MAX_LISTED_EXCLUSIONS) listing.deniedPaths.push(path);
@@ -2390,10 +2387,13 @@ function runGitNow(root: string, args: string[], options: { maxBytes: number; ti
}, options.timeoutMs);
timer.unref?.();
try {
- child = spawn("git", ["-C", root, "-c", "core.quotePath=false", ...args], {
+ // Capture v2: no inherited GIT_DIR/GIT_WORK_TREE (a server started from a git hook), no fsmonitor, and
+ // the session's own root accepted when another user owns it (git's dubious-ownership refusal left
+ // the git block empty while the agent's own git commands worked). See session-archive/repos.ts.
+ child = spawn("git", ["-C", root, "-c", "core.quotePath=false", "-c", "core.fsmonitor=false", "-c", `safe.directory=${resolve(root)}`, ...args], {
stdio: ["ignore", "pipe", "ignore"],
windowsHide: true,
- env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0", GIT_PAGER: "cat", PAGER: "cat" },
+ env: { ...captureGitEnv(), GIT_CEILING_DIRECTORIES: process.env.GIT_CEILING_DIRECTORIES ?? "", LC_ALL: process.env.LC_ALL ?? "" },
});
} catch {
finish(false);
@@ -2551,6 +2551,9 @@ export function filterUploadDiff(raw: string, inputTruncated = false): { diff: s
const path = diffHeaderPath(headerEnd === -1 ? section : section.slice(0, headerEnd));
if (!path || isUploadPathDenied(path)) continue;
if (/^(?:Binary files .* differ|GIT binary patch)/m.test(section)) continue;
+ // Capture v2: a binary type is never diffed as text (its bytes go to the byte-exact archive), and
+ // a section that was not valid UTF-8 (utf8DiffText) never reaches here.
+ if (isKnownBinaryPath(path)) continue;
// A diff is text even when the file is JSON: hunks are not parseable documents.
// Each file's section under the mode its path selects, as the file's snapshot.
const redacted = redactUploadText(section, { context: path, mode: redactModeForPath(path) }).text;
@@ -2568,6 +2571,29 @@ export function filterUploadDiff(raw: string, inputTruncated = false): { diff: s
return { diff: output || null, truncated };
}
+/**
+ * Capture v2: git's diff output as text, without the file sections that are
+ * not valid UTF-8. git diffs any file without a NUL in its first 8000 bytes
+ * as text (a PDF, a latin-1 file); decoding those would replace their bytes
+ * with U+FFFD, so they are dropped here (their bytes are in the byte-exact
+ * archive). A truncated last section may end inside a character.
+ */
+export function utf8DiffText(raw: Buffer): string {
+ if (isUtf8(raw)) return raw.toString("utf8");
+ const marker = Buffer.from("\ndiff --git ");
+ const sections: Buffer[] = [];
+ let start = 0;
+ for (;;) {
+ const next = raw.indexOf(marker, start);
+ const end = next === -1 ? raw.length : next + 1;
+ sections.push(raw.subarray(start, end));
+ if (next === -1) break;
+ start = end;
+ }
+ return sections.filter((section, index) => isUtf8(section) || (index === sections.length - 1 && binaryReason(null, section, true) === null))
+ .map((section) => section.toString("utf8")).join("");
+}
+
/**
* Combined staged + unstaged text diff of the workspace. `--relative` with the
* `.` pathspec limits the diff to the workspace subtree and makes every header
@@ -2581,7 +2607,7 @@ async function gitDiff(root: string, hasHead: boolean): Promise<{ diff: string |
const options = { maxBytes: GIT_DIFF_READ_BYTES, timeoutMs: 20_000 };
if (hasHead) {
const result = await runGit(root, ["diff", "HEAD", ...common, ...scope], options);
- if (result.ok) return filterUploadDiff(result.stdout.toString("utf8"), result.truncated);
+ if (result.ok) return filterUploadDiff(utf8DiffText(result.stdout), result.truncated);
}
// An unborn branch has no HEAD to diff against: combine the index (against
// the empty tree) with the working tree changes on top of it.
@@ -2590,7 +2616,7 @@ async function gitDiff(root: string, hasHead: boolean): Promise<{ diff: string |
runGit(root, ["diff", ...common, ...scope], options),
]);
if (!staged.ok && !unstaged.ok) return { diff: null, truncated: false };
- const text = `${staged.ok ? staged.stdout.toString("utf8") : ""}${unstaged.ok ? unstaged.stdout.toString("utf8") : ""}`;
+ const text = `${staged.ok ? utf8DiffText(staged.stdout) : ""}${unstaged.ok ? utf8DiffText(unstaged.stdout) : ""}`;
return filterUploadDiff(text, staged.truncated || unstaged.truncated);
}
@@ -2602,13 +2628,14 @@ async function gitDiff(root: string, hasHead: boolean): Promise<{ diff: string |
* (porcelain status and the combined text diff), all redacted.
*/
export async function collectGitBlock(root: string): Promise<UploadGitBlock | null> {
- const inside = await gitText(root, ["rev-parse", "--is-inside-work-tree"]);
+ // 20 s, not 5: under load (a full scan hashing the tree) rev-parse timed out and the block was dropped.
+ const inside = await gitText(root, ["rev-parse", "--is-inside-work-tree"], 64 * 1024, 20_000);
if (inside !== "true") return null;
// Where the workspace sits inside the repository ("" at the top level,
// "packages/app/" for a nested workspace). Status, log and diff below are
// scoped to that subtree; without a reliable answer the block is skipped
// rather than risk describing files outside the workspace root.
- const prefixResult = await runGit(root, ["rev-parse", "--show-prefix"], { maxBytes: 64 * 1024, timeoutMs: 5_000 });
+ const prefixResult = await runGit(root, ["rev-parse", "--show-prefix"], { maxBytes: 64 * 1024, timeoutMs: 20_000 });
if (!prefixResult.ok || prefixResult.truncated) return null;
const prefix = prefixResult.stdout.toString("utf8").trim().replaceAll("\\", "/");
const scope = prefix ? ["--", "."] : [];
@@ -3094,6 +3121,8 @@ type ScanContext = {
yielder: LoopYielder;
pool: ReadPool;
ancestors: AncestorCache;
+ /** Capture v2: the binary files and symlinks the scan met, by path. */
+ extras: Map<string, ExtraManifestEntry>;
};
/**
@@ -3125,6 +3154,7 @@ function scanContext(root: string, cache: Map<string, HashCacheEntry>, texts: Re
yielder: new LoopYielder(),
pool: new ReadPool(SCAN_CONCURRENCY),
ancestors: new Map(),
+ extras: new Map(),
};
}
@@ -3147,10 +3177,19 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCach
return null;
}
metrics.fileStats += 1;
- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) return null;
+ if (file.isSymbolicLink()) {
+ // Capture v2: a symlink is listed with its target, also a broken one; never followed.
+ const link = await symlinkExtra(absolute, uploadPathFor(path, undefined), file.mode);
+ if (link) context.extras.set(path, link);
+ return null;
+ }
+ if (!file.isFile() || file.size > MAX_UPLOAD_FILE_BYTES) return null;
const cached = cache.get(path);
const signature = fileSignature(file);
- if (cached && sameFileSignature(cached, signature)) return cached;
+ if (cached && sameFileSignature(cached, signature)) {
+ if (cached.binary) context.extras.set(path, binaryExtra(cached));
+ return cached;
+ }
const scanKey = [
resolve(context.root),
path,
@@ -3168,17 +3207,20 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCach
return entry;
}
const uploadPath = uploadPathFor(path, cached);
- const inFlight = pool.read(absolute, file.size, (buffer) => {
+ const mode = file.mode & 0o7777;
+ const inFlight = pool.read(absolute, file.size, (buffer): HashCacheEntry => {
metrics.fileReads += 1;
- if (isBinary(buffer)) {
- return { ...signature, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };
+ // Capture v2: a known binary type or signature is binary too (a PDF that is all ASCII); its raw digest is kept.
+ if (binaryReason(path, buffer) !== null) {
+ return { ...signature, binary: true, sha256: sha256Hex(buffer), bytes: buffer.length, json: 0, clean: false, uploadPath, mode };
}
- return redactedCacheEntry(path, buffer, signature, uploadPath, metrics, { cache: context.texts, absolute }).entry;
+ return { ...redactedCacheEntry(path, buffer, signature, uploadPath, metrics, { cache: context.texts, absolute }).entry, mode };
}).catch(() => null);
FILE_SCAN_IN_FLIGHT.set(scanKey, inFlight);
try {
const entry = await inFlight;
if (entry) cache.set(path, entry);
+ if (entry?.binary) context.extras.set(path, binaryExtra(entry));
await context.yielder.pause();
return entry;
} finally {
@@ -3186,6 +3228,37 @@ async function inspectFile(context: ScanContext, path: string): Promise<HashCach
}
}
+/** Capture v2: the manifest entry of a binary file the scan read (its raw digest, size and mode). */
+function binaryExtra(entry: HashCacheEntry): ExtraManifestEntry {
+ return { uploadPath: entry.uploadPath, kind: "binary", sha256: entry.sha256, size: entry.bytes, mode: entry.mode ?? 0o644 };
+}
+
+/** Capture v2: the manifest entry of a symlink (its target as written, also when it is broken), or null when unreadable. */
+async function symlinkExtra(absolute: string, uploadPath: string, mode: number): Promise<ExtraManifestEntry | null> {
+ let target: string;
+ try {
+ target = await readlink(absolute);
+ } catch {
+ return null;
+ }
+ let broken = false;
+ try {
+ await stat(absolute);
+ } catch {
+ broken = true;
+ }
+ const clamped = clampUploadText(redactUploadText(target).text, 4_096);
+ return { uploadPath, kind: "symlink", sha256: sha256Hex(Buffer.from(clamped, "utf8")), size: Buffer.byteLength(clamped), mode: mode & 0o7777, target: clamped, broken };
+}
+
+/** One manifest line of a binary file or symlink. */
+function extraEntryJson(entry: ExtraManifestEntry): string {
+ const head = `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.size},"mode":${entry.mode}`;
+ if (entry.kind === "symlink") return `${head},"type":"symlink","target":${JSON.stringify(entry.target ?? "")},"broken":${entry.broken === true}}`;
+ return `${head},"binary":true,"archived":"byte_exact"}`;
+}
+
+
/** One file as files[] carries it, re-read at upload time. */
type UploadContent = { content: string; entry: HashCacheEntry };
@@ -3232,7 +3305,7 @@ async function readUploadContent(
}
}
}
- if (isBinary(buffer)) return null;
+ if (binaryReason(path, buffer) !== null) return null;
const { entry, text } = redactedCacheEntry(path, buffer, signature, uploadPathFor(path, cached), metrics, { cache: texts, absolute });
// The same digest keeps the cached entry, which the manifest may share.
if (cached && cached.sha256 === entry.sha256 && sameFileSignature(cached, signature)) {
@@ -3267,6 +3340,9 @@ type ScanResult = {
binaryPaths?: Map<string, number>;
tooLargePaths?: Map<string, number>;
cappedPaths?: string[];
+ /** Capture v2: binary files and symlinks found; `extrasScope` says whether that is all of them (a full scan) or the inspected paths only. */
+ extras: Map<string, ExtraManifestEntry>;
+ extrasScope: "full" | ReadonlySet<string>;
};
/** A changed file competing for the snapshot's content budget. */
@@ -3349,15 +3425,16 @@ async function startsBinary(absolute: string): Promise<boolean> {
const head = Buffer.alloc(8_192);
try {
const read = await readInto(absolute, head);
- return isBinary(head.subarray(0, read), read === head.length);
+ return binaryReason(absolute, head.subarray(0, read), read === head.length) !== null;
} catch {
return false;
}
}
/** The manifest entry as the envelope carries it: path (redacted), sha256, size. */
-function manifestEntryJson(entry: HashCacheEntry): string {
- return `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.bytes}}`;
+function manifestEntryJson(entry: HashCacheEntry | ExtraManifestEntry): string {
+ if ("kind" in entry) return extraEntryJson(entry);
+ return `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.bytes}${entry.mode !== undefined ? `,"mode":${entry.mode}` : ""}}`;
}
function candidateCost(entry: HashCacheEntry): number {
@@ -3465,6 +3542,8 @@ async function scanWorkspaceFull(
binaryPaths,
tooLargePaths,
cappedPaths,
+ extras: context.extras,
+ extrasScope: "full",
};
}
@@ -3533,6 +3612,8 @@ async function scanDirtyPaths(
removed: countRemoved(previous, manifest),
deniedCount: listing.denied,
manifestTruncated: listing.truncated || manifest.size >= MAX_FILES,
+ extras: context.extras,
+ extrasScope: new Set(targets),
};
}
@@ -3736,7 +3817,7 @@ type EnvelopeBody = {
extras: Record<string, unknown>;
/** Streams files[] one entry at a time through `emit`. */
writeFiles: (emit: (file: UploadFile) => Promise<void>) => Promise<void>;
- manifest: () => Iterable<HashCacheEntry>;
+ manifest: () => Iterable<HashCacheEntry | ExtraManifestEntry>;
/** Computed once files[] is written, so cap omissions are known. */
privacy: () => Record<string, unknown>;
/** Fields written after privacy, once files[] is out (excluded_files). */
@@ -3897,6 +3978,7 @@ export class SessionUploader {
private readonly bases: TurnBaseStore;
private readonly onSessionClosed?: (sessionId: string) => void;
private readonly onPathTouched?: (sessionId: string, path: string) => void;
+ private readonly onBinaryFile?: (sessionId: string, path: string) => void;
private readonly fileUploader?: SessionUploaderOptions["uploadFile"];
private readonly capabilityProbe?: SessionUploaderOptions["capabilities"];
private traceSchemaCache: { version: 2 | 3; expiresAt: number } | null = null;
@@ -3945,6 +4027,7 @@ export class SessionUploader {
});
this.onSessionClosed = options.onSessionClosed;
this.onPathTouched = options.onPathTouched;
+ this.onBinaryFile = options.onBinaryFile;
this.tempDir = stateDir ? join(stateDir, UPLOAD_TEMP_DIRECTORY) : join(tmpdir(), `omnirush-upload-${process.pid}`);
this.bases = new TurnBaseStore(
stateDir ? join(stateDir, UPLOAD_BASE_DIRECTORY) : join(this.tempDir, UPLOAD_BASE_DIRECTORY),
@@ -4180,6 +4263,7 @@ export class SessionUploader {
watchSetup: Promise.resolve(),
trace: [],
changeJournal: new Map(),
+ extras: new Map(),
touchedPaths: new Set(),
changeJournalBytes: 0,
changeCaptureTail: Promise.resolve(),
@@ -5261,7 +5345,7 @@ export class SessionUploader {
} else {
const buffer = await readFile(absolute);
this.metrics.fileReads += 1;
- if (buffer.length > MAX_UPLOAD_FILE_BYTES || isBinary(buffer)) {
+ if (buffer.length > MAX_UPLOAD_FILE_BYTES || binaryReason(path, buffer) !== null) {
if (written) {
state.turnWrittenPaths.set(path, Date.now());
state.turnSkipped.set(path, file.mtimeMs);
@@ -5369,6 +5453,15 @@ export class SessionUploader {
} finally {
releaseScan();
}
+ // Capture v2: the binary files and symlinks the manifest lists next to the text files.
+ const extraEntries = scan.extrasScope === "full" ? new Map(scan.extras) : new Map(state.extras);
+ if (scan.extrasScope !== "full") {
+ for (const path of scan.extrasScope) extraEntries.delete(path);
+ for (const path of ignored) extraEntries.delete(path);
+ for (const [path, entry] of scan.extras) extraEntries.set(path, entry);
+ }
+ // A binary file goes to the byte-exact project archive (never as text).
+ for (const [path, entry] of extraEntries) if (entry.kind === "binary") this.onBinaryFile?.(state.id, path);
// What moved on disk since the last snapshot, where no watcher may have
// seen it (a polled workspace). Only against an accepted snapshot: with
// none yet (the start one was refused), every file counts as changed.
@@ -5432,6 +5525,7 @@ export class SessionUploader {
let fixedBytes = serializedBytes(extras) + SNAPSHOT_WRAPPER_MARGIN_BYTES;
for (const file of leading) fixedBytes += serializedBytes(file) + 1;
for (const entry of scan.manifest.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;
+ for (const entry of extraEntries.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;
const budget = Math.max(0, this.snapshotMaxBytes - fixedBytes);
const selection = selectSnapshotContent(candidates, budget);
const capOmitted: SnapshotCandidate[] = [];
@@ -5477,7 +5571,10 @@ export class SessionUploader {
await this.bases.writable();
}
},
- manifest: () => manifest.values(),
+ manifest: function* () {
+ yield* manifest.values();
+ yield* extraEntries.values();
+ },
privacy: () => ({
...PRIVACY_POLICY,
denied_file_count: scan.deniedCount,
@@ -5518,6 +5615,7 @@ export class SessionUploader {
// next change snapshot reports exactly what moved since this one.
accepted = true;
state.manifest = manifest;
+ state.extras = extraEntries;
state.turnManifest ??= turnBaseline(manifest, state.turnStartedAt, [previous]);
state.listing = { denied: scan.deniedCount, truncated: scan.manifestTruncated };
state.lastHead = git?.commit ?? null;