A desktop coding agent with free access to frontier models.
TypeScript
fix: keep local config values intact
ogsamrat33f50209b068
diff --git a/apps/server/src/__fixtures__/scrub-file-mode.json b/apps/server/src/__fixtures__/scrub-file-mode.jsonindex 74048831..6408ff83 100644--- a/apps/server/src/__fixtures__/scrub-file-mode.json+++ b/apps/server/src/__fixtures__/scrub-file-mode.json@@ -2384,7 +2384,7 @@}},{- "name": "real IPv4 addresses are still redacted",+ "name": "public IPv4 addresses are still redacted; loopback, private and documentation ones are kept","kind": "trace","input": {"schema_version": 1,@@ -2410,7 +2410,7 @@"input": {"command": "curl http://203.0.113.9:8080/health"},- "output": "10.0.0.5\nip=192.168.1.20\nGET http://203.0.113.9:8080 ok\nping from 8.8.8.8 (8.8.8.8): 56 bytes\nlisten 127.0.0.1:5432; peer 172.16.0.1.\n"+ "output": "10.0.0.5\nip=192.168.1.20\nGET http://203.0.113.9:8080 ok\nping from 8.8.8.8 (8.8.8.8): 56 bytes\nlisten 127.0.0.1:5432; peer 172.16.0.1.\nping 52.14.7.21 and 151.101.1.69\n"}}]@@ -2442,9 +2442,9 @@"state": {"status": "completed","input": {- "command": "curl http://[REDACTED_PII]:8080/health"+ "command": "curl http://203.0.113.9:8080/health"},- "output": "[REDACTED_PII]\nip=[REDACTED_PII]\nGET http://[REDACTED_PII]:8080 ok\nping from [REDACTED_PII] ([REDACTED_PII]): 56 bytes\nlisten [REDACTED_PII]:5432; peer [REDACTED_PII].\n"+ "output": "10.0.0.5\nip=192.168.1.20\nGET http://203.0.113.9:8080 ok\nping from [REDACTED_PII] ([REDACTED_PII]): 56 bytes\nlisten 127.0.0.1:5432; peer 172.16.0.1.\nping [REDACTED_PII] and [REDACTED_PII]\n"}}]@@ -2454,5 +2454,651 @@}]}+ },+ {+ "name": "loopback, unspecified, example e-mail and a placeholder token written to code are kept (tester session, anonymised)",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/config.py",+ "content": "BIND_HOST = \"0.0.0.0\"\nDEFAULT_EMAIL = \"[email protected]\"\nFORM_PLACEHOLDER = \"e.g. Jane Doe\"\nAPI_TOKEN = \"fake-not-a-real-token-0000\"\n\n\ndef get_config():\n return {\n \"BIND_HOST\": BIND_HOST,\n \"DEFAULT_EMAIL\": DEFAULT_EMAIL,\n \"FORM_PLACEHOLDER\": FORM_PLACEHOLDER,\n \"API_TOKEN\": API_TOKEN,\n }\n"+ },+ "output": "Wrote file successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_2",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/server.js",+ "content": "const http = require(\"http\");\n\nconst host = \"127.0.0.1\";\nconst port = 3000;\n\nconst server = http.createServer((req, res) => {\n res.end(\"ok\");\n});\n\nserver.listen(port, host, () => {\n console.log(\"serving on http://127.0.0.1:3000\");\n});\n"+ },+ "output": "Wrote file successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_3",+ "tool": "edit",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/README.md",+ "oldString": "# demo\n",+ "newString": "# demo\n\n- `server.js` starts a minimal Node.js HTTP server on `127.0.0.1:3000` and responds with `ok`.\n"+ },+ "output": "Edit applied successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_4",+ "tool": "read",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/config.py"+ },+ "output": "1: BIND_HOST = \"0.0.0.0\"\n2: DEFAULT_EMAIL = \"[email protected]\"\n4: API_TOKEN = \"fake-not-a-real-token-0000\""+ }+ },+ {+ "type": "tool",+ "callID": "call_5",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "node server.js & sleep 3; curl -s http://127.0.0.1:3000"+ },+ "output": "serving on http://127.0.0.1:3000\nok"+ }+ },+ {+ "type": "text",+ "text": "Start server.js bound to 0.0.0.0:3000, then fetch http://localhost:3000. Contact [email protected] or [email protected]."+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/config.py",+ "content": "BIND_HOST = \"0.0.0.0\"\nDEFAULT_EMAIL = \"[email protected]\"\nFORM_PLACEHOLDER = \"e.g. Jane Doe\"\nAPI_TOKEN = \"fake-not-a-real-token-0000\"\n\n\ndef get_config():\n return {\n \"BIND_HOST\": BIND_HOST,\n \"DEFAULT_EMAIL\": DEFAULT_EMAIL,\n \"FORM_PLACEHOLDER\": FORM_PLACEHOLDER,\n \"API_TOKEN\": API_TOKEN,\n }\n"+ },+ "output": "Wrote file successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_2",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/server.js",+ "content": "const http = require(\"http\");\n\nconst host = \"127.0.0.1\";\nconst port = 3000;\n\nconst server = http.createServer((req, res) => {\n res.end(\"ok\");\n});\n\nserver.listen(port, host, () => {\n console.log(\"serving on http://127.0.0.1:3000\");\n});\n"+ },+ "output": "Wrote file successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_3",+ "tool": "edit",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/README.md",+ "oldString": "# demo\n",+ "newString": "# demo\n\n- `server.js` starts a minimal Node.js HTTP server on `127.0.0.1:3000` and responds with `ok`.\n"+ },+ "output": "Edit applied successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_4",+ "tool": "read",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/config.py"+ },+ "output": "1: BIND_HOST = \"0.0.0.0\"\n2: DEFAULT_EMAIL = \"[email protected]\"\n4: API_TOKEN = \"fake-not-a-real-token-0000\""+ }+ },+ {+ "type": "tool",+ "callID": "call_5",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "node server.js & sleep 3; curl -s http://127.0.0.1:3000"+ },+ "output": "serving on http://127.0.0.1:3000\nok"+ }+ },+ {+ "type": "text",+ "text": "Start server.js bound to 0.0.0.0:3000, then fetch http://localhost:3000. Contact [email protected] or [email protected]."+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ }+ },+ {+ "name": "real secrets and personal data next to the kept values are still redacted",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/settings.py",+ "content": "BIND_HOST = \"0.0.0.0\"\nUPSTREAM = \"52.14.7.21\"\nOWNER_EMAIL = \"[email protected]\"\nSUPPORT = \"[email protected]\"\nAPI_TOKEN = \"q7Xk2mP9vR4tL8wZ3nB6\"\nSTRIPE = \"sk_live_51Hx7Gq2eZvKYlo2C\"\nAWS_ACCESS_KEY_ID = \"AKIAIOSFODNN7EXAMPLE\"\n"+ },+ "output": "Wrote file successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_2",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "cat .env"+ },+ "output": "API_TOKEN=q7Xk2mP9vR4tL8wZ3nB6\nDB_PASSWORD=hunter2hunter2\nping 151.101.1.69 ok"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/settings.py",+ "content": "BIND_HOST = \"0.0.0.0\"\nUPSTREAM = \"[REDACTED_PII]\"\nOWNER_EMAIL = \"[REDACTED_PII]\"\nSUPPORT = \"[REDACTED_PII]\"\nAPI_TOKEN = \"[REDACTED]\"\nSTRIPE = \"[REDACTED]\"\nAWS_ACCESS_KEY_ID = \"[REDACTED]\"\n"+ },+ "output": "Wrote file successfully."+ }+ },+ {+ "type": "tool",+ "callID": "call_2",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "cat .env"+ },+ "output": "API_TOKEN=[REDACTED]\nDB_PASSWORD=[REDACTED]\nping [REDACTED_PII] ok"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ }+ },+ {+ "name": "kept IPv4 ranges (private, link-local, loopback, documentation, netmasks); public addresses are redacted",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "ip addr; ip route"+ },+ "output": "inet 10.1.2.3/8 brd 10.255.255.255\ninet 172.16.5.4 172.31.255.1 not-private 172.32.0.1\ninet 192.168.0.10 netmask 255.255.255.0\nlink 169.254.169.254 gw 0.0.0.0 bcast 255.255.255.255 mask 255.255.240.0 odd 255.255.0.255\ndocs 192.0.2.10 198.51.100.20 203.0.113.30 lo 127.0.0.53 ::1\npublic 8.8.8.8 1.1.1.1 52.14.7.21 192.0.3.1 198.51.101.2\n"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "ip addr; ip route"+ },+ "output": "inet 10.1.2.3/8 brd 10.255.255.255\ninet 172.16.5.4 172.31.255.1 not-private [REDACTED_PII]\ninet 192.168.0.10 netmask 255.255.255.0\nlink 169.254.169.254 gw 0.0.0.0 bcast 255.255.255.255 mask 255.255.240.0 odd [REDACTED_PII]\ndocs 192.0.2.10 198.51.100.20 203.0.113.30 lo 127.0.0.53 ::1\npublic [REDACTED_PII] [REDACTED_PII] [REDACTED_PII] [REDACTED_PII] [REDACTED_PII]\n"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ }+ },+ {+ "name": "kept e-mails: reserved, test, placeholder and no-reply addresses; personal ones go",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "git log --format=%ae"+ },+ "output": "[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\n"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "git log --format=%ae"+ },+ "output": "[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\n[REDACTED_PII]\n[REDACTED_PII]\n[REDACTED_PII]\n[REDACTED_PII]\n"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ }+ },+ {+ "name": "placeholder assignment values are kept; secret-shaped ones are not",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/.env.example",+ "content": "API_TOKEN=your-token-here\nAPI_KEY=YOUR_API_KEY\nDB_PASSWORD=changeme\nSECRET_KEY=xxxxxxxxxxxxxxxx\nAUTH_TOKEN=fake-not-a-real-token-0000\nACCESS_TOKEN=replace-me-with-a-token\nSHORT_TOKEN=abc12\nREAL_TOKEN=9f8e7d6c5b4a39281706\nPASSWORD=password1234\nCLIENT_SECRET=Tq3vX8kPz2LmN7rB\n"+ },+ "output": "Wrote file successfully."+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "write",+ "state": {+ "status": "completed",+ "input": {+ "filePath": "/work/proj/.env.example",+ "content": "API_TOKEN=your-token-here\nAPI_KEY=YOUR_API_KEY\nDB_PASSWORD=changeme\nSECRET_KEY=xxxxxxxxxxxxxxxx\nAUTH_TOKEN=fake-not-a-real-token-0000\nACCESS_TOKEN=replace-me-with-a-token\nSHORT_TOKEN=abc12\nREAL_TOKEN=[REDACTED]\nPASSWORD=[REDACTED]\nCLIENT_SECRET=[REDACTED]\n"+ },+ "output": "Wrote file successfully."+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ }+ },+ {+ "name": "Arch and Debian package releases are versions, not IPv4 addresses",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "pacman -Q"+ },+ "output": "alsa-ucm-conf 1.2.15.3-1\nimagemagick 7.1.2.8-1\nlibmodplug 0.8.9.0-7\nsddm-git 0.21.0.12.gbc9eee8-1\nmirror 52.14.7.21\n"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "turn.messages",+ "data": {+ "messages": [+ {+ "info": {+ "id": "msg_1",+ "role": "assistant"+ },+ "parts": [+ {+ "type": "tool",+ "callID": "call_1",+ "tool": "bash",+ "state": {+ "status": "completed",+ "input": {+ "command": "pacman -Q"+ },+ "output": "alsa-ucm-conf 1.2.15.3-1\nimagemagick 7.1.2.8-1\nlibmodplug 0.8.9.0-7\nsddm-git 0.21.0.12.gbc9eee8-1\nmirror [REDACTED_PII]\n"+ }+ }+ ]+ }+ ],+ "part": 1,+ "parts": 1+ }+ }+ ]+ }+ },+ {+ "name": "a package version under a version key is not an IPv4 address; the same digits elsewhere are",+ "kind": "trace",+ "input": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "environment",+ "data": {+ "packages": [+ {+ "name": "update-notifier-common",+ "version": "3.192.1.7"+ },+ {+ "name": "alsa-ucm-conf",+ "version": "1.2.15.3-1"+ }+ ],+ "note": "mirror 3.192.1.7"+ }+ }+ ]+ },+ "expected": {+ "schema_version": 1,+ "session_id": "ses_1",+ "events": [+ {+ "at": "2026-10-04T00:00:00.000Z",+ "type": "environment",+ "data": {+ "packages": [+ {+ "name": "update-notifier-common",+ "version": "3.192.1.7"+ },+ {+ "name": "alsa-ucm-conf",+ "version": "1.2.15.3-1"+ }+ ],+ "note": "mirror [REDACTED_PII]"+ }+ }+ ]+ }}]diff --git a/apps/server/src/capture-client.test.ts b/apps/server/src/capture-client.test.tsindex 94061193..c2197b30 100644--- a/apps/server/src/capture-client.test.ts+++ b/apps/server/src/capture-client.test.ts@@ -42,7 +42,7 @@ async function tempDir(prefix: string): Promise<string> {}async function git(root: string, ...args: string[]): Promise<void> {- await execFileAsync("git", ["-C", root, "-c", "commit.gpgsign=false", "-c", "user.name=Dev", "-c", "[email protected]", ...args]);+ await execFileAsync("git", ["-C", root, "-c", "commit.gpgsign=false", "-c", "user.name=Dev", "-c", "[email protected]", ...args]);}/**@@ -463,7 +463,7 @@ describe("capture worker", () => {expect(capture.mode()).toBe(worker ? "starting" : "local");capture.startSession("session-modes-0001", "workspace-modes", root);expect(capture.hasSession("session-modes-0001")).toBe(true);- expect(capture.recordWebVisit("session-modes-0001", { url: "https://example.com/docs", title: "Docs", text: "mail [email protected]" })).toBe(true);+ expect(capture.recordWebVisit("session-modes-0001", { url: "https://example.com/docs", title: "Docs", text: "mail [email protected]" })).toBe(true);expect(capture.recordWebVisit("session-modes-0001", { url: "http://localhost:3000/" })).toBe(false);await capture.stop();expect(capture.mode()).toBe("down");diff --git a/apps/server/src/session-archive/README.md b/apps/server/src/session-archive/README.mdindex 75e5c4d2..58622076 100644--- a/apps/server/src/session-archive/README.md+++ b/apps/server/src/session-archive/README.md@@ -376,6 +376,9 @@ backend's `docs/omnirush-project-archive.md` ("Capture v2").- **What is archived.** `scanTouchedFiles` looks at the session's touched paths and every file the chain holds (to see it go), nothing else: a plain relative path only; walked from the root with `lstat`, each folder on the way a real folder (a symlink or a file in the way: the path is not in the folder); a touched folder is never expanded; a symlink is followed only to a target that is lexically inside the root (as given or as its real path, at most 8 links; its target goes through the same checks), and one pointing outside is skipped (`special`) without anything outside being looked at. The same exclusions as the whole-folder scan: credential files (`isArchiveCredentialPath`, unless `archiveIncludeCredentialFiles`), app state dirs under the root, `__omnirush__` at the root, special files, unreadable files, lone-surrogate names. A file is hashed from the very file `lstat`ed (same `st_dev` and `st_ino`), with the root's hash cache (read and added to, never pruned: the scan sees only part of the root), and pass 2 packs it with the usual checks (`pack.ts`). Any type and size, byte for byte, unscrubbed, sealed like every archive; `workspace.git` is `null`.- **Manifest.** `"scope": "touched"` in the sealed `manifest.json` of every archive of the chain; `files` lists only touched files (the base: every touched file; a delta: the added and modified ones). A path the manifest does not list is not deleted: a delta deletes, in `deleted`, the files an earlier archive of the chain holds that are no longer a regular file in the folder (gone, a folder, a link, reached through a folder that is now a link). A file that cannot be read for now stays in the baseline as it was.- **Policy off.** With `touched_files` off or absent nothing is written for such a folder: `captureBase` is `not_archivable` after the probe, and no touched path is kept. A touched chain's captures pause (`not_archivable`) while it is off and catch up once it is on (the touched set is kept). The server refuses a `touched` create from a user whose policy is off with `422 archive_marker_not_allowed`: the drain drops the session's chain (its jobs, the session stopped, one `warn` line, no retry) and keeps the policy as off for `touched` until it is asked again (`POLICY_TTL_MS`), so no other chain starts meanwhile. The same holds for `folder`.+- **Project folders (4.6).** GET /archives/key also carries `policy.project_folders` (off unless the boolean `true`; absent on an older backend). With it on, a root with no `.git` that passes every folder refusal and looks like a project (`looksLikeProject`: a manifest or lockfile such as `package.json`, `pyproject.toml`, `requirements*.txt`, `Cargo.toml`, `go.mod`, `pom.xml`, `build.gradle*`, `Gemfile`, `composer.json`, `*.csproj`, `CMakeLists.txt` or a `Makefile` in it or up to two levels below, or a `src/` folder with a code file; a bounded walk of 600 entries and 40 folders that never follows a link nor enters a hidden, regenerable or denied folder) is archived whole with marker `project`, like a plain folder under all folders: regenerable folders left out (`regenerable`), credentials filtered, gitignore applied, `workspace.git` null. Precedence: git, then `folder`, then `project`, then `touched`. The start manifest prescans a project root before the policy is known (a mismatch drops the prescan as before), and no longer prescans a plain root once the kept policy says neither all folders nor project (touched scope or nothing), so a big non-project folder costs no start scan.+ - **Touched-scope listing.** A v2 touched-files chain lists every file of the folder it does not hold in `state.json` `excluded` (`listNotArchived`, at capture time, never in the start gate): `not_archived_touched_scope` with `size` and `sha256` up to 64 MiB (`MAX_EXCLUDED_HASH_BYTES`; larger: `mtime`), the whole-folder rules' reasons for the rest. Its hash cache is kept apart (`listing`), so an unchanged file is read once. The listing holds at most `MAX_EXCLUDED_LISTED` (50,000) entries; `excluded_truncated_count` counts the rest.+ - **Refused roots in the start upload.** The session uploader's start envelope carries `workspace.project_archive: {status: "skipped", reason}` (`projectArchiveSkip`: `credential_root`, `root_too_broad`, `root_app_data`, `root_system`) for a root without git the folder refusals stop, so the export can say why there is no project.- **Reproducible tasks.** Two exceptions to the gitignored rule below, the same as the CLI (2.0.0): every file the agent touched is archived byte for byte, binaries included, even when gitignored, and so is every lockfile (`findLockfiles` in `touched.ts`: `package-lock.json`, `pnpm-lock.yaml`, `Cargo.lock`, `uv.lock`, `go.sum`, ...; a bounded walk, depth 6, 5,000 folders, 200 files, never into `.git/`, never through a link). Neither ever comes from a regenerable folder (`REGENERABLE_DIR_NAMES` / `inRegenerableDir` in `session-uploader.ts`: `node_modules`, `.venv`, `venv`, `__pycache__`, `dist`, `build`, `target`, `vendor`, ... at any depth) or from `.git/`. So every archived chain keeps its touched paths (a whole-folder chain too, not only a touched-files one); a whole-folder scan adds the touched paths and lockfiles it left out with a second `scanTouchedFiles` pass (`includeIgnored: true`); a touched-files chain scans its touched paths and every lockfile with `includeIgnored: true`. The session uploader's end-of-turn artifact events also cover gitignored untracked outputs outside the regenerable folders (`listIgnoredOutputFiles`), and those paths reach the touched set like any other artifact.- **Gitignored content.** Apart from the reproducible-task exceptions above, nothing git ignores is archived: base, delta and final archives, whole-folder and touched-files alike (`ignore.ts`). It can be rebuilt (`node_modules/`, `dist/`, venvs, caches), and it made single projects many gigabytes. Git decides, with all its rules (every `.gitignore`, `.git/info/exclude`, `core.excludesFile`):- **A root in a repository** (a `.git` root or a `git_parent` folder): one `git ls-files -z --others --ignored --exclude-standard --directory` in the root, with `GIT_CEILING_DIRECTORIES` at home like every archive git run. An ignored directory comes back as one `dir/` line and pass 1 leaves it out before `lstat`ing it, so `node_modules/` with 200k files costs one line of git output and nothing else. Tracked files are never ignored (git does not list them, so a force-added `dist/vendor.js` stays). A folder git lists because everything in it is ignored is left out too (only empty subfolders go with it, which git cannot hold either).diff --git a/apps/server/src/session-archive/capture-v2.ts b/apps/server/src/session-archive/capture-v2.tsindex 2a6ab4bf..3ff72cba 100644--- a/apps/server/src/session-archive/capture-v2.ts+++ b/apps/server/src/session-archive/capture-v2.ts@@ -22,7 +22,10 @@ export const STATE_MEMBER = "__omnirush__/state.json";export const STATE_SCHEMA = "omnirush.archive.state.v1";/** The start gate waits this long, at most, for the start manifest before the first prompt goes to the model. */export const START_GATE_MS = 3_000;-export const MAX_EXCLUDED_LISTED = 20_000;+/** At most this many left-out entries are listed; the rest are counted (`excluded_truncated_count`). */+export const MAX_EXCLUDED_LISTED = 50_000;+/** A left-out file up to this size is listed with its SHA-256; a larger one with its size and mtime. */+export const MAX_EXCLUDED_HASH_BYTES = 64 * 1024 * 1024;export type StateKind = "start" | "pre_tool" | "after" | "final";export type StartCapture = "complete" | "partial";@@ -36,9 +39,11 @@ export type ExcludedReason =| "unreadable"| "non_utf8"| "reserved"- | "too_large";+ | "too_large"+ /** A touched-files chain: a file of the folder the agent never touched (listed, not archived). */+ | "not_archived_touched_scope";-export type ExcludedItem = { path: string; type: "file" | "dir"; reason: ExcludedReason; size?: number; sha256?: string };+export type ExcludedItem = { path: string; type: "file" | "dir" | "symlink"; reason: ExcludedReason; size?: number; sha256?: string; mtime?: number };/** The scan's left-out entries, by path (the first reason wins), at most MAX_EXCLUDED_LISTED. */export class ExcludedList {@@ -66,6 +71,15 @@ export class ExcludedList {return this.dropped > 0;}+ /** How many entries were left out of the listing past MAX_EXCLUDED_LISTED. */+ get truncatedCount(): number {+ return this.dropped;+ }++ has(path: string): boolean {+ return this.items.has(path);+ }+list(): ExcludedItem[] {return [...this.items.values()].sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));}@@ -103,6 +117,7 @@ export function stateDocument(input: StateDocumentInput): Buffer {repos: input.repos,excluded: input.excluded.list(),excluded_truncated: input.excluded.truncated,+ excluded_truncated_count: input.excluded.truncatedCount,scrubbed: [...input.scrubbed].sort(),attachments: input.attachments.map(attachmentStateItem),}), "utf8");diff --git a/apps/server/src/session-archive/detect.ts b/apps/server/src/session-archive/detect.tsindex 6425bd98..5e44a75b 100644--- a/apps/server/src/session-archive/detect.ts+++ b/apps/server/src/session-archive/detect.ts@@ -8,18 +8,18 @@* and is not, or is not inside, a credential, app-data or system location.* Git roots never get those refusals.*/-import { lstat, open, realpath } from "node:fs/promises";+import { lstat, open, opendir, realpath } from "node:fs/promises";import { homedir } from "node:os";import path, { isAbsolute, join, parse, posix, relative, resolve, sep, win32 } from "node:path";-import { isUploadDirectoryDenied } from "../session-uploader.js";+import { isUploadDirectoryDenied, REGENERABLE_DIR_NAMES } from "../session-uploader.js";import type { ArchivePolicy } from "./policy.js";export type ArchivableProject = {archivable: boolean;- /** git_dir | git_file | git_parent | folder | touched | no_marker | gitfile_invalid | root_not_directory | root_too_broad */+ /** git_dir | git_file | git_parent | folder | project | touched | no_marker | gitfile_invalid | root_not_directory | root_too_broad */reason: string;- /** The marker that qualified the root (".git", "folder", "touched"), null when not archivable. */+ /** The marker that qualified the root (".git", "folder", "project", "touched"), null when not archivable. */marker: string | null;};@@ -235,6 +235,101 @@ export const defaultProjectDetectors: readonly ProjectMarkerDetector[] = [gitMarexport const FOLDER_MARKER = "folder";/** The marker of the files the agent touched in a plain folder (touched-files policy); the server accepts it only while that policy is on. */export const TOUCHED_MARKER = "touched";+/** The marker of a folder without git that looks like a project (looksLikeProject), archived whole like a git repository; the server accepts it only while the project-folders policy is on. */+export const PROJECT_MARKER = "project";++// --- a folder without git that looks like a project ---------------------------++/** Manifests and lockfiles that make a folder a project (exact names, compared without case). */+const PROJECT_FILE_NAMES = new Set([+ "package.json", "pyproject.toml", "setup.py", "pipfile", "cargo.toml", "go.mod", "pom.xml", "gemfile", "composer.json",+ "cmakelists.txt", "makefile", "gnumakefile", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb",+ "poetry.lock", "uv.lock", "pipfile.lock", "cargo.lock", "go.sum", "gemfile.lock", "composer.lock",+]);+/** `requirements*.txt`, `build.gradle*`, `*.csproj`. */+const PROJECT_FILE_PATTERN = /^(?:requirements[^/]*\.txt|build\.gradle(?:\.kts)?|[^/]+\.csproj)$/i;+/** What a `src/` folder must hold (one file is enough) to make its folder a project. */+const CODE_FILE_PATTERN = /\.(?:c|cc|cpp|cs|go|h|hpp|java|js|jsx|kt|mjs|cjs|php|py|rb|rs|scala|swift|ts|tsx|vue|svelte|dart|lua|sh)$/i;+/** How deep below the root a manifest counts (the root is depth 0). */+const PROJECT_MAX_DEPTH = 2;+/** At most this many directory entries are looked at, and this many folders opened, so a 100k-file folder costs the same. */+const PROJECT_MAX_ENTRIES = 600;+const PROJECT_MAX_DIRS = 40;+const PROJECT_SRC_MAX_ENTRIES = 200;++export type ProjectEvidence = { path: string; kind: "manifest" | "src" };++/**+ * Whether a folder without git looks like a project: the root, or a folder at+ * most two levels below it, holds a manifest or lockfile (package.json,+ * pyproject.toml, requirements*.txt, Cargo.toml, go.mod, pom.xml,+ * build.gradle*, Gemfile, composer.json, *.csproj, CMakeLists.txt, Makefile,+ * ...), or a `src/` folder that holds a code file. A bounded breadth-first+ * walk (PROJECT_MAX_ENTRIES entries, PROJECT_MAX_DIRS folders) that never+ * follows a link and never enters a hidden, regenerable or denied folder+ * (`.git`, `node_modules`, `.venv`, `dist`, ...). The evidence, or null. Never throws.+ */+export async function looksLikeProject(root: string): Promise<ProjectEvidence | null> {+ let budget = PROJECT_MAX_ENTRIES;+ let dirs = 0;+ let level: string[] = [""];+ for (let depth = 0; depth <= PROJECT_MAX_DEPTH && level.length > 0; depth += 1) {+ const next: string[] = [];+ for (const rel of level) {+ if (budget <= 0 || dirs >= PROJECT_MAX_DIRS) return null;+ dirs += 1;+ let handle: Awaited<ReturnType<typeof opendir>>;+ try {+ handle = await opendir(rel ? join(root, ...rel.split("/")) : root);+ } catch {+ continue;+ }+ try {+ for await (const entry of handle) {+ if (budget <= 0) break;+ budget -= 1;+ const name = entry.name;+ const path = rel ? `${rel}/${name}` : name;+ if (entry.isFile()) {+ if (PROJECT_FILE_NAMES.has(name.toLowerCase()) || PROJECT_FILE_PATTERN.test(name)) return { path, kind: "manifest" };+ continue;+ }+ if (!entry.isDirectory() || name.startsWith(".") || REGENERABLE_DIR_NAMES.has(name) || isUploadDirectoryDenied(path)) continue;+ if (name.toLowerCase() === "src" && (await holdsCodeFile(join(root, ...path.split("/"))))) return { path, kind: "src" };+ if (depth < PROJECT_MAX_DEPTH) next.push(path);+ }+ } catch {+ // An unreadable folder says nothing.+ } finally {+ await handle.close().catch(() => undefined);+ }+ }+ level = next;+ }+ return null;+}++/** Whether a `src/` folder holds a code file directly (bounded; links are not followed). */+async function holdsCodeFile(dir: string): Promise<boolean> {+ let handle: Awaited<ReturnType<typeof opendir>>;+ try {+ handle = await opendir(dir);+ } catch {+ return false;+ }+ let seen = 0;+ try {+ for await (const entry of handle) {+ if (++seen > PROJECT_SRC_MAX_ENTRIES) return false;+ if (entry.isFile() && CODE_FILE_PATTERN.test(entry.name)) return true;+ }+ } catch {+ return false;+ } finally {+ await handle.close().catch(() => undefined);+ }+ return false;+}/** Top-level system and app directories of macOS and Linux (one list: the other system's names are absent). */const FOLDER_POSIX_SYSTEM_DIRS = [@@ -364,9 +459,11 @@ export async function folderRootRefusal(root: string, options: FolderGateOptions/*** The folder markers (4.4): a root with no `.git` entry at all, that* folderRootRefusal accepts, is a project with marker `folder` while- * `policy()` has `allFolders` on, or else with marker `touched` (only the- * files the agent touches there) while it has `touchedFiles` on. The policy- * is asked last, only for such a root. Never throws.+ * `policy()` has `allFolders` on, or else with marker `project` (archived+ * whole, like a git repository) when it looks like a project+ * (looksLikeProject) and `projectFolders` is on, or else with marker+ * `touched` (only the files the agent touches there) while it has+ * `touchedFiles` on. The policy is asked last, only for such a root. Never throws.*/export function folderDetector(policy: () => Promise<ArchivePolicy>, options: FolderGateOptions = {}): ProjectMarkerDetector {return async (root) => {@@ -375,6 +472,7 @@ export function folderDetector(policy: () => Promise<ArchivePolicy>, options: Foif (await folderRootRefusal(root, options)) return null;const answer = await policy();if (answer.allFolders) return { archivable: true, reason: "folder", marker: FOLDER_MARKER };+ if (answer.projectFolders === true && (await looksLikeProject(root))) return { archivable: true, reason: "project", marker: PROJECT_MARKER };return answer.touchedFiles ? { archivable: true, reason: "touched", marker: TOUCHED_MARKER } : null;} catch {return null;@@ -382,6 +480,33 @@ export function folderDetector(policy: () => Promise<ArchivePolicy>, options: Fo};}+/** Why a session folder is not archived, as a start upload reports it (`workspace.project_archive.reason`). */+export type ProjectArchiveSkipReason = "credential_root" | "root_too_broad" | "root_app_data" | "root_system";+const SKIP_REASONS: Record<FolderRefusal, ProjectArchiveSkipReason> = {+ root_credentials: "credential_root",+ root_too_broad: "root_too_broad",+ root_app_data: "root_app_data",+ root_system: "root_system",+};++/**+ * What the start upload says about a session folder the archive gate+ * refuses on purpose (a credential store such as `~/.aws`, the home+ * directory, app data, a system directory): `{status: "skipped", reason}`,+ * so the export can say why it has no project. Null for a git repository+ * (or a folder inside one), and for any folder the refusals let through+ * (whether it is archived then depends on the policy). Never throws.+ */+export async function projectArchiveSkip(root: string, options: FolderGateOptions & ProjectGateOptions = {}): Promise<{ status: "skipped"; reason: ProjectArchiveSkipReason } | null> {+ try {+ if ((await isArchivableProject(root, defaultProjectDetectors, options)).archivable) return null;+ const refusal = await folderRootRefusal(root, options);+ return refusal ? { status: "skipped", reason: SKIP_REASONS[refusal] } : null;+ } catch {+ return null;+ }+}+/*** Whether the session root is a project to archive: the root checks first,* then the detectors in order; the first archivable result wins, else thediff --git a/apps/server/src/session-archive/fake-archive-server.ts b/apps/server/src/session-archive/fake-archive-server.tsindex 729732e5..832ec08e 100644--- a/apps/server/src/session-archive/fake-archive-server.ts+++ b/apps/server/src/session-archive/fake-archive-server.ts@@ -151,7 +151,7 @@ export class FakeArchiveServer {}if (kind === "base" && (sequence !== 0 || parentId !== null)) return json(422, { detail: [] });// A folder or touched-files archive only while the key route's policy has its flag on (backend spec 4.4).- const flag = request.marker === "folder" ? "all_folders" : request.marker === "touched" ? "touched_files" : null;+ const flag = request.marker === "folder" ? "all_folders" : request.marker === "touched" ? "touched_files" : request.marker === "project" ? "project_folders" : null;const policy = new Map(typeof this.policy === "object" && this.policy !== null ? Object.entries(this.policy) : []);if (flag && policy.get(flag) !== true) return json(422, { detail: "archive_marker_not_allowed" });if (kid !== testKeys.kid) return json(409, { detail: "archive_kid_unknown" });diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.tsindex ab5c5545..d81ad809 100644--- a/apps/server/src/session-archive/index.ts+++ b/apps/server/src/session-archive/index.ts@@ -24,6 +24,8 @@ import {folderDetector,folderRootRefusal,isArchivableProject,+ looksLikeProject,+ PROJECT_MARKER,TOUCHED_MARKER,type FolderGateOptions,type ProjectMarkerDetector,@@ -60,6 +62,7 @@ import {ARCHIVE_SCHEMA_V2,captureV2Override,ExcludedList,+ MAX_EXCLUDED_HASH_BYTES,START_GATE_MS,STATE_MEMBER,stateDocument,@@ -119,6 +122,8 @@ export const POLICY_TTL_MS = 5 * 60_000;const OUTSIDE_CACHE_SUFFIX = "outside";/** The hash cache of the enclosing repository's .git (enclosing.ts) is kept under the root's key with this suffix. */const ENCLOSING_CACHE_SUFFIX = "enclosing";+/** The hash cache of a touched-files chain's listing of the files it does not hold (listNotArchived). */+const LISTING_CACHE_SUFFIX = "listing";/** Adds an outside scan (outside.ts) to a workspace scan: entries in archive order, gone paths and exclusions. */function mergeScans(scan: { entries: ScannedEntry[]; excluded: ExcludedCounts; gone?: Set<string> }, outside: Pick<TouchedScanResult, "entries" | "gone" | "excluded"> | null): void {@@ -311,9 +316,10 @@ class DiskBudgetPause extends Error {}}-/** Whether `policy` lets a chain with this marker capture: a plain folder needs all_folders, touched files touched_files, git nothing. */+/** Whether `policy` lets a chain with this marker capture: a plain folder needs all_folders, a git-less project project_folders, touched files touched_files, git nothing. */function markerAllowed(marker: string, policy: ArchivePolicy): boolean {if (marker === FOLDER_MARKER) return policy.allFolders;+ if (marker === PROJECT_MARKER) return policy.projectFolders === true;if (marker === TOUCHED_MARKER) return policy.touchedFiles;return true;}@@ -544,7 +550,7 @@ export class SessionArchiver {const next = state.next_sequence === 0 ? turn ?? state.turn_seen ?? 0 : turn ?? (state.last_turn ?? 0) + 1;if (state.last_turn !== null && next <= state.last_turn) return { status: "skipped", reason: "stale_turn" };// A plain folder's or touched files' archives pause while their policy is off; the next one after it is on again catches up.- if ((state.marker === FOLDER_MARKER || touched) && !(await this.policyAllows(state.marker, generation))) return { status: "skipped", reason: "not_archivable" };+ if ((state.marker === FOLDER_MARKER || state.marker === PROJECT_MARKER || touched) && !(await this.policyAllows(state.marker, generation))) return { status: "skipped", reason: "not_archivable" };const key = await this.currentKey();if (key === "disabled") return { status: "skipped", reason: "disabled" };if (key === "unavailable") return { status: "skipped", reason: "unavailable" };@@ -693,7 +699,15 @@ export class SessionArchiver {if (!marker) {const plain = await isArchivableProject(real, [async (candidate) => ((await folderRootRefusal(candidate, this.folderGate)) ? null : { archivable: true, reason: FOLDER_MARKER, marker: FOLDER_MARKER })], { appDirs: this.appDirs });if (!plain.archivable) return null;- marker = FOLDER_MARKER;+ // The marker the base will most likely get, from the policy kept now (the base checks it again;+ // a prescan of another marker is dropped): all folders, else a git-less project, else, with the+ // policy unknown, a whole folder as before. With the policy known and neither on there is no whole+ // folder to take (touched files, or nothing), so a big plain folder costs no start scan.+ const kept = this.policy?.value ?? null;+ if (kept?.allFolders) marker = FOLDER_MARKER;+ else if ((!kept || kept.projectFolders === true) && (await looksLikeProject(real))) marker = PROJECT_MARKER;+ else if (!kept) marker = FOLDER_MARKER;+ else return null;}const cache = await this.loadHashCache(stateKey(real));const excludedList = new ExcludedList();@@ -1045,7 +1059,7 @@ export class SessionArchiver {* the root checks and the folder refusals, and their policy is on.*/private async chainAllowed(state: SessionState, generation: number): Promise<boolean> {- if (state.marker !== FOLDER_MARKER && state.marker !== TOUCHED_MARKER) return (await isArchivableProject(state.root, this.detectors, { appDirs: this.appDirs })).archivable;+ if (state.marker !== FOLDER_MARKER && state.marker !== PROJECT_MARKER && state.marker !== TOUCHED_MARKER) return (await isArchivableProject(state.root, this.detectors, { appDirs: this.appDirs })).archivable;const accepted: ProjectMarkerDetector = async (root) => ((await folderRootRefusal(root, this.folderGate)) ? null : { archivable: true, reason: state.marker, marker: state.marker });if (!(await isArchivableProject(state.root, [accepted], { appDirs: this.appDirs })).archivable) return false;return this.policyAllows(state.marker, generation);@@ -1056,7 +1070,12 @@ export class SessionArchiver {if (generation !== this.generation) return;const value = this.policy?.value ?? POLICY_OFF;this.policy = {- value: { allFolders: value.allFolders && marker !== FOLDER_MARKER, touchedFiles: value.touchedFiles && marker !== TOUCHED_MARKER },+ value: {+ ...value,+ allFolders: value.allFolders && marker !== FOLDER_MARKER,+ touchedFiles: value.touchedFiles && marker !== TOUCHED_MARKER,+ ...(value.projectFolders === true && marker !== PROJECT_MARKER ? { projectFolders: true } : { projectFolders: false }),+ },at: this.now().getTime(),};}@@ -1198,14 +1217,16 @@ export class SessionArchiver {if (generation === this.generation) await this.noteUnchanged(state, kind, turn, options);return { status: "skipped", reason: "unchanged" };}+ // Capture v2: every file of the folder this chain does not hold is listed with its reason (never dropped silently).+ if (excludedList) await this.listNotArchived(state.root, change.next, excludedList, signal);files = change.files;deleted = kind === "delta" ? change.deleted : undefined;excluded = scan.excluded;ignored = scan.ignored;next = change.next;} else {- // A plain folder sends no git block (workspace.git null), even inside a larger repository.- const withGit = state.marker !== FOLDER_MARKER;+ // A plain folder or a git-less project sends no git block (workspace.git null), even inside a larger repository.+ const withGit = state.marker !== FOLDER_MARKER && state.marker !== PROJECT_MARKER;const gitReading = kind === "base" && withGit ? readArchiveGit(state.root) : null;const scan = prescan? { entries: [...prescan.result.scan.entries], excluded: { ...prescan.result.scan.excluded }, ignored: prescan.result.scan.ignored }@@ -1440,8 +1461,11 @@ export class SessionArchiver {if (!prescan) return null;this.dropPrescan(sessionId);const result = await prescan.work;- if (!result || result.marker !== marker || resolve(prescan.root) !== resolve(root)) return null;- if (!v2 && marker === FOLDER_MARKER) return null;+ // A plain folder and a git-less project are scanned alike in a v2 chain (the regenerable folders left out, no git block).+ const plain = (value: string) => value === FOLDER_MARKER || value === PROJECT_MARKER;+ const same = result?.marker === marker || (v2 && result !== null && plain(result.marker) && plain(marker));+ if (!result || !same || resolve(prescan.root) !== resolve(root)) return null;+ if (!v2 && plain(marker)) return null;return { result, capture: prescan.capture };}@@ -1466,7 +1490,8 @@ export class SessionArchiver {includeCredentialFiles: this.includeCredentials,hashCache: cache,...(excludedList ? { excludedList } : {}),- ...(v2 && marker === FOLDER_MARKER ? { pruneRegenerable: true } : {}),+ // A git-less project leaves the regenerable folders out like a v2 plain folder (they have no .gitignore to say so).+ ...((v2 && marker === FOLDER_MARKER) || marker === PROJECT_MARKER ? { pruneRegenerable: true } : {}),...(signal ? { signal } : {}),});// What the ignore rules left out is still archived when the agent@@ -1486,6 +1511,51 @@ export class SessionArchiver {return scan;}+ /**+ * Capture v2, a touched-files chain: lists every entry of the folder the+ * chain does not hold (`held`, its entries after this archive). A file the+ * agent never touched is `not_archived_touched_scope` with its size and+ * SHA-256 (up to MAX_EXCLUDED_HASH_BYTES; a larger one with its mtime);+ * what the whole-folder rules leave out keeps their reason (`credential`,+ * `gitignored`, `regenerable` folders, ...). A stat walk with its own hash+ * cache (an unchanged file is never read twice), at capture time, never+ * in the start gate.+ */+ private async listNotArchived(root: string, held: readonly ArchiveEntry[], excludedList: ExcludedList, signal?: AbortSignal): Promise<void> {+ const cacheKey = stateKey(`${root}\0${LISTING_CACHE_SUFFIX}`);+ const cache = await this.loadHashCache(cacheKey);+ const rules = new ExcludedList();+ let scan: { entries: ScannedEntry[] };+ try {+ scan = await scanArchiveTree(root, {+ excludedDirs: this.appDirs,+ includeCredentialFiles: this.includeCredentials,+ hashCache: cache,+ excludedList: rules,+ pruneRegenerable: true,+ maxHashBytes: MAX_EXCLUDED_HASH_BYTES,+ ...(signal ? { signal } : {}),+ });+ } catch (error) {+ if (signal?.aborted) throw error;+ this.log("warn", "OmniRush touched-files listing failed", { error: errorSummary(error) });+ return;+ }+ const archived = new Set(held.map((entry) => entry.path));+ for (const item of rules.list()) if (!archived.has(item.path)) excludedList.add(item);+ for (const entry of scan.entries) {+ if (entry.type === "dir" || archived.has(entry.path)) continue;+ excludedList.add({+ path: entry.path,+ type: entry.type === "symlink" ? "symlink" : "file",+ reason: "not_archived_touched_scope",+ ...(entry.type === "file" ? { size: entry.size } : {}),+ ...(entry.sha256 ? { sha256: entry.sha256 } : entry.type === "file" ? { mtime: entry.mtime } : {}),+ });+ }+ if (cache.changed) await this.saveHashCache(cacheKey, cache).catch(() => undefined);+ }+/*** Capture v2 (#14): a touched file the folder scan did not hold (gitignored,* in a regenerable folder) larger than the per-file cap is left out withdiff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.tsindex 19d97799..07ef4eeb 100644--- a/apps/server/src/session-archive/manifest.ts+++ b/apps/server/src/session-archive/manifest.ts@@ -365,6 +365,8 @@ export type ScanOptions = {excludedList?: ExcludedList;/** Capture v2 (a folder in no repository): the regenerable folders (node_modules, .venv, target, ...) are left out. */pruneRegenerable?: boolean;+ /** A listing scan (never packed): a file larger than this is not hashed (its sha256 stays null). */+ maxHashBytes?: number;};/**@@ -636,6 +638,10 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):entries.push(entry);return;}+ if (options.maxHashBytes !== undefined && entry.size > options.maxHashBytes) {+ entries.push(entry);+ return;+ }const cached = cache?.lookup(entry) ?? null;if (cached) {entry.sha256 = cached;diff --git a/apps/server/src/session-archive/policy.ts b/apps/server/src/session-archive/policy.tsindex d36991d6..3a3d898f 100644--- a/apps/server/src/session-archive/policy.ts+++ b/apps/server/src/session-archive/policy.ts@@ -2,13 +2,16 @@* The archive policy GET /archives/key sends beside the key (backend spec* 4.4). `policy.all_folders: true` archives every folder a session starts* in, not only git repositories; `policy.touched_files: true` archives, in- * a folder that is neither, the files the agent touched there. Anything+ * a folder that is neither, the files the agent touched there;+ * `policy.project_folders: true` archives a folder without git that looks+ * like a project (a manifest or lockfile, or a `src/` folder with code)+ * whole, like a git repository. Anything* else (no policy, a server that predates it, a value that is not the* boolean true) is off: git only.*/import { z } from "zod";-export type ArchivePolicy = { allFolders: boolean; touchedFiles: boolean; captureV2?: boolean };+export type ArchivePolicy = { allFolders: boolean; touchedFiles: boolean; captureV2?: boolean; projectFolders?: boolean };/** Both off: what a failed probe, a missing policy or a disabled account means. */export const POLICY_OFF: ArchivePolicy = { allFolders: false, touchedFiles: false };@@ -16,6 +19,8 @@ export const POLICY_OFF: ArchivePolicy = { allFolders: false, touchedFiles: falsconst allFoldersSchema = z.object({ policy: z.object({ all_folders: z.literal(true) }) });const touchedFilesSchema = z.object({ policy: z.object({ touched_files: z.literal(true) }) });/** Capture v2 (capture-v2.ts): the server takes the byte-exact state archives. */+/** A git-less project folder (detect.ts `looksLikeProject`) is archived whole, marker `project`. */+const projectFoldersSchema = z.object({ policy: z.object({ project_folders: z.literal(true) }) });const captureV2Schema = z.object({ policy: z.object({ capture_v2: z.literal(true) }) });export function parseArchivePolicy(body: unknown): ArchivePolicy {@@ -24,5 +29,6 @@ export function parseArchivePolicy(body: unknown): ArchivePolicy {touchedFiles: touchedFilesSchema.safeParse(body).success,// Present only when on: a policy without it reads exactly as before....(captureV2Schema.safeParse(body).success ? { captureV2: true } : {}),+ ...(projectFoldersSchema.safeParse(body).success ? { projectFolders: true } : {}),};}diff --git a/apps/server/src/session-archive/project-scope.test.ts b/apps/server/src/session-archive/project-scope.test.tsnew file mode 100644index 00000000..a393e2da--- /dev/null+++ b/apps/server/src/session-archive/project-scope.test.ts@@ -0,0 +1,206 @@+import { afterAll, describe, expect, test } from "bun:test";+import { createHash, randomBytes } from "node:crypto";+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";+import { homedir, tmpdir } from "node:os";+import { dirname, join } from "node:path";+import { zstdDecompressSync } from "node:zlib";++import { SessionUploader } from "../session-uploader.js";+import { looksLikeProject, projectArchiveSkip } from "./detect.js";+import { SessionArchiver } from "./index.js";+import { parseArchivePolicy } from "./policy.js";+import { openSealedBuffer, SEAL_ALG, sealKeyring } from "./seal.js";+import { readTar } from "./test-helpers.js";++// A folder without git: a project-looking one is archived whole (marker+// `project`), any other in touched-files scope lists every file it leaves out,+// and a credential store is refused with a reason the start upload carries.++const cleanups: string[] = [];+const stops: Array<() => Promise<void>> = [];+afterAll(async () => {+ for (const stop of stops) await stop();+ for (const dir of cleanups) rmSync(dir, { recursive: true, force: true });+});+const sha256 = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex");+function temp(prefix: string): string {+ const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix)));+ cleanups.push(dir);+ return dir;+}+function write(file: string, content: string | Buffer): Buffer {+ mkdirSync(dirname(file), { recursive: true });+ writeFileSync(file, content);+ return Buffer.isBuffer(content) ? content : Buffer.from(content);+}++type Item = { path: string; type: string; reason: string; size?: number; sha256?: string; mtime?: number };+function rig(policy: Record<string, unknown>) {+ const stateDir = temp("omnirush-scope-state-");+ const keyring = sealKeyring([randomBytes(32)]);+ const { publicRaw, kid } = [...keyring.values()][0]!;+ const created: Array<Record<string, unknown>> = [];+ const request = async (route: string, init?: { body?: unknown }) => {+ if (route === "archives/key") return Response.json({ public_key: publicRaw.toString("base64"), kid, alg: SEAL_ALG, policy });+ if (route === "archives" && init?.body) created.push(JSON.parse(String(init.body)));+ return new Response("{}", { status: 404 });+ };+ const archiver = new SessionArchiver({ stateDir, request: request as never, excludedDirs: [stateDir], log: () => undefined, touchedFlushMs: 10 });+ stops.push(() => archiver.stop({ budgetMs: 1000 }));+ const open = async (result: Awaited<ReturnType<SessionArchiver["captureBase"]>>) => {+ if (result.status !== "queued") throw new Error(`not queued: ${JSON.stringify(result)}`);+ const sealed = readFileSync(join(stateDir, "omnirush-archive", "pending", `${result.archiveId}.orseal`));+ const members = new Map(readTar(zstdDecompressSync(await openSealedBuffer(sealed, keyring))).map((member) => [member.name.replace(/\/$/, ""), member]));+ const json = (name: string) => JSON.parse(members.get(name)!.content.toString("utf8"));+ return { members, manifest: json("__omnirush__/manifest.json"), state: json("__omnirush__/state.json") as { excluded: Item[]; excluded_truncated: boolean; excluded_truncated_count: number } };+ };+ return { archiver, open, created };+}++describe("looksLikeProject", () => {+ test("a manifest or lockfile in the root or two levels down, or src/ with code, makes a project; deeper or in node_modules does not", async () => {+ for (const [file, expected] of [+ ["package.json", "package.json"], ["requirements-dev.txt", "requirements-dev.txt"], ["app/pyproject.toml", "app/pyproject.toml"],+ ["services/api/go.mod", "services/api/go.mod"], ["x/App.csproj", "x/App.csproj"], ["build.gradle.kts", "build.gradle.kts"],+ ["Makefile", "Makefile"], ["CMakeLists.txt", "CMakeLists.txt"], ["web/Cargo.toml", "web/Cargo.toml"],+ ] as const) {+ const root = temp("omnirush-project-");+ write(join(root, file), "x");+ expect((await looksLikeProject(root))?.path).toBe(expected);+ }+ const src = temp("omnirush-project-src-");+ write(join(src, "src", "main.py"), "print(1)\n");+ expect(await looksLikeProject(src)).toEqual({ path: "src", kind: "src" });+ const none = temp("omnirush-project-none-");+ write(join(none, "a", "b", "c", "package.json"), "{}");+ write(join(none, "node_modules", "package.json"), "{}");+ write(join(none, ".cache", "package.json"), "{}");+ write(join(none, "src", "notes.txt"), "not code");+ write(join(none, "notes.txt"), "hello");+ expect(await looksLikeProject(none)).toBeNull();+ });++ test("bounded on a big folder: 20,000 files without a manifest are judged in milliseconds", async () => {+ const root = temp("omnirush-project-big-");+ for (let bucket = 0; bucket < 40; bucket += 1) for (let n = 0; n < 500; n += 1) write(join(root, `bucket_${bucket}`, `f_${n}.txt`), `${bucket}/${n}\n`);+ const started = performance.now();+ expect(await looksLikeProject(root)).toBeNull();+ expect(performance.now() - started).toBeLessThan(500);+ });+});++describe("credential roots", () => {+ test("a session in ~/.aws (or inside it) is skipped with reason credential_root; a plain folder or a git repository is not", async () => {+ const home = temp("omnirush-home-");+ write(join(home, ".aws", "config"), "[default]\n");+ write(join(home, ".aws", "sso", "cache", "x.json"), "{}");+ write(join(home, "work", "notes.txt"), "hi");+ expect(await projectArchiveSkip(join(home, ".aws"), { homeDir: home })).toEqual({ status: "skipped", reason: "credential_root" });+ expect(await projectArchiveSkip(join(home, ".aws", "sso", "cache"), { homeDir: home })).toEqual({ status: "skipped", reason: "credential_root" });+ expect(await projectArchiveSkip(home, { homeDir: home })).toEqual({ status: "skipped", reason: "root_too_broad" });+ expect(await projectArchiveSkip(join(home, "work"), { homeDir: home })).toBeNull();+ mkdirSync(join(home, "work", ".git"));+ expect(await projectArchiveSkip(join(home, "work"), { homeDir: home })).toBeNull();+ });+});++describe("policy", () => {+ test("project_folders is on only as the boolean true", () => {+ expect(parseArchivePolicy({ policy: { all_folders: false, touched_files: true, project_folders: true } }).projectFolders).toBe(true);+ for (const policy of [{}, { project_folders: false }, { project_folders: "true" }]) expect(parseArchivePolicy({ policy }).projectFolders).toBeUndefined();+ });+});++describe("a folder without git", () => {+ test("a project folder is archived whole with marker project: regenerable folders and credentials listed, no git block", async () => {+ const { archiver, open } = rig({ all_folders: false, touched_files: true, project_folders: true, capture_v2: true });+ const root = temp("omnirush-scope-project-");+ write(join(root, "package.json"), '{"name":"demo"}\n');+ write(join(root, "src", "index.ts"), "export const x = 1;\n");+ for (let n = 0; n < 50; n += 1) write(join(root, "data", `row_${n}.csv`), `${n}\n`);+ write(join(root, "node_modules", "dep", "index.js"), "module.exports = 1;\n");+ write(join(root, ".env"), "API_KEY=sk-live-secret\n");+ const sessionId = "ses_scope_project";+ expect(await archiver.startManifest(sessionId, root)).toBe("complete");+ const base = await open(await archiver.captureBase(sessionId, root, 0));+ expect(base.manifest.workspace.marker).toBe("project");+ expect(base.manifest.workspace.git).toBeNull();+ expect(base.members.has("data/row_49.csv")).toBe(true);+ expect(base.members.has("src/index.ts")).toBe(true);+ expect(base.members.has("node_modules/dep/index.js") || base.members.has(".env")).toBe(false);+ const reasons = new Map(base.state.excluded.map((item) => [item.path, item.reason]));+ expect([reasons.get("node_modules"), reasons.get(".env")]).toEqual(["regenerable", "credential"]);+ });++ test("without project_folders (an older backend) a project folder stays in touched scope", async () => {+ const { archiver } = rig({ all_folders: false, touched_files: true, capture_v2: true });+ const root = temp("omnirush-scope-oldbackend-");+ write(join(root, "package.json"), "{}");+ expect(await archiver.captureBase("ses_scope_oldbackend", root, 0)).toEqual({ status: "skipped", reason: "unchanged" });+ });++ test("touched scope lists every file it does not archive with size and sha256 (a big one with its mtime), never silently", async () => {+ const { archiver, open } = rig({ all_folders: false, touched_files: true, project_folders: true, capture_v2: true });+ const root = temp("omnirush-scope-touched-");+ for (let n = 0; n < 30; n += 1) write(join(root, "bucket", `f_${n}.txt`), `file ${n}\n`);+ const huge = join(root, "huge.bin");+ write(huge, Buffer.alloc(0));+ // Sparse 65 MiB: over the hash cap without writing it.+ const { truncateSync } = await import("node:fs");+ truncateSync(huge, 65 * 1024 * 1024);+ write(join(root, ".env"), "TOKEN=abcdefgh123\n");+ write(join(root, "node_modules", "x", "i.js"), "1\n");+ const sessionId = "ses_scope_touched";+ expect(await archiver.captureBase(sessionId, root, 0)).toEqual({ status: "skipped", reason: "unchanged" });+ write(join(root, "NOTES.txt"), "the agent wrote this\n");+ archiver.recordTouched(sessionId, "NOTES.txt");+ const base = await open(await archiver.captureDelta(sessionId, root, 1));+ expect(base.manifest.scope).toBe("touched");+ expect(base.members.has("NOTES.txt")).toBe(true);+ const items = new Map(base.state.excluded.map((item) => [item.path, item]));+ expect(items.has("NOTES.txt")).toBe(false);+ for (let n = 0; n < 30; n += 1) {+ const item = items.get(`bucket/f_${n}.txt`)!;+ expect(item).toEqual({ path: `bucket/f_${n}.txt`, type: "file", reason: "not_archived_touched_scope", size: `file ${n}\n`.length, sha256: sha256(`file ${n}\n`) });+ }+ const big = items.get("huge.bin")!;+ expect([big.reason, big.size, big.sha256, typeof big.mtime]).toEqual(["not_archived_touched_scope", 65 * 1024 * 1024, undefined, "number"]);+ expect(items.get(".env")!.reason).toBe("credential");+ expect(items.get("node_modules")!.reason).toBe("regenerable");+ expect(base.state.excluded_truncated).toBe(false);+ expect(base.state.excluded_truncated_count).toBe(0);+ });++ test("a non-project folder is not prescanned once the policy says touched scope", async () => {+ const { archiver } = rig({ all_folders: false, touched_files: true, project_folders: true, capture_v2: true });+ const root = temp("omnirush-scope-noprescan-");+ write(join(root, "a.txt"), "a\n");+ // The first session learns the policy; the next one's start gate takes no whole-folder scan.+ await archiver.captureBase("ses_scope_learn_1", root, 0);+ expect(await archiver.startManifest("ses_scope_learn_2", root)).toBe("skipped");+ });+});++describe("start upload", () => {+ test("a session in a credential store says why it has no project archive; a plain folder says nothing", async () => {+ const parent = temp("omnirush-scope-upload-");+ const results: Record<string, unknown> = {};+ for (const name of [".aws", "plain"]) {+ const root = join(parent, name);+ write(join(root, "config"), "[default]\nregion = eu-west-1\n");+ const uploads: Array<Record<string, any>> = [];+ const upload = async (_sessionId: string, compressed: Uint8Array) => {+ uploads.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")));+ return Response.json({ ok: true }, { status: 201 });+ };+ const uploader = new SessionUploader({ upload, fallbackScanMs: 60_000 });+ uploader.startSession(`ses_scope_upload_${name.replace(".", "")}`, "ws", root);+ await uploader.stop();+ const start = uploads.find((item) => item.snapshot_type === "start")!;+ results[name] = start.workspace.project_archive ?? null;+ }+ expect(results).toEqual({ ".aws": { status: "skipped", reason: "credential_root" }, plain: null });+ });+});++void homedir;diff --git a/apps/server/src/session-uploader-file-mode.test.ts b/apps/server/src/session-uploader-file-mode.test.tsindex 7e8fdde2..604e08b6 100644--- a/apps/server/src/session-uploader-file-mode.test.ts+++ b/apps/server/src/session-uploader-file-mode.test.ts@@ -64,7 +64,7 @@ describe("file-aware trace scrub", () => {"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abcdefghijklmnop","Authorization: Bearer abcDEF1234567890xyzQ","postgres://user:[email protected]/app",- "mail [email protected]",+ "mail [email protected]",]) {const state = stateOf(scrubTrace(toolEvent("edit", { input: { filePath: "src/x.py", oldString: "a", newString: shape } })));expect(state.input.newString).not.toBe(shape);diff --git a/apps/server/src/session-uploader.test.ts b/apps/server/src/session-uploader.test.tsindex 8a42ecf2..832d11c0 100644--- a/apps/server/src/session-uploader.test.ts+++ b/apps/server/src/session-uploader.test.ts@@ -68,10 +68,10 @@ describe("session uploader privacy", () => {});test("scrubs common personal identifiers before upload", () => {- const result = redactUploadText("[email protected] +1 (415) 555-0132 192.0.2.25");- expect(result.text).not.toContain("[email protected]");+ const result = redactUploadText("[email protected] +1 (415) 555-0132 52.14.7.25");+ expect(result.text).not.toContain("[email protected]");expect(result.text).not.toContain("415");- expect(result.text).not.toContain("192.0.2.25");+ expect(result.text).not.toContain("52.14.7.25");expect(result.count).toBe(3);});@@ -230,7 +230,7 @@ describe("session uploader privacy", () => {const sessionId = "session-changes-1234";sessionUploader.startSession(sessionId, "workspace-changes", root);await sessionUploader.idle(sessionId);- await writeFile(join(root, "created.txt"), "contact [email protected]");+ await writeFile(join(root, "created.txt"), "contact [email protected]");sessionUploader.recordTrace(sessionId, "file.read", { path: "created.txt" });sessionUploader.flushTrace(sessionId);await new Promise((resolve) => setTimeout(resolve, 40));@@ -241,7 +241,7 @@ describe("session uploader privacy", () => {const changeFile = (change?.files as Array<{ path: string; content: string }>).find((file) => file.path === "__omnirush__/changes.json");expect(changeFile).toBeDefined();expect(changeFile?.content).toContain('"path":"created.txt"');- expect(changeFile?.content).not.toContain("[email protected]");+ expect(changeFile?.content).not.toContain("[email protected]");});test("records a privacy manifest and touched paths without uploading denied files", async () => {@@ -409,7 +409,7 @@ function makeUploads() {}async function git(root: string, ...args: string[]): Promise<string> {- const { stdout } = await execFileAsync("git", ["-C", root, "-c", "commit.gpgsign=false", "-c", "user.name=Dev", "-c", "[email protected]", ...args]);+ const { stdout } = await execFileAsync("git", ["-C", root, "-c", "commit.gpgsign=false", "-c", "user.name=Dev", "-c", "[email protected]", ...args]);return String(stdout).trim();}@@ -527,7 +527,7 @@ describe("session uploader envelope v2", () => {await writeFile(join(root, "credentials.json"), '{"password":"hunter2-committed"}\n');await writeFile(join(root, "blob.bin"), Buffer.from([0, 1, 2, 3, 4]));await git(root, "add", "-A");- await git(root, "commit", "-q", "-m", "initial commit by [email protected]");+ await git(root, "commit", "-q", "-m", "initial commit by [email protected]");await writeFile(join(root, "app.txt"), "line one\nline two added\n");await writeFile(join(root, "credentials.json"), '{"password":"hunter2-modified"}\n');await writeFile(join(root, "blob.bin"), Buffer.from([0, 9, 9, 9, 9, 9]));@@ -578,7 +578,7 @@ describe("session uploader envelope v2", () => {expect(start.manifest.find((entry) => entry.path === "blob.bin")).toMatchObject({ binary: true, archived: "byte_exact", size: 6 });expect(start.files.some((file) => file.path === "blob.bin")).toBe(false);const serialized = JSON.stringify(uploads);- for (const forbidden of ["ghp_secrettoken123456", "oauth2:", "hunter2", "abcdef123456", "[email protected]", ".git/"]) {+ for (const forbidden of ["ghp_secrettoken123456", "oauth2:", "hunter2", "abcdef123456", "[email protected]", ".git/"]) {expect(serialized).not.toContain(forbidden);}});@@ -1495,7 +1495,7 @@ describe("session uploader trace additions", () => {parentSessionId: sessionId,title: "Subtask",agent: "explore",- messages: [{ info: { id: "cmsg_1", role: "user", sessionID: "ses_child_1" }, parts: [{ type: "text", text: "find [email protected]" }] }],+ messages: [{ info: { id: "cmsg_1", role: "user", sessionID: "ses_child_1" }, parts: [{ type: "text", text: "find [email protected]" }] }],lastMessageId: "cmsg_1",});first.recordChildSession(sessionId, {@@ -1523,7 +1523,7 @@ describe("session uploader trace additions", () => {["ses_grandchild_1", "ses_child_1", null, null],]);expect(JSON.stringify(children[0]?.data?.messages)).toContain("cmsg_1");- expect(JSON.stringify(children)).not.toContain("[email protected]");+ expect(JSON.stringify(children)).not.toContain("[email protected]");const end = uploads.at(-1)!;expect(end.session).toEqual({provider_id: "anthropic", model_id: "claude-sonnet-4-5", variant: "high", child_session_ids: ["ses_child_1", "ses_grandchild_1"],@@ -1622,8 +1622,8 @@ describe("session uploader trace additions", () => {const sessionUploader = new SessionUploader({ upload, fallbackScanMs: 60_000 });const sessionId = "session-web-1234";sessionUploader.startSession(sessionId, "workspace-web", root);- const text = `Contact [email protected] about AKIA1234567890123456 ${"page text ".repeat(20_000)}`;- expect(sessionUploader.recordWebVisit(sessionId, { url: "https://user:[email protected]/docs?q=1#top", title: "Docs [email protected]", text })).toBe(true);+ const text = `Contact [email protected] about AKIA1234567890123456 ${"page text ".repeat(20_000)}`;+ expect(sessionUploader.recordWebVisit(sessionId, { url: "https://user:[email protected]/docs?q=1#top", title: "Docs [email protected]", text })).toBe(true);expect(sessionUploader.recordWebVisit(sessionId, { url: "https://example.org/short", title: null, text: "brief" })).toBe(true);for (const url of ["file:///etc/passwd", "data:text/html,<p>hi</p>", "chrome://settings", "http://localhost:3000/app", "http://127.0.0.1:8080/", "https://[::1]/", "ftp://example.com/x"]) {expect(sessionUploader.recordWebVisit(sessionId, { url, title: "local", text: "secret local page" })).toBe(false);@@ -1638,7 +1638,7 @@ describe("session uploader trace additions", () => {expect(long?.data?.title).toBe("Docs [REDACTED_PII]");expect(long?.data?.text_truncated).toBe(true);expect(Buffer.byteLength(String(long?.data?.text))).toBeLessThanOrEqual(MAX_UPLOAD_WEB_VISIT_TEXT_BYTES);- expect(String(long?.data?.text)).not.toContain("[email protected]");+ expect(String(long?.data?.text)).not.toContain("[email protected]");expect(String(long?.data?.text)).not.toContain("AKIA1234567890123456");expect(short?.data).toEqual({ url: "https://example.org/short", title: null, text: "brief", text_truncated: false });expect(JSON.stringify(uploads)).not.toContain("secret local page");@@ -1653,7 +1653,7 @@ describe("session uploader trace additions", () => {const sessionId = "session-attachment-1234";sessionUploader.startSession(sessionId, "workspace-attachment", root);const text = `OPENAI_API_KEY=sk-1234567890abcdefghijklmnop\n${"notes ".repeat(60_000)}`;- sessionUploader.recordAttachment(sessionId, { name: "../notes [email protected] report.txt", mime: "text/plain", bytes: Buffer.byteLength(text), sha256: sha256(text), text });+ sessionUploader.recordAttachment(sessionId, { name: "../notes [email protected] report.txt", mime: "text/plain", bytes: Buffer.byteLength(text), sha256: sha256(text), text });sessionUploader.recordAttachment(sessionId, { name: "photo.png", mime: "image/png", bytes: 12, sha256: sha256("png"), text: null });sessionUploader.flushTrace(sessionId);await sessionUploader.stop();@@ -1982,7 +1982,7 @@ describe("session uploader secret rails", () => {["settings.py", "SECRET = Ab1/Cd2+Ef3=", "SECRET = [REDACTED]"],["ci.yml", `token: ${GITHUB_TOKEN}`, "token: [REDACTED]"],["config.yml", 'api_key: "abcdef1234567890"', 'api_key: "[REDACTED]"'],- ["config.yml", "password: changeme1", "password: [REDACTED]"],+ ["config.yml", "password: hunter7abc", "password: [REDACTED]"],["config.yml", "META_MUSE_API_KEY: 'synthetic'", "META_MUSE_API_KEY: '[REDACTED]'"],["config.yml", 'token: "config.token"', 'token: "[REDACTED]"'],["config.yml", "token: config-token", "token: [REDACTED]"],@@ -2050,14 +2050,14 @@ describe("session uploader secret rails", () => {test("scrubs the trace and .json files as JSON so escapes survive redaction", async () => {const root = await mkdtemp(join(tmpdir(), "omnirush-upload-json-"));roots.push(root);- await writeFile(join(root, "config.json"), '{\n "note": "{\\"password\\":\\"hunter2abc\\"}",\n "nested": {"apiKey": "abcdefghijkl", "author": "Jane <[email protected]>"}\n}\n');+ await writeFile(join(root, "config.json"), '{\n "note": "{\\"password\\":\\"hunter2abc\\"}",\n "nested": {"apiKey": "abcdefghijkl", "author": "Jane <[email protected]>"}\n}\n');await writeFile(join(root, "settings.json"), '// JSON with comments falls back to text scrubbing\n{"password": "hunter2abc"}\n');await writeFile(join(root, "data.json"), '{"a": 1.0, "b": [1, 2]}');const { uploads, upload } = makeUploads();const sessionUploader = new SessionUploader({ upload, fallbackScanMs: 60_000 });const sessionId = "session-json-1";sessionUploader.startSession(sessionId, "workspace-json", root);- sessionUploader.recordTrace(sessionId, "message", { text: 'line one\[email protected](gpu="h100")\[email protected]', token: GITHUB_TOKEN });+ sessionUploader.recordTrace(sessionId, "message", { text: 'line one\[email protected](gpu="h100")\[email protected]', token: GITHUB_TOKEN });sessionUploader.flushTrace(sessionId);await sessionUploader.stop();@@ -2082,7 +2082,7 @@ describe("session uploader secret rails", () => {});test("never leaves a dangling backslash when scrubbing escaped text", () => {- const raw = JSON.stringify({ text: `line one\[email protected](gpu="h100")\[email protected]\[email protected]\tAKIAIOSFODNN7EXAMPLE\nTOKEN=${GITHUB_TOKEN}` });+ const raw = JSON.stringify({ text: `line one\[email protected](gpu="h100")\[email protected]\[email protected]\tAKIAIOSFODNN7EXAMPLE\nTOKEN=${GITHUB_TOKEN}` });const result = redactUploadText(raw);expect(JSON.parse(result.text)).toEqual({ text: 'line one\[email protected](gpu="h100")\n[REDACTED_PII]\n[REDACTED_PII]\t[REDACTED]\nTOKEN=[REDACTED]' });expect(result.count).toBe(4);@@ -2094,7 +2094,7 @@ describe("session uploader secret rails", () => {expect(redactUploadJsonText('{"a":{"toJSON":1},"secret":"abcdefghijkl","n":1e3}')).toBe('{"a":{"toJSON":1},"secret":"[REDACTED]","n":1000}');expect(redactUploadJsonText("not json")).toBeNull();// "secrets" is a family word, the plural "tokens" deliberately is not (tokenizer configs).- expect(redactUploadJson({ at: new Date(0), secrets: ["abcdefghijkl", "short"], tokens: ["abcdefghijkl"], "[email protected]": 1 }))+ expect(redactUploadJson({ at: new Date(0), secrets: ["abcdefghijkl", "short"], tokens: ["abcdefghijkl"], "[email protected]": 1 })).toEqual({ value: { at: "1970-01-01T00:00:00.000Z", secrets: ["[REDACTED]", "short"], tokens: ["abcdefghijkl"], "[REDACTED_PII]": 1 }, count: 2 });expect(redactUploadContent("nested/x.JSON", '{"password":"hunter2abc"}')).toBe('{"password":"[REDACTED]"}');expect(redactUploadContent("x.txt", '{"password":"hunter2abc"}')).toBe('{"password":"[REDACTED]"}');@@ -2111,13 +2111,13 @@ describe("session uploader secret rails", () => {// Both rules are applied only to lines holding their keyword or "@"; the// result must equal a whole-text pass, line endings and escapes included.expect(redactUploadContent("notes.txt", "first line\npassword=hunter2hunter2\r\nlast")).toBe("first line\npassword=[REDACTED]\r\nlast");- expect(redactUploadContent("notes.txt", "a@b\[email protected]\n@x\[email protected]")).toBe("a@b\n[REDACTED_PII]\n@x\n[REDACTED_PII]");+ expect(redactUploadContent("notes.txt", "a@b\[email protected]\n@x\[email protected]")).toBe("a@b\n[REDACTED_PII]\n@x\n[REDACTED_PII]");expect(redactUploadContent("notes.txt", "jane@\nexample.com and key\n=abcdefgh12")).toBe("jane@\nexample.com and key\n=abcdefgh12");expect(redactUploadContent("run.sh", "echo done \\\nexport API_KEY=abc123def456\nTOKEN='S3cretValue99'")).toBe("echo done \\\nexport API_KEY=abc123def456\nTOKEN='[REDACTED]'");- expect(redactUploadContent("app.ts", "const keyName = config.token;\nconst apiKey = \"sk-proj-abcdefghijklmnop1234\";\n// contact: [email protected]"))+ expect(redactUploadContent("app.ts", "const keyName = config.token;\nconst apiKey = \"sk-proj-abcdefghijklmnop1234\";\n// contact: [email protected]")).toBe("const keyName = config.token;\nconst apiKey = \"[REDACTED]\";\n// contact: [REDACTED_PII]");- expect(redactUploadContent("data.json", "{\n \"note\": \"line\\npassword=hunter2hunter2\",\n \"author\": \"[email protected]\"\n}\n"))+ expect(redactUploadContent("data.json", "{\n \"note\": \"line\\npassword=hunter2hunter2\",\n \"author\": \"[email protected]\"\n}\n")).toBe("{\n \"note\": \"line\\npassword=[REDACTED]\",\n \"author\": \"[REDACTED_PII]\"\n}\n");});@@ -2504,7 +2504,7 @@ describe("session uploader scrubber parity: second-sample refinements", () => {test("JSON repeated keys are kept and scrubbed (test_json_repeated_keys_are_kept_and_scrubbed)", () => {const deep = `{"a": 1, "a": ${"[".repeat(600)}${"]".repeat(600)}}`;const cases: Array<[string, string]> = [- ['{"a": "[email protected]", "a": "clean"}', '{"a":"[REDACTED_PII]","a":"clean"}'],+ ['{"a": "[email protected]", "a": "clean"}', '{"a":"[REDACTED_PII]","a":"clean"}'],[`{"a": "${GITHUB_TOKEN}", "a": "clean"}`, '{"a":"[REDACTED]","a":"clean"}'],['{"password": "hunter2abc", "password": "x"}', '{"password":"[REDACTED]","password":"x"}'],[@@ -3508,7 +3508,7 @@ describe("session uploader incremental snapshots", () => {test("a new chat on a captured workspace sends scrubbed files from the redacted-text cache, not the scrubber", async () => {const root = await workspace("texts", 4);- await writeFile(join(root, "src", "contact.txt"), "mail [email protected] about it\n");+ await writeFile(join(root, "src", "contact.txt"), "mail [email protected] about it\n");await writeFile(join(root, "src", "config.yml"), "password: hunter2hunter2\n");const { uploads, upload } = makeUploads();const sessionUploader = new SessionUploader({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });@@ -3532,7 +3532,7 @@ describe("session uploader incremental snapshots", () => {expect(second.manifest).toEqual(first.manifest);// New bytes are scrubbed again, never served from the text of the old ones.- await writeFile(join(root, "src", "contact.txt"), "mail [email protected] instead of jane\n");+ await writeFile(join(root, "src", "contact.txt"), "mail [email protected] instead of jane\n");const edited = { ...sessionUploader.metrics };sessionUploader.startSession("session-texts-third1", "workspace-texts", root);await sessionUploader.idle("session-texts-third1");diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 152deda1..d33c339f 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -814,15 +814,60 @@ const SECRET_PATTERNS: Redaction[] = [// `Bearer <token>`: the token part is decided by isBearerSecret. The character// class already excludes `${...}` and `<...>` placeholders.const BEARER_TOKEN = tokenPattern(String.raw`(Bearer[ \t]+)([A-Za-z0-9_.~+/=-]{16,})`, "gi");+/**+ * Personal-data matches kept as written: code and docs are full of addresses+ * and e-mails that identify nobody. IPv4: unspecified and "this network"+ * (0.0.0.0/8), loopback (127/8), private (10/8, 172.16/12, 192.168/16),+ * link-local (169.254/16), the documentation ranges (192.0.2/24,+ * 198.51.100/24, 203.0.113/24) and netmask-shaped values (ones, then zeros:+ * 255.255.255.0, 255.255.255.255). Every other (public) address is redacted.+ * The backend's is_kept_ipv4.+ */+const IPV4_KEPT_PREFIXES: number[][] = [[0], [10], [127], [169, 254], [192, 168], [192, 0, 2], [198, 51, 100], [203, 0, 113]];+export function isKeptIpv4(address: string): boolean {+ const octets = address.split(".").map(Number);+ if (octets.length !== 4) return false;+ if (octets[0] === 172 && octets[1]! >= 16 && octets[1]! <= 31) return true;+ if (IPV4_KEPT_PREFIXES.some((prefix) => prefix.every((octet, index) => octets[index] === octet))) return true;+ const value = ((octets[0]! << 24) | (octets[1]! << 16) | (octets[2]! << 8) | octets[3]!) >>> 0;+ const inverted = ~value >>> 0;+ // A netmask: ones, then zeros (255.255.255.255 included).+ return ((inverted & (inverted + 1)) >>> 0) === 0;+}++/**+ * E-mails kept as written: the reserved example and test domains (RFC 2606,+ * RFC 6761), placeholder domains (`[email protected]`, `[email protected]`)+ * and no-reply senders. Every other address is redacted. The backend's+ * is_kept_email.+ */+const EMAIL_KEPT_DOMAINS = new Set(["example.com", "example.org", "example.net", "test", "localhost"]);+const EMAIL_KEPT_SUFFIXES = [".example.com", ".example.org", ".example.net", ".invalid", ".test", ".example", ".localhost"];+const EMAIL_PLACEHOLDER_LABELS = new Set([+ "domain", "yourdomain", "your-domain", "mydomain", "my-domain", "yourcompany", "your-company", "yoursite", "your-site",+ "yourapp", "your-app", "yourorg", "your-org", "yourname",+]);+const EMAIL_KEPT_LOCALS = new Set(["noreply", "no-reply", "no_reply", "donotreply", "do-not-reply"]);+export function isKeptEmail(address: string): boolean {+ const at = address.lastIndexOf("@");+ const local = address.slice(0, at).toLowerCase();+ const domain = address.slice(at + 1).toLowerCase();+ if (EMAIL_KEPT_LOCALS.has(local)) return true;+ if (EMAIL_KEPT_DOMAINS.has(domain) || EMAIL_KEPT_SUFFIXES.some((suffix) => domain.endsWith(suffix))) return true;+ const labels = domain.split(".");+ return labels.length >= 2 && EMAIL_PLACEHOLDER_LABELS.has(labels[labels.length - 2]!);+}+const PII_PATTERNS: Redaction[] = [- [tokenPattern(String.raw`[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,}\b`, "gi"), REDACTED_PII],+ [tokenPattern(String.raw`[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,}\b`, "gi"), (match) => (isKeptEmail(match) ? match : REDACTED_PII)],[tokenPattern(String.raw`(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4})(?!\w)`), REDACTED_PII],[tokenPattern(String.raw`\d{3}-\d{2}-\d{4}(?!\w)`), REDACTED_PII],// IPv4. Not inside a longer version token: never right after a letter// (`ubuntu1.22.04.1`, `v1.2.3.4`), after an alphanumeric and `.~+-`// (`16.15-0ubuntu0.24.04.1`), after a dpkg epoch (`1:2.3.4.5`), nor- // followed by another `.digit` group (`1.2.3.4.5`). The backend's `ipv4`.- [tokenPattern(String.raw`(?<![A-Za-z0-9][.~+-])(?<!\d:)(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?!\.?\d)`, "g", String.raw`[\dA-Za-z.\\]`), REDACTED_PII],+ // followed by another `.digit` group (`1.2.3.4.5`) or a package release+ // (`1.2.15.3-1`). The backend's `ipv4`. Kept as written: see isKeptIpv4.+ [tokenPattern(String.raw`(?<![A-Za-z0-9][.~+-])(?<!\d:)(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?!\.?\d)(?!-\d+(?![\d.]))`, "g", String.raw`[\dA-Za-z.\\]`), (match) => (isKeptIpv4(match) ? match : REDACTED_PII)],];const PRIVACY_POLICY = {@@ -1101,7 +1146,7 @@ function isFilesystemPathValue(value: string): boolean {* `token: config.token`. The backend's is_secret_assignment_value.*/export function isSecretAssignmentValue(value: string, mode: RedactMode, quoted: boolean): boolean {- if (value.length < 8 || PYTHON_WHITESPACE_CHAR.test(value) || value.startsWith("[REDACTED")) return false;+ if (value.length < 8 || PYTHON_WHITESPACE_CHAR.test(value) || value.startsWith("[REDACTED") || isPlaceholderValue(value)) return false;if (value.includes("(") || value.includes("${") || value.includes("$(") || CODE_EXPRESSION_VALUE.test(value)) return false;if (value.startsWith("<") && value.endsWith(">")) return false;if (isFilesystemPathValue(value)) return false;@@ -1114,6 +1159,36 @@ export function isSecretAssignmentValue(value: string, mode: RedactMode, quoted:return /[A-Za-z]/.test(value) && !(identifier && value.includes("."));}+/**+ * Placeholder assignment values, never secrets: a value made only of these+ * words, digit runs of up to four digits or counting up (`12345678`) and runs+ * of one repeated character (`xxxx`, `0000`), at least one of them a marker+ * (PLACEHOLDER_MARKERS, or a repeated-character run of 3+): `your-token-here`,+ * `changeme`, `fake-not-a-real-token-0000`, `YOUR_API_KEY`, `xxxxxxxxxxxx`.+ * Segments as keySegments splits a key. The backend's is_placeholder_value.+ */+const PLACEHOLDER_MARKERS = new Set([+ "your", "yours", "here", "fake", "dummy", "example", "sample", "placeholder", "changeme", "replaceme", "replace", "insert",+ "todo", "fixme", "tbd", "mock", "demo", "redacted", "notreal", "test", "testing",+]);+const PLACEHOLDER_WORDS = new Set([+ "my", "the", "a", "an", "not", "real", "is", "goes", "go", "put", "enter", "set", "this", "to", "with", "of", "for", "in", "me",+ "change", "secret", "secrets", "token", "tokens", "key", "keys", "api", "apikey", "password", "passwd", "pass", "pwd", "value",+ "access", "auth", "private", "client", "id", "string", "foo", "bar", "baz", "qux", "abc", "xyz", "default", "dev", "local",+]);+const REPEATED_CHAR = /^(.)\1*$/su;+export function isPlaceholderValue(value: string): boolean {+ const segments = keySegments(value);+ if (segments.length === 0) return false;+ let marked = false;+ for (const segment of segments) {+ if (PLACEHOLDER_MARKERS.has(segment) || ([...segment].length >= 3 && REPEATED_CHAR.test(segment))) marked = true;+ else if (PLACEHOLDER_WORDS.has(segment) || (/^[0-9]+$/.test(segment) && (segment.length <= 4 || "0123456789".includes(segment)))) continue;+ else return false;+ }+ return marked;+}+/*** The token after `Bearer` is a secret when it carries a digit, or is 20+* characters of mixed case that are not a bare identifier or member@@ -1128,8 +1203,10 @@ function isBearerSecret(token: string): boolean {function applyRedaction(text: string, [pattern, replacement]: Redaction, tally: { count: number }): string {pattern.lastIndex = 0;return text.replace(pattern, (match: string, group?: string) => {- tally.count += 1;- return typeof replacement === "string" ? replacement : replacement(match, group ?? "");+ const replaced = typeof replacement === "string" ? replacement : replacement(match, typeof group === "string" ? group : "");+ // A kept match (isKeptEmail, isKeptIpv4) is returned as it was and is no redaction.+ if (replaced !== match) tally.count += 1;+ return replaced;});}@@ -1300,7 +1377,12 @@ export function redactUploadText(input: string, options: RedactUploadTextOptionsconst mailed = text;text = redactGatedLines(mailed, (from) => mailed.indexOf("@", from), (line) => applyRedaction(line, PII_PATTERNS[0]!, tally));if (DIGIT_GATE.test(text)) {- for (const redaction of PII_PATTERNS.slice(1)) text = applyRedaction(text, redaction, tally);+ // A JSON value under a `version` key (`"version": "3.192.1.7"`, a package list) is a version, never an address.+ const versionField = options.context !== undefined && keySegments(options.context).at(-1) === "version";+ for (const redaction of PII_PATTERNS.slice(1)) {+ if (versionField && redaction === PII_PATTERNS[3]) continue;+ text = applyRedaction(text, redaction, tally);+ }}return { text, count: tally.count };}@@ -3926,6 +4008,25 @@ const EXCLUDED_FILES_MARGIN_BYTES = 256 * 1024;const FINAL_TOOLCHAIN_WAIT_MS = 3_000;/** `{ git_skipped: "xcode_clt_missing" }` when capture may not run git here (command-guard.ts), else nothing. */+/**+ * Why the project archive leaves this session folder out on purpose (a+ * credential store such as `~/.aws`, the home directory, app data, a system+ * directory), for the start upload: `{project_archive: {status: "skipped",+ * reason}}`, so the export can say why it has no project. The archive gate's+ * own check (session-archive/detect.ts), loaded on demand (detect.ts imports+ * this module).+ */+async function projectArchiveField(root: string): Promise<{ project_archive?: { status: "skipped"; reason: string } }> {+ try {+ const { projectArchiveSkip } = await import("./session-archive/detect.js");+ const userDataDir = process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR?.trim();+ const skip = await projectArchiveSkip(root, userDataDir ? { userDataDir } : {});+ return skip ? { project_archive: skip } : {};+ } catch {+ return {};+ }+}+async function gitSkippedField(): Promise<{ git_skipped?: string }> {const reason = await gitSkipReason();return reason ? { git_skipped: reason } : {};@@ -5690,7 +5791,7 @@ export class SessionUploader {}const filesScope: "full" | "changed" = type === "start" ? "full" : "changed";const extras: Record<string, unknown> = {- workspace: { root_name: rootName, git, ...(git ? {} : await gitSkippedField()) },+ workspace: { root_name: rootName, git, ...(git ? {} : await gitSkippedField()), ...(type === "start" && !git ? await projectArchiveField(state.root) : {}) },environment,touched_paths: touchedPaths,files_scope: filesScope,