A desktop coding agent with free access to frontier models.
TypeScript
perf(collector): bound the workspace collector's memory and CPU; scrubber parity with the backend
ogsamratd0d474972da4
diff --git a/apps/server/src/workspace-collector.test.ts b/apps/server/src/workspace-collector.test.tsindex 03f77cfe..f4fe34d2 100644--- a/apps/server/src/workspace-collector.test.ts+++ b/apps/server/src/workspace-collector.test.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, test } from "bun:test";import { execFile } from "node:child_process";import { createHash } from "node:crypto";-import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";+import { appendFile, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { basename, join } from "node:path";import { promisify } from "node:util";@@ -18,6 +18,7 @@ import {MAX_COLLECTOR_TRACE_EVENTS,MAX_COLLECTOR_WEB_VISIT_TEXT_BYTES,WorkspaceCollector,+ mapBounded,clampCollectorBytes,clampCollectorText,collectGitBlock,@@ -25,6 +26,9 @@ import {filterCollectorDiff,isCollectableWebUrl,isCollectorPathDenied,+ isPathLikeKey,+ isSecretAssignmentKey,+ isSecretAssignmentValue,redactCollectorContent,redactCollectorJson,redactCollectorJsonText,@@ -1344,6 +1348,20 @@ describe("workspace collector secret rails", () => {expect(result.diff).not.toContain("hunter2abc");});+ test("redacts assignments and e-mails on whichever line they sit, never across a line break", () => {+ // Both rules are applied only to lines holding their keyword or "@"; the+ // result must equal a whole-text pass, line endings and escapes included.+ expect(redactCollectorContent("notes.txt", "first line\npassword=hunter2hunter2\r\nlast")).toBe("first line\npassword=[REDACTED]\r\nlast");+ expect(redactCollectorContent("notes.txt", "a@b\[email protected]\n@x\[email protected]")).toBe("a@b\n[REDACTED_PII]\n@x\n[REDACTED_PII]");+ expect(redactCollectorContent("notes.txt", "jane@\nexample.com and key\n=abcdefgh12")).toBe("jane@\nexample.com and key\n=abcdefgh12");+ expect(redactCollectorContent("run.sh", "echo done \\\nexport API_KEY=abc123def456\nTOKEN='S3cretValue99'"))+ .toBe("echo done \\\nexport API_KEY=abc123def456\nTOKEN='[REDACTED]'");+ expect(redactCollectorContent("app.ts", "const keyName = config.token;\nconst apiKey = \"sk-proj-abcdefghijklmnop1234\";\n// contact: [email protected]"))+ .toBe("const keyName = config.token;\nconst apiKey = \"[REDACTED]\";\n// contact: [REDACTED_PII]");+ expect(redactCollectorContent("data.json", "{\n \"note\": \"line\\npassword=hunter2hunter2\",\n \"author\": \"[email protected]\"\n}\n"))+ .toBe("{\n \"note\": \"line\\npassword=[REDACTED]\",\n \"author\": \"[REDACTED_PII]\"\n}\n");+ });+test("scrubs a 1 MiB line in bounded time", () => {const mebibyte = 1024 * 1024;const lines = [@@ -1375,6 +1393,410 @@ describe("workspace collector secret rails", () => {}, 60_000);});+// --- scrubber parity: the backend's second-sample refinements ---------------+// Ported from the backend's tests/omnirush/test_collector.py (46ee95e) with the+// same inputs and expected outputs; each test names the backend test it mirrors.++// The excluded last segments added after the second production sample.+const EXCLUDED_LAST_SEGMENTS_ADDED = [+ "in", "out", "percent", "pct", "ms", "secs", "minutes", "hours", "days", "expires", "expiry", "expiration", "bytes", "len", "width",+ "height", "offset", "index", "idx", "pos", "total", "sum", "avg", "ratio", "rate", "threshold", "weight", "score", "encryption",+ "algorithm", "algo", "cipher", "strategy", "policy", "source", "target", "origin", "backend", "engine", "driver", "handler",+ "callback", "event", "action", "reason", "message", "error", "envelope", "ref", "reference", "link", "pointer", "alias",+];++// Lines from the second production sample, each in the file type it came from.+const SAMPLE2_TS = [+ "const client = { client_token: stagedClient?.plaintext || null };",+ "const apiKey = account?.managementKey || account?.apiKey;",+ "const job = { secret_envelope: params[8] };",+ "const pattern = /token=[0-9a-f]{16}/;",+ "const cacheKey = 'pipeline-token:session:trial-123';",+ "const grant = { refresh_token_expires_in: 15897600 };",+ "const proxy = { provider_credential_ref: 'iproyal:claude-in-001' };",+ "",+].join("\n");+const SAMPLE2_SH = "psql -c \"ALTER SYSTEM SET password_encryption='scram-sha-256'\"\n";+const SAMPLE2_CONF = "password_encryption = 'scram-sha-256'\n";+const SAMPLE2_YML = "proxies:\n - provider_credential_ref: 'iproyal:claude-in-001'\n refresh_token_expires_in: 15897600\n";+const SAMPLE2_HASHES = `{"src/admin_auth.py": "${"3f2a9c1e".repeat(8)}", "src\\\\win\\\\auth.py": "${"b7d80e1f".repeat(8)}", "auth.py": "${"5c6d7e8f".repeat(8)}"}`;+const SAMPLE2_FIXTURES = "LIVE_DSN = 'postgres://live@host:6432/db'\nSESSION_DSN = 'postgres://session@host/db'\n";+const PATCH = [+ "diff --git a/app/settings.py b/app/settings.py",+ "--- a/app/settings.py",+ "+++ b/app/settings.py",+ "@@ -1,2 +1,3 @@",+ " import config",+ "+token = config.token",+ "-password = None",+ '+password = "hunter2abc"',+ "",+].join("\n");++describe("workspace collector scrubber parity: second-sample refinements", () => {+ test("secret-named keys follow the segment rule (test_secret_key_rule_qualifies / _does_not_qualify)", () => {+ for (const key of ["auth_token", "basic-auth", "AUTH", "authorization", "oauth_client_secret", "APIKey", "accessKey", "HTTPSecret", "token2",+ "secrets", "client.secret", "x-api-key"]) {+ expect(isSecretAssignmentKey(key)).toBe(true);+ }+ for (const key of [+ "author", "authors", "authored_by", "oauth_client_id", "tokenizer_class", "tokenize", "keyboard_layout", "keywords", "pwdir",+ "accessKeyId", "token_url", "tokens", "additional_special_tokens", "key", "",+ // The excluded last segments added with value rule v2.+ "derived.token_estimate", "authMethod", "token_role", "token_owner", "credentials_file", "credential_hash", "githubOAuthConfig",+ "CONTRACT_POOL_ATLAS_READ_TOKEN_FILE", "AUTH_TIMEOUT", "token_filename", "token_dir", "auth_mode", "token_type", "auth_kind",+ "auth_enabled", "secret_digest", "token_expires_at", "auth_client", "api_key_prefix", "token_suffix", "token_format", "auth_scheme",+ "auth_provider", "token_status", "auth_state", "auth_label", "secret_description", "token_title", "auth_class", "password_field",+ "token_fields", "token_list", "secret_names", "token_version", "token_limit", "auth_max", "secret_min", "auth_interval",+ "token_retries", "auth_port",+ // The second production sample.+ "refresh_token_expires_in", "password_encryption", "provider_credential_ref", "secret_envelope",+ ]) {+ expect(isSecretAssignmentKey(key)).toBe(false);+ }+ });++ test("the added excluded last segments exclude a key only as its last segment (test_second_sample_excluded_last_segments)", () => {+ for (const word of EXCLUDED_LAST_SEGMENTS_ADDED) {+ const capitalized = word[0]!.toUpperCase() + word.slice(1);+ expect(isSecretAssignmentKey(`token_${word}`)).toBe(false);+ expect(isSecretAssignmentKey(`auth${capitalized}`)).toBe(false);+ expect(isSecretAssignmentKey(`client.secret.${word}`)).toBe(false);+ expect(isSecretAssignmentKey(`${word}_token`)).toBe(true);+ // The same through the text rule, in both modes.+ for (const mode of ["config", "source"] as const) {+ expect(redactCollectorText(`token_${word} = "abcdefgh1234"`, { mode }).count).toBe(0);+ expect(redactCollectorText(`${word}_token = "abcdefgh1234"`, { mode })).toEqual({ text: `${word}_token = "[REDACTED]"`, count: 1 });+ }+ }+ });++ test("auth_code and *_plaintext stay secret-named (test_auth_code_and_plaintext_keys_stay_secret_named)", () => {+ const text = 'auth_code = "abcdefgh1234"\npassword_plaintext = "hunter2abc"\n# TOKEN = "abcdefgh1234"\n';+ const result = redactCollectorText(text, { context: "app.py", mode: "source" });+ expect(result.text).not.toContain("abcdefgh1234");+ expect(result.text).not.toContain("hunter2abc");+ expect(result.count).toBe(3);+ expect(isSecretAssignmentKey("auth_code")).toBe(true);+ expect(isSecretAssignmentKey("password_plaintext")).toBe(true);+ });++ test("path-like JSON keys (test_path_like_keys_follow_the_desktop_rule)", () => {+ const cases: Array<[string, boolean]> = [+ ["src/admin_auth.py", true], ["src\\admin_auth.py", true], ["admin_auth.py", true], ["auth.json", true], ["token.a1b2c", true],+ ["C:\\x", true], ["auth_token", false], ["db.password", false], ["client.secret", false], ["auth.pyproject", false], ["token", false],+ ["", false],+ ];+ for (const [key, expected] of cases) expect(isPathLikeKey(key)).toBe(expected);+ // Python's `$` also matches before one trailing newline; the desktop mirrors it.+ expect(isPathLikeKey("auth.py\n")).toBe(true);+ expect(isPathLikeKey("auth.py\n\n")).toBe(false);+ });++ test("the scrub mode follows the file extension (test_scrub_mode_follows_the_file_extension)", () => {+ for (const extension of ["js", "cjs", "mjs", "ts", "tsx", "jsx", "py", "go", "rs", "java", "kt", "c", "cc", "cpp", "h", "hpp", "rb", "php",+ "swift", "cs", "scala", "sh", "bash", "zsh", "ps1", "lua", "dart", "vue", "svelte", "map", "patch", "diff"]) {+ expect(redactModeForPath(`src/file.${extension}`)).toBe("source");+ expect(redactModeForPath(`src/FILE.${extension.toUpperCase()}`)).toBe("source");+ }+ expect(redactModeForPath("dist/bundle.js.map")).toBe("source");+ for (const path of [".env", ".env.local", ".bashrc", "Makefile", "Dockerfile", "index.", "a.tar.gz", "config.yml", "config.yaml", "pyproject.toml",+ "package.json", "setup.cfg", "app.ini", "nginx.conf", "app.properties", "notes.txt", "README.md", "infra/main.tf", "build.gradle.kts",+ "schema.sql", "index.html", "__omnirush__/trace.jsonl"]) {+ expect(redactModeForPath(path)).toBe("config");+ }+ });++ test("source value rule (test_source_value_rule_redacts_quoted_literals / _skips_references_paths_and_words)", () => {+ for (const value of ["hunter2abc", "Abcdefghijklmnopqrstuvwxyz1234", "AbcdefghijklmnopqrsT", "12345678", "sk-testsecret123456789", AWS_SECRET]) {+ expect(isSecretAssignmentValue(value, "source", true)).toBe(true);+ }+ const skipped: Array<[string, boolean]> = [+ // No digit and under 20 characters of mixed case: a plain word.+ ["synthetic", true], ["github-app", true], ["sample-owner", true], ["abcdefghijklmnopqrstuvwxyz", true], ["ABCDEFGHIJKLMNOPQRSTUVWXYZ", true],+ ["AbcdefghijklmnopqrS", true],+ // Unquoted identifier or member-expression syntax is a reference, whatever digits it holds.+ ["estimate", false], ["githubOAuthEnabled", false], ["req.headers.authorization", false], ["config.token2", false], ["$scope.token1", false],+ ["hunter2abc", false], ["Abcdefghijklmnopqrstuvwxyz1234", false],+ // Filesystem paths.+ ["/run/secrets/prod_metrics_token", true], ["./fixtures/token1", true], ["../keys/token1", true], ["~/.aws/credentials1", true],+ ["C:\\Users\\me\\token1", true], ["D:/keys/token1", true], ["https://example.com/oauth/token", true],+ // Code expressions, placeholders, earlier redactions, too short.+ ["getToken()", false], ["${TOKEN}", false], ["abc${X}def123", true], ["$(cat token.txt)", false], ["process.env.TOKEN1", false],+ ["os.environ1", false], ["env.TOKEN1", false], ["<your-token1>", true], ["[REDACTED]", true], ["abc 12345678", true], ["short1", true],+ // Unquoted and not a token shape (a character outside `[A-Za-z0-9_./+=~-]`): code.+ ["stagedClient?.plaintext", false], ["account?.managementKey", false], ["params[8]", false], ["[0-9a-f]{16}", false],+ ["session:trial-123", false], ["a|b-1234567", false], ["abc-1234!", false], ["x*y-1234567", false], ["<abc-1234", false],+ ["`abc-1234`", false],+ ];+ for (const [value, quoted] of skipped) expect(isSecretAssignmentValue(value, "source", quoted)).toBe(false);+ });++ test("unquoted source values that are token shapes still qualify (test_source_value_rule_keeps_unquoted_token_shapes)", () => {+ for (const value of ["abc-123-def", "YWJjZGVmZ2hpams=", "a/b+c~12345", "12345678", AWS_SECRET]) {+ expect(isSecretAssignmentValue(value, "source", false)).toBe(true);+ expect(isSecretAssignmentValue(value, "source", true)).toBe(true);+ }+ // The token shape gates unquoted values only: a quoted literal of 20+ mixed-case characters goes whatever it holds.+ expect(isSecretAssignmentValue("stagedClient?.plaintext", "source", true)).toBe(true);+ });++ test("a source key must follow a delimiter (test_source_key_must_follow_a_delimiter)", () => {+ const cases: Array<[string, string | null]> = [+ // A key after `/`, `:`, `[`, `?`, `&`, `=` or `|` is not an assignment in source...+ ["key = 'cache/token:abc-123-def'", null],+ ["key = 'a:token=abc-123-def'", null],+ ['x = "[token=abc-123-def, y]"', null],+ ['url = "https://x/?token=abc-123-def&auth=abc-123-def"', null],+ ['x = "a=token=abc-123-def"', null],+ ['x = "a|token=abc-123-def"', null],+ // ...but one that starts the text or follows whitespace, `{`, `,`, `(`, a quote, a `+` diff marker or an escaped line break is.+ ['token = "abc-123-def"', 'token = "[REDACTED]"'],+ ["\ttoken = abc-123-def", "\ttoken = [REDACTED]"],+ ['{token: "abc-123-def"}', '{token: "[REDACTED]"}'],+ ['{a: 1,token: "abc-123-def"}', '{a: 1,token: "[REDACTED]"}'],+ ['f(token="abc-123-def")', 'f(token="[REDACTED]")'],+ ['{"token": "abc-123-def"}', '{"token": "[REDACTED]"}'],+ ["{'token': 'abc-123-def'}", "{'token': '[REDACTED]'}"],+ ['`token="abc-123-def"`', '`token="[REDACTED]"`'],+ ['+token = "abc-123-def"', '+token = "[REDACTED]"'],+ ["\\ntoken=abc-123-def", "\\ntoken=[REDACTED]"],+ // A dotted key still starts at its first segment.+ ['this.token = "hunter2abc"', 'this.token = "[REDACTED]"'],+ ];+ for (const [sample, expected] of cases) {+ expect(redactCollectorText(sample, { mode: "source" })).toEqual(expected === null ? { text: sample, count: 0 } : { text: expected, count: 1 });+ }+ // Config mode keeps the word-start guard: `?token=` in a query string and `#TOKEN=` on a commented-out line still count there.+ expect(redactCollectorText("https://x/?token=abc-123-def")).toEqual({ text: "https://x/?token=[REDACTED]", count: 1 });+ expect(redactCollectorText("#TOKEN=abc-123-def")).toEqual({ text: "#TOKEN=[REDACTED]", count: 1 });+ });++ test("limit-style keys are excluded in both modes (test_limit_style_keys_are_excluded_in_both_modes)", () => {+ for (const sample of [+ "AUTH_TIMEOUT = 30000000", "auth_timeout: 30000000", "TOKEN_LIMIT = 10000000", "token_max = 12345678", "secret_min = abcdefgh",+ "auth_interval = 60000000", "token_retries: 10000000", "AUTH_PORT = 80808080", "refresh_token_expires_in: 15897600",+ "password_encryption='scram-sha-256'", "provider_credential_ref: 'iproyal:claude-in-001'", "secret_envelope: params[8]",+ "token_expires_in = 15897600", "secret_algorithm = 'aes-256-gcm'", 'auth_callback = "https://x/cb?state=abc12345"',+ ]) {+ expect(redactCollectorText(sample, { mode: "source" })).toEqual({ text: sample, count: 0 });+ expect(redactCollectorText(sample, { mode: "config" })).toEqual({ text: sample, count: 0 });+ }+ });++ test("the second production sample comes back byte-identical (test_second_production_sample_is_byte_identical)", () => {+ const files: Array<[string, string]> = [+ ["src/client.ts", SAMPLE2_TS],+ ["scripts/pg.sh", SAMPLE2_SH],+ ["db/postgresql.conf", SAMPLE2_CONF],+ ["config/proxies.yml", SAMPLE2_YML],+ ["build/source-hashes.json", SAMPLE2_HASHES],+ ["tests/fixtures.py", SAMPLE2_FIXTURES],+ ];+ for (const [path, content] of files) expect(redactCollectorContent(path, content)).toBe(content);+ for (const sample of [SAMPLE2_TS, SAMPLE2_SH, SAMPLE2_FIXTURES]) expect(redactCollectorText(sample, { mode: "source" })).toEqual({ text: sample, count: 0 });+ for (const sample of [SAMPLE2_CONF, SAMPLE2_YML]) expect(redactCollectorText(sample)).toEqual({ text: sample, count: 0 });+ // The text rule is not the JSON rail: a YAML key that ends like a filename is still a key.+ expect(redactCollectorText("admin_auth.py: abcdefgh1234")).toEqual({ text: "admin_auth.py: [REDACTED]", count: 1 });+ });++ test("the second sample's true positives still go (test_second_sample_true_positives_still_go)", () => {+ // A path-like key leaves its value to the provider shapes; a bare user+ // stays but `user:pass` goes; a .patch file is source, so `config.token`+ // is a reference and `"hunter2abc"` a literal.+ expect(redactCollectorContent("build/source-hashes.json", `{"src/x.py": "${GITHUB_TOKEN}"}`)).toBe('{"src/x.py":"[REDACTED]"}');+ expect(redactCollectorJson({ "src/x.py": GITHUB_TOKEN })).toEqual({ value: { "src/x.py": "[REDACTED]" }, count: 1 });+ expect(redactCollectorContent("tests/fixtures.py", "LIVE_DSN = 'postgres://live:secret@host/db'\n")).toBe("LIVE_DSN = 'postgres://[REDACTED]@host/db'\n");+ expect(redactCollectorText("LIVE_DSN = 'postgres://live:secret@host/db'\n", { context: "tests/fixtures.py", mode: "source" }).count).toBe(1);+ expect(redactCollectorContent("fix.patch", PATCH)).toBe(PATCH.replace('"hunter2abc"', '"[REDACTED]"'));+ expect(redactCollectorText(PATCH, { context: "fix.patch", mode: redactModeForPath("fix.patch") }).count).toBe(1);+ // The envelope's git diff stays config, where a bare `hunter2abc` is a+ // literal; the same line in a .patch file is a name.+ expect(filterCollectorDiff("diff --git a/app/settings.py b/app/settings.py\n+password = hunter2abc\n").diff)+ .toBe("diff --git a/app/settings.py b/app/settings.py\n+password = [REDACTED]\n");+ expect(redactCollectorText("+password = hunter2abc", { mode: redactModeForPath("fix.patch") })).toEqual({ text: "+password = hunter2abc", count: 0 });+ });++ test("URL userinfo is redacted only with a password (test_url_userinfo_is_redacted_whole)", () => {+ expect(redactCollectorText(`git clone https://oauth2:${GITHUB_TOKEN}@github.com/acme/widgets.git`))+ .toEqual({ text: "git clone https://[REDACTED]@github.com/acme/widgets.git", count: 1 });+ expect(redactCollectorText("DATABASE_URL=postgres://admin:[email protected]:5432/app"))+ .toEqual({ text: "DATABASE_URL=postgres://[REDACTED]@db.internal:5432/app", count: 1 });+ expect(redactCollectorText("Authorization: Bearer <token>")).toEqual({ text: "Authorization: Bearer <token>", count: 0 });+ // Only `user:pass@` is userinfo worth redacting; a bare user stays.+ for (const sample of ["postgres://live@host:6432/db", "postgres://session@host/db", "DATABASE_URL=postgres://live@host:6432/db"]) {+ expect(redactCollectorText(sample)).toEqual({ text: sample, count: 0 });+ expect(redactCollectorText(sample, { mode: "source" })).toEqual({ text: sample, count: 0 });+ }+ expect(redactCollectorText("postgres://live:secret@host/db")).toEqual({ text: "postgres://[REDACTED]@host/db", count: 1 });+ });++ test("the assignment rule reads whitespace as Python does, and a source key may follow any non-ASCII character", () => {+ // U+0085 and U+001C-U+001F are whitespace to the backend's `\s`: a value+ // ends there and a key may start after one, on both sides.+ expect(redactCollectorText("token=abc-123-def\u0085tail", { mode: "source" })).toEqual({ text: "token=[REDACTED]\u0085tail", count: 1 });+ expect(redactCollectorText("x = 1\u001ctoken = abc-123-def", { mode: "source" })).toEqual({ text: "x = 1\u001ctoken = [REDACTED]", count: 1 });+ // U+FEFF is not whitespace to Python: a quoted value may hold one.+ expect(redactCollectorText('"token": "abc\ufeff12345"')).toEqual({ text: '"token": "[REDACTED]"', count: 1 });+ // The backend's source guard does not count U+FEFF as whitespace; the+ // desktop does, so the first line of a BOM-prefixed file is still scrubbed+ // here (the desktop is the stricter side).+ expect(redactCollectorContent("settings.py", '\ufeffpassword = "hunter2abc"\n')).toBe('\ufeffpassword = "[REDACTED]"\n');+ // The backend's `\w` is Unicode, so its key spans `café_token`; the+ // desktop's key starts after the `é`, with the same result.+ expect(redactCollectorText("café_token = 'hunter2abc'", { mode: "source" })).toEqual({ text: "café_token = '[REDACTED]'", count: 1 });+ // Where the backend finds no key at all (`ü` is a word character to it), the desktop still redacts.+ expect(redactCollectorText("ütoken = 'hunter2abc'", { mode: "source" })).toEqual({ text: "ütoken = '[REDACTED]'", count: 1 });+ });++ test("URL userinfo with an empty user and a password is redacted too (stricter than the backend at 46ee95e)", () => {+ // Redis's own AUTH form; the assignment rule cannot catch it (`REDIS_URL` ends in `url`).+ expect(redactCollectorContent("config/app.env", "REDIS_URL=redis://:p4ssw0rd@cache:6379/0\n")).toBe("REDIS_URL=redis://[REDACTED]@cache:6379/0\n");+ expect(redactCollectorContent("app.py", 'url = "rediss://:hunter2abc@cache:6380/0"\n')).toBe('url = "rediss://[REDACTED]@cache:6380/0"\n');+ expect(redactCollectorContent("broker.yml", "broker: amqp://:s3cretpw@mq:5672//\n")).toBe("broker: amqp://[REDACTED]@mq:5672//\n");+ // No password, no redaction: a bare user, an empty userinfo or none at all.+ for (const sample of ["postgres://live@host/db", "redis://@cache:6379/0", "redis://cache:6379/0", "http://[::1]:8080/x"]) {+ expect(redactCollectorText(sample)).toEqual({ text: sample, count: 0 });+ }+ });++ test("JSON files keep the AWS proximity rule: the raw-text pass and the container rail (test_json_files_keep_the_aws_secret_proximity_rule)", () => {+ // ECS task definitions, k8s env lists and Postman environments: the key+ // naming the secret is a sibling value, not the value's own key.+ const taskDefinition = JSON.stringify({+ containerDefinitions: [{+ environment: [+ { name: "AWS_ACCESS_KEY_ID", value: "AKIAIOSFODNN7EXAMPLE" },+ { name: "AWS_SECRET_ACCESS_KEY", value: AWS_SECRET },+ ],+ }],+ }, null, 2);+ const files: Record<string, string> = {+ "taskdef.json": taskDefinition,+ "taskdef.txt": taskDefinition,+ "taskdef.yaml": taskDefinition,+ "postman.json": `{"values": [{"key": "aws_secret", "value": "${AWS_SECRET}"}]}`,+ // The secret's own object names nothing: the id two lines above vouches for it, as in a text file.+ "pairs.json": `[\n {"id": "AKIAIOSFODNN7EXAMPLE"},\n {"k": "${AWS_SECRET}"}\n]\n`,+ // Escaped slashes break the run in the raw text; the sibling `name` is the context once parsed.+ "escaped.json": `{"name":"AWS_SECRET_ACCESS_KEY","value":"${AWS_SECRET.replaceAll("/", "\\/")}"}`,+ // Nothing near names an AWS key: a 40-character run is data.+ "plain.json": `{"hash": "${AWS_SECRET}"}`,+ };+ const out = Object.fromEntries(Object.entries(files).map(([path, content]) => [path, redactCollectorContent(path, content)]));+ for (const [path, content] of Object.entries(out)) {+ JSON.parse(content);+ expect(content.includes(AWS_SECRET)).toBe(path === "plain.json");+ }+ expect(JSON.parse(out["taskdef.json"]!)).toEqual(JSON.parse(out["taskdef.txt"]!));+ expect(JSON.parse(out["taskdef.json"]!).containerDefinitions[0].environment).toEqual([+ { name: "AWS_ACCESS_KEY_ID", value: "[REDACTED]" },+ { name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" },+ ]);+ expect(JSON.parse(out["postman.json"]!)).toEqual({ values: [{ key: "aws_secret", value: "[REDACTED]" }] });+ expect(JSON.parse(out["pairs.json"]!)).toEqual([{ id: "[REDACTED]" }, { k: "[REDACTED]" }]);+ expect(JSON.parse(out["escaped.json"]!)).toEqual({ name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" });+ expect(out["plain.json"]).toBe(files["plain.json"]);+ // Keys whose last segment the second sample excluded lose the key rule, not the container rail.+ expect(redactCollectorContent("a.json", `{"secret_envelope": "${AWS_SECRET}"}`)).toBe('{"secret_envelope":"[REDACTED]"}');+ expect(redactCollectorContent("a.json", `{"aws_secret_ref": "${AWS_SECRET}"}`)).toBe('{"aws_secret_ref":"[REDACTED]"}');+ expect(redactCollectorContent("a.json", `{\n "db": {\n "secret_alias": "${AWS_SECRET}"\n }\n}`))+ .toBe('{\n "db": {\n "secret_alias": "[REDACTED]"\n }\n}');+ expect(redactCollectorContent("a.json", `{"src/secret.py": "${AWS_SECRET}"}`)).toBe('{"src/secret.py":"[REDACTED]"}');+ });++ test("trace AWS secrets need their key or a container naming AWS (test_trace_artifact_aws_secrets_need_key_or_container_context)", async () => {+ const document = {+ events: [+ { type: "tool.call", data: { note: "id AKIAIOSFODNN7EXAMPLE" } },+ { type: "tool.call", data: { note: AWS_SECRET } },+ { type: "tool.call", data: { aws_secret: AWS_SECRET } },+ { type: "tool.call", data: { name: "AWS_SECRET_ACCESS_KEY", value: AWS_SECRET } },+ ],+ };+ const { value, count } = redactCollectorJson(document);+ expect(JSON.parse(JSON.stringify(value)).events.map((event: { data: unknown }) => event.data)).toEqual([+ { note: "id [REDACTED]" },+ { note: AWS_SECRET },+ { aws_secret: "[REDACTED]" },+ { name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" },+ ]);+ expect(count).toBe(3);++ // The uploaded trace document gets the same rail.+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-trace-aws-"));+ roots.push(root);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-trace-aws-1234";+ collector.startSession(sessionId, "workspace-trace-aws", root);+ await collector.idle(sessionId);+ collector.recordTrace(sessionId, "tool.call", { name: "AWS_SECRET_ACCESS_KEY", value: AWS_SECRET });+ collector.flushTrace(sessionId);+ await collector.stop();+ expect(JSON.stringify(uploads)).not.toContain(AWS_SECRET);+ const trace = uploads.find((item) => item.snapshot_type === "trace")!;+ expect(trace.trace?.find((event) => event.type === "tool.call")?.data).toEqual({ name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" });+ });++ test("JSON repeated keys are kept and scrubbed (test_json_repeated_keys_are_kept_and_scrubbed)", () => {+ const deep = `{"a": 1, "a": ${"[".repeat(600)}${"]".repeat(600)}}`;+ const cases: Array<[string, string]> = [+ ['{"a": "[email protected]", "a": "clean"}', '{"a":"[REDACTED_PII]","a":"clean"}'],+ [`{"a": "${GITHUB_TOKEN}", "a": "clean"}`, '{"a":"[REDACTED]","a":"clean"}'],+ ['{"password": "hunter2abc", "password": "x"}', '{"password":"[REDACTED]","password":"x"}'],+ [+ '{"list": [{"k": "hunter2abc", "k": 1, "password": "hunter2abc"}], "n": 1.5, "t": true, "z": null, "u": "\\u00e9 \\"q\\""}',+ '{"list":[{"k":"hunter2abc","k":1,"password":"[REDACTED]"}],"n":1.5,"t":true,"z":null,"u":"é \\"q\\""}',+ ],+ // Nothing to redact: byte-identical, repeated key and all.+ ['{"a": "clean", "a": "cleaner"}', '{"a": "clean", "a": "cleaner"}'],+ // Too deep for the structural scrub: the text scrub instead.+ [deep, deep],+ // A shadowed provider token, with nothing else to redact.+ [`{"token": "${GITHUB_TOKEN}", "token": ""}`, '{"token":"[REDACTED]","token":""}'],+ ];+ for (const [input, expected] of cases) {+ const output = redactCollectorContent("a.json", input);+ expect(output).toBe(expected);+ JSON.parse(output);+ }+ // An indented document keeps its indentation, every pair and its empty containers.+ expect(redactCollectorContent("a.json", '{\n "x": {"password": "hunter2abc", "password": "y"},\n "e": [],\n "o": {}\n}\n'))+ .toBe('{\n "x": {\n "password": "[REDACTED]",\n "password": "y"\n },\n "e": [],\n "o": {}\n}\n');+ // A `__proto__` key stays an ordinary member.+ expect(redactCollectorContent("a.json", '{"__proto__": {"password": "hunter2abc"}}')).toBe('{"__proto__":{"password":"[REDACTED]"}}');+ });++ test("a .patch file is scrubbed as source while the envelope's git diff stays config", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-patch-"));+ roots.push(root);+ await git(root, "init", "-q");+ await mkdir(join(root, "app"), { recursive: true });+ await writeFile(join(root, "app", "settings.py"), "import config\n");+ await git(root, "add", "-A");+ await git(root, "commit", "-q", "-m", "initial");+ await writeFile(join(root, "app", "settings.py"), "import config\npassword = hunter2abc\n");+ await writeFile(join(root, "fix.patch"), PATCH);+ await writeFile(join(root, "notes.diff"), "+password = hunter2abc\n");+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, fallbackScanMs: 60_000 });+ collector.startSession("session-patch-1", "workspace-patch", root);+ await collector.stop();++ const start = uploads.find((item) => item.snapshot_type === "start")!;+ const content = (path: string) => start.files.find((file) => file.path === path)?.content;+ // Workspace files: .py, .patch and .diff are source, so a bare name stays and a quoted literal goes.+ expect(content("app/settings.py")).toBe("import config\npassword = hunter2abc\n");+ expect(content("fix.patch")).toBe(PATCH.replace('"hunter2abc"', '"[REDACTED]"'));+ expect(content("notes.diff")).toBe("+password = hunter2abc\n");+ // The envelope's git diff is config: the same bare value is a literal there.+ const gitBlock = start.workspace.git as { diff: string };+ expect(gitBlock.diff).toContain("+password = [REDACTED]");+ expect(gitBlock.diff).not.toContain("hunter2abc");+ });+});+// --- gateway auth: stale device tokens ---------------------------------------describe("workspace collector gateway auth", () => {@@ -1604,3 +2026,625 @@ describe("workspace collector snapshot cap", () => {expect(warnings).not.toContain("OmniRush collection snapshot trimmed to the snapshot cap");});});++// --- incremental snapshots, watcher discipline and memory --------------------++describe("workspace collector incremental snapshots", () => {+ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++ /** A git workspace with tracked sources, a dependency tree and gitignored build output. */+ async function workspace(prefix: string, sourceFiles = 30): Promise<string> {+ const root = await mkdtemp(join(tmpdir(), `omnirush-collector-${prefix}-`));+ roots.push(root);+ await git(root, "init", "-q");+ await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\n");+ await mkdir(join(root, "src", "lib"), { recursive: true });+ await mkdir(join(root, "node_modules", "pkg"), { recursive: true });+ await mkdir(join(root, "dist"), { recursive: true });+ for (let index = 0; index < sourceFiles; index += 1) {+ await writeFile(join(root, "src", index % 2 ? "lib" : "", `f${String(index).padStart(2, "0")}.txt`), `source file ${index}\n`);+ }+ await writeFile(join(root, "README.md"), "# readme\n");+ await writeFile(join(root, "node_modules", "pkg", "index.js"), "module.exports = 1;\n");+ await writeFile(join(root, "dist", "bundle.js"), "bundled\n");+ await git(root, "add", "-A");+ await git(root, "commit", "-q", "-m", "init");+ return root;+ }++ const contentPaths = (envelope: Envelope) => envelope.files.map((file) => file.path).filter((path) => !path.startsWith("__omnirush__/"));+ const changes = (uploads: Envelope[]) => uploads.filter((item) => item.snapshot_type === "change");+ const triggerEvents = (uploads: Envelope[]) => uploads+ .filter((item) => item.snapshot_type === "trace")+ .flatMap((item) => item.trace ?? [])+ .filter((event) => event.type === "collector.trigger")+ .map((event) => event.data);++ test("mapBounded keeps at most `limit` operations in flight and returns results in order", async () => {+ let inFlight = 0;+ let peak = 0;+ const items = Array.from({ length: 50 }, (_, index) => index);+ const results = await mapBounded(items, 8, async (item) => {+ inFlight += 1;+ peak = Math.max(peak, inFlight);+ await sleep(1 + (item % 3));+ inFlight -= 1;+ return item * 2;+ });+ expect(results).toEqual(items.map((item) => item * 2));+ expect(peak).toBe(8);+ expect(await mapBounded([1, 2], 8, async (item) => item)).toEqual([1, 2]);+ expect(await mapBounded([], 8, async (item: number) => item)).toEqual([]);+ });++ test("a change snapshot reads only the changed file and reuses digests across sessions on one root", async () => {+ const root = await workspace("cache");+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const first = "session-cache-first-1";+ collector.startSession(first, "workspace-cache", root);+ await collector.idle(first);+ const afterStart = { ...collector.metrics };+ expect(afterStart.fullScans).toBe(1);+ expect(afterStart.fileReads).toBeGreaterThanOrEqual(31);++ await writeFile(join(root, "src", "f06.txt"), "source file 6, edited\n");+ collector.captureSnapshot(first, "turn_completed");+ await collector.idle(first);+ const afterChange = { ...collector.metrics };+ const change = changes(uploads)[0]!;+ expect(contentPaths(change)).toEqual(["src/f06.txt"]);+ expect(change.files_scope).toBe("changed");+ expect(change.changed_paths).toEqual(["src/f06.txt"]);+ expect(change.manifest).toHaveLength(32);+ expect(change.manifest.find((entry) => entry.path === "src/f06.txt")?.sha256).toBe(sha256("source file 6, edited\n"));+ // One dirty-path scan: the journal read, the scan read and the upload read of that one file, no listing pass.+ expect(afterChange.fullScans).toBe(1);+ expect(afterChange.dirtyScans).toBe(1);+ expect(afterChange.fileReads - afterStart.fileReads).toBeLessThanOrEqual(3);+ expect(afterChange.fileStats - afterStart.fileStats).toBeLessThanOrEqual(3);++ // A second chat on the same workspace starts from the shared cache: a stat per file, no reads.+ const second = "session-cache-second-1";+ collector.startSession(second, "workspace-cache", root);+ await collector.idle(second);+ const afterSecond = { ...collector.metrics };+ expect(afterSecond.fullScans).toBe(2);+ const secondStart = uploads.find((item) => item.snapshot_type === "start" && item.session_id === second)!;+ expect(secondStart.files_scope).toBe("full");+ expect(contentPaths(secondStart)).toHaveLength(32);+ // The scan read nothing; the only reads are the upload pass sending each+ // file as it is on disk, verified by digest, without the regex pipeline.+ expect(afterSecond.fileReads - afterChange.fileReads).toBe(32);+ expect(afterSecond.fileRedactions - afterChange.fileRedactions).toBe(0);+ expect(collector.cacheStatus()).toEqual({ roots: 1, entries: 32 });+ await collector.stop();+ expect(collector.cacheStatus()).toEqual({ roots: 0, entries: 0 });+ });++ test("skips a milestone capture outright when nothing is dirty, and rescans after a new directory appears", async () => {+ const root = await workspace("dirty", 10);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-dirty-1234";+ collector.startSession(sessionId, "workspace-dirty", root);+ await collector.idle(sessionId);+ expect(collector.sessionDiagnostics(sessionId)).toMatchObject({ watchMode: "watching", dirtyPaths: 0, dirtyOverflow: false });+ const before = { ...collector.metrics };+ collector.captureSnapshot(sessionId, "turn_completed");+ collector.captureSnapshot(sessionId, "prompt");+ await collector.idle(sessionId);+ const after = { ...collector.metrics };+ expect(after.capturesSkipped - before.capturesSkipped).toBe(2);+ expect(after.fileStats - before.fileStats).toBe(0);+ expect(after.fullScans + after.dirtyScans).toBe(before.fullScans + before.dirtyScans);+ expect(changes(uploads)).toHaveLength(0);++ // Files inside a directory that appears whole arrive without events of+ // their own: the rename of the directory taints the dirty set and the next+ // capture rescans the tree.+ await mkdir(join(root, "feature"));+ await writeFile(join(root, "feature", "new.txt"), "brand new\n");+ await sleep(150);+ expect(collector.sessionDiagnostics(sessionId)?.dirtyOverflow).toBe(true);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(collector.metrics.fullScans).toBe(before.fullScans + 1);+ expect(contentPaths(changes(uploads)[0]!)).toEqual(["feature/new.txt"]);+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths).toContain("feature");+ collector.flushTrace(sessionId);+ await collector.stop();+ expect(triggerEvents(uploads)).toEqual([+ { trigger: "turn_completed", captured: false },+ { trigger: "prompt", captured: false },+ { trigger: "turn_completed", captured: true },+ ]);+ });++ test("watches only directories with eligible files, batches ignore checks and polls past the watched-files cap", async () => {+ const root = await workspace("watch", 6);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 60_000 });+ const sessionId = "session-watch-1234";+ collector.startSession(sessionId, "workspace-watch", root);+ await collector.idle(sessionId);+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths.sort()).toEqual([".", "src"]);+ const before = { ...collector.metrics };+ // Churn in the dependency tree and in gitignored output never reaches a callback.+ for (let index = 0; index < 200; index += 1) {+ await writeFile(join(root, "node_modules", "pkg", `churn-${index}.js`), `// ${index}\n`);+ await writeFile(join(root, "dist", `chunk-${index}.js`), `// ${index}\n`);+ }+ await sleep(250);+ await collector.idle(sessionId);+ expect(collector.metrics.watchEvents - before.watchEvents).toBe(0);+ expect(collector.metrics.ignoreCheckSpawns - before.ignoreCheckSpawns).toBe(0);+ expect(changes(uploads)).toHaveLength(0);+ await collector.stop();+ expect(JSON.stringify(uploads)).not.toContain("churn-");+ expect(JSON.stringify(uploads)).not.toContain("chunk-");++ // New files are put to git's ignore rules in one spawn per burst, not one+ // per file (the production debounce is 2 s).+ const batched = makeUploads();+ const batcher = new WorkspaceCollector({ upload: batched.upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const batchedId = "session-batched-1234";+ batcher.startSession(batchedId, "workspace-batched", root);+ await batcher.idle(batchedId);+ await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\n*.log\n");+ await sleep(120);+ const beforeBurst = batcher.metrics.ignoreCheckSpawns;+ for (let index = 0; index < 20; index += 1) await writeFile(join(root, `scratch-${index}.log`), `log ${index}\n`);+ await writeFile(join(root, "kept.txt"), "kept\n");+ await sleep(150);+ batcher.captureSnapshot(batchedId, "turn_completed");+ await batcher.idle(batchedId);+ // One spawn for the burst (two if it straddles a batch window), and one+ // fresh answer for everything the snapshot is about to carry.+ expect(batcher.metrics.ignoreCheckSpawns - beforeBurst).toBeLessThanOrEqual(3);+ const batchedChange = changes(batched.uploads).at(-1)!;+ expect(contentPaths(batchedChange)).toContain("kept.txt");+ expect(batchedChange.manifest.map((entry) => entry.path)).not.toContain("scratch-0.log");+ await batcher.stop();+ expect(JSON.stringify(batched.uploads)).not.toContain("scratch-");++ // Past the cap the tree is polled: no watchers, and a milestone rescans it.+ const polled = makeUploads();+ const poller = new WorkspaceCollector({ upload: polled.upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000, maxWatchedFiles: 3 });+ const polledId = "session-polled-1234";+ poller.startSession(polledId, "workspace-polled", root);+ await poller.idle(polledId);+ expect(poller.sessionDiagnostics(polledId)).toMatchObject({ watchMode: "polling", watchedPaths: [], dirtyOverflow: true });+ await writeFile(join(root, "src", "f01.txt"), "source file 1, polled edit\n");+ poller.captureSnapshot(polledId, "turn_completed");+ await poller.idle(polledId);+ expect(poller.metrics.fullScans).toBe(2);+ expect(contentPaths(changes(polled.uploads)[0]!)).toEqual(["src/f01.txt"]);+ await poller.stop();+ });++ test("labels snapshot scope and re-sends changes the gateway never accepted", async () => {+ const root = await workspace("scope", 4);+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-collector-scope-state-"));+ roots.push(stateDir);+ let status = 201;+ const uploads: Envelope[] = [];+ const collector = new WorkspaceCollector({+ stateDir,+ upload: async (_sessionId, compressed) => {+ if (status !== 201) return Response.json({ error: "rejected" }, { status });+ uploads.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope);+ return Response.json({ ok: true }, { status: 201 });+ },+ changeDebounceMs: 60_000,+ fallbackScanMs: 60_000,+ });+ const sessionId = "session-scope-1234";+ collector.startSession(sessionId, "workspace-scope", root);+ await collector.idle(sessionId);+ expect(uploads[0]).toMatchObject({ snapshot_type: "start", files_scope: "full" });+ expect(uploads[0]).not.toHaveProperty("changed_paths");++ await writeFile(join(root, "src", "f00.txt"), "rejected edit\n");+ status = 400;+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(changes(uploads)).toHaveLength(0);+ status = 201;+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const resent = changes(uploads)[0]!;+ expect(contentPaths(resent)).toEqual(["src/f00.txt"]);+ expect(resent.changed_paths).toEqual(["src/f00.txt"]);+ expect(resent.files_scope).toBe("changed");++ collector.flushTrace(sessionId);+ await collector.stop();+ const end = uploads.find((item) => item.snapshot_type === "end")!;+ expect(end.files_scope).toBe("changed");+ expect(end.changed_paths).toEqual([]);+ expect(uploads.find((item) => item.snapshot_type === "trace")?.files_scope).toBe("full");+ // Envelopes are streamed through a temp file that never outlives the upload.+ expect((await readdir(join(stateDir, "omnirush-collector-tmp")).catch(() => [])).filter((name) => name.endsWith(".tmp"))).toEqual([]);+ expect(collector.sessionDiagnostics(sessionId)).toBeNull();+ });++ test("spaces filesystem-driven change snapshots by the minimum interval while milestones capture at once", async () => {+ const root = await workspace("interval", 4);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 60_000, minChangeIntervalMs: 500 });+ const sessionId = "session-interval-1234";+ collector.startSession(sessionId, "workspace-interval", root);+ await collector.idle(sessionId);+ await writeFile(join(root, "src", "f00.txt"), "first burst\n");+ await sleep(120);+ await collector.idle(sessionId);+ expect(changes(uploads)).toHaveLength(1);+ const firstAt = Date.parse(changes(uploads)[0]!.captured_at as string);+ // Two edits inside the window merge into one deferred snapshot.+ await writeFile(join(root, "src", "f01.txt"), "second burst a\n");+ await sleep(60);+ await writeFile(join(root, "src", "f02.txt"), "second burst b\n");+ await sleep(120);+ await collector.idle(sessionId);+ expect(changes(uploads)).toHaveLength(1);+ expect(collector.metrics.capturesDeferred).toBeGreaterThanOrEqual(1);+ await sleep(500);+ await collector.idle(sessionId);+ expect(changes(uploads)).toHaveLength(2);+ expect(contentPaths(changes(uploads)[1]!).sort()).toEqual(["src/f01.txt", "src/f02.txt"]);+ expect(Date.parse(changes(uploads)[1]!.captured_at as string) - firstAt).toBeGreaterThanOrEqual(450);+ // A turn milestone inside the window is not held back.+ await writeFile(join(root, "src", "f03.txt"), "milestone edit\n");+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(changes(uploads)).toHaveLength(3);+ expect(contentPaths(changes(uploads)[2]!)).toEqual(["src/f03.txt"]);+ await collector.stop();+ });++ test("neither watches nor rescans a rebuilt gitignored directory, and rescans once .gitignore changes", async () => {+ const root = await workspace("ignored", 6);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-ignored-1234";+ collector.startSession(sessionId, "workspace-ignored", root);+ await collector.idle(sessionId);+ const before = { ...collector.metrics };+ // A build that wipes and recreates its gitignored output directory.+ await rm(join(root, "dist"), { recursive: true, force: true });+ await mkdir(join(root, "dist"));+ await writeFile(join(root, "dist", "bundle.js"), "rebuilt\n");+ await sleep(200);+ expect(collector.sessionDiagnostics(sessionId)).toMatchObject({ dirtyOverflow: false });+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths).not.toContain("dist");+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(collector.metrics.fullScans).toBe(before.fullScans);+ expect(changes(uploads)).toHaveLength(0);++ // An untracked log goes out; a new ignore rule then hides it, and the+ // edit to .gitignore makes the next capture rescan the tree under it.+ await writeFile(join(root, "notes.log"), "scratch notes\n");+ await sleep(150);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(changes(uploads).at(-1)!.manifest.map((entry) => entry.path)).toContain("notes.log");+ await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\n*.log\n");+ await sleep(150);+ expect(collector.sessionDiagnostics(sessionId)?.dirtyOverflow).toBe(true);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const last = changes(uploads).at(-1)!;+ expect(collector.metrics.fullScans).toBe(before.fullScans + 1);+ expect(last.manifest.map((entry) => entry.path)).not.toContain("notes.log");+ expect(last.changed_paths).toEqual([".gitignore"]);+ await collector.stop();+ });++ test("drops the files of a directory moved out of the workspace without rescanning the tree", async () => {+ const root = await workspace("moved", 10);+ const outside = await mkdtemp(join(tmpdir(), "omnirush-collector-moved-out-"));+ roots.push(outside);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-moved-1234";+ collector.startSession(sessionId, "workspace-moved", root);+ await collector.idle(sessionId);+ const before = { ...collector.metrics };+ // One event for the directory, none for the five files inside it.+ await rename(join(root, "src", "lib"), join(outside, "lib"));+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const change = changes(uploads)[0]!;+ const paths = change.manifest.map((entry) => entry.path);+ expect(paths.filter((path) => path.startsWith("src/lib/"))).toEqual([]);+ expect(paths).toContain("src/f00.txt");+ expect(collector.metrics.fullScans).toBe(before.fullScans);+ expect(collector.metrics.dirtyScans).toBe(before.dirtyScans + 1);+ await collector.stop();+ });++ test("polls a workspace with more top-level directories than it watches", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-wide-"));+ roots.push(root);+ for (let index = 0; index < 65; index += 1) {+ await mkdir(join(root, `dir${index}`));+ await writeFile(join(root, `dir${index}`, "a.txt"), `file ${index}\n`);+ }+ const { upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-wide-1234";+ collector.startSession(sessionId, "workspace-wide", root);+ await collector.idle(sessionId);+ expect(collector.sessionDiagnostics(sessionId)).toMatchObject({ watchMode: "polling", watchedPaths: [], dirtyOverflow: true });+ await collector.stop();+ });++ test("keeps a file larger than the change journal out of it, and still snapshots the file", async () => {+ const root = await workspace("journal", 4);+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-journal-1234";+ collector.startSession(sessionId, "workspace-journal", root);+ await collector.idle(sessionId);+ await writeFile(join(root, "src", "small.txt"), "small edit\n");+ await writeFile(join(root, "src", "large.txt"), "large log line\n".repeat(70_000));+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const change = changes(uploads)[0]!;+ expect(contentPaths(change).sort()).toEqual(["src/large.txt", "src/small.txt"]);+ const journal = change.files.find((file) => file.path === "__omnirush__/changes.json")!;+ expect(journal.content).toContain('"path":"src/small.txt"');+ expect(journal.content).not.toContain("src/large.txt");+ await collector.stop();+ });++ test("runs one reconcile pass per interval for every session on a root, and it finds what no watcher reported", async () => {+ const root = await workspace("reconcile", 4);+ // Every watcher slot is taken, so assets/, which held no file at the start, gets none.+ await fillWatchSlots(root);+ await mkdir(join(root, "assets"));+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 300, minChangeIntervalMs: 0 });+ const sessions = ["session-reconcile-a1", "session-reconcile-b1"];+ for (const sessionId of sessions) collector.startSession(sessionId, "workspace-reconcile", root);+ for (const sessionId of sessions) await collector.idle(sessionId);+ expect(collector.sessionDiagnostics(sessions[0]!)).toMatchObject({ watchMode: "watching" });+ expect(collector.sessionDiagnostics(sessions[0]!)?.watchedPaths).toHaveLength(65);+ expect(collector.sessionDiagnostics(sessions[0]!)?.watchedPaths).not.toContain("assets");+ const before = collector.metrics.reconciles;+ await writeFile(join(root, "assets", "found.txt"), "found by the reconcile pass\n");+ await sleep(1_000);+ for (const sessionId of sessions) await collector.idle(sessionId);+ const passes = collector.metrics.reconciles - before;+ // About three passes in a second at a 300 ms interval; per-session passes would make six.+ expect(passes).toBeGreaterThanOrEqual(1);+ expect(passes).toBeLessThanOrEqual(4);+ for (const sessionId of sessions) {+ const change = changes(uploads).find((item) => item.session_id === sessionId);+ expect(change && contentPaths(change)).toEqual(["assets/found.txt"]);+ }+ await collector.stop();+ });++ test("keeps gitignored files out of change snapshots and the journal in a workspace git does not manage", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-nogit-"));+ roots.push(root);+ await writeFile(join(root, ".gitignore"), "*.log\nbuild/\n");+ await mkdir(join(root, "src"));+ await writeFile(join(root, "src", "a.txt"), "a\n");+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-nogit-1234";+ collector.startSession(sessionId, "workspace-nogit", root);+ await collector.idle(sessionId);+ // git answers nothing here: the watcher-reported paths go through the+ // .gitignore files the listing walker applies.+ await writeFile(join(root, "src", "debug.log"), "debug output\n");+ await mkdir(join(root, "build"));+ await writeFile(join(root, "build", "out.txt"), "build output\n");+ await writeFile(join(root, "src", "b.txt"), "b\n");+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const change = changes(uploads)[0]!;+ expect(contentPaths(change)).toEqual(["src/b.txt"]);+ expect(change.manifest.map((entry) => entry.path).sort()).toEqual([".gitignore", "src/a.txt", "src/b.txt"]);+ await collector.stop();+ expect(JSON.stringify(uploads)).not.toContain("debug.log");+ expect(JSON.stringify(uploads)).not.toContain("out.txt");+ });++ /** src/ plus 63 more top-level directories with files: every watcher slot taken. */+ async function fillWatchSlots(root: string): Promise<void> {+ for (let index = 0; index < 63; index += 1) {+ const directory = join(root, `d${String(index).padStart(2, "0")}`);+ await mkdir(directory);+ await writeFile(join(directory, "a.txt"), `file ${index}\n`);+ }+ }++ test("watches the top-level directories the listing did not name, unless git ignores or the denylist names them", async () => {+ const root = await workspace("unlisted", 4);+ await mkdir(join(root, "config"));+ await writeFile(join(root, "config", ".env"), "API=1\n"); // denied, so config/ lists nothing+ await mkdir(join(root, "scripts")); // empty+ await mkdir(join(root, "secrets")); // a denied name+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 60_000, minChangeIntervalMs: 0 });+ const sessionId = "session-unlisted-1234";+ collector.startSession(sessionId, "workspace-unlisted", root);+ await collector.idle(sessionId);+ // Not dist/ (gitignored), node_modules/, secrets/ or .git/ (denied).+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths.sort()).toEqual([".", "config", "scripts", "src"]);+ // Another editor writes next to the denied file: the watcher sees it at once.+ await writeFile(join(root, "config", "app.yml"), "mode: production\n");+ await sleep(300);+ await collector.idle(sessionId);+ const change = changes(uploads)[0]!;+ expect(change.trigger).toBe("fs_change");+ expect(contentPaths(change)).toEqual(["config/app.yml"]);+ await collector.stop();+ expect(JSON.stringify(uploads)).not.toContain("API=1");+ });++ test("a turn's writes where no watcher reaches still land in that turn's change snapshot", async () => {+ const root = await workspace("unwatched", 4);+ await fillWatchSlots(root);+ await mkdir(join(root, "assets"));+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-unwatched-1234";+ collector.startSession(sessionId, "workspace-unwatched", root);+ await collector.idle(sessionId);+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths).not.toContain("assets");+ collector.captureSnapshot(sessionId, "prompt");+ await collector.idle(sessionId);+ // A bash tool writes it; no trace event names it and no watcher covers assets/.+ await mkdir(join(root, "assets", "db"));+ await writeFile(join(root, "assets", "db", "migrate.sh"), "#!/bin/sh\necho migrate\n");+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ // The end-of-turn artifact scan marks it dirty for the turn's own snapshot.+ expect(contentPaths(changes(uploads)[0]!)).toEqual(["assets/db/migrate.sh"]);+ collector.flushTrace(sessionId);+ await collector.stop();+ expect(triggerEvents(uploads)).toEqual([+ { trigger: "prompt", captured: false },+ { trigger: "turn_completed", captured: true },+ ]);+ });++ test.skipIf(process.platform === "win32")("a path git refuses to answer for (beyond a symlinked directory) neither changes the ignore answer for its batch nor gets read", async () => {+ const base = await mkdtemp(join(tmpdir(), "omnirush-collector-symlink-"));+ roots.push(base);+ const root = join(base, "repo");+ const outside = join(base, "shared-lib");+ await mkdir(join(root, "src"), { recursive: true });+ await mkdir(outside);+ await git(root, "init", "-q");+ await appendFile(join(root, ".git", "info", "exclude"), "private-notes.txt\n");+ await writeFile(join(root, "src", "app.ts"), "export const app = 1;\n");+ await writeFile(join(outside, "util.ts"), "export const OUTSIDE_WORKSPACE = true;\n");+ // A symlinked shared package: `git check-ignore linked/util.ts` exits 128.+ await symlink("../shared-lib", join(root, "linked"));+ await git(root, "add", "-A");+ await git(root, "commit", "-q", "-m", "init");+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = "session-symlink-1234";+ collector.startSession(sessionId, "workspace-symlink", root);+ await collector.idle(sessionId);+ // In one ignore batch: a note git ignores only through .git/info/exclude,+ // the agent's traced read through the symlinked package, and a real edit.+ await writeFile(join(root, "private-notes.txt"), "PRIVATE: salary negotiation notes\n");+ collector.recordTrace(sessionId, "tool.read", { filePath: "linked/util.ts" });+ await writeFile(join(root, "src", "app.ts"), "export const app = 2;\n");+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const change = changes(uploads)[0]!;+ expect(contentPaths(change)).toEqual(["src/app.ts"]);+ expect(change.manifest.map((entry) => entry.path)).toEqual(["src/app.ts"]);+ expect(change.files.find((file) => file.path === "__omnirush__/changes.json")?.content).toContain('"path":"src/app.ts"');+ await collector.stop();+ const all = JSON.stringify(uploads);+ expect(all).not.toContain("salary negotiation");+ expect(all).not.toContain("OUTSIDE_WORKSPACE");+ });++ test("a listed file the user then ignores leaves the manifest, files[] and the journal, through .gitignore or .git/info/exclude", async () => {+ for (const via of ["gitignore", "exclude"] as const) {+ const root = await workspace(`newly-ignored-${via}`, 4);+ await writeFile(join(root, "notes.txt"), "scratch v1\n"); // untracked, not ignored yet+ const { uploads, upload } = makeUploads();+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });+ const sessionId = `session-newly-ignored-${via}`;+ collector.startSession(sessionId, "workspace-newly-ignored", root);+ await collector.idle(sessionId);+ expect(uploads[0]!.manifest.map((entry) => entry.path)).toContain("notes.txt");+ // An edit while it is still collected: git's "not ignored" is now cached.+ await writeFile(join(root, "notes.txt"), "scratch v1b\n");+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ expect(contentPaths(changes(uploads)[0]!)).toEqual(["notes.txt"]);+ // The user hides it from git (.git/info/exclude raises no event), then writes something private into it.+ if (via === "gitignore") await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\nnotes.txt\n");+ else await appendFile(join(root, ".git", "info", "exclude"), "notes.txt\n");+ await sleep(150);+ await writeFile(join(root, "notes.txt"), "PRIVATE: do not share v2\n");+ await sleep(200);+ collector.captureSnapshot(sessionId, "turn_completed");+ await collector.idle(sessionId);+ const change = changes(uploads)[1]!;+ expect(change.manifest.map((entry) => entry.path)).not.toContain("notes.txt");+ expect(contentPaths(change)).not.toContain("notes.txt");+ await collector.stop();+ expect(JSON.stringify(uploads)).not.toContain("do not share v2");+ }+ });++ test("snapshots a 60 MiB workspace with peak RSS growth under 120 MB", async () => {+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-memory-"));+ roots.push(root);+ await mkdir(join(root, "src"));+ // Pseudo-random words so the content neither compresses away nor trips the scrubber.+ const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"];+ const filler = (seed: number, bytes: number): string => {+ let out = "";+ let value = seed;+ while (out.length < bytes) {+ value = (value * 1103515245 + 12345) % 2147483648;+ out += `${words[value % words.length]} ${value.toString(36)}${value % 7 === 0 ? "\n" : " "}`;+ }+ return out.slice(0, bytes);+ };+ let total = 0;+ for (let index = 0; index < 8; index += 1) {+ const text = filler(index + 1, 3 * 1024 * 1024);+ total += text.length;+ await writeFile(join(root, "src", `large-${index}.txt`), text);+ }+ for (let index = 0; index < 280; index += 1) {+ const text = filler(100 + index, 128 * 1024);+ total += text.length;+ await writeFile(join(root, "src", `small-${String(index).padStart(3, "0")}.txt`), text);+ }+ // Just under what the 64 MiB snapshot cap leaves for content.+ expect(total).toBeGreaterThanOrEqual(58 * 1024 * 1024);++ const compressedEnvelopes: Uint8Array[] = [];+ const collector = new WorkspaceCollector({+ upload: async (_sessionId, compressed) => {+ compressedEnvelopes.push(compressed);+ return Response.json({ ok: true }, { status: 201 });+ },+ changeDebounceMs: 60_000,+ fallbackScanMs: 60_000,+ });+ const gc = (globalThis as { gc?: () => void }).gc ?? (globalThis as { Bun?: { gc?: (force: boolean) => void } }).Bun?.gc?.bind(null, true);+ gc?.();+ await sleep(50);+ const baseline = process.memoryUsage().rss;+ let peak = baseline;+ const sampler = setInterval(() => { peak = Math.max(peak, process.memoryUsage().rss); }, 20);+ const sessionId = "session-memory-1234";+ collector.startSession(sessionId, "workspace-memory", root);+ await collector.idle(sessionId);+ peak = Math.max(peak, process.memoryUsage().rss);+ clearInterval(sampler);+ await collector.stop();++ const growthMB = (peak - baseline) / (1024 * 1024);+ expect(growthMB).toBeLessThan(120);+ const start = JSON.parse(zstdDecompressSync(compressedEnvelopes[0]!).toString("utf8")) as Envelope;+ expect(start.snapshot_type).toBe("start");+ expect(start.files.reduce((sum, file) => sum + file.content.length, 0)).toBeGreaterThanOrEqual(58 * 1024 * 1024);+ expect(start.privacy).toMatchObject({ files_truncated: false });+ }, 60_000);+});diff --git a/apps/server/src/workspace-collector.ts b/apps/server/src/workspace-collector.tsindex 18acb4b5..b11ae498 100644--- a/apps/server/src/workspace-collector.ts+++ b/apps/server/src/workspace-collector.ts@@ -1,11 +1,11 @@import { createHash, randomBytes } from "node:crypto";import { execFile, spawn } from "node:child_process";-import { createReadStream, watch, type FSWatcher } from "node:fs";-import { lstat, mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";-import { release as osRelease } from "node:os";+import { createReadStream, createWriteStream, watch, type FSWatcher, type WriteStream } from "node:fs";+import { lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";+import { release as osRelease, tmpdir } from "node:os";import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";import { promisify } from "node:util";-import { zstdCompress } from "node:zlib";+import { createZstdCompress } from "node:zlib";import { minimatch } from "minimatch";import { externalFetch } from "./server-fetch.js";@@ -52,7 +52,36 @@ const MAX_WEB_VISIT_TITLE_CHARS = 512;const MAX_ATTACHMENT_NAME_CHARS = 512;const MAX_ATTACHMENT_MIME_CHARS = 128;const CHANGE_DEBOUNCE_MS = 2_000;-const FALLBACK_SCAN_MS = 10_000;+// While a workspace is watched, a reconcile pass (listing + lstat, no reads)+// runs this often to catch events the watcher missed; a polled workspace (see+// MAX_COLLECTOR_WATCHED_FILES) is rescanned on snapshots only, never on a timer.+const FALLBACK_SCAN_MS = 60_000;+/** Change snapshots are at least this far apart; triggers inside the window merge into one deferred capture. */+export const MIN_COLLECTOR_CHANGE_INTERVAL_MS = 30_000;+/** A workspace with more eligible files than this is polled on snapshots instead of watched. */+export const MAX_COLLECTOR_WATCHED_FILES = 20_000;+// Paths the watcher reported since the last capture. Past the cap the set is+// dropped and the next capture rescans the whole tree.+const MAX_DIRTY_PATHS = 8_192;+// Top-level directories watched recursively; a workspace with more is polled.+const MAX_WATCH_ROOTS = 64;+// Reads and stats in flight during a scan, and how long synchronous scan work+// (redaction, hashing) may run before the event loop gets a turn.+const SCAN_CONCURRENCY = 8;+const EVENT_LOOP_YIELD_MS = 12;+// A read slot holds any ordinary source file; larger ones are read whole.+const READ_SLOT_BYTES = 256 * 1024;+// Watcher-reported paths the manifest does not know are put to git's ignore+// rules in batches: one spawn per burst instead of one per event.+const IGNORE_CHECK_BATCH_MS = 50;+// A prompt or turn milestone can land milliseconds after the edit that+// matters; the watcher delivers within tens of milliseconds, so a milestone+// capture waits this long for it before consulting the dirty set.+const MILESTONE_SETTLE_MS = 100;+const IGNORE_CHECK_BATCH_MAX = 2_000;+const MAX_IGNORED_CACHE_ENTRIES = 8_192;+const TEMP_DIRECTORY = "omnirush-collector-tmp";+const ENVELOPE_WRITE_CHUNK_BYTES = 64 * 1024;const MAX_CHANGE_JOURNAL_ENTRIES = 512;const MAX_CHANGE_JOURNAL_BYTES = 768 * 1024;const MAX_SESSION_LEDGER_ENTRIES = 512;@@ -107,12 +136,6 @@ type CollectorFile = {sha256: string;};-type ManifestEntry = {- path: string;- sha256: string;- size: number;-};-type GitRemote = { name: string; url: string };type GitCommit = { sha: string; at: string; subject: string };type GitStatusEntry = { code: string; path: string };@@ -228,7 +251,13 @@ type HashCacheEntry = {mtimeMs: number;binary: boolean;sha256: string;+ /** UTF-8 bytes of the redacted content: the manifest size. */bytes: number;+ /** Bytes of that content once JSON-encoded: what a files[] entry costs. */+ json: number;+ /** Redaction changed nothing, so the raw bytes are the upload form (verified by digest on upload). */+ clean: boolean;+ uploadPath: string;};type SpoolMeta = {@@ -252,20 +281,42 @@ type SessionState = {sequence: number;resumed: boolean;lastMessageId?: string;- lastSignature: string;started: boolean;finished: boolean;changeTimer: ReturnType<typeof setTimeout> | null;changeTrigger: ChangeTrigger | null;+ /** A capture held back by the minimum change interval, and the reason it will run for. */+ deferTimer: ReturnType<typeof setTimeout> | null;+ deferTrigger: ChangeTrigger | null;+ /** The reconcile pass, while watching. */scanTimer: ReturnType<typeof setInterval> | null;- watcher: FSWatcher | null;+ watchers: FSWatcher[];+ watchMode: "starting" | "watching" | "polling";+ /** Workspace-relative directories under a watcher ("." for the root). */+ watchedPaths: string[];+ /** Watchers still being added for top-level directories the listing did not name. */+ watchSetup: Promise<void>;trace: TraceEvent[];changeJournal: Map<string, ChangeJournalEntry>;touchedPaths: Set<string>;changeJournalBytes: number;changeCaptureTail: Promise<void>;- hashCache: Map<string, HashCacheEntry>;- baseline: Map<string, string> | null;+ /** Paths whose journal capture is queued but not started. */+ pendingJournal: Set<string>;+ ignoreBatch: Map<string, Array<(ignored: boolean) => void>>;+ ignoreTimer: ReturnType<typeof setTimeout> | null;+ ignoredCache: Map<string, boolean>;+ /** The last accepted manifest: what the next change snapshot is measured against. */+ manifest: Manifest | null;+ /** The per-root hash cache, shared with every other session open on the same root. */+ cache: Map<string, HashCacheEntry>;+ listing: { denied: number; truncated: boolean };+ /** Paths the watcher (or a trace) reported since the last capture. */+ dirty: Set<string>;+ /** The dirty set cannot be trusted; the next capture scans the whole tree. */+ dirtyOverflow: boolean;+ lastHead: string | null;+ lastChangeAt: number;/** Untracked-but-not-ignored files as they stood when the current turn began. */artifactBaseline: Promise<Map<string, ArtifactStat>> | null;model: CollectorSessionModel | null;@@ -303,6 +354,10 @@ type CollectorOptions = {spoolMaxBytes?: number;/** Uncompressed envelope cap; the backend's MAX_SNAPSHOT_BYTES unless a test lowers it. */snapshotMaxBytes?: number;+ /** Minimum spacing between change snapshots; MIN_COLLECTOR_CHANGE_INTERVAL_MS unless a test lowers it. */+ minChangeIntervalMs?: number;+ /** Watcher cap; MAX_COLLECTOR_WATCHED_FILES unless a test lowers it. */+ maxWatchedFiles?: number;};type TransmitOutcome =@@ -374,16 +429,42 @@ const PRIVATE_KEY_BLOCK: Redaction = [* A backslash escape counts as one unit so the match never splits it. Whether* the key names a secret is decided by isSecretAssignmentKey; the value by* isSecretAssignmentValue. Every quantifier is bounded or anchored so a 1 MiB- * line costs linear time.+ * line costs linear time. Where the key may start depends on the mode: see+ * ASSIGNMENT_PATTERN (CONFIG) and SOURCE_ASSIGNMENT_PATTERN (SOURCE).*/// Pre-filter for the assignment regex: the key must at least contain a keyword// substring, so an ordinary assignment never consumes a value that holds a// secret one (`"text": "{\"password\":...}"`). The exact, segment-bounded// decision is isSecretAssignmentKey.const ASSIGNMENT_KEYWORD_SOURCE = String.raw`secret|passw(?:or)?d|pwd|token|credential|auth|(?:api|access|private|client|session|signing|master|encryption)[_.:-]?key`;-const ASSIGNMENT_PATTERN = new RegExp(- String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))(\\?["']?)((?=[\w.-]{0,63}?(?:${ASSIGNMENT_KEYWORD_SOURCE}))[A-Za-z_-][\w.-]{0,63})\1[ \t]*[=:][ \t]*`- + String.raw`(?:"((?:[^"\s\\]|\\\S){8,})"|'((?:[^'\s\\]|\\\S){8,})'|\\"((?:[^"\s\\]|\\[^"\s]){8,})\\"|((?:[^\s"',;&\\]|\\\S){8,}))`,+// Python's `\s` (the backend's regexes run on str): JS `\s` minus U+FEFF, plus+// U+001C-U+001F and U+0085. The assignment rule spells whitespace this way so a+// value ends, and is rejected for holding whitespace, exactly where the+// backend's does.+const PYTHON_WHITESPACE = String.raw`\t\n\v\f\r \x1c-\x1f\x85\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000`;+const PYTHON_WHITESPACE_CHAR = new RegExp(String.raw`[${PYTHON_WHITESPACE}]`);+const ASSIGNMENT_BODY_SOURCE = String.raw`(\\?["']?)((?=[\w.-]{0,63}?(?:${ASSIGNMENT_KEYWORD_SOURCE}))[A-Za-z_-][\w.-]{0,63})\1[ \t]*[=:][ \t]*`+ + String.raw`(?:"((?:[^"${PYTHON_WHITESPACE}\\]|\\[^${PYTHON_WHITESPACE}]){8,})"`+ + String.raw`|'((?:[^'${PYTHON_WHITESPACE}\\]|\\[^${PYTHON_WHITESPACE}]){8,})'`+ + String.raw`|\\"((?:[^"${PYTHON_WHITESPACE}\\]|\\[^"${PYTHON_WHITESPACE}]){8,})\\"`+ + String.raw`|((?:[^${PYTHON_WHITESPACE}"',;&\\]|\\[^${PYTHON_WHITESPACE}]){8,}))`;+// CONFIG mode: the key starts a word (not after a word character, `.`, `-` or+// a backslash), so `#TOKEN=` on a commented-out line and `?token=` or+// `&token=` in a query string still count.+const ASSIGNMENT_PATTERN = new RegExp(String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))` + ASSIGNMENT_BODY_SOURCE, "gi");+// SOURCE mode: the key starts the text or follows whitespace, `{`, `,`, `(`, a+// quote (`"`, `'`, backtick), a `+` diff marker (a .patch file is source) or+// `#`; never `-`, `.`, a word character or any other punctuation, so `token:`+// after a `/`, `:` or `[` inside a string literal and `?token=` in a URL+// literal are not read as assignments. Both modes accept a key right after a+// `\n`, `\r` or `\t` escape. The backend's _SOURCE_ASSIGNMENT_PATTERN, except+// that any non-ASCII character also counts as a delimiter: the backend's `\w`+// is Unicode, so its key runs through `café_token` from the `c`, while a JS key+// (ASCII `\w`) can only start after the `é`. This keeps every value the+// backend redacts redacted here too (and a key right after a byte order mark,+// the first line of a BOM-prefixed file, still counts on the desktop).+const SOURCE_ASSIGNMENT_PATTERN = new RegExp(+ String.raw`(?:(?<![^${PYTHON_WHITESPACE}\x80-\uffff{,("'\x60+#])|(?<=\\[nrt]))` + ASSIGNMENT_BODY_SOURCE,"gi",);/**@@ -408,6 +489,14 @@ const EXCLUDED_LAST_SEGMENTS = new Set(["file", "filename", "dir", "mode", "method", "role", "owner", "type", "kind", "enabled", "estimate", "hash", "digest", "at","config", "client", "prefix", "suffix", "format", "scheme", "provider", "status", "state", "label", "description", "title","class", "field", "fields", "list", "names", "version", "timeout", "limit", "max", "min", "interval", "retries", "port",+ // Added after the second production sample (`refresh_token_expires_in`,+ // `password_encryption`, `provider_credential_ref`, `secret_envelope`).+ // `code` and `plaintext` are deliberately absent: `auth_code` and+ // `password_plaintext` name secrets.+ "in", "out", "percent", "pct", "ms", "secs", "minutes", "hours", "days", "expires", "expiry", "expiration", "bytes", "len",+ "width", "height", "offset", "index", "idx", "pos", "total", "sum", "avg", "ratio", "rate", "threshold", "weight", "score",+ "encryption", "algorithm", "algo", "cipher", "strategy", "policy", "source", "target", "origin", "backend", "engine", "driver",+ "handler", "callback", "event", "action", "reason", "message", "error", "envelope", "ref", "reference", "link", "pointer", "alias",]);/**@@ -415,13 +504,14 @@ const EXCLUDED_LAST_SEGMENTS = new Set([* extension names a programming language is scrubbed in SOURCE mode, where an* assignment's value must look like a generated literal; everything else* (.env*, ini, cfg, conf, yml, yaml, toml, json, properties, txt, md, no or- * unknown extension), plus git diffs and the trace, is scrubbed in CONFIG- * mode with the permissive rule.+ * unknown extension), plus the envelope's git diff and the trace, is scrubbed+ * in CONFIG mode with the permissive rule. A `.patch` or `.diff` workspace+ * file carries code and is SOURCE.*/export type RedactMode = "source" | "config";const SOURCE_EXTENSIONS = new Set(["js", "cjs", "mjs", "ts", "tsx", "jsx", "py", "go", "rs", "java", "kt", "c", "cc", "cpp", "h", "hpp", "rb", "php", "swift",- "cs", "scala", "sh", "bash", "zsh", "ps1", "lua", "dart", "vue", "svelte", "map",+ "cs", "scala", "sh", "bash", "zsh", "ps1", "lua", "dart", "vue", "svelte", "map", "patch", "diff",]);/** SOURCE for a programming-language extension (case-insensitive), CONFIG otherwise. */@@ -433,11 +523,23 @@ export function redactModeForPath(path: string): RedactMode {const CODE_EXPRESSION_VALUE = /^(?:process\.|os\.|env\.)/;const IDENTIFIER_VALUE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/;const PATH_VALUE = /^(?:\/|\.\/|\.\.\/|~\/|[A-Za-z]:[\\/])/;+// The shape of an unquoted SOURCE value that can be a literal token: letters,+// digits and `_ . / + = ~ -` only (`abc-123-def`, base64 with `+/=`). A `?`,+// `:`, `[`, `]`, `!`, `|`, `&`, `,`, `;`, `)`, quote, backtick, `*`, `<` or+// `>` makes it code (`stagedClient?.plaintext`, `params[8]`, `[0-9a-f]{16}`,+// `session:trial-123`).+const TOKEN_SHAPE_VALUE = /^[A-Za-z0-9_./+=~-]+$/;+// A JSON key that looks like a file path or filename: holds a `/` or `\`, or+// ends in a dot extension of one to five letters or digits. `\n?` mirrors+// Python's `$`, which also matches before one trailing newline.+const PATH_LIKE_KEY = /[/\\]|\.[A-Za-z0-9]{1,5}\n?$/;const MAX_ASSIGNMENT_DEPTH = 4;// An AWS secret access key is 40 base64 characters with no shape of its own,// so a candidate only counts near an access key id or an aws/secret key name.-const AWS_SECRET_QUICK = /[A-Za-z0-9/+]{40}/;+// The quick check only tries a position that starts a run of those characters,+// so it costs one pass over the text instead of one per character of every run.+const AWS_SECRET_QUICK = /(?<![A-Za-z0-9/+])[A-Za-z0-9/+]{40}/;const AWS_SECRET_CANDIDATE = tokenPattern(String.raw`[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+])`, "g", String.raw`[A-Za-z0-9/+\\]`);const AWS_SECRET_CONTEXT = /(?:AKIA|ASIA)[0-9A-Z]{16}|aws|secret/i;const AWS_SECRET_CONTEXT_LINES = 3;@@ -445,10 +547,19 @@ const AWS_SECRET_CONTEXT_LINES = 3;// document (compact JSON, the trace) relies on its JSON key instead.const AWS_SECRET_MIN_LINES = 3;-// Runs before the assignment rule so `https://oauth2:<token>@host/...` keeps-// its host instead of being read as an `oauth2:` assignment.+// `scheme://user:pass@host`: the userinfo is redacted only when it holds a+// password (a `:` after the user), replaced whole by `[REDACTED]`; a bare+// `postgres://live@host/db` in a fixture stays. The user may be empty:+// `redis://:<password>@host` is Redis's own AUTH form. Runs before the+// assignment rule so `https://oauth2:<token>@host/...` keeps its host instead+// of being read as an `oauth2:` assignment. Whitespace is Python's, as in the+// backend's _URL_USERINFO_TEXT; unlike it (at 46ee95e), the user part may be+// empty, so the desktop is the stricter side there.const URL_USERINFO: Redaction = [- tokenPattern(String.raw`([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/@\\]|\\\S)+@`, "gi"),+ tokenPattern(+ String.raw`([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^${PYTHON_WHITESPACE}/@:\\]|\\[^${PYTHON_WHITESPACE}])*:(?:[^${PYTHON_WHITESPACE}/@\\]|\\[^${PYTHON_WHITESPACE}])+@`,+ "gi",+ ),(_match, scheme) => `${scheme}${REDACTED}@`,];@@ -632,7 +743,8 @@ function keySegments(key: string): string[] {.filter(Boolean);}-function isSecretAssignmentKey(key: string): boolean {+/** Whether an assignment key (or JSON object key) names a secret; the backend's is_secret_assignment_key. */+export function isSecretAssignmentKey(key: string): boolean {const segments = keySegments(key);const last = segments.at(-1);if (last === undefined || EXCLUDED_LAST_SEGMENTS.has(last)) return false;@@ -640,6 +752,20 @@ function isSecretAssignmentKey(key: string): boolean {|| SECRET_KEY_PAIRS.some(([first, second]) => segment === `${first}${second}` || (segment === first && segments[index + 1] === second)));}+/**+ * Whether a JSON object key looks like a file path or filename: it holds a `/`+ * or `\`, or ends in `.` plus one to five letters or digits+ * (`src/admin_auth.py`, `admin_auth.py`, `C:\x`). Such a key never names a+ * secret (a source-hashes document maps paths to digests), so the JSON rail+ * skips the key rule for it; its value still gets the text scrub, so a+ * provider shape under it is caught. The text rule is unaffected:+ * `admin_auth.py: abcdefgh1234` in YAML is still an assignment. The backend's+ * is_path_like_key.+ */+export function isPathLikeKey(key: string): boolean {+ return PATH_LIKE_KEY.test(key);+}+/*** A filesystem path: an absolute, relative, home or drive-letter prefix, or a* `/` anywhere with no digit anywhere (`/run/secrets/x`, `C:/Users/me`).@@ -656,20 +782,24 @@ function isFilesystemPathValue(value: string): boolean {* `<|endoftext|>`), not an earlier redaction and not a filesystem path.* SOURCE mode then wants a literal that looks generated: an unquoted value* with identifier or member-expression syntax is a reference (`config.token`,- * `tokenFile`, `req.headers.authorization`), never a literal, and the value+ * `tokenFile`, `req.headers.authorization`), never a literal; an unquoted value+ * must also be a token shape (TOKEN_SHAPE_VALUE: `stagedClient?.plaintext`,+ * `params[8]`, `[0-9a-f]{16}` and `session:trial-123` are code); and the value* must carry a digit or be 20+ characters of mixed case, so `'synthetic'`* and `'github-app'` stay. CONFIG mode keeps the permissive rule (a letter* anywhere), skipping only an unquoted dotted identifier such as YAML's- * `token: config.token`.+ * `token: config.token`. The backend's is_secret_assignment_value.*/-function isSecretAssignmentValue(value: string, mode: RedactMode, quoted: boolean): boolean {- if (value.length < 8 || /\s/.test(value) || value.startsWith("[REDACTED")) return false;+export function isSecretAssignmentValue(value: string, mode: RedactMode, quoted: boolean): boolean {+ if (value.length < 8 || PYTHON_WHITESPACE_CHAR.test(value) || value.startsWith("[REDACTED")) return false;if (value.includes("(") || value.includes("${") || value.includes("$(") || CODE_EXPRESSION_VALUE.test(value)) return false;if (value.startsWith("<") && value.endsWith(">")) return false;if (isFilesystemPathValue(value)) return false;const identifier = !quoted && IDENTIFIER_VALUE.test(value);if (mode === "source") {- return !identifier && (/\d/.test(value) || (value.length >= 20 && /[a-z]/.test(value) && /[A-Z]/.test(value)));+ return !identifier+ && (quoted || TOKEN_SHAPE_VALUE.test(value))+ && (/\d/.test(value) || (value.length >= 20 && /[a-z]/.test(value) && /[A-Z]/.test(value)));}return /[A-Za-z]/.test(value) && !(identifier && value.includes("."));}@@ -695,8 +825,9 @@ function applyRedaction(text: string, [pattern, replacement]: Redaction, tally:/** Redacts the value of every secret-named assignment; one redaction per assignment. */function redactAssignments(text: string, tally: { count: number }, mode: RedactMode, depth = 0): string {- ASSIGNMENT_PATTERN.lastIndex = 0;- return text.replace(ASSIGNMENT_PATTERN, (match: string, _quote: string, key: string, doubleQuoted?: string, singleQuoted?: string, escapedQuoted?: string, bare?: string) => {+ const pattern = mode === "source" ? SOURCE_ASSIGNMENT_PATTERN : ASSIGNMENT_PATTERN;+ pattern.lastIndex = 0;+ return text.replace(pattern, (match: string, _quote: string, key: string, doubleQuoted?: string, singleQuoted?: string, escapedQuoted?: string, bare?: string) => {const value = doubleQuoted ?? singleQuoted ?? escapedQuoted ?? bare ?? "";const quote = doubleQuoted !== undefined ? '"' : singleQuoted !== undefined ? "'" : escapedQuoted !== undefined ? '\\"' : "";const prefix = match.slice(0, match.length - value.length - quote.length * 2);@@ -725,18 +856,25 @@ function redactBearerTokens(text: string, tally: { count: number }): string {* Redacts 40-character mixed-case base64 runs that sit within three lines of* an AWS access key id or of the words aws/secret (or whose surrounding* context, such as the file path or JSON key, names them). Only a document of- * three or more lines (two or more line breaks) is scanned.+ * three or more lines (two or more line breaks) is scanned, unless the+ * enclosing JSON object or array names an AWS key (`awsContext`, the JSON+ * container rail), which vouches for a single string. The backend's+ * _redact_aws_secrets.*/-function redactAwsSecrets(text: string, context: string | undefined, tally: { count: number }): string {+function redactAwsSecrets(text: string, context: string | undefined, tally: { count: number }, awsContext = false): string {+ const nearContext = awsContext || (context !== undefined && AWS_SECRET_CONTEXT.test(context));+ // No context anywhere in the text or around it means no line can qualify.+ // Both checks must pass; this one goes first because a literal scan is+ // many times cheaper than looking for a 40-character run.+ if (!nearContext && !AWS_SECRET_CONTEXT.test(text)) return text;if (!AWS_SECRET_QUICK.test(text)) return text;const lines = text.split("\n");- if (lines.length < AWS_SECRET_MIN_LINES) return text;+ if (lines.length < AWS_SECRET_MIN_LINES && !awsContext) return text;const contextual: Array<boolean | undefined> = new Array(lines.length);const hasContext = (index: number): boolean => {if (contextual[index] === undefined) contextual[index] = AWS_SECRET_CONTEXT.test(lines[index]!);return contextual[index]!;};- const nearContext = context !== undefined && AWS_SECRET_CONTEXT.test(context);let changed = false;for (let index = 0; index < lines.length; index += 1) {const line = lines[index]!;@@ -756,11 +894,76 @@ function redactAwsSecrets(text: string, context: string | undefined, tally: { coreturn changed ? lines.join("\n") : text;}+// Substring gates for the text scrubber: each rule needs a literal the text+// must contain, so a text without it skips that regex pass outright. A gate is+// sound (it never skips a text its rule could match) and changes nothing about+// what is redacted; it only spares the scan's dominant CPU cost. A pattern+// without a gate always runs.+// The assignment rule's key must contain one of these (case-insensitively).+const ASSIGNMENT_LINE_GATE = /secret|passw|pwd|token|credential|auth|key/gi;+const BEARER_GATE = /bearer/i;+const DIGIT_GATE = /\d/;+const DISCORD_TOKEN_GATE = /[MN][\w-]{23,}\./;+const SECRET_PATTERN_GATES: Array<(text: string) => boolean> = [+ (text) => text.includes("AKIA") || text.includes("ASIA"),+ (text) => /gh[pousr]_/.test(text),+ (text) => text.includes("github_pat_"),+ (text) => /xox[abpr]-/.test(text),+ (text) => text.includes("AIza"),+ (text) => /[sr]k_(?:live|test)_/.test(text),+ (text) => text.includes("sk-ant-"),+ (text) => /(?:sk|rk|pk)-/.test(text),+ (text) => text.includes("eyJ"),+ (text) => DISCORD_TOKEN_GATE.test(text),+];+// Every literal the gates above look for, in one scan: when none occurs (and+// no Discord-shaped run either) no provider-token rule can run, so the text+// each later gate would see is unchanged and the whole loop is skipped.+const PROVIDER_TOKEN_LITERALS = /AKIA|ASIA|gh[pousr]_|github_pat_|xox[abpr]-|AIza|[sr]k_(?:live|test)_|sk-ant-|(?:sk|rk|pk)-|eyJ/;++/**+ * Runs `redact` over each line holding a gate match and leaves every other+ * line as it is. Sound only for a rule whose matches never contain a line+ * break and always contain a gate match: the assignment and e-mail rules admit+ * no "\n" in any character class, their keys (or "@") are part of the match's+ * line, and their lookbehinds see a line start's "\n" exactly as they see the+ * start of the text. The result is identical to running the rule over the+ * whole text; only lines that could match pay for the rule's regex.+ */+function redactGatedLines(text: string, nextGate: (from: number) => number, redact: (line: string) => string): string {+ let output = "";+ let copied = 0;+ let changed = false;+ let at = nextGate(0);+ while (at !== -1) {+ const start = text.lastIndexOf("\n", at) + 1;+ const newline = text.indexOf("\n", at);+ const end = newline === -1 ? text.length : newline;+ const line = text.slice(start, end);+ const redacted = redact(line);+ if (redacted !== line) {+ output += text.slice(copied, start) + redacted;+ copied = end;+ changed = true;+ }+ if (newline === -1) break;+ at = nextGate(newline + 1);+ }+ return changed ? output + text.slice(copied) : text;+}++function nextAssignmentKeyword(text: string, from: number): number {+ ASSIGNMENT_LINE_GATE.lastIndex = from;+ return ASSIGNMENT_LINE_GATE.exec(text)?.index ?? -1;+}+export type RedactCollectorTextOptions = {/** Text that counts as context for context-dependent rules: the file path, or the JSON key a value sits under. */context?: string;/** Value rule mode; CONFIG when omitted (git diffs, the trace, JSON). See redactModeForPath. */mode?: RedactMode;+ /** The enclosing JSON object or array names an AWS key: the AWS rule runs whatever the line count. */+ awsContext?: boolean;};/**@@ -770,68 +973,259 @@ export type RedactCollectorTextOptions = {*/export function redactCollectorText(input: string, options: RedactCollectorTextOptions = {}): { text: string; count: number } {const tally = { count: 0 };- let text = applyRedaction(input, PRIVATE_KEY_BLOCK, tally);- text = applyRedaction(text, URL_USERINFO, tally);- text = redactAssignments(text, tally, options.mode ?? "config");- text = redactAwsSecrets(text, options.context, tally);- for (const redaction of SECRET_PATTERNS) text = applyRedaction(text, redaction, tally);- text = redactBearerTokens(text, tally);- for (const redaction of PII_PATTERNS) text = applyRedaction(text, redaction, tally);+ let text = input.includes("-----BEGIN ") ? applyRedaction(input, PRIVATE_KEY_BLOCK, tally) : input;+ if (text.includes("://")) text = applyRedaction(text, URL_USERINFO, tally);+ const mode = options.mode ?? "config";+ const assigned = text;+ text = redactGatedLines(assigned, (from) => nextAssignmentKeyword(assigned, from), (line) => redactAssignments(line, tally, mode));+ text = redactAwsSecrets(text, options.context, tally, options.awsContext);+ if (PROVIDER_TOKEN_LITERALS.test(text) || DISCORD_TOKEN_GATE.test(text)) {+ SECRET_PATTERNS.forEach((redaction, index) => {+ const gate = SECRET_PATTERN_GATES[index];+ if (!gate || gate(text)) text = applyRedaction(text, redaction, tally);+ });+ }+ if (BEARER_GATE.test(text)) text = redactBearerTokens(text, tally);+ // The e-mail rule needs an "@" on the line; the phone, SSN and IP rules need a digit.+ const mailed = text;+ text = redactGatedLines(mailed, (from) => mailed.indexOf("@", from), (line) => applyRedaction(line, PII_PATTERNS[0]!, tally));+ if (DIGIT_GATE.test(text)) {+ for (const redaction of PII_PATTERNS.slice(1)) text = applyRedaction(text, redaction, tally);+ }return { text, count: tally.count };}-function redactJsonValue(value: unknown, key: string | undefined, tally: { count: number }): unknown {+/**+ * A JSON object with a repeated key, kept as its (key, value) pairs in order.+ * JSON.parse keeps only the last value of a repeated key, so a secret in a+ * shadowed value (merge-conflict leftovers, generated configs) would pass the+ * structural scrub untouched and, with nothing else to redact, leave the+ * machine byte for byte. The backend's _Pairs.+ */+class JsonPairs {+ constructor(readonly pairs: Array<[string, unknown]>) {}+}++/** A `.json` file nested deeper than this is scrubbed as text instead (the backend's MAX_TRACE_DEPTH). */+const MAX_JSON_FILE_DEPTH = 512;++class JsonTooDeep extends Error {}++/** Redaction count, plus the object members visited: a repeated key shows up as a shortfall against the text. */+type JsonTally = { count: number; members: number };++/** Whether a JSON item is a string naming an AWS key. */+function namesAws(item: unknown): boolean {+ return typeof item === "string" && AWS_SECRET_CONTEXT.test(item);+}++/**+ * Whether an object's keys or direct string values, or an array's string+ * items, name an AWS key. The backend's _names_aws.+ */+function membersNameAws(members: ReadonlyArray<readonly [string, unknown]>): boolean {+ return members.some(([key, item]) => namesAws(key) || namesAws(item));+}++function redactJsonValue(value: unknown, key: string | undefined, tally: JsonTally, depth = 0, awsContext = false, maxDepth = Infinity): unknown {+ if (depth > maxDepth) throw new JsonTooDeep();if (typeof value === "string") {// A JSON string is always a quoted literal, scrubbed in CONFIG mode.- if (key !== undefined && isSecretAssignmentKey(key) && isSecretAssignmentValue(value, "config", true)) {+ // A path-like key (`{"src/admin_auth.py": "<sha256>"}`) names a file, never a secret.+ if (key !== undefined && !isPathLikeKey(key) && isSecretAssignmentKey(key) && isSecretAssignmentValue(value, "config", true)) {tally.count += 1;return REDACTED;}- const result = redactCollectorText(value, { context: key });+ const result = redactCollectorText(value, { context: key, awsContext });tally.count += result.count;return result.text;}if (!value || typeof value !== "object") return value;- if (Array.isArray(value)) return value.map((item) => redactJsonValue(item, key, tally));+ // The enclosing object or array is AWS context once one of its keys or+ // string items names an AWS key, the structural counterpart of the text+ // rule's three-line window: `{"name": "AWS_SECRET_ACCESS_KEY", "value":+ // "<40 characters>"}` (task definitions, k8s env lists, Postman+ // environments) is redacted although `value` names nothing.+ if (Array.isArray(value)) {+ const context = awsContext || value.some(namesAws);+ return value.map((item) => redactJsonValue(item, key, tally, depth + 1, context, maxDepth));+ }+ if (value instanceof JsonPairs) {+ tally.members += value.pairs.length;+ const context = awsContext || membersNameAws(value.pairs);+ return new JsonPairs(value.pairs.map(([childKey, childValue]) => {+ const scrubbedKey = redactCollectorText(childKey, { awsContext: context });+ tally.count += scrubbedKey.count;+ return [scrubbedKey.text, redactJsonValue(childValue, childKey, tally, depth + 1, context, maxDepth)];+ }));+ }const serializable = value as { toJSON?: unknown };- if (typeof serializable.toJSON === "function") return redactJsonValue((serializable.toJSON as () => unknown)(), key, tally);- const output: Record<string, unknown> = {};- for (const [childKey, childValue] of Object.entries(value as Record<string, unknown>)) {- const scrubbedKey = redactCollectorText(childKey);+ if (typeof serializable.toJSON === "function") return redactJsonValue((serializable.toJSON as () => unknown)(), key, tally, depth, awsContext, maxDepth);+ const entries = Object.entries(value as Record<string, unknown>);+ tally.members += entries.length;+ const context = awsContext || membersNameAws(entries);+ // No prototype, so a `__proto__` key stays an ordinary member.+ const output: Record<string, unknown> = Object.create(null);+ for (const [childKey, childValue] of entries) {+ const scrubbedKey = redactCollectorText(childKey, { awsContext: context });tally.count += scrubbedKey.count;- output[scrubbedKey.text] = redactJsonValue(childValue, childKey, tally);+ output[scrubbedKey.text] = redactJsonValue(childValue, childKey, tally, depth + 1, context, maxDepth);}return output;}/*** Scrubs a JSON value structurally: every string (keys included) goes through- * the text scrubber, and a string under a secret-named key is redacted whole,- * exactly as the `key: value` text rule would treat it. Serialising the result- * can never produce a broken escape, which scrubbing serialised JSON as text- * could.+ * the text scrubber, a string under a secret-named key is redacted whole,+ * exactly as the `key: value` text rule would treat it, and an object or array+ * naming an AWS key makes its values AWS context. Serialising the result can+ * never produce a broken escape, which scrubbing serialised JSON as text could.+ * The backend's _redact_json_value.*/export function redactCollectorJson(value: unknown): { value: unknown; count: number } {- const tally = { count: 0 };+ const tally: JsonTally = { count: 0, members: 0 };return { value: redactJsonValue(value, undefined, tally), count: tally.count };}/**- * Scrubs a JSON document as JSON, keeping its indentation style; the original- * text comes back untouched when nothing needed redacting, and null when the- * text is not valid JSON.+ * Object members in valid JSON text: every `:` outside a string separates one.+ * Fewer members in the parsed value than here means a repeated key.+ */+function countJsonMembers(text: string): number {+ let members = 0;+ let inString = false;+ for (let index = 0; index < text.length; index += 1) {+ const code = text.charCodeAt(index);+ if (inString) {+ if (code === 0x5c) index += 1;+ else if (code === 0x22) inString = false;+ } else if (code === 0x22) {+ inString = true;+ } else if (code === 0x3a) {+ members += 1;+ }+ }+ return members;+}++const JSON_STRING_TOKEN = /"[^"\\]*(?:\\.[^"\\]*)*"/y;+const JSON_NUMBER_TOKEN = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;+const JSON_WHITESPACE = /[ \t\n\r]*/y;++/**+ * Parses text JSON.parse has already accepted, keeping every object as its+ * (key, value) pairs so a repeated key loses nothing. Only used for a document+ * that has one: JSON.parse is several times faster.+ */+function parseJsonPairs(text: string): unknown {+ let at = 0;+ const token = (pattern: RegExp): string => {+ pattern.lastIndex = at;+ const match = pattern.exec(text);+ if (!match) throw new SyntaxError(`unexpected JSON at ${at}`);+ at = pattern.lastIndex;+ return match[0];+ };+ const skip = () => token(JSON_WHITESPACE);+ const value = (depth: number): unknown => {+ if (depth > MAX_JSON_FILE_DEPTH) throw new JsonTooDeep();+ skip();+ const char = text[at];+ if (char === "{") {+ at += 1;+ const pairs: Array<[string, unknown]> = [];+ skip();+ if (text[at] === "}") {+ at += 1;+ return new JsonPairs(pairs);+ }+ for (;;) {+ skip();+ const key = JSON.parse(token(JSON_STRING_TOKEN)) as string;+ skip();+ at += 1; // ":"+ pairs.push([key, value(depth + 1)]);+ skip();+ if (text[at++] === "}") return new JsonPairs(pairs);+ }+ }+ if (char === "[") {+ at += 1;+ const items: unknown[] = [];+ skip();+ if (text[at] === "]") {+ at += 1;+ return items;+ }+ for (;;) {+ items.push(value(depth + 1));+ skip();+ if (text[at++] === "]") return items;+ }+ }+ if (char === '"') return JSON.parse(token(JSON_STRING_TOKEN)) as string;+ for (const [literal, parsed] of [["true", true], ["false", false], ["null", null]] as const) {+ if (text.startsWith(literal, at)) {+ at += literal.length;+ return parsed;+ }+ }+ return Number(token(JSON_NUMBER_TOKEN));+ };+ return value(0);+}++/** JSON.stringify(value, null, indent) for a value that may hold JsonPairs. */+function stringifyJsonPairs(value: unknown, indent: string | undefined, current = ""): string {+ const container = (items: string[], open: string, close: string): string => {+ if (items.length === 0) return `${open}${close}`;+ if (!indent) return `${open}${items.join(",")}${close}`;+ const inner = `${current}${indent}`;+ return `${open}\n${inner}${items.join(`,\n${inner}`)}\n${current}${close}`;+ };+ const nested = `${current}${indent ?? ""}`;+ const member = ([key, item]: [string, unknown]) => `${JSON.stringify(key)}:${indent ? " " : ""}${stringifyJsonPairs(item, indent, nested)}`;+ if (value instanceof JsonPairs) return container(value.pairs.map(member), "{", "}");+ if (Array.isArray(value)) return container(value.map((item) => stringifyJsonPairs(item, indent, nested)), "[", "]");+ if (value && typeof value === "object") return container(Object.entries(value).map(member), "{", "}");+ return JSON.stringify(value);+}++/**+ * Scrubs a JSON document as JSON, keeping its indentation style. The AWS rule+ * runs over the raw text first, where it sees the same lines it sees in a text+ * file; the document is then scrubbed value by value (repeated keys kept, pair+ * by pair). The text comes back untouched when nothing needed redacting (the+ * AWS pass aside), and null when it is not valid JSON or is nested deeper than+ * MAX_JSON_FILE_DEPTH, which the caller scrubs as text. The backend's+ * sanitize_json_file.*/-export function redactCollectorJsonText(text: string): string | null {+export function redactCollectorJsonText(text: string, path?: string): string | null {+ const raw = redactAwsSecrets(text, path, { count: 0 });let parsed: unknown;try {- parsed = JSON.parse(text);+ parsed = JSON.parse(raw);} catch {return null;}- const result = redactCollectorJson(parsed);- if (result.count === 0) return text;- const indent = /\n([ \t]+)\S/.exec(text)?.[1]?.slice(0, 10);- return `${JSON.stringify(result.value, null, indent)}${text.endsWith("\n") ? "\n" : ""}`;+ const tally: JsonTally = { count: 0, members: 0 };+ let value: unknown;+ let pairs = false;+ try {+ value = redactJsonValue(parsed, undefined, tally, 0, false, MAX_JSON_FILE_DEPTH);+ if (tally.members !== countJsonMembers(raw)) {+ pairs = true;+ tally.count = 0;+ value = redactJsonValue(parseJsonPairs(raw), undefined, tally, 0, false, MAX_JSON_FILE_DEPTH);+ }+ } catch (error) {+ if (error instanceof JsonTooDeep) return null;+ throw error;+ }+ if (tally.count === 0) return raw;+ const indent = /\n([ \t]+)\S/.exec(raw)?.[1]?.slice(0, 10);+ const serialized = pairs ? stringifyJsonPairs(value, indent) : JSON.stringify(value, null, indent);+ return `${serialized}${raw.endsWith("\n") ? "\n" : ""}`;}/**@@ -840,11 +1234,16 @@ export function redactCollectorJsonText(text: string): string | null {* as context and the value rule mode the extension selects.*/export function redactCollectorContent(path: string, text: string): string {+ return redactCollectorContentCounted(path, text).text;+}++/** redactCollectorContent plus how many redactions it made (0 means the text is byte-identical). */+function redactCollectorContentCounted(path: string, text: string): { text: string; count: number } {if (/\.json$/i.test(path)) {- const json = redactCollectorJsonText(text);- if (json !== null) return json;+ const json = redactCollectorJsonText(text, path);+ if (json !== null) return { text: json, count: json === text ? 0 : 1 };}- return redactCollectorText(text, { context: path, mode: redactModeForPath(path) }).text;+ return redactCollectorText(text, { context: path, mode: redactModeForPath(path) });}/**@@ -863,13 +1262,15 @@ export function stripRemoteUserinfo(url: string): string {}function isBinary(buffer: Buffer): boolean {- const sample = buffer.subarray(0, Math.min(buffer.length, 8_192));- if (sample.includes(0)) return true;+ const length = Math.min(buffer.length, 8_192);let suspicious = 0;- for (const byte of sample) {+ // An indexed loop: iterating a Buffer is several times slower, once per file.+ for (let index = 0; index < length; index += 1) {+ const byte = buffer[index]!;+ if (byte === 0) return true;if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;}- return sample.length > 0 && suspicious / sample.length > 0.1;+ return length > 0 && suspicious / length > 0.1;}const PATH_FIELD_NAMES = new Set([@@ -914,6 +1315,16 @@ async function readGitignoreFile(directory: string): Promise<string[]> {}}+/** A directory's .gitignore rules re-rooted at the workspace, as walkFallback applies them. */+function scopedIgnoreRules(prefix: string, localRules: string[]): string[] {+ return localRules.map((rule) => {+ const negated = rule.startsWith("!");+ const body = negated ? rule.slice(1) : rule;+ const scoped = prefix ? `${prefix}/${body}` : body;+ return negated ? `!${scoped}` : scoped;+ });+}+function ignoredByRules(path: string, rules: string[]): boolean {let ignored = false;for (const raw of rules) {@@ -925,18 +1336,128 @@ function ignoredByRules(path: string, rules: string[]): boolean {return ignored;}-async function gitIgnoresPath(root: string, path: string): Promise<boolean> {- try {- await execFileAsync("git", ["-C", root, "check-ignore", "--no-index", "-q", "--", path], {- timeout: 5_000,- maxBuffer: 64 * 1024,+/**+ * Asks git which of `paths` its ignore rules cover, in one process: the+ * ignored paths come back NUL-separated. Null when git cannot answer (not a+ * repository, missing, timed out): the caller applies the .gitignore files+ * itself, exactly as the listing walker does.+ */+function gitIgnoredPaths(root: string, paths: string[]): Promise<Set<string> | null> {+ return new Promise((resolvePromise) => {+ const ignored = new Set<string>();+ if (paths.length === 0) {+ resolvePromise(ignored);+ return;+ }+ let child: ReturnType<typeof spawn>;+ try {+ child = spawn("git", ["-C", root, "check-ignore", "--no-index", "-z", "--stdin"], {+ stdio: ["pipe", "pipe", "ignore"],+ windowsHide: true,+ env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" },+ });+ } catch {+ resolvePromise(null);+ return;+ }+ const chunks: Buffer[] = [];+ let total = 0;+ let settled = false;+ // Exit 0: some paths are ignored; 1: none are; anything else: no answer.+ const finish = (answered: boolean) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ if (!answered) {+ resolvePromise(null);+ return;+ }+ for (const path of Buffer.concat(chunks).toString("utf8").split("\0")) {+ if (path) ignored.add(path);+ }+ resolvePromise(ignored);+ };+ const timer = setTimeout(() => child.kill("SIGKILL"), 10_000);+ timer.unref?.();+ child.stdout?.on("data", (chunk: Buffer) => {+ if (total >= 16 * 1024 * 1024) return;+ chunks.push(chunk);+ total += chunk.length;});- return true;- } catch {- // Exit status 1 means the path is not ignored. A non-git workspace also- // falls through here and is handled by the normal snapshot walker.- return false;+ child.on("error", () => finish(false));+ child.on("close", (code) => finish(code === 0 || code === 1));+ child.stdin?.on("error", () => undefined);+ child.stdin?.end(`${paths.join("\0")}\0`);+ });+}++/**+ * gitIgnoredPaths for a batch in which git may refuse single paths: one path+ * beyond a symbolic link (`linked/util.ts`) makes the whole batch exit 128. A+ * refused batch is split in halves until every refusal is a single path, so+ * one bad path never changes the answer for the others (and never sends them+ * to the .gitignore-only fallback, which knows nothing of `.git/info/exclude`+ * or `core.excludesFile`). A path git refuses on its own counts as ignored:+ * it is never collected. Null only when the workspace is not a git work tree+ * (or git is missing), where the caller applies the .gitignore files itself.+ * `onSpawn` counts each `git check-ignore` run.+ */+async function gitIgnoredPathsIsolated(root: string, paths: string[], onSpawn: () => void): Promise<Set<string> | null> {+ onSpawn();+ const whole = await gitIgnoredPaths(root, paths);+ if (whole) return whole;+ if ((await gitText(root, ["rev-parse", "--is-inside-work-tree"])) !== "true") return null;+ const ignored = new Set<string>();+ const isolate = async (refused: string[]): Promise<void> => {+ if (refused.length === 1) {+ ignored.add(refused[0]!);+ return;+ }+ const middle = refused.length >> 1;+ for (const half of [refused.slice(0, middle), refused.slice(middle)]) {+ onSpawn();+ const answer = await gitIgnoredPaths(root, half);+ if (!answer) await isolate(half);+ else for (const path of answer) ignored.add(path);+ }+ };+ await isolate(paths);+ return ignored;+}++/**+ * The listing walker's ignore decision for individual paths, for a workspace+ * git cannot answer for: a path is ignored when it, or any directory above+ * it, matches the .gitignore rules in force at that level.+ */+async function fallbackIgnoredPaths(root: string, paths: string[]): Promise<Set<string>> {+ const rulesByDirectory = new Map<string, Promise<string[]>>();+ const rulesAt = (directory: string): Promise<string[]> => {+ let rules = rulesByDirectory.get(directory);+ if (!rules) {+ const slash = directory.lastIndexOf("/");+ const parent = slash === -1 ? "" : directory.slice(0, slash);+ rules = (async () => [+ ...(directory === "" ? [] : await rulesAt(parent)),+ ...scopedIgnoreRules(directory, await readGitignoreFile(resolve(root, directory))),+ ])();+ rulesByDirectory.set(directory, rules);+ }+ return rules;+ };+ const ignored = new Set<string>();+ for (const path of paths) {+ let directory = "";+ for (const part of pathComponents(path)) {+ const current = directory ? `${directory}/${part}` : part;+ if (ignoredByRules(current, await rulesAt(directory))) {+ ignored.add(path);+ break;+ }+ directory = current;+ }}+ return ignored;}function optionalString(value: unknown): string | undefined {@@ -1016,15 +1537,17 @@ function boundedTracePayload(): Buffer {// The trace is scrubbed as a JSON value, never as serialised text: a text// redaction inside an escaped string (`\[email protected]`) once left a- // dangling backslash that made the whole document unparseable.- const header = redactCollectorJson({+ // dangling backslash that made the whole document unparseable. Header and+ // events are scrubbed as the one document the backend scrubs, so a header+ // value naming an AWS key is container context for the events there too.+ const { events: scrubbed, ...header } = redactCollectorJson({schema_version: 1,session_id: state.id,workspace_id: state.workspaceId,session_segment: state.segment,session_resumed: state.resumed,- }).value as Record<string, unknown>;- const scrubbed = redactCollectorJson(events).value as TraceEvent[];+ events,+ }).value as Record<string, unknown> & { events: TraceEvent[] };const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => Buffer.from(JSON.stringify({...header,trace_truncated: truncated,@@ -1033,17 +1556,26 @@ function boundedTracePayload(}));// One encode settles the common case. Only an oversized trace pays for the- // search, which bisects on the prefix length: every probe is a full- // serialisation, so probing once per event stalled the event loop for- // seconds at the MAX_COLLECTOR_TRACE_EVENTS cap.+ // search, and that search measures each event once: the payload of a prefix+ // is its wrapper plus the events' serialised bytes and the commas between+ // them, so the longest prefix that fits follows from a running total. Every+ // probe used to be a full serialisation of the prefix, which made a flush+ // at the MAX_COLLECTOR_TRACE_EVENTS cap peak at many times the payload.let payload = encode(scrubbed, false);if (payload.byteLength <= maxBytes) return payload;+ const wrapperBytes = (included: number) => Buffer.byteLength(JSON.stringify({+ ...header,+ trace_truncated: true,+ dropped_event_count: Math.max(0, events.length - included),+ events: [],+ }));let fits = 0;- let overflow = scrubbed.length;- while (overflow - fits > 1) {- const middle = Math.floor((fits + overflow) / 2);- if (encode(scrubbed.slice(0, middle), true).byteLength <= maxBytes) fits = middle;- else overflow = middle;+ let eventBytes = 0;+ for (let index = 0; index < scrubbed.length; index += 1) {+ const next = eventBytes + Buffer.byteLength(JSON.stringify(scrubbed[index])) + (index > 0 ? 1 : 0);+ if (wrapperBytes(index + 1) + next > maxBytes) break;+ eventBytes = next;+ fits = index + 1;}const selected = scrubbed.slice(0, fits);payload = encode(selected, true);@@ -1067,14 +1599,7 @@ function boundedTracePayload(type WorkspaceListing = { paths: string[]; denied: number };async function walkFallback(root: string, directory = root, rules: string[] = [], listing: WorkspaceListing = { paths: [], denied: 0 }): Promise<WorkspaceListing> {- const localRules = await readGitignoreFile(directory);- const prefix = portablePath(root, directory);- const ignoreRules = [...rules, ...localRules.map((rule) => {- const negated = rule.startsWith("!");- const body = negated ? rule.slice(1) : rule;- const scoped = prefix ? `${prefix}/${body}` : body;- return negated ? `!${scoped}` : scoped;- })];+ const ignoreRules = [...rules, ...scopedIgnoreRules(portablePath(root, directory), await readGitignoreFile(directory))];let entries;try {entries = await readdir(directory, { withFileTypes: true });@@ -1449,219 +1974,645 @@ async function sha256File(absolute: string): Promise<string> {});}-async function workspaceSignature(root: string): Promise<string> {- const hash = createHash("sha256");- for (const path of (await listWorkspaceFiles(root)).paths) {- try {- const file = await lstat(resolve(root, path));- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) continue;- hash.update(path).update("\0").update(String(file.size)).update("\0").update(String(file.mtimeMs)).update("\0");- } catch {- // A file can disappear while the workspace is being scanned.- }+// --- scanning ----------------------------------------------------------------++/** Lets the event loop run once a stretch of synchronous scan work has gone on long enough. */+class LoopYielder {+ private last = performance.now();++ async pause(): Promise<void> {+ if (performance.now() - this.last < EVENT_LOOP_YIELD_MS) return;+ await new Promise<void>((resolvePromise) => setImmediate(resolvePromise));+ this.last = performance.now();}- // A commit changes the repository without touching any eligible file; fold- // HEAD in so the next prompt or turn still captures the new history.- hash.update("HEAD\0").update((await gitHead(root)) ?? "").update("\0");- return hash.digest("hex");}-type WorkspaceScan = {- manifest: ManifestEntry[];- files: CollectorFile[];- manifestTruncated: boolean;- contentTruncated: boolean;- /** Non-ignored paths the denylist kept out of the snapshot. */- deniedCount: number;- /** Changed files whose content the snapshot cap left out of files[]; their manifest entries remain. */- capOmittedCount: number;- /** Redacted bytes of those files. */- capOmittedBytes: number;-};+/**+ * Reads a whole file into a pooled buffer, so a scan recycles a few buffers+ * instead of allocating and freeing one per file (which leaves the allocator+ * holding hundreds of megabytes of freed pages). A pool lives for one scan or+ * upload pass and is garbage afterwards, so nothing stays allocated between+ * captures. The view handed to `use` is valid until the same slot is used+ * again, so nothing of it may outlive the call except through `toString` or+ * hashing. Null when the file is larger than the cap.+ */+class ReadPool {+ private readonly free: Buffer[] = [];+ private readonly waiters: Array<(buffer: Buffer) => void> = [];+ private allocated = 0;+ /** One buffer at the file cap for the rare file larger than a slot; such reads take turns. */+ private large: Buffer | null = null;+ private largeTail: Promise<unknown> = Promise.resolve();++ constructor(private readonly slots: number, private readonly capacity = READ_SLOT_BYTES) {}++ private acquire(): Promise<Buffer> {+ const buffer = this.free.pop();+ if (buffer) return Promise.resolve(buffer);+ if (this.allocated < this.slots) {+ this.allocated += 1;+ return Promise.resolve(Buffer.allocUnsafeSlow(this.capacity));+ }+ return new Promise((resolvePromise) => this.waiters.push(resolvePromise));+ }-/** A changed file competing for the snapshot's content budget. */-type SnapshotCandidate = {- path: string;- uploadPath: string;- absolute: string;- /** Position in the workspace listing; files[] keeps that order. */- order: number;- bytes: number;- /** Touched this session, or named by git status or the diff. */- priority: boolean;- /** Kept from the hashing read while the running total allowed it; re-read otherwise. */- content?: string;-};+ private release(buffer: Buffer): void {+ const waiter = this.waiters.shift();+ if (waiter) waiter(buffer);+ else this.free.push(buffer);+ }-function serializedBytes(value: unknown): number {- return Buffer.byteLength(JSON.stringify(value));+ /** Runs `use` with the file's bytes; the view must not escape `use`. */+ async read<T>(absolute: string, expectedSize: number, use: (bytes: Buffer) => Promise<T> | T): Promise<T | null> {+ if (expectedSize > MAX_COLLECTOR_FILE_BYTES) return null;+ if (expectedSize < this.capacity) {+ const buffer = await this.acquire();+ try {+ const length = await readInto(absolute, buffer);+ if (length < buffer.length) return await use(buffer.subarray(0, length));+ } finally {+ this.release(buffer);+ }+ // The file grew past its slot since the stat: read it with the large buffer.+ }+ return this.readLarge(absolute, use);+ }++ private readLarge<T>(absolute: string, use: (bytes: Buffer) => Promise<T> | T): Promise<T | null> {+ const run = this.largeTail.then(async () => {+ this.large ??= Buffer.allocUnsafeSlow(MAX_COLLECTOR_FILE_BYTES + 1);+ const length = await readInto(absolute, this.large);+ // Past the cap by now: not a file the collector takes.+ return length > MAX_COLLECTOR_FILE_BYTES ? null : await use(this.large.subarray(0, length));+ });+ this.largeTail = run.catch(() => undefined);+ return run;+ }}-/**- * Paths whose content a capped snapshot keeps first: what the session touched- * plus what git reports as modified, keyed the way files[] names them.- */-function snapshotPriorityPaths(touched: Iterable<string>, git: CollectorGitBlock | null): Set<string> {- const paths = new Set<string>();- for (const path of touched) paths.add(collectorPathForUpload(path));- for (const entry of git?.status ?? []) paths.add(entry.path);- for (const line of git?.diff?.split("\n") ?? []) {- const path = diffHeaderPath(line);- if (path) paths.add(collectorPathForUpload(path));+/** Reads a file from the start into `buffer`, up to its length; resolves with the bytes read. */+async function readInto(absolute: string, buffer: Buffer): Promise<number> {+ const handle = await open(absolute, "r");+ let length = 0;+ try {+ while (length < buffer.length) {+ const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length);+ if (bytesRead === 0) break;+ length += bytesRead;+ }+ } finally {+ await handle.close();}- return paths;+ return length;}/**- * Picks which changed files carry content, smallest first within each- * priority tier, and materialises them in listing order. Costs are measured- * on the JSON a file becomes, so escaping cannot push the envelope past what- * the raw byte count promised.+ * Runs `operation` over `items` with at most `limit` calls in flight and+ * returns the results in item order, so a 50,000-file scan holds a handful of+ * descriptors instead of a burst of them.*/-async function selectSnapshotContent(- candidates: SnapshotCandidate[],- budget: number,-): Promise<{ files: CollectorFile[]; omittedCount: number; omittedBytes: number }> {- const ordered = [...candidates].sort((a, b) => Number(b.priority) - Number(a.priority) || a.bytes - b.bytes || a.order - b.order);- const kept: Array<{ order: number; file: CollectorFile }> = [];- let used = 0;- let omittedCount = 0;- let omittedBytes = 0;- for (const candidate of ordered) {- const overhead = SNAPSHOT_FILE_ENTRY_OVERHEAD_BYTES + Buffer.byteLength(candidate.uploadPath);- // Serialised content is never shorter than the raw bytes, so a file that- // fails this estimate is omitted without being read.- if (used + overhead + candidate.bytes <= budget) {- let content = candidate.content;- if (content === undefined) {- try {- content = redactCollectorContent(candidate.path, (await readFile(candidate.absolute)).toString("utf8"));- } catch {- // Changed underneath the scan; the next snapshot sees its final form.- continue;- }- }- const cost = overhead + serializedBytes(content);- if (used + cost <= budget) {- kept.push({ order: candidate.order, file: { path: candidate.uploadPath, content, sha256: sha256Hex(content) } });- used += cost;- continue;- }+export async function mapBounded<T, R>(items: readonly T[], limit: number, operation: (item: T, index: number) => Promise<R>): Promise<R[]> {+ const results: R[] = new Array(items.length);+ let next = 0;+ const worker = async (): Promise<void> => {+ while (next < items.length) {+ const index = next;+ next += 1;+ results[index] = await operation(items[index]!, index);}- omittedCount += 1;- omittedBytes += candidate.bytes;- }- kept.sort((a, b) => a.order - b.order);- return { files: kept.map((item) => item.file), omittedCount, omittedBytes };+ };+ await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));+ return results;+}++/** Work counters for tests and profiling; never uploaded. */+export type CollectorMetrics = {+ fileStats: number;+ fileReads: number;+ fileRedactions: number;+ fullScans: number;+ dirtyScans: number;+ reconciles: number;+ capturesSkipped: number;+ capturesDeferred: number;+ ignoreCheckSpawns: number;+ watchEvents: number;+ envelopesWritten: number;+};++function freshMetrics(): CollectorMetrics {+ return {+ fileStats: 0, fileReads: 0, fileRedactions: 0, fullScans: 0, dirtyScans: 0, reconciles: 0,+ capturesSkipped: 0, capturesDeferred: 0, ignoreCheckSpawns: 0, watchEvents: 0, envelopesWritten: 0,+ };+}++/** Redacts one file's bytes and describes the result for the cache and the manifest. */+function redactedCacheEntry(+ path: string,+ buffer: Buffer,+ size: number,+ mtimeMs: number,+ uploadPath: string,+ metrics: CollectorMetrics,+): { entry: HashCacheEntry; text: string } {+ const redacted = redactCollectorContentCounted(path, buffer.toString("utf8"));+ metrics.fileRedactions += 1;+ return {+ text: redacted.text,+ entry: {+ size,+ mtimeMs,+ binary: false,+ sha256: sha256Hex(redacted.text),+ bytes: Buffer.byteLength(redacted.text),+ json: Buffer.byteLength(JSON.stringify(redacted.text)),+ clean: redacted.count === 0,+ uploadPath,+ },+ };+}++/** The redacted upload path, sharing the listing's string when redaction left it unchanged. */+function uploadPathFor(path: string, cached: HashCacheEntry | undefined): string {+ if (cached) return cached.uploadPath;+ const uploadPath = collectorPathForUpload(path);+ return uploadPath === path ? path : uploadPath;}+/** Per-pass answers of hasRealAncestors, by workspace-relative directory. */+type AncestorCache = Map<string, Promise<boolean>>;+/**- * Walks every eligible workspace file, hashes its uploadable (redacted) form- * for the manifest, and gathers content for either the whole tree (start- * snapshots) or only the files whose hash differs from the previous manifest.- * Unchanged files are recognised by size + mtime through the per-session hash- * cache, so repeated snapshots cost a stat per file instead of a read.- *- * `byteLimit` bounds the serialised manifest plus files[] together. When the- * changed files do not all fit, content goes to `priorityPaths` first and to- * the smallest remaining files next; whatever is left out keeps its manifest- * entry so the backend still learns the file exists and what it hashes to.+ * Whether no directory between the root and `path` is a symlink (or a Windows+ * junction, which lstat reports as one). A path reached through one+ * (`linked/util.ts` with `linked -> ../shared-lib`) names a file outside the+ * workspace, which git's listing never holds and nothing may upload; checking+ * only the last component, or the resolved path as a string, lets it through.+ * A missing ancestor passes: the file cannot exist either, which the caller's+ * own lstat reports. `seen` shares the answer per directory for one pass.*/-async function scanWorkspace(- root: string,- cache: Map<string, HashCacheEntry>,- baseline: Map<string, string> | null,- byteLimit: number,- includeAll: boolean,- priorityPaths: ReadonlySet<string> = new Set(),-): Promise<WorkspaceScan> {- const manifest: ManifestEntry[] = [];- const candidates: SnapshotCandidate[] = [];- const seen = new Set<string>();- let retained = 0;- const listing = await listWorkspaceFiles(root);- const paths = listing.paths;- for (const path of paths) {- if (manifest.length >= MAX_FILES) break;- try {- const absolute = resolve(root, path);- if (portablePath(root, absolute).startsWith("../")) continue;- const file = await lstat(absolute);- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) continue;- seen.add(path);- const cached = cache.get(path);- let entry: HashCacheEntry | undefined = cached && cached.size === file.size && cached.mtimeMs === file.mtimeMs ? cached : undefined;- let redacted: string | undefined;- if (!entry) {- const buffer = await readFile(absolute);- if (buffer.length > MAX_COLLECTOR_FILE_BYTES || isBinary(buffer)) {- cache.set(path, { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0 });- continue;- }- redacted = redactCollectorContent(path, buffer.toString("utf8"));- entry = { size: file.size, mtimeMs: file.mtimeMs, binary: false, sha256: sha256Hex(redacted), bytes: Buffer.byteLength(redacted) };- cache.set(path, entry);- }- if (entry.binary) continue;- manifest.push({ path, sha256: entry.sha256, size: entry.bytes });- const changed = includeAll || baseline?.get(path) !== entry.sha256;- if (!changed) continue;- // Content already in hand stays in memory while it could still fit, so a- // snapshot under the budget never reads a file twice; beyond that point- // it is dropped and re-read only if the selection picks the file.- const keep = retained + entry.bytes <= byteLimit;- if (keep) retained += entry.bytes;- const uploadPath = collectorPathForUpload(path);- candidates.push({- path,- uploadPath,- absolute,- order: candidates.length,- bytes: entry.bytes,- priority: priorityPaths.has(uploadPath),- ...(keep && redacted !== undefined ? { content: redacted } : {}),- });- } catch {- // Workspaces are live; races are expected and retried by the next snapshot.+async function hasRealAncestors(root: string, path: string, seen: AncestorCache = new Map()): Promise<boolean> {+ const parts = pathComponents(path);+ let directory = "";+ for (let index = 0; index < parts.length - 1; index += 1) {+ directory = directory ? `${directory}/${parts[index]}` : parts[index]!;+ let real = seen.get(directory);+ if (!real) {+ real = lstat(resolve(root, directory)).then((entry) => !entry.isSymbolicLink(), () => true);+ seen.set(directory, real);}+ if (!(await real)) return false;}- for (const key of cache.keys()) {- if (!seen.has(key)) cache.delete(key);- }- const selected = await selectSnapshotContent(candidates, Math.max(0, byteLimit - serializedBytes(manifest)));+ return true;+}++/** What one scan pass shares across the files it inspects. */+type ScanContext = {+ root: string;+ /** The resolved root with a trailing separator: every file inspected must start with it. */+ rootPrefix: string;+ cache: Map<string, HashCacheEntry>;+ metrics: CollectorMetrics;+ yielder: LoopYielder;+ pool: ReadPool;+ ancestors: AncestorCache;+};++function scanContext(root: string, cache: Map<string, HashCacheEntry>, metrics: CollectorMetrics): ScanContext {+ const base = resolve(root);return {- manifest,- files: selected.files,- manifestTruncated: paths.length >= MAX_FILES,- contentTruncated: selected.omittedCount > 0,- deniedCount: listing.denied,- capOmittedCount: selected.omittedCount,- capOmittedBytes: selected.omittedBytes,+ root,+ rootPrefix: base.endsWith(sep) ? base : `${base}${sep}`,+ cache,+ metrics,+ yielder: new LoopYielder(),+ pool: new ReadPool(SCAN_CONCURRENCY),+ ancestors: new Map(),};}/**- * Whether an unauthorized response is the sign-in gate (the broker's or the- * gateway's `omnirush_account_required`) rather than a stale device token.- * Reading the body also releases the connection.+ * Stats one eligible path and, unless the per-root cache already knows the+ * file at this size and mtime, reads, redacts and hashes it. Null for anything+ * that is not a regular file within the size cap (gone, a directory, a+ * symlink, too large) or that resolves outside the root, lexically or through+ * a symlinked directory; a binary file comes back as its binary entry. Content+ * is never kept: the upload pass re-reads what it sends.*/-async function responseSignalsAccountRequired(response: Response): Promise<boolean> {+async function inspectFile(context: ScanContext, path: string): Promise<HashCacheEntry | null> {+ const { cache, metrics, pool } = context;+ const absolute = resolve(context.root, path);+ if (!absolute.startsWith(context.rootPrefix) || !(await hasRealAncestors(context.root, path, context.ancestors))) return null;+ let file;try {- return (await response.text()).includes(ACCOUNT_REQUIRED_MARKER);+ file = await lstat(absolute);} catch {- return false;+ return null;}+ metrics.fileStats += 1;+ if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) return null;+ const cached = cache.get(path);+ if (cached && cached.size === file.size && cached.mtimeMs === file.mtimeMs) return cached;+ const uploadPath = uploadPathFor(path, cached);+ let entry: HashCacheEntry | null = null;+ try {+ entry = await pool.read(absolute, file.size, (buffer) => {+ metrics.fileReads += 1;+ if (isBinary(buffer)) {+ return { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };+ }+ return redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPath, metrics).entry;+ });+ } catch {+ return null;+ }+ if (entry) cache.set(path, entry);+ await context.yielder.pause();+ return entry;}-function compressZstd(payload: Buffer): Promise<Buffer> {- return new Promise((resolvePromise, reject) => {- zstdCompress(payload, (error, result) => {- if (error) reject(error);- else resolvePromise(result);- });+/** One file as files[] carries it, re-read at upload time. */+type UploadContent = { content: string; entry: HashCacheEntry };++/**+ * Reads a file for files[]. A file the scan saw unchanged by redaction is sent+ * as read once its raw bytes hash to the cached digest, which skips the regex+ * pipeline; anything else (changed since the scan, or a file the scrubber+ * touched) is redacted again, and the cache learns the result when it differs.+ */+async function readUploadContent(+ root: string,+ path: string,+ cache: Map<string, HashCacheEntry>,+ metrics: CollectorMetrics,+ pool: ReadPool,+ ancestors: AncestorCache,+): Promise<UploadContent | null> {+ const absolute = resolve(root, path);+ // A directory swapped for a symlink since the scan must not be followed either.+ if (!(await hasRealAncestors(root, path, ancestors))) return null;+ let file;+ try {+ file = await lstat(absolute);+ } catch {+ return null;+ }+ if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) return null;+ const cached = cache.get(path);+ try {+ return await pool.read(absolute, file.size, (buffer): UploadContent | null => {+ metrics.fileReads += 1;+ if (cached && !cached.binary && cached.clean && cached.size === file.size && cached.mtimeMs === file.mtimeMs+ && cached.bytes === buffer.length && sha256Hex(buffer) === cached.sha256) {+ return { content: buffer.toString("utf8"), entry: cached };+ }+ if (isBinary(buffer)) return null;+ const { entry, text } = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, cached), metrics);+ // The same digest keeps the cached entry, which the manifest may share.+ if (cached && cached.sha256 === entry.sha256 && cached.size === entry.size && cached.mtimeMs === entry.mtimeMs) {+ return { content: text, entry: cached };+ }+ cache.set(path, entry);+ return { content: text, entry };+ });+ } catch {+ return null;+ }+}++/**+ * Eligible text files by workspace path, in listing order. Entries are the+ * per-root cache's own (immutable) objects: path (redacted) as `uploadPath`,+ * redacted digest as `sha256`, redacted size as `bytes`.+ */+type Manifest = Map<string, HashCacheEntry>;++type ScanResult = {+ /** Every eligible file: the manifest, and the next baseline once accepted. */+ manifest: Manifest;+ /** Paths whose digest differs from the previous manifest; null when every manifest entry counts (a start snapshot). */+ changed: Set<string> | null;+ /** Previous manifest entries no longer present. */+ removed: number;+ deniedCount: number;+ manifestTruncated: boolean;+};++/** A changed file competing for the snapshot's content budget. */+type SnapshotCandidate = {+ path: string;+ uploadPath: string;+ /** Redacted bytes: the sort key, and what a cap omission reports. */+ bytes: number;+ /** Serialised files[] cost: overhead, path and JSON-encoded content. */+ cost: number;+ /** Touched this session, or named by git status or the diff. */+ priority: boolean;+};++/** The manifest entry as the envelope carries it: path (redacted), sha256, size. */+function manifestEntryJson(entry: HashCacheEntry): string {+ return `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.bytes}}`;+}++function candidateCost(entry: HashCacheEntry): number {+ return SNAPSHOT_FILE_ENTRY_OVERHEAD_BYTES + Buffer.byteLength(entry.uploadPath) + entry.json;+}++/** The changed entries of a manifest as candidates, in manifest order. */+function candidatesFor(scan: ScanResult, priorityPaths: ReadonlySet<string>): SnapshotCandidate[] {+ const candidates: SnapshotCandidate[] = [];+ for (const [path, entry] of scan.manifest) {+ if (scan.changed && !scan.changed.has(path)) continue;+ candidates.push({ path, uploadPath: entry.uploadPath, bytes: entry.bytes, cost: candidateCost(entry), priority: priorityPaths.has(entry.uploadPath) });+ }+ return candidates;+}++function countRemoved(previous: Manifest | null, manifest: Manifest): number {+ let removed = 0;+ for (const path of previous?.keys() ?? []) {+ if (!manifest.has(path)) removed += 1;+ }+ return removed;+}++/**+ * Lists and inspects every eligible file. Unchanged files are recognised by+ * size + mtime through the per-root cache, so a rescan of a settled tree costs+ * a stat per file and no reads. The cache is rebuilt from what the listing+ * still names, so entries for removed paths go with it. `onListing` sees the+ * listing before any file is read.+ */+async function scanWorkspaceFull(+ root: string,+ cache: Map<string, HashCacheEntry>,+ previous: Manifest | null,+ includeAll: boolean,+ metrics: CollectorMetrics,+ onListing?: (paths: readonly string[]) => void,+): Promise<ScanResult> {+ metrics.fullScans += 1;+ const listing = await listWorkspaceFiles(root);+ onListing?.(listing.paths);+ const context = scanContext(root, cache, metrics);+ const entries = await mapBounded(listing.paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));+ cache.clear();+ const manifest: Manifest = new Map();+ const changed = includeAll ? null : new Set<string>();+ listing.paths.forEach((path, index) => {+ const entry = entries[index];+ if (!entry) return;+ cache.set(path, entry);+ if (entry.binary || manifest.size >= MAX_FILES) return;+ manifest.set(path, entry);+ if (changed && previous?.get(path)?.sha256 !== entry.sha256) changed.add(path);});+ return {+ manifest,+ changed,+ removed: countRemoved(previous, manifest),+ deniedCount: listing.denied,+ manifestTruncated: listing.paths.length >= MAX_FILES,+ };+}++/**+ * Inspects only the paths the watcher reported since the last capture and+ * folds them into a copy of the previous manifest: no listing, no stat of the+ * rest of the tree. `ignored` is git's current answer for every reported path+ * (the caller asks in one batch before anything is read): an ignored path is+ * never read, and leaves the manifest if it was there, since a file the user+ * has just added to `.gitignore` or `.git/info/exclude` may not stay listed. A+ * reported path that is no longer a file may have been a directory moved or+ * deleted as a whole (one event, no events for its files): the manifest+ * entries under it are inspected too.+ */+async function scanDirtyPaths(+ root: string,+ cache: Map<string, HashCacheEntry>,+ previous: Manifest,+ dirty: readonly string[],+ ignored: ReadonlySet<string>,+ listing: { denied: number; truncated: boolean },+ metrics: CollectorMetrics,+): Promise<ScanResult> {+ metrics.dirtyScans += 1;+ const context = scanContext(root, cache, metrics);+ const targets = dirty.filter((path) => !ignored.has(path));+ const inspect = (paths: string[]) => mapBounded(paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));+ const entries = await inspect(targets);+ const manifest: Manifest = new Map(previous);+ for (const path of dirty) {+ if (!ignored.has(path)) continue;+ manifest.delete(path);+ cache.delete(path);+ }+ const changed = new Set<string>();+ const vanished = new Set<string>();+ const fold = (path: string, entry: HashCacheEntry | null) => {+ if (!entry || entry.binary) {+ if (!manifest.delete(path) && !entry) vanished.add(path);+ return;+ }+ if (!manifest.has(path) && manifest.size >= MAX_FILES) return;+ manifest.set(path, entry);+ if (previous.get(path)?.sha256 !== entry.sha256) changed.add(path);+ };+ targets.forEach((path, index) => fold(path, entries[index] ?? null));+ if (vanished.size > 0) {+ const reported = new Set(dirty);+ const inside: string[] = [];+ for (const path of manifest.keys()) {+ if (reported.has(path)) continue;+ for (let slash = path.indexOf("/"); slash !== -1; slash = path.indexOf("/", slash + 1)) {+ if (vanished.has(path.slice(0, slash))) {+ inside.push(path);+ break;+ }+ }+ }+ const insideEntries = await inspect(inside);+ inside.forEach((path, index) => fold(path, insideEntries[index] ?? null));+ }+ return {+ manifest,+ changed,+ removed: countRemoved(previous, manifest),+ deniedCount: listing.denied,+ manifestTruncated: listing.truncated || manifest.size >= MAX_FILES,+ };+}++/**+ * Paths whose content a capped snapshot keeps first: what the session touched+ * plus what git reports as modified, keyed the way files[] names them.+ */+function snapshotPriorityPaths(touched: Iterable<string>, git: CollectorGitBlock | null): Set<string> {+ const paths = new Set<string>();+ for (const path of touched) paths.add(collectorPathForUpload(path));+ for (const entry of git?.status ?? []) paths.add(entry.path);+ for (const line of git?.diff?.split("\n") ?? []) {+ const path = diffHeaderPath(line);+ if (path) paths.add(collectorPathForUpload(path));+ }+ return paths;+}++/**+ * Picks which changed files carry content, smallest first within each+ * priority tier, against a budget measured on the JSON a file becomes, so+ * escaping cannot push the envelope past what the raw byte count promised.+ */+function selectSnapshotContent(candidates: SnapshotCandidate[], budget: number): { selected: Set<string>; omittedCount: number; omittedBytes: number } {+ // A stable sort: equal candidates keep manifest order.+ const ordered = [...candidates].sort((a, b) => Number(b.priority) - Number(a.priority) || a.bytes - b.bytes);+ const selected = new Set<string>();+ let used = 0;+ let omittedCount = 0;+ let omittedBytes = 0;+ for (const candidate of ordered) {+ if (used + candidate.cost <= budget) {+ selected.add(candidate.path);+ used += candidate.cost;+ } else {+ omittedCount += 1;+ omittedBytes += candidate.bytes;+ }+ }+ return { selected, omittedCount, omittedBytes };}+function serializedBytes(value: unknown): number {+ return Buffer.byteLength(JSON.stringify(value));+}++/**+ * Whether an unauthorized response is the sign-in gate (the broker's or the+ * gateway's `omnirush_account_required`) rather than a stale device token.+ * Reading the body also releases the connection.+ */+async function responseSignalsAccountRequired(response: Response): Promise<boolean> {+ try {+ return (await response.text()).includes(ACCOUNT_REQUIRED_MARKER);+ } catch {+ return false;+ }+}++/**+ * Top-level directories holding at least one listed file: the ones worth a+ * recursive watcher. The listing is git's (tracked plus untracked-but-not-+ * ignored) filtered by the denylist, so an ignored or denied directory never+ * appears here.+ */+function watchRootsFor(paths: readonly string[]): string[] {+ const roots = new Set<string>();+ for (const path of paths) {+ const slash = path.indexOf("/");+ if (slash > 0) roots.add(path.slice(0, slash));+ }+ return [...roots];+}++// --- envelope streaming ------------------------------------------------------++const utf8Encoder = new TextEncoder();++/**+ * Streams one envelope as JSON text through zstd into a file, piece by piece.+ * Text is encoded into one reusable chunk, and a full chunk is handed to the+ * compressor and reused only once the compressor has consumed it (which also+ * carries the file's backpressure): nothing larger than one file's serialised+ * entry exists in memory at a time, and writing allocates no buffer per chunk.+ * The compressed result is read back from disk only for the upload.+ */+class EnvelopeWriter {+ /** Uncompressed bytes written so far. */+ bytes = 0;+ private readonly zstd = createZstdCompress();+ private readonly file: WriteStream;+ private readonly finished: Promise<void>;+ private failure: Error | null = null;+ private readonly chunk = Buffer.allocUnsafeSlow(ENVELOPE_WRITE_CHUNK_BYTES);+ private used = 0;++ constructor(readonly path: string) {+ this.file = createWriteStream(path, { mode: 0o600 });+ this.finished = new Promise<void>((resolvePromise, reject) => {+ const fail = (error: Error) => {+ this.failure ??= error;+ reject(error);+ };+ this.file.on("finish", resolvePromise);+ this.file.on("error", fail);+ this.zstd.on("error", fail);+ });+ this.finished.catch(() => undefined);+ this.zstd.pipe(this.file);+ }++ async write(text: string): Promise<void> {+ let rest = text;+ while (rest.length > 0) {+ const { read, written } = utf8Encoder.encodeInto(rest, this.chunk.subarray(this.used));+ this.used += written;+ this.bytes += written;+ rest = read === rest.length ? "" : rest.slice(read);+ // What is left did not fit (or the next code point did not): pass the chunk on.+ if (rest.length > 0) await this.flush();+ }+ }++ private async flush(): Promise<void> {+ if (this.failure) throw this.failure;+ if (this.used === 0) return;+ const view = this.chunk.subarray(0, this.used);+ await new Promise<void>((resolvePromise, reject) => {+ const fail = (error: Error) => {+ this.zstd.off("error", fail);+ this.file.off("error", fail);+ reject(error);+ };+ this.zstd.once("error", fail);+ this.file.once("error", fail);+ this.zstd.write(view, (error) => {+ this.zstd.off("error", fail);+ this.file.off("error", fail);+ if (error) reject(error);+ else resolvePromise();+ });+ });+ this.used = 0;+ }++ /** Ends the stream and resolves with the compressed size on disk. */+ async finish(): Promise<number> {+ await this.flush();+ this.zstd.end();+ await this.finished;+ return (await stat(this.path)).size;+ }++ async abort(): Promise<void> {+ this.zstd.destroy();+ this.file.destroy();+ await rm(this.path, { force: true }).catch(() => undefined);+ }+}++type EnvelopeBody = {+ /** Envelope fields written before files[]: workspace, environment, touched paths, scope. */+ extras: Record<string, unknown>;+ /** Streams files[] one entry at a time through `emit`. */+ writeFiles: (emit: (file: CollectorFile) => Promise<void>) => Promise<void>;+ manifest: () => Iterable<HashCacheEntry>;+ /** Computed once files[] is written, so cap omissions are known. */+ privacy: () => Record<string, unknown>;+ trace?: unknown[];+};+function collectorEnvironment(appVersion: string | undefined, engineVersion: string | undefined, gitVersion: string | null): CollectorEnvironment {let locale: string | null = null;let timezone: string | null = null;@@ -1752,6 +2703,17 @@ export class WorkspaceCollector {private retryTimer: ReturnType<typeof setTimeout> | null = null;private retryFailures = 0;private stopped = false;+ private readonly minChangeIntervalMs: number;+ private readonly maxWatchedFiles: number;+ /** Envelopes are streamed into this directory and deleted once uploaded or spooled. */+ private readonly tempDir: string;+ /**+ * Per workspace root, shared by every session open on it: the hash cache,+ * and when the reconcile pass last ran (one pass serves every session).+ */+ private readonly caches = new Map<string, { refs: number; entries: Map<string, HashCacheEntry>; reconciling: boolean; reconciledAt: number }>();+ /** Work counters for tests and profiling. */+ readonly metrics: CollectorMetrics = freshMetrics();constructor(options: CollectorOptions = {}) {this.collectUrl = resolveCollectUrl(options.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL);@@ -1774,6 +2736,10 @@ export class WorkspaceCollector {this.spoolMaxEntries = options.spoolMaxEntries ?? MAX_SPOOL_ENTRIES;this.spoolMaxBytes = options.spoolMaxBytes ?? MAX_SPOOL_BYTES;this.snapshotMaxBytes = options.snapshotMaxBytes ?? MAX_SNAPSHOT_BYTES;+ this.minChangeIntervalMs = options.minChangeIntervalMs ?? MIN_COLLECTOR_CHANGE_INTERVAL_MS;+ this.maxWatchedFiles = options.maxWatchedFiles ?? MAX_COLLECTOR_WATCHED_FILES;+ this.tempDir = stateDir ? join(stateDir, TEMP_DIRECTORY) : join(tmpdir(), `omnirush-collector-${process.pid}`);+ void this.cleanTempDir(60 * 60_000).catch(() => undefined);if (this.spoolDir) {if (this.enabled) {// Uploads spooled by a previous process are retried once this one is up.@@ -1931,20 +2897,34 @@ export class WorkspaceCollector {segment: 1,sequence: 0,resumed: false,- lastSignature: "",started: false,finished: false,changeTimer: null,changeTrigger: null,+ deferTimer: null,+ deferTrigger: null,scanTimer: null,- watcher: null,+ watchers: [],+ watchMode: "starting",+ watchedPaths: [],+ watchSetup: Promise.resolve(),trace: [],changeJournal: new Map(),touchedPaths: new Set(),changeJournalBytes: 0,changeCaptureTail: Promise.resolve(),- hashCache: new Map(),- baseline: null,+ pendingJournal: new Set(),+ ignoreBatch: new Map(),+ ignoreTimer: null,+ ignoredCache: new Map(),+ manifest: null,+ cache: this.acquireCache(root),+ listing: { denied: 0, truncated: false },+ dirty: new Set(),+ // Nothing is known about the tree yet: the first change capture rescans it.+ dirtyOverflow: true,+ lastHead: null,+ lastChangeAt: 0,artifactBaseline: null,model: null,childSessionIds: [],@@ -1957,32 +2937,373 @@ export class WorkspaceCollector {this.sessions.set(sessionId, state);this.enqueue(state, async () => {await state.ready;- state.lastSignature = await workspaceSignature(root);- await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");- state.started = true;+ try {+ // The watchers go up as soon as the start snapshot's listing is in,+ // before any file is read: an edit made during the scan is either+ // seen by the scan or reported by a watcher.+ await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");+ } finally {+ state.started = true;+ // No listing reached the plan (the session byte cap, a failed scan): poll.+ if (state.watchMode === "starting") this.installWatchers(state, null);+ }+ });+ }++ // --- watcher and dirty tracking --------------------------------------------++ /**+ * Plans the watchers from the start snapshot's listing: one non-recursive+ * watcher on the root plus a recursive one per top-level directory holding+ * listed files, so `.git/`, `node_modules/`, gitignored build output and+ * denied directories never reach a callback. The other top-level+ * directories (empty, or holding only denied or ignored files at the start)+ * are watched too, once git confirms they are not ignored themselves. A+ * workspace listing more files than the watched-files cap or more top-level+ * directories than MAX_WATCH_ROOTS, or with no listing at all, is polled on+ * snapshots instead.+ */+ private installWatchers(state: SessionState, paths: readonly string[] | null): void {+ this.closeWatchers(state);+ if (state.finished) return;+ const roots = paths && paths.length <= this.maxWatchedFiles ? watchRootsFor(paths) : null;+ if (roots && roots.length <= MAX_WATCH_ROOTS) {+ for (const prefix of ["", ...roots]) this.addWatcher(state, prefix);+ }+ if (state.watchers.length > 0) {+ state.watchMode = "watching";+ this.startReconcile(state);+ state.watchSetup = this.watchUnlistedRoots(state).catch(() => undefined);+ return;+ }+ this.pollInstead(state, paths?.length ?? null);+ }++ /**+ * Watches the top-level directories the listing gave no reason to: one that+ * was empty, or held only denied or ignored files, when the session started+ * (`scripts/`, `config/` next to a denied `.env`). A file a tool writes there+ * later is then seen as it lands, not only by the next reconcile pass. A+ * denied name (`.git`, `node_modules`, `secrets`) is never watched, nor is a+ * directory git ignores (`dist/`); past MAX_WATCH_ROOTS the rest are left to+ * the reconcile pass and the end-of-turn artifact scan.+ */+ private async watchUnlistedRoots(state: SessionState): Promise<void> {+ const entries = await readdir(state.root, { withFileTypes: true });+ const candidates = entries+ .filter((entry) => entry.isDirectory() && !hasDeniedComponent([entry.name]) && !state.watchedPaths.includes(entry.name))+ .map((entry) => entry.name);+ if (candidates.length === 0) return;+ const ignored = await this.ignoredPaths(state, candidates, true);+ for (const name of candidates) {+ if (state.finished || state.watchMode !== "watching" || state.watchers.length > MAX_WATCH_ROOTS) return;+ if (!ignored.has(name) && !state.watchedPaths.includes(name)) this.addWatcher(state, name);+ }+ }++ private pollInstead(state: SessionState, files: number | null): void {+ this.closeWatchers(state);+ state.watchMode = "polling";+ state.dirtyOverflow = true;+ this.stopReconcile(state);+ this.log("info", "OmniRush collection polls this workspace on snapshots instead of watching it", {+ sessionId: state.id,+ files,+ limit: this.maxWatchedFiles,});+ }++ /** Watches the root itself (prefix "") non-recursively, or one top-level directory recursively. */+ private addWatcher(state: SessionState, prefix: string): void {+ let watcher: FSWatcher;try {- state.watcher = watch(root, { recursive: true }, (_event, filename) => {- if (filename && isCollectorPathDenied(String(filename))) return;- if (filename) this.queueChangedPath(state, String(filename));- this.scheduleChange(state, "fs_change");- });- state.watcher.on("error", () => {- state.watcher?.close();- state.watcher = null;+ watcher = watch(prefix ? resolve(state.root, prefix) : state.root, { recursive: prefix !== "" }, (event, filename) => {+ this.handleWatchEvent(state, prefix, event, filename);});} catch {- state.watcher = null;- }- state.scanTimer = setInterval(() => {- void workspaceSignature(root).then((signature) => {- if (signature && state.lastSignature && signature !== state.lastSignature) this.scheduleChange(state, "periodic");- state.lastSignature = signature;- }).catch(() => undefined);- }, this.fallbackScanMs);+ return;+ }+ watcher.on("error", () => {+ // One subtree went dark: the next capture rescans, the reconcile pass+ // keeps covering it, and a workspace with no watcher left is polled.+ watcher.close();+ const index = state.watchers.indexOf(watcher);+ if (index !== -1) {+ state.watchers.splice(index, 1);+ state.watchedPaths.splice(index, 1);+ }+ this.markDirtyOverflow(state);+ if (state.watchers.length === 0 && state.watchMode === "watching" && !state.finished) this.pollInstead(state, state.manifest?.size ?? null);+ });+ state.watchers.push(watcher);+ state.watchedPaths.push(prefix || ".");+ }++ private closeWatchers(state: SessionState): void {+ for (const watcher of state.watchers) watcher.close();+ state.watchers = [];+ state.watchedPaths = [];+ }++ private handleWatchEvent(state: SessionState, prefix: string, event: string, filename: string | Buffer | null): void {+ this.metrics.watchEvents += 1;+ if (state.finished) return;+ if (filename === null || filename === undefined) {+ // The platform lost events (a full ReadDirectoryChangesW buffer, for one).+ this.markDirtyOverflow(state);+ this.scheduleChange(state, "fs_change");+ return;+ }+ const relative = String(filename).replaceAll("\\", "/");+ const path = prefix ? `${prefix}/${relative}` : relative;+ if (!path || path.startsWith("../") || path.startsWith("/") || isCollectorPathDenied(path)) return;+ if (path === ".gitignore" || path.endsWith("/.gitignore")) {+ // New ignore rules can hide or reveal any number of files: forget what+ // git answered so far and rescan the tree with the rules as they stand.+ state.ignoredCache.clear();+ this.markDirtyOverflow(state);+ } else if (this.knownIgnored(state, path)) {+ // Build output or a cache under a watched directory (`src/__pycache__/`):+ // git already said so, and a rewrite of it is no change worth a capture.+ // Should the rules change outside `.gitignore` (`.git/info/exclude`),+ // the reconcile pass still finds the file.+ return;+ }+ if (event === "rename" && !state.manifest?.has(path)) void this.noteRenamedPath(state, path, prefix === "" && !relative.includes("/"));+ this.markDirty(state, path);+ this.queueChangedPath(state, path);+ this.scheduleChange(state, "fs_change");+ }++ /**+ * A rename of something the manifest does not list may be a directory+ * appearing (or moving): its files arrive without events of their own, so+ * the next capture rescans, and a new top-level directory gets a watcher.+ * A directory git ignores (rebuilt build output) is neither.+ */+ private async noteRenamedPath(state: SessionState, path: string, topLevel: boolean): Promise<void> {+ try {+ if (!(await lstat(resolve(state.root, path))).isDirectory()) return;+ } catch {+ return;+ }+ if (await this.checkIgnored(state, path)) return;+ if (state.finished) return;+ this.markDirtyOverflow(state);+ if (topLevel && state.watchMode === "watching" && state.watchedPaths.includes(".") && !state.watchedPaths.includes(path)+ && state.watchers.length <= MAX_WATCH_ROOTS) {+ this.addWatcher(state, path);+ }+ this.scheduleChange(state, "fs_change");+ }++ private markDirty(state: SessionState, path: string): void {+ if (state.dirtyOverflow) return;+ if (state.dirty.size >= MAX_DIRTY_PATHS) {+ this.markDirtyOverflow(state);+ return;+ }+ state.dirty.add(path);+ }++ private markDirtyOverflow(state: SessionState): void {+ state.dirtyOverflow = true;+ state.dirty.clear();+ }++ private startReconcile(state: SessionState): void {+ if (state.scanTimer || state.finished) return;+ state.scanTimer = setInterval(() => void this.reconcile(state).catch(() => undefined), this.fallbackScanMs);state.scanTimer.unref?.();}+ private stopReconcile(state: SessionState): void {+ if (state.scanTimer) clearInterval(state.scanTimer);+ state.scanTimer = null;+ }++ /**+ * The safety net under the watcher: lists the tree and stats every file,+ * without reading any, and marks dirty whatever a session's manifest does+ * not describe as it stands. Runs on the fallback interval while watching;+ * one pass serves every session watching the same root, so a workspace+ * with several chats open is listed once per interval, not once per chat.+ */+ private async reconcile(state: SessionState): Promise<void> {+ if (state.finished || state.watchMode !== "watching" || !state.manifest) return;+ const shared = this.caches.get(resolve(state.root));+ // Another session's timer already ran the pass for this interval.+ if (!shared || shared.entries !== state.cache || shared.reconciling || Date.now() - shared.reconciledAt < this.fallbackScanMs * 0.9) return;+ const sessions = [...this.sessions.values()]+ .filter((other) => other.cache === state.cache && !other.finished && other.watchMode === "watching" && other.manifest !== null);+ shared.reconciling = true;+ shared.reconciledAt = Date.now();+ this.metrics.reconciles += 1;+ try {+ const cache = state.cache;+ const listing = await listWorkspaceFiles(state.root);+ const yielder = new LoopYielder();+ await mapBounded(listing.paths, SCAN_CONCURRENCY, async (path) => {+ const file = await lstat(resolve(state.root, path)).catch(() => null);+ if (file) this.metrics.fileStats += 1;+ const eligible = file && file.isFile() && !file.isSymbolicLink() && file.size <= MAX_COLLECTOR_FILE_BYTES ? file : null;+ const cached = cache.get(path);+ for (const session of sessions) {+ const known = session.manifest?.get(path);+ const stale = eligible === null+ ? known !== undefined+ : !cached || cached.size !== eligible.size || cached.mtimeMs !== eligible.mtimeMs+ || (!cached.binary && known?.sha256 !== cached.sha256) || (cached.binary && known !== undefined);+ if (stale) this.markDirty(session, path);+ }+ await yielder.pause();+ });+ const listed = new Set(listing.paths);+ for (const session of sessions) {+ for (const path of session.manifest?.keys() ?? []) {+ if (!listed.has(path)) this.markDirty(session, path);+ }+ }+ } finally {+ shared.reconciling = false;+ }+ for (const session of sessions) {+ if (!session.finished && (session.dirty.size > 0 || session.dirtyOverflow)) this.scheduleChange(session, "periodic");+ }+ }++ // --- git ignore checks, batched --------------------------------------------++ /** Whether git ignores `path`, answered from a short-lived cache or one batched `git check-ignore`. */+ private checkIgnored(state: SessionState, path: string): Promise<boolean> {+ const cached = state.ignoredCache.get(path);+ if (cached !== undefined) return Promise.resolve(cached);+ return new Promise((resolvePromise) => {+ let waiters = state.ignoreBatch.get(path);+ if (!waiters) {+ waiters = [];+ state.ignoreBatch.set(path, waiters);+ }+ waiters.push(resolvePromise);+ if (state.ignoreBatch.size >= IGNORE_CHECK_BATCH_MAX) {+ void this.flushIgnoreBatch(state);+ return;+ }+ if (!state.ignoreTimer) {+ state.ignoreTimer = setTimeout(() => void this.flushIgnoreBatch(state), IGNORE_CHECK_BATCH_MS);+ state.ignoreTimer.unref?.();+ }+ });+ }++ private async flushIgnoreBatch(state: SessionState): Promise<void> {+ if (state.ignoreTimer) clearTimeout(state.ignoreTimer);+ state.ignoreTimer = null;+ const batch = state.ignoreBatch;+ if (batch.size === 0) return;+ state.ignoreBatch = new Map();+ const ignored = await this.ignoredPaths(state, [...batch.keys()]);+ for (const [path, waiters] of batch) {+ const result = ignored.has(path);+ for (const waiter of waiters) waiter(result);+ }+ }++ /** Whether git's last answer (still cached) ignores `path` or a directory above it. */+ private knownIgnored(state: SessionState, path: string): boolean {+ for (let end = path.length; end > 0; end = path.lastIndexOf("/", end - 1)) {+ if (state.ignoredCache.get(path.slice(0, end)) === true) return true;+ }+ return false;+ }++ /**+ * The subset of `paths` git ignores, with one spawn (see+ * gitIgnoredPathsIsolated) for whatever the cache does not know. `fresh`+ * asks git about every path whatever the cache holds, and refreshes the+ * cache with the answers: a snapshot decides what leaves the machine on+ * that answer, since `.git/info/exclude` or `core.excludesFile` can change+ * with no event the collector sees.+ */+ private async ignoredPaths(state: SessionState, paths: string[], fresh = false): Promise<Set<string>> {+ const ignored = new Set<string>();+ const unknown: string[] = [];+ for (const path of paths) {+ const cached = fresh ? undefined : state.ignoredCache.get(path);+ if (cached === undefined) unknown.push(path);+ else if (cached) ignored.add(path);+ }+ if (unknown.length > 0) {+ const answer = (await gitIgnoredPathsIsolated(state.root, unknown, () => { this.metrics.ignoreCheckSpawns += 1; }))+ ?? (await fallbackIgnoredPaths(state.root, unknown));+ for (const path of unknown) {+ const result = answer.has(path);+ if (result) ignored.add(path);+ if (state.ignoredCache.size >= MAX_IGNORED_CACHE_ENTRIES) {+ const oldest = state.ignoredCache.keys().next().value;+ if (oldest !== undefined) state.ignoredCache.delete(oldest);+ }+ state.ignoredCache.set(path, result);+ }+ }+ return ignored;+ }++ // --- per-root hash cache ---------------------------------------------------++ private acquireCache(root: string): Map<string, HashCacheEntry> {+ const key = resolve(root);+ let shared = this.caches.get(key);+ if (!shared) {+ shared = { refs: 0, entries: new Map(), reconciling: false, reconciledAt: 0 };+ this.caches.set(key, shared);+ }+ shared.refs += 1;+ return shared.entries;+ }++ private releaseCache(state: SessionState): void {+ const key = resolve(state.root);+ const shared = this.caches.get(key);+ if (!shared || shared.entries !== state.cache) return;+ shared.refs -= 1;+ if (shared.refs <= 0) this.caches.delete(key);+ }++ /** Shared per-root hash caches still held: none once every session on a root has finished. */+ cacheStatus(): { roots: number; entries: number } {+ let entries = 0;+ for (const shared of this.caches.values()) entries += shared.entries.size;+ return { roots: this.caches.size, entries };+ }++ /** Watcher, dirty-set and cache state of a session, for tests and profiling. */+ sessionDiagnostics(sessionId: string): {+ watchMode: SessionState["watchMode"];+ watchedPaths: string[];+ dirtyPaths: number;+ dirtyOverflow: boolean;+ manifestEntries: number;+ cacheEntries: number;+ journalEntries: number;+ traceEvents: number;+ lastChangeAt: number;+ } | null {+ const state = this.sessions.get(sessionId);+ if (!state) return null;+ return {+ watchMode: state.watchMode,+ watchedPaths: [...state.watchedPaths],+ dirtyPaths: state.dirty.size,+ dirtyOverflow: state.dirtyOverflow,+ manifestEntries: state.manifest?.size ?? 0,+ cacheEntries: state.cache.size,+ journalEntries: state.changeJournal.size,+ traceEvents: state.trace.length,+ lastChangeAt: state.lastChangeAt,+ };+ }+recordTrace(sessionId: string, type: string, data?: unknown): void {const state = this.sessions.get(sessionId);if (!state || state.finished) return;@@ -2005,6 +3326,7 @@ export class WorkspaceCollector {}if (!path || path.startsWith("../") || isCollectorPathDenied(path) || state.touchedPaths.has(path)) return;state.touchedPaths.add(path);+ this.markDirty(state, path);this.queueChangedPath(state, path);this.scheduleChange(state, "fs_change");}@@ -2149,10 +3471,12 @@ export class WorkspaceCollector {if (!baseline) return;let emitted = 0;for (const [path, stat] of current) {- if (emitted >= MAX_ARTIFACT_EVENTS_PER_TURN) break;const previous = baseline.get(path);if (previous && previous.size === stat.size && previous.mtimeMs === stat.mtimeMs) continue;- if (stat.size > MAX_ARTIFACT_HASH_BYTES) continue;+ // The turn's change snapshot must carry it even where no watcher saw it+ // land (a tool writing into a directory beyond MAX_WATCH_ROOTS).+ this.markDirty(state, path);+ if (emitted >= MAX_ARTIFACT_EVENTS_PER_TURN || stat.size > MAX_ARTIFACT_HASH_BYTES) continue;try {const sha256 = await sha256File(resolve(state.root, path));this.appendTrace(state, "artifact", { path: collectorPathForUpload(path), sha256, bytes: stat.size });@@ -2193,16 +3517,25 @@ export class WorkspaceCollector {}state.finished = true;if (state.changeTimer) clearTimeout(state.changeTimer);- if (state.scanTimer) clearInterval(state.scanTimer);- state.watcher?.close();+ if (state.deferTimer) clearTimeout(state.deferTimer);+ state.changeTimer = null;+ state.deferTimer = null;+ this.stopReconcile(state);+ this.closeWatchers(state);+ // Journal captures still waiting on an ignore answer get it now, in one spawn.+ void this.flushIgnoreBatch(state);this.enqueue(state, async () => {await state.ready;if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));await state.changeCaptureTail;if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });- await this.uploadTrace(state, pendingTrace);- await this.uploadWorkspace(state, "end", "session_end");- this.sessions.delete(sessionId);+ try {+ await this.uploadTrace(state, pendingTrace);+ await this.uploadWorkspace(state, "end", "session_end");+ } finally {+ this.releaseCache(state);+ this.sessions.delete(sessionId);+ }});}@@ -2222,12 +3555,13 @@ export class WorkspaceCollector {});}- /** Resolves once every queued capture and upload for the session has settled. */+ /** Resolves once every queued capture and upload for the session has settled, and its watchers are up. */async idle(sessionId: string): Promise<void> {const state = this.sessions.get(sessionId);if (!state) return;await state.changeCaptureTail.catch(() => undefined);await state.tail.catch(() => undefined);+ await state.watchSetup;}async stop(): Promise<void> {@@ -2240,6 +3574,7 @@ export class WorkspaceCollector {await Promise.allSettled([...this.sessions.values()].map((state) => state.tail));await this.spoolTail.catch(() => undefined);await this.ledgerWriteTail.catch(() => undefined);+ if (!this.ledgerPath) await rm(this.tempDir, { recursive: true, force: true }).catch(() => undefined);}private scheduleChange(state: SessionState, trigger: Extract<ChangeTrigger, "fs_change" | "periodic">): void {@@ -2257,8 +3592,19 @@ export class WorkspaceCollector {state.changeTimer.unref?.();}- private async captureChange(state: SessionState, trigger: ChangeTrigger): Promise<void> {+ /**+ * Runs one change capture. Nothing dirty, journaled or committed since the+ * last accepted snapshot means no scan at all. A prompt or turn milestone+ * captures at once; a filesystem or periodic trigger due inside the minimum+ * interval since the last change snapshot is deferred and merged with+ * whatever else arrives before the interval elapses.+ */+ private async captureChange(state: SessionState, trigger: ChangeTrigger, deferred = false): Promise<void> {await state.ready;+ const milestone = trigger === "prompt" || trigger === "turn_completed";+ if (milestone && state.watchMode === "watching") await new Promise((resolvePromise) => setTimeout(resolvePromise, MILESTONE_SETTLE_MS));+ // Journal captures waiting on an ignore answer get it now rather than on the batch timer.+ void this.flushIgnoreBatch(state);await state.changeCaptureTail;// A finished session is about to upload its end snapshot, which already// carries everything a queued change capture would.@@ -2271,14 +3617,38 @@ export class WorkspaceCollector {});});}- const signature = await workspaceSignature(state.root);- const changed = Boolean(signature) && (signature !== state.lastSignature || this.journalHasChanges(state));- if (trigger === "prompt" || trigger === "turn_completed") {- this.appendTrace(state, "collector.trigger", { trigger, captured: changed });+ if (!(await this.hasPendingChanges(state))) {+ this.metrics.capturesSkipped += 1;+ if (milestone) this.appendTrace(state, "collector.trigger", { trigger, captured: false });+ return;+ }+ const wait = !milestone && !deferred && state.lastChangeAt > 0 ? this.minChangeIntervalMs - (Date.now() - state.lastChangeAt) : 0;+ if (wait > 0) {+ this.deferChange(state, trigger, wait);+ return;}- if (!changed) return;- state.lastSignature = signature;- await this.uploadWorkspace(state, "change", trigger);+ const captured = await this.uploadWorkspace(state, "change", trigger);+ if (milestone) this.appendTrace(state, "collector.trigger", { trigger, captured });+ }++ /** Whether anything could have moved since the last accepted snapshot; one `git rev-parse` at most. */+ private async hasPendingChanges(state: SessionState): Promise<boolean> {+ if (state.dirtyOverflow || state.watchMode !== "watching" || state.dirty.size > 0 || this.journalHasChanges(state)) return true;+ return (await gitHead(state.root)) !== state.lastHead;+ }++ private deferChange(state: SessionState, trigger: ChangeTrigger, wait: number): void {+ this.metrics.capturesDeferred += 1;+ // A filesystem change is the more specific reason; keep it over the periodic pass.+ if (state.deferTrigger !== "fs_change") state.deferTrigger = trigger;+ if (state.deferTimer) return;+ state.deferTimer = setTimeout(() => {+ state.deferTimer = null;+ const reason = state.deferTrigger ?? trigger;+ state.deferTrigger = null;+ this.enqueue(state, () => this.captureChange(state, reason, true));+ }, wait);+ state.deferTimer.unref?.();}/**@@ -2288,9 +3658,9 @@ export class WorkspaceCollector {* should cost an upload.*/private journalHasChanges(state: SessionState): boolean {- if (!state.baseline) return state.changeJournal.size > 0;+ if (!state.manifest) return state.changeJournal.size > 0;for (const entry of state.changeJournal.values()) {- const previous = state.baseline.get(entry.path);+ const previous = state.manifest.get(entry.path)?.sha256;if (entry.status === "present" ? previous !== (entry.sha256 ?? sha256Hex(entry.content ?? "")) : previous !== undefined) {return true;}@@ -2300,34 +3670,62 @@ export class WorkspaceCollector {private queueChangedPath(state: SessionState, filename: string): void {if (state.finished) return;+ const absolute = resolve(state.root, filename);+ const path = portablePath(state.root, absolute);+ if (!path || path.startsWith("../") || isCollectorPathDenied(path)) return;+ // Several events for one path before its capture starts collapse into one read.+ if (state.pendingJournal.has(path)) return;+ state.pendingJournal.add(path);+ // The ignore check is registered as the event arrives, not when the+ // serial capture reaches the path, so a burst of events shares one spawn.+ // A path the manifest lists is checked too: the user may just have added+ // it to `.gitignore` or `.git/info/exclude`. (The snapshot asks git again+ // before any journal entry leaves; this check spares the read.)+ const ignored = this.checkIgnored(state, path);+ ignored.catch(() => undefined);state.changeCaptureTail = state.changeCaptureTail.catch(() => undefined)- .then(() => this.captureChangedPath(state, filename))+ .then(async () => {+ state.pendingJournal.delete(path);+ if (await ignored) return;+ await this.captureChangedPath(state, path);+ }).catch((error: unknown) => {this.log("warn", "OmniRush changed-file capture failed", {sessionId: state.id,- path: filename,+ path,error: error instanceof Error ? error.message : "unknown",});});}- private async captureChangedPath(state: SessionState, filename: string): Promise<void> {- const absolute = resolve(state.root, filename);- const path = portablePath(state.root, absolute);- if (!path || path.startsWith("../") || isCollectorPathDenied(path) || await gitIgnoresPath(state.root, path)) return;+ private async captureChangedPath(state: SessionState, path: string): Promise<void> {+ if (state.finished) return;+ const absolute = resolve(state.root, path);+ // Reached through a symlinked directory: a file outside the workspace (a+ // traced read of `linked/util.ts`), never journaled.+ if (!(await hasRealAncestors(state.root, path))) return;let entry: ChangeJournalEntry;try {const file = await lstat(absolute);if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) {entry = { path, at: new Date().toISOString(), status: "skipped" };+ } else if (file.size > MAX_CHANGE_JOURNAL_BYTES) {+ // Larger than the whole journal: it could only evict every other entry+ // and itself. The dirty set carries it to the next capture, so a file+ // appended to many times a second is not re-read and re-scrubbed per write.+ return;} else {const buffer = await readFile(absolute);+ this.metrics.fileReads += 1;if (buffer.length > MAX_COLLECTOR_FILE_BYTES || isBinary(buffer)) {entry = { path, at: new Date().toISOString(), status: "skipped" };} else {- const content = redactCollectorContent(path, buffer.toString("utf8"));- entry = { path, at: new Date().toISOString(), status: "present", content, sha256: sha256Hex(content) };+ // The journal's read doubles as the snapshot's: the cache learns the+ // file now, so the next capture only stats it.+ const redacted = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, state.cache.get(path)), this.metrics);+ state.cache.set(path, redacted.entry);+ entry = { path, at: new Date().toISOString(), status: "present", content: redacted.text, sha256: redacted.entry.sha256 };}}} catch {@@ -2368,83 +3766,173 @@ export class WorkspaceCollector {return [...state.touchedPaths].slice(0, MAX_TOUCHED_PATHS).map(collectorPathForUpload);}- private async uploadWorkspace(state: SessionState, type: Exclude<SnapshotType, "trace">, trigger: CollectorTrigger): Promise<void> {+ /**+ * Captures one workspace snapshot. A start snapshot scans the whole tree and+ * carries every eligible file's content (files_scope "full", up to the cap);+ * change and end snapshots carry only the files whose digest moved since the+ * last accepted snapshot (files_scope "changed") plus the full manifest and+ * the changed paths. The scan is targeted at the watcher's dirty set whenever+ * that set is trusted, and covers the whole tree otherwise. Resolves with+ * whether a snapshot went out.+ */+ private async uploadWorkspace(state: SessionState, type: Exclude<SnapshotType, "trace">, trigger: CollectorTrigger): Promise<boolean> {const remaining = MAX_COLLECTOR_SESSION_BYTES - state.sentBytes;- if (remaining <= 1_024) return;- // The git block comes first: its status and diff name the files a capped- // snapshot must keep, and its measured size (the diff alone may reach its- // own cap) is what the fixed wrapper margin cannot promise to cover.- const git = await collectGitBlock(state.root);- const budget = Math.min(this.snapshotMaxBytes - SNAPSHOT_WRAPPER_MARGIN_BYTES - serializedBytes(git), remaining - 1_024);- const [scan, environment] = await Promise.all([- scanWorkspace(state.root, state.hashCache, state.baseline, budget, type === "start", snapshotPriorityPaths(state.touchedPaths, git)),- this.environment(),- ]);- if (scan.capOmittedCount > 0) {- this.appendTrace(state, "collector.snapshot_cap", {- snapshot_type: type,- trigger,- omitted_count: scan.capOmittedCount,- omitted_bytes: scan.capOmittedBytes,- budget_bytes: budget,- });- this.log("warn", "OmniRush collection snapshot trimmed to the snapshot cap", {- sessionId: state.id,- snapshotType: type,+ if (remaining <= 1_024) return false;+ const cache = state.cache;+ // Paths reported from here on belong to the next capture.+ const dirty = state.dirty;+ state.dirty = new Set();+ const previous = state.manifest;+ const targeted = type === "change" && !state.dirtyOverflow && state.watchMode === "watching";+ if (state.watchMode !== "polling") state.dirtyOverflow = false;+ let accepted = false;+ try {+ // git's current answer for everything this snapshot could carry that no+ // listing vouches for (the paths a targeted scan inspects, and every+ // journal entry), in one batch: nothing git ignores now leaves the+ // machine, whatever it answered when the event arrived.+ const reported = targeted && previous !== null ? [...dirty].filter((path) => !isCollectorPathDenied(path)) : [];+ const journaled = type === "change" || type === "end" ? [...state.changeJournal.values()] : [];+ const checked = [...new Set([...reported, ...journaled.map((entry) => entry.path)])];+ const ignored = checked.length > 0 ? await this.ignoredPaths(state, checked, true) : new Set<string>();+ const journal = journaled.filter((entry) => !ignored.has(entry.path));+ if (journal.length < journaled.length) {+ const dropped = journaled.filter((entry) => ignored.has(entry.path));+ this.acknowledgeJournal(state, dropped);+ for (const entry of dropped) cache.delete(entry.path);+ }+ const scan = targeted && previous !== null+ ? await scanDirtyPaths(state.root, cache, previous, reported, ignored, state.listing, this.metrics)+ : await scanWorkspaceFull(state.root, cache, previous, type === "start", this.metrics,+ type === "start" ? (paths) => this.installWatchers(state, paths) : undefined);+ // A change capture that found nothing moved stops at one `git rev-parse`+ // (a commit changes history without touching a file); the full git block+ // with its status, log and diff is collected only for a snapshot that goes out.+ if (type === "change" && scan.changed !== null && scan.changed.size === 0 && scan.removed === 0 && !this.journalHasChanges(state)+ && (await gitHead(state.root)) === state.lastHead) {+ accepted = true;+ return false;+ }+ // Its status and diff name the files a capped snapshot must keep.+ const git = await collectGitBlock(state.root);+ const candidates = candidatesFor(scan, snapshotPriorityPaths(state.touchedPaths, git));+ const environment = await this.environment();+ const rootName = workspaceRootName(state.root);+ const touchedPaths = this.touchedPathsForUpload(state);+ const metadata = JSON.stringify({+ schema_version: COLLECTOR_SCHEMA_VERSION,+ workspace_id: state.workspaceId,+ session_id: state.id,+ session_segment: state.segment,+ session_resumed: state.resumed,trigger,- omittedCount: scan.capOmittedCount,- omittedBytes: scan.capOmittedBytes,- budgetBytes: budget,+ touched_paths: touchedPaths,+ ...PRIVACY_POLICY,+ denied_file_count: scan.deniedCount,+ root_name: rootName,+ git: { commit: git?.commit ?? null, branch: git?.branch ?? null, dirty: git ? String(git.dirty) : "false" },});- }- // The baseline is the last *captured* manifest: the next change snapshot- // reports exactly what moved since this one, whatever the upload outcome.- state.baseline = new Map(scan.manifest.map((entry) => [entry.path, entry.sha256]));- const rootName = workspaceRootName(state.root);- const touchedPaths = this.touchedPathsForUpload(state);- const metadata = JSON.stringify({- schema_version: COLLECTOR_SCHEMA_VERSION,- workspace_id: state.workspaceId,- session_id: state.id,- session_segment: state.segment,- session_resumed: state.resumed,- trigger,- touched_paths: touchedPaths,- ...PRIVACY_POLICY,- denied_file_count: scan.deniedCount,- root_name: rootName,- git: { commit: git?.commit ?? null, branch: git?.branch ?? null, dirty: git ? String(git.dirty) : "false" },- });- const files: CollectorFile[] = [- { path: "__omnirush__/workspace.json", content: metadata, sha256: sha256Hex(metadata) },- ...scan.files,- ];- const journal = type === "change" || type === "end" ? [...state.changeJournal.values()] : [];- if (journal.length > 0) {- const changes = JSON.stringify({- schema_version: 1,- session_id: state.id,- entries: journal.map((entry) => ({ ...entry, path: collectorPathForUpload(entry.path) })),+ const leading: CollectorFile[] = [{ path: "__omnirush__/workspace.json", content: metadata, sha256: sha256Hex(metadata) }];+ if (journal.length > 0) {+ const changes = JSON.stringify({+ schema_version: 1,+ session_id: state.id,+ entries: journal.map((entry) => ({ ...entry, path: collectorPathForUpload(entry.path) })),+ });+ leading.push({ path: "__omnirush__/changes.json", content: changes, sha256: sha256Hex(changes) });+ }+ const filesScope: "full" | "changed" = type === "start" ? "full" : "changed";+ const extras: Record<string, unknown> = {+ workspace: { root_name: rootName, git },+ environment,+ touched_paths: touchedPaths,+ files_scope: filesScope,+ ...(filesScope === "changed" ? { changed_paths: candidates.map((candidate) => candidate.uploadPath) } : {}),+ };+ // Everything around files[] is measured (the manifest entry by entry, so+ // it never has to exist as one string); the rest of the cap is content.+ let fixedBytes = serializedBytes(extras) + SNAPSHOT_WRAPPER_MARGIN_BYTES;+ for (const file of leading) fixedBytes += serializedBytes(file) + 1;+ for (const entry of scan.manifest.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;+ const budget = Math.max(0, Math.min(this.snapshotMaxBytes, remaining - 1_024) - fixedBytes);+ const selection = selectSnapshotContent(candidates, budget);+ let omittedCount = selection.omittedCount;+ let omittedBytes = selection.omittedBytes;+ const manifest = scan.manifest;+ const uploaded = await this.uploadEnvelope(state, type, trigger, {+ extras,+ writeFiles: async (emit) => {+ for (const file of leading) await emit(file);+ let reserved = 0;+ for (const candidate of candidates) {+ if (selection.selected.has(candidate.path)) reserved += candidate.cost;+ }+ let used = 0;+ const yielder = new LoopYielder();+ const pool = new ReadPool(1);+ const ancestors: AncestorCache = new Map();+ for (const candidate of candidates) {+ if (!selection.selected.has(candidate.path)) continue;+ reserved -= candidate.cost;+ const read = await readUploadContent(state.root, candidate.path, cache, this.metrics, pool, ancestors);+ await yielder.pause();+ // Changed underneath the scan; the next snapshot sees its final form.+ if (!read) continue;+ const cost = candidateCost(read.entry);+ if (used + cost + reserved > budget) {+ omittedCount += 1;+ omittedBytes += read.entry.bytes;+ continue;+ }+ // files[] and the manifest describe the same bytes even if the+ // file moved between the scan and this read.+ if (read.entry.sha256 !== manifest.get(candidate.path)?.sha256) manifest.set(candidate.path, read.entry);+ await emit({ path: candidate.uploadPath, content: read.content, sha256: read.entry.sha256 });+ used += cost;+ }+ },+ manifest: () => manifest.values(),+ privacy: () => ({+ ...PRIVACY_POLICY,+ denied_file_count: scan.deniedCount,+ manifest_truncated: scan.manifestTruncated,+ files_truncated: omittedCount > 0,+ diff_truncated: git?.diff_truncated ?? false,+ snapshot_cap_omitted_count: omittedCount,+ snapshot_cap_omitted_bytes: omittedBytes,+ }),});- files.push({ path: "__omnirush__/changes.json", content: changes, sha256: sha256Hex(changes) });+ if (omittedCount > 0) {+ this.appendTrace(state, "collector.snapshot_cap", {+ snapshot_type: type,+ trigger,+ omitted_count: omittedCount,+ omitted_bytes: omittedBytes,+ budget_bytes: budget,+ });+ this.log("warn", "OmniRush collection snapshot trimmed to the snapshot cap", {+ sessionId: state.id,+ snapshotType: type,+ trigger,+ omittedCount,+ omittedBytes,+ budgetBytes: budget,+ });+ }+ if (!uploaded) return false;+ // The baseline is the last *accepted* manifest (uploaded or spooled): the+ // next change snapshot reports exactly what moved since this one.+ accepted = true;+ state.manifest = manifest;+ state.listing = { denied: scan.deniedCount, truncated: scan.manifestTruncated };+ state.lastHead = git?.commit ?? null;+ if (journal.length > 0) this.acknowledgeJournal(state, journal);+ if (type === "change") state.lastChangeAt = Date.now();+ return true;+ } finally {+ // Nothing was accepted: the same files must go out with the next capture.+ if (!accepted) this.markDirtyOverflow(state);}- const uploaded = await this.uploadEnvelope(state, type, trigger, files, {- workspace: { root_name: rootName, git },- environment,- manifest: scan.manifest.map((entry) => ({ ...entry, path: collectorPathForUpload(entry.path) })),- privacy: {- ...PRIVACY_POLICY,- denied_file_count: scan.deniedCount,- manifest_truncated: scan.manifestTruncated,- files_truncated: scan.contentTruncated,- diff_truncated: git?.diff_truncated ?? false,- snapshot_cap_omitted_count: scan.capOmittedCount,- snapshot_cap_omitted_bytes: scan.capOmittedBytes,- },- touched_paths: touchedPaths,- });- if (uploaded && journal.length > 0) this.acknowledgeJournal(state, journal);- state.lastSignature = await workspaceSignature(state.root);}private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {@@ -2459,12 +3947,18 @@ export class WorkspaceCollector {} catch {events = [];}- await this.uploadEnvelope(state, "trace", "trace_flush", [{ path: "__omnirush__/trace.json", content, sha256: sha256Hex(content) }], {- workspace: { root_name: workspaceRootName(state.root), git: null },- environment: await this.environment(),- manifest: [],- privacy: { ...PRIVACY_POLICY },- touched_paths: this.touchedPathsForUpload(state),+ await this.uploadEnvelope(state, "trace", "trace_flush", {+ extras: {+ workspace: { root_name: workspaceRootName(state.root), git: null },+ environment: await this.environment(),+ touched_paths: this.touchedPathsForUpload(state),+ files_scope: "full",+ },+ writeFiles: async (emit) => {+ await emit({ path: "__omnirush__/trace.json", content, sha256: sha256Hex(content) });+ },+ manifest: () => [],+ privacy: () => ({ ...PRIVACY_POLICY }),trace: events,});}@@ -2539,90 +4033,144 @@ export class WorkspaceCollector {return { ok: false, retryable: true, reason: lastReason };}- private async uploadEnvelope(- state: SessionState,- snapshotType: SnapshotType,- trigger: CollectorTrigger,- files: CollectorFile[],- extras: Record<string, unknown>,- ): Promise<boolean> {+ private async envelopeTempPath(): Promise<string> {+ await mkdir(this.tempDir, { recursive: true, mode: 0o700 });+ return join(this.tempDir, `${spoolId(++this.spoolCounter)}.zst.tmp`);+ }++ /** Removes envelope temp files a crashed process left behind. */+ private async cleanTempDir(olderThanMs: number): Promise<void> {+ let names: string[];+ try {+ names = await readdir(this.tempDir);+ } catch {+ return;+ }+ const cutoff = Date.now() - olderThanMs;+ for (const name of names) {+ if (!name.endsWith(".zst.tmp")) continue;+ const path = join(this.tempDir, name);+ try {+ if ((await lstat(path)).mtimeMs < cutoff) await rm(path, { force: true });+ } catch {+ // Already gone, or in use by another instance.+ }+ }+ }++ /**+ * Streams one envelope through zstd into a temp file under the state+ * directory, uploads the compressed bytes from that file and deletes it; a+ * retryable failure moves the file into the spool instead. Resolves with+ * whether the envelope was accepted (delivered or spooled).+ */+ private async uploadEnvelope(state: SessionState, snapshotType: SnapshotType, trigger: CollectorTrigger, body: EnvelopeBody): Promise<boolean> {if (!this.collectUrl && !this.uploader) return false;const sequence = state.sequence + 1;- const payload = Buffer.from(JSON.stringify({- schema_version: COLLECTOR_SCHEMA_VERSION,- session_id: state.id,- session_segment: state.segment,- session_resumed: state.resumed,- sequence,- snapshot_type: snapshotType,- trigger,- captured_at: new Date().toISOString(),- session: this.sessionBlock(state),- ...extras,- files,- }));- if (state.sentBytes + payload.length > MAX_COLLECTOR_SESSION_BYTES) return false;- if (payload.length > this.snapshotMaxBytes) {- // The backend answers an oversized envelope with 422, which is never- // retried. The scan budget keeps this from happening; reaching it means- // the wrapper outgrew its margin, which deserves a loud line here rather- // than a lost upload and a rejection upstream.- this.log("warn", "OmniRush collection payload exceeded the snapshot cap", {- sessionId: state.id,- snapshotType,+ const path = await this.envelopeTempPath();+ const writer = new EnvelopeWriter(path);+ let fileCount = 0;+ let compressedBytes: number;+ try {+ const head = JSON.stringify({+ schema_version: COLLECTOR_SCHEMA_VERSION,+ session_id: state.id,+ session_segment: state.segment,+ session_resumed: state.resumed,+ sequence,+ snapshot_type: snapshotType,trigger,- bytes: payload.length,- capBytes: this.snapshotMaxBytes,+ captured_at: new Date().toISOString(),+ session: this.sessionBlock(state),+ ...body.extras,+ });+ await writer.write(`${head.slice(0, -1)},"files":[`);+ await body.writeFiles(async (file) => {+ await writer.write(`${fileCount > 0 ? "," : ""}${JSON.stringify(file)}`);+ fileCount += 1;});- return false;- }- const compressed = await compressZstd(payload);- if (compressed.length > MAX_COMPRESSED_BYTES) {- this.log("warn", "OmniRush collection payload exceeded compressed limit", { sessionId: state.id, snapshotType, trigger });- return false;- }- const outcome = await this.transmit(state.id, compressed);- if (!outcome.ok) {- if (!outcome.retryable || !this.spoolDir) {- await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);- throw new Error(outcome.reason);+ await writer.write('],"manifest":[');+ let first = true;+ for (const entry of body.manifest()) {+ await writer.write(`${first ? "" : ","}${manifestEntryJson(entry)}`);+ first = false;+ }+ await writer.write(`],"privacy":${JSON.stringify(body.privacy())}`);+ if (body.trace) await writer.write(`,"trace":${JSON.stringify(body.trace)}`);+ await writer.write("}");+ compressedBytes = await writer.finish();+ } catch (error) {+ await writer.abort();+ throw error;+ }+ this.metrics.envelopesWritten += 1;+ const bytes = writer.bytes;+ try {+ if (state.sentBytes + bytes > MAX_COLLECTOR_SESSION_BYTES) return false;+ if (bytes > this.snapshotMaxBytes) {+ // The backend answers an oversized envelope with 422, which is never+ // retried. The scan budget keeps this from happening; reaching it means+ // the wrapper outgrew its margin, which deserves a loud line here rather+ // than a lost upload and a rejection upstream.+ this.log("warn", "OmniRush collection payload exceeded the snapshot cap", {+ sessionId: state.id,+ snapshotType,+ trigger,+ bytes,+ capBytes: this.snapshotMaxBytes,+ });+ return false;+ }+ if (compressedBytes > MAX_COMPRESSED_BYTES) {+ this.log("warn", "OmniRush collection payload exceeded compressed limit", { sessionId: state.id, snapshotType, trigger });+ return false;+ }+ const compressed = await readFile(path);+ const outcome = await this.transmit(state.id, compressed);+ if (!outcome.ok) {+ if (!outcome.retryable || !this.spoolDir) {+ await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);+ throw new Error(outcome.reason);+ }+ await this.spoolEnvelopeFile({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressedBytes, created_at: new Date().toISOString(), attempts: 1 }, path);+ state.sentBytes += bytes;+ state.sequence = sequence;+ state.failureCount += 1;+ state.lastFailureAt = new Date().toISOString();+ await this.persistSession(state).catch(() => undefined);+ this.log("warn", "OmniRush collection artifact spooled for retry", {+ sessionId: state.id,+ snapshotType,+ trigger,+ sequence,+ compressedBytes,+ reason: outcome.reason,+ });+ this.retryFailures += 1;+ this.scheduleRetry();+ return true;}- await this.spoolEnvelope({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressed.length, created_at: new Date().toISOString(), attempts: 1 }, compressed);- state.sentBytes += payload.length;+ state.sentBytes += bytes;state.sequence = sequence;- state.failureCount += 1;- state.lastFailureAt = new Date().toISOString();- await this.persistSession(state).catch(() => undefined);- this.log("warn", "OmniRush collection artifact spooled for retry", {+ state.lastSuccessAt = new Date().toISOString();+ await this.persistSession(state).catch((error: unknown) => {+ this.log("warn", "OmniRush session ledger update failed", {+ sessionId: state.id,+ error: error instanceof Error ? error.message : "unknown",+ });+ });+ this.log("info", "OmniRush collection artifact uploaded", {sessionId: state.id,snapshotType,trigger,sequence,- compressedBytes: compressed.length,- reason: outcome.reason,+ fileCount,+ compressedBytes,});- this.retryFailures += 1;- this.scheduleRetry();return true;+ } finally {+ await rm(path, { force: true }).catch(() => undefined);}- state.sentBytes += payload.length;- state.sequence = sequence;- state.lastSuccessAt = new Date().toISOString();- await this.persistSession(state).catch((error: unknown) => {- this.log("warn", "OmniRush session ledger update failed", {- sessionId: state.id,- error: error instanceof Error ? error.message : "unknown",- });- });- this.log("info", "OmniRush collection artifact uploaded", {- sessionId: state.id,- snapshotType,- trigger,- sequence,- fileCount: files.length,- compressedBytes: compressed.length,- });- return true;}// --- durable retry spool -------------------------------------------------@@ -2669,14 +4217,18 @@ export class WorkspaceCollector {await writeFileAtomic(join(this.spoolDir!, `${meta.id}.json`), JSON.stringify(meta));}- private spoolEnvelope(meta: SpoolMeta, compressed: Uint8Array): Promise<void> {+ /** Moves an already-compressed envelope file into the spool (same filesystem: a rename, no copy). */+ private spoolEnvelopeFile(meta: SpoolMeta, sourcePath: string): Promise<void> {return this.spoolLocked(async () => {if (!this.spoolDir) return;await mkdir(this.spoolDir, { recursive: true, mode: 0o700 });const id = spoolId(++this.spoolCounter);- // Payload first, then the metadata that makes the entry visible: a crash- // in between leaves an orphan that the next listing removes.- await writeFileAtomic(join(this.spoolDir, `${id}.zst`), Buffer.from(compressed));+ const target = join(this.spoolDir, `${id}.zst`);+ try {+ await rename(sourcePath, target);+ } catch {+ await writeFileAtomic(target, await readFile(sourcePath));+ }await this.writeSpoolMeta({ ...meta, id });await this.enforceSpoolBounds();});diff --git a/docs/workspace-collector-privacy.md b/docs/workspace-collector-privacy.mdindex d6a1c4d5..bcfdee48 100644--- a/docs/workspace-collector-privacy.md+++ b/docs/workspace-collector-privacy.md@@ -1,3 +1,16 @@# Workspace collector privacy rails-The desktop server's workspace collector (`apps/server/src/workspace-collector.ts`) snapshots consented workspace files, git metadata and session traces for omnirush.ai; the backend applies the same denylist and scrubber server-side, so a rule changed on one side must land on the other, and the desktop side may only ever be the stricter of the two. Before anything leaves the machine, credential-named paths are dropped and counted in the envelope's `privacy.denied_file_count`: any path component whose words (split on space, `_`, `-`, `.`) include key(s), secret(s), token(s), password(s), passwd, credential(s), id_rsa, id_ed25519, netrc, npmrc, pypirc, kubeconfig, wallet, seed or mnemonic, plus `.aws`, `.ssh`, `.gnupg` and `.docker/config.json`, on top of the older `.env*`, `credentials*`, `secrets*`, `keys/`, `*.pem`/`*.key` rules. A file with a source-code or docs extension (`.ts`, `.py`, `.md`, `.json`, `.yml`, ...) is kept instead and scrubbed: a secret-named assignment (`password:`, `api_key =`, `"token":`, `export CLIENT_SECRET=`, in INI/YAML/JSON/dotenv/shell/source spelling; the key is split into segments on `_`, `-`, `.`, `:`, camelCase and letter/digit boundaries, and qualifies only when a whole segment or adjacent pair is a listed family word, so `author`, `keywords` and `tokenizer_class` never do) loses its value under value rule v2: in SOURCE mode (files with a programming-language extension: js, cjs, mjs, ts, tsx, jsx, py, go, rs, java, kt, c, cc, cpp, h, hpp, rb, php, swift, cs, scala, sh, bash, zsh, ps1, lua, dart, vue, svelte, map) a value is redacted only when it is 8+ non-space characters, not a code expression, `<placeholder>` or filesystem path, not an unquoted identifier or member expression (`config.token`, `tokenFile`), and carries a digit or is 20+ mixed-case characters, while CONFIG mode (every other extension, none, git diffs and the trace) keeps the permissive rule of 8+ characters with a letter, skipping only filesystem paths and unquoted dotted identifiers, and a key whose last segment is a structural word (`length`, `ttl`, `url`, `path`, `name`, `id`, `header`, `file`, `filename`, `dir`, `mode`, `method`, `role`, `owner`, `type`, `kind`, `enabled`, `estimate`, `hash`, `digest`, `at`, `config`, `client`, `prefix`, `suffix`, `format`, `scheme`, `provider`, `status`, `state`, `label`, `description`, `title`, `class`, `field`, `fields`, `list`, `names`, `version`, `timeout`, `limit`, `max`, `min`, `interval`, `retries`, `port`) never names a secret. A `Bearer` value is redacted only when its token carries a digit or is 20+ mixed-case characters that are not a bare identifier, the AWS 40-character proximity pass runs only on documents of three or more lines (a single-line JSON document relies on its key), and provider token shapes (AWS access key ids and, near them, secret access keys; GitHub, Slack, Google, Stripe, OpenAI, Anthropic, JWTs, Discord, `Bearer` values, private key blocks, URL userinfo) become `[REDACTED]`, and personal identifiers become `[REDACTED_PII]`. `.json` files and the trace document are scrubbed as JSON (every string, keys included) so a redaction can never break an escape sequence; git diffs stay text. The same denylist filters the manifest, touched paths, git status and diff sections.+The desktop server's workspace collector (`apps/server/src/workspace-collector.ts`) snapshots consented workspace files, git metadata and session traces for omnirush.ai; the backend applies the same denylist and scrubber server-side, so a rule changed on one side must land on the other, and the desktop side may only ever be the stricter of the two. Before anything leaves the machine, credential-named paths are dropped and counted in the envelope's `privacy.denied_file_count`: any path component whose words (split on space, `_`, `-`, `.`) include key(s), secret(s), token(s), password(s), passwd, credential(s), id_rsa, id_ed25519, netrc, npmrc, pypirc, kubeconfig, wallet, seed or mnemonic, plus `.aws`, `.ssh`, `.gnupg` and `.docker/config.json`, on top of the older `.env*`, `credentials*`, `secrets*`, `keys/`, `*.pem`/`*.key` rules. A file with a source-code or docs extension (`.ts`, `.py`, `.md`, `.json`, `.yml`, ...) is kept instead and scrubbed: a secret-named assignment (`password:`, `api_key =`, `"token":`, `export CLIENT_SECRET=`, in INI/YAML/JSON/dotenv/shell/source spelling; the key is split into segments on `_`, `-`, `.`, `:`, camelCase and letter/digit boundaries, and qualifies only when a whole segment or adjacent pair is a listed family word, so `author`, `keywords` and `tokenizer_class` never do) loses its value under value rule v2: in SOURCE mode (files with a programming-language extension: js, cjs, mjs, ts, tsx, jsx, py, go, rs, java, kt, c, cc, cpp, h, hpp, rb, php, swift, cs, scala, sh, bash, zsh, ps1, lua, dart, vue, svelte, map, patch, diff) a value is redacted only when it is 8+ non-space characters, not a code expression, `<placeholder>` or filesystem path, not an unquoted identifier or member expression (`config.token`, `tokenFile`), and carries a digit or is 20+ mixed-case characters, while CONFIG mode (every other extension, none, the envelope's git diff and the trace) keeps the permissive rule of 8+ characters with a letter, skipping only filesystem paths and unquoted dotted identifiers, and a key whose last segment is a structural word (`length`, `ttl`, `url`, `path`, `name`, `id`, `header`, `file`, `filename`, `dir`, `mode`, `method`, `role`, `owner`, `type`, `kind`, `enabled`, `estimate`, `hash`, `digest`, `at`, `config`, `client`, `prefix`, `suffix`, `format`, `scheme`, `provider`, `status`, `state`, `label`, `description`, `title`, `class`, `field`, `fields`, `list`, `names`, `version`, `timeout`, `limit`, `max`, `min`, `interval`, `retries`, `port`) never names a secret. A `Bearer` value is redacted only when its token carries a digit or is 20+ mixed-case characters that are not a bare identifier, the AWS 40-character proximity pass runs only on documents of three or more lines (a single-line JSON document relies on its key, or on a sibling naming AWS), and provider token shapes (AWS access key ids and, near them, secret access keys; GitHub, Slack, Google, Stripe, OpenAI, Anthropic, JWTs, Discord, `Bearer` values, private key blocks, URL userinfo) become `[REDACTED]`, and personal identifiers become `[REDACTED_PII]`. `.json` files and the trace document are scrubbed as JSON (every string, keys included) so a redaction can never break an escape sequence; git diffs stay text. As on the backend, a `.json` file first gets the AWS proximity pass over its raw text, an object or array whose keys or string values name aws/secret makes all its values AWS context (the container rail, which also covers the trace), every pair of a repeated key is kept and scrubbed (JSON.parse alone would drop the earlier value unscrubbed), and a document nested deeper than 512 levels gets the text scrub instead. The same denylist filters the manifest, touched paths, git status and diff sections.++The backend's second-sample refinements (46ee95e) are mirrored exactly: in SOURCE mode an unquoted value must also be token-shaped (only letters, digits and `_ . / + = ~ -`, so `stagedClient?.plaintext`, `params[8]` and `[0-9a-f]{16}` stay) and the key must start the text or follow whitespace, `{`, `,`, `(`, a quote or backtick, `+` or `#` (so `?token=` or `pipeline-token:...` inside a string literal stays), while CONFIG keeps its word-start guard. 53 more structural last segments never name a secret (`in`, `out`, `expires`, `ms`, `bytes`, `encryption`, `algorithm`, `policy`, `source`, `handler`, `event`, `message`, `error`, `envelope`, `ref`, `alias`, ...; `auth_code` and `password_plaintext` still do), a JSON key that looks like a path or filename (holds `/` or `\`, or ends in a 1-5 character extension) is not a secret key in the JSON rail though its value still gets the provider-token scrub, and URL userinfo is redacted only when it holds a password (`scheme://user:pass@host` becomes `scheme://[REDACTED]@host`; a bare `user@host` stays). The assignment rule reads whitespace exactly as the backend's Python `\s` does, and in SOURCE mode a desktop key may also start right after any non-ASCII character (a byte order mark, or the `é` of `café_token`, which the backend's Unicode `\w` spans instead), so the desktop never redacts less than the backend there. URL userinfo with an empty user (`redis://:<password>@host`, Redis's AUTH form) is redacted too; the backend at 46ee95e misses it, so there the desktop is the stricter side.++## Snapshot scope, incremental captures and the `files_scope` field++Every envelope (contract v2) carries `files_scope`, which tells a consumer what `files[]` covers:++- `"full"`: `files[]` holds the content of every eligible file the snapshot cap allowed (the `start` snapshot of a session, whether new or resumed) or, for a `trace` envelope, its single trace document.+- `"changed"`: `files[]` holds only the files whose redacted sha256 differs from the previous snapshot the gateway accepted (delivered or spooled) for this session; a `change` or `end` snapshot. The envelope also carries `changed_paths`, the redacted paths of every such file, including any the snapshot cap kept out of `files[]`, and the `manifest` still lists every eligible file with its sha256 and size, so a consumer can tell a deleted file (gone from the manifest) from an unchanged one (same digest as before) without content. A snapshot that failed to upload advances nothing: the same files are sent again with the next one.++Neither field changes what may leave the machine: `files[]`, `changed_paths` and the manifest go through the same denylist and scrubber, and a manifest digest is always the sha256 of the redacted UTF-8 content.++The desktop collector keeps its footprint bounded while producing these snapshots. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path reached through a symlinked directory is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once.