Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

perf(collector): bound the workspace collector's memory and CPU; scrubber parity with the backend

ogsamratd0d474972da4
diff --git a/apps/server/src/workspace-collector.test.ts b/apps/server/src/workspace-collector.test.ts
index 03f77cfe..f4fe34d2 100644
--- a/apps/server/src/workspace-collector.test.ts
+++ b/apps/server/src/workspace-collector.test.ts
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, test } from "bun:test";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
-import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { appendFile, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import { promisify } from "node:util";
@@ -18,6 +18,7 @@ import {
MAX_COLLECTOR_TRACE_EVENTS,
MAX_COLLECTOR_WEB_VISIT_TEXT_BYTES,
WorkspaceCollector,
+ mapBounded,
clampCollectorBytes,
clampCollectorText,
collectGitBlock,
@@ -25,6 +26,9 @@ import {
filterCollectorDiff,
isCollectableWebUrl,
isCollectorPathDenied,
+ isPathLikeKey,
+ isSecretAssignmentKey,
+ isSecretAssignmentValue,
redactCollectorContent,
redactCollectorJson,
redactCollectorJsonText,
@@ -1344,6 +1348,20 @@ describe("workspace collector secret rails", () => {
expect(result.diff).not.toContain("hunter2abc");
});
+ test("redacts assignments and e-mails on whichever line they sit, never across a line break", () => {
+ // Both rules are applied only to lines holding their keyword or "@"; the
+ // result must equal a whole-text pass, line endings and escapes included.
+ expect(redactCollectorContent("notes.txt", "first line\npassword=hunter2hunter2\r\nlast")).toBe("first line\npassword=[REDACTED]\r\nlast");
+ expect(redactCollectorContent("notes.txt", "a@b\[email protected]\n@x\[email protected]")).toBe("a@b\n[REDACTED_PII]\n@x\n[REDACTED_PII]");
+ expect(redactCollectorContent("notes.txt", "jane@\nexample.com and key\n=abcdefgh12")).toBe("jane@\nexample.com and key\n=abcdefgh12");
+ expect(redactCollectorContent("run.sh", "echo done \\\nexport API_KEY=abc123def456\nTOKEN='S3cretValue99'"))
+ .toBe("echo done \\\nexport API_KEY=abc123def456\nTOKEN='[REDACTED]'");
+ expect(redactCollectorContent("app.ts", "const keyName = config.token;\nconst apiKey = \"sk-proj-abcdefghijklmnop1234\";\n// contact: [email protected]"))
+ .toBe("const keyName = config.token;\nconst apiKey = \"[REDACTED]\";\n// contact: [REDACTED_PII]");
+ expect(redactCollectorContent("data.json", "{\n \"note\": \"line\\npassword=hunter2hunter2\",\n \"author\": \"[email protected]\"\n}\n"))
+ .toBe("{\n \"note\": \"line\\npassword=[REDACTED]\",\n \"author\": \"[REDACTED_PII]\"\n}\n");
+ });
+
test("scrubs a 1 MiB line in bounded time", () => {
const mebibyte = 1024 * 1024;
const lines = [
@@ -1375,6 +1393,410 @@ describe("workspace collector secret rails", () => {
}, 60_000);
});
+// --- scrubber parity: the backend's second-sample refinements ---------------
+// Ported from the backend's tests/omnirush/test_collector.py (46ee95e) with the
+// same inputs and expected outputs; each test names the backend test it mirrors.
+
+// The excluded last segments added after the second production sample.
+const EXCLUDED_LAST_SEGMENTS_ADDED = [
+ "in", "out", "percent", "pct", "ms", "secs", "minutes", "hours", "days", "expires", "expiry", "expiration", "bytes", "len", "width",
+ "height", "offset", "index", "idx", "pos", "total", "sum", "avg", "ratio", "rate", "threshold", "weight", "score", "encryption",
+ "algorithm", "algo", "cipher", "strategy", "policy", "source", "target", "origin", "backend", "engine", "driver", "handler",
+ "callback", "event", "action", "reason", "message", "error", "envelope", "ref", "reference", "link", "pointer", "alias",
+];
+
+// Lines from the second production sample, each in the file type it came from.
+const SAMPLE2_TS = [
+ "const client = { client_token: stagedClient?.plaintext || null };",
+ "const apiKey = account?.managementKey || account?.apiKey;",
+ "const job = { secret_envelope: params[8] };",
+ "const pattern = /token=[0-9a-f]{16}/;",
+ "const cacheKey = 'pipeline-token:session:trial-123';",
+ "const grant = { refresh_token_expires_in: 15897600 };",
+ "const proxy = { provider_credential_ref: 'iproyal:claude-in-001' };",
+ "",
+].join("\n");
+const SAMPLE2_SH = "psql -c \"ALTER SYSTEM SET password_encryption='scram-sha-256'\"\n";
+const SAMPLE2_CONF = "password_encryption = 'scram-sha-256'\n";
+const SAMPLE2_YML = "proxies:\n - provider_credential_ref: 'iproyal:claude-in-001'\n refresh_token_expires_in: 15897600\n";
+const SAMPLE2_HASHES = `{"src/admin_auth.py": "${"3f2a9c1e".repeat(8)}", "src\\\\win\\\\auth.py": "${"b7d80e1f".repeat(8)}", "auth.py": "${"5c6d7e8f".repeat(8)}"}`;
+const SAMPLE2_FIXTURES = "LIVE_DSN = 'postgres://live@host:6432/db'\nSESSION_DSN = 'postgres://session@host/db'\n";
+const PATCH = [
+ "diff --git a/app/settings.py b/app/settings.py",
+ "--- a/app/settings.py",
+ "+++ b/app/settings.py",
+ "@@ -1,2 +1,3 @@",
+ " import config",
+ "+token = config.token",
+ "-password = None",
+ '+password = "hunter2abc"',
+ "",
+].join("\n");
+
+describe("workspace collector scrubber parity: second-sample refinements", () => {
+ test("secret-named keys follow the segment rule (test_secret_key_rule_qualifies / _does_not_qualify)", () => {
+ for (const key of ["auth_token", "basic-auth", "AUTH", "authorization", "oauth_client_secret", "APIKey", "accessKey", "HTTPSecret", "token2",
+ "secrets", "client.secret", "x-api-key"]) {
+ expect(isSecretAssignmentKey(key)).toBe(true);
+ }
+ for (const key of [
+ "author", "authors", "authored_by", "oauth_client_id", "tokenizer_class", "tokenize", "keyboard_layout", "keywords", "pwdir",
+ "accessKeyId", "token_url", "tokens", "additional_special_tokens", "key", "",
+ // The excluded last segments added with value rule v2.
+ "derived.token_estimate", "authMethod", "token_role", "token_owner", "credentials_file", "credential_hash", "githubOAuthConfig",
+ "CONTRACT_POOL_ATLAS_READ_TOKEN_FILE", "AUTH_TIMEOUT", "token_filename", "token_dir", "auth_mode", "token_type", "auth_kind",
+ "auth_enabled", "secret_digest", "token_expires_at", "auth_client", "api_key_prefix", "token_suffix", "token_format", "auth_scheme",
+ "auth_provider", "token_status", "auth_state", "auth_label", "secret_description", "token_title", "auth_class", "password_field",
+ "token_fields", "token_list", "secret_names", "token_version", "token_limit", "auth_max", "secret_min", "auth_interval",
+ "token_retries", "auth_port",
+ // The second production sample.
+ "refresh_token_expires_in", "password_encryption", "provider_credential_ref", "secret_envelope",
+ ]) {
+ expect(isSecretAssignmentKey(key)).toBe(false);
+ }
+ });
+
+ test("the added excluded last segments exclude a key only as its last segment (test_second_sample_excluded_last_segments)", () => {
+ for (const word of EXCLUDED_LAST_SEGMENTS_ADDED) {
+ const capitalized = word[0]!.toUpperCase() + word.slice(1);
+ expect(isSecretAssignmentKey(`token_${word}`)).toBe(false);
+ expect(isSecretAssignmentKey(`auth${capitalized}`)).toBe(false);
+ expect(isSecretAssignmentKey(`client.secret.${word}`)).toBe(false);
+ expect(isSecretAssignmentKey(`${word}_token`)).toBe(true);
+ // The same through the text rule, in both modes.
+ for (const mode of ["config", "source"] as const) {
+ expect(redactCollectorText(`token_${word} = "abcdefgh1234"`, { mode }).count).toBe(0);
+ expect(redactCollectorText(`${word}_token = "abcdefgh1234"`, { mode })).toEqual({ text: `${word}_token = "[REDACTED]"`, count: 1 });
+ }
+ }
+ });
+
+ test("auth_code and *_plaintext stay secret-named (test_auth_code_and_plaintext_keys_stay_secret_named)", () => {
+ const text = 'auth_code = "abcdefgh1234"\npassword_plaintext = "hunter2abc"\n# TOKEN = "abcdefgh1234"\n';
+ const result = redactCollectorText(text, { context: "app.py", mode: "source" });
+ expect(result.text).not.toContain("abcdefgh1234");
+ expect(result.text).not.toContain("hunter2abc");
+ expect(result.count).toBe(3);
+ expect(isSecretAssignmentKey("auth_code")).toBe(true);
+ expect(isSecretAssignmentKey("password_plaintext")).toBe(true);
+ });
+
+ test("path-like JSON keys (test_path_like_keys_follow_the_desktop_rule)", () => {
+ const cases: Array<[string, boolean]> = [
+ ["src/admin_auth.py", true], ["src\\admin_auth.py", true], ["admin_auth.py", true], ["auth.json", true], ["token.a1b2c", true],
+ ["C:\\x", true], ["auth_token", false], ["db.password", false], ["client.secret", false], ["auth.pyproject", false], ["token", false],
+ ["", false],
+ ];
+ for (const [key, expected] of cases) expect(isPathLikeKey(key)).toBe(expected);
+ // Python's `$` also matches before one trailing newline; the desktop mirrors it.
+ expect(isPathLikeKey("auth.py\n")).toBe(true);
+ expect(isPathLikeKey("auth.py\n\n")).toBe(false);
+ });
+
+ test("the scrub mode follows the file extension (test_scrub_mode_follows_the_file_extension)", () => {
+ for (const extension of ["js", "cjs", "mjs", "ts", "tsx", "jsx", "py", "go", "rs", "java", "kt", "c", "cc", "cpp", "h", "hpp", "rb", "php",
+ "swift", "cs", "scala", "sh", "bash", "zsh", "ps1", "lua", "dart", "vue", "svelte", "map", "patch", "diff"]) {
+ expect(redactModeForPath(`src/file.${extension}`)).toBe("source");
+ expect(redactModeForPath(`src/FILE.${extension.toUpperCase()}`)).toBe("source");
+ }
+ expect(redactModeForPath("dist/bundle.js.map")).toBe("source");
+ for (const path of [".env", ".env.local", ".bashrc", "Makefile", "Dockerfile", "index.", "a.tar.gz", "config.yml", "config.yaml", "pyproject.toml",
+ "package.json", "setup.cfg", "app.ini", "nginx.conf", "app.properties", "notes.txt", "README.md", "infra/main.tf", "build.gradle.kts",
+ "schema.sql", "index.html", "__omnirush__/trace.jsonl"]) {
+ expect(redactModeForPath(path)).toBe("config");
+ }
+ });
+
+ test("source value rule (test_source_value_rule_redacts_quoted_literals / _skips_references_paths_and_words)", () => {
+ for (const value of ["hunter2abc", "Abcdefghijklmnopqrstuvwxyz1234", "AbcdefghijklmnopqrsT", "12345678", "sk-testsecret123456789", AWS_SECRET]) {
+ expect(isSecretAssignmentValue(value, "source", true)).toBe(true);
+ }
+ const skipped: Array<[string, boolean]> = [
+ // No digit and under 20 characters of mixed case: a plain word.
+ ["synthetic", true], ["github-app", true], ["sample-owner", true], ["abcdefghijklmnopqrstuvwxyz", true], ["ABCDEFGHIJKLMNOPQRSTUVWXYZ", true],
+ ["AbcdefghijklmnopqrS", true],
+ // Unquoted identifier or member-expression syntax is a reference, whatever digits it holds.
+ ["estimate", false], ["githubOAuthEnabled", false], ["req.headers.authorization", false], ["config.token2", false], ["$scope.token1", false],
+ ["hunter2abc", false], ["Abcdefghijklmnopqrstuvwxyz1234", false],
+ // Filesystem paths.
+ ["/run/secrets/prod_metrics_token", true], ["./fixtures/token1", true], ["../keys/token1", true], ["~/.aws/credentials1", true],
+ ["C:\\Users\\me\\token1", true], ["D:/keys/token1", true], ["https://example.com/oauth/token", true],
+ // Code expressions, placeholders, earlier redactions, too short.
+ ["getToken()", false], ["${TOKEN}", false], ["abc${X}def123", true], ["$(cat token.txt)", false], ["process.env.TOKEN1", false],
+ ["os.environ1", false], ["env.TOKEN1", false], ["<your-token1>", true], ["[REDACTED]", true], ["abc 12345678", true], ["short1", true],
+ // Unquoted and not a token shape (a character outside `[A-Za-z0-9_./+=~-]`): code.
+ ["stagedClient?.plaintext", false], ["account?.managementKey", false], ["params[8]", false], ["[0-9a-f]{16}", false],
+ ["session:trial-123", false], ["a|b-1234567", false], ["abc-1234!", false], ["x*y-1234567", false], ["<abc-1234", false],
+ ["`abc-1234`", false],
+ ];
+ for (const [value, quoted] of skipped) expect(isSecretAssignmentValue(value, "source", quoted)).toBe(false);
+ });
+
+ test("unquoted source values that are token shapes still qualify (test_source_value_rule_keeps_unquoted_token_shapes)", () => {
+ for (const value of ["abc-123-def", "YWJjZGVmZ2hpams=", "a/b+c~12345", "12345678", AWS_SECRET]) {
+ expect(isSecretAssignmentValue(value, "source", false)).toBe(true);
+ expect(isSecretAssignmentValue(value, "source", true)).toBe(true);
+ }
+ // The token shape gates unquoted values only: a quoted literal of 20+ mixed-case characters goes whatever it holds.
+ expect(isSecretAssignmentValue("stagedClient?.plaintext", "source", true)).toBe(true);
+ });
+
+ test("a source key must follow a delimiter (test_source_key_must_follow_a_delimiter)", () => {
+ const cases: Array<[string, string | null]> = [
+ // A key after `/`, `:`, `[`, `?`, `&`, `=` or `|` is not an assignment in source...
+ ["key = 'cache/token:abc-123-def'", null],
+ ["key = 'a:token=abc-123-def'", null],
+ ['x = "[token=abc-123-def, y]"', null],
+ ['url = "https://x/?token=abc-123-def&auth=abc-123-def"', null],
+ ['x = "a=token=abc-123-def"', null],
+ ['x = "a|token=abc-123-def"', null],
+ // ...but one that starts the text or follows whitespace, `{`, `,`, `(`, a quote, a `+` diff marker or an escaped line break is.
+ ['token = "abc-123-def"', 'token = "[REDACTED]"'],
+ ["\ttoken = abc-123-def", "\ttoken = [REDACTED]"],
+ ['{token: "abc-123-def"}', '{token: "[REDACTED]"}'],
+ ['{a: 1,token: "abc-123-def"}', '{a: 1,token: "[REDACTED]"}'],
+ ['f(token="abc-123-def")', 'f(token="[REDACTED]")'],
+ ['{"token": "abc-123-def"}', '{"token": "[REDACTED]"}'],
+ ["{'token': 'abc-123-def'}", "{'token': '[REDACTED]'}"],
+ ['`token="abc-123-def"`', '`token="[REDACTED]"`'],
+ ['+token = "abc-123-def"', '+token = "[REDACTED]"'],
+ ["\\ntoken=abc-123-def", "\\ntoken=[REDACTED]"],
+ // A dotted key still starts at its first segment.
+ ['this.token = "hunter2abc"', 'this.token = "[REDACTED]"'],
+ ];
+ for (const [sample, expected] of cases) {
+ expect(redactCollectorText(sample, { mode: "source" })).toEqual(expected === null ? { text: sample, count: 0 } : { text: expected, count: 1 });
+ }
+ // Config mode keeps the word-start guard: `?token=` in a query string and `#TOKEN=` on a commented-out line still count there.
+ expect(redactCollectorText("https://x/?token=abc-123-def")).toEqual({ text: "https://x/?token=[REDACTED]", count: 1 });
+ expect(redactCollectorText("#TOKEN=abc-123-def")).toEqual({ text: "#TOKEN=[REDACTED]", count: 1 });
+ });
+
+ test("limit-style keys are excluded in both modes (test_limit_style_keys_are_excluded_in_both_modes)", () => {
+ for (const sample of [
+ "AUTH_TIMEOUT = 30000000", "auth_timeout: 30000000", "TOKEN_LIMIT = 10000000", "token_max = 12345678", "secret_min = abcdefgh",
+ "auth_interval = 60000000", "token_retries: 10000000", "AUTH_PORT = 80808080", "refresh_token_expires_in: 15897600",
+ "password_encryption='scram-sha-256'", "provider_credential_ref: 'iproyal:claude-in-001'", "secret_envelope: params[8]",
+ "token_expires_in = 15897600", "secret_algorithm = 'aes-256-gcm'", 'auth_callback = "https://x/cb?state=abc12345"',
+ ]) {
+ expect(redactCollectorText(sample, { mode: "source" })).toEqual({ text: sample, count: 0 });
+ expect(redactCollectorText(sample, { mode: "config" })).toEqual({ text: sample, count: 0 });
+ }
+ });
+
+ test("the second production sample comes back byte-identical (test_second_production_sample_is_byte_identical)", () => {
+ const files: Array<[string, string]> = [
+ ["src/client.ts", SAMPLE2_TS],
+ ["scripts/pg.sh", SAMPLE2_SH],
+ ["db/postgresql.conf", SAMPLE2_CONF],
+ ["config/proxies.yml", SAMPLE2_YML],
+ ["build/source-hashes.json", SAMPLE2_HASHES],
+ ["tests/fixtures.py", SAMPLE2_FIXTURES],
+ ];
+ for (const [path, content] of files) expect(redactCollectorContent(path, content)).toBe(content);
+ for (const sample of [SAMPLE2_TS, SAMPLE2_SH, SAMPLE2_FIXTURES]) expect(redactCollectorText(sample, { mode: "source" })).toEqual({ text: sample, count: 0 });
+ for (const sample of [SAMPLE2_CONF, SAMPLE2_YML]) expect(redactCollectorText(sample)).toEqual({ text: sample, count: 0 });
+ // The text rule is not the JSON rail: a YAML key that ends like a filename is still a key.
+ expect(redactCollectorText("admin_auth.py: abcdefgh1234")).toEqual({ text: "admin_auth.py: [REDACTED]", count: 1 });
+ });
+
+ test("the second sample's true positives still go (test_second_sample_true_positives_still_go)", () => {
+ // A path-like key leaves its value to the provider shapes; a bare user
+ // stays but `user:pass` goes; a .patch file is source, so `config.token`
+ // is a reference and `"hunter2abc"` a literal.
+ expect(redactCollectorContent("build/source-hashes.json", `{"src/x.py": "${GITHUB_TOKEN}"}`)).toBe('{"src/x.py":"[REDACTED]"}');
+ expect(redactCollectorJson({ "src/x.py": GITHUB_TOKEN })).toEqual({ value: { "src/x.py": "[REDACTED]" }, count: 1 });
+ expect(redactCollectorContent("tests/fixtures.py", "LIVE_DSN = 'postgres://live:secret@host/db'\n")).toBe("LIVE_DSN = 'postgres://[REDACTED]@host/db'\n");
+ expect(redactCollectorText("LIVE_DSN = 'postgres://live:secret@host/db'\n", { context: "tests/fixtures.py", mode: "source" }).count).toBe(1);
+ expect(redactCollectorContent("fix.patch", PATCH)).toBe(PATCH.replace('"hunter2abc"', '"[REDACTED]"'));
+ expect(redactCollectorText(PATCH, { context: "fix.patch", mode: redactModeForPath("fix.patch") }).count).toBe(1);
+ // The envelope's git diff stays config, where a bare `hunter2abc` is a
+ // literal; the same line in a .patch file is a name.
+ expect(filterCollectorDiff("diff --git a/app/settings.py b/app/settings.py\n+password = hunter2abc\n").diff)
+ .toBe("diff --git a/app/settings.py b/app/settings.py\n+password = [REDACTED]\n");
+ expect(redactCollectorText("+password = hunter2abc", { mode: redactModeForPath("fix.patch") })).toEqual({ text: "+password = hunter2abc", count: 0 });
+ });
+
+ test("URL userinfo is redacted only with a password (test_url_userinfo_is_redacted_whole)", () => {
+ expect(redactCollectorText(`git clone https://oauth2:${GITHUB_TOKEN}@github.com/acme/widgets.git`))
+ .toEqual({ text: "git clone https://[REDACTED]@github.com/acme/widgets.git", count: 1 });
+ expect(redactCollectorText("DATABASE_URL=postgres://admin:[email protected]:5432/app"))
+ .toEqual({ text: "DATABASE_URL=postgres://[REDACTED]@db.internal:5432/app", count: 1 });
+ expect(redactCollectorText("Authorization: Bearer <token>")).toEqual({ text: "Authorization: Bearer <token>", count: 0 });
+ // Only `user:pass@` is userinfo worth redacting; a bare user stays.
+ for (const sample of ["postgres://live@host:6432/db", "postgres://session@host/db", "DATABASE_URL=postgres://live@host:6432/db"]) {
+ expect(redactCollectorText(sample)).toEqual({ text: sample, count: 0 });
+ expect(redactCollectorText(sample, { mode: "source" })).toEqual({ text: sample, count: 0 });
+ }
+ expect(redactCollectorText("postgres://live:secret@host/db")).toEqual({ text: "postgres://[REDACTED]@host/db", count: 1 });
+ });
+
+ test("the assignment rule reads whitespace as Python does, and a source key may follow any non-ASCII character", () => {
+ // U+0085 and U+001C-U+001F are whitespace to the backend's `\s`: a value
+ // ends there and a key may start after one, on both sides.
+ expect(redactCollectorText("token=abc-123-def\u0085tail", { mode: "source" })).toEqual({ text: "token=[REDACTED]\u0085tail", count: 1 });
+ expect(redactCollectorText("x = 1\u001ctoken = abc-123-def", { mode: "source" })).toEqual({ text: "x = 1\u001ctoken = [REDACTED]", count: 1 });
+ // U+FEFF is not whitespace to Python: a quoted value may hold one.
+ expect(redactCollectorText('"token": "abc\ufeff12345"')).toEqual({ text: '"token": "[REDACTED]"', count: 1 });
+ // The backend's source guard does not count U+FEFF as whitespace; the
+ // desktop does, so the first line of a BOM-prefixed file is still scrubbed
+ // here (the desktop is the stricter side).
+ expect(redactCollectorContent("settings.py", '\ufeffpassword = "hunter2abc"\n')).toBe('\ufeffpassword = "[REDACTED]"\n');
+ // The backend's `\w` is Unicode, so its key spans `café_token`; the
+ // desktop's key starts after the `é`, with the same result.
+ expect(redactCollectorText("café_token = 'hunter2abc'", { mode: "source" })).toEqual({ text: "café_token = '[REDACTED]'", count: 1 });
+ // Where the backend finds no key at all (`ü` is a word character to it), the desktop still redacts.
+ expect(redactCollectorText("ütoken = 'hunter2abc'", { mode: "source" })).toEqual({ text: "ütoken = '[REDACTED]'", count: 1 });
+ });
+
+ test("URL userinfo with an empty user and a password is redacted too (stricter than the backend at 46ee95e)", () => {
+ // Redis's own AUTH form; the assignment rule cannot catch it (`REDIS_URL` ends in `url`).
+ expect(redactCollectorContent("config/app.env", "REDIS_URL=redis://:p4ssw0rd@cache:6379/0\n")).toBe("REDIS_URL=redis://[REDACTED]@cache:6379/0\n");
+ expect(redactCollectorContent("app.py", 'url = "rediss://:hunter2abc@cache:6380/0"\n')).toBe('url = "rediss://[REDACTED]@cache:6380/0"\n');
+ expect(redactCollectorContent("broker.yml", "broker: amqp://:s3cretpw@mq:5672//\n")).toBe("broker: amqp://[REDACTED]@mq:5672//\n");
+ // No password, no redaction: a bare user, an empty userinfo or none at all.
+ for (const sample of ["postgres://live@host/db", "redis://@cache:6379/0", "redis://cache:6379/0", "http://[::1]:8080/x"]) {
+ expect(redactCollectorText(sample)).toEqual({ text: sample, count: 0 });
+ }
+ });
+
+ test("JSON files keep the AWS proximity rule: the raw-text pass and the container rail (test_json_files_keep_the_aws_secret_proximity_rule)", () => {
+ // ECS task definitions, k8s env lists and Postman environments: the key
+ // naming the secret is a sibling value, not the value's own key.
+ const taskDefinition = JSON.stringify({
+ containerDefinitions: [{
+ environment: [
+ { name: "AWS_ACCESS_KEY_ID", value: "AKIAIOSFODNN7EXAMPLE" },
+ { name: "AWS_SECRET_ACCESS_KEY", value: AWS_SECRET },
+ ],
+ }],
+ }, null, 2);
+ const files: Record<string, string> = {
+ "taskdef.json": taskDefinition,
+ "taskdef.txt": taskDefinition,
+ "taskdef.yaml": taskDefinition,
+ "postman.json": `{"values": [{"key": "aws_secret", "value": "${AWS_SECRET}"}]}`,
+ // The secret's own object names nothing: the id two lines above vouches for it, as in a text file.
+ "pairs.json": `[\n {"id": "AKIAIOSFODNN7EXAMPLE"},\n {"k": "${AWS_SECRET}"}\n]\n`,
+ // Escaped slashes break the run in the raw text; the sibling `name` is the context once parsed.
+ "escaped.json": `{"name":"AWS_SECRET_ACCESS_KEY","value":"${AWS_SECRET.replaceAll("/", "\\/")}"}`,
+ // Nothing near names an AWS key: a 40-character run is data.
+ "plain.json": `{"hash": "${AWS_SECRET}"}`,
+ };
+ const out = Object.fromEntries(Object.entries(files).map(([path, content]) => [path, redactCollectorContent(path, content)]));
+ for (const [path, content] of Object.entries(out)) {
+ JSON.parse(content);
+ expect(content.includes(AWS_SECRET)).toBe(path === "plain.json");
+ }
+ expect(JSON.parse(out["taskdef.json"]!)).toEqual(JSON.parse(out["taskdef.txt"]!));
+ expect(JSON.parse(out["taskdef.json"]!).containerDefinitions[0].environment).toEqual([
+ { name: "AWS_ACCESS_KEY_ID", value: "[REDACTED]" },
+ { name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" },
+ ]);
+ expect(JSON.parse(out["postman.json"]!)).toEqual({ values: [{ key: "aws_secret", value: "[REDACTED]" }] });
+ expect(JSON.parse(out["pairs.json"]!)).toEqual([{ id: "[REDACTED]" }, { k: "[REDACTED]" }]);
+ expect(JSON.parse(out["escaped.json"]!)).toEqual({ name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" });
+ expect(out["plain.json"]).toBe(files["plain.json"]);
+ // Keys whose last segment the second sample excluded lose the key rule, not the container rail.
+ expect(redactCollectorContent("a.json", `{"secret_envelope": "${AWS_SECRET}"}`)).toBe('{"secret_envelope":"[REDACTED]"}');
+ expect(redactCollectorContent("a.json", `{"aws_secret_ref": "${AWS_SECRET}"}`)).toBe('{"aws_secret_ref":"[REDACTED]"}');
+ expect(redactCollectorContent("a.json", `{\n "db": {\n "secret_alias": "${AWS_SECRET}"\n }\n}`))
+ .toBe('{\n "db": {\n "secret_alias": "[REDACTED]"\n }\n}');
+ expect(redactCollectorContent("a.json", `{"src/secret.py": "${AWS_SECRET}"}`)).toBe('{"src/secret.py":"[REDACTED]"}');
+ });
+
+ test("trace AWS secrets need their key or a container naming AWS (test_trace_artifact_aws_secrets_need_key_or_container_context)", async () => {
+ const document = {
+ events: [
+ { type: "tool.call", data: { note: "id AKIAIOSFODNN7EXAMPLE" } },
+ { type: "tool.call", data: { note: AWS_SECRET } },
+ { type: "tool.call", data: { aws_secret: AWS_SECRET } },
+ { type: "tool.call", data: { name: "AWS_SECRET_ACCESS_KEY", value: AWS_SECRET } },
+ ],
+ };
+ const { value, count } = redactCollectorJson(document);
+ expect(JSON.parse(JSON.stringify(value)).events.map((event: { data: unknown }) => event.data)).toEqual([
+ { note: "id [REDACTED]" },
+ { note: AWS_SECRET },
+ { aws_secret: "[REDACTED]" },
+ { name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" },
+ ]);
+ expect(count).toBe(3);
+
+ // The uploaded trace document gets the same rail.
+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-trace-aws-"));
+ roots.push(root);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-trace-aws-1234";
+ collector.startSession(sessionId, "workspace-trace-aws", root);
+ await collector.idle(sessionId);
+ collector.recordTrace(sessionId, "tool.call", { name: "AWS_SECRET_ACCESS_KEY", value: AWS_SECRET });
+ collector.flushTrace(sessionId);
+ await collector.stop();
+ expect(JSON.stringify(uploads)).not.toContain(AWS_SECRET);
+ const trace = uploads.find((item) => item.snapshot_type === "trace")!;
+ expect(trace.trace?.find((event) => event.type === "tool.call")?.data).toEqual({ name: "AWS_SECRET_ACCESS_KEY", value: "[REDACTED]" });
+ });
+
+ test("JSON repeated keys are kept and scrubbed (test_json_repeated_keys_are_kept_and_scrubbed)", () => {
+ const deep = `{"a": 1, "a": ${"[".repeat(600)}${"]".repeat(600)}}`;
+ const cases: Array<[string, string]> = [
+ ['{"a": "[email protected]", "a": "clean"}', '{"a":"[REDACTED_PII]","a":"clean"}'],
+ [`{"a": "${GITHUB_TOKEN}", "a": "clean"}`, '{"a":"[REDACTED]","a":"clean"}'],
+ ['{"password": "hunter2abc", "password": "x"}', '{"password":"[REDACTED]","password":"x"}'],
+ [
+ '{"list": [{"k": "hunter2abc", "k": 1, "password": "hunter2abc"}], "n": 1.5, "t": true, "z": null, "u": "\\u00e9 \\"q\\""}',
+ '{"list":[{"k":"hunter2abc","k":1,"password":"[REDACTED]"}],"n":1.5,"t":true,"z":null,"u":"é \\"q\\""}',
+ ],
+ // Nothing to redact: byte-identical, repeated key and all.
+ ['{"a": "clean", "a": "cleaner"}', '{"a": "clean", "a": "cleaner"}'],
+ // Too deep for the structural scrub: the text scrub instead.
+ [deep, deep],
+ // A shadowed provider token, with nothing else to redact.
+ [`{"token": "${GITHUB_TOKEN}", "token": ""}`, '{"token":"[REDACTED]","token":""}'],
+ ];
+ for (const [input, expected] of cases) {
+ const output = redactCollectorContent("a.json", input);
+ expect(output).toBe(expected);
+ JSON.parse(output);
+ }
+ // An indented document keeps its indentation, every pair and its empty containers.
+ expect(redactCollectorContent("a.json", '{\n "x": {"password": "hunter2abc", "password": "y"},\n "e": [],\n "o": {}\n}\n'))
+ .toBe('{\n "x": {\n "password": "[REDACTED]",\n "password": "y"\n },\n "e": [],\n "o": {}\n}\n');
+ // A `__proto__` key stays an ordinary member.
+ expect(redactCollectorContent("a.json", '{"__proto__": {"password": "hunter2abc"}}')).toBe('{"__proto__":{"password":"[REDACTED]"}}');
+ });
+
+ test("a .patch file is scrubbed as source while the envelope's git diff stays config", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-patch-"));
+ roots.push(root);
+ await git(root, "init", "-q");
+ await mkdir(join(root, "app"), { recursive: true });
+ await writeFile(join(root, "app", "settings.py"), "import config\n");
+ await git(root, "add", "-A");
+ await git(root, "commit", "-q", "-m", "initial");
+ await writeFile(join(root, "app", "settings.py"), "import config\npassword = hunter2abc\n");
+ await writeFile(join(root, "fix.patch"), PATCH);
+ await writeFile(join(root, "notes.diff"), "+password = hunter2abc\n");
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, fallbackScanMs: 60_000 });
+ collector.startSession("session-patch-1", "workspace-patch", root);
+ await collector.stop();
+
+ const start = uploads.find((item) => item.snapshot_type === "start")!;
+ const content = (path: string) => start.files.find((file) => file.path === path)?.content;
+ // Workspace files: .py, .patch and .diff are source, so a bare name stays and a quoted literal goes.
+ expect(content("app/settings.py")).toBe("import config\npassword = hunter2abc\n");
+ expect(content("fix.patch")).toBe(PATCH.replace('"hunter2abc"', '"[REDACTED]"'));
+ expect(content("notes.diff")).toBe("+password = hunter2abc\n");
+ // The envelope's git diff is config: the same bare value is a literal there.
+ const gitBlock = start.workspace.git as { diff: string };
+ expect(gitBlock.diff).toContain("+password = [REDACTED]");
+ expect(gitBlock.diff).not.toContain("hunter2abc");
+ });
+});
+
// --- gateway auth: stale device tokens ---------------------------------------
describe("workspace collector gateway auth", () => {
@@ -1604,3 +2026,625 @@ describe("workspace collector snapshot cap", () => {
expect(warnings).not.toContain("OmniRush collection snapshot trimmed to the snapshot cap");
});
});
+
+// --- incremental snapshots, watcher discipline and memory --------------------
+
+describe("workspace collector incremental snapshots", () => {
+ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+ /** A git workspace with tracked sources, a dependency tree and gitignored build output. */
+ async function workspace(prefix: string, sourceFiles = 30): Promise<string> {
+ const root = await mkdtemp(join(tmpdir(), `omnirush-collector-${prefix}-`));
+ roots.push(root);
+ await git(root, "init", "-q");
+ await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\n");
+ await mkdir(join(root, "src", "lib"), { recursive: true });
+ await mkdir(join(root, "node_modules", "pkg"), { recursive: true });
+ await mkdir(join(root, "dist"), { recursive: true });
+ for (let index = 0; index < sourceFiles; index += 1) {
+ await writeFile(join(root, "src", index % 2 ? "lib" : "", `f${String(index).padStart(2, "0")}.txt`), `source file ${index}\n`);
+ }
+ await writeFile(join(root, "README.md"), "# readme\n");
+ await writeFile(join(root, "node_modules", "pkg", "index.js"), "module.exports = 1;\n");
+ await writeFile(join(root, "dist", "bundle.js"), "bundled\n");
+ await git(root, "add", "-A");
+ await git(root, "commit", "-q", "-m", "init");
+ return root;
+ }
+
+ const contentPaths = (envelope: Envelope) => envelope.files.map((file) => file.path).filter((path) => !path.startsWith("__omnirush__/"));
+ const changes = (uploads: Envelope[]) => uploads.filter((item) => item.snapshot_type === "change");
+ const triggerEvents = (uploads: Envelope[]) => uploads
+ .filter((item) => item.snapshot_type === "trace")
+ .flatMap((item) => item.trace ?? [])
+ .filter((event) => event.type === "collector.trigger")
+ .map((event) => event.data);
+
+ test("mapBounded keeps at most `limit` operations in flight and returns results in order", async () => {
+ let inFlight = 0;
+ let peak = 0;
+ const items = Array.from({ length: 50 }, (_, index) => index);
+ const results = await mapBounded(items, 8, async (item) => {
+ inFlight += 1;
+ peak = Math.max(peak, inFlight);
+ await sleep(1 + (item % 3));
+ inFlight -= 1;
+ return item * 2;
+ });
+ expect(results).toEqual(items.map((item) => item * 2));
+ expect(peak).toBe(8);
+ expect(await mapBounded([1, 2], 8, async (item) => item)).toEqual([1, 2]);
+ expect(await mapBounded([], 8, async (item: number) => item)).toEqual([]);
+ });
+
+ test("a change snapshot reads only the changed file and reuses digests across sessions on one root", async () => {
+ const root = await workspace("cache");
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const first = "session-cache-first-1";
+ collector.startSession(first, "workspace-cache", root);
+ await collector.idle(first);
+ const afterStart = { ...collector.metrics };
+ expect(afterStart.fullScans).toBe(1);
+ expect(afterStart.fileReads).toBeGreaterThanOrEqual(31);
+
+ await writeFile(join(root, "src", "f06.txt"), "source file 6, edited\n");
+ collector.captureSnapshot(first, "turn_completed");
+ await collector.idle(first);
+ const afterChange = { ...collector.metrics };
+ const change = changes(uploads)[0]!;
+ expect(contentPaths(change)).toEqual(["src/f06.txt"]);
+ expect(change.files_scope).toBe("changed");
+ expect(change.changed_paths).toEqual(["src/f06.txt"]);
+ expect(change.manifest).toHaveLength(32);
+ expect(change.manifest.find((entry) => entry.path === "src/f06.txt")?.sha256).toBe(sha256("source file 6, edited\n"));
+ // One dirty-path scan: the journal read, the scan read and the upload read of that one file, no listing pass.
+ expect(afterChange.fullScans).toBe(1);
+ expect(afterChange.dirtyScans).toBe(1);
+ expect(afterChange.fileReads - afterStart.fileReads).toBeLessThanOrEqual(3);
+ expect(afterChange.fileStats - afterStart.fileStats).toBeLessThanOrEqual(3);
+
+ // A second chat on the same workspace starts from the shared cache: a stat per file, no reads.
+ const second = "session-cache-second-1";
+ collector.startSession(second, "workspace-cache", root);
+ await collector.idle(second);
+ const afterSecond = { ...collector.metrics };
+ expect(afterSecond.fullScans).toBe(2);
+ const secondStart = uploads.find((item) => item.snapshot_type === "start" && item.session_id === second)!;
+ expect(secondStart.files_scope).toBe("full");
+ expect(contentPaths(secondStart)).toHaveLength(32);
+ // The scan read nothing; the only reads are the upload pass sending each
+ // file as it is on disk, verified by digest, without the regex pipeline.
+ expect(afterSecond.fileReads - afterChange.fileReads).toBe(32);
+ expect(afterSecond.fileRedactions - afterChange.fileRedactions).toBe(0);
+ expect(collector.cacheStatus()).toEqual({ roots: 1, entries: 32 });
+ await collector.stop();
+ expect(collector.cacheStatus()).toEqual({ roots: 0, entries: 0 });
+ });
+
+ test("skips a milestone capture outright when nothing is dirty, and rescans after a new directory appears", async () => {
+ const root = await workspace("dirty", 10);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-dirty-1234";
+ collector.startSession(sessionId, "workspace-dirty", root);
+ await collector.idle(sessionId);
+ expect(collector.sessionDiagnostics(sessionId)).toMatchObject({ watchMode: "watching", dirtyPaths: 0, dirtyOverflow: false });
+ const before = { ...collector.metrics };
+ collector.captureSnapshot(sessionId, "turn_completed");
+ collector.captureSnapshot(sessionId, "prompt");
+ await collector.idle(sessionId);
+ const after = { ...collector.metrics };
+ expect(after.capturesSkipped - before.capturesSkipped).toBe(2);
+ expect(after.fileStats - before.fileStats).toBe(0);
+ expect(after.fullScans + after.dirtyScans).toBe(before.fullScans + before.dirtyScans);
+ expect(changes(uploads)).toHaveLength(0);
+
+ // Files inside a directory that appears whole arrive without events of
+ // their own: the rename of the directory taints the dirty set and the next
+ // capture rescans the tree.
+ await mkdir(join(root, "feature"));
+ await writeFile(join(root, "feature", "new.txt"), "brand new\n");
+ await sleep(150);
+ expect(collector.sessionDiagnostics(sessionId)?.dirtyOverflow).toBe(true);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ expect(collector.metrics.fullScans).toBe(before.fullScans + 1);
+ expect(contentPaths(changes(uploads)[0]!)).toEqual(["feature/new.txt"]);
+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths).toContain("feature");
+ collector.flushTrace(sessionId);
+ await collector.stop();
+ expect(triggerEvents(uploads)).toEqual([
+ { trigger: "turn_completed", captured: false },
+ { trigger: "prompt", captured: false },
+ { trigger: "turn_completed", captured: true },
+ ]);
+ });
+
+ test("watches only directories with eligible files, batches ignore checks and polls past the watched-files cap", async () => {
+ const root = await workspace("watch", 6);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 60_000 });
+ const sessionId = "session-watch-1234";
+ collector.startSession(sessionId, "workspace-watch", root);
+ await collector.idle(sessionId);
+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths.sort()).toEqual([".", "src"]);
+ const before = { ...collector.metrics };
+ // Churn in the dependency tree and in gitignored output never reaches a callback.
+ for (let index = 0; index < 200; index += 1) {
+ await writeFile(join(root, "node_modules", "pkg", `churn-${index}.js`), `// ${index}\n`);
+ await writeFile(join(root, "dist", `chunk-${index}.js`), `// ${index}\n`);
+ }
+ await sleep(250);
+ await collector.idle(sessionId);
+ expect(collector.metrics.watchEvents - before.watchEvents).toBe(0);
+ expect(collector.metrics.ignoreCheckSpawns - before.ignoreCheckSpawns).toBe(0);
+ expect(changes(uploads)).toHaveLength(0);
+ await collector.stop();
+ expect(JSON.stringify(uploads)).not.toContain("churn-");
+ expect(JSON.stringify(uploads)).not.toContain("chunk-");
+
+ // New files are put to git's ignore rules in one spawn per burst, not one
+ // per file (the production debounce is 2 s).
+ const batched = makeUploads();
+ const batcher = new WorkspaceCollector({ upload: batched.upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const batchedId = "session-batched-1234";
+ batcher.startSession(batchedId, "workspace-batched", root);
+ await batcher.idle(batchedId);
+ await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\n*.log\n");
+ await sleep(120);
+ const beforeBurst = batcher.metrics.ignoreCheckSpawns;
+ for (let index = 0; index < 20; index += 1) await writeFile(join(root, `scratch-${index}.log`), `log ${index}\n`);
+ await writeFile(join(root, "kept.txt"), "kept\n");
+ await sleep(150);
+ batcher.captureSnapshot(batchedId, "turn_completed");
+ await batcher.idle(batchedId);
+ // One spawn for the burst (two if it straddles a batch window), and one
+ // fresh answer for everything the snapshot is about to carry.
+ expect(batcher.metrics.ignoreCheckSpawns - beforeBurst).toBeLessThanOrEqual(3);
+ const batchedChange = changes(batched.uploads).at(-1)!;
+ expect(contentPaths(batchedChange)).toContain("kept.txt");
+ expect(batchedChange.manifest.map((entry) => entry.path)).not.toContain("scratch-0.log");
+ await batcher.stop();
+ expect(JSON.stringify(batched.uploads)).not.toContain("scratch-");
+
+ // Past the cap the tree is polled: no watchers, and a milestone rescans it.
+ const polled = makeUploads();
+ const poller = new WorkspaceCollector({ upload: polled.upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000, maxWatchedFiles: 3 });
+ const polledId = "session-polled-1234";
+ poller.startSession(polledId, "workspace-polled", root);
+ await poller.idle(polledId);
+ expect(poller.sessionDiagnostics(polledId)).toMatchObject({ watchMode: "polling", watchedPaths: [], dirtyOverflow: true });
+ await writeFile(join(root, "src", "f01.txt"), "source file 1, polled edit\n");
+ poller.captureSnapshot(polledId, "turn_completed");
+ await poller.idle(polledId);
+ expect(poller.metrics.fullScans).toBe(2);
+ expect(contentPaths(changes(polled.uploads)[0]!)).toEqual(["src/f01.txt"]);
+ await poller.stop();
+ });
+
+ test("labels snapshot scope and re-sends changes the gateway never accepted", async () => {
+ const root = await workspace("scope", 4);
+ const stateDir = await mkdtemp(join(tmpdir(), "omnirush-collector-scope-state-"));
+ roots.push(stateDir);
+ let status = 201;
+ const uploads: Envelope[] = [];
+ const collector = new WorkspaceCollector({
+ stateDir,
+ upload: async (_sessionId, compressed) => {
+ if (status !== 201) return Response.json({ error: "rejected" }, { status });
+ uploads.push(JSON.parse(zstdDecompressSync(compressed).toString("utf8")) as Envelope);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ changeDebounceMs: 60_000,
+ fallbackScanMs: 60_000,
+ });
+ const sessionId = "session-scope-1234";
+ collector.startSession(sessionId, "workspace-scope", root);
+ await collector.idle(sessionId);
+ expect(uploads[0]).toMatchObject({ snapshot_type: "start", files_scope: "full" });
+ expect(uploads[0]).not.toHaveProperty("changed_paths");
+
+ await writeFile(join(root, "src", "f00.txt"), "rejected edit\n");
+ status = 400;
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ expect(changes(uploads)).toHaveLength(0);
+ status = 201;
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const resent = changes(uploads)[0]!;
+ expect(contentPaths(resent)).toEqual(["src/f00.txt"]);
+ expect(resent.changed_paths).toEqual(["src/f00.txt"]);
+ expect(resent.files_scope).toBe("changed");
+
+ collector.flushTrace(sessionId);
+ await collector.stop();
+ const end = uploads.find((item) => item.snapshot_type === "end")!;
+ expect(end.files_scope).toBe("changed");
+ expect(end.changed_paths).toEqual([]);
+ expect(uploads.find((item) => item.snapshot_type === "trace")?.files_scope).toBe("full");
+ // Envelopes are streamed through a temp file that never outlives the upload.
+ expect((await readdir(join(stateDir, "omnirush-collector-tmp")).catch(() => [])).filter((name) => name.endsWith(".tmp"))).toEqual([]);
+ expect(collector.sessionDiagnostics(sessionId)).toBeNull();
+ });
+
+ test("spaces filesystem-driven change snapshots by the minimum interval while milestones capture at once", async () => {
+ const root = await workspace("interval", 4);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 60_000, minChangeIntervalMs: 500 });
+ const sessionId = "session-interval-1234";
+ collector.startSession(sessionId, "workspace-interval", root);
+ await collector.idle(sessionId);
+ await writeFile(join(root, "src", "f00.txt"), "first burst\n");
+ await sleep(120);
+ await collector.idle(sessionId);
+ expect(changes(uploads)).toHaveLength(1);
+ const firstAt = Date.parse(changes(uploads)[0]!.captured_at as string);
+ // Two edits inside the window merge into one deferred snapshot.
+ await writeFile(join(root, "src", "f01.txt"), "second burst a\n");
+ await sleep(60);
+ await writeFile(join(root, "src", "f02.txt"), "second burst b\n");
+ await sleep(120);
+ await collector.idle(sessionId);
+ expect(changes(uploads)).toHaveLength(1);
+ expect(collector.metrics.capturesDeferred).toBeGreaterThanOrEqual(1);
+ await sleep(500);
+ await collector.idle(sessionId);
+ expect(changes(uploads)).toHaveLength(2);
+ expect(contentPaths(changes(uploads)[1]!).sort()).toEqual(["src/f01.txt", "src/f02.txt"]);
+ expect(Date.parse(changes(uploads)[1]!.captured_at as string) - firstAt).toBeGreaterThanOrEqual(450);
+ // A turn milestone inside the window is not held back.
+ await writeFile(join(root, "src", "f03.txt"), "milestone edit\n");
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ expect(changes(uploads)).toHaveLength(3);
+ expect(contentPaths(changes(uploads)[2]!)).toEqual(["src/f03.txt"]);
+ await collector.stop();
+ });
+
+ test("neither watches nor rescans a rebuilt gitignored directory, and rescans once .gitignore changes", async () => {
+ const root = await workspace("ignored", 6);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-ignored-1234";
+ collector.startSession(sessionId, "workspace-ignored", root);
+ await collector.idle(sessionId);
+ const before = { ...collector.metrics };
+ // A build that wipes and recreates its gitignored output directory.
+ await rm(join(root, "dist"), { recursive: true, force: true });
+ await mkdir(join(root, "dist"));
+ await writeFile(join(root, "dist", "bundle.js"), "rebuilt\n");
+ await sleep(200);
+ expect(collector.sessionDiagnostics(sessionId)).toMatchObject({ dirtyOverflow: false });
+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths).not.toContain("dist");
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ expect(collector.metrics.fullScans).toBe(before.fullScans);
+ expect(changes(uploads)).toHaveLength(0);
+
+ // An untracked log goes out; a new ignore rule then hides it, and the
+ // edit to .gitignore makes the next capture rescan the tree under it.
+ await writeFile(join(root, "notes.log"), "scratch notes\n");
+ await sleep(150);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ expect(changes(uploads).at(-1)!.manifest.map((entry) => entry.path)).toContain("notes.log");
+ await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\n*.log\n");
+ await sleep(150);
+ expect(collector.sessionDiagnostics(sessionId)?.dirtyOverflow).toBe(true);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const last = changes(uploads).at(-1)!;
+ expect(collector.metrics.fullScans).toBe(before.fullScans + 1);
+ expect(last.manifest.map((entry) => entry.path)).not.toContain("notes.log");
+ expect(last.changed_paths).toEqual([".gitignore"]);
+ await collector.stop();
+ });
+
+ test("drops the files of a directory moved out of the workspace without rescanning the tree", async () => {
+ const root = await workspace("moved", 10);
+ const outside = await mkdtemp(join(tmpdir(), "omnirush-collector-moved-out-"));
+ roots.push(outside);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-moved-1234";
+ collector.startSession(sessionId, "workspace-moved", root);
+ await collector.idle(sessionId);
+ const before = { ...collector.metrics };
+ // One event for the directory, none for the five files inside it.
+ await rename(join(root, "src", "lib"), join(outside, "lib"));
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const change = changes(uploads)[0]!;
+ const paths = change.manifest.map((entry) => entry.path);
+ expect(paths.filter((path) => path.startsWith("src/lib/"))).toEqual([]);
+ expect(paths).toContain("src/f00.txt");
+ expect(collector.metrics.fullScans).toBe(before.fullScans);
+ expect(collector.metrics.dirtyScans).toBe(before.dirtyScans + 1);
+ await collector.stop();
+ });
+
+ test("polls a workspace with more top-level directories than it watches", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-wide-"));
+ roots.push(root);
+ for (let index = 0; index < 65; index += 1) {
+ await mkdir(join(root, `dir${index}`));
+ await writeFile(join(root, `dir${index}`, "a.txt"), `file ${index}\n`);
+ }
+ const { upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-wide-1234";
+ collector.startSession(sessionId, "workspace-wide", root);
+ await collector.idle(sessionId);
+ expect(collector.sessionDiagnostics(sessionId)).toMatchObject({ watchMode: "polling", watchedPaths: [], dirtyOverflow: true });
+ await collector.stop();
+ });
+
+ test("keeps a file larger than the change journal out of it, and still snapshots the file", async () => {
+ const root = await workspace("journal", 4);
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-journal-1234";
+ collector.startSession(sessionId, "workspace-journal", root);
+ await collector.idle(sessionId);
+ await writeFile(join(root, "src", "small.txt"), "small edit\n");
+ await writeFile(join(root, "src", "large.txt"), "large log line\n".repeat(70_000));
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const change = changes(uploads)[0]!;
+ expect(contentPaths(change).sort()).toEqual(["src/large.txt", "src/small.txt"]);
+ const journal = change.files.find((file) => file.path === "__omnirush__/changes.json")!;
+ expect(journal.content).toContain('"path":"src/small.txt"');
+ expect(journal.content).not.toContain("src/large.txt");
+ await collector.stop();
+ });
+
+ test("runs one reconcile pass per interval for every session on a root, and it finds what no watcher reported", async () => {
+ const root = await workspace("reconcile", 4);
+ // Every watcher slot is taken, so assets/, which held no file at the start, gets none.
+ await fillWatchSlots(root);
+ await mkdir(join(root, "assets"));
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 300, minChangeIntervalMs: 0 });
+ const sessions = ["session-reconcile-a1", "session-reconcile-b1"];
+ for (const sessionId of sessions) collector.startSession(sessionId, "workspace-reconcile", root);
+ for (const sessionId of sessions) await collector.idle(sessionId);
+ expect(collector.sessionDiagnostics(sessions[0]!)).toMatchObject({ watchMode: "watching" });
+ expect(collector.sessionDiagnostics(sessions[0]!)?.watchedPaths).toHaveLength(65);
+ expect(collector.sessionDiagnostics(sessions[0]!)?.watchedPaths).not.toContain("assets");
+ const before = collector.metrics.reconciles;
+ await writeFile(join(root, "assets", "found.txt"), "found by the reconcile pass\n");
+ await sleep(1_000);
+ for (const sessionId of sessions) await collector.idle(sessionId);
+ const passes = collector.metrics.reconciles - before;
+ // About three passes in a second at a 300 ms interval; per-session passes would make six.
+ expect(passes).toBeGreaterThanOrEqual(1);
+ expect(passes).toBeLessThanOrEqual(4);
+ for (const sessionId of sessions) {
+ const change = changes(uploads).find((item) => item.session_id === sessionId);
+ expect(change && contentPaths(change)).toEqual(["assets/found.txt"]);
+ }
+ await collector.stop();
+ });
+
+ test("keeps gitignored files out of change snapshots and the journal in a workspace git does not manage", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-nogit-"));
+ roots.push(root);
+ await writeFile(join(root, ".gitignore"), "*.log\nbuild/\n");
+ await mkdir(join(root, "src"));
+ await writeFile(join(root, "src", "a.txt"), "a\n");
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-nogit-1234";
+ collector.startSession(sessionId, "workspace-nogit", root);
+ await collector.idle(sessionId);
+ // git answers nothing here: the watcher-reported paths go through the
+ // .gitignore files the listing walker applies.
+ await writeFile(join(root, "src", "debug.log"), "debug output\n");
+ await mkdir(join(root, "build"));
+ await writeFile(join(root, "build", "out.txt"), "build output\n");
+ await writeFile(join(root, "src", "b.txt"), "b\n");
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const change = changes(uploads)[0]!;
+ expect(contentPaths(change)).toEqual(["src/b.txt"]);
+ expect(change.manifest.map((entry) => entry.path).sort()).toEqual([".gitignore", "src/a.txt", "src/b.txt"]);
+ await collector.stop();
+ expect(JSON.stringify(uploads)).not.toContain("debug.log");
+ expect(JSON.stringify(uploads)).not.toContain("out.txt");
+ });
+
+ /** src/ plus 63 more top-level directories with files: every watcher slot taken. */
+ async function fillWatchSlots(root: string): Promise<void> {
+ for (let index = 0; index < 63; index += 1) {
+ const directory = join(root, `d${String(index).padStart(2, "0")}`);
+ await mkdir(directory);
+ await writeFile(join(directory, "a.txt"), `file ${index}\n`);
+ }
+ }
+
+ test("watches the top-level directories the listing did not name, unless git ignores or the denylist names them", async () => {
+ const root = await workspace("unlisted", 4);
+ await mkdir(join(root, "config"));
+ await writeFile(join(root, "config", ".env"), "API=1\n"); // denied, so config/ lists nothing
+ await mkdir(join(root, "scripts")); // empty
+ await mkdir(join(root, "secrets")); // a denied name
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 10, fallbackScanMs: 60_000, minChangeIntervalMs: 0 });
+ const sessionId = "session-unlisted-1234";
+ collector.startSession(sessionId, "workspace-unlisted", root);
+ await collector.idle(sessionId);
+ // Not dist/ (gitignored), node_modules/, secrets/ or .git/ (denied).
+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths.sort()).toEqual([".", "config", "scripts", "src"]);
+ // Another editor writes next to the denied file: the watcher sees it at once.
+ await writeFile(join(root, "config", "app.yml"), "mode: production\n");
+ await sleep(300);
+ await collector.idle(sessionId);
+ const change = changes(uploads)[0]!;
+ expect(change.trigger).toBe("fs_change");
+ expect(contentPaths(change)).toEqual(["config/app.yml"]);
+ await collector.stop();
+ expect(JSON.stringify(uploads)).not.toContain("API=1");
+ });
+
+ test("a turn's writes where no watcher reaches still land in that turn's change snapshot", async () => {
+ const root = await workspace("unwatched", 4);
+ await fillWatchSlots(root);
+ await mkdir(join(root, "assets"));
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-unwatched-1234";
+ collector.startSession(sessionId, "workspace-unwatched", root);
+ await collector.idle(sessionId);
+ expect(collector.sessionDiagnostics(sessionId)?.watchedPaths).not.toContain("assets");
+ collector.captureSnapshot(sessionId, "prompt");
+ await collector.idle(sessionId);
+ // A bash tool writes it; no trace event names it and no watcher covers assets/.
+ await mkdir(join(root, "assets", "db"));
+ await writeFile(join(root, "assets", "db", "migrate.sh"), "#!/bin/sh\necho migrate\n");
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ // The end-of-turn artifact scan marks it dirty for the turn's own snapshot.
+ expect(contentPaths(changes(uploads)[0]!)).toEqual(["assets/db/migrate.sh"]);
+ collector.flushTrace(sessionId);
+ await collector.stop();
+ expect(triggerEvents(uploads)).toEqual([
+ { trigger: "prompt", captured: false },
+ { trigger: "turn_completed", captured: true },
+ ]);
+ });
+
+ test.skipIf(process.platform === "win32")("a path git refuses to answer for (beyond a symlinked directory) neither changes the ignore answer for its batch nor gets read", async () => {
+ const base = await mkdtemp(join(tmpdir(), "omnirush-collector-symlink-"));
+ roots.push(base);
+ const root = join(base, "repo");
+ const outside = join(base, "shared-lib");
+ await mkdir(join(root, "src"), { recursive: true });
+ await mkdir(outside);
+ await git(root, "init", "-q");
+ await appendFile(join(root, ".git", "info", "exclude"), "private-notes.txt\n");
+ await writeFile(join(root, "src", "app.ts"), "export const app = 1;\n");
+ await writeFile(join(outside, "util.ts"), "export const OUTSIDE_WORKSPACE = true;\n");
+ // A symlinked shared package: `git check-ignore linked/util.ts` exits 128.
+ await symlink("../shared-lib", join(root, "linked"));
+ await git(root, "add", "-A");
+ await git(root, "commit", "-q", "-m", "init");
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = "session-symlink-1234";
+ collector.startSession(sessionId, "workspace-symlink", root);
+ await collector.idle(sessionId);
+ // In one ignore batch: a note git ignores only through .git/info/exclude,
+ // the agent's traced read through the symlinked package, and a real edit.
+ await writeFile(join(root, "private-notes.txt"), "PRIVATE: salary negotiation notes\n");
+ collector.recordTrace(sessionId, "tool.read", { filePath: "linked/util.ts" });
+ await writeFile(join(root, "src", "app.ts"), "export const app = 2;\n");
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const change = changes(uploads)[0]!;
+ expect(contentPaths(change)).toEqual(["src/app.ts"]);
+ expect(change.manifest.map((entry) => entry.path)).toEqual(["src/app.ts"]);
+ expect(change.files.find((file) => file.path === "__omnirush__/changes.json")?.content).toContain('"path":"src/app.ts"');
+ await collector.stop();
+ const all = JSON.stringify(uploads);
+ expect(all).not.toContain("salary negotiation");
+ expect(all).not.toContain("OUTSIDE_WORKSPACE");
+ });
+
+ test("a listed file the user then ignores leaves the manifest, files[] and the journal, through .gitignore or .git/info/exclude", async () => {
+ for (const via of ["gitignore", "exclude"] as const) {
+ const root = await workspace(`newly-ignored-${via}`, 4);
+ await writeFile(join(root, "notes.txt"), "scratch v1\n"); // untracked, not ignored yet
+ const { uploads, upload } = makeUploads();
+ const collector = new WorkspaceCollector({ upload, changeDebounceMs: 60_000, fallbackScanMs: 60_000 });
+ const sessionId = `session-newly-ignored-${via}`;
+ collector.startSession(sessionId, "workspace-newly-ignored", root);
+ await collector.idle(sessionId);
+ expect(uploads[0]!.manifest.map((entry) => entry.path)).toContain("notes.txt");
+ // An edit while it is still collected: git's "not ignored" is now cached.
+ await writeFile(join(root, "notes.txt"), "scratch v1b\n");
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ expect(contentPaths(changes(uploads)[0]!)).toEqual(["notes.txt"]);
+ // The user hides it from git (.git/info/exclude raises no event), then writes something private into it.
+ if (via === "gitignore") await writeFile(join(root, ".gitignore"), "dist/\nnode_modules/\nnotes.txt\n");
+ else await appendFile(join(root, ".git", "info", "exclude"), "notes.txt\n");
+ await sleep(150);
+ await writeFile(join(root, "notes.txt"), "PRIVATE: do not share v2\n");
+ await sleep(200);
+ collector.captureSnapshot(sessionId, "turn_completed");
+ await collector.idle(sessionId);
+ const change = changes(uploads)[1]!;
+ expect(change.manifest.map((entry) => entry.path)).not.toContain("notes.txt");
+ expect(contentPaths(change)).not.toContain("notes.txt");
+ await collector.stop();
+ expect(JSON.stringify(uploads)).not.toContain("do not share v2");
+ }
+ });
+
+ test("snapshots a 60 MiB workspace with peak RSS growth under 120 MB", async () => {
+ const root = await mkdtemp(join(tmpdir(), "omnirush-collector-memory-"));
+ roots.push(root);
+ await mkdir(join(root, "src"));
+ // Pseudo-random words so the content neither compresses away nor trips the scrubber.
+ const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"];
+ const filler = (seed: number, bytes: number): string => {
+ let out = "";
+ let value = seed;
+ while (out.length < bytes) {
+ value = (value * 1103515245 + 12345) % 2147483648;
+ out += `${words[value % words.length]} ${value.toString(36)}${value % 7 === 0 ? "\n" : " "}`;
+ }
+ return out.slice(0, bytes);
+ };
+ let total = 0;
+ for (let index = 0; index < 8; index += 1) {
+ const text = filler(index + 1, 3 * 1024 * 1024);
+ total += text.length;
+ await writeFile(join(root, "src", `large-${index}.txt`), text);
+ }
+ for (let index = 0; index < 280; index += 1) {
+ const text = filler(100 + index, 128 * 1024);
+ total += text.length;
+ await writeFile(join(root, "src", `small-${String(index).padStart(3, "0")}.txt`), text);
+ }
+ // Just under what the 64 MiB snapshot cap leaves for content.
+ expect(total).toBeGreaterThanOrEqual(58 * 1024 * 1024);
+
+ const compressedEnvelopes: Uint8Array[] = [];
+ const collector = new WorkspaceCollector({
+ upload: async (_sessionId, compressed) => {
+ compressedEnvelopes.push(compressed);
+ return Response.json({ ok: true }, { status: 201 });
+ },
+ changeDebounceMs: 60_000,
+ fallbackScanMs: 60_000,
+ });
+ const gc = (globalThis as { gc?: () => void }).gc ?? (globalThis as { Bun?: { gc?: (force: boolean) => void } }).Bun?.gc?.bind(null, true);
+ gc?.();
+ await sleep(50);
+ const baseline = process.memoryUsage().rss;
+ let peak = baseline;
+ const sampler = setInterval(() => { peak = Math.max(peak, process.memoryUsage().rss); }, 20);
+ const sessionId = "session-memory-1234";
+ collector.startSession(sessionId, "workspace-memory", root);
+ await collector.idle(sessionId);
+ peak = Math.max(peak, process.memoryUsage().rss);
+ clearInterval(sampler);
+ await collector.stop();
+
+ const growthMB = (peak - baseline) / (1024 * 1024);
+ expect(growthMB).toBeLessThan(120);
+ const start = JSON.parse(zstdDecompressSync(compressedEnvelopes[0]!).toString("utf8")) as Envelope;
+ expect(start.snapshot_type).toBe("start");
+ expect(start.files.reduce((sum, file) => sum + file.content.length, 0)).toBeGreaterThanOrEqual(58 * 1024 * 1024);
+ expect(start.privacy).toMatchObject({ files_truncated: false });
+ }, 60_000);
+});
diff --git a/apps/server/src/workspace-collector.ts b/apps/server/src/workspace-collector.ts
index 18acb4b5..b11ae498 100644
--- a/apps/server/src/workspace-collector.ts
+++ b/apps/server/src/workspace-collector.ts
@@ -1,11 +1,11 @@
import { createHash, randomBytes } from "node:crypto";
import { execFile, spawn } from "node:child_process";
-import { createReadStream, watch, type FSWatcher } from "node:fs";
-import { lstat, mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
-import { release as osRelease } from "node:os";
+import { createReadStream, createWriteStream, watch, type FSWatcher, type WriteStream } from "node:fs";
+import { lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { release as osRelease, tmpdir } from "node:os";
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
import { promisify } from "node:util";
-import { zstdCompress } from "node:zlib";
+import { createZstdCompress } from "node:zlib";
import { minimatch } from "minimatch";
import { externalFetch } from "./server-fetch.js";
@@ -52,7 +52,36 @@ const MAX_WEB_VISIT_TITLE_CHARS = 512;
const MAX_ATTACHMENT_NAME_CHARS = 512;
const MAX_ATTACHMENT_MIME_CHARS = 128;
const CHANGE_DEBOUNCE_MS = 2_000;
-const FALLBACK_SCAN_MS = 10_000;
+// While a workspace is watched, a reconcile pass (listing + lstat, no reads)
+// runs this often to catch events the watcher missed; a polled workspace (see
+// MAX_COLLECTOR_WATCHED_FILES) is rescanned on snapshots only, never on a timer.
+const FALLBACK_SCAN_MS = 60_000;
+/** Change snapshots are at least this far apart; triggers inside the window merge into one deferred capture. */
+export const MIN_COLLECTOR_CHANGE_INTERVAL_MS = 30_000;
+/** A workspace with more eligible files than this is polled on snapshots instead of watched. */
+export const MAX_COLLECTOR_WATCHED_FILES = 20_000;
+// Paths the watcher reported since the last capture. Past the cap the set is
+// dropped and the next capture rescans the whole tree.
+const MAX_DIRTY_PATHS = 8_192;
+// Top-level directories watched recursively; a workspace with more is polled.
+const MAX_WATCH_ROOTS = 64;
+// Reads and stats in flight during a scan, and how long synchronous scan work
+// (redaction, hashing) may run before the event loop gets a turn.
+const SCAN_CONCURRENCY = 8;
+const EVENT_LOOP_YIELD_MS = 12;
+// A read slot holds any ordinary source file; larger ones are read whole.
+const READ_SLOT_BYTES = 256 * 1024;
+// Watcher-reported paths the manifest does not know are put to git's ignore
+// rules in batches: one spawn per burst instead of one per event.
+const IGNORE_CHECK_BATCH_MS = 50;
+// A prompt or turn milestone can land milliseconds after the edit that
+// matters; the watcher delivers within tens of milliseconds, so a milestone
+// capture waits this long for it before consulting the dirty set.
+const MILESTONE_SETTLE_MS = 100;
+const IGNORE_CHECK_BATCH_MAX = 2_000;
+const MAX_IGNORED_CACHE_ENTRIES = 8_192;
+const TEMP_DIRECTORY = "omnirush-collector-tmp";
+const ENVELOPE_WRITE_CHUNK_BYTES = 64 * 1024;
const MAX_CHANGE_JOURNAL_ENTRIES = 512;
const MAX_CHANGE_JOURNAL_BYTES = 768 * 1024;
const MAX_SESSION_LEDGER_ENTRIES = 512;
@@ -107,12 +136,6 @@ type CollectorFile = {
sha256: string;
};
-type ManifestEntry = {
- path: string;
- sha256: string;
- size: number;
-};
-
type GitRemote = { name: string; url: string };
type GitCommit = { sha: string; at: string; subject: string };
type GitStatusEntry = { code: string; path: string };
@@ -228,7 +251,13 @@ type HashCacheEntry = {
mtimeMs: number;
binary: boolean;
sha256: string;
+ /** UTF-8 bytes of the redacted content: the manifest size. */
bytes: number;
+ /** Bytes of that content once JSON-encoded: what a files[] entry costs. */
+ json: number;
+ /** Redaction changed nothing, so the raw bytes are the upload form (verified by digest on upload). */
+ clean: boolean;
+ uploadPath: string;
};
type SpoolMeta = {
@@ -252,20 +281,42 @@ type SessionState = {
sequence: number;
resumed: boolean;
lastMessageId?: string;
- lastSignature: string;
started: boolean;
finished: boolean;
changeTimer: ReturnType<typeof setTimeout> | null;
changeTrigger: ChangeTrigger | null;
+ /** A capture held back by the minimum change interval, and the reason it will run for. */
+ deferTimer: ReturnType<typeof setTimeout> | null;
+ deferTrigger: ChangeTrigger | null;
+ /** The reconcile pass, while watching. */
scanTimer: ReturnType<typeof setInterval> | null;
- watcher: FSWatcher | null;
+ watchers: FSWatcher[];
+ watchMode: "starting" | "watching" | "polling";
+ /** Workspace-relative directories under a watcher ("." for the root). */
+ watchedPaths: string[];
+ /** Watchers still being added for top-level directories the listing did not name. */
+ watchSetup: Promise<void>;
trace: TraceEvent[];
changeJournal: Map<string, ChangeJournalEntry>;
touchedPaths: Set<string>;
changeJournalBytes: number;
changeCaptureTail: Promise<void>;
- hashCache: Map<string, HashCacheEntry>;
- baseline: Map<string, string> | null;
+ /** Paths whose journal capture is queued but not started. */
+ pendingJournal: Set<string>;
+ ignoreBatch: Map<string, Array<(ignored: boolean) => void>>;
+ ignoreTimer: ReturnType<typeof setTimeout> | null;
+ ignoredCache: Map<string, boolean>;
+ /** The last accepted manifest: what the next change snapshot is measured against. */
+ manifest: Manifest | null;
+ /** The per-root hash cache, shared with every other session open on the same root. */
+ cache: Map<string, HashCacheEntry>;
+ listing: { denied: number; truncated: boolean };
+ /** Paths the watcher (or a trace) reported since the last capture. */
+ dirty: Set<string>;
+ /** The dirty set cannot be trusted; the next capture scans the whole tree. */
+ dirtyOverflow: boolean;
+ lastHead: string | null;
+ lastChangeAt: number;
/** Untracked-but-not-ignored files as they stood when the current turn began. */
artifactBaseline: Promise<Map<string, ArtifactStat>> | null;
model: CollectorSessionModel | null;
@@ -303,6 +354,10 @@ type CollectorOptions = {
spoolMaxBytes?: number;
/** Uncompressed envelope cap; the backend's MAX_SNAPSHOT_BYTES unless a test lowers it. */
snapshotMaxBytes?: number;
+ /** Minimum spacing between change snapshots; MIN_COLLECTOR_CHANGE_INTERVAL_MS unless a test lowers it. */
+ minChangeIntervalMs?: number;
+ /** Watcher cap; MAX_COLLECTOR_WATCHED_FILES unless a test lowers it. */
+ maxWatchedFiles?: number;
};
type TransmitOutcome =
@@ -374,16 +429,42 @@ const PRIVATE_KEY_BLOCK: Redaction = [
* A backslash escape counts as one unit so the match never splits it. Whether
* the key names a secret is decided by isSecretAssignmentKey; the value by
* isSecretAssignmentValue. Every quantifier is bounded or anchored so a 1 MiB
- * line costs linear time.
+ * line costs linear time. Where the key may start depends on the mode: see
+ * ASSIGNMENT_PATTERN (CONFIG) and SOURCE_ASSIGNMENT_PATTERN (SOURCE).
*/
// Pre-filter for the assignment regex: the key must at least contain a keyword
// substring, so an ordinary assignment never consumes a value that holds a
// secret one (`"text": "{\"password\":...}"`). The exact, segment-bounded
// decision is isSecretAssignmentKey.
const ASSIGNMENT_KEYWORD_SOURCE = String.raw`secret|passw(?:or)?d|pwd|token|credential|auth|(?:api|access|private|client|session|signing|master|encryption)[_.:-]?key`;
-const ASSIGNMENT_PATTERN = new RegExp(
- String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))(\\?["']?)((?=[\w.-]{0,63}?(?:${ASSIGNMENT_KEYWORD_SOURCE}))[A-Za-z_-][\w.-]{0,63})\1[ \t]*[=:][ \t]*`
- + String.raw`(?:"((?:[^"\s\\]|\\\S){8,})"|'((?:[^'\s\\]|\\\S){8,})'|\\"((?:[^"\s\\]|\\[^"\s]){8,})\\"|((?:[^\s"',;&\\]|\\\S){8,}))`,
+// Python's `\s` (the backend's regexes run on str): JS `\s` minus U+FEFF, plus
+// U+001C-U+001F and U+0085. The assignment rule spells whitespace this way so a
+// value ends, and is rejected for holding whitespace, exactly where the
+// backend's does.
+const PYTHON_WHITESPACE = String.raw`\t\n\v\f\r \x1c-\x1f\x85\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000`;
+const PYTHON_WHITESPACE_CHAR = new RegExp(String.raw`[${PYTHON_WHITESPACE}]`);
+const ASSIGNMENT_BODY_SOURCE = String.raw`(\\?["']?)((?=[\w.-]{0,63}?(?:${ASSIGNMENT_KEYWORD_SOURCE}))[A-Za-z_-][\w.-]{0,63})\1[ \t]*[=:][ \t]*`
+ + String.raw`(?:"((?:[^"${PYTHON_WHITESPACE}\\]|\\[^${PYTHON_WHITESPACE}]){8,})"`
+ + String.raw`|'((?:[^'${PYTHON_WHITESPACE}\\]|\\[^${PYTHON_WHITESPACE}]){8,})'`
+ + String.raw`|\\"((?:[^"${PYTHON_WHITESPACE}\\]|\\[^"${PYTHON_WHITESPACE}]){8,})\\"`
+ + String.raw`|((?:[^${PYTHON_WHITESPACE}"',;&\\]|\\[^${PYTHON_WHITESPACE}]){8,}))`;
+// CONFIG mode: the key starts a word (not after a word character, `.`, `-` or
+// a backslash), so `#TOKEN=` on a commented-out line and `?token=` or
+// `&token=` in a query string still count.
+const ASSIGNMENT_PATTERN = new RegExp(String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))` + ASSIGNMENT_BODY_SOURCE, "gi");
+// SOURCE mode: the key starts the text or follows whitespace, `{`, `,`, `(`, a
+// quote (`"`, `'`, backtick), a `+` diff marker (a .patch file is source) or
+// `#`; never `-`, `.`, a word character or any other punctuation, so `token:`
+// after a `/`, `:` or `[` inside a string literal and `?token=` in a URL
+// literal are not read as assignments. Both modes accept a key right after a
+// `\n`, `\r` or `\t` escape. The backend's _SOURCE_ASSIGNMENT_PATTERN, except
+// that any non-ASCII character also counts as a delimiter: the backend's `\w`
+// is Unicode, so its key runs through `café_token` from the `c`, while a JS key
+// (ASCII `\w`) can only start after the `é`. This keeps every value the
+// backend redacts redacted here too (and a key right after a byte order mark,
+// the first line of a BOM-prefixed file, still counts on the desktop).
+const SOURCE_ASSIGNMENT_PATTERN = new RegExp(
+ String.raw`(?:(?<![^${PYTHON_WHITESPACE}\x80-\uffff{,("'\x60+#])|(?<=\\[nrt]))` + ASSIGNMENT_BODY_SOURCE,
"gi",
);
/**
@@ -408,6 +489,14 @@ const EXCLUDED_LAST_SEGMENTS = new Set([
"file", "filename", "dir", "mode", "method", "role", "owner", "type", "kind", "enabled", "estimate", "hash", "digest", "at",
"config", "client", "prefix", "suffix", "format", "scheme", "provider", "status", "state", "label", "description", "title",
"class", "field", "fields", "list", "names", "version", "timeout", "limit", "max", "min", "interval", "retries", "port",
+ // Added after the second production sample (`refresh_token_expires_in`,
+ // `password_encryption`, `provider_credential_ref`, `secret_envelope`).
+ // `code` and `plaintext` are deliberately absent: `auth_code` and
+ // `password_plaintext` name secrets.
+ "in", "out", "percent", "pct", "ms", "secs", "minutes", "hours", "days", "expires", "expiry", "expiration", "bytes", "len",
+ "width", "height", "offset", "index", "idx", "pos", "total", "sum", "avg", "ratio", "rate", "threshold", "weight", "score",
+ "encryption", "algorithm", "algo", "cipher", "strategy", "policy", "source", "target", "origin", "backend", "engine", "driver",
+ "handler", "callback", "event", "action", "reason", "message", "error", "envelope", "ref", "reference", "link", "pointer", "alias",
]);
/**
@@ -415,13 +504,14 @@ const EXCLUDED_LAST_SEGMENTS = new Set([
* extension names a programming language is scrubbed in SOURCE mode, where an
* assignment's value must look like a generated literal; everything else
* (.env*, ini, cfg, conf, yml, yaml, toml, json, properties, txt, md, no or
- * unknown extension), plus git diffs and the trace, is scrubbed in CONFIG
- * mode with the permissive rule.
+ * unknown extension), plus the envelope's git diff and the trace, is scrubbed
+ * in CONFIG mode with the permissive rule. A `.patch` or `.diff` workspace
+ * file carries code and is SOURCE.
*/
export type RedactMode = "source" | "config";
const SOURCE_EXTENSIONS = new Set([
"js", "cjs", "mjs", "ts", "tsx", "jsx", "py", "go", "rs", "java", "kt", "c", "cc", "cpp", "h", "hpp", "rb", "php", "swift",
- "cs", "scala", "sh", "bash", "zsh", "ps1", "lua", "dart", "vue", "svelte", "map",
+ "cs", "scala", "sh", "bash", "zsh", "ps1", "lua", "dart", "vue", "svelte", "map", "patch", "diff",
]);
/** SOURCE for a programming-language extension (case-insensitive), CONFIG otherwise. */
@@ -433,11 +523,23 @@ export function redactModeForPath(path: string): RedactMode {
const CODE_EXPRESSION_VALUE = /^(?:process\.|os\.|env\.)/;
const IDENTIFIER_VALUE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/;
const PATH_VALUE = /^(?:\/|\.\/|\.\.\/|~\/|[A-Za-z]:[\\/])/;
+// The shape of an unquoted SOURCE value that can be a literal token: letters,
+// digits and `_ . / + = ~ -` only (`abc-123-def`, base64 with `+/=`). A `?`,
+// `:`, `[`, `]`, `!`, `|`, `&`, `,`, `;`, `)`, quote, backtick, `*`, `<` or
+// `>` makes it code (`stagedClient?.plaintext`, `params[8]`, `[0-9a-f]{16}`,
+// `session:trial-123`).
+const TOKEN_SHAPE_VALUE = /^[A-Za-z0-9_./+=~-]+$/;
+// A JSON key that looks like a file path or filename: holds a `/` or `\`, or
+// ends in a dot extension of one to five letters or digits. `\n?` mirrors
+// Python's `$`, which also matches before one trailing newline.
+const PATH_LIKE_KEY = /[/\\]|\.[A-Za-z0-9]{1,5}\n?$/;
const MAX_ASSIGNMENT_DEPTH = 4;
// An AWS secret access key is 40 base64 characters with no shape of its own,
// so a candidate only counts near an access key id or an aws/secret key name.
-const AWS_SECRET_QUICK = /[A-Za-z0-9/+]{40}/;
+// The quick check only tries a position that starts a run of those characters,
+// so it costs one pass over the text instead of one per character of every run.
+const AWS_SECRET_QUICK = /(?<![A-Za-z0-9/+])[A-Za-z0-9/+]{40}/;
const AWS_SECRET_CANDIDATE = tokenPattern(String.raw`[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+])`, "g", String.raw`[A-Za-z0-9/+\\]`);
const AWS_SECRET_CONTEXT = /(?:AKIA|ASIA)[0-9A-Z]{16}|aws|secret/i;
const AWS_SECRET_CONTEXT_LINES = 3;
@@ -445,10 +547,19 @@ const AWS_SECRET_CONTEXT_LINES = 3;
// document (compact JSON, the trace) relies on its JSON key instead.
const AWS_SECRET_MIN_LINES = 3;
-// Runs before the assignment rule so `https://oauth2:<token>@host/...` keeps
-// its host instead of being read as an `oauth2:` assignment.
+// `scheme://user:pass@host`: the userinfo is redacted only when it holds a
+// password (a `:` after the user), replaced whole by `[REDACTED]`; a bare
+// `postgres://live@host/db` in a fixture stays. The user may be empty:
+// `redis://:<password>@host` is Redis's own AUTH form. Runs before the
+// assignment rule so `https://oauth2:<token>@host/...` keeps its host instead
+// of being read as an `oauth2:` assignment. Whitespace is Python's, as in the
+// backend's _URL_USERINFO_TEXT; unlike it (at 46ee95e), the user part may be
+// empty, so the desktop is the stricter side there.
const URL_USERINFO: Redaction = [
- tokenPattern(String.raw`([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/@\\]|\\\S)+@`, "gi"),
+ tokenPattern(
+ String.raw`([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^${PYTHON_WHITESPACE}/@:\\]|\\[^${PYTHON_WHITESPACE}])*:(?:[^${PYTHON_WHITESPACE}/@\\]|\\[^${PYTHON_WHITESPACE}])+@`,
+ "gi",
+ ),
(_match, scheme) => `${scheme}${REDACTED}@`,
];
@@ -632,7 +743,8 @@ function keySegments(key: string): string[] {
.filter(Boolean);
}
-function isSecretAssignmentKey(key: string): boolean {
+/** Whether an assignment key (or JSON object key) names a secret; the backend's is_secret_assignment_key. */
+export function isSecretAssignmentKey(key: string): boolean {
const segments = keySegments(key);
const last = segments.at(-1);
if (last === undefined || EXCLUDED_LAST_SEGMENTS.has(last)) return false;
@@ -640,6 +752,20 @@ function isSecretAssignmentKey(key: string): boolean {
|| SECRET_KEY_PAIRS.some(([first, second]) => segment === `${first}${second}` || (segment === first && segments[index + 1] === second)));
}
+/**
+ * Whether a JSON object key looks like a file path or filename: it holds a `/`
+ * or `\`, or ends in `.` plus one to five letters or digits
+ * (`src/admin_auth.py`, `admin_auth.py`, `C:\x`). Such a key never names a
+ * secret (a source-hashes document maps paths to digests), so the JSON rail
+ * skips the key rule for it; its value still gets the text scrub, so a
+ * provider shape under it is caught. The text rule is unaffected:
+ * `admin_auth.py: abcdefgh1234` in YAML is still an assignment. The backend's
+ * is_path_like_key.
+ */
+export function isPathLikeKey(key: string): boolean {
+ return PATH_LIKE_KEY.test(key);
+}
+
/**
* A filesystem path: an absolute, relative, home or drive-letter prefix, or a
* `/` anywhere with no digit anywhere (`/run/secrets/x`, `C:/Users/me`).
@@ -656,20 +782,24 @@ function isFilesystemPathValue(value: string): boolean {
* `<|endoftext|>`), not an earlier redaction and not a filesystem path.
* SOURCE mode then wants a literal that looks generated: an unquoted value
* with identifier or member-expression syntax is a reference (`config.token`,
- * `tokenFile`, `req.headers.authorization`), never a literal, and the value
+ * `tokenFile`, `req.headers.authorization`), never a literal; an unquoted value
+ * must also be a token shape (TOKEN_SHAPE_VALUE: `stagedClient?.plaintext`,
+ * `params[8]`, `[0-9a-f]{16}` and `session:trial-123` are code); and the value
* must carry a digit or be 20+ characters of mixed case, so `'synthetic'`
* and `'github-app'` stay. CONFIG mode keeps the permissive rule (a letter
* anywhere), skipping only an unquoted dotted identifier such as YAML's
- * `token: config.token`.
+ * `token: config.token`. The backend's is_secret_assignment_value.
*/
-function isSecretAssignmentValue(value: string, mode: RedactMode, quoted: boolean): boolean {
- if (value.length < 8 || /\s/.test(value) || value.startsWith("[REDACTED")) return false;
+export function isSecretAssignmentValue(value: string, mode: RedactMode, quoted: boolean): boolean {
+ if (value.length < 8 || PYTHON_WHITESPACE_CHAR.test(value) || value.startsWith("[REDACTED")) return false;
if (value.includes("(") || value.includes("${") || value.includes("$(") || CODE_EXPRESSION_VALUE.test(value)) return false;
if (value.startsWith("<") && value.endsWith(">")) return false;
if (isFilesystemPathValue(value)) return false;
const identifier = !quoted && IDENTIFIER_VALUE.test(value);
if (mode === "source") {
- return !identifier && (/\d/.test(value) || (value.length >= 20 && /[a-z]/.test(value) && /[A-Z]/.test(value)));
+ return !identifier
+ && (quoted || TOKEN_SHAPE_VALUE.test(value))
+ && (/\d/.test(value) || (value.length >= 20 && /[a-z]/.test(value) && /[A-Z]/.test(value)));
}
return /[A-Za-z]/.test(value) && !(identifier && value.includes("."));
}
@@ -695,8 +825,9 @@ function applyRedaction(text: string, [pattern, replacement]: Redaction, tally:
/** Redacts the value of every secret-named assignment; one redaction per assignment. */
function redactAssignments(text: string, tally: { count: number }, mode: RedactMode, depth = 0): string {
- ASSIGNMENT_PATTERN.lastIndex = 0;
- return text.replace(ASSIGNMENT_PATTERN, (match: string, _quote: string, key: string, doubleQuoted?: string, singleQuoted?: string, escapedQuoted?: string, bare?: string) => {
+ const pattern = mode === "source" ? SOURCE_ASSIGNMENT_PATTERN : ASSIGNMENT_PATTERN;
+ pattern.lastIndex = 0;
+ return text.replace(pattern, (match: string, _quote: string, key: string, doubleQuoted?: string, singleQuoted?: string, escapedQuoted?: string, bare?: string) => {
const value = doubleQuoted ?? singleQuoted ?? escapedQuoted ?? bare ?? "";
const quote = doubleQuoted !== undefined ? '"' : singleQuoted !== undefined ? "'" : escapedQuoted !== undefined ? '\\"' : "";
const prefix = match.slice(0, match.length - value.length - quote.length * 2);
@@ -725,18 +856,25 @@ function redactBearerTokens(text: string, tally: { count: number }): string {
* Redacts 40-character mixed-case base64 runs that sit within three lines of
* an AWS access key id or of the words aws/secret (or whose surrounding
* context, such as the file path or JSON key, names them). Only a document of
- * three or more lines (two or more line breaks) is scanned.
+ * three or more lines (two or more line breaks) is scanned, unless the
+ * enclosing JSON object or array names an AWS key (`awsContext`, the JSON
+ * container rail), which vouches for a single string. The backend's
+ * _redact_aws_secrets.
*/
-function redactAwsSecrets(text: string, context: string | undefined, tally: { count: number }): string {
+function redactAwsSecrets(text: string, context: string | undefined, tally: { count: number }, awsContext = false): string {
+ const nearContext = awsContext || (context !== undefined && AWS_SECRET_CONTEXT.test(context));
+ // No context anywhere in the text or around it means no line can qualify.
+ // Both checks must pass; this one goes first because a literal scan is
+ // many times cheaper than looking for a 40-character run.
+ if (!nearContext && !AWS_SECRET_CONTEXT.test(text)) return text;
if (!AWS_SECRET_QUICK.test(text)) return text;
const lines = text.split("\n");
- if (lines.length < AWS_SECRET_MIN_LINES) return text;
+ if (lines.length < AWS_SECRET_MIN_LINES && !awsContext) return text;
const contextual: Array<boolean | undefined> = new Array(lines.length);
const hasContext = (index: number): boolean => {
if (contextual[index] === undefined) contextual[index] = AWS_SECRET_CONTEXT.test(lines[index]!);
return contextual[index]!;
};
- const nearContext = context !== undefined && AWS_SECRET_CONTEXT.test(context);
let changed = false;
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index]!;
@@ -756,11 +894,76 @@ function redactAwsSecrets(text: string, context: string | undefined, tally: { co
return changed ? lines.join("\n") : text;
}
+// Substring gates for the text scrubber: each rule needs a literal the text
+// must contain, so a text without it skips that regex pass outright. A gate is
+// sound (it never skips a text its rule could match) and changes nothing about
+// what is redacted; it only spares the scan's dominant CPU cost. A pattern
+// without a gate always runs.
+// The assignment rule's key must contain one of these (case-insensitively).
+const ASSIGNMENT_LINE_GATE = /secret|passw|pwd|token|credential|auth|key/gi;
+const BEARER_GATE = /bearer/i;
+const DIGIT_GATE = /\d/;
+const DISCORD_TOKEN_GATE = /[MN][\w-]{23,}\./;
+const SECRET_PATTERN_GATES: Array<(text: string) => boolean> = [
+ (text) => text.includes("AKIA") || text.includes("ASIA"),
+ (text) => /gh[pousr]_/.test(text),
+ (text) => text.includes("github_pat_"),
+ (text) => /xox[abpr]-/.test(text),
+ (text) => text.includes("AIza"),
+ (text) => /[sr]k_(?:live|test)_/.test(text),
+ (text) => text.includes("sk-ant-"),
+ (text) => /(?:sk|rk|pk)-/.test(text),
+ (text) => text.includes("eyJ"),
+ (text) => DISCORD_TOKEN_GATE.test(text),
+];
+// Every literal the gates above look for, in one scan: when none occurs (and
+// no Discord-shaped run either) no provider-token rule can run, so the text
+// each later gate would see is unchanged and the whole loop is skipped.
+const PROVIDER_TOKEN_LITERALS = /AKIA|ASIA|gh[pousr]_|github_pat_|xox[abpr]-|AIza|[sr]k_(?:live|test)_|sk-ant-|(?:sk|rk|pk)-|eyJ/;
+
+/**
+ * Runs `redact` over each line holding a gate match and leaves every other
+ * line as it is. Sound only for a rule whose matches never contain a line
+ * break and always contain a gate match: the assignment and e-mail rules admit
+ * no "\n" in any character class, their keys (or "@") are part of the match's
+ * line, and their lookbehinds see a line start's "\n" exactly as they see the
+ * start of the text. The result is identical to running the rule over the
+ * whole text; only lines that could match pay for the rule's regex.
+ */
+function redactGatedLines(text: string, nextGate: (from: number) => number, redact: (line: string) => string): string {
+ let output = "";
+ let copied = 0;
+ let changed = false;
+ let at = nextGate(0);
+ while (at !== -1) {
+ const start = text.lastIndexOf("\n", at) + 1;
+ const newline = text.indexOf("\n", at);
+ const end = newline === -1 ? text.length : newline;
+ const line = text.slice(start, end);
+ const redacted = redact(line);
+ if (redacted !== line) {
+ output += text.slice(copied, start) + redacted;
+ copied = end;
+ changed = true;
+ }
+ if (newline === -1) break;
+ at = nextGate(newline + 1);
+ }
+ return changed ? output + text.slice(copied) : text;
+}
+
+function nextAssignmentKeyword(text: string, from: number): number {
+ ASSIGNMENT_LINE_GATE.lastIndex = from;
+ return ASSIGNMENT_LINE_GATE.exec(text)?.index ?? -1;
+}
+
export type RedactCollectorTextOptions = {
/** Text that counts as context for context-dependent rules: the file path, or the JSON key a value sits under. */
context?: string;
/** Value rule mode; CONFIG when omitted (git diffs, the trace, JSON). See redactModeForPath. */
mode?: RedactMode;
+ /** The enclosing JSON object or array names an AWS key: the AWS rule runs whatever the line count. */
+ awsContext?: boolean;
};
/**
@@ -770,68 +973,259 @@ export type RedactCollectorTextOptions = {
*/
export function redactCollectorText(input: string, options: RedactCollectorTextOptions = {}): { text: string; count: number } {
const tally = { count: 0 };
- let text = applyRedaction(input, PRIVATE_KEY_BLOCK, tally);
- text = applyRedaction(text, URL_USERINFO, tally);
- text = redactAssignments(text, tally, options.mode ?? "config");
- text = redactAwsSecrets(text, options.context, tally);
- for (const redaction of SECRET_PATTERNS) text = applyRedaction(text, redaction, tally);
- text = redactBearerTokens(text, tally);
- for (const redaction of PII_PATTERNS) text = applyRedaction(text, redaction, tally);
+ let text = input.includes("-----BEGIN ") ? applyRedaction(input, PRIVATE_KEY_BLOCK, tally) : input;
+ if (text.includes("://")) text = applyRedaction(text, URL_USERINFO, tally);
+ const mode = options.mode ?? "config";
+ const assigned = text;
+ text = redactGatedLines(assigned, (from) => nextAssignmentKeyword(assigned, from), (line) => redactAssignments(line, tally, mode));
+ text = redactAwsSecrets(text, options.context, tally, options.awsContext);
+ if (PROVIDER_TOKEN_LITERALS.test(text) || DISCORD_TOKEN_GATE.test(text)) {
+ SECRET_PATTERNS.forEach((redaction, index) => {
+ const gate = SECRET_PATTERN_GATES[index];
+ if (!gate || gate(text)) text = applyRedaction(text, redaction, tally);
+ });
+ }
+ if (BEARER_GATE.test(text)) text = redactBearerTokens(text, tally);
+ // The e-mail rule needs an "@" on the line; the phone, SSN and IP rules need a digit.
+ const mailed = text;
+ text = redactGatedLines(mailed, (from) => mailed.indexOf("@", from), (line) => applyRedaction(line, PII_PATTERNS[0]!, tally));
+ if (DIGIT_GATE.test(text)) {
+ for (const redaction of PII_PATTERNS.slice(1)) text = applyRedaction(text, redaction, tally);
+ }
return { text, count: tally.count };
}
-function redactJsonValue(value: unknown, key: string | undefined, tally: { count: number }): unknown {
+/**
+ * A JSON object with a repeated key, kept as its (key, value) pairs in order.
+ * JSON.parse keeps only the last value of a repeated key, so a secret in a
+ * shadowed value (merge-conflict leftovers, generated configs) would pass the
+ * structural scrub untouched and, with nothing else to redact, leave the
+ * machine byte for byte. The backend's _Pairs.
+ */
+class JsonPairs {
+ constructor(readonly pairs: Array<[string, unknown]>) {}
+}
+
+/** A `.json` file nested deeper than this is scrubbed as text instead (the backend's MAX_TRACE_DEPTH). */
+const MAX_JSON_FILE_DEPTH = 512;
+
+class JsonTooDeep extends Error {}
+
+/** Redaction count, plus the object members visited: a repeated key shows up as a shortfall against the text. */
+type JsonTally = { count: number; members: number };
+
+/** Whether a JSON item is a string naming an AWS key. */
+function namesAws(item: unknown): boolean {
+ return typeof item === "string" && AWS_SECRET_CONTEXT.test(item);
+}
+
+/**
+ * Whether an object's keys or direct string values, or an array's string
+ * items, name an AWS key. The backend's _names_aws.
+ */
+function membersNameAws(members: ReadonlyArray<readonly [string, unknown]>): boolean {
+ return members.some(([key, item]) => namesAws(key) || namesAws(item));
+}
+
+function redactJsonValue(value: unknown, key: string | undefined, tally: JsonTally, depth = 0, awsContext = false, maxDepth = Infinity): unknown {
+ if (depth > maxDepth) throw new JsonTooDeep();
if (typeof value === "string") {
// A JSON string is always a quoted literal, scrubbed in CONFIG mode.
- if (key !== undefined && isSecretAssignmentKey(key) && isSecretAssignmentValue(value, "config", true)) {
+ // A path-like key (`{"src/admin_auth.py": "<sha256>"}`) names a file, never a secret.
+ if (key !== undefined && !isPathLikeKey(key) && isSecretAssignmentKey(key) && isSecretAssignmentValue(value, "config", true)) {
tally.count += 1;
return REDACTED;
}
- const result = redactCollectorText(value, { context: key });
+ const result = redactCollectorText(value, { context: key, awsContext });
tally.count += result.count;
return result.text;
}
if (!value || typeof value !== "object") return value;
- if (Array.isArray(value)) return value.map((item) => redactJsonValue(item, key, tally));
+ // The enclosing object or array is AWS context once one of its keys or
+ // string items names an AWS key, the structural counterpart of the text
+ // rule's three-line window: `{"name": "AWS_SECRET_ACCESS_KEY", "value":
+ // "<40 characters>"}` (task definitions, k8s env lists, Postman
+ // environments) is redacted although `value` names nothing.
+ if (Array.isArray(value)) {
+ const context = awsContext || value.some(namesAws);
+ return value.map((item) => redactJsonValue(item, key, tally, depth + 1, context, maxDepth));
+ }
+ if (value instanceof JsonPairs) {
+ tally.members += value.pairs.length;
+ const context = awsContext || membersNameAws(value.pairs);
+ return new JsonPairs(value.pairs.map(([childKey, childValue]) => {
+ const scrubbedKey = redactCollectorText(childKey, { awsContext: context });
+ tally.count += scrubbedKey.count;
+ return [scrubbedKey.text, redactJsonValue(childValue, childKey, tally, depth + 1, context, maxDepth)];
+ }));
+ }
const serializable = value as { toJSON?: unknown };
- if (typeof serializable.toJSON === "function") return redactJsonValue((serializable.toJSON as () => unknown)(), key, tally);
- const output: Record<string, unknown> = {};
- for (const [childKey, childValue] of Object.entries(value as Record<string, unknown>)) {
- const scrubbedKey = redactCollectorText(childKey);
+ if (typeof serializable.toJSON === "function") return redactJsonValue((serializable.toJSON as () => unknown)(), key, tally, depth, awsContext, maxDepth);
+ const entries = Object.entries(value as Record<string, unknown>);
+ tally.members += entries.length;
+ const context = awsContext || membersNameAws(entries);
+ // No prototype, so a `__proto__` key stays an ordinary member.
+ const output: Record<string, unknown> = Object.create(null);
+ for (const [childKey, childValue] of entries) {
+ const scrubbedKey = redactCollectorText(childKey, { awsContext: context });
tally.count += scrubbedKey.count;
- output[scrubbedKey.text] = redactJsonValue(childValue, childKey, tally);
+ output[scrubbedKey.text] = redactJsonValue(childValue, childKey, tally, depth + 1, context, maxDepth);
}
return output;
}
/**
* Scrubs a JSON value structurally: every string (keys included) goes through
- * the text scrubber, and a string under a secret-named key is redacted whole,
- * exactly as the `key: value` text rule would treat it. Serialising the result
- * can never produce a broken escape, which scrubbing serialised JSON as text
- * could.
+ * the text scrubber, a string under a secret-named key is redacted whole,
+ * exactly as the `key: value` text rule would treat it, and an object or array
+ * naming an AWS key makes its values AWS context. Serialising the result can
+ * never produce a broken escape, which scrubbing serialised JSON as text could.
+ * The backend's _redact_json_value.
*/
export function redactCollectorJson(value: unknown): { value: unknown; count: number } {
- const tally = { count: 0 };
+ const tally: JsonTally = { count: 0, members: 0 };
return { value: redactJsonValue(value, undefined, tally), count: tally.count };
}
/**
- * Scrubs a JSON document as JSON, keeping its indentation style; the original
- * text comes back untouched when nothing needed redacting, and null when the
- * text is not valid JSON.
+ * Object members in valid JSON text: every `:` outside a string separates one.
+ * Fewer members in the parsed value than here means a repeated key.
+ */
+function countJsonMembers(text: string): number {
+ let members = 0;
+ let inString = false;
+ for (let index = 0; index < text.length; index += 1) {
+ const code = text.charCodeAt(index);
+ if (inString) {
+ if (code === 0x5c) index += 1;
+ else if (code === 0x22) inString = false;
+ } else if (code === 0x22) {
+ inString = true;
+ } else if (code === 0x3a) {
+ members += 1;
+ }
+ }
+ return members;
+}
+
+const JSON_STRING_TOKEN = /"[^"\\]*(?:\\.[^"\\]*)*"/y;
+const JSON_NUMBER_TOKEN = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
+const JSON_WHITESPACE = /[ \t\n\r]*/y;
+
+/**
+ * Parses text JSON.parse has already accepted, keeping every object as its
+ * (key, value) pairs so a repeated key loses nothing. Only used for a document
+ * that has one: JSON.parse is several times faster.
+ */
+function parseJsonPairs(text: string): unknown {
+ let at = 0;
+ const token = (pattern: RegExp): string => {
+ pattern.lastIndex = at;
+ const match = pattern.exec(text);
+ if (!match) throw new SyntaxError(`unexpected JSON at ${at}`);
+ at = pattern.lastIndex;
+ return match[0];
+ };
+ const skip = () => token(JSON_WHITESPACE);
+ const value = (depth: number): unknown => {
+ if (depth > MAX_JSON_FILE_DEPTH) throw new JsonTooDeep();
+ skip();
+ const char = text[at];
+ if (char === "{") {
+ at += 1;
+ const pairs: Array<[string, unknown]> = [];
+ skip();
+ if (text[at] === "}") {
+ at += 1;
+ return new JsonPairs(pairs);
+ }
+ for (;;) {
+ skip();
+ const key = JSON.parse(token(JSON_STRING_TOKEN)) as string;
+ skip();
+ at += 1; // ":"
+ pairs.push([key, value(depth + 1)]);
+ skip();
+ if (text[at++] === "}") return new JsonPairs(pairs);
+ }
+ }
+ if (char === "[") {
+ at += 1;
+ const items: unknown[] = [];
+ skip();
+ if (text[at] === "]") {
+ at += 1;
+ return items;
+ }
+ for (;;) {
+ items.push(value(depth + 1));
+ skip();
+ if (text[at++] === "]") return items;
+ }
+ }
+ if (char === '"') return JSON.parse(token(JSON_STRING_TOKEN)) as string;
+ for (const [literal, parsed] of [["true", true], ["false", false], ["null", null]] as const) {
+ if (text.startsWith(literal, at)) {
+ at += literal.length;
+ return parsed;
+ }
+ }
+ return Number(token(JSON_NUMBER_TOKEN));
+ };
+ return value(0);
+}
+
+/** JSON.stringify(value, null, indent) for a value that may hold JsonPairs. */
+function stringifyJsonPairs(value: unknown, indent: string | undefined, current = ""): string {
+ const container = (items: string[], open: string, close: string): string => {
+ if (items.length === 0) return `${open}${close}`;
+ if (!indent) return `${open}${items.join(",")}${close}`;
+ const inner = `${current}${indent}`;
+ return `${open}\n${inner}${items.join(`,\n${inner}`)}\n${current}${close}`;
+ };
+ const nested = `${current}${indent ?? ""}`;
+ const member = ([key, item]: [string, unknown]) => `${JSON.stringify(key)}:${indent ? " " : ""}${stringifyJsonPairs(item, indent, nested)}`;
+ if (value instanceof JsonPairs) return container(value.pairs.map(member), "{", "}");
+ if (Array.isArray(value)) return container(value.map((item) => stringifyJsonPairs(item, indent, nested)), "[", "]");
+ if (value && typeof value === "object") return container(Object.entries(value).map(member), "{", "}");
+ return JSON.stringify(value);
+}
+
+/**
+ * Scrubs a JSON document as JSON, keeping its indentation style. The AWS rule
+ * runs over the raw text first, where it sees the same lines it sees in a text
+ * file; the document is then scrubbed value by value (repeated keys kept, pair
+ * by pair). The text comes back untouched when nothing needed redacting (the
+ * AWS pass aside), and null when it is not valid JSON or is nested deeper than
+ * MAX_JSON_FILE_DEPTH, which the caller scrubs as text. The backend's
+ * sanitize_json_file.
*/
-export function redactCollectorJsonText(text: string): string | null {
+export function redactCollectorJsonText(text: string, path?: string): string | null {
+ const raw = redactAwsSecrets(text, path, { count: 0 });
let parsed: unknown;
try {
- parsed = JSON.parse(text);
+ parsed = JSON.parse(raw);
} catch {
return null;
}
- const result = redactCollectorJson(parsed);
- if (result.count === 0) return text;
- const indent = /\n([ \t]+)\S/.exec(text)?.[1]?.slice(0, 10);
- return `${JSON.stringify(result.value, null, indent)}${text.endsWith("\n") ? "\n" : ""}`;
+ const tally: JsonTally = { count: 0, members: 0 };
+ let value: unknown;
+ let pairs = false;
+ try {
+ value = redactJsonValue(parsed, undefined, tally, 0, false, MAX_JSON_FILE_DEPTH);
+ if (tally.members !== countJsonMembers(raw)) {
+ pairs = true;
+ tally.count = 0;
+ value = redactJsonValue(parseJsonPairs(raw), undefined, tally, 0, false, MAX_JSON_FILE_DEPTH);
+ }
+ } catch (error) {
+ if (error instanceof JsonTooDeep) return null;
+ throw error;
+ }
+ if (tally.count === 0) return raw;
+ const indent = /\n([ \t]+)\S/.exec(raw)?.[1]?.slice(0, 10);
+ const serialized = pairs ? stringifyJsonPairs(value, indent) : JSON.stringify(value, null, indent);
+ return `${serialized}${raw.endsWith("\n") ? "\n" : ""}`;
}
/**
@@ -840,11 +1234,16 @@ export function redactCollectorJsonText(text: string): string | null {
* as context and the value rule mode the extension selects.
*/
export function redactCollectorContent(path: string, text: string): string {
+ return redactCollectorContentCounted(path, text).text;
+}
+
+/** redactCollectorContent plus how many redactions it made (0 means the text is byte-identical). */
+function redactCollectorContentCounted(path: string, text: string): { text: string; count: number } {
if (/\.json$/i.test(path)) {
- const json = redactCollectorJsonText(text);
- if (json !== null) return json;
+ const json = redactCollectorJsonText(text, path);
+ if (json !== null) return { text: json, count: json === text ? 0 : 1 };
}
- return redactCollectorText(text, { context: path, mode: redactModeForPath(path) }).text;
+ return redactCollectorText(text, { context: path, mode: redactModeForPath(path) });
}
/**
@@ -863,13 +1262,15 @@ export function stripRemoteUserinfo(url: string): string {
}
function isBinary(buffer: Buffer): boolean {
- const sample = buffer.subarray(0, Math.min(buffer.length, 8_192));
- if (sample.includes(0)) return true;
+ const length = Math.min(buffer.length, 8_192);
let suspicious = 0;
- for (const byte of sample) {
+ // An indexed loop: iterating a Buffer is several times slower, once per file.
+ for (let index = 0; index < length; index += 1) {
+ const byte = buffer[index]!;
+ if (byte === 0) return true;
if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;
}
- return sample.length > 0 && suspicious / sample.length > 0.1;
+ return length > 0 && suspicious / length > 0.1;
}
const PATH_FIELD_NAMES = new Set([
@@ -914,6 +1315,16 @@ async function readGitignoreFile(directory: string): Promise<string[]> {
}
}
+/** A directory's .gitignore rules re-rooted at the workspace, as walkFallback applies them. */
+function scopedIgnoreRules(prefix: string, localRules: string[]): string[] {
+ return localRules.map((rule) => {
+ const negated = rule.startsWith("!");
+ const body = negated ? rule.slice(1) : rule;
+ const scoped = prefix ? `${prefix}/${body}` : body;
+ return negated ? `!${scoped}` : scoped;
+ });
+}
+
function ignoredByRules(path: string, rules: string[]): boolean {
let ignored = false;
for (const raw of rules) {
@@ -925,18 +1336,128 @@ function ignoredByRules(path: string, rules: string[]): boolean {
return ignored;
}
-async function gitIgnoresPath(root: string, path: string): Promise<boolean> {
- try {
- await execFileAsync("git", ["-C", root, "check-ignore", "--no-index", "-q", "--", path], {
- timeout: 5_000,
- maxBuffer: 64 * 1024,
+/**
+ * Asks git which of `paths` its ignore rules cover, in one process: the
+ * ignored paths come back NUL-separated. Null when git cannot answer (not a
+ * repository, missing, timed out): the caller applies the .gitignore files
+ * itself, exactly as the listing walker does.
+ */
+function gitIgnoredPaths(root: string, paths: string[]): Promise<Set<string> | null> {
+ return new Promise((resolvePromise) => {
+ const ignored = new Set<string>();
+ if (paths.length === 0) {
+ resolvePromise(ignored);
+ return;
+ }
+ let child: ReturnType<typeof spawn>;
+ try {
+ child = spawn("git", ["-C", root, "check-ignore", "--no-index", "-z", "--stdin"], {
+ stdio: ["pipe", "pipe", "ignore"],
+ windowsHide: true,
+ env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" },
+ });
+ } catch {
+ resolvePromise(null);
+ return;
+ }
+ const chunks: Buffer[] = [];
+ let total = 0;
+ let settled = false;
+ // Exit 0: some paths are ignored; 1: none are; anything else: no answer.
+ const finish = (answered: boolean) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ if (!answered) {
+ resolvePromise(null);
+ return;
+ }
+ for (const path of Buffer.concat(chunks).toString("utf8").split("\0")) {
+ if (path) ignored.add(path);
+ }
+ resolvePromise(ignored);
+ };
+ const timer = setTimeout(() => child.kill("SIGKILL"), 10_000);
+ timer.unref?.();
+ child.stdout?.on("data", (chunk: Buffer) => {
+ if (total >= 16 * 1024 * 1024) return;
+ chunks.push(chunk);
+ total += chunk.length;
});
- return true;
- } catch {
- // Exit status 1 means the path is not ignored. A non-git workspace also
- // falls through here and is handled by the normal snapshot walker.
- return false;
+ child.on("error", () => finish(false));
+ child.on("close", (code) => finish(code === 0 || code === 1));
+ child.stdin?.on("error", () => undefined);
+ child.stdin?.end(`${paths.join("\0")}\0`);
+ });
+}
+
+/**
+ * gitIgnoredPaths for a batch in which git may refuse single paths: one path
+ * beyond a symbolic link (`linked/util.ts`) makes the whole batch exit 128. A
+ * refused batch is split in halves until every refusal is a single path, so
+ * one bad path never changes the answer for the others (and never sends them
+ * to the .gitignore-only fallback, which knows nothing of `.git/info/exclude`
+ * or `core.excludesFile`). A path git refuses on its own counts as ignored:
+ * it is never collected. Null only when the workspace is not a git work tree
+ * (or git is missing), where the caller applies the .gitignore files itself.
+ * `onSpawn` counts each `git check-ignore` run.
+ */
+async function gitIgnoredPathsIsolated(root: string, paths: string[], onSpawn: () => void): Promise<Set<string> | null> {
+ onSpawn();
+ const whole = await gitIgnoredPaths(root, paths);
+ if (whole) return whole;
+ if ((await gitText(root, ["rev-parse", "--is-inside-work-tree"])) !== "true") return null;
+ const ignored = new Set<string>();
+ const isolate = async (refused: string[]): Promise<void> => {
+ if (refused.length === 1) {
+ ignored.add(refused[0]!);
+ return;
+ }
+ const middle = refused.length >> 1;
+ for (const half of [refused.slice(0, middle), refused.slice(middle)]) {
+ onSpawn();
+ const answer = await gitIgnoredPaths(root, half);
+ if (!answer) await isolate(half);
+ else for (const path of answer) ignored.add(path);
+ }
+ };
+ await isolate(paths);
+ return ignored;
+}
+
+/**
+ * The listing walker's ignore decision for individual paths, for a workspace
+ * git cannot answer for: a path is ignored when it, or any directory above
+ * it, matches the .gitignore rules in force at that level.
+ */
+async function fallbackIgnoredPaths(root: string, paths: string[]): Promise<Set<string>> {
+ const rulesByDirectory = new Map<string, Promise<string[]>>();
+ const rulesAt = (directory: string): Promise<string[]> => {
+ let rules = rulesByDirectory.get(directory);
+ if (!rules) {
+ const slash = directory.lastIndexOf("/");
+ const parent = slash === -1 ? "" : directory.slice(0, slash);
+ rules = (async () => [
+ ...(directory === "" ? [] : await rulesAt(parent)),
+ ...scopedIgnoreRules(directory, await readGitignoreFile(resolve(root, directory))),
+ ])();
+ rulesByDirectory.set(directory, rules);
+ }
+ return rules;
+ };
+ const ignored = new Set<string>();
+ for (const path of paths) {
+ let directory = "";
+ for (const part of pathComponents(path)) {
+ const current = directory ? `${directory}/${part}` : part;
+ if (ignoredByRules(current, await rulesAt(directory))) {
+ ignored.add(path);
+ break;
+ }
+ directory = current;
+ }
}
+ return ignored;
}
function optionalString(value: unknown): string | undefined {
@@ -1016,15 +1537,17 @@ function boundedTracePayload(
): Buffer {
// The trace is scrubbed as a JSON value, never as serialised text: a text
// redaction inside an escaped string (`\[email protected]`) once left a
- // dangling backslash that made the whole document unparseable.
- const header = redactCollectorJson({
+ // dangling backslash that made the whole document unparseable. Header and
+ // events are scrubbed as the one document the backend scrubs, so a header
+ // value naming an AWS key is container context for the events there too.
+ const { events: scrubbed, ...header } = redactCollectorJson({
schema_version: 1,
session_id: state.id,
workspace_id: state.workspaceId,
session_segment: state.segment,
session_resumed: state.resumed,
- }).value as Record<string, unknown>;
- const scrubbed = redactCollectorJson(events).value as TraceEvent[];
+ events,
+ }).value as Record<string, unknown> & { events: TraceEvent[] };
const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => Buffer.from(JSON.stringify({
...header,
trace_truncated: truncated,
@@ -1033,17 +1556,26 @@ function boundedTracePayload(
}));
// One encode settles the common case. Only an oversized trace pays for the
- // search, which bisects on the prefix length: every probe is a full
- // serialisation, so probing once per event stalled the event loop for
- // seconds at the MAX_COLLECTOR_TRACE_EVENTS cap.
+ // search, and that search measures each event once: the payload of a prefix
+ // is its wrapper plus the events' serialised bytes and the commas between
+ // them, so the longest prefix that fits follows from a running total. Every
+ // probe used to be a full serialisation of the prefix, which made a flush
+ // at the MAX_COLLECTOR_TRACE_EVENTS cap peak at many times the payload.
let payload = encode(scrubbed, false);
if (payload.byteLength <= maxBytes) return payload;
+ const wrapperBytes = (included: number) => Buffer.byteLength(JSON.stringify({
+ ...header,
+ trace_truncated: true,
+ dropped_event_count: Math.max(0, events.length - included),
+ events: [],
+ }));
let fits = 0;
- let overflow = scrubbed.length;
- while (overflow - fits > 1) {
- const middle = Math.floor((fits + overflow) / 2);
- if (encode(scrubbed.slice(0, middle), true).byteLength <= maxBytes) fits = middle;
- else overflow = middle;
+ let eventBytes = 0;
+ for (let index = 0; index < scrubbed.length; index += 1) {
+ const next = eventBytes + Buffer.byteLength(JSON.stringify(scrubbed[index])) + (index > 0 ? 1 : 0);
+ if (wrapperBytes(index + 1) + next > maxBytes) break;
+ eventBytes = next;
+ fits = index + 1;
}
const selected = scrubbed.slice(0, fits);
payload = encode(selected, true);
@@ -1067,14 +1599,7 @@ function boundedTracePayload(
type WorkspaceListing = { paths: string[]; denied: number };
async function walkFallback(root: string, directory = root, rules: string[] = [], listing: WorkspaceListing = { paths: [], denied: 0 }): Promise<WorkspaceListing> {
- const localRules = await readGitignoreFile(directory);
- const prefix = portablePath(root, directory);
- const ignoreRules = [...rules, ...localRules.map((rule) => {
- const negated = rule.startsWith("!");
- const body = negated ? rule.slice(1) : rule;
- const scoped = prefix ? `${prefix}/${body}` : body;
- return negated ? `!${scoped}` : scoped;
- })];
+ const ignoreRules = [...rules, ...scopedIgnoreRules(portablePath(root, directory), await readGitignoreFile(directory))];
let entries;
try {
entries = await readdir(directory, { withFileTypes: true });
@@ -1449,219 +1974,645 @@ async function sha256File(absolute: string): Promise<string> {
});
}
-async function workspaceSignature(root: string): Promise<string> {
- const hash = createHash("sha256");
- for (const path of (await listWorkspaceFiles(root)).paths) {
- try {
- const file = await lstat(resolve(root, path));
- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) continue;
- hash.update(path).update("\0").update(String(file.size)).update("\0").update(String(file.mtimeMs)).update("\0");
- } catch {
- // A file can disappear while the workspace is being scanned.
- }
+// --- scanning ----------------------------------------------------------------
+
+/** Lets the event loop run once a stretch of synchronous scan work has gone on long enough. */
+class LoopYielder {
+ private last = performance.now();
+
+ async pause(): Promise<void> {
+ if (performance.now() - this.last < EVENT_LOOP_YIELD_MS) return;
+ await new Promise<void>((resolvePromise) => setImmediate(resolvePromise));
+ this.last = performance.now();
}
- // A commit changes the repository without touching any eligible file; fold
- // HEAD in so the next prompt or turn still captures the new history.
- hash.update("HEAD\0").update((await gitHead(root)) ?? "").update("\0");
- return hash.digest("hex");
}
-type WorkspaceScan = {
- manifest: ManifestEntry[];
- files: CollectorFile[];
- manifestTruncated: boolean;
- contentTruncated: boolean;
- /** Non-ignored paths the denylist kept out of the snapshot. */
- deniedCount: number;
- /** Changed files whose content the snapshot cap left out of files[]; their manifest entries remain. */
- capOmittedCount: number;
- /** Redacted bytes of those files. */
- capOmittedBytes: number;
-};
+/**
+ * Reads a whole file into a pooled buffer, so a scan recycles a few buffers
+ * instead of allocating and freeing one per file (which leaves the allocator
+ * holding hundreds of megabytes of freed pages). A pool lives for one scan or
+ * upload pass and is garbage afterwards, so nothing stays allocated between
+ * captures. The view handed to `use` is valid until the same slot is used
+ * again, so nothing of it may outlive the call except through `toString` or
+ * hashing. Null when the file is larger than the cap.
+ */
+class ReadPool {
+ private readonly free: Buffer[] = [];
+ private readonly waiters: Array<(buffer: Buffer) => void> = [];
+ private allocated = 0;
+ /** One buffer at the file cap for the rare file larger than a slot; such reads take turns. */
+ private large: Buffer | null = null;
+ private largeTail: Promise<unknown> = Promise.resolve();
+
+ constructor(private readonly slots: number, private readonly capacity = READ_SLOT_BYTES) {}
+
+ private acquire(): Promise<Buffer> {
+ const buffer = this.free.pop();
+ if (buffer) return Promise.resolve(buffer);
+ if (this.allocated < this.slots) {
+ this.allocated += 1;
+ return Promise.resolve(Buffer.allocUnsafeSlow(this.capacity));
+ }
+ return new Promise((resolvePromise) => this.waiters.push(resolvePromise));
+ }
-/** A changed file competing for the snapshot's content budget. */
-type SnapshotCandidate = {
- path: string;
- uploadPath: string;
- absolute: string;
- /** Position in the workspace listing; files[] keeps that order. */
- order: number;
- bytes: number;
- /** Touched this session, or named by git status or the diff. */
- priority: boolean;
- /** Kept from the hashing read while the running total allowed it; re-read otherwise. */
- content?: string;
-};
+ private release(buffer: Buffer): void {
+ const waiter = this.waiters.shift();
+ if (waiter) waiter(buffer);
+ else this.free.push(buffer);
+ }
-function serializedBytes(value: unknown): number {
- return Buffer.byteLength(JSON.stringify(value));
+ /** Runs `use` with the file's bytes; the view must not escape `use`. */
+ async read<T>(absolute: string, expectedSize: number, use: (bytes: Buffer) => Promise<T> | T): Promise<T | null> {
+ if (expectedSize > MAX_COLLECTOR_FILE_BYTES) return null;
+ if (expectedSize < this.capacity) {
+ const buffer = await this.acquire();
+ try {
+ const length = await readInto(absolute, buffer);
+ if (length < buffer.length) return await use(buffer.subarray(0, length));
+ } finally {
+ this.release(buffer);
+ }
+ // The file grew past its slot since the stat: read it with the large buffer.
+ }
+ return this.readLarge(absolute, use);
+ }
+
+ private readLarge<T>(absolute: string, use: (bytes: Buffer) => Promise<T> | T): Promise<T | null> {
+ const run = this.largeTail.then(async () => {
+ this.large ??= Buffer.allocUnsafeSlow(MAX_COLLECTOR_FILE_BYTES + 1);
+ const length = await readInto(absolute, this.large);
+ // Past the cap by now: not a file the collector takes.
+ return length > MAX_COLLECTOR_FILE_BYTES ? null : await use(this.large.subarray(0, length));
+ });
+ this.largeTail = run.catch(() => undefined);
+ return run;
+ }
}
-/**
- * Paths whose content a capped snapshot keeps first: what the session touched
- * plus what git reports as modified, keyed the way files[] names them.
- */
-function snapshotPriorityPaths(touched: Iterable<string>, git: CollectorGitBlock | null): Set<string> {
- const paths = new Set<string>();
- for (const path of touched) paths.add(collectorPathForUpload(path));
- for (const entry of git?.status ?? []) paths.add(entry.path);
- for (const line of git?.diff?.split("\n") ?? []) {
- const path = diffHeaderPath(line);
- if (path) paths.add(collectorPathForUpload(path));
+/** Reads a file from the start into `buffer`, up to its length; resolves with the bytes read. */
+async function readInto(absolute: string, buffer: Buffer): Promise<number> {
+ const handle = await open(absolute, "r");
+ let length = 0;
+ try {
+ while (length < buffer.length) {
+ const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length);
+ if (bytesRead === 0) break;
+ length += bytesRead;
+ }
+ } finally {
+ await handle.close();
}
- return paths;
+ return length;
}
/**
- * Picks which changed files carry content, smallest first within each
- * priority tier, and materialises them in listing order. Costs are measured
- * on the JSON a file becomes, so escaping cannot push the envelope past what
- * the raw byte count promised.
+ * Runs `operation` over `items` with at most `limit` calls in flight and
+ * returns the results in item order, so a 50,000-file scan holds a handful of
+ * descriptors instead of a burst of them.
*/
-async function selectSnapshotContent(
- candidates: SnapshotCandidate[],
- budget: number,
-): Promise<{ files: CollectorFile[]; omittedCount: number; omittedBytes: number }> {
- const ordered = [...candidates].sort((a, b) => Number(b.priority) - Number(a.priority) || a.bytes - b.bytes || a.order - b.order);
- const kept: Array<{ order: number; file: CollectorFile }> = [];
- let used = 0;
- let omittedCount = 0;
- let omittedBytes = 0;
- for (const candidate of ordered) {
- const overhead = SNAPSHOT_FILE_ENTRY_OVERHEAD_BYTES + Buffer.byteLength(candidate.uploadPath);
- // Serialised content is never shorter than the raw bytes, so a file that
- // fails this estimate is omitted without being read.
- if (used + overhead + candidate.bytes <= budget) {
- let content = candidate.content;
- if (content === undefined) {
- try {
- content = redactCollectorContent(candidate.path, (await readFile(candidate.absolute)).toString("utf8"));
- } catch {
- // Changed underneath the scan; the next snapshot sees its final form.
- continue;
- }
- }
- const cost = overhead + serializedBytes(content);
- if (used + cost <= budget) {
- kept.push({ order: candidate.order, file: { path: candidate.uploadPath, content, sha256: sha256Hex(content) } });
- used += cost;
- continue;
- }
+export async function mapBounded<T, R>(items: readonly T[], limit: number, operation: (item: T, index: number) => Promise<R>): Promise<R[]> {
+ const results: R[] = new Array(items.length);
+ let next = 0;
+ const worker = async (): Promise<void> => {
+ while (next < items.length) {
+ const index = next;
+ next += 1;
+ results[index] = await operation(items[index]!, index);
}
- omittedCount += 1;
- omittedBytes += candidate.bytes;
- }
- kept.sort((a, b) => a.order - b.order);
- return { files: kept.map((item) => item.file), omittedCount, omittedBytes };
+ };
+ await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
+ return results;
+}
+
+/** Work counters for tests and profiling; never uploaded. */
+export type CollectorMetrics = {
+ fileStats: number;
+ fileReads: number;
+ fileRedactions: number;
+ fullScans: number;
+ dirtyScans: number;
+ reconciles: number;
+ capturesSkipped: number;
+ capturesDeferred: number;
+ ignoreCheckSpawns: number;
+ watchEvents: number;
+ envelopesWritten: number;
+};
+
+function freshMetrics(): CollectorMetrics {
+ return {
+ fileStats: 0, fileReads: 0, fileRedactions: 0, fullScans: 0, dirtyScans: 0, reconciles: 0,
+ capturesSkipped: 0, capturesDeferred: 0, ignoreCheckSpawns: 0, watchEvents: 0, envelopesWritten: 0,
+ };
+}
+
+/** Redacts one file's bytes and describes the result for the cache and the manifest. */
+function redactedCacheEntry(
+ path: string,
+ buffer: Buffer,
+ size: number,
+ mtimeMs: number,
+ uploadPath: string,
+ metrics: CollectorMetrics,
+): { entry: HashCacheEntry; text: string } {
+ const redacted = redactCollectorContentCounted(path, buffer.toString("utf8"));
+ metrics.fileRedactions += 1;
+ return {
+ text: redacted.text,
+ entry: {
+ size,
+ mtimeMs,
+ binary: false,
+ sha256: sha256Hex(redacted.text),
+ bytes: Buffer.byteLength(redacted.text),
+ json: Buffer.byteLength(JSON.stringify(redacted.text)),
+ clean: redacted.count === 0,
+ uploadPath,
+ },
+ };
+}
+
+/** The redacted upload path, sharing the listing's string when redaction left it unchanged. */
+function uploadPathFor(path: string, cached: HashCacheEntry | undefined): string {
+ if (cached) return cached.uploadPath;
+ const uploadPath = collectorPathForUpload(path);
+ return uploadPath === path ? path : uploadPath;
}
+/** Per-pass answers of hasRealAncestors, by workspace-relative directory. */
+type AncestorCache = Map<string, Promise<boolean>>;
+
/**
- * Walks every eligible workspace file, hashes its uploadable (redacted) form
- * for the manifest, and gathers content for either the whole tree (start
- * snapshots) or only the files whose hash differs from the previous manifest.
- * Unchanged files are recognised by size + mtime through the per-session hash
- * cache, so repeated snapshots cost a stat per file instead of a read.
- *
- * `byteLimit` bounds the serialised manifest plus files[] together. When the
- * changed files do not all fit, content goes to `priorityPaths` first and to
- * the smallest remaining files next; whatever is left out keeps its manifest
- * entry so the backend still learns the file exists and what it hashes to.
+ * Whether no directory between the root and `path` is a symlink (or a Windows
+ * junction, which lstat reports as one). A path reached through one
+ * (`linked/util.ts` with `linked -> ../shared-lib`) names a file outside the
+ * workspace, which git's listing never holds and nothing may upload; checking
+ * only the last component, or the resolved path as a string, lets it through.
+ * A missing ancestor passes: the file cannot exist either, which the caller's
+ * own lstat reports. `seen` shares the answer per directory for one pass.
*/
-async function scanWorkspace(
- root: string,
- cache: Map<string, HashCacheEntry>,
- baseline: Map<string, string> | null,
- byteLimit: number,
- includeAll: boolean,
- priorityPaths: ReadonlySet<string> = new Set(),
-): Promise<WorkspaceScan> {
- const manifest: ManifestEntry[] = [];
- const candidates: SnapshotCandidate[] = [];
- const seen = new Set<string>();
- let retained = 0;
- const listing = await listWorkspaceFiles(root);
- const paths = listing.paths;
- for (const path of paths) {
- if (manifest.length >= MAX_FILES) break;
- try {
- const absolute = resolve(root, path);
- if (portablePath(root, absolute).startsWith("../")) continue;
- const file = await lstat(absolute);
- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) continue;
- seen.add(path);
- const cached = cache.get(path);
- let entry: HashCacheEntry | undefined = cached && cached.size === file.size && cached.mtimeMs === file.mtimeMs ? cached : undefined;
- let redacted: string | undefined;
- if (!entry) {
- const buffer = await readFile(absolute);
- if (buffer.length > MAX_COLLECTOR_FILE_BYTES || isBinary(buffer)) {
- cache.set(path, { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0 });
- continue;
- }
- redacted = redactCollectorContent(path, buffer.toString("utf8"));
- entry = { size: file.size, mtimeMs: file.mtimeMs, binary: false, sha256: sha256Hex(redacted), bytes: Buffer.byteLength(redacted) };
- cache.set(path, entry);
- }
- if (entry.binary) continue;
- manifest.push({ path, sha256: entry.sha256, size: entry.bytes });
- const changed = includeAll || baseline?.get(path) !== entry.sha256;
- if (!changed) continue;
- // Content already in hand stays in memory while it could still fit, so a
- // snapshot under the budget never reads a file twice; beyond that point
- // it is dropped and re-read only if the selection picks the file.
- const keep = retained + entry.bytes <= byteLimit;
- if (keep) retained += entry.bytes;
- const uploadPath = collectorPathForUpload(path);
- candidates.push({
- path,
- uploadPath,
- absolute,
- order: candidates.length,
- bytes: entry.bytes,
- priority: priorityPaths.has(uploadPath),
- ...(keep && redacted !== undefined ? { content: redacted } : {}),
- });
- } catch {
- // Workspaces are live; races are expected and retried by the next snapshot.
+async function hasRealAncestors(root: string, path: string, seen: AncestorCache = new Map()): Promise<boolean> {
+ const parts = pathComponents(path);
+ let directory = "";
+ for (let index = 0; index < parts.length - 1; index += 1) {
+ directory = directory ? `${directory}/${parts[index]}` : parts[index]!;
+ let real = seen.get(directory);
+ if (!real) {
+ real = lstat(resolve(root, directory)).then((entry) => !entry.isSymbolicLink(), () => true);
+ seen.set(directory, real);
}
+ if (!(await real)) return false;
}
- for (const key of cache.keys()) {
- if (!seen.has(key)) cache.delete(key);
- }
- const selected = await selectSnapshotContent(candidates, Math.max(0, byteLimit - serializedBytes(manifest)));
+ return true;
+}
+
+/** What one scan pass shares across the files it inspects. */
+type ScanContext = {
+ root: string;
+ /** The resolved root with a trailing separator: every file inspected must start with it. */
+ rootPrefix: string;
+ cache: Map<string, HashCacheEntry>;
+ metrics: CollectorMetrics;
+ yielder: LoopYielder;
+ pool: ReadPool;
+ ancestors: AncestorCache;
+};
+
+function scanContext(root: string, cache: Map<string, HashCacheEntry>, metrics: CollectorMetrics): ScanContext {
+ const base = resolve(root);
return {
- manifest,
- files: selected.files,
- manifestTruncated: paths.length >= MAX_FILES,
- contentTruncated: selected.omittedCount > 0,
- deniedCount: listing.denied,
- capOmittedCount: selected.omittedCount,
- capOmittedBytes: selected.omittedBytes,
+ root,
+ rootPrefix: base.endsWith(sep) ? base : `${base}${sep}`,
+ cache,
+ metrics,
+ yielder: new LoopYielder(),
+ pool: new ReadPool(SCAN_CONCURRENCY),
+ ancestors: new Map(),
};
}
/**
- * Whether an unauthorized response is the sign-in gate (the broker's or the
- * gateway's `omnirush_account_required`) rather than a stale device token.
- * Reading the body also releases the connection.
+ * Stats one eligible path and, unless the per-root cache already knows the
+ * file at this size and mtime, reads, redacts and hashes it. Null for anything
+ * that is not a regular file within the size cap (gone, a directory, a
+ * symlink, too large) or that resolves outside the root, lexically or through
+ * a symlinked directory; a binary file comes back as its binary entry. Content
+ * is never kept: the upload pass re-reads what it sends.
*/
-async function responseSignalsAccountRequired(response: Response): Promise<boolean> {
+async function inspectFile(context: ScanContext, path: string): Promise<HashCacheEntry | null> {
+ const { cache, metrics, pool } = context;
+ const absolute = resolve(context.root, path);
+ if (!absolute.startsWith(context.rootPrefix) || !(await hasRealAncestors(context.root, path, context.ancestors))) return null;
+ let file;
try {
- return (await response.text()).includes(ACCOUNT_REQUIRED_MARKER);
+ file = await lstat(absolute);
} catch {
- return false;
+ return null;
}
+ metrics.fileStats += 1;
+ if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) return null;
+ const cached = cache.get(path);
+ if (cached && cached.size === file.size && cached.mtimeMs === file.mtimeMs) return cached;
+ const uploadPath = uploadPathFor(path, cached);
+ let entry: HashCacheEntry | null = null;
+ try {
+ entry = await pool.read(absolute, file.size, (buffer) => {
+ metrics.fileReads += 1;
+ if (isBinary(buffer)) {
+ return { size: file.size, mtimeMs: file.mtimeMs, binary: true, sha256: "", bytes: 0, json: 0, clean: false, uploadPath };
+ }
+ return redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPath, metrics).entry;
+ });
+ } catch {
+ return null;
+ }
+ if (entry) cache.set(path, entry);
+ await context.yielder.pause();
+ return entry;
}
-function compressZstd(payload: Buffer): Promise<Buffer> {
- return new Promise((resolvePromise, reject) => {
- zstdCompress(payload, (error, result) => {
- if (error) reject(error);
- else resolvePromise(result);
- });
+/** One file as files[] carries it, re-read at upload time. */
+type UploadContent = { content: string; entry: HashCacheEntry };
+
+/**
+ * Reads a file for files[]. A file the scan saw unchanged by redaction is sent
+ * as read once its raw bytes hash to the cached digest, which skips the regex
+ * pipeline; anything else (changed since the scan, or a file the scrubber
+ * touched) is redacted again, and the cache learns the result when it differs.
+ */
+async function readUploadContent(
+ root: string,
+ path: string,
+ cache: Map<string, HashCacheEntry>,
+ metrics: CollectorMetrics,
+ pool: ReadPool,
+ ancestors: AncestorCache,
+): Promise<UploadContent | null> {
+ const absolute = resolve(root, path);
+ // A directory swapped for a symlink since the scan must not be followed either.
+ if (!(await hasRealAncestors(root, path, ancestors))) return null;
+ let file;
+ try {
+ file = await lstat(absolute);
+ } catch {
+ return null;
+ }
+ if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) return null;
+ const cached = cache.get(path);
+ try {
+ return await pool.read(absolute, file.size, (buffer): UploadContent | null => {
+ metrics.fileReads += 1;
+ if (cached && !cached.binary && cached.clean && cached.size === file.size && cached.mtimeMs === file.mtimeMs
+ && cached.bytes === buffer.length && sha256Hex(buffer) === cached.sha256) {
+ return { content: buffer.toString("utf8"), entry: cached };
+ }
+ if (isBinary(buffer)) return null;
+ const { entry, text } = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, cached), metrics);
+ // The same digest keeps the cached entry, which the manifest may share.
+ if (cached && cached.sha256 === entry.sha256 && cached.size === entry.size && cached.mtimeMs === entry.mtimeMs) {
+ return { content: text, entry: cached };
+ }
+ cache.set(path, entry);
+ return { content: text, entry };
+ });
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Eligible text files by workspace path, in listing order. Entries are the
+ * per-root cache's own (immutable) objects: path (redacted) as `uploadPath`,
+ * redacted digest as `sha256`, redacted size as `bytes`.
+ */
+type Manifest = Map<string, HashCacheEntry>;
+
+type ScanResult = {
+ /** Every eligible file: the manifest, and the next baseline once accepted. */
+ manifest: Manifest;
+ /** Paths whose digest differs from the previous manifest; null when every manifest entry counts (a start snapshot). */
+ changed: Set<string> | null;
+ /** Previous manifest entries no longer present. */
+ removed: number;
+ deniedCount: number;
+ manifestTruncated: boolean;
+};
+
+/** A changed file competing for the snapshot's content budget. */
+type SnapshotCandidate = {
+ path: string;
+ uploadPath: string;
+ /** Redacted bytes: the sort key, and what a cap omission reports. */
+ bytes: number;
+ /** Serialised files[] cost: overhead, path and JSON-encoded content. */
+ cost: number;
+ /** Touched this session, or named by git status or the diff. */
+ priority: boolean;
+};
+
+/** The manifest entry as the envelope carries it: path (redacted), sha256, size. */
+function manifestEntryJson(entry: HashCacheEntry): string {
+ return `{"path":${JSON.stringify(entry.uploadPath)},"sha256":"${entry.sha256}","size":${entry.bytes}}`;
+}
+
+function candidateCost(entry: HashCacheEntry): number {
+ return SNAPSHOT_FILE_ENTRY_OVERHEAD_BYTES + Buffer.byteLength(entry.uploadPath) + entry.json;
+}
+
+/** The changed entries of a manifest as candidates, in manifest order. */
+function candidatesFor(scan: ScanResult, priorityPaths: ReadonlySet<string>): SnapshotCandidate[] {
+ const candidates: SnapshotCandidate[] = [];
+ for (const [path, entry] of scan.manifest) {
+ if (scan.changed && !scan.changed.has(path)) continue;
+ candidates.push({ path, uploadPath: entry.uploadPath, bytes: entry.bytes, cost: candidateCost(entry), priority: priorityPaths.has(entry.uploadPath) });
+ }
+ return candidates;
+}
+
+function countRemoved(previous: Manifest | null, manifest: Manifest): number {
+ let removed = 0;
+ for (const path of previous?.keys() ?? []) {
+ if (!manifest.has(path)) removed += 1;
+ }
+ return removed;
+}
+
+/**
+ * Lists and inspects every eligible file. Unchanged files are recognised by
+ * size + mtime through the per-root cache, so a rescan of a settled tree costs
+ * a stat per file and no reads. The cache is rebuilt from what the listing
+ * still names, so entries for removed paths go with it. `onListing` sees the
+ * listing before any file is read.
+ */
+async function scanWorkspaceFull(
+ root: string,
+ cache: Map<string, HashCacheEntry>,
+ previous: Manifest | null,
+ includeAll: boolean,
+ metrics: CollectorMetrics,
+ onListing?: (paths: readonly string[]) => void,
+): Promise<ScanResult> {
+ metrics.fullScans += 1;
+ const listing = await listWorkspaceFiles(root);
+ onListing?.(listing.paths);
+ const context = scanContext(root, cache, metrics);
+ const entries = await mapBounded(listing.paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));
+ cache.clear();
+ const manifest: Manifest = new Map();
+ const changed = includeAll ? null : new Set<string>();
+ listing.paths.forEach((path, index) => {
+ const entry = entries[index];
+ if (!entry) return;
+ cache.set(path, entry);
+ if (entry.binary || manifest.size >= MAX_FILES) return;
+ manifest.set(path, entry);
+ if (changed && previous?.get(path)?.sha256 !== entry.sha256) changed.add(path);
});
+ return {
+ manifest,
+ changed,
+ removed: countRemoved(previous, manifest),
+ deniedCount: listing.denied,
+ manifestTruncated: listing.paths.length >= MAX_FILES,
+ };
+}
+
+/**
+ * Inspects only the paths the watcher reported since the last capture and
+ * folds them into a copy of the previous manifest: no listing, no stat of the
+ * rest of the tree. `ignored` is git's current answer for every reported path
+ * (the caller asks in one batch before anything is read): an ignored path is
+ * never read, and leaves the manifest if it was there, since a file the user
+ * has just added to `.gitignore` or `.git/info/exclude` may not stay listed. A
+ * reported path that is no longer a file may have been a directory moved or
+ * deleted as a whole (one event, no events for its files): the manifest
+ * entries under it are inspected too.
+ */
+async function scanDirtyPaths(
+ root: string,
+ cache: Map<string, HashCacheEntry>,
+ previous: Manifest,
+ dirty: readonly string[],
+ ignored: ReadonlySet<string>,
+ listing: { denied: number; truncated: boolean },
+ metrics: CollectorMetrics,
+): Promise<ScanResult> {
+ metrics.dirtyScans += 1;
+ const context = scanContext(root, cache, metrics);
+ const targets = dirty.filter((path) => !ignored.has(path));
+ const inspect = (paths: string[]) => mapBounded(paths, SCAN_CONCURRENCY, (path) => inspectFile(context, path));
+ const entries = await inspect(targets);
+ const manifest: Manifest = new Map(previous);
+ for (const path of dirty) {
+ if (!ignored.has(path)) continue;
+ manifest.delete(path);
+ cache.delete(path);
+ }
+ const changed = new Set<string>();
+ const vanished = new Set<string>();
+ const fold = (path: string, entry: HashCacheEntry | null) => {
+ if (!entry || entry.binary) {
+ if (!manifest.delete(path) && !entry) vanished.add(path);
+ return;
+ }
+ if (!manifest.has(path) && manifest.size >= MAX_FILES) return;
+ manifest.set(path, entry);
+ if (previous.get(path)?.sha256 !== entry.sha256) changed.add(path);
+ };
+ targets.forEach((path, index) => fold(path, entries[index] ?? null));
+ if (vanished.size > 0) {
+ const reported = new Set(dirty);
+ const inside: string[] = [];
+ for (const path of manifest.keys()) {
+ if (reported.has(path)) continue;
+ for (let slash = path.indexOf("/"); slash !== -1; slash = path.indexOf("/", slash + 1)) {
+ if (vanished.has(path.slice(0, slash))) {
+ inside.push(path);
+ break;
+ }
+ }
+ }
+ const insideEntries = await inspect(inside);
+ inside.forEach((path, index) => fold(path, insideEntries[index] ?? null));
+ }
+ return {
+ manifest,
+ changed,
+ removed: countRemoved(previous, manifest),
+ deniedCount: listing.denied,
+ manifestTruncated: listing.truncated || manifest.size >= MAX_FILES,
+ };
+}
+
+/**
+ * Paths whose content a capped snapshot keeps first: what the session touched
+ * plus what git reports as modified, keyed the way files[] names them.
+ */
+function snapshotPriorityPaths(touched: Iterable<string>, git: CollectorGitBlock | null): Set<string> {
+ const paths = new Set<string>();
+ for (const path of touched) paths.add(collectorPathForUpload(path));
+ for (const entry of git?.status ?? []) paths.add(entry.path);
+ for (const line of git?.diff?.split("\n") ?? []) {
+ const path = diffHeaderPath(line);
+ if (path) paths.add(collectorPathForUpload(path));
+ }
+ return paths;
+}
+
+/**
+ * Picks which changed files carry content, smallest first within each
+ * priority tier, against a budget measured on the JSON a file becomes, so
+ * escaping cannot push the envelope past what the raw byte count promised.
+ */
+function selectSnapshotContent(candidates: SnapshotCandidate[], budget: number): { selected: Set<string>; omittedCount: number; omittedBytes: number } {
+ // A stable sort: equal candidates keep manifest order.
+ const ordered = [...candidates].sort((a, b) => Number(b.priority) - Number(a.priority) || a.bytes - b.bytes);
+ const selected = new Set<string>();
+ let used = 0;
+ let omittedCount = 0;
+ let omittedBytes = 0;
+ for (const candidate of ordered) {
+ if (used + candidate.cost <= budget) {
+ selected.add(candidate.path);
+ used += candidate.cost;
+ } else {
+ omittedCount += 1;
+ omittedBytes += candidate.bytes;
+ }
+ }
+ return { selected, omittedCount, omittedBytes };
}
+function serializedBytes(value: unknown): number {
+ return Buffer.byteLength(JSON.stringify(value));
+}
+
+/**
+ * Whether an unauthorized response is the sign-in gate (the broker's or the
+ * gateway's `omnirush_account_required`) rather than a stale device token.
+ * Reading the body also releases the connection.
+ */
+async function responseSignalsAccountRequired(response: Response): Promise<boolean> {
+ try {
+ return (await response.text()).includes(ACCOUNT_REQUIRED_MARKER);
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * Top-level directories holding at least one listed file: the ones worth a
+ * recursive watcher. The listing is git's (tracked plus untracked-but-not-
+ * ignored) filtered by the denylist, so an ignored or denied directory never
+ * appears here.
+ */
+function watchRootsFor(paths: readonly string[]): string[] {
+ const roots = new Set<string>();
+ for (const path of paths) {
+ const slash = path.indexOf("/");
+ if (slash > 0) roots.add(path.slice(0, slash));
+ }
+ return [...roots];
+}
+
+// --- envelope streaming ------------------------------------------------------
+
+const utf8Encoder = new TextEncoder();
+
+/**
+ * Streams one envelope as JSON text through zstd into a file, piece by piece.
+ * Text is encoded into one reusable chunk, and a full chunk is handed to the
+ * compressor and reused only once the compressor has consumed it (which also
+ * carries the file's backpressure): nothing larger than one file's serialised
+ * entry exists in memory at a time, and writing allocates no buffer per chunk.
+ * The compressed result is read back from disk only for the upload.
+ */
+class EnvelopeWriter {
+ /** Uncompressed bytes written so far. */
+ bytes = 0;
+ private readonly zstd = createZstdCompress();
+ private readonly file: WriteStream;
+ private readonly finished: Promise<void>;
+ private failure: Error | null = null;
+ private readonly chunk = Buffer.allocUnsafeSlow(ENVELOPE_WRITE_CHUNK_BYTES);
+ private used = 0;
+
+ constructor(readonly path: string) {
+ this.file = createWriteStream(path, { mode: 0o600 });
+ this.finished = new Promise<void>((resolvePromise, reject) => {
+ const fail = (error: Error) => {
+ this.failure ??= error;
+ reject(error);
+ };
+ this.file.on("finish", resolvePromise);
+ this.file.on("error", fail);
+ this.zstd.on("error", fail);
+ });
+ this.finished.catch(() => undefined);
+ this.zstd.pipe(this.file);
+ }
+
+ async write(text: string): Promise<void> {
+ let rest = text;
+ while (rest.length > 0) {
+ const { read, written } = utf8Encoder.encodeInto(rest, this.chunk.subarray(this.used));
+ this.used += written;
+ this.bytes += written;
+ rest = read === rest.length ? "" : rest.slice(read);
+ // What is left did not fit (or the next code point did not): pass the chunk on.
+ if (rest.length > 0) await this.flush();
+ }
+ }
+
+ private async flush(): Promise<void> {
+ if (this.failure) throw this.failure;
+ if (this.used === 0) return;
+ const view = this.chunk.subarray(0, this.used);
+ await new Promise<void>((resolvePromise, reject) => {
+ const fail = (error: Error) => {
+ this.zstd.off("error", fail);
+ this.file.off("error", fail);
+ reject(error);
+ };
+ this.zstd.once("error", fail);
+ this.file.once("error", fail);
+ this.zstd.write(view, (error) => {
+ this.zstd.off("error", fail);
+ this.file.off("error", fail);
+ if (error) reject(error);
+ else resolvePromise();
+ });
+ });
+ this.used = 0;
+ }
+
+ /** Ends the stream and resolves with the compressed size on disk. */
+ async finish(): Promise<number> {
+ await this.flush();
+ this.zstd.end();
+ await this.finished;
+ return (await stat(this.path)).size;
+ }
+
+ async abort(): Promise<void> {
+ this.zstd.destroy();
+ this.file.destroy();
+ await rm(this.path, { force: true }).catch(() => undefined);
+ }
+}
+
+type EnvelopeBody = {
+ /** Envelope fields written before files[]: workspace, environment, touched paths, scope. */
+ extras: Record<string, unknown>;
+ /** Streams files[] one entry at a time through `emit`. */
+ writeFiles: (emit: (file: CollectorFile) => Promise<void>) => Promise<void>;
+ manifest: () => Iterable<HashCacheEntry>;
+ /** Computed once files[] is written, so cap omissions are known. */
+ privacy: () => Record<string, unknown>;
+ trace?: unknown[];
+};
+
function collectorEnvironment(appVersion: string | undefined, engineVersion: string | undefined, gitVersion: string | null): CollectorEnvironment {
let locale: string | null = null;
let timezone: string | null = null;
@@ -1752,6 +2703,17 @@ export class WorkspaceCollector {
private retryTimer: ReturnType<typeof setTimeout> | null = null;
private retryFailures = 0;
private stopped = false;
+ private readonly minChangeIntervalMs: number;
+ private readonly maxWatchedFiles: number;
+ /** Envelopes are streamed into this directory and deleted once uploaded or spooled. */
+ private readonly tempDir: string;
+ /**
+ * Per workspace root, shared by every session open on it: the hash cache,
+ * and when the reconcile pass last ran (one pass serves every session).
+ */
+ private readonly caches = new Map<string, { refs: number; entries: Map<string, HashCacheEntry>; reconciling: boolean; reconciledAt: number }>();
+ /** Work counters for tests and profiling. */
+ readonly metrics: CollectorMetrics = freshMetrics();
constructor(options: CollectorOptions = {}) {
this.collectUrl = resolveCollectUrl(options.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL);
@@ -1774,6 +2736,10 @@ export class WorkspaceCollector {
this.spoolMaxEntries = options.spoolMaxEntries ?? MAX_SPOOL_ENTRIES;
this.spoolMaxBytes = options.spoolMaxBytes ?? MAX_SPOOL_BYTES;
this.snapshotMaxBytes = options.snapshotMaxBytes ?? MAX_SNAPSHOT_BYTES;
+ this.minChangeIntervalMs = options.minChangeIntervalMs ?? MIN_COLLECTOR_CHANGE_INTERVAL_MS;
+ this.maxWatchedFiles = options.maxWatchedFiles ?? MAX_COLLECTOR_WATCHED_FILES;
+ this.tempDir = stateDir ? join(stateDir, TEMP_DIRECTORY) : join(tmpdir(), `omnirush-collector-${process.pid}`);
+ void this.cleanTempDir(60 * 60_000).catch(() => undefined);
if (this.spoolDir) {
if (this.enabled) {
// Uploads spooled by a previous process are retried once this one is up.
@@ -1931,20 +2897,34 @@ export class WorkspaceCollector {
segment: 1,
sequence: 0,
resumed: false,
- lastSignature: "",
started: false,
finished: false,
changeTimer: null,
changeTrigger: null,
+ deferTimer: null,
+ deferTrigger: null,
scanTimer: null,
- watcher: null,
+ watchers: [],
+ watchMode: "starting",
+ watchedPaths: [],
+ watchSetup: Promise.resolve(),
trace: [],
changeJournal: new Map(),
touchedPaths: new Set(),
changeJournalBytes: 0,
changeCaptureTail: Promise.resolve(),
- hashCache: new Map(),
- baseline: null,
+ pendingJournal: new Set(),
+ ignoreBatch: new Map(),
+ ignoreTimer: null,
+ ignoredCache: new Map(),
+ manifest: null,
+ cache: this.acquireCache(root),
+ listing: { denied: 0, truncated: false },
+ dirty: new Set(),
+ // Nothing is known about the tree yet: the first change capture rescans it.
+ dirtyOverflow: true,
+ lastHead: null,
+ lastChangeAt: 0,
artifactBaseline: null,
model: null,
childSessionIds: [],
@@ -1957,32 +2937,373 @@ export class WorkspaceCollector {
this.sessions.set(sessionId, state);
this.enqueue(state, async () => {
await state.ready;
- state.lastSignature = await workspaceSignature(root);
- await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");
- state.started = true;
+ try {
+ // The watchers go up as soon as the start snapshot's listing is in,
+ // before any file is read: an edit made during the scan is either
+ // seen by the scan or reported by a watcher.
+ await this.uploadWorkspace(state, "start", state.resumed ? "resume" : "session_start");
+ } finally {
+ state.started = true;
+ // No listing reached the plan (the session byte cap, a failed scan): poll.
+ if (state.watchMode === "starting") this.installWatchers(state, null);
+ }
+ });
+ }
+
+ // --- watcher and dirty tracking --------------------------------------------
+
+ /**
+ * Plans the watchers from the start snapshot's listing: one non-recursive
+ * watcher on the root plus a recursive one per top-level directory holding
+ * listed files, so `.git/`, `node_modules/`, gitignored build output and
+ * denied directories never reach a callback. The other top-level
+ * directories (empty, or holding only denied or ignored files at the start)
+ * are watched too, once git confirms they are not ignored themselves. A
+ * workspace listing more files than the watched-files cap or more top-level
+ * directories than MAX_WATCH_ROOTS, or with no listing at all, is polled on
+ * snapshots instead.
+ */
+ private installWatchers(state: SessionState, paths: readonly string[] | null): void {
+ this.closeWatchers(state);
+ if (state.finished) return;
+ const roots = paths && paths.length <= this.maxWatchedFiles ? watchRootsFor(paths) : null;
+ if (roots && roots.length <= MAX_WATCH_ROOTS) {
+ for (const prefix of ["", ...roots]) this.addWatcher(state, prefix);
+ }
+ if (state.watchers.length > 0) {
+ state.watchMode = "watching";
+ this.startReconcile(state);
+ state.watchSetup = this.watchUnlistedRoots(state).catch(() => undefined);
+ return;
+ }
+ this.pollInstead(state, paths?.length ?? null);
+ }
+
+ /**
+ * Watches the top-level directories the listing gave no reason to: one that
+ * was empty, or held only denied or ignored files, when the session started
+ * (`scripts/`, `config/` next to a denied `.env`). A file a tool writes there
+ * later is then seen as it lands, not only by the next reconcile pass. A
+ * denied name (`.git`, `node_modules`, `secrets`) is never watched, nor is a
+ * directory git ignores (`dist/`); past MAX_WATCH_ROOTS the rest are left to
+ * the reconcile pass and the end-of-turn artifact scan.
+ */
+ private async watchUnlistedRoots(state: SessionState): Promise<void> {
+ const entries = await readdir(state.root, { withFileTypes: true });
+ const candidates = entries
+ .filter((entry) => entry.isDirectory() && !hasDeniedComponent([entry.name]) && !state.watchedPaths.includes(entry.name))
+ .map((entry) => entry.name);
+ if (candidates.length === 0) return;
+ const ignored = await this.ignoredPaths(state, candidates, true);
+ for (const name of candidates) {
+ if (state.finished || state.watchMode !== "watching" || state.watchers.length > MAX_WATCH_ROOTS) return;
+ if (!ignored.has(name) && !state.watchedPaths.includes(name)) this.addWatcher(state, name);
+ }
+ }
+
+ private pollInstead(state: SessionState, files: number | null): void {
+ this.closeWatchers(state);
+ state.watchMode = "polling";
+ state.dirtyOverflow = true;
+ this.stopReconcile(state);
+ this.log("info", "OmniRush collection polls this workspace on snapshots instead of watching it", {
+ sessionId: state.id,
+ files,
+ limit: this.maxWatchedFiles,
});
+ }
+
+ /** Watches the root itself (prefix "") non-recursively, or one top-level directory recursively. */
+ private addWatcher(state: SessionState, prefix: string): void {
+ let watcher: FSWatcher;
try {
- state.watcher = watch(root, { recursive: true }, (_event, filename) => {
- if (filename && isCollectorPathDenied(String(filename))) return;
- if (filename) this.queueChangedPath(state, String(filename));
- this.scheduleChange(state, "fs_change");
- });
- state.watcher.on("error", () => {
- state.watcher?.close();
- state.watcher = null;
+ watcher = watch(prefix ? resolve(state.root, prefix) : state.root, { recursive: prefix !== "" }, (event, filename) => {
+ this.handleWatchEvent(state, prefix, event, filename);
});
} catch {
- state.watcher = null;
- }
- state.scanTimer = setInterval(() => {
- void workspaceSignature(root).then((signature) => {
- if (signature && state.lastSignature && signature !== state.lastSignature) this.scheduleChange(state, "periodic");
- state.lastSignature = signature;
- }).catch(() => undefined);
- }, this.fallbackScanMs);
+ return;
+ }
+ watcher.on("error", () => {
+ // One subtree went dark: the next capture rescans, the reconcile pass
+ // keeps covering it, and a workspace with no watcher left is polled.
+ watcher.close();
+ const index = state.watchers.indexOf(watcher);
+ if (index !== -1) {
+ state.watchers.splice(index, 1);
+ state.watchedPaths.splice(index, 1);
+ }
+ this.markDirtyOverflow(state);
+ if (state.watchers.length === 0 && state.watchMode === "watching" && !state.finished) this.pollInstead(state, state.manifest?.size ?? null);
+ });
+ state.watchers.push(watcher);
+ state.watchedPaths.push(prefix || ".");
+ }
+
+ private closeWatchers(state: SessionState): void {
+ for (const watcher of state.watchers) watcher.close();
+ state.watchers = [];
+ state.watchedPaths = [];
+ }
+
+ private handleWatchEvent(state: SessionState, prefix: string, event: string, filename: string | Buffer | null): void {
+ this.metrics.watchEvents += 1;
+ if (state.finished) return;
+ if (filename === null || filename === undefined) {
+ // The platform lost events (a full ReadDirectoryChangesW buffer, for one).
+ this.markDirtyOverflow(state);
+ this.scheduleChange(state, "fs_change");
+ return;
+ }
+ const relative = String(filename).replaceAll("\\", "/");
+ const path = prefix ? `${prefix}/${relative}` : relative;
+ if (!path || path.startsWith("../") || path.startsWith("/") || isCollectorPathDenied(path)) return;
+ if (path === ".gitignore" || path.endsWith("/.gitignore")) {
+ // New ignore rules can hide or reveal any number of files: forget what
+ // git answered so far and rescan the tree with the rules as they stand.
+ state.ignoredCache.clear();
+ this.markDirtyOverflow(state);
+ } else if (this.knownIgnored(state, path)) {
+ // Build output or a cache under a watched directory (`src/__pycache__/`):
+ // git already said so, and a rewrite of it is no change worth a capture.
+ // Should the rules change outside `.gitignore` (`.git/info/exclude`),
+ // the reconcile pass still finds the file.
+ return;
+ }
+ if (event === "rename" && !state.manifest?.has(path)) void this.noteRenamedPath(state, path, prefix === "" && !relative.includes("/"));
+ this.markDirty(state, path);
+ this.queueChangedPath(state, path);
+ this.scheduleChange(state, "fs_change");
+ }
+
+ /**
+ * A rename of something the manifest does not list may be a directory
+ * appearing (or moving): its files arrive without events of their own, so
+ * the next capture rescans, and a new top-level directory gets a watcher.
+ * A directory git ignores (rebuilt build output) is neither.
+ */
+ private async noteRenamedPath(state: SessionState, path: string, topLevel: boolean): Promise<void> {
+ try {
+ if (!(await lstat(resolve(state.root, path))).isDirectory()) return;
+ } catch {
+ return;
+ }
+ if (await this.checkIgnored(state, path)) return;
+ if (state.finished) return;
+ this.markDirtyOverflow(state);
+ if (topLevel && state.watchMode === "watching" && state.watchedPaths.includes(".") && !state.watchedPaths.includes(path)
+ && state.watchers.length <= MAX_WATCH_ROOTS) {
+ this.addWatcher(state, path);
+ }
+ this.scheduleChange(state, "fs_change");
+ }
+
+ private markDirty(state: SessionState, path: string): void {
+ if (state.dirtyOverflow) return;
+ if (state.dirty.size >= MAX_DIRTY_PATHS) {
+ this.markDirtyOverflow(state);
+ return;
+ }
+ state.dirty.add(path);
+ }
+
+ private markDirtyOverflow(state: SessionState): void {
+ state.dirtyOverflow = true;
+ state.dirty.clear();
+ }
+
+ private startReconcile(state: SessionState): void {
+ if (state.scanTimer || state.finished) return;
+ state.scanTimer = setInterval(() => void this.reconcile(state).catch(() => undefined), this.fallbackScanMs);
state.scanTimer.unref?.();
}
+ private stopReconcile(state: SessionState): void {
+ if (state.scanTimer) clearInterval(state.scanTimer);
+ state.scanTimer = null;
+ }
+
+ /**
+ * The safety net under the watcher: lists the tree and stats every file,
+ * without reading any, and marks dirty whatever a session's manifest does
+ * not describe as it stands. Runs on the fallback interval while watching;
+ * one pass serves every session watching the same root, so a workspace
+ * with several chats open is listed once per interval, not once per chat.
+ */
+ private async reconcile(state: SessionState): Promise<void> {
+ if (state.finished || state.watchMode !== "watching" || !state.manifest) return;
+ const shared = this.caches.get(resolve(state.root));
+ // Another session's timer already ran the pass for this interval.
+ if (!shared || shared.entries !== state.cache || shared.reconciling || Date.now() - shared.reconciledAt < this.fallbackScanMs * 0.9) return;
+ const sessions = [...this.sessions.values()]
+ .filter((other) => other.cache === state.cache && !other.finished && other.watchMode === "watching" && other.manifest !== null);
+ shared.reconciling = true;
+ shared.reconciledAt = Date.now();
+ this.metrics.reconciles += 1;
+ try {
+ const cache = state.cache;
+ const listing = await listWorkspaceFiles(state.root);
+ const yielder = new LoopYielder();
+ await mapBounded(listing.paths, SCAN_CONCURRENCY, async (path) => {
+ const file = await lstat(resolve(state.root, path)).catch(() => null);
+ if (file) this.metrics.fileStats += 1;
+ const eligible = file && file.isFile() && !file.isSymbolicLink() && file.size <= MAX_COLLECTOR_FILE_BYTES ? file : null;
+ const cached = cache.get(path);
+ for (const session of sessions) {
+ const known = session.manifest?.get(path);
+ const stale = eligible === null
+ ? known !== undefined
+ : !cached || cached.size !== eligible.size || cached.mtimeMs !== eligible.mtimeMs
+ || (!cached.binary && known?.sha256 !== cached.sha256) || (cached.binary && known !== undefined);
+ if (stale) this.markDirty(session, path);
+ }
+ await yielder.pause();
+ });
+ const listed = new Set(listing.paths);
+ for (const session of sessions) {
+ for (const path of session.manifest?.keys() ?? []) {
+ if (!listed.has(path)) this.markDirty(session, path);
+ }
+ }
+ } finally {
+ shared.reconciling = false;
+ }
+ for (const session of sessions) {
+ if (!session.finished && (session.dirty.size > 0 || session.dirtyOverflow)) this.scheduleChange(session, "periodic");
+ }
+ }
+
+ // --- git ignore checks, batched --------------------------------------------
+
+ /** Whether git ignores `path`, answered from a short-lived cache or one batched `git check-ignore`. */
+ private checkIgnored(state: SessionState, path: string): Promise<boolean> {
+ const cached = state.ignoredCache.get(path);
+ if (cached !== undefined) return Promise.resolve(cached);
+ return new Promise((resolvePromise) => {
+ let waiters = state.ignoreBatch.get(path);
+ if (!waiters) {
+ waiters = [];
+ state.ignoreBatch.set(path, waiters);
+ }
+ waiters.push(resolvePromise);
+ if (state.ignoreBatch.size >= IGNORE_CHECK_BATCH_MAX) {
+ void this.flushIgnoreBatch(state);
+ return;
+ }
+ if (!state.ignoreTimer) {
+ state.ignoreTimer = setTimeout(() => void this.flushIgnoreBatch(state), IGNORE_CHECK_BATCH_MS);
+ state.ignoreTimer.unref?.();
+ }
+ });
+ }
+
+ private async flushIgnoreBatch(state: SessionState): Promise<void> {
+ if (state.ignoreTimer) clearTimeout(state.ignoreTimer);
+ state.ignoreTimer = null;
+ const batch = state.ignoreBatch;
+ if (batch.size === 0) return;
+ state.ignoreBatch = new Map();
+ const ignored = await this.ignoredPaths(state, [...batch.keys()]);
+ for (const [path, waiters] of batch) {
+ const result = ignored.has(path);
+ for (const waiter of waiters) waiter(result);
+ }
+ }
+
+ /** Whether git's last answer (still cached) ignores `path` or a directory above it. */
+ private knownIgnored(state: SessionState, path: string): boolean {
+ for (let end = path.length; end > 0; end = path.lastIndexOf("/", end - 1)) {
+ if (state.ignoredCache.get(path.slice(0, end)) === true) return true;
+ }
+ return false;
+ }
+
+ /**
+ * The subset of `paths` git ignores, with one spawn (see
+ * gitIgnoredPathsIsolated) for whatever the cache does not know. `fresh`
+ * asks git about every path whatever the cache holds, and refreshes the
+ * cache with the answers: a snapshot decides what leaves the machine on
+ * that answer, since `.git/info/exclude` or `core.excludesFile` can change
+ * with no event the collector sees.
+ */
+ private async ignoredPaths(state: SessionState, paths: string[], fresh = false): Promise<Set<string>> {
+ const ignored = new Set<string>();
+ const unknown: string[] = [];
+ for (const path of paths) {
+ const cached = fresh ? undefined : state.ignoredCache.get(path);
+ if (cached === undefined) unknown.push(path);
+ else if (cached) ignored.add(path);
+ }
+ if (unknown.length > 0) {
+ const answer = (await gitIgnoredPathsIsolated(state.root, unknown, () => { this.metrics.ignoreCheckSpawns += 1; }))
+ ?? (await fallbackIgnoredPaths(state.root, unknown));
+ for (const path of unknown) {
+ const result = answer.has(path);
+ if (result) ignored.add(path);
+ if (state.ignoredCache.size >= MAX_IGNORED_CACHE_ENTRIES) {
+ const oldest = state.ignoredCache.keys().next().value;
+ if (oldest !== undefined) state.ignoredCache.delete(oldest);
+ }
+ state.ignoredCache.set(path, result);
+ }
+ }
+ return ignored;
+ }
+
+ // --- per-root hash cache ---------------------------------------------------
+
+ private acquireCache(root: string): Map<string, HashCacheEntry> {
+ const key = resolve(root);
+ let shared = this.caches.get(key);
+ if (!shared) {
+ shared = { refs: 0, entries: new Map(), reconciling: false, reconciledAt: 0 };
+ this.caches.set(key, shared);
+ }
+ shared.refs += 1;
+ return shared.entries;
+ }
+
+ private releaseCache(state: SessionState): void {
+ const key = resolve(state.root);
+ const shared = this.caches.get(key);
+ if (!shared || shared.entries !== state.cache) return;
+ shared.refs -= 1;
+ if (shared.refs <= 0) this.caches.delete(key);
+ }
+
+ /** Shared per-root hash caches still held: none once every session on a root has finished. */
+ cacheStatus(): { roots: number; entries: number } {
+ let entries = 0;
+ for (const shared of this.caches.values()) entries += shared.entries.size;
+ return { roots: this.caches.size, entries };
+ }
+
+ /** Watcher, dirty-set and cache state of a session, for tests and profiling. */
+ sessionDiagnostics(sessionId: string): {
+ watchMode: SessionState["watchMode"];
+ watchedPaths: string[];
+ dirtyPaths: number;
+ dirtyOverflow: boolean;
+ manifestEntries: number;
+ cacheEntries: number;
+ journalEntries: number;
+ traceEvents: number;
+ lastChangeAt: number;
+ } | null {
+ const state = this.sessions.get(sessionId);
+ if (!state) return null;
+ return {
+ watchMode: state.watchMode,
+ watchedPaths: [...state.watchedPaths],
+ dirtyPaths: state.dirty.size,
+ dirtyOverflow: state.dirtyOverflow,
+ manifestEntries: state.manifest?.size ?? 0,
+ cacheEntries: state.cache.size,
+ journalEntries: state.changeJournal.size,
+ traceEvents: state.trace.length,
+ lastChangeAt: state.lastChangeAt,
+ };
+ }
+
recordTrace(sessionId: string, type: string, data?: unknown): void {
const state = this.sessions.get(sessionId);
if (!state || state.finished) return;
@@ -2005,6 +3326,7 @@ export class WorkspaceCollector {
}
if (!path || path.startsWith("../") || isCollectorPathDenied(path) || state.touchedPaths.has(path)) return;
state.touchedPaths.add(path);
+ this.markDirty(state, path);
this.queueChangedPath(state, path);
this.scheduleChange(state, "fs_change");
}
@@ -2149,10 +3471,12 @@ export class WorkspaceCollector {
if (!baseline) return;
let emitted = 0;
for (const [path, stat] of current) {
- if (emitted >= MAX_ARTIFACT_EVENTS_PER_TURN) break;
const previous = baseline.get(path);
if (previous && previous.size === stat.size && previous.mtimeMs === stat.mtimeMs) continue;
- if (stat.size > MAX_ARTIFACT_HASH_BYTES) continue;
+ // The turn's change snapshot must carry it even where no watcher saw it
+ // land (a tool writing into a directory beyond MAX_WATCH_ROOTS).
+ this.markDirty(state, path);
+ if (emitted >= MAX_ARTIFACT_EVENTS_PER_TURN || stat.size > MAX_ARTIFACT_HASH_BYTES) continue;
try {
const sha256 = await sha256File(resolve(state.root, path));
this.appendTrace(state, "artifact", { path: collectorPathForUpload(path), sha256, bytes: stat.size });
@@ -2193,16 +3517,25 @@ export class WorkspaceCollector {
}
state.finished = true;
if (state.changeTimer) clearTimeout(state.changeTimer);
- if (state.scanTimer) clearInterval(state.scanTimer);
- state.watcher?.close();
+ if (state.deferTimer) clearTimeout(state.deferTimer);
+ state.changeTimer = null;
+ state.deferTimer = null;
+ this.stopReconcile(state);
+ this.closeWatchers(state);
+ // Journal captures still waiting on an ignore answer get it now, in one spawn.
+ void this.flushIgnoreBatch(state);
this.enqueue(state, async () => {
await state.ready;
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
await state.changeCaptureTail;
if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
- await this.uploadTrace(state, pendingTrace);
- await this.uploadWorkspace(state, "end", "session_end");
- this.sessions.delete(sessionId);
+ try {
+ await this.uploadTrace(state, pendingTrace);
+ await this.uploadWorkspace(state, "end", "session_end");
+ } finally {
+ this.releaseCache(state);
+ this.sessions.delete(sessionId);
+ }
});
}
@@ -2222,12 +3555,13 @@ export class WorkspaceCollector {
});
}
- /** Resolves once every queued capture and upload for the session has settled. */
+ /** Resolves once every queued capture and upload for the session has settled, and its watchers are up. */
async idle(sessionId: string): Promise<void> {
const state = this.sessions.get(sessionId);
if (!state) return;
await state.changeCaptureTail.catch(() => undefined);
await state.tail.catch(() => undefined);
+ await state.watchSetup;
}
async stop(): Promise<void> {
@@ -2240,6 +3574,7 @@ export class WorkspaceCollector {
await Promise.allSettled([...this.sessions.values()].map((state) => state.tail));
await this.spoolTail.catch(() => undefined);
await this.ledgerWriteTail.catch(() => undefined);
+ if (!this.ledgerPath) await rm(this.tempDir, { recursive: true, force: true }).catch(() => undefined);
}
private scheduleChange(state: SessionState, trigger: Extract<ChangeTrigger, "fs_change" | "periodic">): void {
@@ -2257,8 +3592,19 @@ export class WorkspaceCollector {
state.changeTimer.unref?.();
}
- private async captureChange(state: SessionState, trigger: ChangeTrigger): Promise<void> {
+ /**
+ * Runs one change capture. Nothing dirty, journaled or committed since the
+ * last accepted snapshot means no scan at all. A prompt or turn milestone
+ * captures at once; a filesystem or periodic trigger due inside the minimum
+ * interval since the last change snapshot is deferred and merged with
+ * whatever else arrives before the interval elapses.
+ */
+ private async captureChange(state: SessionState, trigger: ChangeTrigger, deferred = false): Promise<void> {
await state.ready;
+ const milestone = trigger === "prompt" || trigger === "turn_completed";
+ if (milestone && state.watchMode === "watching") await new Promise((resolvePromise) => setTimeout(resolvePromise, MILESTONE_SETTLE_MS));
+ // Journal captures waiting on an ignore answer get it now rather than on the batch timer.
+ void this.flushIgnoreBatch(state);
await state.changeCaptureTail;
// A finished session is about to upload its end snapshot, which already
// carries everything a queued change capture would.
@@ -2271,14 +3617,38 @@ export class WorkspaceCollector {
});
});
}
- const signature = await workspaceSignature(state.root);
- const changed = Boolean(signature) && (signature !== state.lastSignature || this.journalHasChanges(state));
- if (trigger === "prompt" || trigger === "turn_completed") {
- this.appendTrace(state, "collector.trigger", { trigger, captured: changed });
+ if (!(await this.hasPendingChanges(state))) {
+ this.metrics.capturesSkipped += 1;
+ if (milestone) this.appendTrace(state, "collector.trigger", { trigger, captured: false });
+ return;
+ }
+ const wait = !milestone && !deferred && state.lastChangeAt > 0 ? this.minChangeIntervalMs - (Date.now() - state.lastChangeAt) : 0;
+ if (wait > 0) {
+ this.deferChange(state, trigger, wait);
+ return;
}
- if (!changed) return;
- state.lastSignature = signature;
- await this.uploadWorkspace(state, "change", trigger);
+ const captured = await this.uploadWorkspace(state, "change", trigger);
+ if (milestone) this.appendTrace(state, "collector.trigger", { trigger, captured });
+ }
+
+ /** Whether anything could have moved since the last accepted snapshot; one `git rev-parse` at most. */
+ private async hasPendingChanges(state: SessionState): Promise<boolean> {
+ if (state.dirtyOverflow || state.watchMode !== "watching" || state.dirty.size > 0 || this.journalHasChanges(state)) return true;
+ return (await gitHead(state.root)) !== state.lastHead;
+ }
+
+ private deferChange(state: SessionState, trigger: ChangeTrigger, wait: number): void {
+ this.metrics.capturesDeferred += 1;
+ // A filesystem change is the more specific reason; keep it over the periodic pass.
+ if (state.deferTrigger !== "fs_change") state.deferTrigger = trigger;
+ if (state.deferTimer) return;
+ state.deferTimer = setTimeout(() => {
+ state.deferTimer = null;
+ const reason = state.deferTrigger ?? trigger;
+ state.deferTrigger = null;
+ this.enqueue(state, () => this.captureChange(state, reason, true));
+ }, wait);
+ state.deferTimer.unref?.();
}
/**
@@ -2288,9 +3658,9 @@ export class WorkspaceCollector {
* should cost an upload.
*/
private journalHasChanges(state: SessionState): boolean {
- if (!state.baseline) return state.changeJournal.size > 0;
+ if (!state.manifest) return state.changeJournal.size > 0;
for (const entry of state.changeJournal.values()) {
- const previous = state.baseline.get(entry.path);
+ const previous = state.manifest.get(entry.path)?.sha256;
if (entry.status === "present" ? previous !== (entry.sha256 ?? sha256Hex(entry.content ?? "")) : previous !== undefined) {
return true;
}
@@ -2300,34 +3670,62 @@ export class WorkspaceCollector {
private queueChangedPath(state: SessionState, filename: string): void {
if (state.finished) return;
+ const absolute = resolve(state.root, filename);
+ const path = portablePath(state.root, absolute);
+ if (!path || path.startsWith("../") || isCollectorPathDenied(path)) return;
+ // Several events for one path before its capture starts collapse into one read.
+ if (state.pendingJournal.has(path)) return;
+ state.pendingJournal.add(path);
+ // The ignore check is registered as the event arrives, not when the
+ // serial capture reaches the path, so a burst of events shares one spawn.
+ // A path the manifest lists is checked too: the user may just have added
+ // it to `.gitignore` or `.git/info/exclude`. (The snapshot asks git again
+ // before any journal entry leaves; this check spares the read.)
+ const ignored = this.checkIgnored(state, path);
+ ignored.catch(() => undefined);
state.changeCaptureTail = state.changeCaptureTail
.catch(() => undefined)
- .then(() => this.captureChangedPath(state, filename))
+ .then(async () => {
+ state.pendingJournal.delete(path);
+ if (await ignored) return;
+ await this.captureChangedPath(state, path);
+ })
.catch((error: unknown) => {
this.log("warn", "OmniRush changed-file capture failed", {
sessionId: state.id,
- path: filename,
+ path,
error: error instanceof Error ? error.message : "unknown",
});
});
}
- private async captureChangedPath(state: SessionState, filename: string): Promise<void> {
- const absolute = resolve(state.root, filename);
- const path = portablePath(state.root, absolute);
- if (!path || path.startsWith("../") || isCollectorPathDenied(path) || await gitIgnoresPath(state.root, path)) return;
+ private async captureChangedPath(state: SessionState, path: string): Promise<void> {
+ if (state.finished) return;
+ const absolute = resolve(state.root, path);
+ // Reached through a symlinked directory: a file outside the workspace (a
+ // traced read of `linked/util.ts`), never journaled.
+ if (!(await hasRealAncestors(state.root, path))) return;
let entry: ChangeJournalEntry;
try {
const file = await lstat(absolute);
if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_COLLECTOR_FILE_BYTES) {
entry = { path, at: new Date().toISOString(), status: "skipped" };
+ } else if (file.size > MAX_CHANGE_JOURNAL_BYTES) {
+ // Larger than the whole journal: it could only evict every other entry
+ // and itself. The dirty set carries it to the next capture, so a file
+ // appended to many times a second is not re-read and re-scrubbed per write.
+ return;
} else {
const buffer = await readFile(absolute);
+ this.metrics.fileReads += 1;
if (buffer.length > MAX_COLLECTOR_FILE_BYTES || isBinary(buffer)) {
entry = { path, at: new Date().toISOString(), status: "skipped" };
} else {
- const content = redactCollectorContent(path, buffer.toString("utf8"));
- entry = { path, at: new Date().toISOString(), status: "present", content, sha256: sha256Hex(content) };
+ // The journal's read doubles as the snapshot's: the cache learns the
+ // file now, so the next capture only stats it.
+ const redacted = redactedCacheEntry(path, buffer, file.size, file.mtimeMs, uploadPathFor(path, state.cache.get(path)), this.metrics);
+ state.cache.set(path, redacted.entry);
+ entry = { path, at: new Date().toISOString(), status: "present", content: redacted.text, sha256: redacted.entry.sha256 };
}
}
} catch {
@@ -2368,83 +3766,173 @@ export class WorkspaceCollector {
return [...state.touchedPaths].slice(0, MAX_TOUCHED_PATHS).map(collectorPathForUpload);
}
- private async uploadWorkspace(state: SessionState, type: Exclude<SnapshotType, "trace">, trigger: CollectorTrigger): Promise<void> {
+ /**
+ * Captures one workspace snapshot. A start snapshot scans the whole tree and
+ * carries every eligible file's content (files_scope "full", up to the cap);
+ * change and end snapshots carry only the files whose digest moved since the
+ * last accepted snapshot (files_scope "changed") plus the full manifest and
+ * the changed paths. The scan is targeted at the watcher's dirty set whenever
+ * that set is trusted, and covers the whole tree otherwise. Resolves with
+ * whether a snapshot went out.
+ */
+ private async uploadWorkspace(state: SessionState, type: Exclude<SnapshotType, "trace">, trigger: CollectorTrigger): Promise<boolean> {
const remaining = MAX_COLLECTOR_SESSION_BYTES - state.sentBytes;
- if (remaining <= 1_024) return;
- // The git block comes first: its status and diff name the files a capped
- // snapshot must keep, and its measured size (the diff alone may reach its
- // own cap) is what the fixed wrapper margin cannot promise to cover.
- const git = await collectGitBlock(state.root);
- const budget = Math.min(this.snapshotMaxBytes - SNAPSHOT_WRAPPER_MARGIN_BYTES - serializedBytes(git), remaining - 1_024);
- const [scan, environment] = await Promise.all([
- scanWorkspace(state.root, state.hashCache, state.baseline, budget, type === "start", snapshotPriorityPaths(state.touchedPaths, git)),
- this.environment(),
- ]);
- if (scan.capOmittedCount > 0) {
- this.appendTrace(state, "collector.snapshot_cap", {
- snapshot_type: type,
- trigger,
- omitted_count: scan.capOmittedCount,
- omitted_bytes: scan.capOmittedBytes,
- budget_bytes: budget,
- });
- this.log("warn", "OmniRush collection snapshot trimmed to the snapshot cap", {
- sessionId: state.id,
- snapshotType: type,
+ if (remaining <= 1_024) return false;
+ const cache = state.cache;
+ // Paths reported from here on belong to the next capture.
+ const dirty = state.dirty;
+ state.dirty = new Set();
+ const previous = state.manifest;
+ const targeted = type === "change" && !state.dirtyOverflow && state.watchMode === "watching";
+ if (state.watchMode !== "polling") state.dirtyOverflow = false;
+ let accepted = false;
+ try {
+ // git's current answer for everything this snapshot could carry that no
+ // listing vouches for (the paths a targeted scan inspects, and every
+ // journal entry), in one batch: nothing git ignores now leaves the
+ // machine, whatever it answered when the event arrived.
+ const reported = targeted && previous !== null ? [...dirty].filter((path) => !isCollectorPathDenied(path)) : [];
+ const journaled = type === "change" || type === "end" ? [...state.changeJournal.values()] : [];
+ const checked = [...new Set([...reported, ...journaled.map((entry) => entry.path)])];
+ const ignored = checked.length > 0 ? await this.ignoredPaths(state, checked, true) : new Set<string>();
+ const journal = journaled.filter((entry) => !ignored.has(entry.path));
+ if (journal.length < journaled.length) {
+ const dropped = journaled.filter((entry) => ignored.has(entry.path));
+ this.acknowledgeJournal(state, dropped);
+ for (const entry of dropped) cache.delete(entry.path);
+ }
+ const scan = targeted && previous !== null
+ ? await scanDirtyPaths(state.root, cache, previous, reported, ignored, state.listing, this.metrics)
+ : await scanWorkspaceFull(state.root, cache, previous, type === "start", this.metrics,
+ type === "start" ? (paths) => this.installWatchers(state, paths) : undefined);
+ // A change capture that found nothing moved stops at one `git rev-parse`
+ // (a commit changes history without touching a file); the full git block
+ // with its status, log and diff is collected only for a snapshot that goes out.
+ if (type === "change" && scan.changed !== null && scan.changed.size === 0 && scan.removed === 0 && !this.journalHasChanges(state)
+ && (await gitHead(state.root)) === state.lastHead) {
+ accepted = true;
+ return false;
+ }
+ // Its status and diff name the files a capped snapshot must keep.
+ const git = await collectGitBlock(state.root);
+ const candidates = candidatesFor(scan, snapshotPriorityPaths(state.touchedPaths, git));
+ const environment = await this.environment();
+ const rootName = workspaceRootName(state.root);
+ const touchedPaths = this.touchedPathsForUpload(state);
+ const metadata = JSON.stringify({
+ schema_version: COLLECTOR_SCHEMA_VERSION,
+ workspace_id: state.workspaceId,
+ session_id: state.id,
+ session_segment: state.segment,
+ session_resumed: state.resumed,
trigger,
- omittedCount: scan.capOmittedCount,
- omittedBytes: scan.capOmittedBytes,
- budgetBytes: budget,
+ touched_paths: touchedPaths,
+ ...PRIVACY_POLICY,
+ denied_file_count: scan.deniedCount,
+ root_name: rootName,
+ git: { commit: git?.commit ?? null, branch: git?.branch ?? null, dirty: git ? String(git.dirty) : "false" },
});
- }
- // The baseline is the last *captured* manifest: the next change snapshot
- // reports exactly what moved since this one, whatever the upload outcome.
- state.baseline = new Map(scan.manifest.map((entry) => [entry.path, entry.sha256]));
- const rootName = workspaceRootName(state.root);
- const touchedPaths = this.touchedPathsForUpload(state);
- const metadata = JSON.stringify({
- schema_version: COLLECTOR_SCHEMA_VERSION,
- workspace_id: state.workspaceId,
- session_id: state.id,
- session_segment: state.segment,
- session_resumed: state.resumed,
- trigger,
- touched_paths: touchedPaths,
- ...PRIVACY_POLICY,
- denied_file_count: scan.deniedCount,
- root_name: rootName,
- git: { commit: git?.commit ?? null, branch: git?.branch ?? null, dirty: git ? String(git.dirty) : "false" },
- });
- const files: CollectorFile[] = [
- { path: "__omnirush__/workspace.json", content: metadata, sha256: sha256Hex(metadata) },
- ...scan.files,
- ];
- const journal = type === "change" || type === "end" ? [...state.changeJournal.values()] : [];
- if (journal.length > 0) {
- const changes = JSON.stringify({
- schema_version: 1,
- session_id: state.id,
- entries: journal.map((entry) => ({ ...entry, path: collectorPathForUpload(entry.path) })),
+ const leading: CollectorFile[] = [{ path: "__omnirush__/workspace.json", content: metadata, sha256: sha256Hex(metadata) }];
+ if (journal.length > 0) {
+ const changes = JSON.stringify({
+ schema_version: 1,
+ session_id: state.id,
+ entries: journal.map((entry) => ({ ...entry, path: collectorPathForUpload(entry.path) })),
+ });
+ leading.push({ path: "__omnirush__/changes.json", content: changes, sha256: sha256Hex(changes) });
+ }
+ const filesScope: "full" | "changed" = type === "start" ? "full" : "changed";
+ const extras: Record<string, unknown> = {
+ workspace: { root_name: rootName, git },
+ environment,
+ touched_paths: touchedPaths,
+ files_scope: filesScope,
+ ...(filesScope === "changed" ? { changed_paths: candidates.map((candidate) => candidate.uploadPath) } : {}),
+ };
+ // Everything around files[] is measured (the manifest entry by entry, so
+ // it never has to exist as one string); the rest of the cap is content.
+ let fixedBytes = serializedBytes(extras) + SNAPSHOT_WRAPPER_MARGIN_BYTES;
+ for (const file of leading) fixedBytes += serializedBytes(file) + 1;
+ for (const entry of scan.manifest.values()) fixedBytes += Buffer.byteLength(manifestEntryJson(entry)) + 1;
+ const budget = Math.max(0, Math.min(this.snapshotMaxBytes, remaining - 1_024) - fixedBytes);
+ const selection = selectSnapshotContent(candidates, budget);
+ let omittedCount = selection.omittedCount;
+ let omittedBytes = selection.omittedBytes;
+ const manifest = scan.manifest;
+ const uploaded = await this.uploadEnvelope(state, type, trigger, {
+ extras,
+ writeFiles: async (emit) => {
+ for (const file of leading) await emit(file);
+ let reserved = 0;
+ for (const candidate of candidates) {
+ if (selection.selected.has(candidate.path)) reserved += candidate.cost;
+ }
+ let used = 0;
+ const yielder = new LoopYielder();
+ const pool = new ReadPool(1);
+ const ancestors: AncestorCache = new Map();
+ for (const candidate of candidates) {
+ if (!selection.selected.has(candidate.path)) continue;
+ reserved -= candidate.cost;
+ const read = await readUploadContent(state.root, candidate.path, cache, this.metrics, pool, ancestors);
+ await yielder.pause();
+ // Changed underneath the scan; the next snapshot sees its final form.
+ if (!read) continue;
+ const cost = candidateCost(read.entry);
+ if (used + cost + reserved > budget) {
+ omittedCount += 1;
+ omittedBytes += read.entry.bytes;
+ continue;
+ }
+ // files[] and the manifest describe the same bytes even if the
+ // file moved between the scan and this read.
+ if (read.entry.sha256 !== manifest.get(candidate.path)?.sha256) manifest.set(candidate.path, read.entry);
+ await emit({ path: candidate.uploadPath, content: read.content, sha256: read.entry.sha256 });
+ used += cost;
+ }
+ },
+ manifest: () => manifest.values(),
+ privacy: () => ({
+ ...PRIVACY_POLICY,
+ denied_file_count: scan.deniedCount,
+ manifest_truncated: scan.manifestTruncated,
+ files_truncated: omittedCount > 0,
+ diff_truncated: git?.diff_truncated ?? false,
+ snapshot_cap_omitted_count: omittedCount,
+ snapshot_cap_omitted_bytes: omittedBytes,
+ }),
});
- files.push({ path: "__omnirush__/changes.json", content: changes, sha256: sha256Hex(changes) });
+ if (omittedCount > 0) {
+ this.appendTrace(state, "collector.snapshot_cap", {
+ snapshot_type: type,
+ trigger,
+ omitted_count: omittedCount,
+ omitted_bytes: omittedBytes,
+ budget_bytes: budget,
+ });
+ this.log("warn", "OmniRush collection snapshot trimmed to the snapshot cap", {
+ sessionId: state.id,
+ snapshotType: type,
+ trigger,
+ omittedCount,
+ omittedBytes,
+ budgetBytes: budget,
+ });
+ }
+ if (!uploaded) return false;
+ // The baseline is the last *accepted* manifest (uploaded or spooled): the
+ // next change snapshot reports exactly what moved since this one.
+ accepted = true;
+ state.manifest = manifest;
+ state.listing = { denied: scan.deniedCount, truncated: scan.manifestTruncated };
+ state.lastHead = git?.commit ?? null;
+ if (journal.length > 0) this.acknowledgeJournal(state, journal);
+ if (type === "change") state.lastChangeAt = Date.now();
+ return true;
+ } finally {
+ // Nothing was accepted: the same files must go out with the next capture.
+ if (!accepted) this.markDirtyOverflow(state);
}
- const uploaded = await this.uploadEnvelope(state, type, trigger, files, {
- workspace: { root_name: rootName, git },
- environment,
- manifest: scan.manifest.map((entry) => ({ ...entry, path: collectorPathForUpload(entry.path) })),
- privacy: {
- ...PRIVACY_POLICY,
- denied_file_count: scan.deniedCount,
- manifest_truncated: scan.manifestTruncated,
- files_truncated: scan.contentTruncated,
- diff_truncated: git?.diff_truncated ?? false,
- snapshot_cap_omitted_count: scan.capOmittedCount,
- snapshot_cap_omitted_bytes: scan.capOmittedBytes,
- },
- touched_paths: touchedPaths,
- });
- if (uploaded && journal.length > 0) this.acknowledgeJournal(state, journal);
- state.lastSignature = await workspaceSignature(state.root);
}
private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {
@@ -2459,12 +3947,18 @@ export class WorkspaceCollector {
} catch {
events = [];
}
- await this.uploadEnvelope(state, "trace", "trace_flush", [{ path: "__omnirush__/trace.json", content, sha256: sha256Hex(content) }], {
- workspace: { root_name: workspaceRootName(state.root), git: null },
- environment: await this.environment(),
- manifest: [],
- privacy: { ...PRIVACY_POLICY },
- touched_paths: this.touchedPathsForUpload(state),
+ await this.uploadEnvelope(state, "trace", "trace_flush", {
+ extras: {
+ workspace: { root_name: workspaceRootName(state.root), git: null },
+ environment: await this.environment(),
+ touched_paths: this.touchedPathsForUpload(state),
+ files_scope: "full",
+ },
+ writeFiles: async (emit) => {
+ await emit({ path: "__omnirush__/trace.json", content, sha256: sha256Hex(content) });
+ },
+ manifest: () => [],
+ privacy: () => ({ ...PRIVACY_POLICY }),
trace: events,
});
}
@@ -2539,90 +4033,144 @@ export class WorkspaceCollector {
return { ok: false, retryable: true, reason: lastReason };
}
- private async uploadEnvelope(
- state: SessionState,
- snapshotType: SnapshotType,
- trigger: CollectorTrigger,
- files: CollectorFile[],
- extras: Record<string, unknown>,
- ): Promise<boolean> {
+ private async envelopeTempPath(): Promise<string> {
+ await mkdir(this.tempDir, { recursive: true, mode: 0o700 });
+ return join(this.tempDir, `${spoolId(++this.spoolCounter)}.zst.tmp`);
+ }
+
+ /** Removes envelope temp files a crashed process left behind. */
+ private async cleanTempDir(olderThanMs: number): Promise<void> {
+ let names: string[];
+ try {
+ names = await readdir(this.tempDir);
+ } catch {
+ return;
+ }
+ const cutoff = Date.now() - olderThanMs;
+ for (const name of names) {
+ if (!name.endsWith(".zst.tmp")) continue;
+ const path = join(this.tempDir, name);
+ try {
+ if ((await lstat(path)).mtimeMs < cutoff) await rm(path, { force: true });
+ } catch {
+ // Already gone, or in use by another instance.
+ }
+ }
+ }
+
+ /**
+ * Streams one envelope through zstd into a temp file under the state
+ * directory, uploads the compressed bytes from that file and deletes it; a
+ * retryable failure moves the file into the spool instead. Resolves with
+ * whether the envelope was accepted (delivered or spooled).
+ */
+ private async uploadEnvelope(state: SessionState, snapshotType: SnapshotType, trigger: CollectorTrigger, body: EnvelopeBody): Promise<boolean> {
if (!this.collectUrl && !this.uploader) return false;
const sequence = state.sequence + 1;
- const payload = Buffer.from(JSON.stringify({
- schema_version: COLLECTOR_SCHEMA_VERSION,
- session_id: state.id,
- session_segment: state.segment,
- session_resumed: state.resumed,
- sequence,
- snapshot_type: snapshotType,
- trigger,
- captured_at: new Date().toISOString(),
- session: this.sessionBlock(state),
- ...extras,
- files,
- }));
- if (state.sentBytes + payload.length > MAX_COLLECTOR_SESSION_BYTES) return false;
- if (payload.length > this.snapshotMaxBytes) {
- // The backend answers an oversized envelope with 422, which is never
- // retried. The scan budget keeps this from happening; reaching it means
- // the wrapper outgrew its margin, which deserves a loud line here rather
- // than a lost upload and a rejection upstream.
- this.log("warn", "OmniRush collection payload exceeded the snapshot cap", {
- sessionId: state.id,
- snapshotType,
+ const path = await this.envelopeTempPath();
+ const writer = new EnvelopeWriter(path);
+ let fileCount = 0;
+ let compressedBytes: number;
+ try {
+ const head = JSON.stringify({
+ schema_version: COLLECTOR_SCHEMA_VERSION,
+ session_id: state.id,
+ session_segment: state.segment,
+ session_resumed: state.resumed,
+ sequence,
+ snapshot_type: snapshotType,
trigger,
- bytes: payload.length,
- capBytes: this.snapshotMaxBytes,
+ captured_at: new Date().toISOString(),
+ session: this.sessionBlock(state),
+ ...body.extras,
+ });
+ await writer.write(`${head.slice(0, -1)},"files":[`);
+ await body.writeFiles(async (file) => {
+ await writer.write(`${fileCount > 0 ? "," : ""}${JSON.stringify(file)}`);
+ fileCount += 1;
});
- return false;
- }
- const compressed = await compressZstd(payload);
- if (compressed.length > MAX_COMPRESSED_BYTES) {
- this.log("warn", "OmniRush collection payload exceeded compressed limit", { sessionId: state.id, snapshotType, trigger });
- return false;
- }
- const outcome = await this.transmit(state.id, compressed);
- if (!outcome.ok) {
- if (!outcome.retryable || !this.spoolDir) {
- await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);
- throw new Error(outcome.reason);
+ await writer.write('],"manifest":[');
+ let first = true;
+ for (const entry of body.manifest()) {
+ await writer.write(`${first ? "" : ","}${manifestEntryJson(entry)}`);
+ first = false;
+ }
+ await writer.write(`],"privacy":${JSON.stringify(body.privacy())}`);
+ if (body.trace) await writer.write(`,"trace":${JSON.stringify(body.trace)}`);
+ await writer.write("}");
+ compressedBytes = await writer.finish();
+ } catch (error) {
+ await writer.abort();
+ throw error;
+ }
+ this.metrics.envelopesWritten += 1;
+ const bytes = writer.bytes;
+ try {
+ if (state.sentBytes + bytes > MAX_COLLECTOR_SESSION_BYTES) return false;
+ if (bytes > this.snapshotMaxBytes) {
+ // The backend answers an oversized envelope with 422, which is never
+ // retried. The scan budget keeps this from happening; reaching it means
+ // the wrapper outgrew its margin, which deserves a loud line here rather
+ // than a lost upload and a rejection upstream.
+ this.log("warn", "OmniRush collection payload exceeded the snapshot cap", {
+ sessionId: state.id,
+ snapshotType,
+ trigger,
+ bytes,
+ capBytes: this.snapshotMaxBytes,
+ });
+ return false;
+ }
+ if (compressedBytes > MAX_COMPRESSED_BYTES) {
+ this.log("warn", "OmniRush collection payload exceeded compressed limit", { sessionId: state.id, snapshotType, trigger });
+ return false;
+ }
+ const compressed = await readFile(path);
+ const outcome = await this.transmit(state.id, compressed);
+ if (!outcome.ok) {
+ if (!outcome.retryable || !this.spoolDir) {
+ await this.recordLedgerOutcome(state.id, "failure").catch(() => undefined);
+ throw new Error(outcome.reason);
+ }
+ await this.spoolEnvelopeFile({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressedBytes, created_at: new Date().toISOString(), attempts: 1 }, path);
+ state.sentBytes += bytes;
+ state.sequence = sequence;
+ state.failureCount += 1;
+ state.lastFailureAt = new Date().toISOString();
+ await this.persistSession(state).catch(() => undefined);
+ this.log("warn", "OmniRush collection artifact spooled for retry", {
+ sessionId: state.id,
+ snapshotType,
+ trigger,
+ sequence,
+ compressedBytes,
+ reason: outcome.reason,
+ });
+ this.retryFailures += 1;
+ this.scheduleRetry();
+ return true;
}
- await this.spoolEnvelope({ id: "", session_id: state.id, snapshot_type: snapshotType, trigger, sequence, bytes: compressed.length, created_at: new Date().toISOString(), attempts: 1 }, compressed);
- state.sentBytes += payload.length;
+ state.sentBytes += bytes;
state.sequence = sequence;
- state.failureCount += 1;
- state.lastFailureAt = new Date().toISOString();
- await this.persistSession(state).catch(() => undefined);
- this.log("warn", "OmniRush collection artifact spooled for retry", {
+ state.lastSuccessAt = new Date().toISOString();
+ await this.persistSession(state).catch((error: unknown) => {
+ this.log("warn", "OmniRush session ledger update failed", {
+ sessionId: state.id,
+ error: error instanceof Error ? error.message : "unknown",
+ });
+ });
+ this.log("info", "OmniRush collection artifact uploaded", {
sessionId: state.id,
snapshotType,
trigger,
sequence,
- compressedBytes: compressed.length,
- reason: outcome.reason,
+ fileCount,
+ compressedBytes,
});
- this.retryFailures += 1;
- this.scheduleRetry();
return true;
+ } finally {
+ await rm(path, { force: true }).catch(() => undefined);
}
- state.sentBytes += payload.length;
- state.sequence = sequence;
- state.lastSuccessAt = new Date().toISOString();
- await this.persistSession(state).catch((error: unknown) => {
- this.log("warn", "OmniRush session ledger update failed", {
- sessionId: state.id,
- error: error instanceof Error ? error.message : "unknown",
- });
- });
- this.log("info", "OmniRush collection artifact uploaded", {
- sessionId: state.id,
- snapshotType,
- trigger,
- sequence,
- fileCount: files.length,
- compressedBytes: compressed.length,
- });
- return true;
}
// --- durable retry spool -------------------------------------------------
@@ -2669,14 +4217,18 @@ export class WorkspaceCollector {
await writeFileAtomic(join(this.spoolDir!, `${meta.id}.json`), JSON.stringify(meta));
}
- private spoolEnvelope(meta: SpoolMeta, compressed: Uint8Array): Promise<void> {
+ /** Moves an already-compressed envelope file into the spool (same filesystem: a rename, no copy). */
+ private spoolEnvelopeFile(meta: SpoolMeta, sourcePath: string): Promise<void> {
return this.spoolLocked(async () => {
if (!this.spoolDir) return;
await mkdir(this.spoolDir, { recursive: true, mode: 0o700 });
const id = spoolId(++this.spoolCounter);
- // Payload first, then the metadata that makes the entry visible: a crash
- // in between leaves an orphan that the next listing removes.
- await writeFileAtomic(join(this.spoolDir, `${id}.zst`), Buffer.from(compressed));
+ const target = join(this.spoolDir, `${id}.zst`);
+ try {
+ await rename(sourcePath, target);
+ } catch {
+ await writeFileAtomic(target, await readFile(sourcePath));
+ }
await this.writeSpoolMeta({ ...meta, id });
await this.enforceSpoolBounds();
});
diff --git a/docs/workspace-collector-privacy.md b/docs/workspace-collector-privacy.md
index d6a1c4d5..bcfdee48 100644
--- a/docs/workspace-collector-privacy.md
+++ b/docs/workspace-collector-privacy.md
@@ -1,3 +1,16 @@
# Workspace collector privacy rails
-The desktop server's workspace collector (`apps/server/src/workspace-collector.ts`) snapshots consented workspace files, git metadata and session traces for omnirush.ai; the backend applies the same denylist and scrubber server-side, so a rule changed on one side must land on the other, and the desktop side may only ever be the stricter of the two. Before anything leaves the machine, credential-named paths are dropped and counted in the envelope's `privacy.denied_file_count`: any path component whose words (split on space, `_`, `-`, `.`) include key(s), secret(s), token(s), password(s), passwd, credential(s), id_rsa, id_ed25519, netrc, npmrc, pypirc, kubeconfig, wallet, seed or mnemonic, plus `.aws`, `.ssh`, `.gnupg` and `.docker/config.json`, on top of the older `.env*`, `credentials*`, `secrets*`, `keys/`, `*.pem`/`*.key` rules. A file with a source-code or docs extension (`.ts`, `.py`, `.md`, `.json`, `.yml`, ...) is kept instead and scrubbed: a secret-named assignment (`password:`, `api_key =`, `"token":`, `export CLIENT_SECRET=`, in INI/YAML/JSON/dotenv/shell/source spelling; the key is split into segments on `_`, `-`, `.`, `:`, camelCase and letter/digit boundaries, and qualifies only when a whole segment or adjacent pair is a listed family word, so `author`, `keywords` and `tokenizer_class` never do) loses its value under value rule v2: in SOURCE mode (files with a programming-language extension: js, cjs, mjs, ts, tsx, jsx, py, go, rs, java, kt, c, cc, cpp, h, hpp, rb, php, swift, cs, scala, sh, bash, zsh, ps1, lua, dart, vue, svelte, map) a value is redacted only when it is 8+ non-space characters, not a code expression, `<placeholder>` or filesystem path, not an unquoted identifier or member expression (`config.token`, `tokenFile`), and carries a digit or is 20+ mixed-case characters, while CONFIG mode (every other extension, none, git diffs and the trace) keeps the permissive rule of 8+ characters with a letter, skipping only filesystem paths and unquoted dotted identifiers, and a key whose last segment is a structural word (`length`, `ttl`, `url`, `path`, `name`, `id`, `header`, `file`, `filename`, `dir`, `mode`, `method`, `role`, `owner`, `type`, `kind`, `enabled`, `estimate`, `hash`, `digest`, `at`, `config`, `client`, `prefix`, `suffix`, `format`, `scheme`, `provider`, `status`, `state`, `label`, `description`, `title`, `class`, `field`, `fields`, `list`, `names`, `version`, `timeout`, `limit`, `max`, `min`, `interval`, `retries`, `port`) never names a secret. A `Bearer` value is redacted only when its token carries a digit or is 20+ mixed-case characters that are not a bare identifier, the AWS 40-character proximity pass runs only on documents of three or more lines (a single-line JSON document relies on its key), and provider token shapes (AWS access key ids and, near them, secret access keys; GitHub, Slack, Google, Stripe, OpenAI, Anthropic, JWTs, Discord, `Bearer` values, private key blocks, URL userinfo) become `[REDACTED]`, and personal identifiers become `[REDACTED_PII]`. `.json` files and the trace document are scrubbed as JSON (every string, keys included) so a redaction can never break an escape sequence; git diffs stay text. The same denylist filters the manifest, touched paths, git status and diff sections.
+The desktop server's workspace collector (`apps/server/src/workspace-collector.ts`) snapshots consented workspace files, git metadata and session traces for omnirush.ai; the backend applies the same denylist and scrubber server-side, so a rule changed on one side must land on the other, and the desktop side may only ever be the stricter of the two. Before anything leaves the machine, credential-named paths are dropped and counted in the envelope's `privacy.denied_file_count`: any path component whose words (split on space, `_`, `-`, `.`) include key(s), secret(s), token(s), password(s), passwd, credential(s), id_rsa, id_ed25519, netrc, npmrc, pypirc, kubeconfig, wallet, seed or mnemonic, plus `.aws`, `.ssh`, `.gnupg` and `.docker/config.json`, on top of the older `.env*`, `credentials*`, `secrets*`, `keys/`, `*.pem`/`*.key` rules. A file with a source-code or docs extension (`.ts`, `.py`, `.md`, `.json`, `.yml`, ...) is kept instead and scrubbed: a secret-named assignment (`password:`, `api_key =`, `"token":`, `export CLIENT_SECRET=`, in INI/YAML/JSON/dotenv/shell/source spelling; the key is split into segments on `_`, `-`, `.`, `:`, camelCase and letter/digit boundaries, and qualifies only when a whole segment or adjacent pair is a listed family word, so `author`, `keywords` and `tokenizer_class` never do) loses its value under value rule v2: in SOURCE mode (files with a programming-language extension: js, cjs, mjs, ts, tsx, jsx, py, go, rs, java, kt, c, cc, cpp, h, hpp, rb, php, swift, cs, scala, sh, bash, zsh, ps1, lua, dart, vue, svelte, map, patch, diff) a value is redacted only when it is 8+ non-space characters, not a code expression, `<placeholder>` or filesystem path, not an unquoted identifier or member expression (`config.token`, `tokenFile`), and carries a digit or is 20+ mixed-case characters, while CONFIG mode (every other extension, none, the envelope's git diff and the trace) keeps the permissive rule of 8+ characters with a letter, skipping only filesystem paths and unquoted dotted identifiers, and a key whose last segment is a structural word (`length`, `ttl`, `url`, `path`, `name`, `id`, `header`, `file`, `filename`, `dir`, `mode`, `method`, `role`, `owner`, `type`, `kind`, `enabled`, `estimate`, `hash`, `digest`, `at`, `config`, `client`, `prefix`, `suffix`, `format`, `scheme`, `provider`, `status`, `state`, `label`, `description`, `title`, `class`, `field`, `fields`, `list`, `names`, `version`, `timeout`, `limit`, `max`, `min`, `interval`, `retries`, `port`) never names a secret. A `Bearer` value is redacted only when its token carries a digit or is 20+ mixed-case characters that are not a bare identifier, the AWS 40-character proximity pass runs only on documents of three or more lines (a single-line JSON document relies on its key, or on a sibling naming AWS), and provider token shapes (AWS access key ids and, near them, secret access keys; GitHub, Slack, Google, Stripe, OpenAI, Anthropic, JWTs, Discord, `Bearer` values, private key blocks, URL userinfo) become `[REDACTED]`, and personal identifiers become `[REDACTED_PII]`. `.json` files and the trace document are scrubbed as JSON (every string, keys included) so a redaction can never break an escape sequence; git diffs stay text. As on the backend, a `.json` file first gets the AWS proximity pass over its raw text, an object or array whose keys or string values name aws/secret makes all its values AWS context (the container rail, which also covers the trace), every pair of a repeated key is kept and scrubbed (JSON.parse alone would drop the earlier value unscrubbed), and a document nested deeper than 512 levels gets the text scrub instead. The same denylist filters the manifest, touched paths, git status and diff sections.
+
+The backend's second-sample refinements (46ee95e) are mirrored exactly: in SOURCE mode an unquoted value must also be token-shaped (only letters, digits and `_ . / + = ~ -`, so `stagedClient?.plaintext`, `params[8]` and `[0-9a-f]{16}` stay) and the key must start the text or follow whitespace, `{`, `,`, `(`, a quote or backtick, `+` or `#` (so `?token=` or `pipeline-token:...` inside a string literal stays), while CONFIG keeps its word-start guard. 53 more structural last segments never name a secret (`in`, `out`, `expires`, `ms`, `bytes`, `encryption`, `algorithm`, `policy`, `source`, `handler`, `event`, `message`, `error`, `envelope`, `ref`, `alias`, ...; `auth_code` and `password_plaintext` still do), a JSON key that looks like a path or filename (holds `/` or `\`, or ends in a 1-5 character extension) is not a secret key in the JSON rail though its value still gets the provider-token scrub, and URL userinfo is redacted only when it holds a password (`scheme://user:pass@host` becomes `scheme://[REDACTED]@host`; a bare `user@host` stays). The assignment rule reads whitespace exactly as the backend's Python `\s` does, and in SOURCE mode a desktop key may also start right after any non-ASCII character (a byte order mark, or the `é` of `café_token`, which the backend's Unicode `\w` spans instead), so the desktop never redacts less than the backend there. URL userinfo with an empty user (`redis://:<password>@host`, Redis's AUTH form) is redacted too; the backend at 46ee95e misses it, so there the desktop is the stricter side.
+
+## Snapshot scope, incremental captures and the `files_scope` field
+
+Every envelope (contract v2) carries `files_scope`, which tells a consumer what `files[]` covers:
+
+- `"full"`: `files[]` holds the content of every eligible file the snapshot cap allowed (the `start` snapshot of a session, whether new or resumed) or, for a `trace` envelope, its single trace document.
+- `"changed"`: `files[]` holds only the files whose redacted sha256 differs from the previous snapshot the gateway accepted (delivered or spooled) for this session; a `change` or `end` snapshot. The envelope also carries `changed_paths`, the redacted paths of every such file, including any the snapshot cap kept out of `files[]`, and the `manifest` still lists every eligible file with its sha256 and size, so a consumer can tell a deleted file (gone from the manifest) from an unchanged one (same digest as before) without content. A snapshot that failed to upload advances nothing: the same files are sent again with the next one.
+
+Neither field changes what may leave the machine: `files[]`, `changed_paths` and the manifest go through the same denylist and scrubber, and a manifest digest is always the sha256 of the redacted UTF-8 content.
+
+The desktop collector keeps its footprint bounded while producing these snapshots. Envelopes are streamed as JSON through zstd into a temp file under the app state directory (`omnirush-collector-tmp`) and uploaded from that file, so no envelope string or buffer is ever held whole; a failed upload moves the file into the retry spool (same filesystem, a rename) and a delivered one is deleted at once. Files are read with bounded concurrency and the scan yields to the event loop between files. A per-root digest cache keyed by path, size and mtime, shared by every chat open on the same workspace, means unchanged files are neither re-read nor re-hashed, and a file the scrubber left byte-identical is uploaded as read (verified by digest) without a second pass through the redaction rules. The scrubber runs the assignment and e-mail rules only over lines holding their keyword or an "@" and skips a rule whose literal the text lacks; neither shortcut changes a single output byte. Change snapshots come from watcher-driven dirty tracking: a prompt or completed turn with nothing dirty, journaled or committed since the last accepted snapshot records only its `collector.trigger` event; a change is scanned by dirty path, and before anything is read git is asked afresh, in one batched `git check-ignore`, about every reported path and every journal entry (or the workspace's own `.gitignore` files are applied where git cannot answer), so a file the user has just ignored through `.gitignore`, `.git/info/exclude` or `core.excludesFile` leaves the manifest and the journal at once; a path git refuses to answer for (one beyond a symlinked directory) is isolated by splitting the batch, so it cannot change the answer for the others, and counts as ignored. No path reached through a symlinked directory is ever read or listed. The whole tree is rescanned only when the dirty set overflowed, a directory appeared, a `.gitignore` changed, or the workspace is polled. The watchers go up from the start snapshot's listing before any file is read and cover the root, the top-level directories that hold listed files and then every other top-level directory git does not ignore and the denylist does not name (an empty `scripts/`, a `config/` holding only a denied `.env`), so `.git/`, `node_modules/`, gitignored build output and denied directories never reach them, and a gitignored directory that appears later (a rebuilt `dist/`) is neither watched nor rescanned; at the end of a turn every untracked file the turn created or changed is marked dirty too, so the turn's own snapshot carries it even where no watcher reaches; a workspace with more than 20,000 eligible files, or more than 64 top-level directories holding listed files, is polled on snapshots instead of watched (past 64 watched directories in all, the rest are left to the reconcile pass and the end-of-turn scan), and one reconcile pass (listing and stat only) per workspace every 60 seconds covers every chat watching it. A file larger than the change journal's budget is not journaled on every write; the next snapshot carries it. Filesystem- and reconcile-driven change snapshots are spaced at least 30 seconds apart, with the triggers in between merged into one; prompt and turn milestones capture at once.