Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat: shared upload rules v3

ogsamratb054a84e0d4e
diff --git a/apps/server/src/capture-context.test.ts b/apps/server/src/capture-context.test.ts
index 65e55c11..dca44ef8 100644
--- a/apps/server/src/capture-context.test.ts
+++ b/apps/server/src/capture-context.test.ts
@@ -683,7 +683,7 @@ test("CLI/desktop parity: the context modules match PARITY.sha256 (the same list
const contextDir = path.join(import.meta.dir, "context");
const lines = fs.readFileSync(path.join(contextDir, "PARITY.sha256"), "utf8").split("\n").filter((line) => line && !line.startsWith("#"));
const listed = new Map(lines.map((line) => { const [hash, name] = line.split(/\s+/); return [name, hash]; }));
- const files = fs.readdirSync(contextDir).filter((name) => name.endsWith(".ts")).sort();
+ const files = fs.readdirSync(contextDir).filter((name) => name.endsWith(".ts") || name.endsWith(".json")).sort();
assert.deepEqual([...listed.keys()].sort(), files);
for (const name of files) assert.equal(createHash("sha256").update(fs.readFileSync(path.join(contextDir, name))).digest("hex"), listed.get(name), `${name} changed: port it to the CLI and update PARITY.sha256 in both`);
});
diff --git a/apps/server/src/context/PARITY.sha256 b/apps/server/src/context/PARITY.sha256
index 5d175192..1bfbdfd4 100644
--- a/apps/server/src/context/PARITY.sha256
+++ b/apps/server/src/context/PARITY.sha256
@@ -9,6 +9,8 @@ d772119f2785b6926cd765a44589c171542d5e79b3677bfa076851775d794480 outside.ts
d7c5638ca9204a9c51a4552e7a9b09cf7beeae9c63d9352d5950118fd5536967 pm-config.ts
d13231905671ad40fdd73e3fe7af81e44dcc909ace606f45a79c215984b6b35d privacy.ts
0e04845a2d0ad2136c01a61d3ed1193b62ef99f5b82e1f555b7c97bb6dc18773 processes.ts
+dfe4efad20d114462bc0f9e3f3178eb7e44c4241d119b7deb1524183bf08b481 redact-policy.ts
+16f6bbf34934e279bd1ff3f6db5841bcd3e9f5a9ec05363dc39f5e7108a16e6a redact-vectors.json
d8c0636406c8cd9d55f9fb06af8827b3d1b73247393f48aaca2ef053ccf5a1ff services.ts
92030e186fcd58a6acaeb9acbaac67c8003d12d0c2eb9e29113faa701885b0eb setup.ts
d51d434283f757f0ed45bb6996a74fb5bf9e1d56f34146e99df3c392cde5ea19 shell-aliases.ts
diff --git a/apps/server/src/context/redact-policy.ts b/apps/server/src/context/redact-policy.ts
new file mode 100644
index 00000000..9cacd2d9
--- /dev/null
+++ b/apps/server/src/context/redact-policy.ts
@@ -0,0 +1,442 @@
+// Client rules v3 ("client-v3"): what the uploader removes on the device once
+// the server turns them on (`policy.redaction_v3` in GET /archives/key).
+// Credentials only, at high precision, the variable name always kept; personal
+// data (e-mail, phone, SSN, IPv4) is left to the server's export pass, and
+// opaque data (encrypted reasoning, signatures, base64 blobs, `data:` URIs) is
+// never touched. Until the server says so the uploader keeps its old rules.
+//
+// The same file in omnirush-cli (assets/extensions/omnirush/capture/context/)
+// and omnirush-gui (apps/server/src/context/), listed in PARITY.sha256; the
+// backend runs the same vectors (redact-vectors.json) against its port
+// (omnirush/credential_rules.py).
+
+export const CLIENT_V3 = "client-v3";
+export type RedactionPolicy = "legacy" | typeof CLIENT_V3;
+
+// One switch per process, kept on globalThis so every copy of this module
+// (a bundle, a loader without a module cache) reads the same answer.
+const STATE = Symbol.for("omnirush.redactionPolicy");
+type PolicyHolder = { [STATE]?: RedactionPolicy };
+
+/** Switches the uploader's rules; anything but `client-v3` is the old rule set. */
+export function setRedactionPolicy(policy: string | null | undefined): void {
+ (globalThis as PolicyHolder)[STATE] = policy === CLIENT_V3 ? CLIENT_V3 : "legacy";
+}
+
+export function redactionPolicy(): RedactionPolicy {
+ return (globalThis as PolicyHolder)[STATE] === CLIENT_V3 ? CLIENT_V3 : "legacy";
+}
+
+export function redactionV3(): boolean {
+ return redactionPolicy() === CLIENT_V3;
+}
+
+const REDACTED = "[REDACTED]";
+
+/** JSON keys whose string values are opaque provider data, never scanned. */
+const OPAQUE_KEYS = new Set([
+ "reasoningencryptedcontent", "reasoning_encrypted_content", "encrypted_content", "encryptedcontent",
+ "signature", "thinkingsignature", "thinking_signature", "redacted_thinking", "redactedthinking",
+]);
+
+export function isOpaqueKey(key: string | undefined): boolean {
+ return key !== undefined && OPAQUE_KEYS.has(key.toLowerCase());
+}
+
+// A run of base64 or base64url characters this long with no break is a blob
+// (encrypted content, an image, a signature), not a credential.
+const OPAQUE_RUN = /[A-Za-z0-9+/_-]{200,}={0,2}/g;
+const DATA_URI = /data:[A-Za-z0-9.+/-]*(?:;[A-Za-z0-9.+=-]+)*,[A-Za-z0-9+/=%._~-]*/g;
+
+/** A whole string that is a `data:` URI or one base64 blob. */
+export function isOpaqueString(value: string): boolean {
+ if (value.startsWith("data:") && /^data:[A-Za-z0-9.+/-]*(?:;[A-Za-z0-9.+=-]+)*,/.test(value)) return true;
+ return value.length >= 200 && /^[A-Za-z0-9+/_-]+={0,2}$/.test(value);
+}
+
+function opaqueSpans(text: string): Array<[number, number]> {
+ const spans: Array<[number, number]> = [];
+ if (text.length < 200 && !text.includes("data:")) return spans;
+ for (const pattern of [OPAQUE_RUN, DATA_URI]) {
+ pattern.lastIndex = 0;
+ for (let match = pattern.exec(text); match; match = pattern.exec(text)) {
+ if (match[0].length === 0) {
+ pattern.lastIndex += 1;
+ continue;
+ }
+ spans.push([match.index, match.index + match[0].length]);
+ }
+ }
+ return spans;
+}
+
+/** A match lies inside an opaque span (and is not the whole span: a long token on its own still counts). */
+function insideOpaque(spans: Array<[number, number]>, start: number, end: number): boolean {
+ return spans.some(([from, to]) => start >= from && end <= to && !(start === from && end === to));
+}
+
+/** Shannon entropy in bits per character. */
+export function shannonEntropy(value: string): number {
+ if (!value) return 0;
+ const counts = new Map<string, number>();
+ for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);
+ const length = [...value].length;
+ let entropy = 0;
+ for (const count of counts.values()) {
+ const p = count / length;
+ entropy -= p * Math.log2(p);
+ }
+ return entropy;
+}
+
+function charClasses(value: string): number {
+ return [/[a-z]/, /[A-Z]/, /[0-9]/, /[^A-Za-z0-9]/].filter((pattern) => pattern.test(value)).length;
+}
+
+/** 12+ characters, two or more character classes and 3.0+ bits per character. */
+export function looksRandom(value: string): boolean {
+ return value.length >= 12 && charClasses(value) >= 2 && shannonEntropy(value) >= 3.0;
+}
+
+/**
+ * Credential files: any non-placeholder value of a secret-named key there is
+ * a credential. `.env*` except examples and templates, `.npmrc`, `.pypirc`,
+ * `.netrc`, `.pgpass`, `.git-credentials`, `credentials` (`.aws/credentials`),
+ * kubeconfig.
+ */
+export function isCredentialFile(path: string | undefined): boolean {
+ if (!path) return false;
+ const name = path.split(/[\\/]/).pop()!.toLowerCase();
+ if (name.startsWith(".env")) return !/(?:example|sample|template|dist|defaults?)$/.test(name);
+ return [".npmrc", ".pypirc", ".netrc", "_netrc", ".pgpass", ".git-credentials", "credentials", "kubeconfig", "config.kube"].includes(name)
+ || name.endsWith(".kubeconfig");
+}
+
+// --- secret-named assignments ----------------------------------------------
+
+// Python's `\s`, so a value ends where the backend's does.
+const WS = String.raw`\t\n\v\f\r \x1c-\x1f\x85\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000`;
+const KEYWORD = String.raw`secret|passw(?:or)?d|pwd|token|credential|auth|(?:api|access|private|client|session|signing|master|encryption)[_.:-]?key`;
+const ASSIGNMENT_BODY = String.raw`(\\?["']?)((?=[\w.-]{0,63}?(?:${KEYWORD}))[A-Za-z_-][\w.-]{0,63})\1[ \t]*[=:][ \t]*`
+ + String.raw`(?:"((?:[^"${WS}\\]|\\[^${WS}]){8,})"`
+ + String.raw`|'((?:[^'${WS}\\]|\\[^${WS}]){8,})'`
+ + String.raw`|\\"((?:[^"${WS}\\]|\\[^"${WS}]){8,})\\"`
+ + String.raw`|((?:[^${WS}"',;&\\]|\\[^${WS}]){8,}))`;
+const CONFIG_ASSIGNMENT = new RegExp(String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");
+const SOURCE_ASSIGNMENT = new RegExp(String.raw`(?:(?<![^${WS}\x80-\uffff{,("'\x60+#])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");
+
+const SECRET_SEGMENTS = new Set([
+ "secret", "secrets", "password", "passwords", "passwd", "pwd", "token", "credential", "credentials",
+ "auth", "authorization", "authtoken", "authkey", "apikey",
+]);
+const SECRET_PAIRS: Array<[string, string]> = [
+ ["api", "key"], ["access", "key"], ["secret", "key"], ["private", "key"], ["client", "key"], ["client", "secret"],
+ ["session", "key"], ["signing", "key"], ["master", "key"], ["encryption", "key"],
+];
+const EXCLUDED_LAST = new Set([
+ "length", "ttl", "seconds", "count", "size", "url", "path", "name", "id", "header",
+ "file", "filename", "dir", "mode", "method", "role", "owner", "type", "kind", "enabled", "estimate", "hash", "digest", "at",
+ "config", "client", "prefix", "suffix", "format", "scheme", "provider", "status", "state", "label", "description", "title",
+ "class", "field", "fields", "list", "names", "version", "timeout", "limit", "max", "min", "interval", "retries", "port",
+ "in", "out", "percent", "pct", "ms", "secs", "minutes", "hours", "days", "expires", "expiry", "expiration", "bytes", "len",
+ "width", "height", "offset", "index", "idx", "pos", "total", "sum", "avg", "ratio", "rate", "threshold", "weight", "score",
+ "encryption", "algorithm", "algo", "cipher", "strategy", "policy", "source", "target", "origin", "backend", "engine", "driver",
+ "handler", "callback", "event", "action", "reason", "message", "error", "envelope", "ref", "reference", "link", "pointer", "alias",
+]);
+
+function keySegments(key: string): string[] {
+ return key
+ .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+ .replace(/([A-Z])([A-Z][a-z])/g, "$1 $2")
+ .replace(/([A-Za-z])(\d)/g, "$1 $2")
+ .replace(/(\d)([A-Za-z])/g, "$1 $2")
+ .toLowerCase()
+ .split(/[\s_.:-]+/)
+ .filter(Boolean);
+}
+
+/** Whether a key names a secret (`auth_token`, `APIKey`, `client_secret`; not `author`, `token_url`, `accessKeyId`). */
+export function isSecretKey(key: string): boolean {
+ const segments = keySegments(key);
+ const last = segments.at(-1);
+ if (last === undefined || EXCLUDED_LAST.has(last)) return false;
+ return segments.some((segment, index) => SECRET_SEGMENTS.has(segment)
+ || SECRET_PAIRS.some(([first, second]) => segment === `${first}${second}` || (segment === first && segments[index + 1] === second)));
+}
+
+const PLACEHOLDER_MARKERS = new Set([
+ "your", "yours", "here", "fake", "dummy", "example", "sample", "placeholder", "changeme", "replaceme", "replace", "insert",
+ "todo", "fixme", "tbd", "mock", "demo", "redacted", "notreal", "test", "testing",
+]);
+const PLACEHOLDER_WORDS = new Set([
+ "my", "the", "a", "an", "not", "real", "is", "goes", "go", "put", "enter", "set", "this", "to", "with", "of", "for", "in", "me",
+ "change", "secret", "secrets", "token", "tokens", "key", "keys", "api", "apikey", "password", "passwd", "pass", "pwd", "value",
+ "access", "auth", "private", "client", "id", "string", "foo", "bar", "baz", "qux", "abc", "xyz", "default", "dev", "local",
+]);
+const REPEATED_CHAR = /^(.)\1*$/su;
+
+/** `your-token-here`, `changeme`, `xxxxxxxx`, `YOUR_API_KEY`: never a secret. */
+export function isPlaceholder(value: string): boolean {
+ const segments = keySegments(value);
+ if (segments.length === 0) return false;
+ let marked = false;
+ for (const segment of segments) {
+ if (PLACEHOLDER_MARKERS.has(segment) || ([...segment].length >= 3 && REPEATED_CHAR.test(segment))) marked = true;
+ else if (PLACEHOLDER_WORDS.has(segment) || (/^[0-9]+$/.test(segment) && (segment.length <= 4 || "0123456789".includes(segment)))) continue;
+ else return false;
+ }
+ return marked;
+}
+
+/** A variable or an expression, never a literal: `$VAR`, `${...}`, `$(...)`, `%VAR%`, `process.env.X`, `os.environ[...]`, `getenv(`. */
+export function isReference(value: string): boolean {
+ return /^\$[A-Za-z_]\w*$/.test(value) || value.includes("${") || value.includes("$(") || /^%[A-Za-z_]\w*%$/.test(value)
+ || /(?:^|\W)(?:process\.env|import\.meta\.env|os\.environ|getenv\(|ENV\[|env\()/.test(value)
+ || /^(?:process|os|env|self|this|config|settings|secrets|vars|inputs)\./.test(value)
+ || /\{\{[^}]*\}\}|\{%[^%]*%\}/.test(value);
+}
+
+const PATH_VALUE = /^(?:\/|\.\/|\.\.\/|~\/|[A-Za-z]:[\\/])/;
+const IDENTIFIER_VALUE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/;
+
+export type ValueContext = { mode?: "source" | "config"; quoted?: boolean; credentialFile?: boolean };
+
+/**
+ * The value of a secret-named key is removed when it is a literal that looks
+ * generated (12+ characters, two classes, 3.0+ bits), carries a known token,
+ * or sits in a credential file. Never a reference, a placeholder, a path or
+ * an earlier marker; in source code only a quoted literal.
+ */
+export function isSecretValue(value: string, context: ValueContext = {}): boolean {
+ if (value.length < 8 || new RegExp(`[${WS}]`).test(value) || value.startsWith("[REDACTED")) return false;
+ if (isReference(value) || isPlaceholder(value)) return false;
+ if (value.startsWith("<") && value.endsWith(">")) return false;
+ if (PATH_VALUE.test(value) || (value.includes("/") && !/\d/.test(value))) return false;
+ if (containsKnownToken(value)) return true;
+ if (value.includes("(")) return false;
+ if (context.credentialFile) return true;
+ if (context.mode === "source" && !context.quoted) return false;
+ if (!context.quoted && IDENTIFIER_VALUE.test(value)) return false;
+ return looksRandom(value);
+}
+
+// --- known token shapes ----------------------------------------------------
+
+/** Start of a token: never inside a word or on an escape's letter; right after `\n`/`\r`/`\t` escapes still counts. */
+function token(source: string, flags = "g"): RegExp {
+ return new RegExp(String.raw`(?:(?<![A-Za-z0-9_\\])|(?<=\\[nrt]))(?:${source})`, flags);
+}
+
+type TokenRule = { name: string; gate: string | RegExp; pattern: RegExp; check?: (match: string) => boolean; keep?: number };
+
+function jwtHeader(match: string): boolean {
+ try {
+ const header = JSON.parse(Buffer.from(match.slice(0, match.indexOf(".")), "base64url").toString("utf8")) as unknown;
+ return typeof header === "object" && header !== null && !Array.isArray(header) && "alg" in header;
+ } catch {
+ return false;
+ }
+}
+
+const TOKEN_RULES: TokenRule[] = [
+ { name: "aws_access_key_id", gate: /AKIA|ASIA/, pattern: token(String.raw`(?:AKIA|ASIA)[0-9A-Z]{16}(?![0-9A-Za-z])`), check: (m) => !m.endsWith("EXAMPLE") },
+ { name: "github_token", gate: /gh[pousr]_/, pattern: token(String.raw`gh[pousr]_[A-Za-z0-9]{36}(?![A-Za-z0-9])`) },
+ { name: "github_pat", gate: "github_pat_", pattern: token(String.raw`github_pat_[A-Za-z0-9_]{60,}`) },
+ { name: "gitlab_token", gate: "glpat-", pattern: token(String.raw`glpat-[A-Za-z0-9_-]{20,}`) },
+ { name: "slack_token", gate: /xox[abprs]-/, pattern: token(String.raw`xox[abprs]-[A-Za-z0-9-]{10,}`) },
+ { name: "slack_webhook", gate: "hooks.slack.com", pattern: /(https:\/\/hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/)([A-Za-z0-9]{20,})/g, keep: 1 },
+ { name: "discord_webhook", gate: "/api/webhooks/", pattern: /(https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/)([A-Za-z0-9_-]{50,})/g, keep: 1 },
+ { name: "google_api_key", gate: "AIza", pattern: token(String.raw`AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])`) },
+ { name: "stripe_key", gate: /[sr]k_live_/, pattern: token(String.raw`[sr]k_live_[A-Za-z0-9]{16,}`) },
+ { name: "stripe_webhook", gate: "whsec_", pattern: token(String.raw`whsec_[A-Za-z0-9+/=]{24,}`) },
+ { name: "anthropic_key", gate: "sk-ant-", pattern: token(String.raw`sk-ant-(?:api|admin)\d\d-[A-Za-z0-9_-]{32,}`) },
+ { name: "openai_key", gate: "sk-", pattern: token(String.raw`sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{40,}|sk-[A-Za-z0-9]{48}(?![A-Za-z0-9])`), check: (m) => shannonEntropy(m) >= 3.5 },
+ { name: "npm_token", gate: "npm_", pattern: token(String.raw`npm_[A-Za-z0-9]{36}(?![A-Za-z0-9])`) },
+ { name: "pypi_token", gate: "pypi-AgE", pattern: token(String.raw`pypi-AgE[A-Za-z0-9_-]{50,}`) },
+ { name: "huggingface_token", gate: "hf_", pattern: token(String.raw`hf_[A-Za-z0-9]{34,}(?![A-Za-z0-9])`) },
+ { name: "sendgrid_key", gate: "SG.", pattern: token(String.raw`SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}(?![A-Za-z0-9_-])`) },
+ { name: "jwt", gate: "eyJ", pattern: token(String.raw`eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), check: jwtHeader },
+ { name: "discord_token", gate: /[MN][\w-]{23,}\./, pattern: token(String.raw`[MN][A-Za-z0-9_-]{23,}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}`), check: (m) => shannonEntropy(m) >= 3.5 },
+];
+
+function gateOpen(gate: string | RegExp, text: string): boolean {
+ return typeof gate === "string" ? text.includes(gate) : gate.test(text);
+}
+
+/** Whether a value holds one known credential token (its whole shape and checks). */
+export function containsKnownToken(value: string): boolean {
+ for (const rule of TOKEN_RULES) {
+ if (!gateOpen(rule.gate, value)) continue;
+ rule.pattern.lastIndex = 0;
+ for (let match = rule.pattern.exec(value); match; match = rule.pattern.exec(value)) {
+ const secret = rule.keep ? match[rule.keep + 1]! : match[0];
+ if (!rule.check || rule.check(secret)) return true;
+ }
+ }
+ return false;
+}
+
+// --- the text pass ---------------------------------------------------------
+
+export type CredentialOptions = {
+ /** The file path or JSON key the text belongs to. */
+ context?: string;
+ /** Source code (quoted literals only) or config/trace text. */
+ mode?: "source" | "config";
+ /** The enclosing JSON object names an AWS key. */
+ awsContext?: boolean;
+ /** What a removed value is written as; `[REDACTED]` by default. */
+ replacement?: string;
+};
+
+type Tally = { count: number; rules: Record<string, number> };
+
+function note(tally: Tally, rule: string): void {
+ tally.count += 1;
+ tally.rules[rule] = (tally.rules[rule] ?? 0) + 1;
+}
+
+const PRIVATE_KEY = /(?<!\\)(-----BEGIN ([A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?)-----)([\s\S]*?)(-----END \2-----)/g;
+
+function redactPrivateKeys(text: string, hidden: string, tally: Tally): string {
+ PRIVATE_KEY.lastIndex = 0;
+ return text.replace(PRIVATE_KEY, (match: string, begin: string, _kind: string, body: string, end: string) => {
+ const plain = body.replace(/\\[nrt]/g, " ");
+ if (plain.includes("...") || plain.includes("…") || plain.replace(/[^A-Za-z0-9+/=]/g, "").length < 64) return match;
+ const lead = /^(?:\s|\\[nr])*/.exec(body)![0];
+ const trail = /(?:\s|\\[nr])*$/.exec(body)![0];
+ note(tally, "private_key");
+ return `${begin}${lead}${hidden}${trail}${end}`;
+ });
+}
+
+const URL_PASSWORD = new RegExp(
+ String.raw`(?:(?<![A-Za-z0-9_\\])|(?<=\\[nrt]))([a-z][a-z0-9+.-]{0,31}://)((?:[^${WS}/@:\\]|\\[^${WS}])*):((?:[^${WS}/@\\]|\\[^${WS}])+)@`,
+ "gi",
+);
+const URL_PASSWORD_PLACEHOLDERS = new Set(["password", "pass", "passwd", "pwd", "secret", "changeme", "xxx", "***", "<password>", "[redacted]"]);
+
+function urlPasswordPlaceholder(password: string): boolean {
+ const lowered = password.toLowerCase();
+ return URL_PASSWORD_PLACEHOLDERS.has(lowered) || lowered.startsWith("[redacted") || isReference(password) || /^<[^>]*>$/.test(password)
+ || /^\*+$/.test(password) || isPlaceholder(password);
+}
+
+function redactUrlPasswords(text: string, hidden: string, tally: Tally, spans: Array<[number, number]>): string {
+ URL_PASSWORD.lastIndex = 0;
+ return text.replace(URL_PASSWORD, (match: string, scheme: string, user: string, password: string, offset: number) => {
+ if (insideOpaque(spans, offset, offset + match.length) || urlPasswordPlaceholder(password)) return match;
+ note(tally, "url_password");
+ return `${scheme}${user}:${hidden}@`;
+ });
+}
+
+const MAX_ASSIGNMENT_DEPTH = 4;
+
+function redactAssignments(
+ text: string, hidden: string, tally: Tally, mode: "source" | "config", credentialFile: boolean, spans: Array<[number, number]>, depth = 0,
+): string {
+ // A fresh expression per call: the pass recurses into values while the outer replace is still running.
+ const pattern = new RegExp(mode === "source" ? SOURCE_ASSIGNMENT : CONFIG_ASSIGNMENT);
+ return text.replace(pattern, (match: string, _quote: string, key: string, doubleQuoted: string | undefined, singleQuoted: string | undefined, escapedQuoted: string | undefined, bare: string | undefined, offset: number) => {
+ if (depth === 0 && insideOpaque(spans, offset, offset + match.length)) return match;
+ const value = doubleQuoted ?? singleQuoted ?? escapedQuoted ?? bare ?? "";
+ const quote = doubleQuoted !== undefined ? '"' : singleQuoted !== undefined ? "'" : escapedQuoted !== undefined ? '\\"' : "";
+ const prefix = match.slice(0, match.length - value.length - quote.length * 2);
+ if (isSecretKey(key) && isSecretValue(value, { mode, quoted: bare === undefined, credentialFile })) {
+ note(tally, "secret_assignment");
+ return `${prefix}${quote}${hidden}${quote}`;
+ }
+ if (depth >= MAX_ASSIGNMENT_DEPTH) return match;
+ return `${prefix}${quote}${redactAssignments(value, hidden, tally, mode, credentialFile, [], depth + 1)}${quote}`;
+ });
+}
+
+const ASSIGNMENT_GATE = /secret|passw|pwd|token|credential|auth|key/i;
+
+const AWS_ID = /(?:AKIA|ASIA)[0-9A-Z]{16}/;
+const AWS_CONTEXT = /(?:AKIA|ASIA)[0-9A-Z]{16}|aws[_ .-]?secret[_ .-]?access[_ .-]?key/i;
+const AWS_SECRET = token(String.raw`[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+])`);
+
+function redactAwsSecrets(text: string, hidden: string, tally: Tally, nearContext: boolean, spans: Array<[number, number]>): string {
+ if (!nearContext && !AWS_CONTEXT.test(text)) return text;
+ if (!/[A-Za-z0-9/+]{40}/.test(text)) return text;
+ const lines = text.split("\n");
+ const context = lines.map((line) => AWS_CONTEXT.test(line));
+ let offset = 0;
+ let changed = false;
+ for (let index = 0; index < lines.length; index += 1) {
+ const line = lines[index]!;
+ const start = offset;
+ offset += line.length + 1;
+ if (line.length < 40) continue;
+ let near = nearContext;
+ for (let other = Math.max(0, index - 3); !near && other <= Math.min(lines.length - 1, index + 3); other += 1) near = context[other]!;
+ if (!near) continue;
+ AWS_SECRET.lastIndex = 0;
+ lines[index] = line.replace(AWS_SECRET, (candidate: string, at: number) => {
+ if (!/[a-z]/.test(candidate) || !/[A-Z]/.test(candidate) || !/\d/.test(candidate) || AWS_ID.test(candidate)) return candidate;
+ if (insideOpaque(spans, start + at, start + at + candidate.length) || shannonEntropy(candidate) < 3.5) return candidate;
+ note(tally, "aws_secret");
+ changed = true;
+ return hidden;
+ });
+ }
+ return changed ? lines.join("\n") : text;
+}
+
+function redactTokens(text: string, hidden: string, tally: Tally, spans: Array<[number, number]>): string {
+ for (const rule of TOKEN_RULES) {
+ if (!gateOpen(rule.gate, text)) continue;
+ rule.pattern.lastIndex = 0;
+ text = text.replace(rule.pattern, (...args: unknown[]) => {
+ const match = args[0] as string;
+ const offset = args.find((arg, index) => index > 0 && typeof arg === "number") as number;
+ const secret = rule.keep ? args[rule.keep + 1] as string : match;
+ if (rule.check && !rule.check(secret)) return match;
+ if (insideOpaque(spans, offset, offset + match.length)) return match;
+ note(tally, rule.name);
+ return rule.keep ? `${args[rule.keep] as string}${hidden}` : hidden;
+ });
+ }
+ return text;
+}
+
+const AUTH_HEADER = /(?<![A-Za-z0-9_])((?:Bearer|Basic)[ \t]+)([A-Za-z0-9_.~+/=-]{16,})/gi;
+
+function authSecret(token: string): boolean {
+ if (shannonEntropy(token) < 3.0 || isPlaceholder(token)) return false;
+ if (/\d/.test(token)) return true;
+ return token.length >= 20 && /[a-z]/.test(token) && /[A-Z]/.test(token) && !IDENTIFIER_VALUE.test(token);
+}
+
+function redactAuthHeaders(text: string, hidden: string, tally: Tally, spans: Array<[number, number]>): string {
+ AUTH_HEADER.lastIndex = 0;
+ return text.replace(AUTH_HEADER, (match: string, prefix: string, value: string, offset: number) => {
+ if (!authSecret(value) || insideOpaque(spans, offset, offset + match.length)) return match;
+ note(tally, "auth_header");
+ return `${prefix}${hidden}`;
+ });
+}
+
+/**
+ * Removes credentials from free text (a trace string, a file, a diff), the
+ * name always kept: private key bodies, URL passwords, secret-named values
+ * that look generated (any value in a credential file), AWS secrets beside
+ * their key id, known token shapes and `Bearer`/`Basic` values. Nothing
+ * inside a `data:` URI or a base64 blob is touched.
+ */
+export function redactCredentials(input: string, options: CredentialOptions = {}): { text: string; count: number; rules: Record<string, number> } {
+ const tally: Tally = { count: 0, rules: {} };
+ if (!input) return { text: input, count: 0, rules: tally.rules };
+ const hidden = options.replacement ?? REDACTED;
+ const credentialFile = isCredentialFile(options.context);
+ let text = input.includes("-----BEGIN ") ? redactPrivateKeys(input, hidden, tally) : input;
+ if (text.includes("://")) text = redactUrlPasswords(text, hidden, tally, opaqueSpans(text));
+ if (ASSIGNMENT_GATE.test(text)) text = redactAssignments(text, hidden, tally, options.mode ?? "config", credentialFile, opaqueSpans(text));
+ const nearAws = options.awsContext === true || (options.context !== undefined && AWS_CONTEXT.test(options.context));
+ text = redactAwsSecrets(text, hidden, tally, nearAws, opaqueSpans(text));
+ text = redactTokens(text, hidden, tally, opaqueSpans(text));
+ if (/bearer|basic/i.test(text)) text = redactAuthHeaders(text, hidden, tally, opaqueSpans(text));
+ return { text, count: tally.count, rules: tally.rules };
+}
diff --git a/apps/server/src/context/redact-vectors.json b/apps/server/src/context/redact-vectors.json
new file mode 100644
index 00000000..dcb076d2
--- /dev/null
+++ b/apps/server/src/context/redact-vectors.json
@@ -0,0 +1,272 @@
+{
+ "join": "⟨+⟩",
+ "redacted": "[REDACTED]",
+ "opaque_keys": [
+ "reasoningEncryptedContent",
+ "encrypted_content",
+ "encryptedContent",
+ "signature",
+ "thinkingSignature",
+ "redacted_thinking"
+ ],
+ "cases": [
+ {
+ "name": "env var reference kept",
+ "input": "GITHUB_TOKEN=$GITHUB_TOKEN",
+ "expect": "GITHUB_TOKEN=$GITHUB_TOKEN"
+ },
+ {
+ "name": "braced env var kept",
+ "input": "export API_KEY=\"${OPENAI_API_KEY}\"",
+ "expect": "export API_KEY=\"${OPENAI_API_KEY}\""
+ },
+ {
+ "name": "process.env kept",
+ "input": "const token = process.env.SLACK_TOKEN_VALUE_X",
+ "expect": "const token = process.env.SLACK_TOKEN_VALUE_X",
+ "mode": "source"
+ },
+ {
+ "name": "os.environ kept",
+ "input": "password = os.environ[\"DB_PASSWORD\"]",
+ "expect": "password = os.environ[\"DB_PASSWORD\"]",
+ "mode": "source"
+ },
+ {
+ "name": "template kept",
+ "input": "auth_token: \"{{ vault_auth_token_value }}\"",
+ "expect": "auth_token: \"{{ vault_auth_token_value }}\""
+ },
+ {
+ "name": "windows var kept",
+ "input": "set SECRET_KEY=%MY_SECRET_KEY%",
+ "expect": "set SECRET_KEY=%MY_SECRET_KEY%"
+ },
+ {
+ "name": "email kept on the client",
+ "input": "contact [email protected] now",
+ "expect": "contact [email protected] now"
+ },
+ {
+ "name": "phone kept on the client",
+ "input": "call +1 415 555 0101",
+ "expect": "call +1 415 555 0101"
+ },
+ {
+ "name": "ssn kept on the client",
+ "input": "id 123-45-6789",
+ "expect": "id 123-45-6789"
+ },
+ {
+ "name": "public ipv4 kept on the client",
+ "input": "ssh [email protected] -p 22",
+ "expect": "ssh [email protected] -p 22"
+ },
+ {
+ "name": "version kept",
+ "input": "billiard==3.6.4.0 and version = \"2.4.7.3\"",
+ "expect": "billiard==3.6.4.0 and version = \"2.4.7.3\""
+ },
+ {
+ "name": "kernel kept",
+ "input": "Linux version 5.15.153.1-microsoft-standard-WSL2",
+ "expect": "Linux version 5.15.153.1-microsoft-standard-WSL2"
+ },
+ {
+ "name": "decorator kept",
+ "input": "[email protected](\"/x\")",
+ "expect": "[email protected](\"/x\")"
+ },
+ {
+ "name": "git ssh kept",
+ "input": "[email protected]:owner/repo.git",
+ "expect": "[email protected]:owner/repo.git"
+ },
+ {
+ "name": "generic sk- in a blob kept",
+ "input": "blob EuSxsYMIUla2_LmDMWGoVW6MfRSyeybbbyqw0VOHDMwcSs-vwzRk0JKrq5B_I9IWX72IroVtEYd9QRVg3ZBHHwTdLfAjX-sSVCaJ8DXLipJ8ZHwrrVcDj0k6sk-1t8FTrUvkHKdhoIuX2NpfytPseqA0PGxhaTdUise7sZ1d5qlHFRyT_-Z_kd6HhbAPqxpB7LFRaFcqPK1tfGO0Ir7mjqk5VAu1u7CJiShQOBjOYbzQJgWZXdK5MFPa2khz2XtL-psgPtt59sngsuKl4JiPwKBkWIuvNEIbyRRuAlxvxaAV end",
+ "expect": "blob EuSxsYMIUla2_LmDMWGoVW6MfRSyeybbbyqw0VOHDMwcSs-vwzRk0JKrq5B_I9IWX72IroVtEYd9QRVg3ZBHHwTdLfAjX-sSVCaJ8DXLipJ8ZHwrrVcDj0k6sk-1t8FTrUvkHKdhoIuX2NpfytPseqA0PGxhaTdUise7sZ1d5qlHFRyT_-Z_kd6HhbAPqxpB7LFRaFcqPK1tfGO0Ir7mjqk5VAu1u7CJiShQOBjOYbzQJgWZXdK5MFPa2khz2XtL-psgPtt59sngsuKl4JiPwKBkWIuvNEIbyRRuAlxvxaAV end"
+ },
+ {
+ "name": "data uri kept",
+ "input": "src=\"data:image/png;base64,ipx5utjfTxzUn/XZK54xFb2n75zrqbl6q3t/OXwT2ZeZLV06Agdf4T8h4PwFi06l4L2ok4XwVqk3WtcOsk-WVGALBh6bRKfkr1vMRNcsvFUqYI7tg==\"",
+ "expect": null,
+ "same": true
+ },
+ {
+ "name": "word password kept",
+ "input": "password: hunter2hunter",
+ "expect": "password: hunter2hunter"
+ },
+ {
+ "name": "placeholder kept",
+ "input": "API_KEY=your-api-key-here",
+ "expect": "API_KEY=your-api-key-here"
+ },
+ {
+ "name": "aws example id kept",
+ "input": "AKIA⟨+⟩IOSFODNN7EXAMPLE",
+ "expect": "AKIA⟨+⟩IOSFODNN7EXAMPLE"
+ },
+ {
+ "name": "function param kept",
+ "input": "function f(token: string) { return token }",
+ "expect": "function f(token: string) { return token }",
+ "mode": "source"
+ },
+ {
+ "name": "integrity hash kept",
+ "input": "\"integrity\": \"sha512-eyVpl09JmcJpaXZ5xbsrBEYLtg5908t1hfCmB8vI32dTy1IATELbOlZi0vSVy/dCiT8Kb2gWt4SwTH8Q5hPV3M==\"",
+ "expect": null,
+ "same": true
+ },
+ {
+ "name": "uuid under api_key looks random",
+ "input": "api_key = \"3f2b8a4e-9c1d-4e7a-b5f6-0a1b2c3d4e5f\"",
+ "expect": "api_key = \"[REDACTED]\""
+ },
+ {
+ "name": "short secret value kept",
+ "input": "token=abc123",
+ "expect": "token=abc123"
+ },
+ {
+ "name": "truncated private key kept",
+ "input": "-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----",
+ "expect": "-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----"
+ },
+ {
+ "name": "url user without password kept",
+ "input": "postgres://[email protected]:5432/app",
+ "expect": "postgres://[email protected]:5432/app"
+ },
+ {
+ "name": "url placeholder password kept",
+ "input": "postgres://user:${DB_PASS}@localhost:5432/app",
+ "expect": "postgres://user:${DB_PASS}@localhost:5432/app"
+ },
+ {
+ "name": "bearer placeholder kept",
+ "input": "Authorization: Bearer yourAccessTokenGoesHere",
+ "expect": "Authorization: Bearer yourAccessTokenGoesHere"
+ },
+ {
+ "name": "non-jwt eyJ kept",
+ "input": "ey⟨+⟩JraWQiOiJ4IiwidHlwIjoieSJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkZha2UgVXNlciIsImlhdCI6MTUxNjIzOTAyMn0.HnRI5zBguaFFIU3rkbPU",
+ "expect": "ey⟨+⟩JraWQiOiJ4IiwidHlwIjoieSJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkZha2UgVXNlciIsImlhdCI6MTUxNjIzOTAyMn0.HnRI5zBguaFFIU3rkbPU"
+ },
+ {
+ "name": "openai project key",
+ "input": "OPENAI_API_KEY=sk⟨+⟩-proj-k0t2-IBx0MSFjLIHgdtCHfaBsoexCYqsFdec8ZnbsrQau0e9D6W5Sosy-ZzgMTZ6pDE0O1Vbx6UcE4flEOBRH8gUFXo0dItNV6dby1aGWll5oflgb-XSWpl4kt4O8ItCjaBSnEhCLzw_XAlGQAFIrEEUc5d1",
+ "expect": "OPENAI_API_KEY=[REDACTED]"
+ },
+ {
+ "name": "anthropic key in prose",
+ "input": "use sk⟨+⟩-ant-api03-BI11bAx-D8Kv5xjY64Y7gxOO6E_tSz-2LaqC47M5PEyGbPW4eWwhvH8cUauOv-g1850wDoxpJPC1bLaoZDqiUVdrlhJ47 please",
+ "expect": "use [REDACTED] please"
+ },
+ {
+ "name": "github token",
+ "input": "gh auth: gh⟨+⟩p_b4z0PMbDeCxAqT4kiRhFv5FnMXlMWbiohfGV",
+ "expect": "gh auth: [REDACTED]"
+ },
+ {
+ "name": "aws id and secret",
+ "input": "aws_access_key_id = AK⟨+⟩IAWGN7A7YSXASSWMRT\naws_secret_access_key = RbqEoYnLY/TZ3v44hCp4oQzLdlL2cd0SVjlHRjD5",
+ "expect": "aws_access_key_id = [REDACTED]\naws_secret_access_key = [REDACTED]"
+ },
+ {
+ "name": "aws secret beside id",
+ "input": "id AK⟨+⟩IAWGN7A7YSXASSWMRT\nkey RbqEoYnLY/TZ3v44hCp4oQzLdlL2cd0SVjlHRjD5",
+ "expect": "id [REDACTED]\nkey [REDACTED]"
+ },
+ {
+ "name": "jwt",
+ "input": "cookie=ey⟨+⟩JhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkZha2UgVXNlciIsImlhdCI6MTUxNjIzOTAyMn0.SDfGQZPTGwvY6aLP4vySLMnlhkIOYLh6XrYf1xBiQN1",
+ "expect": "cookie=[REDACTED]"
+ },
+ {
+ "name": "bearer header",
+ "input": "Authorization: Bearer I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY",
+ "expect": "Authorization: Bearer [REDACTED]"
+ },
+ {
+ "name": "basic header",
+ "input": "Authorization: Basic YWRtaW46WWlKZXV5d3V5cWt1Z2dQeg==",
+ "expect": "Authorization: Basic [REDACTED]"
+ },
+ {
+ "name": "url password keeps user",
+ "input": "postgresql+asyncpg://app_user:[email protected]:5432/app",
+ "expect": "postgresql+asyncpg://app_user:[REDACTED]@db.example.net:5432/app"
+ },
+ {
+ "name": "url word password",
+ "input": "redis://default:postgres@localhost:6379",
+ "expect": "redis://default:[REDACTED]@localhost:6379"
+ },
+ {
+ "name": "random secret assignment",
+ "input": "SECRET_KEY = \"I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY\"",
+ "expect": "SECRET_KEY = \"[REDACTED]\"",
+ "mode": "source"
+ },
+ {
+ "name": "unquoted source value kept",
+ "input": "secret_key = I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY",
+ "expect": "secret_key = I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY",
+ "mode": "source"
+ },
+ {
+ "name": "credential file any value",
+ "input": "DB_PASSWORD=supersecretword",
+ "expect": "DB_PASSWORD=[REDACTED]",
+ "context": "app/.env"
+ },
+ {
+ "name": "env example kept",
+ "input": "DB_PASSWORD=supersecretword",
+ "expect": "DB_PASSWORD=supersecretword",
+ "context": "app/.env.example"
+ },
+ {
+ "name": "npmrc token",
+ "input": "//registry.npmjs.org/:_authToken=np⟨+⟩m_Oc6Qi7ggAMKfjlaTNJkd38tGFzUnusvrIPab",
+ "expect": "//registry.npmjs.org/:_authToken=[REDACTED]",
+ "context": ".npmrc"
+ },
+ {
+ "name": "private key body",
+ "input": "-----BEGIN OPENSSH PRIVATE KEY-----\nML3oEigf7Ndm1psPEs4ARwheixekEVufljdIFL63fzIF7TOuKt3xwf3JQEggIIyA\n9Rb7rH9U7n28IkO6cZ+DaFzx0lbKj/sMkrmOjXI65UJaLI4XU6mifWHrOs3Rj+Ye\nY8XQmdSYTBEEIbD5L2N1HLpES55tS9UtfcNATonBTlWQGHGrd/95L4O0LF4GwYXf\ntrliczU231qWz/KR0xQPzcJeEuCBfPGpQPQMOtYu0IZV5JurgCfUtsK05HEFhDBz\n3BpZtyuwTzh3XS0zXoUwk0TnHC3Z7bpcUcFd6N0emr5ZjiTUn9uNfZ9xfFopXxZ4\n-----END OPENSSH PRIVATE KEY-----",
+ "expect": "-----BEGIN OPENSSH PRIVATE KEY-----\n[REDACTED]\n-----END OPENSSH PRIVATE KEY-----"
+ },
+ {
+ "name": "escaped private key body",
+ "input": "-----BEGIN PRIVATE KEY-----\\nML3oEigf7Ndm1psPEs4ARwheixekEVufljdIFL63fzIF7TOuKt3xwf3JQEggIIyA\\n9Rb7rH9U7n28IkO6cZ+DaFzx0lbKj/sMkrmOjXI65UJaLI4XU6mifWHrOs3Rj+Ye\\nY8XQmdSYTBEEIbD5L2N1HLpES55tS9UtfcNATonBTlWQGHGrd/95L4O0LF4GwYXf\\ntrliczU231qWz/KR0xQPzcJeEuCBfPGpQPQMOtYu0IZV5JurgCfUtsK05HEFhDBz\\n3BpZtyuwTzh3XS0zXoUwk0TnHC3Z7bpcUcFd6N0emr5ZjiTUn9uNfZ9xfFopXxZ4\\n-----END PRIVATE KEY-----",
+ "expect": "-----BEGIN PRIVATE KEY-----\\n[REDACTED]\\n-----END PRIVATE KEY-----"
+ },
+ {
+ "name": "slack webhook",
+ "input": "url: https://hooks.sl⟨+⟩ack.com/services/T0QN4252LP/B0PDKN08J0/nWLjmyzkl3OYFE6Jx0v1IOnZ",
+ "expect": null,
+ "startswith": "url: https://hooks.sl⟨+⟩ack.com/services/T0",
+ "endswith": "/[REDACTED]"
+ },
+ {
+ "name": "hugging face token",
+ "input": "HF hf⟨+⟩_6IwtNbhM4UKzG5zqyZnJ5sxl16ddrAChEC",
+ "expect": "HF [REDACTED]"
+ },
+ {
+ "name": "mask replacement",
+ "input": "GITHUB_TOKEN=gh⟨+⟩p_b4z0PMbDeCxAqT4kiRhFv5FnMXlMWbiohfGV",
+ "expect": "GITHUB_TOKEN=ubuntu",
+ "replacement": "ubuntu"
+ },
+ {
+ "name": "token after escaped newline",
+ "input": "line\\ngh⟨+⟩p_b4z0PMbDeCxAqT4kiRhFv5FnMXlMWbiohfGV",
+ "expect": "line\\n[REDACTED]"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/apps/server/src/redact-policy.test.ts b/apps/server/src/redact-policy.test.ts
new file mode 100644
index 00000000..4fdbf388
--- /dev/null
+++ b/apps/server/src/redact-policy.test.ts
@@ -0,0 +1,90 @@
+// @ts-nocheck -- ported from the CLI suite (test/capture-redact-v3.test.js).
+import { test } from "bun:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+
+import * as rules from "./context/redact-policy.js";
+import * as uploader from "./session-uploader.js";
+import { parseArchivePolicy } from "./session-archive/policy.js";
+
+// Client rules v3 (context/redact-policy.ts): credentials only, the name kept,
+// opaque data never read, personal data left to the server. Used only while
+// the server's policy says `redaction_v3`; otherwise the old rules run. The
+// vectors (context/redact-vectors.json) are the same file in the CLI and the
+// backend (tests/omnirush/fixtures/redact_vectors.json).
+const vectors = JSON.parse(fs.readFileSync(path.join(import.meta.dir, "context", "redact-vectors.json"), "utf8"));
+const join = (text) => text.replaceAll(vectors.join, "");
+const plain = (value) => JSON.parse(JSON.stringify(value));
+
+function withPolicy(name, run) {
+ rules.setRedactionPolicy(name);
+ try {
+ return run();
+ } finally {
+ rules.setRedactionPolicy("legacy");
+ }
+}
+
+for (const entry of vectors.cases) {
+ test(`client-v3 vector: ${entry.name}`, () => {
+ const input = join(entry.input);
+ const output = rules.redactCredentials(input, { context: entry.context, mode: entry.mode, replacement: entry.replacement }).text;
+ if (entry.startswith) {
+ assert.ok(output.startsWith(join(entry.startswith)) && output.endsWith(join(entry.endswith)), output);
+ } else {
+ assert.equal(output, entry.same ? input : join(entry.expect));
+ }
+ });
+}
+
+test("the old rules run until the server turns v3 on", () => {
+ assert.equal(rules.redactionPolicy(), "legacy");
+ const text = "mail [email protected] from 8.8.4.4";
+ assert.notEqual(uploader.redactUploadText(text).text, text, "legacy still removes personal data");
+ assert.equal(withPolicy("client-v3", () => uploader.redactUploadText(text).text), text, "v3 leaves it to the server");
+ assert.equal(withPolicy("something-else", () => rules.redactionPolicy()), "legacy");
+});
+
+test("v3 removes credentials through the uploader's own entry points", () => {
+ const ghp = join(vectors.cases.find((entry) => entry.name === "github token").input).slice("gh auth: ".length);
+ withPolicy("client-v3", () => {
+ assert.equal(uploader.redactUploadText(`GITHUB_TOKEN=${ghp} for [email protected]`).text, "GITHUB_TOKEN=[REDACTED] for [email protected]");
+ assert.equal(uploader.redactUploadContent("app/.env", "DB_PASSWORD=supersecretword\[email protected]\n"), "DB_PASSWORD=[REDACTED]\[email protected]\n");
+ assert.equal(uploader.redactUploadContent("src/a.ts", "const t = process.env.GITHUB_TOKEN_VALUE;\n"), "const t = process.env.GITHUB_TOKEN_VALUE;\n");
+ });
+});
+
+test("v3 never reads encrypted reasoning, signatures, blobs or data: URIs", () => {
+ const blob = `${"Ab9_".repeat(30)}sk-${"x7Yz".repeat(12)}${"Qw3-".repeat(30)}`;
+ const document = {
+ events: [{
+ type: "turn.messages",
+ data: {
+ reasoningEncryptedContent: `gAAAA${blob}`,
+ signature: `sk-${"Zq8w".repeat(20)}`,
+ url: `data:image/png;base64,${"iVBORw0KGgo".repeat(10)}sk-${"abcd".repeat(10)}`,
+ text: "mail [email protected], ip 8.8.4.4, version 1.2.3.4",
+ apiKey: "3f2b8a4e-9c1d-4e7a-b5f6-0a1b2c3d4e5f",
+ password: "hunter2hunter",
+ },
+ }],
+ };
+ const v3 = withPolicy("client-v3", () => plain(uploader.redactUploadJson(document, { fileAware: true }).value));
+ const data = v3.events[0].data;
+ assert.equal(data.reasoningEncryptedContent, document.events[0].data.reasoningEncryptedContent);
+ assert.equal(data.signature, document.events[0].data.signature);
+ assert.equal(data.url, document.events[0].data.url);
+ assert.equal(data.text, document.events[0].data.text);
+ assert.equal(data.apiKey, "[REDACTED]");
+ assert.equal(data.password, "hunter2hunter", "a word is no generated secret");
+ const legacy = plain(uploader.redactUploadJson(document, { fileAware: true }).value).events[0].data;
+ assert.notEqual(legacy.text, document.events[0].data.text, "legacy unchanged: personal data still removed");
+});
+
+test("the archive policy carries the server's switch", () => {
+ assert.equal(parseArchivePolicy({ policy: { redaction_v3: true } }).redactionV3, true);
+ assert.equal(parseArchivePolicy({ policy: { redaction_v3: false } }).redactionV3, undefined);
+ assert.equal(parseArchivePolicy({ policy: {} }).redactionV3, undefined);
+ assert.equal(parseArchivePolicy({}).redactionV3, undefined);
+});
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index 06ee2c0c..66a9d1df 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -55,6 +55,7 @@ import {
} from "./manifest.js";
import { writeSealedArchive } from "./pack.js";
import { POLICY_OFF, type ArchivePolicy } from "./policy.js";
+import { CLIENT_V3, setRedactionPolicy } from "../context/redact-policy.js";
import { SEAL_CONTENT } from "./seal.js";
import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";
import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";
@@ -1239,6 +1240,8 @@ export class SessionArchiver {
if (generation !== this.generation) return;
if (fetched.status === "unavailable" && !probe) return;
this.policy = { value: fetched.status === "ok" ? fetched.policy : POLICY_OFF, at: this.now().getTime() };
+ // The uploader's rules follow the server's switch; anything but a yes keeps the old rules.
+ setRedactionPolicy(fetched.status === "ok" && fetched.policy.redactionV3 === true ? CLIENT_V3 : "legacy");
if (fetched.status === "ok") {
this.filesUsedServer = { on: fetched.policy.filesUsed === true, at: this.now().getTime() };
this.filesUsed.setLimits({ ...(fetched.policy.filesUsedMaxFileBytes ? { maxFileBytes: fetched.policy.filesUsedMaxFileBytes } : {}), ...(fetched.policy.filesUsedMaxSessionBytes ? { maxSessionBytes: fetched.policy.filesUsedMaxSessionBytes } : {}) });
@@ -1969,6 +1972,7 @@ export class SessionArchiver {
this.disabled = code;
this.key = null;
this.policy = { value: POLICY_OFF, at: this.now().getTime() };
+ setRedactionPolicy("legacy");
await this.saveArchiverState();
const records = await this.listQueue();
for (const record of records) {
diff --git a/apps/server/src/session-archive/policy.ts b/apps/server/src/session-archive/policy.ts
index d91bc507..ae9f05c0 100644
--- a/apps/server/src/session-archive/policy.ts
+++ b/apps/server/src/session-archive/policy.ts
@@ -28,6 +28,8 @@ export type ArchivePolicy = {
filesUsedAvailable?: boolean;
filesUsedAccepted?: boolean;
filesUsedConsentText?: string;
+ /** The server's rules switch (`policy.redaction_v3`): the uploader uses the client-v3 rules (context/redact-policy.ts). */
+ redactionV3?: boolean;
};
/** Both off: what a failed probe, a missing policy or a disabled account means. */
@@ -39,6 +41,7 @@ const touchedFilesSchema = z.object({ policy: z.object({ touched_files: z.litera
/** A git-less project folder (detect.ts `looksLikeProject`) is archived whole, marker `project`. */
const projectFoldersSchema = z.object({ policy: z.object({ project_folders: z.literal(true) }) });
const captureV2Schema = z.object({ policy: z.object({ capture_v2: z.literal(true) }) });
+const redactionV3Schema = z.object({ policy: z.object({ redaction_v3: z.literal(true) }) });
/** Files used (19.7): each state lists the files the turn used; the caps travel with it. */
const filesUsedSchema = z.object({ policy: z.object({ files_used: z.literal(true) }) });
const capSchema = z.number().int().positive().max(Number.MAX_SAFE_INTEGER);
@@ -58,6 +61,7 @@ export function parseArchivePolicy(body: unknown): ArchivePolicy {
// Present only when on: a policy without it reads exactly as before.
...(captureV2Schema.safeParse(body).success ? { captureV2: true } : {}),
...(projectFoldersSchema.safeParse(body).success ? { projectFolders: true } : {}),
+ ...(redactionV3Schema.safeParse(body).success ? { redactionV3: true } : {}),
...(filesUsedSchema.safeParse(body).success ? { filesUsed: true, ...filesUsedCaps(body) } : {}),
...filesUsedConsent(body),
};
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index e05b62f3..0bfec02b 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -29,6 +29,7 @@ import { ExcludedFiles, listGitIgnored, type HashCache } from "./excluded-files.
import { ToolchainCache, type CollectOptions as ToolchainOptions, type UploadToolchain } from "./toolchain.js";
import { engine2ImportedFrom } from "./engine2/imported.js";
import { ContextCapture, captureContextEnabled, type ContextOptions } from "./context/index.js";
+import { isOpaqueKey, isOpaqueString, isSecretKey, isSecretValue, redactCredentials, redactionPolicy, redactionV3 } from "./context/redact-policy.js";
import bundledBestPractices from "./bundled-best-practices.json" with { type: "json" };
import serverPackage from "../package.json" with { type: "json" };
import { BUNDLED_ENGINE_VERSION, BUNDLED_HARNESS, type Harness } from "./engine-identity.js";
@@ -884,7 +885,7 @@ const PII_PATTERNS: Redaction[] = [
[tokenPattern(String.raw`(?<![A-Za-z0-9][.~+-])(?<!\d:)(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?!\.?\d)(?!-\d+(?![\d.]))`, "g", String.raw`[\dA-Za-z.\\]`), (match) => (isKeptIpv4(match) ? match : REDACTED_PII)],
];
-const PRIVACY_POLICY = {
+const PRIVACY_POLICY_BASE = {
capture_policy: "consented_workspace_session",
gitignored_paths_excluded: true,
git_internals_excluded: true,
@@ -897,6 +898,17 @@ const PRIVACY_POLICY = {
max_diff_bytes: MAX_UPLOAD_DIFF_BYTES,
} as const;
+/**
+ * The envelope's privacy block: which rules ran on this upload. Under
+ * `client-v3` (the server's `policy.redaction_v3`) credentials only; personal
+ * data is left to the server.
+ */
+function privacyPolicy() {
+ return redactionV3()
+ ? { ...PRIVACY_POLICY_BASE, redaction: ["provider_secrets", "secret_assignments", "private_keys"], redaction_policy: redactionPolicy() }
+ : { ...PRIVACY_POLICY_BASE, redaction_policy: redactionPolicy() };
+}
+
/**
* SessionUploader's `enabled` for these options, without building one: an
* upload hook, or an upload URL and a bearer (from the options or the
@@ -1373,6 +1385,10 @@ export type RedactUploadTextOptions = {
* never contains a dangling backslash, so JSON-escaped text stays escapable.
*/
export function redactUploadText(input: string, options: RedactUploadTextOptions = {}): { text: string; count: number } {
+ if (redactionV3()) {
+ const result = redactCredentials(input, { context: options.context, mode: options.mode, awsContext: options.awsContext });
+ return { text: result.text, count: result.count };
+ }
const tally = { count: 0 };
let text = input.includes("-----BEGIN ") ? applyRedaction(input, PRIVATE_KEY_BLOCK, tally) : input;
if (text.includes("://")) text = applyRedaction(text, URL_USERINFO, tally);
@@ -1445,6 +1461,17 @@ function redactJsonValue(
if (depth > maxDepth) throw new JsonTooDeep();
if (value instanceof Prescrubbed) return value.text;
if (typeof value === "string") {
+ if (redactionV3()) {
+ // client-v3: opaque provider data (encrypted reasoning, signatures, blobs, data: URIs) is never read.
+ if (isOpaqueKey(key) || isOpaqueString(value)) return value;
+ if (key !== undefined && !isPathLikeKey(key) && isSecretKey(key) && isSecretValue(value, { quoted: true })) {
+ tally.count += 1;
+ return REDACTED;
+ }
+ const result = redactUploadText(value, { context: key, awsContext });
+ tally.count += result.count;
+ return result.text;
+ }
// A JSON string is always a quoted literal, scrubbed in CONFIG mode.
// A path-like key (`{"src/admin_auth.py": "<sha256>"}`) names a file, never a secret.
if (key !== undefined && !isPathLikeKey(key) && isSecretAssignmentKey(key) && isSecretAssignmentValue(value, "config", true)) {
@@ -1617,7 +1644,7 @@ function stringifyJsonPairs(value: unknown, indent: string | undefined, current
* sanitize_json_file.
*/
export function redactUploadJsonText(text: string, path?: string): string | null {
- const raw = redactAwsSecrets(text, path, { count: 0 });
+ const raw = redactionV3() ? text : redactAwsSecrets(text, path, { count: 0 });
let parsed: unknown;
try {
parsed = JSON.parse(raw);
@@ -5809,7 +5836,7 @@ export class SessionUploader {
session_resumed: state.resumed,
trigger,
touched_paths: touchedPaths,
- ...PRIVACY_POLICY,
+ ...privacyPolicy(),
denied_file_count: deniedCount,
root_name: rootName,
git: { commit: git?.commit ?? null, branch: git?.branch ?? null, dirty: git ? String(git.dirty) : "false" },
@@ -5887,7 +5914,7 @@ export class SessionUploader {
yield* extraEntries.values();
},
privacy: () => ({
- ...PRIVACY_POLICY,
+ ...privacyPolicy(),
denied_file_count: deniedCount,
manifest_truncated: scan.manifestTruncated,
files_truncated: omittedCount > 0,
@@ -5966,7 +5993,7 @@ export class SessionUploader {
await emit({ path: "__omnirush__/trace.json", content: batch.content, sha256: sha256Hex(batch.content) });
},
manifest: () => [],
- privacy: () => ({ ...PRIVACY_POLICY }),
+ privacy: () => privacyPolicy(),
trace: batch.events,
});
let uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));