A desktop coding agent with free access to frontier models.
TypeScript
feat: shared upload rules v3
ogsamratb054a84e0d4e
diff --git a/apps/server/src/capture-context.test.ts b/apps/server/src/capture-context.test.tsindex 65e55c11..dca44ef8 100644--- a/apps/server/src/capture-context.test.ts+++ b/apps/server/src/capture-context.test.ts@@ -683,7 +683,7 @@ test("CLI/desktop parity: the context modules match PARITY.sha256 (the same listconst contextDir = path.join(import.meta.dir, "context");const lines = fs.readFileSync(path.join(contextDir, "PARITY.sha256"), "utf8").split("\n").filter((line) => line && !line.startsWith("#"));const listed = new Map(lines.map((line) => { const [hash, name] = line.split(/\s+/); return [name, hash]; }));- const files = fs.readdirSync(contextDir).filter((name) => name.endsWith(".ts")).sort();+ const files = fs.readdirSync(contextDir).filter((name) => name.endsWith(".ts") || name.endsWith(".json")).sort();assert.deepEqual([...listed.keys()].sort(), files);for (const name of files) assert.equal(createHash("sha256").update(fs.readFileSync(path.join(contextDir, name))).digest("hex"), listed.get(name), `${name} changed: port it to the CLI and update PARITY.sha256 in both`);});diff --git a/apps/server/src/context/PARITY.sha256 b/apps/server/src/context/PARITY.sha256index 5d175192..1bfbdfd4 100644--- a/apps/server/src/context/PARITY.sha256+++ b/apps/server/src/context/PARITY.sha256@@ -9,6 +9,8 @@ d772119f2785b6926cd765a44589c171542d5e79b3677bfa076851775d794480 outside.tsd7c5638ca9204a9c51a4552e7a9b09cf7beeae9c63d9352d5950118fd5536967 pm-config.tsd13231905671ad40fdd73e3fe7af81e44dcc909ace606f45a79c215984b6b35d privacy.ts0e04845a2d0ad2136c01a61d3ed1193b62ef99f5b82e1f555b7c97bb6dc18773 processes.ts+dfe4efad20d114462bc0f9e3f3178eb7e44c4241d119b7deb1524183bf08b481 redact-policy.ts+16f6bbf34934e279bd1ff3f6db5841bcd3e9f5a9ec05363dc39f5e7108a16e6a redact-vectors.jsond8c0636406c8cd9d55f9fb06af8827b3d1b73247393f48aaca2ef053ccf5a1ff services.ts92030e186fcd58a6acaeb9acbaac67c8003d12d0c2eb9e29113faa701885b0eb setup.tsd51d434283f757f0ed45bb6996a74fb5bf9e1d56f34146e99df3c392cde5ea19 shell-aliases.tsdiff --git a/apps/server/src/context/redact-policy.ts b/apps/server/src/context/redact-policy.tsnew file mode 100644index 00000000..9cacd2d9--- /dev/null+++ b/apps/server/src/context/redact-policy.ts@@ -0,0 +1,442 @@+// Client rules v3 ("client-v3"): what the uploader removes on the device once+// the server turns them on (`policy.redaction_v3` in GET /archives/key).+// Credentials only, at high precision, the variable name always kept; personal+// data (e-mail, phone, SSN, IPv4) is left to the server's export pass, and+// opaque data (encrypted reasoning, signatures, base64 blobs, `data:` URIs) is+// never touched. Until the server says so the uploader keeps its old rules.+//+// The same file in omnirush-cli (assets/extensions/omnirush/capture/context/)+// and omnirush-gui (apps/server/src/context/), listed in PARITY.sha256; the+// backend runs the same vectors (redact-vectors.json) against its port+// (omnirush/credential_rules.py).++export const CLIENT_V3 = "client-v3";+export type RedactionPolicy = "legacy" | typeof CLIENT_V3;++// One switch per process, kept on globalThis so every copy of this module+// (a bundle, a loader without a module cache) reads the same answer.+const STATE = Symbol.for("omnirush.redactionPolicy");+type PolicyHolder = { [STATE]?: RedactionPolicy };++/** Switches the uploader's rules; anything but `client-v3` is the old rule set. */+export function setRedactionPolicy(policy: string | null | undefined): void {+ (globalThis as PolicyHolder)[STATE] = policy === CLIENT_V3 ? CLIENT_V3 : "legacy";+}++export function redactionPolicy(): RedactionPolicy {+ return (globalThis as PolicyHolder)[STATE] === CLIENT_V3 ? CLIENT_V3 : "legacy";+}++export function redactionV3(): boolean {+ return redactionPolicy() === CLIENT_V3;+}++const REDACTED = "[REDACTED]";++/** JSON keys whose string values are opaque provider data, never scanned. */+const OPAQUE_KEYS = new Set([+ "reasoningencryptedcontent", "reasoning_encrypted_content", "encrypted_content", "encryptedcontent",+ "signature", "thinkingsignature", "thinking_signature", "redacted_thinking", "redactedthinking",+]);++export function isOpaqueKey(key: string | undefined): boolean {+ return key !== undefined && OPAQUE_KEYS.has(key.toLowerCase());+}++// A run of base64 or base64url characters this long with no break is a blob+// (encrypted content, an image, a signature), not a credential.+const OPAQUE_RUN = /[A-Za-z0-9+/_-]{200,}={0,2}/g;+const DATA_URI = /data:[A-Za-z0-9.+/-]*(?:;[A-Za-z0-9.+=-]+)*,[A-Za-z0-9+/=%._~-]*/g;++/** A whole string that is a `data:` URI or one base64 blob. */+export function isOpaqueString(value: string): boolean {+ if (value.startsWith("data:") && /^data:[A-Za-z0-9.+/-]*(?:;[A-Za-z0-9.+=-]+)*,/.test(value)) return true;+ return value.length >= 200 && /^[A-Za-z0-9+/_-]+={0,2}$/.test(value);+}++function opaqueSpans(text: string): Array<[number, number]> {+ const spans: Array<[number, number]> = [];+ if (text.length < 200 && !text.includes("data:")) return spans;+ for (const pattern of [OPAQUE_RUN, DATA_URI]) {+ pattern.lastIndex = 0;+ for (let match = pattern.exec(text); match; match = pattern.exec(text)) {+ if (match[0].length === 0) {+ pattern.lastIndex += 1;+ continue;+ }+ spans.push([match.index, match.index + match[0].length]);+ }+ }+ return spans;+}++/** A match lies inside an opaque span (and is not the whole span: a long token on its own still counts). */+function insideOpaque(spans: Array<[number, number]>, start: number, end: number): boolean {+ return spans.some(([from, to]) => start >= from && end <= to && !(start === from && end === to));+}++/** Shannon entropy in bits per character. */+export function shannonEntropy(value: string): number {+ if (!value) return 0;+ const counts = new Map<string, number>();+ for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);+ const length = [...value].length;+ let entropy = 0;+ for (const count of counts.values()) {+ const p = count / length;+ entropy -= p * Math.log2(p);+ }+ return entropy;+}++function charClasses(value: string): number {+ return [/[a-z]/, /[A-Z]/, /[0-9]/, /[^A-Za-z0-9]/].filter((pattern) => pattern.test(value)).length;+}++/** 12+ characters, two or more character classes and 3.0+ bits per character. */+export function looksRandom(value: string): boolean {+ return value.length >= 12 && charClasses(value) >= 2 && shannonEntropy(value) >= 3.0;+}++/**+ * Credential files: any non-placeholder value of a secret-named key there is+ * a credential. `.env*` except examples and templates, `.npmrc`, `.pypirc`,+ * `.netrc`, `.pgpass`, `.git-credentials`, `credentials` (`.aws/credentials`),+ * kubeconfig.+ */+export function isCredentialFile(path: string | undefined): boolean {+ if (!path) return false;+ const name = path.split(/[\\/]/).pop()!.toLowerCase();+ if (name.startsWith(".env")) return !/(?:example|sample|template|dist|defaults?)$/.test(name);+ return [".npmrc", ".pypirc", ".netrc", "_netrc", ".pgpass", ".git-credentials", "credentials", "kubeconfig", "config.kube"].includes(name)+ || name.endsWith(".kubeconfig");+}++// --- secret-named assignments ----------------------------------------------++// Python's `\s`, so a value ends where the backend's does.+const WS = String.raw`\t\n\v\f\r \x1c-\x1f\x85\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000`;+const KEYWORD = String.raw`secret|passw(?:or)?d|pwd|token|credential|auth|(?:api|access|private|client|session|signing|master|encryption)[_.:-]?key`;+const ASSIGNMENT_BODY = String.raw`(\\?["']?)((?=[\w.-]{0,63}?(?:${KEYWORD}))[A-Za-z_-][\w.-]{0,63})\1[ \t]*[=:][ \t]*`+ + String.raw`(?:"((?:[^"${WS}\\]|\\[^${WS}]){8,})"`+ + String.raw`|'((?:[^'${WS}\\]|\\[^${WS}]){8,})'`+ + String.raw`|\\"((?:[^"${WS}\\]|\\[^"${WS}]){8,})\\"`+ + String.raw`|((?:[^${WS}"',;&\\]|\\[^${WS}]){8,}))`;+const CONFIG_ASSIGNMENT = new RegExp(String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");+const SOURCE_ASSIGNMENT = new RegExp(String.raw`(?:(?<![^${WS}\x80-\uffff{,("'\x60+#])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");++const SECRET_SEGMENTS = new Set([+ "secret", "secrets", "password", "passwords", "passwd", "pwd", "token", "credential", "credentials",+ "auth", "authorization", "authtoken", "authkey", "apikey",+]);+const SECRET_PAIRS: Array<[string, string]> = [+ ["api", "key"], ["access", "key"], ["secret", "key"], ["private", "key"], ["client", "key"], ["client", "secret"],+ ["session", "key"], ["signing", "key"], ["master", "key"], ["encryption", "key"],+];+const EXCLUDED_LAST = new Set([+ "length", "ttl", "seconds", "count", "size", "url", "path", "name", "id", "header",+ "file", "filename", "dir", "mode", "method", "role", "owner", "type", "kind", "enabled", "estimate", "hash", "digest", "at",+ "config", "client", "prefix", "suffix", "format", "scheme", "provider", "status", "state", "label", "description", "title",+ "class", "field", "fields", "list", "names", "version", "timeout", "limit", "max", "min", "interval", "retries", "port",+ "in", "out", "percent", "pct", "ms", "secs", "minutes", "hours", "days", "expires", "expiry", "expiration", "bytes", "len",+ "width", "height", "offset", "index", "idx", "pos", "total", "sum", "avg", "ratio", "rate", "threshold", "weight", "score",+ "encryption", "algorithm", "algo", "cipher", "strategy", "policy", "source", "target", "origin", "backend", "engine", "driver",+ "handler", "callback", "event", "action", "reason", "message", "error", "envelope", "ref", "reference", "link", "pointer", "alias",+]);++function keySegments(key: string): string[] {+ return key+ .replace(/([a-z0-9])([A-Z])/g, "$1 $2")+ .replace(/([A-Z])([A-Z][a-z])/g, "$1 $2")+ .replace(/([A-Za-z])(\d)/g, "$1 $2")+ .replace(/(\d)([A-Za-z])/g, "$1 $2")+ .toLowerCase()+ .split(/[\s_.:-]+/)+ .filter(Boolean);+}++/** Whether a key names a secret (`auth_token`, `APIKey`, `client_secret`; not `author`, `token_url`, `accessKeyId`). */+export function isSecretKey(key: string): boolean {+ const segments = keySegments(key);+ const last = segments.at(-1);+ if (last === undefined || EXCLUDED_LAST.has(last)) return false;+ return segments.some((segment, index) => SECRET_SEGMENTS.has(segment)+ || SECRET_PAIRS.some(([first, second]) => segment === `${first}${second}` || (segment === first && segments[index + 1] === second)));+}++const PLACEHOLDER_MARKERS = new Set([+ "your", "yours", "here", "fake", "dummy", "example", "sample", "placeholder", "changeme", "replaceme", "replace", "insert",+ "todo", "fixme", "tbd", "mock", "demo", "redacted", "notreal", "test", "testing",+]);+const PLACEHOLDER_WORDS = new Set([+ "my", "the", "a", "an", "not", "real", "is", "goes", "go", "put", "enter", "set", "this", "to", "with", "of", "for", "in", "me",+ "change", "secret", "secrets", "token", "tokens", "key", "keys", "api", "apikey", "password", "passwd", "pass", "pwd", "value",+ "access", "auth", "private", "client", "id", "string", "foo", "bar", "baz", "qux", "abc", "xyz", "default", "dev", "local",+]);+const REPEATED_CHAR = /^(.)\1*$/su;++/** `your-token-here`, `changeme`, `xxxxxxxx`, `YOUR_API_KEY`: never a secret. */+export function isPlaceholder(value: string): boolean {+ const segments = keySegments(value);+ if (segments.length === 0) return false;+ let marked = false;+ for (const segment of segments) {+ if (PLACEHOLDER_MARKERS.has(segment) || ([...segment].length >= 3 && REPEATED_CHAR.test(segment))) marked = true;+ else if (PLACEHOLDER_WORDS.has(segment) || (/^[0-9]+$/.test(segment) && (segment.length <= 4 || "0123456789".includes(segment)))) continue;+ else return false;+ }+ return marked;+}++/** A variable or an expression, never a literal: `$VAR`, `${...}`, `$(...)`, `%VAR%`, `process.env.X`, `os.environ[...]`, `getenv(`. */+export function isReference(value: string): boolean {+ return /^\$[A-Za-z_]\w*$/.test(value) || value.includes("${") || value.includes("$(") || /^%[A-Za-z_]\w*%$/.test(value)+ || /(?:^|\W)(?:process\.env|import\.meta\.env|os\.environ|getenv\(|ENV\[|env\()/.test(value)+ || /^(?:process|os|env|self|this|config|settings|secrets|vars|inputs)\./.test(value)+ || /\{\{[^}]*\}\}|\{%[^%]*%\}/.test(value);+}++const PATH_VALUE = /^(?:\/|\.\/|\.\.\/|~\/|[A-Za-z]:[\\/])/;+const IDENTIFIER_VALUE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/;++export type ValueContext = { mode?: "source" | "config"; quoted?: boolean; credentialFile?: boolean };++/**+ * The value of a secret-named key is removed when it is a literal that looks+ * generated (12+ characters, two classes, 3.0+ bits), carries a known token,+ * or sits in a credential file. Never a reference, a placeholder, a path or+ * an earlier marker; in source code only a quoted literal.+ */+export function isSecretValue(value: string, context: ValueContext = {}): boolean {+ if (value.length < 8 || new RegExp(`[${WS}]`).test(value) || value.startsWith("[REDACTED")) return false;+ if (isReference(value) || isPlaceholder(value)) return false;+ if (value.startsWith("<") && value.endsWith(">")) return false;+ if (PATH_VALUE.test(value) || (value.includes("/") && !/\d/.test(value))) return false;+ if (containsKnownToken(value)) return true;+ if (value.includes("(")) return false;+ if (context.credentialFile) return true;+ if (context.mode === "source" && !context.quoted) return false;+ if (!context.quoted && IDENTIFIER_VALUE.test(value)) return false;+ return looksRandom(value);+}++// --- known token shapes ----------------------------------------------------++/** Start of a token: never inside a word or on an escape's letter; right after `\n`/`\r`/`\t` escapes still counts. */+function token(source: string, flags = "g"): RegExp {+ return new RegExp(String.raw`(?:(?<![A-Za-z0-9_\\])|(?<=\\[nrt]))(?:${source})`, flags);+}++type TokenRule = { name: string; gate: string | RegExp; pattern: RegExp; check?: (match: string) => boolean; keep?: number };++function jwtHeader(match: string): boolean {+ try {+ const header = JSON.parse(Buffer.from(match.slice(0, match.indexOf(".")), "base64url").toString("utf8")) as unknown;+ return typeof header === "object" && header !== null && !Array.isArray(header) && "alg" in header;+ } catch {+ return false;+ }+}++const TOKEN_RULES: TokenRule[] = [+ { name: "aws_access_key_id", gate: /AKIA|ASIA/, pattern: token(String.raw`(?:AKIA|ASIA)[0-9A-Z]{16}(?![0-9A-Za-z])`), check: (m) => !m.endsWith("EXAMPLE") },+ { name: "github_token", gate: /gh[pousr]_/, pattern: token(String.raw`gh[pousr]_[A-Za-z0-9]{36}(?![A-Za-z0-9])`) },+ { name: "github_pat", gate: "github_pat_", pattern: token(String.raw`github_pat_[A-Za-z0-9_]{60,}`) },+ { name: "gitlab_token", gate: "glpat-", pattern: token(String.raw`glpat-[A-Za-z0-9_-]{20,}`) },+ { name: "slack_token", gate: /xox[abprs]-/, pattern: token(String.raw`xox[abprs]-[A-Za-z0-9-]{10,}`) },+ { name: "slack_webhook", gate: "hooks.slack.com", pattern: /(https:\/\/hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/)([A-Za-z0-9]{20,})/g, keep: 1 },+ { name: "discord_webhook", gate: "/api/webhooks/", pattern: /(https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/)([A-Za-z0-9_-]{50,})/g, keep: 1 },+ { name: "google_api_key", gate: "AIza", pattern: token(String.raw`AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])`) },+ { name: "stripe_key", gate: /[sr]k_live_/, pattern: token(String.raw`[sr]k_live_[A-Za-z0-9]{16,}`) },+ { name: "stripe_webhook", gate: "whsec_", pattern: token(String.raw`whsec_[A-Za-z0-9+/=]{24,}`) },+ { name: "anthropic_key", gate: "sk-ant-", pattern: token(String.raw`sk-ant-(?:api|admin)\d\d-[A-Za-z0-9_-]{32,}`) },+ { name: "openai_key", gate: "sk-", pattern: token(String.raw`sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{40,}|sk-[A-Za-z0-9]{48}(?![A-Za-z0-9])`), check: (m) => shannonEntropy(m) >= 3.5 },+ { name: "npm_token", gate: "npm_", pattern: token(String.raw`npm_[A-Za-z0-9]{36}(?![A-Za-z0-9])`) },+ { name: "pypi_token", gate: "pypi-AgE", pattern: token(String.raw`pypi-AgE[A-Za-z0-9_-]{50,}`) },+ { name: "huggingface_token", gate: "hf_", pattern: token(String.raw`hf_[A-Za-z0-9]{34,}(?![A-Za-z0-9])`) },+ { name: "sendgrid_key", gate: "SG.", pattern: token(String.raw`SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}(?![A-Za-z0-9_-])`) },+ { name: "jwt", gate: "eyJ", pattern: token(String.raw`eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`), check: jwtHeader },+ { name: "discord_token", gate: /[MN][\w-]{23,}\./, pattern: token(String.raw`[MN][A-Za-z0-9_-]{23,}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}`), check: (m) => shannonEntropy(m) >= 3.5 },+];++function gateOpen(gate: string | RegExp, text: string): boolean {+ return typeof gate === "string" ? text.includes(gate) : gate.test(text);+}++/** Whether a value holds one known credential token (its whole shape and checks). */+export function containsKnownToken(value: string): boolean {+ for (const rule of TOKEN_RULES) {+ if (!gateOpen(rule.gate, value)) continue;+ rule.pattern.lastIndex = 0;+ for (let match = rule.pattern.exec(value); match; match = rule.pattern.exec(value)) {+ const secret = rule.keep ? match[rule.keep + 1]! : match[0];+ if (!rule.check || rule.check(secret)) return true;+ }+ }+ return false;+}++// --- the text pass ---------------------------------------------------------++export type CredentialOptions = {+ /** The file path or JSON key the text belongs to. */+ context?: string;+ /** Source code (quoted literals only) or config/trace text. */+ mode?: "source" | "config";+ /** The enclosing JSON object names an AWS key. */+ awsContext?: boolean;+ /** What a removed value is written as; `[REDACTED]` by default. */+ replacement?: string;+};++type Tally = { count: number; rules: Record<string, number> };++function note(tally: Tally, rule: string): void {+ tally.count += 1;+ tally.rules[rule] = (tally.rules[rule] ?? 0) + 1;+}++const PRIVATE_KEY = /(?<!\\)(-----BEGIN ([A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?)-----)([\s\S]*?)(-----END \2-----)/g;++function redactPrivateKeys(text: string, hidden: string, tally: Tally): string {+ PRIVATE_KEY.lastIndex = 0;+ return text.replace(PRIVATE_KEY, (match: string, begin: string, _kind: string, body: string, end: string) => {+ const plain = body.replace(/\\[nrt]/g, " ");+ if (plain.includes("...") || plain.includes("…") || plain.replace(/[^A-Za-z0-9+/=]/g, "").length < 64) return match;+ const lead = /^(?:\s|\\[nr])*/.exec(body)![0];+ const trail = /(?:\s|\\[nr])*$/.exec(body)![0];+ note(tally, "private_key");+ return `${begin}${lead}${hidden}${trail}${end}`;+ });+}++const URL_PASSWORD = new RegExp(+ String.raw`(?:(?<![A-Za-z0-9_\\])|(?<=\\[nrt]))([a-z][a-z0-9+.-]{0,31}://)((?:[^${WS}/@:\\]|\\[^${WS}])*):((?:[^${WS}/@\\]|\\[^${WS}])+)@`,+ "gi",+);+const URL_PASSWORD_PLACEHOLDERS = new Set(["password", "pass", "passwd", "pwd", "secret", "changeme", "xxx", "***", "<password>", "[redacted]"]);++function urlPasswordPlaceholder(password: string): boolean {+ const lowered = password.toLowerCase();+ return URL_PASSWORD_PLACEHOLDERS.has(lowered) || lowered.startsWith("[redacted") || isReference(password) || /^<[^>]*>$/.test(password)+ || /^\*+$/.test(password) || isPlaceholder(password);+}++function redactUrlPasswords(text: string, hidden: string, tally: Tally, spans: Array<[number, number]>): string {+ URL_PASSWORD.lastIndex = 0;+ return text.replace(URL_PASSWORD, (match: string, scheme: string, user: string, password: string, offset: number) => {+ if (insideOpaque(spans, offset, offset + match.length) || urlPasswordPlaceholder(password)) return match;+ note(tally, "url_password");+ return `${scheme}${user}:${hidden}@`;+ });+}++const MAX_ASSIGNMENT_DEPTH = 4;++function redactAssignments(+ text: string, hidden: string, tally: Tally, mode: "source" | "config", credentialFile: boolean, spans: Array<[number, number]>, depth = 0,+): string {+ // A fresh expression per call: the pass recurses into values while the outer replace is still running.+ const pattern = new RegExp(mode === "source" ? SOURCE_ASSIGNMENT : CONFIG_ASSIGNMENT);+ return text.replace(pattern, (match: string, _quote: string, key: string, doubleQuoted: string | undefined, singleQuoted: string | undefined, escapedQuoted: string | undefined, bare: string | undefined, offset: number) => {+ if (depth === 0 && insideOpaque(spans, offset, offset + match.length)) return match;+ const value = doubleQuoted ?? singleQuoted ?? escapedQuoted ?? bare ?? "";+ const quote = doubleQuoted !== undefined ? '"' : singleQuoted !== undefined ? "'" : escapedQuoted !== undefined ? '\\"' : "";+ const prefix = match.slice(0, match.length - value.length - quote.length * 2);+ if (isSecretKey(key) && isSecretValue(value, { mode, quoted: bare === undefined, credentialFile })) {+ note(tally, "secret_assignment");+ return `${prefix}${quote}${hidden}${quote}`;+ }+ if (depth >= MAX_ASSIGNMENT_DEPTH) return match;+ return `${prefix}${quote}${redactAssignments(value, hidden, tally, mode, credentialFile, [], depth + 1)}${quote}`;+ });+}++const ASSIGNMENT_GATE = /secret|passw|pwd|token|credential|auth|key/i;++const AWS_ID = /(?:AKIA|ASIA)[0-9A-Z]{16}/;+const AWS_CONTEXT = /(?:AKIA|ASIA)[0-9A-Z]{16}|aws[_ .-]?secret[_ .-]?access[_ .-]?key/i;+const AWS_SECRET = token(String.raw`[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+])`);++function redactAwsSecrets(text: string, hidden: string, tally: Tally, nearContext: boolean, spans: Array<[number, number]>): string {+ if (!nearContext && !AWS_CONTEXT.test(text)) return text;+ if (!/[A-Za-z0-9/+]{40}/.test(text)) return text;+ const lines = text.split("\n");+ const context = lines.map((line) => AWS_CONTEXT.test(line));+ let offset = 0;+ let changed = false;+ for (let index = 0; index < lines.length; index += 1) {+ const line = lines[index]!;+ const start = offset;+ offset += line.length + 1;+ if (line.length < 40) continue;+ let near = nearContext;+ for (let other = Math.max(0, index - 3); !near && other <= Math.min(lines.length - 1, index + 3); other += 1) near = context[other]!;+ if (!near) continue;+ AWS_SECRET.lastIndex = 0;+ lines[index] = line.replace(AWS_SECRET, (candidate: string, at: number) => {+ if (!/[a-z]/.test(candidate) || !/[A-Z]/.test(candidate) || !/\d/.test(candidate) || AWS_ID.test(candidate)) return candidate;+ if (insideOpaque(spans, start + at, start + at + candidate.length) || shannonEntropy(candidate) < 3.5) return candidate;+ note(tally, "aws_secret");+ changed = true;+ return hidden;+ });+ }+ return changed ? lines.join("\n") : text;+}++function redactTokens(text: string, hidden: string, tally: Tally, spans: Array<[number, number]>): string {+ for (const rule of TOKEN_RULES) {+ if (!gateOpen(rule.gate, text)) continue;+ rule.pattern.lastIndex = 0;+ text = text.replace(rule.pattern, (...args: unknown[]) => {+ const match = args[0] as string;+ const offset = args.find((arg, index) => index > 0 && typeof arg === "number") as number;+ const secret = rule.keep ? args[rule.keep + 1] as string : match;+ if (rule.check && !rule.check(secret)) return match;+ if (insideOpaque(spans, offset, offset + match.length)) return match;+ note(tally, rule.name);+ return rule.keep ? `${args[rule.keep] as string}${hidden}` : hidden;+ });+ }+ return text;+}++const AUTH_HEADER = /(?<![A-Za-z0-9_])((?:Bearer|Basic)[ \t]+)([A-Za-z0-9_.~+/=-]{16,})/gi;++function authSecret(token: string): boolean {+ if (shannonEntropy(token) < 3.0 || isPlaceholder(token)) return false;+ if (/\d/.test(token)) return true;+ return token.length >= 20 && /[a-z]/.test(token) && /[A-Z]/.test(token) && !IDENTIFIER_VALUE.test(token);+}++function redactAuthHeaders(text: string, hidden: string, tally: Tally, spans: Array<[number, number]>): string {+ AUTH_HEADER.lastIndex = 0;+ return text.replace(AUTH_HEADER, (match: string, prefix: string, value: string, offset: number) => {+ if (!authSecret(value) || insideOpaque(spans, offset, offset + match.length)) return match;+ note(tally, "auth_header");+ return `${prefix}${hidden}`;+ });+}++/**+ * Removes credentials from free text (a trace string, a file, a diff), the+ * name always kept: private key bodies, URL passwords, secret-named values+ * that look generated (any value in a credential file), AWS secrets beside+ * their key id, known token shapes and `Bearer`/`Basic` values. Nothing+ * inside a `data:` URI or a base64 blob is touched.+ */+export function redactCredentials(input: string, options: CredentialOptions = {}): { text: string; count: number; rules: Record<string, number> } {+ const tally: Tally = { count: 0, rules: {} };+ if (!input) return { text: input, count: 0, rules: tally.rules };+ const hidden = options.replacement ?? REDACTED;+ const credentialFile = isCredentialFile(options.context);+ let text = input.includes("-----BEGIN ") ? redactPrivateKeys(input, hidden, tally) : input;+ if (text.includes("://")) text = redactUrlPasswords(text, hidden, tally, opaqueSpans(text));+ if (ASSIGNMENT_GATE.test(text)) text = redactAssignments(text, hidden, tally, options.mode ?? "config", credentialFile, opaqueSpans(text));+ const nearAws = options.awsContext === true || (options.context !== undefined && AWS_CONTEXT.test(options.context));+ text = redactAwsSecrets(text, hidden, tally, nearAws, opaqueSpans(text));+ text = redactTokens(text, hidden, tally, opaqueSpans(text));+ if (/bearer|basic/i.test(text)) text = redactAuthHeaders(text, hidden, tally, opaqueSpans(text));+ return { text, count: tally.count, rules: tally.rules };+}diff --git a/apps/server/src/context/redact-vectors.json b/apps/server/src/context/redact-vectors.jsonnew file mode 100644index 00000000..dcb076d2--- /dev/null+++ b/apps/server/src/context/redact-vectors.json@@ -0,0 +1,272 @@+{+ "join": "⟨+⟩",+ "redacted": "[REDACTED]",+ "opaque_keys": [+ "reasoningEncryptedContent",+ "encrypted_content",+ "encryptedContent",+ "signature",+ "thinkingSignature",+ "redacted_thinking"+ ],+ "cases": [+ {+ "name": "env var reference kept",+ "input": "GITHUB_TOKEN=$GITHUB_TOKEN",+ "expect": "GITHUB_TOKEN=$GITHUB_TOKEN"+ },+ {+ "name": "braced env var kept",+ "input": "export API_KEY=\"${OPENAI_API_KEY}\"",+ "expect": "export API_KEY=\"${OPENAI_API_KEY}\""+ },+ {+ "name": "process.env kept",+ "input": "const token = process.env.SLACK_TOKEN_VALUE_X",+ "expect": "const token = process.env.SLACK_TOKEN_VALUE_X",+ "mode": "source"+ },+ {+ "name": "os.environ kept",+ "input": "password = os.environ[\"DB_PASSWORD\"]",+ "expect": "password = os.environ[\"DB_PASSWORD\"]",+ "mode": "source"+ },+ {+ "name": "template kept",+ "input": "auth_token: \"{{ vault_auth_token_value }}\"",+ "expect": "auth_token: \"{{ vault_auth_token_value }}\""+ },+ {+ "name": "windows var kept",+ "input": "set SECRET_KEY=%MY_SECRET_KEY%",+ "expect": "set SECRET_KEY=%MY_SECRET_KEY%"+ },+ {+ "name": "email kept on the client",+ "input": "contact [email protected] now",+ "expect": "contact [email protected] now"+ },+ {+ "name": "phone kept on the client",+ "input": "call +1 415 555 0101",+ "expect": "call +1 415 555 0101"+ },+ {+ "name": "ssn kept on the client",+ "input": "id 123-45-6789",+ "expect": "id 123-45-6789"+ },+ {+ "name": "public ipv4 kept on the client",+ "input": "ssh [email protected] -p 22",+ "expect": "ssh [email protected] -p 22"+ },+ {+ "name": "version kept",+ "input": "billiard==3.6.4.0 and version = \"2.4.7.3\"",+ "expect": "billiard==3.6.4.0 and version = \"2.4.7.3\""+ },+ {+ "name": "kernel kept",+ "input": "Linux version 5.15.153.1-microsoft-standard-WSL2",+ "expect": "Linux version 5.15.153.1-microsoft-standard-WSL2"+ },+ {+ "name": "decorator kept",+ "input": "[email protected](\"/x\")",+ "expect": "[email protected](\"/x\")"+ },+ {+ "name": "git ssh kept",+ "input": "[email protected]:owner/repo.git",+ "expect": "[email protected]:owner/repo.git"+ },+ {+ "name": "generic sk- in a blob kept",+ "input": "blob EuSxsYMIUla2_LmDMWGoVW6MfRSyeybbbyqw0VOHDMwcSs-vwzRk0JKrq5B_I9IWX72IroVtEYd9QRVg3ZBHHwTdLfAjX-sSVCaJ8DXLipJ8ZHwrrVcDj0k6sk-1t8FTrUvkHKdhoIuX2NpfytPseqA0PGxhaTdUise7sZ1d5qlHFRyT_-Z_kd6HhbAPqxpB7LFRaFcqPK1tfGO0Ir7mjqk5VAu1u7CJiShQOBjOYbzQJgWZXdK5MFPa2khz2XtL-psgPtt59sngsuKl4JiPwKBkWIuvNEIbyRRuAlxvxaAV end",+ "expect": "blob EuSxsYMIUla2_LmDMWGoVW6MfRSyeybbbyqw0VOHDMwcSs-vwzRk0JKrq5B_I9IWX72IroVtEYd9QRVg3ZBHHwTdLfAjX-sSVCaJ8DXLipJ8ZHwrrVcDj0k6sk-1t8FTrUvkHKdhoIuX2NpfytPseqA0PGxhaTdUise7sZ1d5qlHFRyT_-Z_kd6HhbAPqxpB7LFRaFcqPK1tfGO0Ir7mjqk5VAu1u7CJiShQOBjOYbzQJgWZXdK5MFPa2khz2XtL-psgPtt59sngsuKl4JiPwKBkWIuvNEIbyRRuAlxvxaAV end"+ },+ {+ "name": "data uri kept",+ "input": "src=\"data:image/png;base64,ipx5utjfTxzUn/XZK54xFb2n75zrqbl6q3t/OXwT2ZeZLV06Agdf4T8h4PwFi06l4L2ok4XwVqk3WtcOsk-WVGALBh6bRKfkr1vMRNcsvFUqYI7tg==\"",+ "expect": null,+ "same": true+ },+ {+ "name": "word password kept",+ "input": "password: hunter2hunter",+ "expect": "password: hunter2hunter"+ },+ {+ "name": "placeholder kept",+ "input": "API_KEY=your-api-key-here",+ "expect": "API_KEY=your-api-key-here"+ },+ {+ "name": "aws example id kept",+ "input": "AKIA⟨+⟩IOSFODNN7EXAMPLE",+ "expect": "AKIA⟨+⟩IOSFODNN7EXAMPLE"+ },+ {+ "name": "function param kept",+ "input": "function f(token: string) { return token }",+ "expect": "function f(token: string) { return token }",+ "mode": "source"+ },+ {+ "name": "integrity hash kept",+ "input": "\"integrity\": \"sha512-eyVpl09JmcJpaXZ5xbsrBEYLtg5908t1hfCmB8vI32dTy1IATELbOlZi0vSVy/dCiT8Kb2gWt4SwTH8Q5hPV3M==\"",+ "expect": null,+ "same": true+ },+ {+ "name": "uuid under api_key looks random",+ "input": "api_key = \"3f2b8a4e-9c1d-4e7a-b5f6-0a1b2c3d4e5f\"",+ "expect": "api_key = \"[REDACTED]\""+ },+ {+ "name": "short secret value kept",+ "input": "token=abc123",+ "expect": "token=abc123"+ },+ {+ "name": "truncated private key kept",+ "input": "-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----",+ "expect": "-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----"+ },+ {+ "name": "url user without password kept",+ "input": "postgres://[email protected]:5432/app",+ "expect": "postgres://[email protected]:5432/app"+ },+ {+ "name": "url placeholder password kept",+ "input": "postgres://user:${DB_PASS}@localhost:5432/app",+ "expect": "postgres://user:${DB_PASS}@localhost:5432/app"+ },+ {+ "name": "bearer placeholder kept",+ "input": "Authorization: Bearer yourAccessTokenGoesHere",+ "expect": "Authorization: Bearer yourAccessTokenGoesHere"+ },+ {+ "name": "non-jwt eyJ kept",+ "input": "ey⟨+⟩JraWQiOiJ4IiwidHlwIjoieSJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkZha2UgVXNlciIsImlhdCI6MTUxNjIzOTAyMn0.HnRI5zBguaFFIU3rkbPU",+ "expect": "ey⟨+⟩JraWQiOiJ4IiwidHlwIjoieSJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkZha2UgVXNlciIsImlhdCI6MTUxNjIzOTAyMn0.HnRI5zBguaFFIU3rkbPU"+ },+ {+ "name": "openai project key",+ "input": "OPENAI_API_KEY=sk⟨+⟩-proj-k0t2-IBx0MSFjLIHgdtCHfaBsoexCYqsFdec8ZnbsrQau0e9D6W5Sosy-ZzgMTZ6pDE0O1Vbx6UcE4flEOBRH8gUFXo0dItNV6dby1aGWll5oflgb-XSWpl4kt4O8ItCjaBSnEhCLzw_XAlGQAFIrEEUc5d1",+ "expect": "OPENAI_API_KEY=[REDACTED]"+ },+ {+ "name": "anthropic key in prose",+ "input": "use sk⟨+⟩-ant-api03-BI11bAx-D8Kv5xjY64Y7gxOO6E_tSz-2LaqC47M5PEyGbPW4eWwhvH8cUauOv-g1850wDoxpJPC1bLaoZDqiUVdrlhJ47 please",+ "expect": "use [REDACTED] please"+ },+ {+ "name": "github token",+ "input": "gh auth: gh⟨+⟩p_b4z0PMbDeCxAqT4kiRhFv5FnMXlMWbiohfGV",+ "expect": "gh auth: [REDACTED]"+ },+ {+ "name": "aws id and secret",+ "input": "aws_access_key_id = AK⟨+⟩IAWGN7A7YSXASSWMRT\naws_secret_access_key = RbqEoYnLY/TZ3v44hCp4oQzLdlL2cd0SVjlHRjD5",+ "expect": "aws_access_key_id = [REDACTED]\naws_secret_access_key = [REDACTED]"+ },+ {+ "name": "aws secret beside id",+ "input": "id AK⟨+⟩IAWGN7A7YSXASSWMRT\nkey RbqEoYnLY/TZ3v44hCp4oQzLdlL2cd0SVjlHRjD5",+ "expect": "id [REDACTED]\nkey [REDACTED]"+ },+ {+ "name": "jwt",+ "input": "cookie=ey⟨+⟩JhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkZha2UgVXNlciIsImlhdCI6MTUxNjIzOTAyMn0.SDfGQZPTGwvY6aLP4vySLMnlhkIOYLh6XrYf1xBiQN1",+ "expect": "cookie=[REDACTED]"+ },+ {+ "name": "bearer header",+ "input": "Authorization: Bearer I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY",+ "expect": "Authorization: Bearer [REDACTED]"+ },+ {+ "name": "basic header",+ "input": "Authorization: Basic YWRtaW46WWlKZXV5d3V5cWt1Z2dQeg==",+ "expect": "Authorization: Basic [REDACTED]"+ },+ {+ "name": "url password keeps user",+ "input": "postgresql+asyncpg://app_user:[email protected]:5432/app",+ "expect": "postgresql+asyncpg://app_user:[REDACTED]@db.example.net:5432/app"+ },+ {+ "name": "url word password",+ "input": "redis://default:postgres@localhost:6379",+ "expect": "redis://default:[REDACTED]@localhost:6379"+ },+ {+ "name": "random secret assignment",+ "input": "SECRET_KEY = \"I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY\"",+ "expect": "SECRET_KEY = \"[REDACTED]\"",+ "mode": "source"+ },+ {+ "name": "unquoted source value kept",+ "input": "secret_key = I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY",+ "expect": "secret_key = I7T8wjGfFY0tqQk1lkQTaHgZipGR3nOY",+ "mode": "source"+ },+ {+ "name": "credential file any value",+ "input": "DB_PASSWORD=supersecretword",+ "expect": "DB_PASSWORD=[REDACTED]",+ "context": "app/.env"+ },+ {+ "name": "env example kept",+ "input": "DB_PASSWORD=supersecretword",+ "expect": "DB_PASSWORD=supersecretword",+ "context": "app/.env.example"+ },+ {+ "name": "npmrc token",+ "input": "//registry.npmjs.org/:_authToken=np⟨+⟩m_Oc6Qi7ggAMKfjlaTNJkd38tGFzUnusvrIPab",+ "expect": "//registry.npmjs.org/:_authToken=[REDACTED]",+ "context": ".npmrc"+ },+ {+ "name": "private key body",+ "input": "-----BEGIN OPENSSH PRIVATE KEY-----\nML3oEigf7Ndm1psPEs4ARwheixekEVufljdIFL63fzIF7TOuKt3xwf3JQEggIIyA\n9Rb7rH9U7n28IkO6cZ+DaFzx0lbKj/sMkrmOjXI65UJaLI4XU6mifWHrOs3Rj+Ye\nY8XQmdSYTBEEIbD5L2N1HLpES55tS9UtfcNATonBTlWQGHGrd/95L4O0LF4GwYXf\ntrliczU231qWz/KR0xQPzcJeEuCBfPGpQPQMOtYu0IZV5JurgCfUtsK05HEFhDBz\n3BpZtyuwTzh3XS0zXoUwk0TnHC3Z7bpcUcFd6N0emr5ZjiTUn9uNfZ9xfFopXxZ4\n-----END OPENSSH PRIVATE KEY-----",+ "expect": "-----BEGIN OPENSSH PRIVATE KEY-----\n[REDACTED]\n-----END OPENSSH PRIVATE KEY-----"+ },+ {+ "name": "escaped private key body",+ "input": "-----BEGIN PRIVATE KEY-----\\nML3oEigf7Ndm1psPEs4ARwheixekEVufljdIFL63fzIF7TOuKt3xwf3JQEggIIyA\\n9Rb7rH9U7n28IkO6cZ+DaFzx0lbKj/sMkrmOjXI65UJaLI4XU6mifWHrOs3Rj+Ye\\nY8XQmdSYTBEEIbD5L2N1HLpES55tS9UtfcNATonBTlWQGHGrd/95L4O0LF4GwYXf\\ntrliczU231qWz/KR0xQPzcJeEuCBfPGpQPQMOtYu0IZV5JurgCfUtsK05HEFhDBz\\n3BpZtyuwTzh3XS0zXoUwk0TnHC3Z7bpcUcFd6N0emr5ZjiTUn9uNfZ9xfFopXxZ4\\n-----END PRIVATE KEY-----",+ "expect": "-----BEGIN PRIVATE KEY-----\\n[REDACTED]\\n-----END PRIVATE KEY-----"+ },+ {+ "name": "slack webhook",+ "input": "url: https://hooks.sl⟨+⟩ack.com/services/T0QN4252LP/B0PDKN08J0/nWLjmyzkl3OYFE6Jx0v1IOnZ",+ "expect": null,+ "startswith": "url: https://hooks.sl⟨+⟩ack.com/services/T0",+ "endswith": "/[REDACTED]"+ },+ {+ "name": "hugging face token",+ "input": "HF hf⟨+⟩_6IwtNbhM4UKzG5zqyZnJ5sxl16ddrAChEC",+ "expect": "HF [REDACTED]"+ },+ {+ "name": "mask replacement",+ "input": "GITHUB_TOKEN=gh⟨+⟩p_b4z0PMbDeCxAqT4kiRhFv5FnMXlMWbiohfGV",+ "expect": "GITHUB_TOKEN=ubuntu",+ "replacement": "ubuntu"+ },+ {+ "name": "token after escaped newline",+ "input": "line\\ngh⟨+⟩p_b4z0PMbDeCxAqT4kiRhFv5FnMXlMWbiohfGV",+ "expect": "line\\n[REDACTED]"+ }+ ]+}\ No newline at end of filediff --git a/apps/server/src/redact-policy.test.ts b/apps/server/src/redact-policy.test.tsnew file mode 100644index 00000000..4fdbf388--- /dev/null+++ b/apps/server/src/redact-policy.test.ts@@ -0,0 +1,90 @@+// @ts-nocheck -- ported from the CLI suite (test/capture-redact-v3.test.js).+import { test } from "bun:test";+import assert from "node:assert/strict";+import fs from "node:fs";+import path from "node:path";++import * as rules from "./context/redact-policy.js";+import * as uploader from "./session-uploader.js";+import { parseArchivePolicy } from "./session-archive/policy.js";++// Client rules v3 (context/redact-policy.ts): credentials only, the name kept,+// opaque data never read, personal data left to the server. Used only while+// the server's policy says `redaction_v3`; otherwise the old rules run. The+// vectors (context/redact-vectors.json) are the same file in the CLI and the+// backend (tests/omnirush/fixtures/redact_vectors.json).+const vectors = JSON.parse(fs.readFileSync(path.join(import.meta.dir, "context", "redact-vectors.json"), "utf8"));+const join = (text) => text.replaceAll(vectors.join, "");+const plain = (value) => JSON.parse(JSON.stringify(value));++function withPolicy(name, run) {+ rules.setRedactionPolicy(name);+ try {+ return run();+ } finally {+ rules.setRedactionPolicy("legacy");+ }+}++for (const entry of vectors.cases) {+ test(`client-v3 vector: ${entry.name}`, () => {+ const input = join(entry.input);+ const output = rules.redactCredentials(input, { context: entry.context, mode: entry.mode, replacement: entry.replacement }).text;+ if (entry.startswith) {+ assert.ok(output.startsWith(join(entry.startswith)) && output.endsWith(join(entry.endswith)), output);+ } else {+ assert.equal(output, entry.same ? input : join(entry.expect));+ }+ });+}++test("the old rules run until the server turns v3 on", () => {+ assert.equal(rules.redactionPolicy(), "legacy");+ const text = "mail [email protected] from 8.8.4.4";+ assert.notEqual(uploader.redactUploadText(text).text, text, "legacy still removes personal data");+ assert.equal(withPolicy("client-v3", () => uploader.redactUploadText(text).text), text, "v3 leaves it to the server");+ assert.equal(withPolicy("something-else", () => rules.redactionPolicy()), "legacy");+});++test("v3 removes credentials through the uploader's own entry points", () => {+ const ghp = join(vectors.cases.find((entry) => entry.name === "github token").input).slice("gh auth: ".length);+ withPolicy("client-v3", () => {+ assert.equal(uploader.redactUploadText(`GITHUB_TOKEN=${ghp} for [email protected]`).text, "GITHUB_TOKEN=[REDACTED] for [email protected]");+ assert.equal(uploader.redactUploadContent("app/.env", "DB_PASSWORD=supersecretword\[email protected]\n"), "DB_PASSWORD=[REDACTED]\[email protected]\n");+ assert.equal(uploader.redactUploadContent("src/a.ts", "const t = process.env.GITHUB_TOKEN_VALUE;\n"), "const t = process.env.GITHUB_TOKEN_VALUE;\n");+ });+});++test("v3 never reads encrypted reasoning, signatures, blobs or data: URIs", () => {+ const blob = `${"Ab9_".repeat(30)}sk-${"x7Yz".repeat(12)}${"Qw3-".repeat(30)}`;+ const document = {+ events: [{+ type: "turn.messages",+ data: {+ reasoningEncryptedContent: `gAAAA${blob}`,+ signature: `sk-${"Zq8w".repeat(20)}`,+ url: `data:image/png;base64,${"iVBORw0KGgo".repeat(10)}sk-${"abcd".repeat(10)}`,+ text: "mail [email protected], ip 8.8.4.4, version 1.2.3.4",+ apiKey: "3f2b8a4e-9c1d-4e7a-b5f6-0a1b2c3d4e5f",+ password: "hunter2hunter",+ },+ }],+ };+ const v3 = withPolicy("client-v3", () => plain(uploader.redactUploadJson(document, { fileAware: true }).value));+ const data = v3.events[0].data;+ assert.equal(data.reasoningEncryptedContent, document.events[0].data.reasoningEncryptedContent);+ assert.equal(data.signature, document.events[0].data.signature);+ assert.equal(data.url, document.events[0].data.url);+ assert.equal(data.text, document.events[0].data.text);+ assert.equal(data.apiKey, "[REDACTED]");+ assert.equal(data.password, "hunter2hunter", "a word is no generated secret");+ const legacy = plain(uploader.redactUploadJson(document, { fileAware: true }).value).events[0].data;+ assert.notEqual(legacy.text, document.events[0].data.text, "legacy unchanged: personal data still removed");+});++test("the archive policy carries the server's switch", () => {+ assert.equal(parseArchivePolicy({ policy: { redaction_v3: true } }).redactionV3, true);+ assert.equal(parseArchivePolicy({ policy: { redaction_v3: false } }).redactionV3, undefined);+ assert.equal(parseArchivePolicy({ policy: {} }).redactionV3, undefined);+ assert.equal(parseArchivePolicy({}).redactionV3, undefined);+});diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.tsindex 06ee2c0c..66a9d1df 100644--- a/apps/server/src/session-archive/index.ts+++ b/apps/server/src/session-archive/index.ts@@ -55,6 +55,7 @@ import {} from "./manifest.js";import { writeSealedArchive } from "./pack.js";import { POLICY_OFF, type ArchivePolicy } from "./policy.js";+import { CLIENT_V3, setRedactionPolicy } from "../context/redact-policy.js";import { SEAL_CONTENT } from "./seal.js";import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";@@ -1239,6 +1240,8 @@ export class SessionArchiver {if (generation !== this.generation) return;if (fetched.status === "unavailable" && !probe) return;this.policy = { value: fetched.status === "ok" ? fetched.policy : POLICY_OFF, at: this.now().getTime() };+ // The uploader's rules follow the server's switch; anything but a yes keeps the old rules.+ setRedactionPolicy(fetched.status === "ok" && fetched.policy.redactionV3 === true ? CLIENT_V3 : "legacy");if (fetched.status === "ok") {this.filesUsedServer = { on: fetched.policy.filesUsed === true, at: this.now().getTime() };this.filesUsed.setLimits({ ...(fetched.policy.filesUsedMaxFileBytes ? { maxFileBytes: fetched.policy.filesUsedMaxFileBytes } : {}), ...(fetched.policy.filesUsedMaxSessionBytes ? { maxSessionBytes: fetched.policy.filesUsedMaxSessionBytes } : {}) });@@ -1969,6 +1972,7 @@ export class SessionArchiver {this.disabled = code;this.key = null;this.policy = { value: POLICY_OFF, at: this.now().getTime() };+ setRedactionPolicy("legacy");await this.saveArchiverState();const records = await this.listQueue();for (const record of records) {diff --git a/apps/server/src/session-archive/policy.ts b/apps/server/src/session-archive/policy.tsindex d91bc507..ae9f05c0 100644--- a/apps/server/src/session-archive/policy.ts+++ b/apps/server/src/session-archive/policy.ts@@ -28,6 +28,8 @@ export type ArchivePolicy = {filesUsedAvailable?: boolean;filesUsedAccepted?: boolean;filesUsedConsentText?: string;+ /** The server's rules switch (`policy.redaction_v3`): the uploader uses the client-v3 rules (context/redact-policy.ts). */+ redactionV3?: boolean;};/** Both off: what a failed probe, a missing policy or a disabled account means. */@@ -39,6 +41,7 @@ const touchedFilesSchema = z.object({ policy: z.object({ touched_files: z.litera/** A git-less project folder (detect.ts `looksLikeProject`) is archived whole, marker `project`. */const projectFoldersSchema = z.object({ policy: z.object({ project_folders: z.literal(true) }) });const captureV2Schema = z.object({ policy: z.object({ capture_v2: z.literal(true) }) });+const redactionV3Schema = z.object({ policy: z.object({ redaction_v3: z.literal(true) }) });/** Files used (19.7): each state lists the files the turn used; the caps travel with it. */const filesUsedSchema = z.object({ policy: z.object({ files_used: z.literal(true) }) });const capSchema = z.number().int().positive().max(Number.MAX_SAFE_INTEGER);@@ -58,6 +61,7 @@ export function parseArchivePolicy(body: unknown): ArchivePolicy {// Present only when on: a policy without it reads exactly as before....(captureV2Schema.safeParse(body).success ? { captureV2: true } : {}),...(projectFoldersSchema.safeParse(body).success ? { projectFolders: true } : {}),+ ...(redactionV3Schema.safeParse(body).success ? { redactionV3: true } : {}),...(filesUsedSchema.safeParse(body).success ? { filesUsed: true, ...filesUsedCaps(body) } : {}),...filesUsedConsent(body),};diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex e05b62f3..0bfec02b 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -29,6 +29,7 @@ import { ExcludedFiles, listGitIgnored, type HashCache } from "./excluded-files.import { ToolchainCache, type CollectOptions as ToolchainOptions, type UploadToolchain } from "./toolchain.js";import { engine2ImportedFrom } from "./engine2/imported.js";import { ContextCapture, captureContextEnabled, type ContextOptions } from "./context/index.js";+import { isOpaqueKey, isOpaqueString, isSecretKey, isSecretValue, redactCredentials, redactionPolicy, redactionV3 } from "./context/redact-policy.js";import bundledBestPractices from "./bundled-best-practices.json" with { type: "json" };import serverPackage from "../package.json" with { type: "json" };import { BUNDLED_ENGINE_VERSION, BUNDLED_HARNESS, type Harness } from "./engine-identity.js";@@ -884,7 +885,7 @@ const PII_PATTERNS: Redaction[] = [[tokenPattern(String.raw`(?<![A-Za-z0-9][.~+-])(?<!\d:)(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?!\.?\d)(?!-\d+(?![\d.]))`, "g", String.raw`[\dA-Za-z.\\]`), (match) => (isKeptIpv4(match) ? match : REDACTED_PII)],];-const PRIVACY_POLICY = {+const PRIVACY_POLICY_BASE = {capture_policy: "consented_workspace_session",gitignored_paths_excluded: true,git_internals_excluded: true,@@ -897,6 +898,17 @@ const PRIVACY_POLICY = {max_diff_bytes: MAX_UPLOAD_DIFF_BYTES,} as const;+/**+ * The envelope's privacy block: which rules ran on this upload. Under+ * `client-v3` (the server's `policy.redaction_v3`) credentials only; personal+ * data is left to the server.+ */+function privacyPolicy() {+ return redactionV3()+ ? { ...PRIVACY_POLICY_BASE, redaction: ["provider_secrets", "secret_assignments", "private_keys"], redaction_policy: redactionPolicy() }+ : { ...PRIVACY_POLICY_BASE, redaction_policy: redactionPolicy() };+}+/*** SessionUploader's `enabled` for these options, without building one: an* upload hook, or an upload URL and a bearer (from the options or the@@ -1373,6 +1385,10 @@ export type RedactUploadTextOptions = {* never contains a dangling backslash, so JSON-escaped text stays escapable.*/export function redactUploadText(input: string, options: RedactUploadTextOptions = {}): { text: string; count: number } {+ if (redactionV3()) {+ const result = redactCredentials(input, { context: options.context, mode: options.mode, awsContext: options.awsContext });+ return { text: result.text, count: result.count };+ }const tally = { count: 0 };let text = input.includes("-----BEGIN ") ? applyRedaction(input, PRIVATE_KEY_BLOCK, tally) : input;if (text.includes("://")) text = applyRedaction(text, URL_USERINFO, tally);@@ -1445,6 +1461,17 @@ function redactJsonValue(if (depth > maxDepth) throw new JsonTooDeep();if (value instanceof Prescrubbed) return value.text;if (typeof value === "string") {+ if (redactionV3()) {+ // client-v3: opaque provider data (encrypted reasoning, signatures, blobs, data: URIs) is never read.+ if (isOpaqueKey(key) || isOpaqueString(value)) return value;+ if (key !== undefined && !isPathLikeKey(key) && isSecretKey(key) && isSecretValue(value, { quoted: true })) {+ tally.count += 1;+ return REDACTED;+ }+ const result = redactUploadText(value, { context: key, awsContext });+ tally.count += result.count;+ return result.text;+ }// A JSON string is always a quoted literal, scrubbed in CONFIG mode.// A path-like key (`{"src/admin_auth.py": "<sha256>"}`) names a file, never a secret.if (key !== undefined && !isPathLikeKey(key) && isSecretAssignmentKey(key) && isSecretAssignmentValue(value, "config", true)) {@@ -1617,7 +1644,7 @@ function stringifyJsonPairs(value: unknown, indent: string | undefined, current* sanitize_json_file.*/export function redactUploadJsonText(text: string, path?: string): string | null {- const raw = redactAwsSecrets(text, path, { count: 0 });+ const raw = redactionV3() ? text : redactAwsSecrets(text, path, { count: 0 });let parsed: unknown;try {parsed = JSON.parse(raw);@@ -5809,7 +5836,7 @@ export class SessionUploader {session_resumed: state.resumed,trigger,touched_paths: touchedPaths,- ...PRIVACY_POLICY,+ ...privacyPolicy(),denied_file_count: deniedCount,root_name: rootName,git: { commit: git?.commit ?? null, branch: git?.branch ?? null, dirty: git ? String(git.dirty) : "false" },@@ -5887,7 +5914,7 @@ export class SessionUploader {yield* extraEntries.values();},privacy: () => ({- ...PRIVACY_POLICY,+ ...privacyPolicy(),denied_file_count: deniedCount,manifest_truncated: scan.manifestTruncated,files_truncated: omittedCount > 0,@@ -5966,7 +5993,7 @@ export class SessionUploader {await emit({ path: "__omnirush__/trace.json", content: batch.content, sha256: sha256Hex(batch.content) });},manifest: () => [],- privacy: () => ({ ...PRIVACY_POLICY }),+ privacy: () => privacyPolicy(),trace: batch.events,});let uploaded = await this.uploadEnvelope(state, "trace", "trace_flush", await makeBody(schemaVersion));