A desktop coding agent with free access to frontier models.
TypeScript
fix: catch more key shapes
ogsamrat430d66e9b070
diff --git a/apps/server/src/context/PARITY.sha256 b/apps/server/src/context/PARITY.sha256index 1fe283fe..abd276a2 100644--- a/apps/server/src/context/PARITY.sha256+++ b/apps/server/src/context/PARITY.sha256@@ -9,8 +9,8 @@ d772119f2785b6926cd765a44589c171542d5e79b3677bfa076851775d794480 outside.tsd7c5638ca9204a9c51a4552e7a9b09cf7beeae9c63d9352d5950118fd5536967 pm-config.ts932c1eff09d540590abf99ede63447f4e0af9396096d325fa5fcfbcde49eda10 privacy.ts0e04845a2d0ad2136c01a61d3ed1193b62ef99f5b82e1f555b7c97bb6dc18773 processes.ts-5061a781045788dc72e04dec2ef7abab936677fbf58ad1ee0329336df62dc2ce redact-policy.ts-2fdf90c962aa42e48f4dd0a53057f1778ff7f0996a6a0deea3eec6a42c2063b2 redact-vectors.json+d7cb7384da5a3b2c7e82a45f05f1b9258bb84a7f3510de8eb6f1da478b4157f0 redact-policy.ts+ecb628437615d261d3f7f1f02bc05ce000b16b1040c5804e8ffaf9d7ed9fe471 redact-vectors.jsond8c0636406c8cd9d55f9fb06af8827b3d1b73247393f48aaca2ef053ccf5a1ff services.ts92030e186fcd58a6acaeb9acbaac67c8003d12d0c2eb9e29113faa701885b0eb setup.tsd51d434283f757f0ed45bb6996a74fb5bf9e1d56f34146e99df3c392cde5ea19 shell-aliases.tsdiff --git a/apps/server/src/context/redact-policy.ts b/apps/server/src/context/redact-policy.tsindex 4dfac67a..f45b12b8 100644--- a/apps/server/src/context/redact-policy.ts+++ b/apps/server/src/context/redact-policy.ts@@ -114,6 +114,33 @@ export function looksRandom(value: string): boolean {return shannonEntropy(value) >= 3.0;}+function charClass(char: string): string | undefined {+ return /[0-9]/.test(char) ? "d" : /[A-Z]/.test(char) ? "u" : /[a-z]/.test(char) ? "l" : undefined;+}++/**+ * An unquoted value with a name's shape that is still generated+ * (`client-secret=Xk7pQ2vR...` in a `.properties` file): 16+ characters,+ * no dot, two classes, 3.5+ bits per character, and the class changes+ * between neighbours (digit, upper, lower; a capital starting a word does+ * not count) at 30%+ of them. Names change class only between words.+ */+export function looksRandomName(value: string): boolean {+ if (value.length < 16 || value.includes(".")) return false;+ const classes = Number(/[a-z]/.test(value)) + Number(/[A-Z]/.test(value)) + Number(/[0-9]/.test(value));+ if (classes < 2 || shannonEntropy(value) < 3.5) return false;+ let pairs = 0;+ let changes = 0;+ for (let index = 1; index < value.length; index += 1) {+ const before = charClass(value[index - 1]!);+ const after = charClass(value[index]!);+ if (before === undefined || after === undefined) continue;+ pairs += 1;+ if (before !== after && !(before === "u" && after === "l")) changes += 1;+ }+ return pairs > 0 && changes >= 0.3 * pairs;+}+/*** Credential files: any non-placeholder value of a secret-named key there is* a credential. `.env*` except examples and templates, `.npmrc`, `.pypirc`,@@ -137,13 +164,16 @@ const ASSIGNMENT_BODY = String.raw`(\\?["']?)((?=[\w.-]{0,63}?(?:${KEYWORD}))[A-+ String.raw`(?:"((?:[^"${WS}\\]|\\[^${WS}]){8,})"`+ String.raw`|'((?:[^'${WS}\\]|\\[^${WS}]){8,})'`+ String.raw`|\\"((?:[^"${WS}\\]|\\[^"${WS}]){8,})\\"`- + String.raw`|((?:[^${WS}"',;&\\]|\\[^${WS}]){8,}))`;+ // An unquoted value ends at an escaped line break or tab (`KEY=value\n+NEXT=...`).+ + String.raw`|((?:[^${WS}"',;&\\]|\\[^${WS}nrt]){8,}))`;const CONFIG_ASSIGNMENT = new RegExp(String.raw`(?:(?<![\w.\\-])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");-const SOURCE_ASSIGNMENT = new RegExp(String.raw`(?:(?<![^${WS}\x80-\uffff{,("'\x60+#])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");+// In code the name follows a space, an opening bracket, a quote, `+`, `#` or+// a sigil (`$password = '...'` in PHP, Perl, PowerShell).+const SOURCE_ASSIGNMENT = new RegExp(String.raw`(?:(?<![^${WS}\x80-\uffff{,("'\x60+#$])|(?<=\\[nrt]))` + ASSIGNMENT_BODY, "gi");const SECRET_SEGMENTS = new Set(["secret", "secrets", "password", "passwords", "passwd", "pwd", "token", "credential", "credentials",- "auth", "authorization", "authtoken", "authkey", "apikey",+ "auth", "authorization", "authtoken", "authkey", "apikey", "pgpassword",]);const SECRET_PAIRS: Array<[string, string]> = [["api", "key"], ["access", "key"], ["secret", "key"], ["private", "key"], ["client", "key"], ["client", "secret"],@@ -232,7 +262,7 @@ export function isSecretValue(value: string, context: ValueContext = {}): booleaif (value.includes("(")) return false;if (context.credentialFile) return true;if (context.mode === "source" && !context.quoted) return false;- if (!context.quoted && IDENTIFIER_VALUE.test(value)) return false;+ if (!context.quoted && IDENTIFIER_VALUE.test(value)) return looksRandomName(value);return looksRandom(value);}@@ -349,9 +379,21 @@ function redactUrlPasswords(text: string, hidden: string, tally: Tally, spans: Aconst MAX_ASSIGNMENT_DEPTH = 4;+/** Shell scripts: there an unquoted `NAME=value` is a literal, never an expression. */+export function isShellFile(path: string | undefined): boolean {+ return path !== undefined && /\.(?:sh|bash|zsh|ksh)$/i.test(path);+}++// A query parameter that carries a one-time credential although its name ends+// in a word the key rules skip (`?token_hash=` in Supabase sign-in links).+const QUERY_SECRET_KEYS = new Set(["token hash"]);++type AssignmentContext = { mode: "source" | "config"; credentialFile: boolean; shellFile: boolean };+function redactAssignments(- text: string, hidden: string, tally: Tally, mode: "source" | "config", credentialFile: boolean, spans: Array<[number, number]>, depth = 0,+ text: string, hidden: string, tally: Tally, context: AssignmentContext, spans: Array<[number, number]>, depth = 0,): string {+ const { mode, credentialFile, shellFile } = context;// A fresh expression per call: the pass recurses into values while the outer replace is still running.const pattern = new RegExp(mode === "source" ? SOURCE_ASSIGNMENT : CONFIG_ASSIGNMENT);return text.replace(pattern, (match: string, _quote: string, key: string, doubleQuoted: string | undefined, singleQuoted: string | undefined, escapedQuoted: string | undefined, bare: string | undefined, offset: number) => {@@ -359,32 +401,58 @@ function redactAssignments(const value = doubleQuoted ?? singleQuoted ?? escapedQuoted ?? bare ?? "";const quote = doubleQuoted !== undefined ? '"' : singleQuoted !== undefined ? "'" : escapedQuoted !== undefined ? '\\"' : "";const prefix = match.slice(0, match.length - value.length - quote.length * 2);- if (isSecretKey(key) && isSecretValue(value, { mode, quoted: bare === undefined, credentialFile })) {+ // A URL query value (`?name=value&`) and a shell `NAME=value` are literals like a quoted one.+ const query = bare !== undefined && offset > 0 && (text[offset - 1] === "?" || text[offset - 1] === "&");+ const shellLiteral = shellFile && bare !== undefined && prefix === `${key}=` && !/[$`]/.test(value);+ const secretKey = isSecretKey(key) || (query && QUERY_SECRET_KEYS.has(keySegments(key).join(" ")));+ if (secretKey && isSecretValue(value, { mode: query || shellLiteral ? "config" : mode, quoted: bare === undefined || query || shellLiteral, credentialFile })) {note(tally, "secret_assignment");return `${prefix}${quote}${hidden}${quote}`;}if (depth >= MAX_ASSIGNMENT_DEPTH) return match;- return `${prefix}${quote}${redactAssignments(value, hidden, tally, mode, credentialFile, [], depth + 1)}${quote}`;+ return `${prefix}${quote}${redactAssignments(value, hidden, tally, context, [], depth + 1)}${quote}`;});}const ASSIGNMENT_GATE = /secret|passw|pwd|token|credential|auth|key/i;// A label on its own line and the value alone on the next one (notes,-// READMEs, `.txt` files): `API_KEY:` then `abc123...`. The whole next line-// must be one token; it is judged like any other value.-const NEXT_LINE_VALUE = /(^|\n)([^\S\n]*[-*#>]*[^\S\n]*[*_`"']*)([A-Za-z_][\w.-]{0,63})([*_`"']*[^\S\n]*[:=][^\S\n]*[*_`"']*\r?\n[^\S\n]*)([`"']?)([^\s`"']{12,})\5(?=[^\S\n]*(?:\r?\n|$))/g;+// READMEs, `.txt` and `.properties` files): `API_KEY:` then `abc123...`, one+// blank line between them allowed. The whole value line must be one token;+// it is judged like any other value. Line by line, so a label right under+// another empty label is still read.+// Python's `\s` without the line break, so both ports split values alike.+const HS = String.raw`\t\v\f\r \x1c-\x1f\x85\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000`;+const LABEL_LINE = new RegExp(String.raw`^([${HS}]*[-*#>]*[${HS}]*[*_\x60"']*)([A-Za-z_][\w.-]{0,63})([*_\x60"']*[${HS}]*[:=][${HS}]*[*_\x60"']*)$`);+const VALUE_LINE = new RegExp(String.raw`^([${HS}]*)([\x60"']?)([^${WS}\x60"']{12,})\2([${HS}]*)$`);+const BLANK_LINE = new RegExp(String.raw`^[${HS}]*$`);function redactNextLineValues(text: string, hidden: string, tally: Tally, credentialFile: boolean, spans: Array<[number, number]>): string {- NEXT_LINE_VALUE.lastIndex = 0;- return text.replace(NEXT_LINE_VALUE, (match: string, start: string, lead: string, key: string, sep: string, quote: string, value: string, offset: number) => {- if (insideOpaque(spans, offset, offset + match.length) || !isSecretKey(key)) return match;+ const lines = text.split("\n");+ const starts: number[] = [];+ let at = 0;+ for (const line of lines) {+ starts.push(at);+ at += line.length + 1;+ }+ let changed = false;+ for (let index = 0; index < lines.length - 1; index += 1) {+ const label = LABEL_LINE.exec(lines[index]!);+ if (!label || !isSecretKey(label[2]!)) continue;+ let target = index + 1;+ if (BLANK_LINE.test(lines[target]!) && target + 1 < lines.length) target += 1;+ const found = VALUE_LINE.exec(lines[target]!);+ if (!found) continue;+ const value = found[3]!;// The next line is an assignment of its own (`API_KEY=` left empty above `OTHER=value`).- if (/^[A-Za-z_][\w.-]*[:=]/.test(value)) return match;- if (!isSecretValue(value, { mode: "config", quoted: true, credentialFile })) return match;+ if (/^[A-Za-z_][\w.-]*[:=]/.test(value)) continue;+ if (insideOpaque(spans, starts[index]!, starts[target]! + lines[target]!.length)) continue;+ if (!isSecretValue(value, { mode: "config", quoted: true, credentialFile })) continue;note(tally, "secret_assignment");- return `${start}${lead}${key}${sep}${quote}${hidden}${quote}`;- });+ lines[target] = `${found[1]!}${found[2]!}${hidden}${found[2]!}${found[4]!}`;+ changed = true;+ }+ return changed ? lines.join("\n") : text;}const AWS_ID = /(?:AKIA|ASIA)[0-9A-Z]{16}/;@@ -467,7 +535,7 @@ export function redactCredentials(input: string, options: CredentialOptions = {}let text = input.includes("-----BEGIN ") ? redactPrivateKeys(input, hidden, tally) : input;if (text.includes("://")) text = redactUrlPasswords(text, hidden, tally, opaqueSpans(text));if (ASSIGNMENT_GATE.test(text)) {- text = redactAssignments(text, hidden, tally, options.mode ?? "config", credentialFile, opaqueSpans(text));+ text = redactAssignments(text, hidden, tally, { mode: options.mode ?? "config", credentialFile, shellFile: isShellFile(options.context) }, opaqueSpans(text));if (text.includes("\n") && options.mode !== "source") text = redactNextLineValues(text, hidden, tally, credentialFile, opaqueSpans(text));}const nearAws = options.awsContext === true || (options.context !== undefined && AWS_CONTEXT.test(options.context));diff --git a/apps/server/src/context/redact-vectors.json b/apps/server/src/context/redact-vectors.jsonindex d8a1e646..c9162e5a 100644--- a/apps/server/src/context/redact-vectors.json+++ b/apps/server/src/context/redact-vectors.json@@ -345,6 +345,131 @@"name": "escaped empty key above another assignment kept","input": "SOME_SECRET_KEY=\\nSERVICE_NAME=svc-alpha-77beta","expect": "SOME_SECRET_KEY=\\nSERVICE_NAME=svc-alpha-77beta"+ },+ {+ "name": "label then value on the next line under another label",+ "input": "server.port=\r\nspring.datasource.app-db.password=\r\nq8⟨+⟩zr2lmt5vxp9k3d-wq7n4b6h1j0f2g8sy5c\r\nmax-retries=10\r\n",+ "expect": "server.port=\r\nspring.datasource.app-db.password=\r\n[REDACTED]\r\nmax-retries=10\r\n",+ "context": "application.properties"+ },+ {+ "name": "label, blank line, then value",+ "input": "OPENAI_API_KEY=\n\nXK⟨+⟩7PQ2VRT9MZ-LW3NC8BD4HJ6FS\n\nANTHROPIC_API_KEY=\n",+ "expect": "OPENAI_API_KEY=\n\n[REDACTED]\n\nANTHROPIC_API_KEY=\n",+ "context": "keys.txt"+ },+ {+ "name": "label= then a long value on the next line",+ "input": "notes-for-me-v2-a1\nOPENAI_COMPATIBLE_API_KEY=\nXk⟨+⟩7pQ2vR-t9mZ_Lw3Nc8Bd4Hj6Fs1Gq5Yt0Ue2Ia7Ob3Pc9Rd4Se6Tf\nOTHER_THING_NAME=x\n",+ "expect": "notes-for-me-v2-a1\nOPENAI_COMPATIBLE_API_KEY=\n[REDACTED]\nOTHER_THING_NAME=x\n",+ "context": "README.md"+ },+ {+ "name": "empty labels and a comment after a blank line kept",+ "input": "OPENAI_API_KEY=\n\nANTHROPIC_API_KEY=\n\n# filled in by CI\n",+ "expect": null,+ "same": true,+ "context": ".env.example"+ },+ {+ "name": "label, blank line, then prose kept",+ "input": "password:\n\nask the admin for it\n",+ "expect": null,+ "same": true+ },+ {+ "name": "unquoted value ends at an escaped line break",+ "input": "\"diff\": \"--- a/.env\\n+++ b/.env\\n@@ -1 +1 @@\\n-x\\n+NEXTA_CLIENT_SECRET=a8⟨+⟩f3k2m9q7w4e1r6t5y0u3i8o2p7s4d6\\n+NEXTA_CLIENT_URL=https://example.com/cb\"",+ "expect": "\"diff\": \"--- a/.env\\n+++ b/.env\\n@@ -1 +1 @@\\n-x\\n+NEXTA_CLIENT_SECRET=[REDACTED]\\n+NEXTA_CLIENT_URL=https://example.com/cb\"",+ "context": "NOTES.md"+ },+ {+ "name": "generated value with a name shape, unquoted",+ "input": "oauth2.client-id=web\nspring.security.oauth2.client-secret=Xk⟨+⟩7pQ2vRt9mZLw3Nc8Bd4Hj6Fs1Gq5Yt0Ue2Ia7\n",+ "expect": "oauth2.client-id=web\nspring.security.oauth2.client-secret=[REDACTED]\n",+ "context": "application.properties"+ },+ {+ "name": "lowercase generated value, unquoted",+ "input": "API_KEY=ak⟨+⟩_7f3k2m9q7w4e1r6t5y0u3i8o2p7s4d6g9h1j\n",+ "expect": "API_KEY=[REDACTED]\n",+ "context": ".env.example"+ },+ {+ "name": "camelCase names as values kept",+ "input": "password = userPasswordHash\ntoken: accessTokenExpiresAt\nAPI_KEY=myApiToken2024Prod\nsecret=getAuthTokenForUser123\n",+ "expect": null,+ "same": true,+ "context": "notes.md"+ },+ {+ "name": "uuid with a name-like key kept",+ "input": "app.api-key-id=3f2b8a4e-9c1d-4e7a-b5f6-0a1b2c3d4e5f\n",+ "expect": null,+ "same": true,+ "context": "a.properties"+ },+ {+ "name": "shell script: unquoted password ending in !",+ "input": "#!/bin/bash\nDB_PASSWORD=Xk⟨+⟩7pQ2vRt9mZ!\nexport ADMIN_PASSWORD=Hn⟨+⟩4rT8wq2Lzp!\n",+ "expect": "#!/bin/bash\nDB_PASSWORD=[REDACTED]\nexport ADMIN_PASSWORD=[REDACTED]\n",+ "context": "deploy.sh",+ "mode": "source"+ },+ {+ "name": "shell script: PGPASSWORD before a command",+ "input": "PGPASSWORD=Su⟨+⟩mmer2024!x psql -h db\n",+ "expect": "PGPASSWORD=[REDACTED] psql -h db\n",+ "context": "run.sh",+ "mode": "source"+ },+ {+ "name": "shell script: references kept",+ "input": "DB_PASSWORD=$(cat /run/secrets/db)\nAPI_TOKEN=${API_TOKEN:-}\nexport SECRET_KEY=\"$1\"\nTOKEN=`gen-token`\n",+ "expect": null,+ "same": true,+ "context": "run.sh",+ "mode": "source"+ },+ {+ "name": "shell script: spaced or non-shell file unquoted kept",+ "input": "password = generatedPass2024xyz\n",+ "expect": null,+ "same": true,+ "context": "app.py",+ "mode": "source"+ },+ {+ "name": "php variable assignment",+ "input": "<?php\n$db_password = 'q8⟨+⟩Zr2LmT5vXp9K';\n$api_key = \"f9⟨+⟩Kd83LmQp2Zx7Rt\";\n",+ "expect": "<?php\n$db_password = '[REDACTED]';\n$api_key = \"[REDACTED]\";\n",+ "context": "config.php",+ "mode": "source"+ },+ {+ "name": "php references kept",+ "input": "<?php\n$password = getenv('DB_PASSWORD');\n$token = $request->get('token');\n$secret = $config['app_secret_value'];\n",+ "expect": null,+ "same": true,+ "context": "a.php",+ "mode": "source"+ },+ {+ "name": "token_hash in a logged url",+ "input": "GET https://abc.supabase.co/auth/v1/verify?token_hash=pk⟨+⟩ce_8f3a9b2c7d1e4f5a6b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c&type=signup 200\n",+ "expect": "GET https://abc.supabase.co/auth/v1/verify?token_hash=[REDACTED]&type=signup 200\n",+ "context": "server.log"+ },+ {+ "name": "token_hash hex after &",+ "input": "http://localhost:3000/auth/confirm?type=email&token_hash=8f⟨+⟩3a9b2c7d1e4f5a6b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c",+ "expect": "http://localhost:3000/auth/confirm?type=email&token_hash=[REDACTED]"+ },+ {+ "name": "token_hash outside a query, placeholders and short values kept",+ "input": "token_hash=8f3a9b2c7d1e4f5a6b8c9d0e1f2a3b4c5d6e7f8a\n/verify?token_hash={{ .TokenHash }}&type=x\n/verify?token_hash=YOUR_TOKEN_HASH&x=1\n/search?q=hello-world-2024-x&page_token_hash=abc\n",+ "expect": null,+ "same": true}]}\ No newline at end of file