A desktop coding agent with free access to frontier models.
TypeScript
fix: check named source files by content
ogsamrat44a2aa6f9693
diff --git a/apps/server/src/session-archive/PARITY.sha256 b/apps/server/src/session-archive/PARITY.sha256index d8b6ae28..5472f171 100644--- a/apps/server/src/session-archive/PARITY.sha256+++ b/apps/server/src/session-archive/PARITY.sha256@@ -6,4 +6,4 @@38dc65a461de6905134c22837f5230d03383ec0bdd6df35090dd269160446edf turn-overlap.ts1906cab8c93370abd1d024d3552fe919a0006af9ebef084ada7b64c290d1bc95 turn-writers.ts55e5e322b2c372b77714eb3677bee6ea0f3e0a11120ba6edc7d202c264a3d7e8 turn-prompts.ts-a64925d3546bf3b087677ac46d9a3b18a2f8178bb1258246e4f9504e1085d5fa code-files.ts+ca3bd7e89143925e6d35ae37757e736b9f8c20f6e5911a4f66433684fa185f1c code-files.tsdiff --git a/apps/server/src/session-archive/code-files.ts b/apps/server/src/session-archive/code-files.tsindex fd9ce575..9b1354e1 100644--- a/apps/server/src/session-archive/code-files.ts+++ b/apps/server/src/session-archive/code-files.ts@@ -2,16 +2,32 @@* Source files in the project archive. The credential denylist the archive* shares with the session uploader judges a path by its words: a folder* named `keys`, `secrets` or `credentials`, a name like `secret.rs` or- * `token.php`. For program source that is a wrong call: `src/config/secret.rs`,- * `lib/core/secrets/secret_store.dart` or `PhpParser/Token.php` are code the- * agent reads and edits, and leaving them out leaves a hole in the project.- * A file with a source-code extension is therefore archived whatever its name- * or folder says (its text goes through the export pass like any project- * file), unless a folder above it is a credential store of its own (`.ssh`,- * `.aws`, `.gnupg`). Data, config, docs and script files (`.json`, `.yml`,- * `.env`, `.md`, `.sh`, `.cmd`, ...) keep the denylist: they are where real- * keys are kept. Identical in the CLI and the desktop app (PARITY.sha256).+ * `token.php`. For program source that is often a wrong call:+ * `src/config/secret.rs`, `lib/core/secrets/secret_store.dart` or+ * `PhpParser/Token.php` are code the agent reads and edits.+ *+ * The same check applies to a source file whose name or folder only carries+ * such a word (`config/keys.js`, `auth/token.ts`), which the denylist lets+ * through by its extension. Such a file is archived when all of these hold:+ * - it has a program-source extension and no folder above it is a+ * credential store of its own (`.ssh`, `.aws`, `.gnupg`);+ * - it is not ignored by git (the caller checks that);+ * - it is a regular UTF-8 text file of at most MAX_CODE_CHECK_BYTES;+ * - the client rules (context/redact-policy.ts, source mode) find no+ * credential in its text.+ * Otherwise it stays out as `credential`. The archive then carries exactly+ * the bytes that were checked. Data, config, docs and script files (`.json`,+ * `.yml`, `.env`, `.md`, `.sh`, `.cmd`, ...) always keep the denylist.+ * Identical in the CLI and the desktop app (PARITY.sha256).*/+import { createHash } from "node:crypto";+import { constants as fsConstants, type BigIntStats } from "node:fs";+import { open } from "node:fs/promises";++import { redactCredentials } from "../context/redact-policy.js";++/** A denylisted source file larger than this is never read for the check: it stays out. */+export const MAX_CODE_CHECK_BYTES = 1024 * 1024;/** Program source extensions (lower case, with the dot). */export const ARCHIVE_CODE_EXTENSIONS: ReadonlySet<string> = new Set([@@ -26,13 +42,23 @@ export const ARCHIVE_CODE_EXTENSIONS: ReadonlySet<string> = new Set([".sol", ".proto", ".graphql", ".gql",]);+/** Words (a path component split on space, `.`, `_`, `-`) that name a credential store; the denylist's word list. */+const CREDENTIAL_WORDS: ReadonlySet<string> = new Set([+ "key", "keys", "secret", "secrets", "token", "tokens", "password", "passwords", "passwd", "credential", "credentials",+ "id_rsa", "id_ed25519", "netrc", "npmrc", "pypirc", "kubeconfig", "wallet", "seed", "mnemonic",+]);+const CREDENTIAL_NAME = /(^|[._-])(?:credentials?|secrets?|private[_-]?keys?)([._-]|$)/i;+/** Folders that are credential stores whatever they hold. */const CREDENTIAL_STORE_DIRS: ReadonlySet<string> = new Set([".ssh", ".aws", ".gnupg"]);+const READ_FLAGS = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0) | (fsConstants.O_NONBLOCK ?? 0) | (fsConstants.O_NOCTTY ?? 0);+const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });+/**- * Whether a project-relative path (split on `/`) is program source that the- * archive keeps even when the credential denylist matches its name or a- * folder above it.+ * Whether a project-relative path (split on `/`) is program source that may+ * be archived although the credential denylist matches its name or a folder+ * above it, subject to the content check (readCleanCodeFile).*/export function isArchiveCodeFile(parts: readonly string[]): boolean {const name = parts[parts.length - 1]?.toLowerCase() ?? "";@@ -41,3 +67,61 @@ export function isArchiveCodeFile(parts: readonly string[]): boolean {if (dot <= 0 || !ARCHIVE_CODE_EXTENSIONS.has(name.slice(dot))) return false;return !parts.slice(0, -1).some((part) => CREDENTIAL_STORE_DIRS.has(part.toLowerCase()));}++/**+ * Whether a source file is checked by content before it is archived: its+ * name or a folder above it reads like a credential store (whether or not+ * the denylist itself already leaves it out).+ */+export function needsCodeCheck(parts: readonly string[]): boolean {+ if (!isArchiveCodeFile(parts)) return false;+ return parts.some((part) => {+ const lower = part.toLowerCase();+ if (CREDENTIAL_NAME.test(lower)) return true;+ const words = lower.split(/[\s._-]+/).filter(Boolean);+ return words.some((word, index) => CREDENTIAL_WORDS.has(word) || (index > 0 && CREDENTIAL_WORDS.has(`${words[index - 1]}_${word}`)));+ });+}++export type CleanCodeFile = { bytes: Buffer; sha256: string };++/**+ * The content check of a denylisted source file at `absolute` (`rel` its+ * project-relative portable path): its bytes and their SHA-256 when it is a+ * regular file (`sameFile` confirms it is the one the scan lstat'ed) of at+ * most MAX_CODE_CHECK_BYTES, valid UTF-8 without NUL bytes, in which the+ * client credential rules find nothing; null otherwise (it stays out).+ */+export async function readCleanCodeFile(absolute: string, rel: string, sameFile: (stats: BigIntStats) => boolean): Promise<CleanCodeFile | null> {+ if (!isArchiveCodeFile(rel.split("/"))) return null;+ let handle: Awaited<ReturnType<typeof open>>;+ try {+ handle = await open(absolute, READ_FLAGS);+ } catch {+ return null;+ }+ try {+ const stats = await handle.stat({ bigint: true });+ if (!stats.isFile() || !sameFile(stats) || stats.size > BigInt(MAX_CODE_CHECK_BYTES)) return null;+ const bytes = Buffer.alloc(Number(stats.size));+ let offset = 0;+ while (offset < bytes.length) {+ const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset);+ if (bytesRead === 0) return null;+ offset += bytesRead;+ }+ if (bytes.includes(0)) return null;+ let text: string;+ try {+ text = UTF8.decode(bytes);+ } catch {+ return null;+ }+ if (redactCredentials(text, { context: rel, mode: "source" }).count > 0) return null;+ return { bytes, sha256: createHash("sha256").update(bytes).digest("hex") };+ } catch {+ return null;+ } finally {+ await handle.close().catch(() => undefined);+ }+}diff --git a/apps/server/src/session-archive/manifest.test.ts b/apps/server/src/session-archive/manifest.test.tsindex 4a99cfb0..8d23db08 100644--- a/apps/server/src/session-archive/manifest.test.ts+++ b/apps/server/src/session-archive/manifest.test.ts@@ -18,6 +18,7 @@ import {type ScannedEntry,} from "./manifest.js";import { cleanupTempDirs, tempDir } from "./test-helpers.js";+import { scanTouchedFiles } from "./touched.js";const execFileAsync = promisify(execFile);@@ -62,8 +63,11 @@ describe("credential filter", () => {describe("source files named like keys", () => {// Program source whose name or folder reads like a credential store is- // project code: a plain project folder and a git repository both keep it,- // while real key, env and config files stay out.+ // project code: a plain project folder and a git repository both keep it+ // when the client credential rules find nothing in its text. Real key, env+ // and config files stay out, and so does such a source file that holds a+ // credential, is not UTF-8 text, is over the size cap or is gitignored.+ // Every value below is invented for the test.const code = ["src/config/secret.rs","lib/core/secrets/secret_store.dart",@@ -72,30 +76,44 @@ describe("source files named like keys", () => {"src/keys/keys.mjs","app/credentials/provider.ts",];- const leftOut = [".env", "config/secrets.json", "credentials.yml", "certs/server.key", "Set Keys.cmd", "deploy/.ssh/helper.py"];-- test("the credential filter keeps them", () => {- for (const path of code) expect(isArchiveCredentialPath(path)).toBe(false);- for (const path of leftOut) expect(isArchiveCredentialPath(path)).toBe(true);+ const special: Record<string, string | Buffer> = {+ "config/keys.js": 'module.exports = { mongoURI: "mongodb://appuser:[email protected]:27017/app" };\n',+ "secrets.py": 'API_KEY = "Zq7Lm2Rt9Vb4Nw8Kp1Xc6Hd3"\n',+ "src/secrets/blob.ts": Buffer.from([0x2f, 0x2f, 0x20, 0xff, 0xfe, 0x0a]),+ "src/keys/big.ts": `// ${"x".repeat(1024 * 1024)}\n`,+ };+ const leftOut = [".env", "config/secrets.json", "credentials.yml", "certs/server.key", "Set Keys.cmd", "deploy/.ssh/helper.py", ...Object.keys(special)];+ const ignoredCode = "local/keys/local_keys.ts";++ test("the path rule alone still denies them (the scans decide by content)", () => {+ for (const path of code) expect(isArchiveCredentialPath(path)).toBe(true);+ expect(isArchiveCredentialPath("src/app.ts")).toBe(false);});for (const withGit of [false, true]) {- test(`${withGit ? "a git repository" : "a plain project folder"} archives them`, async () => {+ test(`${withGit ? "a git repository" : "a plain project folder"} archives the clean ones only`, async () => {const root = await tempDir("codenames");- for (const path of ["package.json", ...code, ...leftOut]) {+ for (const path of ["package.json", ".gitignore", ignoredCode, ...code, ...leftOut]) {await mkdir(join(root, path, ".."), { recursive: true });- await writeFile(join(root, path), `${path}\n`);+ await writeFile(join(root, path), path === ".gitignore" ? "local/\n" : special[path] ?? `// ${path}\n`);}if (withGit) {await git(root, "init", "-q");- await git(root, "add", "package.json", ...code);+ await git(root, "add", "package.json", ".gitignore", ...code);await git(root, "commit", "-qm", "init");}const { entries, excluded } = await scanArchiveTree(root);const paths = entries.map((entry) => entry.path);for (const path of code) expect(paths).toContain(path);- for (const path of leftOut) expect(paths).not.toContain(path);+ for (const path of [...leftOut, ignoredCode]) expect(paths).not.toContain(path);expect(excluded.credential).toBe(leftOut.length);+ const secretRs = entries.find((entry) => entry.path === "src/config/secret.rs")!;+ expect(secretRs.content?.toString("utf8")).toBe("// src/config/secret.rs\n");+ // A touched path goes through the same rule, also where ignored files are kept (the archiver's keepers).+ for (const includeIgnored of [false, true]) {+ const kept = await scanTouchedFiles(root, [...code, ...leftOut, ignoredCode], { includeIgnored });+ expect(kept.entries.map((entry) => entry.path).sort()).toEqual([...code].sort());+ }});}});diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.tsindex 34c85f27..8942aebd 100644--- a/apps/server/src/session-archive/manifest.ts+++ b/apps/server/src/session-archive/manifest.ts@@ -14,7 +14,7 @@ import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";import { clampUploadBytes, isUploadPathDenied, REGENERABLE_DIR_NAMES, stripRemoteUserinfo } from "../session-uploader.js";import { ATTACHMENTS_ROOT_NAME } from "./attachments.js";-import { isArchiveCodeFile } from "./code-files.js";+import { isArchiveCodeFile, needsCodeCheck, readCleanCodeFile } from "./code-files.js";import type { ExcludedList, ExcludedReason } from "./capture-v2.js";import { gitSkipReason } from "../command-guard.js";import { hintGarbageCollection } from "./files.js";@@ -172,15 +172,16 @@ export function compareArchivePaths(left: string, right: string): number {/*** True when a file or symlink must be left out of the archive (section 5.3):- * the session uploader's credential denylist, with git internals exempt,- * `node_modules` not counted as a denial, and program source kept whatever- * its name or folder says (code-files.ts).+ * the session uploader's credential denylist, with git internals exempt and+ * `node_modules` not counted as a denial. A whole-folder or touched-files+ * scan still keeps a denylisted source file that passes the content check+ * of code-files.ts.*/export function isArchiveCredentialPath(relPath: string): boolean {const parts = relPath.split("/");if (parts.some((part) => part.toLowerCase() === ".git")) return false;const rest = parts.filter((part) => part.toLowerCase() !== "node_modules");- return rest.length > 0 && isUploadPathDenied(rest.join("/")) && !isArchiveCodeFile(rest);+ return rest.length > 0 && isUploadPathDenied(rest.join("/"));}/** workspace.label: the root's basename, at most 255 UTF-8 bytes. */@@ -612,10 +613,15 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):note(rel, "special", "file");return;}- if (!includeCredentials && isArchiveCredentialPath(rel)) {- excluded.credential += 1;- note(rel, "credential", "file");- return;+ // A source file named like a key store is decided by its content once it is known not to be ignored (code-files.ts).+ let codeCheck = false;+ if (!includeCredentials && (isArchiveCredentialPath(rel) || (stats.isFile() && needsCodeCheck(rel.split("/"))))) {+ if (!stats.isFile() || !isArchiveCodeFile(rel.split("/"))) {+ excluded.credential += 1;+ note(rel, "credential", "file");+ return;+ }+ codeCheck = true;}if (stats.isSymbolicLink()) {let target: string | null;@@ -637,6 +643,20 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):return;}const entry: ScannedEntry = { path: rel, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };+ if (codeCheck) {+ const clean = await readCleanCodeFile(abs, rel, (opened) => isSameFileIdentity(entry, opened));+ if (!clean) {+ excluded.credential += 1;+ note(rel, "credential", "file");+ return;+ }+ // Pass 2 writes exactly the checked bytes.+ entry.content = clean.bytes;+ entry.size = clean.bytes.length;+ entry.sha256 = clean.sha256;+ entries.push(entry);+ return;+ }if (isGitConfigPath(rel) && entry.size > MAX_SCRUBBED_CONFIG_BYTES) {// Too large to scrub: never archived raw (it could carry credentials). Listed with its size.excluded.special += 1;diff --git a/apps/server/src/session-archive/touched.ts b/apps/server/src/session-archive/touched.tsindex 71ea746a..54a7aa4c 100644--- a/apps/server/src/session-archive/touched.ts+++ b/apps/server/src/session-archive/touched.ts@@ -39,6 +39,7 @@ import {type ExcludedCounts,type ScannedEntry,} from "./manifest.js";+import { isArchiveCodeFile, needsCodeCheck, readCleanCodeFile } from "./code-files.js";import { stateKey } from "./files.js";import { touchedIgnoredPaths } from "./ignore.js";import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";@@ -196,6 +197,7 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, opconst gone = new Set<string>();const entries: ScannedEntry[] = [];const seen = new Set<string>();+ const codeChecks = new Set<ScannedEntry>();const dirs = new Map<string, Promise<DirState>>();/** Whether `relDir` is a real folder under real folders (a symlink or a file in the way is "gone": the path is not inside the root). */@@ -240,9 +242,14 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, opexcluded.app_state += 1;return;}- if (!includeCredentials && isArchiveCredentialPath(path)) {- excluded.credential += 1;- return;+ // A source file named like a key store is decided once it is known not to be ignored, by its content (code-files.ts).+ let codeCheck = false;+ if (!includeCredentials && (isArchiveCredentialPath(path) || needsCodeCheck(parts))) {+ if (!isArchiveCodeFile(parts)) {+ excluded.credential += 1;+ return;+ }+ codeCheck = true;}if (parts.length > 1) {const parent = await dirState(parts, parts.length - 1);@@ -292,11 +299,41 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, opexcluded.special += 1;return;}- entries.push({ path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) });+ const entry: ScannedEntry = { path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };+ entries.push(entry);+ if (codeCheck) codeChecks.add(entry);};await forEachBounded([...paths], STAT_CONCURRENCY, (path) => visit(path, 0));+ // Source files named like key stores: never when git ignores them (even where ignored files are kept), else only with clean content.+ if (codeChecks.size > 0) {+ const ignoredCode = await touchedIgnoredPaths(base, [...codeChecks].map((entry) => entry.path), signal);+ const dropped = new Set<ScannedEntry>();+ await forEachBounded([...codeChecks], HASH_CONCURRENCY, async (entry) => {+ signal?.throwIfAborted();+ const absolute = join(base, ...entry.path.split("/"));+ access(absolute);+ const clean = ignoredCode.has(entry.path) ? null : await readCleanCodeFile(absolute, entry.path, (opened) => isSameFileIdentity(entry, opened));+ if (!clean) {+ dropped.add(entry);+ return;+ }+ // Pass 2 writes exactly the checked bytes.+ entry.content = clean.bytes;+ entry.size = clean.bytes.length;+ entry.sha256 = clean.sha256;+ });+ if (dropped.size > 0) {+ excluded.credential += dropped.size;+ // An archived file that now fails the check leaves the chain like a gone one.+ for (const entry of dropped) gone.add(entry.path);+ const remaining = entries.filter((entry) => !dropped.has(entry));+ entries.length = 0;+ entries.push(...remaining);+ }+ }+// Git's answer for just the files found, so the folder is never walked (none asked when ignored files are kept).const ignoredPaths = options.includeIgnored === true ? new Set<string>() : await touchedIgnoredPaths(base, entries.map((entry) => entry.path), signal);signal?.throwIfAborted();@@ -313,6 +350,7 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, opconst buffers: Buffer[] = [];await forEachBounded(entries, HASH_CONCURRENCY, async (entry) => {signal?.throwIfAborted();+ if (entry.content) return;const cached = cache?.lookup(entry) ?? null;if (cached) {entry.sha256 = cached;