Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: check named source files by content

ogsamrat44a2aa6f9693
diff --git a/apps/server/src/session-archive/PARITY.sha256 b/apps/server/src/session-archive/PARITY.sha256
index d8b6ae28..5472f171 100644
--- a/apps/server/src/session-archive/PARITY.sha256
+++ b/apps/server/src/session-archive/PARITY.sha256
@@ -6,4 +6,4 @@
38dc65a461de6905134c22837f5230d03383ec0bdd6df35090dd269160446edf turn-overlap.ts
1906cab8c93370abd1d024d3552fe919a0006af9ebef084ada7b64c290d1bc95 turn-writers.ts
55e5e322b2c372b77714eb3677bee6ea0f3e0a11120ba6edc7d202c264a3d7e8 turn-prompts.ts
-a64925d3546bf3b087677ac46d9a3b18a2f8178bb1258246e4f9504e1085d5fa code-files.ts
+ca3bd7e89143925e6d35ae37757e736b9f8c20f6e5911a4f66433684fa185f1c code-files.ts
diff --git a/apps/server/src/session-archive/code-files.ts b/apps/server/src/session-archive/code-files.ts
index fd9ce575..9b1354e1 100644
--- a/apps/server/src/session-archive/code-files.ts
+++ b/apps/server/src/session-archive/code-files.ts
@@ -2,16 +2,32 @@
* Source files in the project archive. The credential denylist the archive
* shares with the session uploader judges a path by its words: a folder
* named `keys`, `secrets` or `credentials`, a name like `secret.rs` or
- * `token.php`. For program source that is a wrong call: `src/config/secret.rs`,
- * `lib/core/secrets/secret_store.dart` or `PhpParser/Token.php` are code the
- * agent reads and edits, and leaving them out leaves a hole in the project.
- * A file with a source-code extension is therefore archived whatever its name
- * or folder says (its text goes through the export pass like any project
- * file), unless a folder above it is a credential store of its own (`.ssh`,
- * `.aws`, `.gnupg`). Data, config, docs and script files (`.json`, `.yml`,
- * `.env`, `.md`, `.sh`, `.cmd`, ...) keep the denylist: they are where real
- * keys are kept. Identical in the CLI and the desktop app (PARITY.sha256).
+ * `token.php`. For program source that is often a wrong call:
+ * `src/config/secret.rs`, `lib/core/secrets/secret_store.dart` or
+ * `PhpParser/Token.php` are code the agent reads and edits.
+ *
+ * The same check applies to a source file whose name or folder only carries
+ * such a word (`config/keys.js`, `auth/token.ts`), which the denylist lets
+ * through by its extension. Such a file is archived when all of these hold:
+ * - it has a program-source extension and no folder above it is a
+ * credential store of its own (`.ssh`, `.aws`, `.gnupg`);
+ * - it is not ignored by git (the caller checks that);
+ * - it is a regular UTF-8 text file of at most MAX_CODE_CHECK_BYTES;
+ * - the client rules (context/redact-policy.ts, source mode) find no
+ * credential in its text.
+ * Otherwise it stays out as `credential`. The archive then carries exactly
+ * the bytes that were checked. Data, config, docs and script files (`.json`,
+ * `.yml`, `.env`, `.md`, `.sh`, `.cmd`, ...) always keep the denylist.
+ * Identical in the CLI and the desktop app (PARITY.sha256).
*/
+import { createHash } from "node:crypto";
+import { constants as fsConstants, type BigIntStats } from "node:fs";
+import { open } from "node:fs/promises";
+
+import { redactCredentials } from "../context/redact-policy.js";
+
+/** A denylisted source file larger than this is never read for the check: it stays out. */
+export const MAX_CODE_CHECK_BYTES = 1024 * 1024;
/** Program source extensions (lower case, with the dot). */
export const ARCHIVE_CODE_EXTENSIONS: ReadonlySet<string> = new Set([
@@ -26,13 +42,23 @@ export const ARCHIVE_CODE_EXTENSIONS: ReadonlySet<string> = new Set([
".sol", ".proto", ".graphql", ".gql",
]);
+/** Words (a path component split on space, `.`, `_`, `-`) that name a credential store; the denylist's word list. */
+const CREDENTIAL_WORDS: ReadonlySet<string> = new Set([
+ "key", "keys", "secret", "secrets", "token", "tokens", "password", "passwords", "passwd", "credential", "credentials",
+ "id_rsa", "id_ed25519", "netrc", "npmrc", "pypirc", "kubeconfig", "wallet", "seed", "mnemonic",
+]);
+const CREDENTIAL_NAME = /(^|[._-])(?:credentials?|secrets?|private[_-]?keys?)([._-]|$)/i;
+
/** Folders that are credential stores whatever they hold. */
const CREDENTIAL_STORE_DIRS: ReadonlySet<string> = new Set([".ssh", ".aws", ".gnupg"]);
+const READ_FLAGS = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0) | (fsConstants.O_NONBLOCK ?? 0) | (fsConstants.O_NOCTTY ?? 0);
+const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+
/**
- * Whether a project-relative path (split on `/`) is program source that the
- * archive keeps even when the credential denylist matches its name or a
- * folder above it.
+ * Whether a project-relative path (split on `/`) is program source that may
+ * be archived although the credential denylist matches its name or a folder
+ * above it, subject to the content check (readCleanCodeFile).
*/
export function isArchiveCodeFile(parts: readonly string[]): boolean {
const name = parts[parts.length - 1]?.toLowerCase() ?? "";
@@ -41,3 +67,61 @@ export function isArchiveCodeFile(parts: readonly string[]): boolean {
if (dot <= 0 || !ARCHIVE_CODE_EXTENSIONS.has(name.slice(dot))) return false;
return !parts.slice(0, -1).some((part) => CREDENTIAL_STORE_DIRS.has(part.toLowerCase()));
}
+
+/**
+ * Whether a source file is checked by content before it is archived: its
+ * name or a folder above it reads like a credential store (whether or not
+ * the denylist itself already leaves it out).
+ */
+export function needsCodeCheck(parts: readonly string[]): boolean {
+ if (!isArchiveCodeFile(parts)) return false;
+ return parts.some((part) => {
+ const lower = part.toLowerCase();
+ if (CREDENTIAL_NAME.test(lower)) return true;
+ const words = lower.split(/[\s._-]+/).filter(Boolean);
+ return words.some((word, index) => CREDENTIAL_WORDS.has(word) || (index > 0 && CREDENTIAL_WORDS.has(`${words[index - 1]}_${word}`)));
+ });
+}
+
+export type CleanCodeFile = { bytes: Buffer; sha256: string };
+
+/**
+ * The content check of a denylisted source file at `absolute` (`rel` its
+ * project-relative portable path): its bytes and their SHA-256 when it is a
+ * regular file (`sameFile` confirms it is the one the scan lstat'ed) of at
+ * most MAX_CODE_CHECK_BYTES, valid UTF-8 without NUL bytes, in which the
+ * client credential rules find nothing; null otherwise (it stays out).
+ */
+export async function readCleanCodeFile(absolute: string, rel: string, sameFile: (stats: BigIntStats) => boolean): Promise<CleanCodeFile | null> {
+ if (!isArchiveCodeFile(rel.split("/"))) return null;
+ let handle: Awaited<ReturnType<typeof open>>;
+ try {
+ handle = await open(absolute, READ_FLAGS);
+ } catch {
+ return null;
+ }
+ try {
+ const stats = await handle.stat({ bigint: true });
+ if (!stats.isFile() || !sameFile(stats) || stats.size > BigInt(MAX_CODE_CHECK_BYTES)) return null;
+ const bytes = Buffer.alloc(Number(stats.size));
+ let offset = 0;
+ while (offset < bytes.length) {
+ const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset);
+ if (bytesRead === 0) return null;
+ offset += bytesRead;
+ }
+ if (bytes.includes(0)) return null;
+ let text: string;
+ try {
+ text = UTF8.decode(bytes);
+ } catch {
+ return null;
+ }
+ if (redactCredentials(text, { context: rel, mode: "source" }).count > 0) return null;
+ return { bytes, sha256: createHash("sha256").update(bytes).digest("hex") };
+ } catch {
+ return null;
+ } finally {
+ await handle.close().catch(() => undefined);
+ }
+}
diff --git a/apps/server/src/session-archive/manifest.test.ts b/apps/server/src/session-archive/manifest.test.ts
index 4a99cfb0..8d23db08 100644
--- a/apps/server/src/session-archive/manifest.test.ts
+++ b/apps/server/src/session-archive/manifest.test.ts
@@ -18,6 +18,7 @@ import {
type ScannedEntry,
} from "./manifest.js";
import { cleanupTempDirs, tempDir } from "./test-helpers.js";
+import { scanTouchedFiles } from "./touched.js";
const execFileAsync = promisify(execFile);
@@ -62,8 +63,11 @@ describe("credential filter", () => {
describe("source files named like keys", () => {
// Program source whose name or folder reads like a credential store is
- // project code: a plain project folder and a git repository both keep it,
- // while real key, env and config files stay out.
+ // project code: a plain project folder and a git repository both keep it
+ // when the client credential rules find nothing in its text. Real key, env
+ // and config files stay out, and so does such a source file that holds a
+ // credential, is not UTF-8 text, is over the size cap or is gitignored.
+ // Every value below is invented for the test.
const code = [
"src/config/secret.rs",
"lib/core/secrets/secret_store.dart",
@@ -72,30 +76,44 @@ describe("source files named like keys", () => {
"src/keys/keys.mjs",
"app/credentials/provider.ts",
];
- const leftOut = [".env", "config/secrets.json", "credentials.yml", "certs/server.key", "Set Keys.cmd", "deploy/.ssh/helper.py"];
-
- test("the credential filter keeps them", () => {
- for (const path of code) expect(isArchiveCredentialPath(path)).toBe(false);
- for (const path of leftOut) expect(isArchiveCredentialPath(path)).toBe(true);
+ const special: Record<string, string | Buffer> = {
+ "config/keys.js": 'module.exports = { mongoURI: "mongodb://appuser:[email protected]:27017/app" };\n',
+ "secrets.py": 'API_KEY = "Zq7Lm2Rt9Vb4Nw8Kp1Xc6Hd3"\n',
+ "src/secrets/blob.ts": Buffer.from([0x2f, 0x2f, 0x20, 0xff, 0xfe, 0x0a]),
+ "src/keys/big.ts": `// ${"x".repeat(1024 * 1024)}\n`,
+ };
+ const leftOut = [".env", "config/secrets.json", "credentials.yml", "certs/server.key", "Set Keys.cmd", "deploy/.ssh/helper.py", ...Object.keys(special)];
+ const ignoredCode = "local/keys/local_keys.ts";
+
+ test("the path rule alone still denies them (the scans decide by content)", () => {
+ for (const path of code) expect(isArchiveCredentialPath(path)).toBe(true);
+ expect(isArchiveCredentialPath("src/app.ts")).toBe(false);
});
for (const withGit of [false, true]) {
- test(`${withGit ? "a git repository" : "a plain project folder"} archives them`, async () => {
+ test(`${withGit ? "a git repository" : "a plain project folder"} archives the clean ones only`, async () => {
const root = await tempDir("codenames");
- for (const path of ["package.json", ...code, ...leftOut]) {
+ for (const path of ["package.json", ".gitignore", ignoredCode, ...code, ...leftOut]) {
await mkdir(join(root, path, ".."), { recursive: true });
- await writeFile(join(root, path), `${path}\n`);
+ await writeFile(join(root, path), path === ".gitignore" ? "local/\n" : special[path] ?? `// ${path}\n`);
}
if (withGit) {
await git(root, "init", "-q");
- await git(root, "add", "package.json", ...code);
+ await git(root, "add", "package.json", ".gitignore", ...code);
await git(root, "commit", "-qm", "init");
}
const { entries, excluded } = await scanArchiveTree(root);
const paths = entries.map((entry) => entry.path);
for (const path of code) expect(paths).toContain(path);
- for (const path of leftOut) expect(paths).not.toContain(path);
+ for (const path of [...leftOut, ignoredCode]) expect(paths).not.toContain(path);
expect(excluded.credential).toBe(leftOut.length);
+ const secretRs = entries.find((entry) => entry.path === "src/config/secret.rs")!;
+ expect(secretRs.content?.toString("utf8")).toBe("// src/config/secret.rs\n");
+ // A touched path goes through the same rule, also where ignored files are kept (the archiver's keepers).
+ for (const includeIgnored of [false, true]) {
+ const kept = await scanTouchedFiles(root, [...code, ...leftOut, ignoredCode], { includeIgnored });
+ expect(kept.entries.map((entry) => entry.path).sort()).toEqual([...code].sort());
+ }
});
}
});
diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.ts
index 34c85f27..8942aebd 100644
--- a/apps/server/src/session-archive/manifest.ts
+++ b/apps/server/src/session-archive/manifest.ts
@@ -14,7 +14,7 @@ import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";
import { clampUploadBytes, isUploadPathDenied, REGENERABLE_DIR_NAMES, stripRemoteUserinfo } from "../session-uploader.js";
import { ATTACHMENTS_ROOT_NAME } from "./attachments.js";
-import { isArchiveCodeFile } from "./code-files.js";
+import { isArchiveCodeFile, needsCodeCheck, readCleanCodeFile } from "./code-files.js";
import type { ExcludedList, ExcludedReason } from "./capture-v2.js";
import { gitSkipReason } from "../command-guard.js";
import { hintGarbageCollection } from "./files.js";
@@ -172,15 +172,16 @@ export function compareArchivePaths(left: string, right: string): number {
/**
* True when a file or symlink must be left out of the archive (section 5.3):
- * the session uploader's credential denylist, with git internals exempt,
- * `node_modules` not counted as a denial, and program source kept whatever
- * its name or folder says (code-files.ts).
+ * the session uploader's credential denylist, with git internals exempt and
+ * `node_modules` not counted as a denial. A whole-folder or touched-files
+ * scan still keeps a denylisted source file that passes the content check
+ * of code-files.ts.
*/
export function isArchiveCredentialPath(relPath: string): boolean {
const parts = relPath.split("/");
if (parts.some((part) => part.toLowerCase() === ".git")) return false;
const rest = parts.filter((part) => part.toLowerCase() !== "node_modules");
- return rest.length > 0 && isUploadPathDenied(rest.join("/")) && !isArchiveCodeFile(rest);
+ return rest.length > 0 && isUploadPathDenied(rest.join("/"));
}
/** workspace.label: the root's basename, at most 255 UTF-8 bytes. */
@@ -612,10 +613,15 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
note(rel, "special", "file");
return;
}
- if (!includeCredentials && isArchiveCredentialPath(rel)) {
- excluded.credential += 1;
- note(rel, "credential", "file");
- return;
+ // A source file named like a key store is decided by its content once it is known not to be ignored (code-files.ts).
+ let codeCheck = false;
+ if (!includeCredentials && (isArchiveCredentialPath(rel) || (stats.isFile() && needsCodeCheck(rel.split("/"))))) {
+ if (!stats.isFile() || !isArchiveCodeFile(rel.split("/"))) {
+ excluded.credential += 1;
+ note(rel, "credential", "file");
+ return;
+ }
+ codeCheck = true;
}
if (stats.isSymbolicLink()) {
let target: string | null;
@@ -637,6 +643,20 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
return;
}
const entry: ScannedEntry = { path: rel, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };
+ if (codeCheck) {
+ const clean = await readCleanCodeFile(abs, rel, (opened) => isSameFileIdentity(entry, opened));
+ if (!clean) {
+ excluded.credential += 1;
+ note(rel, "credential", "file");
+ return;
+ }
+ // Pass 2 writes exactly the checked bytes.
+ entry.content = clean.bytes;
+ entry.size = clean.bytes.length;
+ entry.sha256 = clean.sha256;
+ entries.push(entry);
+ return;
+ }
if (isGitConfigPath(rel) && entry.size > MAX_SCRUBBED_CONFIG_BYTES) {
// Too large to scrub: never archived raw (it could carry credentials). Listed with its size.
excluded.special += 1;
diff --git a/apps/server/src/session-archive/touched.ts b/apps/server/src/session-archive/touched.ts
index 71ea746a..54a7aa4c 100644
--- a/apps/server/src/session-archive/touched.ts
+++ b/apps/server/src/session-archive/touched.ts
@@ -39,6 +39,7 @@ import {
type ExcludedCounts,
type ScannedEntry,
} from "./manifest.js";
+import { isArchiveCodeFile, needsCodeCheck, readCleanCodeFile } from "./code-files.js";
import { stateKey } from "./files.js";
import { touchedIgnoredPaths } from "./ignore.js";
import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";
@@ -196,6 +197,7 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, op
const gone = new Set<string>();
const entries: ScannedEntry[] = [];
const seen = new Set<string>();
+ const codeChecks = new Set<ScannedEntry>();
const dirs = new Map<string, Promise<DirState>>();
/** Whether `relDir` is a real folder under real folders (a symlink or a file in the way is "gone": the path is not inside the root). */
@@ -240,9 +242,14 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, op
excluded.app_state += 1;
return;
}
- if (!includeCredentials && isArchiveCredentialPath(path)) {
- excluded.credential += 1;
- return;
+ // A source file named like a key store is decided once it is known not to be ignored, by its content (code-files.ts).
+ let codeCheck = false;
+ if (!includeCredentials && (isArchiveCredentialPath(path) || needsCodeCheck(parts))) {
+ if (!isArchiveCodeFile(parts)) {
+ excluded.credential += 1;
+ return;
+ }
+ codeCheck = true;
}
if (parts.length > 1) {
const parent = await dirState(parts, parts.length - 1);
@@ -292,11 +299,41 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, op
excluded.special += 1;
return;
}
- entries.push({ path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) });
+ const entry: ScannedEntry = { path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };
+ entries.push(entry);
+ if (codeCheck) codeChecks.add(entry);
};
await forEachBounded([...paths], STAT_CONCURRENCY, (path) => visit(path, 0));
+ // Source files named like key stores: never when git ignores them (even where ignored files are kept), else only with clean content.
+ if (codeChecks.size > 0) {
+ const ignoredCode = await touchedIgnoredPaths(base, [...codeChecks].map((entry) => entry.path), signal);
+ const dropped = new Set<ScannedEntry>();
+ await forEachBounded([...codeChecks], HASH_CONCURRENCY, async (entry) => {
+ signal?.throwIfAborted();
+ const absolute = join(base, ...entry.path.split("/"));
+ access(absolute);
+ const clean = ignoredCode.has(entry.path) ? null : await readCleanCodeFile(absolute, entry.path, (opened) => isSameFileIdentity(entry, opened));
+ if (!clean) {
+ dropped.add(entry);
+ return;
+ }
+ // Pass 2 writes exactly the checked bytes.
+ entry.content = clean.bytes;
+ entry.size = clean.bytes.length;
+ entry.sha256 = clean.sha256;
+ });
+ if (dropped.size > 0) {
+ excluded.credential += dropped.size;
+ // An archived file that now fails the check leaves the chain like a gone one.
+ for (const entry of dropped) gone.add(entry.path);
+ const remaining = entries.filter((entry) => !dropped.has(entry));
+ entries.length = 0;
+ entries.push(...remaining);
+ }
+ }
+
// Git's answer for just the files found, so the folder is never walked (none asked when ignored files are kept).
const ignoredPaths = options.includeIgnored === true ? new Set<string>() : await touchedIgnoredPaths(base, entries.map((entry) => entry.path), signal);
signal?.throwIfAborted();
@@ -313,6 +350,7 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, op
const buffers: Buffer[] = [];
await forEachBounded(entries, HASH_CONCURRENCY, async (entry) => {
signal?.throwIfAborted();
+ if (entry.content) return;
const cached = cache?.lookup(entry) ?? null;
if (cached) {
entry.sha256 = cached;