Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(capture): archive outside files a shell glob, loop or find reached, and files used during the call under named folders

ogsamrat7ac401ad5b0c
diff --git a/apps/server/src/session-archive/outside.test.ts b/apps/server/src/session-archive/outside.test.ts
index 169a810e..54e736e3 100644
--- a/apps/server/src/session-archive/outside.test.ts
+++ b/apps/server/src/session-archive/outside.test.ts
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, test } from "bun:test";
import { execFile } from "node:child_process";
import { createHash, randomBytes } from "node:crypto";
-import { mkdir, rm, symlink, writeFile } from "node:fs/promises";
+import { mkdir, rm, symlink, utimes, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { promisify } from "node:util";
@@ -10,7 +10,7 @@ import { SessionArchiver } from "./index.js";
import { outsideArchivePath, outsideSourcePath } from "./outside.js";
import { cleanupTempDirs, manifestOf, openArchive, tempDir } from "./test-helpers.js";
import { TouchedPathStore } from "./touched.js";
-import { recordTurnFiles, shellPathCandidates } from "./turn-files.js";
+import { expandShellTargets, recordTurnFiles, shellGlobCandidates, shellPathCandidates } from "./turn-files.js";
// Files outside the session's workspace that the agent touches are archived
// byte for byte under `__outside__/<absolute path>`, with the archive's
@@ -178,4 +178,73 @@ describe("files outside the workspace", () => {
expect(byPath.has(".env.local")).toBe(false);
expect(traced).toHaveLength(3);
});
+
+ test("shell glob words that look like paths are glob candidates; variables, braces and bare patterns are not", () => {
+ const found = shellGlobCandidates(`for f in /tmp/fx/*; do wc -c "$f"; done; cat dir/*.txt ~/x/**/*.csv; find /data -name '*.log' -exec cat {} \\; ; ls $HOME/*.md /a/{b,c}/*`);
+ expect(found.sort()).toEqual(["/tmp/fx/*", "dir/*.txt", "~/x/**/*.csv"].sort());
+ });
+
+ test("files a shell glob, a loop or `find -exec` reached are archived; files used during the call under a named folder too", async () => {
+ const root = await gitProject();
+ const away = await tempDir("outside-glob");
+ const home = join(away, "home");
+ const server = new FakeArchiveServer();
+ const stateDir = await tempDir("outside-glob-state");
+ const archive = archiver(server, stateDir);
+ const sessionId = "ses_outside_glob_1";
+ expect((await archive.captureBase(sessionId, root)).status).toBe("queued");
+ const past = new Date(Date.now() - 3_600_000);
+ const fixtures = join(away, "fixtures");
+ const looped: Array<[string, Buffer]> = [];
+ for (const name of ["a.bin", "b.txt", "c.gz"]) looped.push([name, await write(join(fixtures, name), randomBytes(500))]);
+ await write(join(fixtures, ".env"), "API_KEY=sk-secret\n");
+ await write(join(fixtures, "key.pem"), "-----BEGIN PRIVATE KEY-----\n");
+ await symlink(join(fixtures, "a.bin"), join(fixtures, "link.bin"));
+ for (const name of ["one.txt", "two.txt"]) await write(join(away, "notes", name), `${name}\n`);
+ await write(join(away, "notes", "skip.md"), "not matched\n");
+ const logs = join(away, "logs");
+ await write(join(logs, "deep", "used.log"), "used\n");
+ await write(join(logs, "untouched.log"), "untouched\n");
+ await write(join(logs, "node_modules", "dep.log"), "dep\n");
+ for (const file of ["deep/used.log", "untouched.log", "node_modules/dep.log"]) await utimes(join(logs, file), past, past);
+ const start = Date.now();
+ await utimes(join(logs, "deep", "used.log"), new Date(start + 100), past);
+ await write(join(home, "secret-notes.txt"), "home\n");
+ const messages = [{
+ info: { id: "msg_1", role: "assistant" },
+ parts: [
+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `for f in ${fixtures}/*; do wc -c "$f"; done` }, time: { start: start - 60_000, end: start - 59_000 } } },
+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `cat ${away}/notes/*.txt ~/*.txt` } } },
+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `find ${logs} -name '*.log' -exec cat {} \\;` }, time: { start, end: start + 500 } } },
+ ],
+ }];
+ const reported: string[] = [];
+ await recordTurnFiles({
+ sessionId, root, messages, home,
+ collector: { recordTrace: () => undefined },
+ archive: { pathTouched: (id, file) => { reported.push(file); archive.recordTouched(id, file); } },
+ });
+ const expected = [...looped.map(([name]) => join(fixtures, name)), join(away, "notes", "one.txt"), join(away, "notes", "two.txt"), join(logs, "deep", "used.log")];
+ expect(new Set(reported)).toEqual(new Set(expected));
+ expect((await archive.captureDelta(sessionId, root, 1)).status).toBe("queued");
+ await archive.drain();
+ const delta = await lastArchive(server);
+ for (const [name, bytes] of looped) expect(delta.names.get(encoded(join(fixtures, name)))!.equals(bytes)).toBe(true);
+ expect(delta.names.get(encoded(join(logs, "deep", "used.log")))!.toString()).toBe("used\n");
+ expect([...delta.names.keys()].some((name) => /\.env|key\.pem|link\.bin|untouched|node_modules|skip\.md|secret-notes/.test(name))).toBe(false);
+ await archive.stop();
+ });
+
+ test("glob and folder expansion never walks a temp root, the home folder, the file system root or the workspace", async () => {
+ const root = await tempDir("expand-root");
+ const away = await tempDir("expand-away");
+ await write(join(away, "a.txt"), "a\n");
+ await write(join(root, "data", "x.csv"), "x\n");
+ const now = Date.now();
+ const targets = [`${away}/*`, "/*", "/etc/*", `${root}/../*`].map((path) => ({ path, glob: true, start: null as number | null, end: null as number | null }));
+ targets.push({ path: away, glob: false, start: now - 1000, end: now + 1000 }, { path: "/", glob: false, start: now, end: now });
+ expect(await expandShellTargets(targets, root, { home: away, tempRoots: [] })).toEqual([]);
+ expect(await expandShellTargets(targets, root, { home: null, tempRoots: [away] })).toEqual([]);
+ expect(await expandShellTargets([{ path: `${root}/data/*.csv`, glob: true, start: null, end: null }], root, { home: null, tempRoots: [] })).toEqual([join(root, "data", "x.csv")]);
+ });
});
diff --git a/apps/server/src/session-archive/turn-files.ts b/apps/server/src/session-archive/turn-files.ts
index b489571f..d72cb1b3 100644
--- a/apps/server/src/session-archive/turn-files.ts
+++ b/apps/server/src/session-archive/turn-files.ts
@@ -14,17 +14,29 @@
* show, still says which files it read. Paths are workspace-relative, or
* the `__outside__/...` archive path (`outside: true`).
*
+ * A shell command that reaches files without naming each one is expanded
+ * too (expandShellTargets): a glob word (`for f in /data/*`, `cat dir/*.txt`)
+ * names every file it matches, and an outside folder the command names
+ * (`find /data -exec ...`, `ls /data`, the folder of an outside script it
+ * ran) contributes the files read or modified during the call (atime or
+ * mtime within the tool call's time window; best effort where the file
+ * system does not update atime). Folders are walked without following
+ * symlinks, within caps, and never the file system root, the home folder,
+ * a temp root, a system folder, an ancestor of the workspace, or a folder
+ * the archive excludes.
+ *
* The archive's exclusions apply to both: credential files and stores, the
* app's state, dependency and cache folders, symlinks, special files, and
* files over MAX_OUTSIDE_FILE_BYTES.
*/
import { createHash } from "node:crypto";
-import { lstat, open, realpath } from "node:fs/promises";
-import { homedir } from "node:os";
+import { lstat, open, readdir, realpath } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import { HASH_READ_BYTES, OPEN_ENTRY_FLAGS, isArchiveCredentialPath } from "./manifest.js";
import { MAX_OUTSIDE_FILE_BYTES, outsideArchivePath, outsideExclusion } from "./outside.js";
+import { isHomeSettingsFolder, isSecretFolder } from "../context/privacy.js";
export type TurnFileAccess = "read" | "write" | "shell";
export type TurnFileEvidence = { path: string; access: TurnFileAccess };
@@ -43,6 +55,16 @@ const MAX_TURN_CANDIDATES = 1_024;
const MAX_TURN_FILE_EVENTS = 500;
const MAX_PATH_CHARS = 4_096;
const MAX_COMMAND_CHARS = 256 * 1024;
+/** Shell expansion caps (per turn): globs and folders expanded, files one glob or folder adds, directory entries read. */
+const MAX_TURN_EXPANSIONS = 32;
+const MAX_EXPANSION_FILES = 256;
+const MAX_EXPANSION_ENTRIES = 20_000;
+const MAX_EXPANSION_DEPTH = 6;
+/** Clock and timestamp granularity slack around a tool call's window. */
+const CALL_WINDOW_SLACK_MS = 2_000;
+/** Folders never walked when a command names them or a glob starts in them. */
+const SKIPPED_EXPANSION_DIRS = new Set([".git", ".hg", ".svn", "node_modules", ".venv", "venv", "__pycache__", ".tox", ".mypy_cache", ".pytest_cache", ".cache"]);
+const SYSTEM_FOLDER = /^\/(?:usr|etc|bin|sbin|lib|lib32|lib64|libx32|proc|sys|dev|run|boot|var\/lib|var\/log|var\/cache|snap|System|Library|private\/etc|private\/var\/db|Applications)(?:\/|$)/;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -147,6 +169,303 @@ export function shellPathCandidates(command: string): string[] {
return [...out];
}
+/**
+ * Glob words of a shell command that look like paths (`/data/*`,
+ * `dir/*.txt`, `~/x/**\/*.csv`): `*`, `?` or `[...]` in a word with a `/`.
+ * Words with a variable, a command substitution or a brace are left out.
+ */
+export function shellGlobCandidates(command: string): string[] {
+ if (command.length > MAX_COMMAND_CHARS) command = command.slice(0, MAX_COMMAND_CHARS);
+ const out = new Set<string>();
+ for (const word of shellWords(command)) {
+ if (word.startsWith("-") || word.length > MAX_PATH_CHARS || word.includes("://") || /[$`{}]/.test(word)) continue;
+ if (!word.includes("/") || !/[*?[]/.test(word)) continue;
+ out.add(word);
+ }
+ return [...out];
+}
+
+/** A shell command's glob and folder targets, absolute, with the call's time window (ms) when the engine recorded it. */
+export type ShellTarget = { path: string; glob: boolean; start: number | null; end: number | null };
+
+function callWindow(state: Record<string, unknown>): { start: number | null; end: number | null } {
+ const time = isRecord(state.time) ? state.time : {};
+ const start = typeof time.start === "number" && Number.isFinite(time.start) && time.start > 0 ? time.start : null;
+ const end = typeof time.end === "number" && Number.isFinite(time.end) && time.end > 0 ? time.end : null;
+ return { start, end };
+}
+
+/** The glob words and path words (folder candidates) of a turn's shell commands, absolute, with each call's window. */
+export function turnShellTargets(messages: unknown, root: string, home: string | null = safeHome()): ShellTarget[] {
+ const out: ShellTarget[] = [];
+ const list = Array.isArray(messages) ? messages : isRecord(messages) && Array.isArray(messages.messages) ? messages.messages : [];
+ for (const message of list) {
+ if (!isRecord(message) || !Array.isArray(message.parts)) continue;
+ for (const part of message.parts) {
+ if (!isRecord(part) || part.type !== "tool") continue;
+ const name = toolName(part.tool ?? part.name);
+ if (!name || !SHELL_TOOLS.has(name)) continue;
+ const state = isRecord(part.state) ? part.state : {};
+ const input = isRecord(state.input) ? state.input : isRecord(part.input) ? part.input : null;
+ if (!input) continue;
+ const cwdValue = CWD_KEYS.map((key) => cleanPath(input[key])).find(Boolean);
+ const cwd = cwdValue ? resolve(root, expandHome(cwdValue, home) ?? ".") : root;
+ const window = callWindow(state);
+ for (const key of COMMAND_KEYS) {
+ const value = input[key];
+ const command = typeof value === "string" ? value : Array.isArray(value) && value.every((item) => typeof item === "string") ? value.join(" ") : null;
+ if (!command) continue;
+ const push = (raw: string, glob: boolean) => {
+ if (out.length >= MAX_TURN_CANDIDATES) return;
+ const expanded = expandHome(raw, home);
+ if (expanded) out.push({ path: resolve(cwd, expanded), glob, ...window });
+ };
+ for (const word of shellGlobCandidates(command)) push(word, true);
+ for (const word of shellPathCandidates(command)) push(word, false);
+ }
+ }
+ }
+ return out;
+}
+
+/** One glob component as a regular expression (`*`, `?`, `[...]`/`[!...]`), or null when it has no glob. */
+function componentPattern(component: string): RegExp | null {
+ if (!/[*?[]/.test(component)) return null;
+ let source = "";
+ for (let index = 0; index < component.length; index += 1) {
+ const char = component[index]!;
+ if (char === "*") source += "[^/]*";
+ else if (char === "?") source += "[^/]";
+ else if (char === "[") {
+ const close = component.indexOf("]", index + 2);
+ if (close < 0) {
+ source += "\\[";
+ continue;
+ }
+ let body = component.slice(index + 1, close);
+ const negated = body.startsWith("!") || body.startsWith("^");
+ if (negated) body = body.slice(1);
+ source += `[${negated ? "^" : ""}${body.replace(/[\\\]^]/g, "\\$&")}]`;
+ index = close;
+ } else source += char.replace(/[.+^${}()|\\]/g, "\\$&");
+ }
+ try {
+ return new RegExp(`^${source}$`, "u");
+ } catch {
+ return null;
+ }
+}
+
+type ExpansionContext = {
+ roots: string[];
+ home: string | null;
+ temps: string[];
+ exclusion: (absolute: string) => unknown;
+ budget: { entries: number };
+};
+
+function tempFolders(): string[] {
+ const out = new Set<string>();
+ for (const value of [tmpdir(), process.env.TMPDIR, process.env.TEMP, process.env.TMP, "/tmp", "/var/tmp", "/private/tmp", "/private/var/tmp"]) {
+ if (value && isAbsolute(value)) out.add(resolve(value));
+ }
+ return [...out];
+}
+
+/**
+ * A folder that is never walked: the file system root or a top-level
+ * folder, the home folder or an ancestor of it, a temp root itself, a system
+ * folder, the workspace or an ancestor of it, a secret store, a folder the
+ * archive excludes (app state, dependency/cache folders).
+ */
+function unsafeFolder(dir: string, context: ExpansionContext): boolean {
+ const portable = dir.replaceAll("\\", "/");
+ if (resolve(dir) === resolve(dir, "/") || /^[A-Za-z]:[\\/]?$/.test(dir) || /^\/[^/]*$/.test(portable) || /^[A-Za-z]:[\\/][^\\/]*$/.test(dir)) return true;
+ if (context.home && (resolve(dir) === resolve(context.home) || within(dir, context.home) !== null)) return true;
+ if (context.temps.some((temp) => resolve(temp) === resolve(dir) || within(dir, temp) !== null)) return true;
+ if (/^(?:\/private)?\/var\/folders\/[^/]+\/[^/]+(?:\/T)?$/.test(portable)) return true;
+ if (sep === "/" && SYSTEM_FOLDER.test(portable)) return true;
+ if (/^[A-Za-z]:\\(?:Windows|Program Files|Program Files \(x86\)|ProgramData)(?:\\|$)/i.test(dir)) return true;
+ if (context.roots.some((root) => resolve(root) === resolve(dir) || within(dir, root) !== null)) return true;
+ if (isSecretFolder(dir)) return true;
+ return context.exclusion(dir) !== null;
+}
+
+/** The files a glob matches (no dotfiles unless the component starts with `.`; symlinks never followed). */
+async function expandGlob(pattern: string, context: ExpansionContext): Promise<string[]> {
+ const parts = pattern.split(sep === "\\" ? /[\\/]/ : "/");
+ const first = parts.findIndex((part) => /[*?[]/.test(part));
+ if (first <= 0) return [];
+ const base = parts.slice(0, first).join(sep) || sep;
+ let realBase: string;
+ try {
+ realBase = await realpath(base);
+ } catch {
+ return [];
+ }
+ if (unsafeFolder(realBase, context)) return [];
+ const rest = parts.slice(first);
+ const found: string[] = [];
+ const walk = async (dir: string, index: number, depth: number): Promise<void> => {
+ if (found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0 || depth > MAX_EXPANSION_DEPTH * 2) return;
+ const component = rest[index];
+ if (component === undefined) return;
+ const last = index === rest.length - 1;
+ if (component === "**") {
+ // Zero folders, then each folder below (bounded).
+ await walk(dir, index + 1, depth + 1);
+ if (depth >= MAX_EXPANSION_DEPTH) return;
+ }
+ const literal = component !== "**" && !/[*?[]/.test(component);
+ if (literal) {
+ const next = join(dir, component);
+ if (last) found.push(next);
+ else {
+ try {
+ const stats = await lstat(next);
+ if (stats.isDirectory() && !SKIPPED_EXPANSION_DIRS.has(component)) await walk(next, index + 1, depth + 1);
+ } catch {
+ // Not there.
+ }
+ }
+ return;
+ }
+ const matcher = component === "**" ? null : componentPattern(component);
+ let children;
+ try {
+ children = await readdir(dir, { withFileTypes: true });
+ } catch {
+ return;
+ }
+ children.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+ for (const child of children) {
+ if (found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0) return;
+ context.budget.entries -= 1;
+ if (child.name.startsWith(".") && !(component.startsWith("."))) continue;
+ const path = join(dir, child.name);
+ if (component === "**") {
+ if (child.isDirectory() && !SKIPPED_EXPANSION_DIRS.has(child.name) && !isSecretFolder(child.name)) await walk(path, index, depth + 1);
+ continue;
+ }
+ if (!matcher?.test(child.name)) continue;
+ if (last) {
+ if (child.isFile()) found.push(path);
+ } else if (child.isDirectory() && !SKIPPED_EXPANSION_DIRS.has(child.name) && !isSecretFolder(child.name)) {
+ await walk(path, index + 1, depth + 1);
+ }
+ }
+ };
+ await walk(realBase, 0, 0);
+ return found;
+}
+
+/**
+ * The files under an outside folder that were read or modified during a
+ * tool call: atime or mtime within [start - slack, end + slack]. Walked
+ * without following symlinks, regenerable and secret folders skipped.
+ */
+async function changedDuringCall(dir: string, start: number, end: number, context: ExpansionContext): Promise<string[]> {
+ const found: string[] = [];
+ const low = start - CALL_WINDOW_SLACK_MS;
+ const high = end + CALL_WINDOW_SLACK_MS;
+ const walk = async (folder: string, depth: number): Promise<void> => {
+ if (depth > MAX_EXPANSION_DEPTH || found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0) return;
+ let children;
+ try {
+ children = await readdir(folder, { withFileTypes: true });
+ } catch {
+ return;
+ }
+ children.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+ for (const child of children) {
+ if (found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0) return;
+ context.budget.entries -= 1;
+ const path = join(folder, child.name);
+ if (child.isDirectory()) {
+ if (!SKIPPED_EXPANSION_DIRS.has(child.name) && !isSecretFolder(child.name)) await walk(path, depth + 1);
+ continue;
+ }
+ if (!child.isFile()) continue;
+ try {
+ const stats = await lstat(path);
+ const used = (stats.atimeMs >= low && stats.atimeMs <= high) || (stats.mtimeMs >= low && stats.mtimeMs <= high);
+ if (stats.isFile() && used) found.push(path);
+ } catch {
+ // Gone.
+ }
+ }
+ };
+ await walk(dir, 0);
+ return found;
+}
+
+/**
+ * The files a turn's shell commands reached without naming them: glob
+ * matches, and the files read or modified during the call under an outside
+ * folder the command names (or the folder of an outside file it names).
+ * Absolute, deduplicated, bounded. Never throws.
+ */
+export async function expandShellTargets(
+ targets: readonly ShellTarget[],
+ root: string,
+ options: { home?: string | null; exclusion?: (absolute: string) => unknown; tempRoots?: readonly string[] } = {},
+): Promise<string[]> {
+ const out = new Set<string>();
+ try {
+ const home = options.home === undefined ? safeHome() : options.home;
+ const roots = [resolve(root)];
+ try {
+ roots.push(await realpath(root));
+ } catch {
+ // Gone: the root as given.
+ }
+ const context: ExpansionContext = {
+ roots,
+ home,
+ temps: [...(options.tempRoots ?? tempFolders())],
+ exclusion: options.exclusion ?? (() => null),
+ budget: { entries: MAX_EXPANSION_ENTRIES },
+ };
+ const insideRoot = (path: string) => roots.some((candidate) => within(candidate, path) !== null);
+ const scanned = new Set<string>();
+ let expansions = 0;
+ for (const target of targets) {
+ if (expansions >= MAX_TURN_EXPANSIONS || context.budget.entries <= 0) break;
+ if (target.glob) {
+ expansions += 1;
+ for (const file of await expandGlob(target.path, context)) out.add(file);
+ continue;
+ }
+ // A folder (or the folder of a file) outside the workspace, with the call's window: files used during the call.
+ if (target.start === null || insideRoot(target.path)) continue;
+ let stats;
+ try {
+ stats = await lstat(target.path);
+ } catch {
+ continue;
+ }
+ let dir: string;
+ if (stats.isDirectory()) dir = target.path;
+ else if (stats.isFile()) dir = dirname(target.path);
+ else continue;
+ try {
+ dir = await realpath(dir);
+ } catch {
+ continue;
+ }
+ if (scanned.has(dir) || insideRoot(dir) || unsafeFolder(dir, context)) continue;
+ if (isHomeSettingsFolder(dir, home)) continue;
+ scanned.add(dir);
+ expansions += 1;
+ const end = target.end ?? target.start;
+ for (const file of await changedDuringCall(dir, target.start, Math.max(end, target.start), context)) out.add(file);
+ }
+ } catch {
+ // Best effort.
+ }
+ return [...out];
+}
+
/** Every file path a turn's tool calls name, absolute, with how it was touched (deduplicated, bounded). */
export function turnFileEvidence(messages: unknown, root: string, home: string | null = safeHome()): TurnFileEvidence[] {
const found = new Map<string, TurnFileAccess>();
@@ -253,6 +572,18 @@ export async function recordTurnFiles(input: TurnFilesInput): Promise<TurnFilesR
try {
const home = input.home === undefined ? safeHome() : input.home;
const evidence = turnFileEvidence(input.messages, input.root, home);
+ // Files reached through a glob, a folder or a script the commands named (bounded; see expandShellTargets).
+ const named = new Set(evidence.map((item) => item.path));
+ const expansionDirs = (input.excludedDirs ?? []).map((dir) => resolve(dir));
+ const expanded = await expandShellTargets(turnShellTargets(input.messages, input.root, home), input.root, {
+ home,
+ exclusion: (absolute) => outsideExclusion(absolute, { appDirs: expansionDirs, home, includeCredentialFiles: input.includeCredentialFiles === true }),
+ });
+ for (const path of expanded) {
+ if (named.has(path) || evidence.length >= MAX_TURN_CANDIDATES + MAX_EXPANSION_FILES) continue;
+ named.add(path);
+ evidence.push({ path, access: "shell" });
+ }
if (evidence.length === 0) return result;
const roots = new Set([resolve(input.root)]);
try {