A desktop coding agent with free access to frontier models.
TypeScript
fix(capture): archive outside files a shell glob, loop or find reached, and files used during the call under named folders
ogsamrat7ac401ad5b0c
diff --git a/apps/server/src/session-archive/outside.test.ts b/apps/server/src/session-archive/outside.test.tsindex 169a810e..54e736e3 100644--- a/apps/server/src/session-archive/outside.test.ts+++ b/apps/server/src/session-archive/outside.test.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, test } from "bun:test";import { execFile } from "node:child_process";import { createHash, randomBytes } from "node:crypto";-import { mkdir, rm, symlink, writeFile } from "node:fs/promises";+import { mkdir, rm, symlink, utimes, writeFile } from "node:fs/promises";import { dirname, join } from "node:path";import { promisify } from "node:util";@@ -10,7 +10,7 @@ import { SessionArchiver } from "./index.js";import { outsideArchivePath, outsideSourcePath } from "./outside.js";import { cleanupTempDirs, manifestOf, openArchive, tempDir } from "./test-helpers.js";import { TouchedPathStore } from "./touched.js";-import { recordTurnFiles, shellPathCandidates } from "./turn-files.js";+import { expandShellTargets, recordTurnFiles, shellGlobCandidates, shellPathCandidates } from "./turn-files.js";// Files outside the session's workspace that the agent touches are archived// byte for byte under `__outside__/<absolute path>`, with the archive's@@ -178,4 +178,73 @@ describe("files outside the workspace", () => {expect(byPath.has(".env.local")).toBe(false);expect(traced).toHaveLength(3);});++ test("shell glob words that look like paths are glob candidates; variables, braces and bare patterns are not", () => {+ const found = shellGlobCandidates(`for f in /tmp/fx/*; do wc -c "$f"; done; cat dir/*.txt ~/x/**/*.csv; find /data -name '*.log' -exec cat {} \\; ; ls $HOME/*.md /a/{b,c}/*`);+ expect(found.sort()).toEqual(["/tmp/fx/*", "dir/*.txt", "~/x/**/*.csv"].sort());+ });++ test("files a shell glob, a loop or `find -exec` reached are archived; files used during the call under a named folder too", async () => {+ const root = await gitProject();+ const away = await tempDir("outside-glob");+ const home = join(away, "home");+ const server = new FakeArchiveServer();+ const stateDir = await tempDir("outside-glob-state");+ const archive = archiver(server, stateDir);+ const sessionId = "ses_outside_glob_1";+ expect((await archive.captureBase(sessionId, root)).status).toBe("queued");+ const past = new Date(Date.now() - 3_600_000);+ const fixtures = join(away, "fixtures");+ const looped: Array<[string, Buffer]> = [];+ for (const name of ["a.bin", "b.txt", "c.gz"]) looped.push([name, await write(join(fixtures, name), randomBytes(500))]);+ await write(join(fixtures, ".env"), "API_KEY=sk-secret\n");+ await write(join(fixtures, "key.pem"), "-----BEGIN PRIVATE KEY-----\n");+ await symlink(join(fixtures, "a.bin"), join(fixtures, "link.bin"));+ for (const name of ["one.txt", "two.txt"]) await write(join(away, "notes", name), `${name}\n`);+ await write(join(away, "notes", "skip.md"), "not matched\n");+ const logs = join(away, "logs");+ await write(join(logs, "deep", "used.log"), "used\n");+ await write(join(logs, "untouched.log"), "untouched\n");+ await write(join(logs, "node_modules", "dep.log"), "dep\n");+ for (const file of ["deep/used.log", "untouched.log", "node_modules/dep.log"]) await utimes(join(logs, file), past, past);+ const start = Date.now();+ await utimes(join(logs, "deep", "used.log"), new Date(start + 100), past);+ await write(join(home, "secret-notes.txt"), "home\n");+ const messages = [{+ info: { id: "msg_1", role: "assistant" },+ parts: [+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `for f in ${fixtures}/*; do wc -c "$f"; done` }, time: { start: start - 60_000, end: start - 59_000 } } },+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `cat ${away}/notes/*.txt ~/*.txt` } } },+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `find ${logs} -name '*.log' -exec cat {} \\;` }, time: { start, end: start + 500 } } },+ ],+ }];+ const reported: string[] = [];+ await recordTurnFiles({+ sessionId, root, messages, home,+ collector: { recordTrace: () => undefined },+ archive: { pathTouched: (id, file) => { reported.push(file); archive.recordTouched(id, file); } },+ });+ const expected = [...looped.map(([name]) => join(fixtures, name)), join(away, "notes", "one.txt"), join(away, "notes", "two.txt"), join(logs, "deep", "used.log")];+ expect(new Set(reported)).toEqual(new Set(expected));+ expect((await archive.captureDelta(sessionId, root, 1)).status).toBe("queued");+ await archive.drain();+ const delta = await lastArchive(server);+ for (const [name, bytes] of looped) expect(delta.names.get(encoded(join(fixtures, name)))!.equals(bytes)).toBe(true);+ expect(delta.names.get(encoded(join(logs, "deep", "used.log")))!.toString()).toBe("used\n");+ expect([...delta.names.keys()].some((name) => /\.env|key\.pem|link\.bin|untouched|node_modules|skip\.md|secret-notes/.test(name))).toBe(false);+ await archive.stop();+ });++ test("glob and folder expansion never walks a temp root, the home folder, the file system root or the workspace", async () => {+ const root = await tempDir("expand-root");+ const away = await tempDir("expand-away");+ await write(join(away, "a.txt"), "a\n");+ await write(join(root, "data", "x.csv"), "x\n");+ const now = Date.now();+ const targets = [`${away}/*`, "/*", "/etc/*", `${root}/../*`].map((path) => ({ path, glob: true, start: null as number | null, end: null as number | null }));+ targets.push({ path: away, glob: false, start: now - 1000, end: now + 1000 }, { path: "/", glob: false, start: now, end: now });+ expect(await expandShellTargets(targets, root, { home: away, tempRoots: [] })).toEqual([]);+ expect(await expandShellTargets(targets, root, { home: null, tempRoots: [away] })).toEqual([]);+ expect(await expandShellTargets([{ path: `${root}/data/*.csv`, glob: true, start: null, end: null }], root, { home: null, tempRoots: [] })).toEqual([join(root, "data", "x.csv")]);+ });});diff --git a/apps/server/src/session-archive/turn-files.ts b/apps/server/src/session-archive/turn-files.tsindex b489571f..d72cb1b3 100644--- a/apps/server/src/session-archive/turn-files.ts+++ b/apps/server/src/session-archive/turn-files.ts@@ -14,17 +14,29 @@* show, still says which files it read. Paths are workspace-relative, or* the `__outside__/...` archive path (`outside: true`).*+ * A shell command that reaches files without naming each one is expanded+ * too (expandShellTargets): a glob word (`for f in /data/*`, `cat dir/*.txt`)+ * names every file it matches, and an outside folder the command names+ * (`find /data -exec ...`, `ls /data`, the folder of an outside script it+ * ran) contributes the files read or modified during the call (atime or+ * mtime within the tool call's time window; best effort where the file+ * system does not update atime). Folders are walked without following+ * symlinks, within caps, and never the file system root, the home folder,+ * a temp root, a system folder, an ancestor of the workspace, or a folder+ * the archive excludes.+ ** The archive's exclusions apply to both: credential files and stores, the* app's state, dependency and cache folders, symlinks, special files, and* files over MAX_OUTSIDE_FILE_BYTES.*/import { createHash } from "node:crypto";-import { lstat, open, realpath } from "node:fs/promises";-import { homedir } from "node:os";+import { lstat, open, readdir, realpath } from "node:fs/promises";+import { homedir, tmpdir } from "node:os";import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";import { HASH_READ_BYTES, OPEN_ENTRY_FLAGS, isArchiveCredentialPath } from "./manifest.js";import { MAX_OUTSIDE_FILE_BYTES, outsideArchivePath, outsideExclusion } from "./outside.js";+import { isHomeSettingsFolder, isSecretFolder } from "../context/privacy.js";export type TurnFileAccess = "read" | "write" | "shell";export type TurnFileEvidence = { path: string; access: TurnFileAccess };@@ -43,6 +55,16 @@ const MAX_TURN_CANDIDATES = 1_024;const MAX_TURN_FILE_EVENTS = 500;const MAX_PATH_CHARS = 4_096;const MAX_COMMAND_CHARS = 256 * 1024;+/** Shell expansion caps (per turn): globs and folders expanded, files one glob or folder adds, directory entries read. */+const MAX_TURN_EXPANSIONS = 32;+const MAX_EXPANSION_FILES = 256;+const MAX_EXPANSION_ENTRIES = 20_000;+const MAX_EXPANSION_DEPTH = 6;+/** Clock and timestamp granularity slack around a tool call's window. */+const CALL_WINDOW_SLACK_MS = 2_000;+/** Folders never walked when a command names them or a glob starts in them. */+const SKIPPED_EXPANSION_DIRS = new Set([".git", ".hg", ".svn", "node_modules", ".venv", "venv", "__pycache__", ".tox", ".mypy_cache", ".pytest_cache", ".cache"]);+const SYSTEM_FOLDER = /^\/(?:usr|etc|bin|sbin|lib|lib32|lib64|libx32|proc|sys|dev|run|boot|var\/lib|var\/log|var\/cache|snap|System|Library|private\/etc|private\/var\/db|Applications)(?:\/|$)/;function isRecord(value: unknown): value is Record<string, unknown> {return typeof value === "object" && value !== null && !Array.isArray(value);@@ -147,6 +169,303 @@ export function shellPathCandidates(command: string): string[] {return [...out];}+/**+ * Glob words of a shell command that look like paths (`/data/*`,+ * `dir/*.txt`, `~/x/**\/*.csv`): `*`, `?` or `[...]` in a word with a `/`.+ * Words with a variable, a command substitution or a brace are left out.+ */+export function shellGlobCandidates(command: string): string[] {+ if (command.length > MAX_COMMAND_CHARS) command = command.slice(0, MAX_COMMAND_CHARS);+ const out = new Set<string>();+ for (const word of shellWords(command)) {+ if (word.startsWith("-") || word.length > MAX_PATH_CHARS || word.includes("://") || /[$`{}]/.test(word)) continue;+ if (!word.includes("/") || !/[*?[]/.test(word)) continue;+ out.add(word);+ }+ return [...out];+}++/** A shell command's glob and folder targets, absolute, with the call's time window (ms) when the engine recorded it. */+export type ShellTarget = { path: string; glob: boolean; start: number | null; end: number | null };++function callWindow(state: Record<string, unknown>): { start: number | null; end: number | null } {+ const time = isRecord(state.time) ? state.time : {};+ const start = typeof time.start === "number" && Number.isFinite(time.start) && time.start > 0 ? time.start : null;+ const end = typeof time.end === "number" && Number.isFinite(time.end) && time.end > 0 ? time.end : null;+ return { start, end };+}++/** The glob words and path words (folder candidates) of a turn's shell commands, absolute, with each call's window. */+export function turnShellTargets(messages: unknown, root: string, home: string | null = safeHome()): ShellTarget[] {+ const out: ShellTarget[] = [];+ const list = Array.isArray(messages) ? messages : isRecord(messages) && Array.isArray(messages.messages) ? messages.messages : [];+ for (const message of list) {+ if (!isRecord(message) || !Array.isArray(message.parts)) continue;+ for (const part of message.parts) {+ if (!isRecord(part) || part.type !== "tool") continue;+ const name = toolName(part.tool ?? part.name);+ if (!name || !SHELL_TOOLS.has(name)) continue;+ const state = isRecord(part.state) ? part.state : {};+ const input = isRecord(state.input) ? state.input : isRecord(part.input) ? part.input : null;+ if (!input) continue;+ const cwdValue = CWD_KEYS.map((key) => cleanPath(input[key])).find(Boolean);+ const cwd = cwdValue ? resolve(root, expandHome(cwdValue, home) ?? ".") : root;+ const window = callWindow(state);+ for (const key of COMMAND_KEYS) {+ const value = input[key];+ const command = typeof value === "string" ? value : Array.isArray(value) && value.every((item) => typeof item === "string") ? value.join(" ") : null;+ if (!command) continue;+ const push = (raw: string, glob: boolean) => {+ if (out.length >= MAX_TURN_CANDIDATES) return;+ const expanded = expandHome(raw, home);+ if (expanded) out.push({ path: resolve(cwd, expanded), glob, ...window });+ };+ for (const word of shellGlobCandidates(command)) push(word, true);+ for (const word of shellPathCandidates(command)) push(word, false);+ }+ }+ }+ return out;+}++/** One glob component as a regular expression (`*`, `?`, `[...]`/`[!...]`), or null when it has no glob. */+function componentPattern(component: string): RegExp | null {+ if (!/[*?[]/.test(component)) return null;+ let source = "";+ for (let index = 0; index < component.length; index += 1) {+ const char = component[index]!;+ if (char === "*") source += "[^/]*";+ else if (char === "?") source += "[^/]";+ else if (char === "[") {+ const close = component.indexOf("]", index + 2);+ if (close < 0) {+ source += "\\[";+ continue;+ }+ let body = component.slice(index + 1, close);+ const negated = body.startsWith("!") || body.startsWith("^");+ if (negated) body = body.slice(1);+ source += `[${negated ? "^" : ""}${body.replace(/[\\\]^]/g, "\\$&")}]`;+ index = close;+ } else source += char.replace(/[.+^${}()|\\]/g, "\\$&");+ }+ try {+ return new RegExp(`^${source}$`, "u");+ } catch {+ return null;+ }+}++type ExpansionContext = {+ roots: string[];+ home: string | null;+ temps: string[];+ exclusion: (absolute: string) => unknown;+ budget: { entries: number };+};++function tempFolders(): string[] {+ const out = new Set<string>();+ for (const value of [tmpdir(), process.env.TMPDIR, process.env.TEMP, process.env.TMP, "/tmp", "/var/tmp", "/private/tmp", "/private/var/tmp"]) {+ if (value && isAbsolute(value)) out.add(resolve(value));+ }+ return [...out];+}++/**+ * A folder that is never walked: the file system root or a top-level+ * folder, the home folder or an ancestor of it, a temp root itself, a system+ * folder, the workspace or an ancestor of it, a secret store, a folder the+ * archive excludes (app state, dependency/cache folders).+ */+function unsafeFolder(dir: string, context: ExpansionContext): boolean {+ const portable = dir.replaceAll("\\", "/");+ if (resolve(dir) === resolve(dir, "/") || /^[A-Za-z]:[\\/]?$/.test(dir) || /^\/[^/]*$/.test(portable) || /^[A-Za-z]:[\\/][^\\/]*$/.test(dir)) return true;+ if (context.home && (resolve(dir) === resolve(context.home) || within(dir, context.home) !== null)) return true;+ if (context.temps.some((temp) => resolve(temp) === resolve(dir) || within(dir, temp) !== null)) return true;+ if (/^(?:\/private)?\/var\/folders\/[^/]+\/[^/]+(?:\/T)?$/.test(portable)) return true;+ if (sep === "/" && SYSTEM_FOLDER.test(portable)) return true;+ if (/^[A-Za-z]:\\(?:Windows|Program Files|Program Files \(x86\)|ProgramData)(?:\\|$)/i.test(dir)) return true;+ if (context.roots.some((root) => resolve(root) === resolve(dir) || within(dir, root) !== null)) return true;+ if (isSecretFolder(dir)) return true;+ return context.exclusion(dir) !== null;+}++/** The files a glob matches (no dotfiles unless the component starts with `.`; symlinks never followed). */+async function expandGlob(pattern: string, context: ExpansionContext): Promise<string[]> {+ const parts = pattern.split(sep === "\\" ? /[\\/]/ : "/");+ const first = parts.findIndex((part) => /[*?[]/.test(part));+ if (first <= 0) return [];+ const base = parts.slice(0, first).join(sep) || sep;+ let realBase: string;+ try {+ realBase = await realpath(base);+ } catch {+ return [];+ }+ if (unsafeFolder(realBase, context)) return [];+ const rest = parts.slice(first);+ const found: string[] = [];+ const walk = async (dir: string, index: number, depth: number): Promise<void> => {+ if (found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0 || depth > MAX_EXPANSION_DEPTH * 2) return;+ const component = rest[index];+ if (component === undefined) return;+ const last = index === rest.length - 1;+ if (component === "**") {+ // Zero folders, then each folder below (bounded).+ await walk(dir, index + 1, depth + 1);+ if (depth >= MAX_EXPANSION_DEPTH) return;+ }+ const literal = component !== "**" && !/[*?[]/.test(component);+ if (literal) {+ const next = join(dir, component);+ if (last) found.push(next);+ else {+ try {+ const stats = await lstat(next);+ if (stats.isDirectory() && !SKIPPED_EXPANSION_DIRS.has(component)) await walk(next, index + 1, depth + 1);+ } catch {+ // Not there.+ }+ }+ return;+ }+ const matcher = component === "**" ? null : componentPattern(component);+ let children;+ try {+ children = await readdir(dir, { withFileTypes: true });+ } catch {+ return;+ }+ children.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));+ for (const child of children) {+ if (found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0) return;+ context.budget.entries -= 1;+ if (child.name.startsWith(".") && !(component.startsWith("."))) continue;+ const path = join(dir, child.name);+ if (component === "**") {+ if (child.isDirectory() && !SKIPPED_EXPANSION_DIRS.has(child.name) && !isSecretFolder(child.name)) await walk(path, index, depth + 1);+ continue;+ }+ if (!matcher?.test(child.name)) continue;+ if (last) {+ if (child.isFile()) found.push(path);+ } else if (child.isDirectory() && !SKIPPED_EXPANSION_DIRS.has(child.name) && !isSecretFolder(child.name)) {+ await walk(path, index + 1, depth + 1);+ }+ }+ };+ await walk(realBase, 0, 0);+ return found;+}++/**+ * The files under an outside folder that were read or modified during a+ * tool call: atime or mtime within [start - slack, end + slack]. Walked+ * without following symlinks, regenerable and secret folders skipped.+ */+async function changedDuringCall(dir: string, start: number, end: number, context: ExpansionContext): Promise<string[]> {+ const found: string[] = [];+ const low = start - CALL_WINDOW_SLACK_MS;+ const high = end + CALL_WINDOW_SLACK_MS;+ const walk = async (folder: string, depth: number): Promise<void> => {+ if (depth > MAX_EXPANSION_DEPTH || found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0) return;+ let children;+ try {+ children = await readdir(folder, { withFileTypes: true });+ } catch {+ return;+ }+ children.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));+ for (const child of children) {+ if (found.length >= MAX_EXPANSION_FILES || context.budget.entries <= 0) return;+ context.budget.entries -= 1;+ const path = join(folder, child.name);+ if (child.isDirectory()) {+ if (!SKIPPED_EXPANSION_DIRS.has(child.name) && !isSecretFolder(child.name)) await walk(path, depth + 1);+ continue;+ }+ if (!child.isFile()) continue;+ try {+ const stats = await lstat(path);+ const used = (stats.atimeMs >= low && stats.atimeMs <= high) || (stats.mtimeMs >= low && stats.mtimeMs <= high);+ if (stats.isFile() && used) found.push(path);+ } catch {+ // Gone.+ }+ }+ };+ await walk(dir, 0);+ return found;+}++/**+ * The files a turn's shell commands reached without naming them: glob+ * matches, and the files read or modified during the call under an outside+ * folder the command names (or the folder of an outside file it names).+ * Absolute, deduplicated, bounded. Never throws.+ */+export async function expandShellTargets(+ targets: readonly ShellTarget[],+ root: string,+ options: { home?: string | null; exclusion?: (absolute: string) => unknown; tempRoots?: readonly string[] } = {},+): Promise<string[]> {+ const out = new Set<string>();+ try {+ const home = options.home === undefined ? safeHome() : options.home;+ const roots = [resolve(root)];+ try {+ roots.push(await realpath(root));+ } catch {+ // Gone: the root as given.+ }+ const context: ExpansionContext = {+ roots,+ home,+ temps: [...(options.tempRoots ?? tempFolders())],+ exclusion: options.exclusion ?? (() => null),+ budget: { entries: MAX_EXPANSION_ENTRIES },+ };+ const insideRoot = (path: string) => roots.some((candidate) => within(candidate, path) !== null);+ const scanned = new Set<string>();+ let expansions = 0;+ for (const target of targets) {+ if (expansions >= MAX_TURN_EXPANSIONS || context.budget.entries <= 0) break;+ if (target.glob) {+ expansions += 1;+ for (const file of await expandGlob(target.path, context)) out.add(file);+ continue;+ }+ // A folder (or the folder of a file) outside the workspace, with the call's window: files used during the call.+ if (target.start === null || insideRoot(target.path)) continue;+ let stats;+ try {+ stats = await lstat(target.path);+ } catch {+ continue;+ }+ let dir: string;+ if (stats.isDirectory()) dir = target.path;+ else if (stats.isFile()) dir = dirname(target.path);+ else continue;+ try {+ dir = await realpath(dir);+ } catch {+ continue;+ }+ if (scanned.has(dir) || insideRoot(dir) || unsafeFolder(dir, context)) continue;+ if (isHomeSettingsFolder(dir, home)) continue;+ scanned.add(dir);+ expansions += 1;+ const end = target.end ?? target.start;+ for (const file of await changedDuringCall(dir, target.start, Math.max(end, target.start), context)) out.add(file);+ }+ } catch {+ // Best effort.+ }+ return [...out];+}+/** Every file path a turn's tool calls name, absolute, with how it was touched (deduplicated, bounded). */export function turnFileEvidence(messages: unknown, root: string, home: string | null = safeHome()): TurnFileEvidence[] {const found = new Map<string, TurnFileAccess>();@@ -253,6 +572,18 @@ export async function recordTurnFiles(input: TurnFilesInput): Promise<TurnFilesRtry {const home = input.home === undefined ? safeHome() : input.home;const evidence = turnFileEvidence(input.messages, input.root, home);+ // Files reached through a glob, a folder or a script the commands named (bounded; see expandShellTargets).+ const named = new Set(evidence.map((item) => item.path));+ const expansionDirs = (input.excludedDirs ?? []).map((dir) => resolve(dir));+ const expanded = await expandShellTargets(turnShellTargets(input.messages, input.root, home), input.root, {+ home,+ exclusion: (absolute) => outsideExclusion(absolute, { appDirs: expansionDirs, home, includeCredentialFiles: input.includeCredentialFiles === true }),+ });+ for (const path of expanded) {+ if (named.has(path) || evidence.length >= MAX_TURN_CANDIDATES + MAX_EXPANSION_FILES) continue;+ named.add(path);+ evidence.push({ path, access: "shell" });+ }if (evidence.length === 0) return result;const roots = new Set([resolve(input.root)]);try {