A desktop coding agent with free access to frontier models.
TypeScript
/**
* Files outside the session's workspace that the agent touched (read,
* wrote or edited with its file tools, or named in a shell command it ran).
* They are archived byte for byte with the session's project archive, under
* `__outside__/<absolute path>`: `/tmp/x/a.bin` is `__outside__/tmp/x/a.bin`,
* `C:\data\a.bin` is `__outside__/C/data/a.bin`. Only the specific files
* touched are ever looked at (a touched folder is never expanded), with the
* archive's exclusions: credential files and stores, the app's own state
* directories, dependency and cache folders, the kernel's pseudo file
* systems, symlinks (never followed), special files, and files larger than
* MAX_OUTSIDE_FILE_BYTES.
*/
import { createHash } from "node:crypto";
import type { BigIntStats } from "node:fs";
import { lstat, open, realpath } from "node:fs/promises";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import {
HASH_CONCURRENCY,
HASH_READ_BYTES,
OPEN_ENTRY_FLAGS,
OUTSIDE_ROOT_NAME,
STAT_CONCURRENCY,
compareArchivePaths,
emptyExcludedCounts,
forEachBounded,
isArchiveCredentialPath,
isSameFileIdentity,
statFields,
type ArchiveHashCache,
type ScannedEntry,
} from "./manifest.js";
import type { TouchedScanResult } from "./touched.js";
import { isHomeSettingsFolder, isSecretFile } from "../context/privacy.js";
import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";
/** The regenerable folders that hold build output: a file there may be the agent's own (not a dependency or cache). */
const BUILD_OUTPUT_DIR_NAMES = new Set(["dist", "build", "target", "out", ".next", ".nuxt", ".svelte-kit"]);
/** Whether a portable path lies in a dependency or cache folder (regenerable, not build output), or is `.git/` content. */
function inDependencyDir(path: string): boolean {
return path.split("/").slice(0, -1).some((part) => part === ".git" || (REGENERABLE_DIR_NAMES.has(part) && !BUILD_OUTPUT_DIR_NAMES.has(part)));
}
export { OUTSIDE_ROOT_NAME };
/** An outside file larger than this is left out (1 GiB, the cap of a session's own build output). */
export const MAX_OUTSIDE_FILE_BYTES = 1024 * 1024 * 1024;
const MAX_OUTSIDE_PATH_CHARS = 4_096;
/** Pseudo and device file systems: nothing under them is a file the agent made or read as data. */
const SYSTEM_PREFIXES = ["/proc", "/sys", "/dev", "/run", "/private/var/run", "/System/Volumes"];
/**
* Folders under the home directory that only hold caches, package stores or
* other apps' state (dependency and cache folders, beyond the per-component
* names inDependencyDir knows).
*/
const HOME_CACHE_DIRS = [
".cache", ".npm", ".pnpm-store", ".yarn", ".bun/install", ".cargo/registry", ".cargo/git", ".rustup", ".gradle", ".m2",
".nuget", ".local/share/pnpm", ".local/share/opencode", ".local/state/opencode", ".config/opencode", ".omnirush", ".pi",
".config/omnirush", ".config/OmniRush", "Library/Application Support/OmniRush", "AppData/Roaming/OmniRush",
"Library/Caches", "AppData/Local/Temp", "AppData/Local/npm-cache", "go/pkg/mod",
];
const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
export function isOutsideArchivePath(path: string): boolean {
return path.startsWith(`${OUTSIDE_ROOT_NAME}/`);
}
/**
* The archive path of an absolute file path, or null when it cannot be one
* (relative, a UNC share or device path, `.`/`..`/empty components, NUL).
* `win32` forces the Windows form (tests).
*/
export function outsideArchivePath(absolute: string, win32 = sep === "\\"): string | null {
if (!absolute || absolute.length > MAX_OUTSIDE_PATH_CHARS || absolute.includes("\0") || LONE_SURROGATE.test(absolute)) return null;
let parts: string[];
if (win32) {
const match = /^([A-Za-z]):[\\/](.*)$/.exec(absolute);
if (!match) return null;
parts = [match[1]!.toUpperCase(), ...match[2]!.split(/[\\/]/)];
if (parts.slice(1).some((part) => part.includes(":"))) return null;
} else {
if (!absolute.startsWith("/")) return null;
parts = absolute.slice(1).split("/");
}
if (parts.length < 2 || parts.some((part) => part === "" || part === "." || part === "..")) return null;
return `${OUTSIDE_ROOT_NAME}/${parts.join("/")}`;
}
/** The absolute file path an outside archive path stands for (the inverse of outsideArchivePath), or null. */
export function outsideSourcePath(archivePath: string, win32 = sep === "\\"): string | null {
if (!isOutsideArchivePath(archivePath)) return null;
const parts = archivePath.slice(OUTSIDE_ROOT_NAME.length + 1).split("/");
if (parts.length < 2 || parts.some((part) => part === "" || part === "." || part === "..")) return null;
if (win32) {
if (!/^[A-Z]$/.test(parts[0]!)) return null;
return `${parts[0]}:\\${parts.slice(1).join("\\")}`;
}
return `/${parts.join("/")}`;
}
function inside(parent: string, child: string): boolean {
const rel = relative(parent, child);
return rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel));
}
/** Forms of a directory to compare against: as given and its real path. */
async function forms(dir: string): Promise<string[]> {
const out = new Set([resolve(dir)]);
try {
out.add(await realpath(dir));
} catch {
// Not there (yet): only its resolved form can contain anything.
}
return [...out];
}
export type OutsideScanOptions = {
/** The workspace root: a path inside it is a workspace file, never an outside one. */
root: string;
/** Absolute app state/temp/data directories: nothing under them is archived. */
excludedDirs?: readonly string[];
/** archiveIncludeCredentialFiles: turns the credential filter off entirely. */
includeCredentialFiles?: boolean;
/** Hash cache for outside files (looked up and added to, never pruned). */
hashCache?: ArchiveHashCache;
signal?: AbortSignal;
/** The home directory whose cache folders are excluded (os.homedir() by default). */
home?: string;
/** Tests: every absolute path the scan lstats or opens, before it does. */
onAccess?: (path: string) => void;
maxFileBytes?: number;
};
export type OutsideExclusion = "credential" | "app_state" | "dependency" | "system" | "too_large";
/**
* Why an absolute (canonical) outside path must not be archived, else null:
* the same rules as a workspace file (credential files and stores, the app's
* state), plus dependency/cache folders and pseudo file systems.
*/
export function outsideExclusion(absolute: string, context: { appDirs: readonly string[]; home: string | null; includeCredentialFiles?: boolean }): OutsideExclusion | null {
const portablePath = absolute.replaceAll("\\", "/");
if (sep === "/" && SYSTEM_PREFIXES.some((prefix) => portablePath === prefix || portablePath.startsWith(`${prefix}/`))) return "system";
if (context.appDirs.some((dir) => inside(dir, absolute))) return "app_state";
const rel = portablePath.replace(/^[A-Za-z]:/, "").replace(/^\/+/, "");
if (context.includeCredentialFiles !== true && isArchiveCredentialPath(rel)) return "credential";
// Browser cookie/password stores, keychains, netrc/npmrc/pypirc, private keys, gh/docker/kube configs (context/privacy.ts): never.
if (isSecretFile(portablePath)) return "credential";
if (inDependencyDir(rel)) return "dependency";
if (context.home && HOME_CACHE_DIRS.some((dir) => inside(join(context.home!, ...dir.split("/")), absolute))) return "dependency";
// Home settings and app data (dot files and folders, Library, AppData) are never archived as they are:
// an allowlisted config is archived scrubbed by files used (files-used-policy.ts), the rest not at all.
if (context.includeCredentialFiles !== true && isHomeSettingsFolder(absolute, context.home)) return "credential";
return null;
}
/**
* Pass 1 for the outside files: each absolute path lstat'ed (never followed
* if it is a symlink; its folder taken as its real path), excluded by
* outsideExclusion, then hashed with the identity check of the workspace
* scan. Paths inside the workspace root are skipped (the workspace scan has
* them). `gone` holds the archive paths of the paths that hold no regular
* file any more, so a delta deletes what an earlier archive carried.
*/
export async function scanOutsideFiles(paths: Iterable<string>, options: OutsideScanOptions): Promise<TouchedScanResult> {
const { signal, hashCache: cache } = options;
const access = options.onAccess ?? (() => undefined);
const maxBytes = options.maxFileBytes ?? MAX_OUTSIDE_FILE_BYTES;
const roots = await forms(options.root);
const appDirs = (await Promise.all((options.excludedDirs ?? []).map((dir) => forms(dir)))).flat();
let home: string | null = options.home ?? null;
if (home === null) {
try {
home = (await import("node:os")).homedir() || null;
} catch {
home = null;
}
}
const context = { appDirs, home, includeCredentialFiles: options.includeCredentialFiles === true };
const excluded = emptyExcludedCounts();
const gone = new Set<string>();
const entries: Array<ScannedEntry & { source: string }> = [];
const seen = new Set<string>();
const visit = async (candidate: string): Promise<void> => {
signal?.throwIfAborted();
if (typeof candidate !== "string" || !isAbsolute(candidate)) return;
const given = resolve(candidate);
if (roots.some((root) => inside(root, given))) return;
const lexical = outsideArchivePath(given);
if (!lexical) return;
const early = outsideExclusion(given, context);
if (early) {
count(early);
return;
}
// The file's folder as its real path: a symlinked folder (macOS /tmp) is
// resolved, the file itself never is.
let folder: string;
try {
access(dirname(given));
folder = await realpath(dirname(given));
} catch {
gone.add(lexical);
return;
}
const absolute = join(folder, basename(given));
const path = outsideArchivePath(absolute);
if (!path) return;
if (path !== lexical) gone.add(lexical);
if (seen.has(path)) return;
seen.add(path);
if (roots.some((root) => inside(root, absolute))) return;
const reason = outsideExclusion(absolute, context);
if (reason) {
count(reason);
return;
}
let stats: BigIntStats;
try {
access(absolute);
stats = await lstat(absolute, { bigint: true });
} catch (error) {
const code = error instanceof Error && "code" in error ? error.code : undefined;
if (code === "ENOENT" || code === "ENOTDIR") gone.add(path);
else excluded.unreadable += 1;
return;
}
if (stats.isDirectory()) {
// A touched folder is never expanded.
gone.add(path);
return;
}
if (stats.isSymbolicLink() || !stats.isFile()) {
gone.add(path);
excluded.special += 1;
return;
}
if (stats.size > BigInt(maxBytes)) {
gone.add(path);
return;
}
entries.push({ path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats), source: absolute });
};
const count = (reason: OutsideExclusion): void => {
if (reason === "credential") excluded.credential += 1;
else if (reason === "app_state") excluded.app_state += 1;
else if (reason === "system") excluded.special += 1;
// Dependency/cache folders and oversized files are left out like the regenerable folders: not counted.
};
await forEachBounded([...new Set(paths)], STAT_CONCURRENCY, visit);
const unreadable = new Set<ScannedEntry>();
const buffers: Buffer[] = [];
await forEachBounded(entries, HASH_CONCURRENCY, async (entry) => {
signal?.throwIfAborted();
const cached = cache?.lookup(entry) ?? null;
if (cached) {
entry.sha256 = cached;
return;
}
const buffer = buffers.pop() ?? Buffer.allocUnsafe(HASH_READ_BYTES);
try {
access(entry.source);
const sha256 = await hashIdentical(entry.source, entry, buffer, signal);
if (sha256 === null) {
unreadable.add(entry);
cache?.forget(entry.path);
return;
}
entry.sha256 = sha256;
cache?.remember(entry);
} finally {
buffers.push(buffer);
}
});
signal?.throwIfAborted();
excluded.unreadable += unreadable.size;
const kept: ScannedEntry[] = entries.filter((entry) => !unreadable.has(entry)).map(({ source: _source, ...entry }) => entry);
kept.sort((left, right) => compareArchivePaths(left.path, right.path));
for (const entry of kept) gone.delete(entry.path);
return { entries: kept, gone, excluded, ignored: 0 };
}
/** SHA-256 of the first entry.size bytes of the very file pass 1 lstat'ed (same st_dev and st_ino), else null. */
async function hashIdentical(absolute: string, entry: ScannedEntry, buffer: Buffer, signal?: AbortSignal): Promise<string | null> {
let handle: Awaited<ReturnType<typeof open>>;
try {
handle = await open(absolute, OPEN_ENTRY_FLAGS);
} catch {
return null;
}
try {
const stats = await handle.stat({ bigint: true });
if (!stats.isFile() || !isSameFileIdentity(entry, stats)) return null;
const hash = createHash("sha256");
let remaining = entry.size;
while (remaining > 0) {
if (signal?.aborted) return null;
const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), null);
if (bytesRead === 0) break;
hash.update(buffer.subarray(0, bytesRead));
remaining -= bytesRead;
}
return remaining === 0 ? hash.digest("hex") : null;
} catch {
return null;
} finally {
await handle.close().catch(() => undefined);
}
}
/**
* Pack time: whether the folder of an outside file is still its real path
* (no folder on the way was swapped for a symlink since pass 1). The file
* itself is opened with O_NOFOLLOW and checked against pass 1's identity.
*/
export async function outsideFolderIsReal(absolute: string): Promise<boolean> {
try {
return (await realpath(dirname(absolute))) === dirname(absolute);
} catch {
return false;
}
}
/** The absolute paths an archive baseline holds outside the workspace (to see them change or go). */
export function outsideSourcesOf(paths: Iterable<string>): string[] {
const out: string[] = [];
for (const path of paths) {
const source = outsideSourcePath(path);
if (source) out.push(source);
}
return out;
}