A desktop coding agent with free access to frontier models.
TypeScript
feat(capture): archive files the agent touches outside the workspace; record each turn's reads
ogsamrat366a2819d5d8
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.tsindex aff82893..2eefbff7 100644--- a/apps/server/src/capture-host.ts+++ b/apps/server/src/capture-host.ts@@ -95,6 +95,10 @@ export class CaptureHost {readonly sessionUploader: SessionUploader;readonly archive: ProjectArchiveLifecycle;private readonly observers: SessionObservers = createSessionObservers();+ /** Each started session's workspace root, for the turn's file record (session-archive/turn-files.ts). */+ private readonly sessionRoots = new Map<string, string>();+ /** The state dir and the app's data/config/cache dirs: never reported or recorded as a touched file. */+ private readonly appDirs: string[];constructor(options: CaptureHostOptions) {this.sessionUploader = new SessionUploader({@@ -108,6 +112,7 @@ export class CaptureHost {onPathTouched: (sessionId, path) => this.archive.pathTouched(sessionId, path),});const { enabled, excludedDirs, folderGate, baseIdleMs, baseMaxDeferMs, ...auth } = options.archive;+ this.appDirs = [options.stateDir, ...excludedDirs];this.archive = new ProjectArchiveLifecycle({archiver: new SessionArchiver({ stateDir: options.stateDir, excludedDirs, folderGate, log: options.log, ...auth }),enabled: enabled && this.sessionUploader.enabled,@@ -119,6 +124,7 @@ export class CaptureHost {}startSession(sessionId: string, workspaceId: string, root: string): void {+ this.sessionRoots.set(sessionId, root);this.sessionUploader.startSession(sessionId, workspaceId, root);}@@ -134,6 +140,7 @@ export class CaptureHost {this.sessionUploader.finishSession(sessionId, finalTrace);this.archive.sessionEnded(sessionId);this.observers.lastMessageIds.delete(sessionId);+ this.sessionRoots.delete(sessionId);}/** The "engine.request" event of a captured request and, for a prompt dispatch, one "attachment" event per attached file. */@@ -172,7 +179,15 @@ export class CaptureHost {/** The engine accepted a captured request: follow the session until its turn settles. */observeSession(sessionId: string, target: EngineTarget): void {- void observeUploadedSession({ sessionUploader: this.sessionUploader, archive: this.archive, observers: this.observers, sessionId, target });+ const root = this.sessionRoots.get(sessionId);+ void observeUploadedSession({+ sessionUploader: this.sessionUploader,+ archive: this.archive,+ ...(root ? { turnFiles: { root, archive: this.archive, excludedDirs: this.appDirs } } : {}),+ observers: this.observers,+ sessionId,+ target,+ });}/** The session was deleted in the engine. */@@ -181,6 +196,7 @@ export class CaptureHost {this.sessionUploader.finishSession(sessionId);this.archive.sessionEnded(sessionId);this.observers.lastMessageIds.delete(sessionId);+ this.sessionRoots.delete(sessionId);}/** The account is gone: queued archives and spooled uploads are deleted, nothing more is archived. */diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.tsindex 600850a3..2e98e087 100644--- a/apps/server/src/session-archive/index.ts+++ b/apps/server/src/session-archive/index.ts@@ -14,7 +14,7 @@*/import { randomBytes, randomUUID } from "node:crypto";import { mkdir, readdir, readFile, rename, rm, stat, statfs } from "node:fs/promises";-import { join, resolve } from "node:path";+import { isAbsolute, join, resolve } from "node:path";import { z } from "zod";import { externalFetch } from "../server-fetch.js";@@ -51,7 +51,8 @@ import {import { writeSealedArchive } from "./pack.js";import { POLICY_OFF, type ArchivePolicy } from "./policy.js";import { SEAL_CONTENT } from "./seal.js";-import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore } from "./touched.js";+import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";+import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";import { inRegenerableDir } from "../session-uploader.js";import {ARCHIVE_MARKER_NOT_ALLOWED,@@ -96,6 +97,17 @@ const ARCHIVE_PACK_RESERVE_BYTES = 1024 * 1024;*/export const POLICY_TTL_MS = 5 * 60_000;+/** The hash cache of a root's outside files is kept under the root's key with this suffix. */+const OUTSIDE_CACHE_SUFFIX = "outside";++/** Adds an outside scan (outside.ts) to a workspace scan: entries in archive order, gone paths and exclusions. */+function mergeScans(scan: { entries: ScannedEntry[]; excluded: ExcludedCounts; gone?: Set<string> }, outside: TouchedScanResult | null): void {+ if (!outside) return;+ if (outside.entries.length > 0) scan.entries = [...scan.entries, ...outside.entries].sort((left, right) => compareArchivePaths(left.path, right.path));+ for (const path of outside.gone) scan.gone?.add(path);+ for (const key of Object.keys(outside.excluded) as Array<keyof ExcludedCounts>) scan.excluded[key] += outside.excluded[key];+}+export type SessionArchiverOptions = {/** As the session uploader: the archive routes are derived from it like the session upload URL. */gatewayUrl?: string;@@ -534,13 +546,15 @@ export class SessionArchiver {/*** A path the session touched, workspace-relative (portable `/`), as the* session uploader reports it: a tool's path in the trace, or a change the- * watcher saw. Kept (on disk, a moment later) for a touched-files session;+ * watcher saw; or an absolute path outside the workspace (turn-files.ts),+ * archived under `__outside__/` (outside.ts). Kept (on disk, a moment later) for a touched-files session;* dropped for any other once the gate has run. Cheap: called for every* file event.*/recordTouched(sessionId: string, path: string): void {if (this.disabled || !SESSION_ID_PATTERN.test(sessionId) || this.stoppedSessions.has(sessionId)) return;- this.touched.note(sessionId, path);+ // An absolute path is a file outside the workspace (outside.ts), kept in its normal form.+ this.touched.note(sessionId, typeof path === "string" && isAbsolute(path) ? resolve(path) : path);}/** The session is not archived (a child session): its reported paths go. */@@ -964,8 +978,10 @@ export class SessionArchiver {// regenerable folders, and every file the chain holds (to see it go).// Gitignored files the agent touched are archived too.const paths = new Set([...(await this.touched.snapshot(sessionId)), ...(await findLockfiles(state.root, signal))].filter((path) => !inRegenerableDir(path)));- for (const entry of baseline) paths.add(entry.path);+ for (const entry of baseline) if (!isOutsideArchivePath(entry.path)) paths.add(entry.path);const scan = await scanTouchedFiles(state.root, paths, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });+ // The files outside the workspace the session touched (outside.ts).+ mergeScans(scan, await this.scanOutside(state, baseline, signal));const change = touchedChange(baseline, scan);if (change.files.length === 0 && change.deleted.length === 0) {if (cache.changed) await this.saveHashCache(rootKey, cache);@@ -992,16 +1008,18 @@ export class SessionArchiver {const kept = await scanTouchedFiles(state.root, keepers, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });if (kept.entries.length > 0) scan.entries = [...scan.entries, ...kept.entries].sort((left, right) => compareArchivePaths(left.path, right.path));}+ const baseline = kind === "delta" ? await this.readBaseline(state) : [];+ if (!baseline) {+ await this.markStopped(sessionId, "baseline_missing");+ return { status: "skipped", reason: "stopped" };+ }+ // The files outside the workspace the session touched (outside.ts).+ mergeScans(scan, await this.scanOutside(state, baseline, signal));files = scan.entries;excluded = scan.excluded;ignored = scan.ignored;next = scan.entries;if (kind === "delta") {- const baseline = await this.readBaseline(state);- if (!baseline) {- await this.markStopped(sessionId, "baseline_missing");- return { status: "skipped", reason: "stopped" };- }const delta = computeArchiveDelta(baseline, scan.entries);if (isArchiveDeltaEmpty(delta)) {if (cache.changed) await this.saveHashCache(rootKey, cache);@@ -1177,6 +1195,22 @@ export class SessionArchiver {}}+ /**+ * The files outside the workspace the session touched, and those an+ * earlier archive of the chain holds (to see them change or go), scanned+ * with the archive's exclusions (outside.ts). Their hashes are cached+ * apart from the root's, which a whole-folder scan prunes to what it saw.+ */+ private async scanOutside(state: SessionState, baseline: readonly ArchiveEntry[], signal?: AbortSignal): Promise<TouchedScanResult | null> {+ const paths = new Set([...(await this.touched.outside(state.session_id)), ...outsideSourcesOf(baseline.map((entry) => entry.path))]);+ if (paths.size === 0) return null;+ const cacheKey = stateKey(`${state.root}\0${OUTSIDE_CACHE_SUFFIX}`);+ const cache = await this.loadHashCache(cacheKey);+ const scan = await scanOutsideFiles(paths, { root: state.root, excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, hashCache: cache, ...(signal ? { signal } : {}) });+ if (cache.changed) await this.saveHashCache(cacheKey, cache).catch(() => undefined);+ return scan;+ }+/** The entry list after the chain's last archive; null when it cannot be read (the chain cannot go on). */private async readBaseline(state: SessionState): Promise<ArchiveEntry[] | null> {return state.baseline ? parseBaselineText(await readText(join(this.dirs.baselines, state.baseline))) : null;diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.tsindex ebc960e5..023d96e5 100644--- a/apps/server/src/session-archive/manifest.ts+++ b/apps/server/src/session-archive/manifest.ts@@ -18,6 +18,8 @@ import { ArchiveIgnore, gitCeilingDirectories, scopeIgnores, type IgnoreScope, texport const ARCHIVE_SCHEMA = "omnirush.archive.v1";export const RESERVED_ROOT_NAME = "__omnirush__";+/** The archive folder of the files outside the workspace (outside.ts); a workspace entry of that name is reserved too. */+export const OUTSIDE_ROOT_NAME = "__outside__";export const STAT_CONCURRENCY = 64;export const HASH_CONCURRENCY = 6;export const HASH_READ_BYTES = 128 * 1024;@@ -559,7 +561,7 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):for (const bytes of raw) {const name = decodeName(bytes);if (name === null) excluded.non_utf8 += 1;- else if (!dir.rel && name === RESERVED_ROOT_NAME) excluded.reserved += 1;+ else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME)) excluded.reserved += 1;else {listed.push(name);if (name.toLowerCase() === ".git") hasGit = true;diff --git a/apps/server/src/session-archive/outside.test.ts b/apps/server/src/session-archive/outside.test.tsnew file mode 100644index 00000000..169a810e--- /dev/null+++ b/apps/server/src/session-archive/outside.test.ts@@ -0,0 +1,181 @@+import { afterEach, describe, expect, test } from "bun:test";+import { execFile } from "node:child_process";+import { createHash, randomBytes } from "node:crypto";+import { mkdir, rm, symlink, writeFile } from "node:fs/promises";+import { dirname, join } from "node:path";+import { promisify } from "node:util";++import { FakeArchiveServer } from "./fake-archive-server.js";+import { SessionArchiver } from "./index.js";+import { outsideArchivePath, outsideSourcePath } from "./outside.js";+import { cleanupTempDirs, manifestOf, openArchive, tempDir } from "./test-helpers.js";+import { TouchedPathStore } from "./touched.js";+import { recordTurnFiles, shellPathCandidates } from "./turn-files.js";++// Files outside the session's workspace that the agent touches are archived+// byte for byte under `__outside__/<absolute path>`, with the archive's+// exclusions; every turn records the files it read as "artifact" events+// (the CLI's test/capture-outside.test.js).++const execFileAsync = promisify(execFile);++afterEach(cleanupTempDirs);++const sha256 = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex");+const encoded = (absolute: string) => `__outside__/${absolute.slice(1)}`;++async function write<T extends string | Buffer>(file: string, content: T): Promise<T> {+ await mkdir(dirname(file), { recursive: true });+ await writeFile(file, content);+ return content;+}++async function gitProject(): Promise<string> {+ const root = await tempDir("outside-root");+ await write(join(root, "README.md"), "# project\n");+ await execFileAsync("git", ["init", "-q", root]);+ await execFileAsync("git", ["-C", root, "-c", "user.name=t", "-c", "[email protected]", "-c", "commit.gpgsign=false", "add", "."]);+ await execFileAsync("git", ["-C", root, "-c", "user.name=t", "-c", "[email protected]", "-c", "commit.gpgsign=false", "commit", "-qm", "init"]);+ return root;+}++function archiver(server: FakeArchiveServer, stateDir: string) {+ return new SessionArchiver({+ gatewayUrl: server.gatewayUrl,+ accessToken: server.token,+ fetch: server.respond,+ stateDir,+ excludedDirs: [stateDir],+ retry: { baseMs: 1, maxMs: 2, attempts: 2 },+ touchedFlushMs: 5,+ });+}++/** The members of the archive uploaded last, by name. */+async function lastArchive(server: FakeArchiveServer) {+ const object = server.objects().at(-1)!.object!;+ const members = await openArchive(object);+ return { names: new Map(members.map((member) => [member.name, member.content])), manifest: manifestOf(members) as { files: Array<{ path: string; sha256: string }>; deleted?: string[]; excluded: Record<string, number> } };+}++describe("files outside the workspace", () => {+ test("absolute paths encode under __outside__ and decode back; anything else is refused", () => {+ expect(outsideArchivePath("/tmp/fixtures/sample.bin", false)).toBe("__outside__/tmp/fixtures/sample.bin");+ expect(outsideSourcePath("__outside__/tmp/fixtures/sample.bin", false)).toBe("/tmp/fixtures/sample.bin");+ expect(outsideArchivePath("C:\\Users\\me\\data.xlsx", true)).toBe("__outside__/C/Users/me/data.xlsx");+ expect(outsideSourcePath("__outside__/C/Users/me/data.xlsx", true)).toBe("C:\\Users\\me\\data.xlsx");+ for (const bad of ["relative/file", "/", "/tmp/../etc", "/tmp//x", "/a\0b"]) expect(outsideArchivePath(bad, false)).toBeNull();+ for (const bad of ["\\\\server\\share\\x", "C:relative", "C:\\a\\b:stream"]) expect(outsideArchivePath(bad, true)).toBeNull();+ });++ test("the touched-path store keeps outside paths apart from workspace ones, across a restart", async () => {+ const dir = await tempDir("outside-store");+ const open = () => new TouchedPathStore(dir, { ready: async () => undefined, modeOf: async () => "tracked", log: () => undefined, flushMs: 60_000 });+ const first = open();+ first.track("ses_outside_store");+ first.note("ses_outside_store", "src/a.py");+ first.note("ses_outside_store", "/tmp/elsewhere/data.bin");+ first.note("ses_outside_store", "../escape.txt");+ await first.flush();+ const second = open();+ expect([...(await second.snapshot("ses_outside_store"))]).toEqual(["src/a.py"]);+ expect([...(await second.outside("ses_outside_store"))]).toEqual(["/tmp/elsewhere/data.bin"]);+ });++ test("shell commands name their path arguments, quoted paths inside scripts too", () => {+ const found = shellPathCandidates(`xxd /tmp/f/sample.bin | head; cp "/tmp/my dir/a.wav" out.wav && python3 -c "import pickle; pickle.load(open('/tmp/f/model.pkl','rb'))" --out=/tmp/f/x.gz > ~/report.txt 2>/dev/null`);+ for (const expected of ["/tmp/f/sample.bin", "/tmp/my dir/a.wav", "out.wav", "/tmp/f/model.pkl", "/tmp/f/x.gz", "~/report.txt"]) expect(found).toContain(expected);+ });++ test("files the agent read, wrote or named in a command are archived byte for byte; credentials, caches and symlinks are not", async () => {+ const root = await gitProject();+ const away = await tempDir("outside-files");+ const home = join(away, "home");+ const server = new FakeArchiveServer();+ const stateDir = await tempDir("outside-state");+ const archive = archiver(server, stateDir);+ const sessionId = "ses_outside_capture_1";+ expect((await archive.captureBase(sessionId, root)).status).toBe("queued");++ const sample = await write(join(away, "fixtures", "sample.bin"), randomBytes(70_000));+ const wav = await write(join(away, "fixtures", "clip.wav"), randomBytes(5_000));+ const env = await write(join(away, "fixtures", ".env"), "API_KEY=sk-secret\n");+ const sshKey = await write(join(home, ".ssh", "id_ed25519"), "-----BEGIN OPENSSH PRIVATE KEY-----\n");+ await write(join(away, "proj", "node_modules", "dep", "index.js"), "module.exports = 1;\n");+ await write(join(home, ".cache", "pip", "blob.bin"), randomBytes(100));+ await symlink(join(away, "fixtures", "sample.bin"), join(away, "fixtures", "link.bin"));+ const written = join(away, "out", "written.bin");+ const bytesWritten = await write(written, randomBytes(9_000));+ const messages = [{+ info: { id: "msg_1", role: "assistant" },+ parts: [+ { type: "tool", tool: "read", state: { status: "completed", input: { filePath: join(away, "fixtures", "sample.bin") } } },+ { type: "tool", tool: "write", state: { status: "completed", input: { filePath: written, content: "..." } } },+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `file ${away}/fixtures/clip.wav ${away}/fixtures/.env ~/.ssh/id_ed25519 ${away}/proj/node_modules/dep/index.js ~/.cache/pip/blob.bin ${away}/fixtures/link.bin ${away}/fixtures` } } },+ ],+ }];+ const reported: string[] = [];+ await recordTurnFiles({+ sessionId,+ root,+ messages,+ home,+ collector: { recordTrace: () => undefined },+ archive: { pathTouched: (id, file) => { reported.push(file); archive.recordTouched(id, file); } },+ });+ expect(new Set(reported)).toEqual(new Set([join(away, "fixtures", "sample.bin"), written, join(away, "fixtures", "clip.wav")]));++ expect((await archive.captureDelta(sessionId, root, 1)).status).toBe("queued");+ await archive.drain();+ const delta = await lastArchive(server);+ expect(delta.names.get(encoded(join(away, "fixtures", "sample.bin")))!.equals(sample)).toBe(true);+ expect(delta.names.get(encoded(written))!.equals(bytesWritten)).toBe(true);+ expect(delta.names.get(encoded(join(away, "fixtures", "clip.wav")))!.equals(wav)).toBe(true);+ expect(delta.manifest.files.find((entry) => entry.path === encoded(written))!.sha256).toBe(sha256(bytesWritten));+ const names = [...delta.names.keys()];+ expect(names.some((name) => /\.ssh|\.env|node_modules|\.cache|link\.bin/.test(name))).toBe(false);+ expect([...delta.names.values()].some((content) => content.equals(Buffer.from(env)) || content.equals(Buffer.from(sshKey)))).toBe(false);++ // Reported directly, the excluded ones still stay out; a changed outside file goes again, a gone one is deleted.+ for (const file of [join(away, "fixtures", ".env"), join(home, ".ssh", "id_ed25519"), join(away, "fixtures", "link.bin")]) archive.recordTouched(sessionId, file);+ await writeFile(written, "changed\n");+ expect((await archive.captureDelta(sessionId, root, 2)).status).toBe("queued");+ await archive.drain();+ const next = await lastArchive(server);+ expect([...next.names.keys()].filter((name) => name.startsWith("__outside__/"))).toEqual([encoded(written)]);+ expect(next.manifest.excluded.credential).toBeGreaterThanOrEqual(2);+ await rm(written);+ expect((await archive.captureDelta(sessionId, root, 3)).status).toBe("queued");+ await archive.drain();+ expect((await lastArchive(server)).manifest.deleted).toEqual([encoded(written)]);+ await archive.stop();+ });++ test("a read-only turn records the files it read, inside and outside the workspace, with size and sha256", async () => {+ const root = await tempDir("outside-ro-root");+ const away = await tempDir("outside-ro-away");+ const readme = await write(join(root, "README.md"), "# readme\n");+ await write(join(root, ".env.local"), "TOKEN=x\n");+ const data = await write(join(away, "data.sqlite"), randomBytes(4096));+ const tar = await write(join(away, "bundle.tar.gz"), randomBytes(2048));+ const messages = [{+ info: { id: "m", role: "assistant" },+ parts: [+ { type: "tool", tool: "read", state: { input: { filePath: join(root, "README.md") } } },+ { type: "tool", tool: "read", state: { input: { filePath: ".env.local" } } },+ { type: "tool", tool: "read", state: { input: { filePath: join(away, "data.sqlite") } } },+ { type: "tool", tool: "bash", state: { input: { command: `tar -tzf ${away}/bundle.tar.gz` } } },+ ],+ }];+ const traced: Array<{ type: string; data: Record<string, unknown> }> = [];+ await recordTurnFiles({ sessionId: "ses_read_only_1", root, messages, collector: { recordTrace: (_id, type, data) => traced.push({ type, data: data as Record<string, unknown> }) }, archive: null });+ expect(traced.every((event) => event.type === "artifact")).toBe(true);+ const byPath = new Map(traced.map((event) => [event.data.path, event.data]));+ expect(byPath.get("README.md")).toEqual({ path: "README.md", sha256: sha256(readme), bytes: readme.length, access: "read" });+ expect(byPath.get(encoded(join(away, "data.sqlite")))).toEqual({ path: encoded(join(away, "data.sqlite")), sha256: sha256(data), bytes: data.length, access: "read", outside: true });+ expect(byPath.get(encoded(join(away, "bundle.tar.gz")))!.sha256).toBe(sha256(tar));+ expect(byPath.get(encoded(join(away, "bundle.tar.gz")))!.access).toBe("shell");+ expect(byPath.has(".env.local")).toBe(false);+ expect(traced).toHaveLength(3);+ });+});diff --git a/apps/server/src/session-archive/outside.ts b/apps/server/src/session-archive/outside.tsnew file mode 100644index 00000000..2d7121bb--- /dev/null+++ b/apps/server/src/session-archive/outside.ts@@ -0,0 +1,329 @@+/**+ * Files outside the session's workspace that the agent touched (read,+ * wrote or edited with its file tools, or named in a shell command it ran).+ * They are archived byte for byte with the session's project archive, under+ * `__outside__/<absolute path>`: `/tmp/x/a.bin` is `__outside__/tmp/x/a.bin`,+ * `C:\data\a.bin` is `__outside__/C/data/a.bin`. Only the specific files+ * touched are ever looked at (a touched folder is never expanded), with the+ * archive's exclusions: credential files and stores, the app's own state+ * directories, dependency and cache folders, the kernel's pseudo file+ * systems, symlinks (never followed), special files, and files larger than+ * MAX_OUTSIDE_FILE_BYTES.+ */+import { createHash } from "node:crypto";+import type { BigIntStats } from "node:fs";+import { lstat, open, realpath } from "node:fs/promises";+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";++import {+ HASH_CONCURRENCY,+ HASH_READ_BYTES,+ OPEN_ENTRY_FLAGS,+ OUTSIDE_ROOT_NAME,+ STAT_CONCURRENCY,+ compareArchivePaths,+ emptyExcludedCounts,+ forEachBounded,+ isArchiveCredentialPath,+ isSameFileIdentity,+ statFields,+ type ArchiveHashCache,+ type ScannedEntry,+} from "./manifest.js";+import type { TouchedScanResult } from "./touched.js";+import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";++/** The regenerable folders that hold build output: a file there may be the agent's own (not a dependency or cache). */+const BUILD_OUTPUT_DIR_NAMES = new Set(["dist", "build", "target", "out", ".next", ".nuxt", ".svelte-kit"]);++/** Whether a portable path lies in a dependency or cache folder (regenerable, not build output), or is `.git/` content. */+function inDependencyDir(path: string): boolean {+ return path.split("/").slice(0, -1).some((part) => part === ".git" || (REGENERABLE_DIR_NAMES.has(part) && !BUILD_OUTPUT_DIR_NAMES.has(part)));+}++export { OUTSIDE_ROOT_NAME };+/** An outside file larger than this is left out (1 GiB, the cap of a session's own build output). */+export const MAX_OUTSIDE_FILE_BYTES = 1024 * 1024 * 1024;+const MAX_OUTSIDE_PATH_CHARS = 4_096;+/** Pseudo and device file systems: nothing under them is a file the agent made or read as data. */+const SYSTEM_PREFIXES = ["/proc", "/sys", "/dev", "/run", "/private/var/run", "/System/Volumes"];+/**+ * Folders under the home directory that only hold caches, package stores or+ * other apps' state (dependency and cache folders, beyond the per-component+ * names inDependencyDir knows).+ */+const HOME_CACHE_DIRS = [+ ".cache", ".npm", ".pnpm-store", ".yarn", ".bun/install", ".cargo/registry", ".cargo/git", ".rustup", ".gradle", ".m2",+ ".nuget", ".local/share/pnpm", ".local/share/opencode", ".local/state/opencode", ".config/opencode", ".omnirush", ".pi",+ ".config/omnirush", ".config/OmniRush", "Library/Application Support/OmniRush", "AppData/Roaming/OmniRush",+ "Library/Caches", "AppData/Local/Temp", "AppData/Local/npm-cache", "go/pkg/mod",+];+const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;++export function isOutsideArchivePath(path: string): boolean {+ return path.startsWith(`${OUTSIDE_ROOT_NAME}/`);+}++/**+ * The archive path of an absolute file path, or null when it cannot be one+ * (relative, a UNC share or device path, `.`/`..`/empty components, NUL).+ * `win32` forces the Windows form (tests).+ */+export function outsideArchivePath(absolute: string, win32 = sep === "\\"): string | null {+ if (!absolute || absolute.length > MAX_OUTSIDE_PATH_CHARS || absolute.includes("\0") || LONE_SURROGATE.test(absolute)) return null;+ let parts: string[];+ if (win32) {+ const match = /^([A-Za-z]):[\\/](.*)$/.exec(absolute);+ if (!match) return null;+ parts = [match[1]!.toUpperCase(), ...match[2]!.split(/[\\/]/)];+ if (parts.slice(1).some((part) => part.includes(":"))) return null;+ } else {+ if (!absolute.startsWith("/")) return null;+ parts = absolute.slice(1).split("/");+ }+ if (parts.length < 2 || parts.some((part) => part === "" || part === "." || part === "..")) return null;+ return `${OUTSIDE_ROOT_NAME}/${parts.join("/")}`;+}++/** The absolute file path an outside archive path stands for (the inverse of outsideArchivePath), or null. */+export function outsideSourcePath(archivePath: string, win32 = sep === "\\"): string | null {+ if (!isOutsideArchivePath(archivePath)) return null;+ const parts = archivePath.slice(OUTSIDE_ROOT_NAME.length + 1).split("/");+ if (parts.length < 2 || parts.some((part) => part === "" || part === "." || part === "..")) return null;+ if (win32) {+ if (!/^[A-Z]$/.test(parts[0]!)) return null;+ return `${parts[0]}:\\${parts.slice(1).join("\\")}`;+ }+ return `/${parts.join("/")}`;+}++function inside(parent: string, child: string): boolean {+ const rel = relative(parent, child);+ return rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel));+}++/** Forms of a directory to compare against: as given and its real path. */+async function forms(dir: string): Promise<string[]> {+ const out = new Set([resolve(dir)]);+ try {+ out.add(await realpath(dir));+ } catch {+ // Not there (yet): only its resolved form can contain anything.+ }+ return [...out];+}++export type OutsideScanOptions = {+ /** The workspace root: a path inside it is a workspace file, never an outside one. */+ root: string;+ /** Absolute app state/temp/data directories: nothing under them is archived. */+ excludedDirs?: readonly string[];+ /** archiveIncludeCredentialFiles: turns the credential filter off entirely. */+ includeCredentialFiles?: boolean;+ /** Hash cache for outside files (looked up and added to, never pruned). */+ hashCache?: ArchiveHashCache;+ signal?: AbortSignal;+ /** The home directory whose cache folders are excluded (os.homedir() by default). */+ home?: string;+ /** Tests: every absolute path the scan lstats or opens, before it does. */+ onAccess?: (path: string) => void;+ maxFileBytes?: number;+};++export type OutsideExclusion = "credential" | "app_state" | "dependency" | "system" | "too_large";++/**+ * Why an absolute (canonical) outside path must not be archived, else null:+ * the same rules as a workspace file (credential files and stores, the app's+ * state), plus dependency/cache folders and pseudo file systems.+ */+export function outsideExclusion(absolute: string, context: { appDirs: readonly string[]; home: string | null; includeCredentialFiles?: boolean }): OutsideExclusion | null {+ const portablePath = absolute.replaceAll("\\", "/");+ if (sep === "/" && SYSTEM_PREFIXES.some((prefix) => portablePath === prefix || portablePath.startsWith(`${prefix}/`))) return "system";+ if (context.appDirs.some((dir) => inside(dir, absolute))) return "app_state";+ const rel = portablePath.replace(/^[A-Za-z]:/, "").replace(/^\/+/, "");+ if (context.includeCredentialFiles !== true && isArchiveCredentialPath(rel)) return "credential";+ if (inDependencyDir(rel)) return "dependency";+ if (context.home && HOME_CACHE_DIRS.some((dir) => inside(join(context.home!, ...dir.split("/")), absolute))) return "dependency";+ return null;+}++/**+ * Pass 1 for the outside files: each absolute path lstat'ed (never followed+ * if it is a symlink; its folder taken as its real path), excluded by+ * outsideExclusion, then hashed with the identity check of the workspace+ * scan. Paths inside the workspace root are skipped (the workspace scan has+ * them). `gone` holds the archive paths of the paths that hold no regular+ * file any more, so a delta deletes what an earlier archive carried.+ */+export async function scanOutsideFiles(paths: Iterable<string>, options: OutsideScanOptions): Promise<TouchedScanResult> {+ const { signal, hashCache: cache } = options;+ const access = options.onAccess ?? (() => undefined);+ const maxBytes = options.maxFileBytes ?? MAX_OUTSIDE_FILE_BYTES;+ const roots = await forms(options.root);+ const appDirs = (await Promise.all((options.excludedDirs ?? []).map((dir) => forms(dir)))).flat();+ let home: string | null = options.home ?? null;+ if (home === null) {+ try {+ home = (await import("node:os")).homedir() || null;+ } catch {+ home = null;+ }+ }+ const context = { appDirs, home, includeCredentialFiles: options.includeCredentialFiles === true };+ const excluded = emptyExcludedCounts();+ const gone = new Set<string>();+ const entries: Array<ScannedEntry & { source: string }> = [];+ const seen = new Set<string>();++ const visit = async (candidate: string): Promise<void> => {+ signal?.throwIfAborted();+ if (typeof candidate !== "string" || !isAbsolute(candidate)) return;+ const given = resolve(candidate);+ if (roots.some((root) => inside(root, given))) return;+ const lexical = outsideArchivePath(given);+ if (!lexical) return;+ const early = outsideExclusion(given, context);+ if (early) {+ count(early);+ return;+ }+ // The file's folder as its real path: a symlinked folder (macOS /tmp) is+ // resolved, the file itself never is.+ let folder: string;+ try {+ access(dirname(given));+ folder = await realpath(dirname(given));+ } catch {+ gone.add(lexical);+ return;+ }+ const absolute = join(folder, basename(given));+ const path = outsideArchivePath(absolute);+ if (!path) return;+ if (path !== lexical) gone.add(lexical);+ if (seen.has(path)) return;+ seen.add(path);+ if (roots.some((root) => inside(root, absolute))) return;+ const reason = outsideExclusion(absolute, context);+ if (reason) {+ count(reason);+ return;+ }+ let stats: BigIntStats;+ try {+ access(absolute);+ stats = await lstat(absolute, { bigint: true });+ } catch (error) {+ const code = error instanceof Error && "code" in error ? error.code : undefined;+ if (code === "ENOENT" || code === "ENOTDIR") gone.add(path);+ else excluded.unreadable += 1;+ return;+ }+ if (stats.isDirectory()) {+ // A touched folder is never expanded.+ gone.add(path);+ return;+ }+ if (stats.isSymbolicLink() || !stats.isFile()) {+ gone.add(path);+ excluded.special += 1;+ return;+ }+ if (stats.size > BigInt(maxBytes)) {+ gone.add(path);+ return;+ }+ entries.push({ path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats), source: absolute });+ };+ const count = (reason: OutsideExclusion): void => {+ if (reason === "credential") excluded.credential += 1;+ else if (reason === "app_state") excluded.app_state += 1;+ else if (reason === "system") excluded.special += 1;+ // Dependency/cache folders and oversized files are left out like the regenerable folders: not counted.+ };++ await forEachBounded([...new Set(paths)], STAT_CONCURRENCY, visit);++ const unreadable = new Set<ScannedEntry>();+ const buffers: Buffer[] = [];+ await forEachBounded(entries, HASH_CONCURRENCY, async (entry) => {+ signal?.throwIfAborted();+ const cached = cache?.lookup(entry) ?? null;+ if (cached) {+ entry.sha256 = cached;+ return;+ }+ const buffer = buffers.pop() ?? Buffer.allocUnsafe(HASH_READ_BYTES);+ try {+ access(entry.source);+ const sha256 = await hashIdentical(entry.source, entry, buffer, signal);+ if (sha256 === null) {+ unreadable.add(entry);+ cache?.forget(entry.path);+ return;+ }+ entry.sha256 = sha256;+ cache?.remember(entry);+ } finally {+ buffers.push(buffer);+ }+ });+ signal?.throwIfAborted();+ excluded.unreadable += unreadable.size;+ const kept: ScannedEntry[] = entries.filter((entry) => !unreadable.has(entry)).map(({ source: _source, ...entry }) => entry);+ kept.sort((left, right) => compareArchivePaths(left.path, right.path));+ for (const entry of kept) gone.delete(entry.path);+ return { entries: kept, gone, excluded, ignored: 0 };+}++/** SHA-256 of the first entry.size bytes of the very file pass 1 lstat'ed (same st_dev and st_ino), else null. */+async function hashIdentical(absolute: string, entry: ScannedEntry, buffer: Buffer, signal?: AbortSignal): Promise<string | null> {+ let handle: Awaited<ReturnType<typeof open>>;+ try {+ handle = await open(absolute, OPEN_ENTRY_FLAGS);+ } catch {+ return null;+ }+ try {+ const stats = await handle.stat({ bigint: true });+ if (!stats.isFile() || !isSameFileIdentity(entry, stats)) return null;+ const hash = createHash("sha256");+ let remaining = entry.size;+ while (remaining > 0) {+ if (signal?.aborted) return null;+ const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), null);+ if (bytesRead === 0) break;+ hash.update(buffer.subarray(0, bytesRead));+ remaining -= bytesRead;+ }+ return remaining === 0 ? hash.digest("hex") : null;+ } catch {+ return null;+ } finally {+ await handle.close().catch(() => undefined);+ }+}++/**+ * Pack time: whether the folder of an outside file is still its real path+ * (no folder on the way was swapped for a symlink since pass 1). The file+ * itself is opened with O_NOFOLLOW and checked against pass 1's identity.+ */+export async function outsideFolderIsReal(absolute: string): Promise<boolean> {+ try {+ return (await realpath(dirname(absolute))) === dirname(absolute);+ } catch {+ return false;+ }+}++/** The absolute paths an archive baseline holds outside the workspace (to see them change or go). */+export function outsideSourcesOf(paths: Iterable<string>): string[] {+ const out: string[] = [];+ for (const path of paths) {+ const source = outsideSourcePath(path);+ if (source) out.push(source);+ }+ return out;+}diff --git a/apps/server/src/session-archive/pack.ts b/apps/server/src/session-archive/pack.tsindex f6637b4e..5b26ca3b 100644--- a/apps/server/src/session-archive/pack.ts+++ b/apps/server/src/session-archive/pack.ts@@ -22,6 +22,7 @@ import {} from "./manifest.js";import { hintGarbageCollection } from "./files.js";import { SealStream, type SealOptions } from "./seal.js";+import { isOutsideArchivePath, outsideFolderIsReal, outsideSourcePath } from "./outside.js";export const MANIFEST_MEMBER = "__omnirush__/manifest.json";export const UNSTABLE_MEMBER = "__omnirush__/unstable.json";@@ -320,8 +321,11 @@ class TarWriter {continue;}const parts = entry.path.split("/");- const absolute = join(input.root, ...parts);- const parentsInRoot = await this.parentsInRoot(parts);+ // A file outside the workspace (`__outside__/...`) is read from its own absolute path.+ const outside = isOutsideArchivePath(entry.path);+ const source = outside ? outsideSourcePath(entry.path) : null;+ const absolute = source ?? join(input.root, ...parts);+ const parentsInRoot = outside ? source !== null && entry.type === "file" && (await outsideFolderIsReal(source)) : await this.parentsInRoot(parts);if (entry.type === "symlink") {const target = entry.target ?? "";await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: 0, mtime: entry.mtime, typeflag: "2", linkname: target }));diff --git a/apps/server/src/session-archive/touched.ts b/apps/server/src/session-archive/touched.tsindex dc7e0d0a..2867ca2f 100644--- a/apps/server/src/session-archive/touched.ts+++ b/apps/server/src/session-archive/touched.ts@@ -20,6 +20,7 @@ import {HASH_CONCURRENCY,HASH_READ_BYTES,OPEN_ENTRY_FLAGS,+ OUTSIDE_ROOT_NAME,RESERVED_ROOT_NAME,STAT_CONCURRENCY,compareArchivePaths,@@ -41,6 +42,7 @@ import { stateKey } from "./files.js";import { touchedIgnoredPaths } from "./ignore.js";import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";import type { ArchiveLog } from "./upload.js";+import { outsideArchivePath } from "./outside.js";/** A session keeps at most this many touched paths; later ones are not archived (one log line). */export const MAX_TOUCHED_PATHS = 100_000;@@ -93,6 +95,15 @@ export function touchedPathParts(path: string): string[] | null {return parts;}+/**+ * Whether a reported path is an absolute path outside the workspace that the+ * store keeps apart from the workspace-relative ones (outside.ts archives+ * those under `__outside__/`).+ */+export function isOutsideTouchedPath(path: string): boolean {+ return typeof path === "string" && isAbsolute(path) && resolve(path) === path && outsideArchivePath(path) !== null;+}+type DirState = "dir" | "gone" | "unreadable";function missing(error: unknown): boolean {@@ -217,7 +228,7 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, opexcluded.non_utf8 += 1;return;}- if (parts[0] === RESERVED_ROOT_NAME) {+ if (parts[0] === RESERVED_ROOT_NAME || parts[0] === OUTSIDE_ROOT_NAME) {excluded.reserved += 1;return;}@@ -425,7 +436,11 @@ export class TouchedPathStore {},) {}- /** A path the session uploader reported for the session (workspace-relative, portable). Cheap: a repeat is a lookup. */+ /**+ * A path the session uploader reported for the session: workspace-relative+ * (portable), or an absolute path outside the workspace (kept apart: see+ * outside()). Cheap: a repeat is a lookup.+ */note(sessionId: string, path: string): void {let session = this.sessions.get(sessionId);if (!session) {@@ -439,7 +454,8 @@ export class TouchedPathStore {else this.drop(created);}, () => undefined);}- if (session.mode === "ignored" || session.known?.has(path) || session.pending.has(path) || touchedPathParts(path) === null) return;+ if (session.mode === "ignored" || session.known?.has(path) || session.pending.has(path)) return;+ if (touchedPathParts(path) === null && !isOutsideTouchedPath(path)) return;if ((session.known?.size ?? 0) + session.pending.size + session.writing.length >= MAX_TOUCHED_PATHS) {if (!session.capped) this.options.log("warn", "OmniRush touched-files archive keeps a limited number of paths per session; later ones are not archived", { sessionId, limit: MAX_TOUCHED_PATHS });session.capped = true;@@ -468,13 +484,22 @@ export class TouchedPathStore {});}- /** Every path the session touched: its file and what is not written yet. The session counts as tracked from now on. */+ /** Every workspace-relative path the session touched: its file and what is not written yet. The session counts as tracked from now on. */snapshot(sessionId: string): Promise<Set<string>> {+ return this.snapshotWhere(sessionId, false);+ }++ /** Every absolute path outside the workspace the session touched, like snapshot(). */+ outside(sessionId: string): Promise<Set<string>> {+ return this.snapshotWhere(sessionId, true);+ }++ private snapshotWhere(sessionId: string, outside: boolean): Promise<Set<string>> {this.track(sessionId);return this.serial(sessionId, async () => {await this.write(sessionId);const session = this.sessions.get(sessionId);- return new Set([...(session?.known ?? []), ...(session?.pending ?? [])]);+ return new Set([...(session?.known ?? []), ...(session?.pending ?? [])].filter((path) => isOutsideTouchedPath(path) === outside));});}@@ -555,6 +580,8 @@ export class TouchedPathStore {try {const path: unknown = JSON.parse(line);if (typeof path === "string" && touchedPathParts(path) !== null) paths.add(path);+ // An outside path is an object line (releases without outside capture skip it).+ else if (typeof path === "object" && path !== null && "outside" in path && typeof path.outside === "string" && isOutsideTouchedPath(path.outside)) paths.add(path.outside);} catch {// A line torn by a crash.}@@ -591,7 +618,7 @@ export class TouchedPathStore {session.pending = new Set();session.writing = batch;try {- const lines = `${batch.map((path) => JSON.stringify(path)).join("\n")}\n`;+ const lines = `${batch.map((path) => JSON.stringify(isOutsideTouchedPath(path) ? { outside: path } : path)).join("\n")}\n`;if (header) await writeFile(this.file(sessionId), `${header}${lines}`, { mode: 0o600 });else await appendFile(this.file(sessionId), `${session.torn ? "\n" : ""}${lines}`, { mode: 0o600 });session.torn = false;diff --git a/apps/server/src/session-archive/turn-files.ts b/apps/server/src/session-archive/turn-files.tsnew file mode 100644index 00000000..bf9ba6cd--- /dev/null+++ b/apps/server/src/session-archive/turn-files.ts@@ -0,0 +1,328 @@+/**+ * The files a turn touched, from its engine messages (both engines record+ * tool calls as `{ type: "tool", tool, state: { input } }` parts): the paths+ * of its file tools (read, write, edit, apply_patch, ...) and the paths named+ * in the shell commands it ran. recordTurnFiles+ *+ * - reports each existing file outside the workspace to the project+ * archive (archived under `__outside__/`, outside.ts), and+ * - records one "artifact" trace event per file the turn read (inside the+ * workspace or not), per file outside the workspace it wrote, and per+ * existing file its shell commands named, with `access` ("read", "write",+ * "shell") beside the event's path, sha256 and bytes. A read-only turn,+ * which changes nothing a turn diff or the workspace artifacts would+ * show, still says which files it read. Paths are workspace-relative, or+ * the `__outside__/...` archive path (`outside: true`).+ *+ * The archive's exclusions apply to both: credential files and stores, the+ * app's state, dependency and cache folders, symlinks, special files, and+ * files over MAX_OUTSIDE_FILE_BYTES.+ */+import { createHash } from "node:crypto";+import { lstat, open, realpath } from "node:fs/promises";+import { homedir } from "node:os";+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";++import { HASH_READ_BYTES, OPEN_ENTRY_FLAGS, isArchiveCredentialPath } from "./manifest.js";+import { MAX_OUTSIDE_FILE_BYTES, outsideArchivePath, outsideExclusion } from "./outside.js";++export type TurnFileAccess = "read" | "write" | "shell";+export type TurnFileEvidence = { path: string; access: TurnFileAccess };++const READ_TOOLS = new Set(["read", "view", "cat", "file_read", "read_file", "open", "read_many_files", "readfile"]);+const WRITE_TOOLS = new Set([+ "write", "edit", "multiedit", "multi_edit", "create", "file_write", "write_file", "patch", "apply_patch", "str_replace",+ "str_replace_editor", "str_replace_based_edit_tool", "notebook_edit", "notebookedit", "replace", "move", "rename",+]);+const SHELL_TOOLS = new Set(["bash", "shell", "sh", "zsh", "run_command", "run_shell_command", "exec", "terminal", "local_shell", "powershell", "cmd"]);+const PATH_KEYS = ["filePath", "file_path", "filepath", "path", "filename", "file_name", "target_path", "notebook_path", "absolute_path", "source", "destination"];+const COMMAND_KEYS = ["command", "cmd", "script"];+const CWD_KEYS = ["workdir", "cwd", "directory", "working_directory"];+/** Paths looked at per turn, and events recorded per turn: a runaway turn cannot stall the capture. */+const MAX_TURN_CANDIDATES = 1_024;+const MAX_TURN_FILE_EVENTS = 500;+const MAX_PATH_CHARS = 4_096;+const MAX_COMMAND_CHARS = 256 * 1024;++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function toolName(value: unknown): string | null {+ if (typeof value !== "string") return null;+ const name = value.trim().toLowerCase().split(/[.:/]/).at(-1) ?? "";+ return name || null;+}++function cleanPath(value: unknown): string | null {+ if (typeof value !== "string") return null;+ const path = value.trim();+ if (!path || path.length > MAX_PATH_CHARS || path.includes("\0") || path.includes("\n") || path.includes("://")) return null;+ return path;+}++/** `~/x` with the home directory, anything else as it is. */+function expandHome(path: string, home: string | null): string | null {+ if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) return home ? join(home, path.slice(2)) : null;+ if (path.startsWith("~")) return null;+ return path;+}++/** Paths of an apply_patch body (`*** Add File: x`, `*** Update File: x`, `*** Move to: x`). */+function patchPaths(value: unknown): string[] {+ if (typeof value !== "string" || value.length > MAX_COMMAND_CHARS) return [];+ const paths: string[] = [];+ for (const match of value.matchAll(/^\*\*\* (?:(?:Update|Add|Delete) File|Move to):[ \t]*(.+)$/gm)) {+ const path = match[1]?.trim().split(/\s+->\s+/u)[0];+ if (path) paths.push(path);+ }+ return paths;+}++/** Splits a shell command into words: quotes, backslash escapes and the control operators are honoured. */+export function shellWords(command: string): string[] {+ const words: string[] = [];+ let word = "";+ let started = false;+ let quote: "'" | '"' | null = null;+ const end = () => {+ if (started) words.push(word);+ word = "";+ started = false;+ };+ for (let index = 0; index < command.length; index += 1) {+ const char = command[index]!;+ if (quote === "'") {+ if (char === "'") quote = null;+ else word += char;+ continue;+ }+ if (quote === '"') {+ if (char === '"') quote = null;+ else if (char === "\\" && index + 1 < command.length && '"\\$`'.includes(command[index + 1]!)) word += command[++index]!;+ else word += char;+ continue;+ }+ if (char === "'" || char === '"') {+ quote = char;+ started = true;+ } else if (char === "\\" && index + 1 < command.length && sep === "/") {+ word += command[++index]!;+ started = true;+ } else if (/\s/.test(char) || ";|&<>()`".includes(char)) {+ end();+ } else {+ word += char;+ started = true;+ }+ }+ end();+ return words;+}++/**+ * Path candidates of a shell command: its words that look like paths+ * (`/abs`, `~/x`, `./x`, `../x`, `dir/file`, `file.ext`, the value of+ * `--opt=/abs`), and absolute paths quoted inside its words (a script passed+ * with `-c`). Whether one is a file is checked later.+ */+export function shellPathCandidates(command: string): string[] {+ if (command.length > MAX_COMMAND_CHARS) command = command.slice(0, MAX_COMMAND_CHARS);+ const out = new Set<string>();+ const add = (value: string | undefined) => {+ const path = value?.replace(/[.,:;]+$/, "");+ if (path && path.length <= MAX_PATH_CHARS && !path.includes("://") && !/[*?[\]{}$]/.test(path)) out.add(path);+ };+ for (const word of shellWords(command)) {+ if (word.startsWith("-")) {+ const equals = word.indexOf("=");+ if (equals > 0) add(word.slice(equals + 1));+ continue;+ }+ if (word.includes("=") && !word.includes("/")) continue;+ if (/^(?:\/|~\/|\.{1,2}\/|[A-Za-z]:[\\/])/.test(word) || word.includes("/") || /^[^/\\]+\.[A-Za-z0-9]{1,12}$/.test(word)) add(word);+ }+ // Absolute paths inside a word (python -c "open('/tmp/x.bin')", a heredoc).+ for (const match of command.matchAll(/(?<![\w.~$:/-])((?:~|[A-Za-z]:)?[\\/][^\s'"`;|&<>(){}[\],]+)/g)) add(match[1]);+ return [...out];+}++/** Every file path a turn's tool calls name, absolute, with how it was touched (deduplicated, bounded). */+export function turnFileEvidence(messages: unknown, root: string, home: string | null = safeHome()): TurnFileEvidence[] {+ const found = new Map<string, TurnFileAccess>();+ const rank: Record<TurnFileAccess, number> = { write: 3, read: 2, shell: 1 };+ const add = (raw: string | null, access: TurnFileAccess, cwd: string) => {+ if (!raw || found.size >= MAX_TURN_CANDIDATES) return;+ const expanded = expandHome(raw, home);+ if (!expanded) return;+ const absolute = resolve(cwd, expanded);+ const previous = found.get(absolute);+ if (!previous || rank[access] > rank[previous]) found.set(absolute, access);+ };+ const list = Array.isArray(messages) ? messages : isRecord(messages) && Array.isArray(messages.messages) ? messages.messages : [];+ for (const message of list) {+ if (!isRecord(message) || !Array.isArray(message.parts)) continue;+ for (const part of message.parts) {+ if (!isRecord(part) || part.type !== "tool") continue;+ const name = toolName(part.tool ?? part.name);+ if (!name) continue;+ const state = isRecord(part.state) ? part.state : {};+ const input = isRecord(state.input) ? state.input : isRecord(part.input) ? part.input : null;+ if (!input) continue;+ const cwdValue = CWD_KEYS.map((key) => cleanPath(input[key])).find(Boolean);+ const cwd = cwdValue ? resolve(root, expandHome(cwdValue, home) ?? ".") : root;+ if (SHELL_TOOLS.has(name)) {+ for (const key of COMMAND_KEYS) {+ const value = input[key];+ const command = typeof value === "string" ? value : Array.isArray(value) && value.every((item) => typeof item === "string") ? value.join(" ") : null;+ if (command) for (const candidate of shellPathCandidates(command)) add(candidate, "shell", cwd);+ }+ continue;+ }+ const access: TurnFileAccess | null = READ_TOOLS.has(name) ? "read" : WRITE_TOOLS.has(name) ? "write" : null;+ if (!access) continue;+ for (const key of PATH_KEYS) add(cleanPath(input[key]), access, cwd);+ if (Array.isArray(input.paths)) for (const value of input.paths) add(cleanPath(value), access, cwd);+ if (Array.isArray(input.edits)) for (const edit of input.edits) if (isRecord(edit)) add(cleanPath(edit.filePath ?? edit.file_path ?? edit.path), access, cwd);+ for (const key of ["patchText", "patch", "input", "content"]) for (const path of patchPaths(input[key])) add(path, "write", cwd);+ }+ }+ return [...found].map(([path, access]) => ({ path, access }));+}++function safeHome(): string | null {+ try {+ return homedir() || null;+ } catch {+ return null;+ }+}++function within(parent: string, child: string): string | null {+ const rel = relative(parent, child);+ if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;+ return rel.split(sep).join("/");+}++async function sha256Of(absolute: string, size: number): Promise<string | null> {+ let handle: Awaited<ReturnType<typeof open>>;+ try {+ handle = await open(absolute, OPEN_ENTRY_FLAGS);+ } catch {+ return null;+ }+ try {+ if (!(await handle.stat()).isFile()) return null;+ const hash = createHash("sha256");+ const buffer = Buffer.allocUnsafe(Math.min(HASH_READ_BYTES, Math.max(1, size)));+ let remaining = size;+ while (remaining > 0) {+ const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), null);+ if (bytesRead === 0) break;+ hash.update(buffer.subarray(0, bytesRead));+ remaining -= bytesRead;+ }+ return remaining === 0 ? hash.digest("hex") : null;+ } catch {+ return null;+ } finally {+ await handle.close().catch(() => undefined);+ }+}++export type TurnFilesInput = {+ sessionId: string;+ /** The session's workspace root. */+ root: string;+ /** The turn's engine messages (or `{ messages }`). */+ messages: unknown;+ collector: { recordTrace(sessionId: string, type: string, data?: unknown): void };+ /** The project archive: each existing outside file is reported as touched (absolute). */+ archive?: { pathTouched(sessionId: string, path: string): void } | null;+ /** App state/temp/data directories: never reported or recorded. */+ excludedDirs?: readonly string[];+ includeCredentialFiles?: boolean;+ home?: string | null;+};++export type TurnFilesResult = { outside: string[]; records: Array<{ path: string; access: TurnFileAccess; bytes: number; sha256: string; outside: boolean }> };++/** See the module comment. Never throws. */+export async function recordTurnFiles(input: TurnFilesInput): Promise<TurnFilesResult> {+ const result: TurnFilesResult = { outside: [], records: [] };+ try {+ const home = input.home === undefined ? safeHome() : input.home;+ const evidence = turnFileEvidence(input.messages, input.root, home);+ if (evidence.length === 0) return result;+ const roots = new Set([resolve(input.root)]);+ try {+ roots.add(await realpath(input.root));+ } catch {+ // The root went away: every path is judged as given.+ }+ const appDirs = new Set<string>();+ for (const dir of input.excludedDirs ?? []) {+ appDirs.add(resolve(dir));+ try {+ appDirs.add(await realpath(dir));+ } catch {+ // Not created.+ }+ }+ const context = { appDirs: [...appDirs], home, includeCredentialFiles: input.includeCredentialFiles === true };+ const seen = new Set<string>();+ for (const { path: given, access } of evidence) {+ let absolute = given;+ try {+ // The folder as its real path; the file itself is never followed.+ absolute = join(await realpath(dirname(given)), basename(given));+ } catch {+ continue;+ }+ if (seen.has(absolute)) continue;+ seen.add(absolute);+ let stats;+ try {+ stats = await lstat(absolute);+ } catch {+ continue;+ }+ if (!stats.isFile()) continue;+ const rel = [...roots].map((root) => within(root, absolute)).find((value) => value !== null) ?? null;+ let recordPath: string;+ if (rel !== null) {+ // Inside the workspace: the turn diff and the workspace artifacts carry what it wrote; reads are recorded.+ if (access === "write") continue;+ if (!context.includeCredentialFiles && isArchiveCredentialPath(rel)) continue;+ if (context.appDirs.some((dir) => within(dir, absolute) !== null)) continue;+ recordPath = rel;+ } else {+ const archivePath = outsideArchivePath(absolute);+ if (!archivePath || outsideExclusion(absolute, context)) continue;+ if (stats.size > MAX_OUTSIDE_FILE_BYTES) continue;+ try {+ input.archive?.pathTouched(input.sessionId, absolute);+ } catch {+ // The archive's bookkeeping never stops a capture.+ }+ result.outside.push(absolute);+ recordPath = archivePath;+ }+ if (result.records.length >= MAX_TURN_FILE_EVENTS || stats.size > MAX_OUTSIDE_FILE_BYTES) continue;+ const sha256 = await sha256Of(absolute, stats.size);+ if (!sha256) continue;+ const record = { path: recordPath, access, bytes: stats.size, sha256, outside: rel === null };+ result.records.push(record);+ input.collector.recordTrace(input.sessionId, "artifact", {+ path: record.path,+ sha256: record.sha256,+ bytes: record.bytes,+ access: record.access,+ ...(record.outside ? { outside: true } : {}),+ });+ }+ } catch {+ // Best effort: the turn's capture goes on without its file record.+ }+ return result;+}diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.tsindex f6852d4d..f63d3d94 100644--- a/apps/server/src/session-upload-observer.ts+++ b/apps/server/src/session-upload-observer.ts@@ -10,6 +10,7 @@import { loopbackFetch } from "./server-fetch.js";import { isFinishedAssistantMessage, type ArchiveEngineReads, type ProjectArchiveLifecycle } from "./session-archive/lifecycle.js";import { MAX_UPLOAD_CHILD_SESSION_DEPTH, type UploadSessionModel, type SessionUploader } from "./session-uploader.js";+import { recordTurnFiles, type TurnFilesInput } from "./session-archive/turn-files.js";/** The engine a captured request went to: base URL, request headers (the engine's auth), query and API generation. */export type EngineTarget = {@@ -701,6 +702,12 @@ export function projectArchiveEngineReads(target: EngineTarget | (() => EngineTaexport function observeUploadedSession(input: {sessionUploader: ObservedUploader;archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete">;+ /**+ * The session's workspace: each settled turn reports the files outside it+ * that the turn touched to the archive, and records the files it read+ * (session-archive/turn-files.ts). Without it, neither.+ */+ turnFiles?: Omit<TurnFilesInput, "sessionId" | "messages" | "collector">;observers: SessionObservers;sessionId: string;target: EngineTarget;@@ -851,6 +858,8 @@ export function observeUploadedSession(input: {}room(newest.bytes);sessionUploader.recordTrace(sessionId, "session.idle", { status });+ // The files the turn read, and those outside the workspace it touched, before the snapshot and the flush.+ if (history && input.turnFiles) await recordTurnFiles({ ...input.turnFiles, sessionId, messages: history.delta, collector: sessionUploader });// The turn snapshot runs first so the artifacts it discovers are part of// the trace flushed right behind it.captureTurnSnapshot();