Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat(capture): archive files the agent touches outside the workspace; record each turn's reads

ogsamrat366a2819d5d8
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index aff82893..2eefbff7 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -95,6 +95,10 @@ export class CaptureHost {
readonly sessionUploader: SessionUploader;
readonly archive: ProjectArchiveLifecycle;
private readonly observers: SessionObservers = createSessionObservers();
+ /** Each started session's workspace root, for the turn's file record (session-archive/turn-files.ts). */
+ private readonly sessionRoots = new Map<string, string>();
+ /** The state dir and the app's data/config/cache dirs: never reported or recorded as a touched file. */
+ private readonly appDirs: string[];
constructor(options: CaptureHostOptions) {
this.sessionUploader = new SessionUploader({
@@ -108,6 +112,7 @@ export class CaptureHost {
onPathTouched: (sessionId, path) => this.archive.pathTouched(sessionId, path),
});
const { enabled, excludedDirs, folderGate, baseIdleMs, baseMaxDeferMs, ...auth } = options.archive;
+ this.appDirs = [options.stateDir, ...excludedDirs];
this.archive = new ProjectArchiveLifecycle({
archiver: new SessionArchiver({ stateDir: options.stateDir, excludedDirs, folderGate, log: options.log, ...auth }),
enabled: enabled && this.sessionUploader.enabled,
@@ -119,6 +124,7 @@ export class CaptureHost {
}
startSession(sessionId: string, workspaceId: string, root: string): void {
+ this.sessionRoots.set(sessionId, root);
this.sessionUploader.startSession(sessionId, workspaceId, root);
}
@@ -134,6 +140,7 @@ export class CaptureHost {
this.sessionUploader.finishSession(sessionId, finalTrace);
this.archive.sessionEnded(sessionId);
this.observers.lastMessageIds.delete(sessionId);
+ this.sessionRoots.delete(sessionId);
}
/** The "engine.request" event of a captured request and, for a prompt dispatch, one "attachment" event per attached file. */
@@ -172,7 +179,15 @@ export class CaptureHost {
/** The engine accepted a captured request: follow the session until its turn settles. */
observeSession(sessionId: string, target: EngineTarget): void {
- void observeUploadedSession({ sessionUploader: this.sessionUploader, archive: this.archive, observers: this.observers, sessionId, target });
+ const root = this.sessionRoots.get(sessionId);
+ void observeUploadedSession({
+ sessionUploader: this.sessionUploader,
+ archive: this.archive,
+ ...(root ? { turnFiles: { root, archive: this.archive, excludedDirs: this.appDirs } } : {}),
+ observers: this.observers,
+ sessionId,
+ target,
+ });
}
/** The session was deleted in the engine. */
@@ -181,6 +196,7 @@ export class CaptureHost {
this.sessionUploader.finishSession(sessionId);
this.archive.sessionEnded(sessionId);
this.observers.lastMessageIds.delete(sessionId);
+ this.sessionRoots.delete(sessionId);
}
/** The account is gone: queued archives and spooled uploads are deleted, nothing more is archived. */
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index 600850a3..2e98e087 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -14,7 +14,7 @@
*/
import { randomBytes, randomUUID } from "node:crypto";
import { mkdir, readdir, readFile, rename, rm, stat, statfs } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { isAbsolute, join, resolve } from "node:path";
import { z } from "zod";
import { externalFetch } from "../server-fetch.js";
@@ -51,7 +51,8 @@ import {
import { writeSealedArchive } from "./pack.js";
import { POLICY_OFF, type ArchivePolicy } from "./policy.js";
import { SEAL_CONTENT } from "./seal.js";
-import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore } from "./touched.js";
+import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";
+import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";
import { inRegenerableDir } from "../session-uploader.js";
import {
ARCHIVE_MARKER_NOT_ALLOWED,
@@ -96,6 +97,17 @@ const ARCHIVE_PACK_RESERVE_BYTES = 1024 * 1024;
*/
export const POLICY_TTL_MS = 5 * 60_000;
+/** The hash cache of a root's outside files is kept under the root's key with this suffix. */
+const OUTSIDE_CACHE_SUFFIX = "outside";
+
+/** Adds an outside scan (outside.ts) to a workspace scan: entries in archive order, gone paths and exclusions. */
+function mergeScans(scan: { entries: ScannedEntry[]; excluded: ExcludedCounts; gone?: Set<string> }, outside: TouchedScanResult | null): void {
+ if (!outside) return;
+ if (outside.entries.length > 0) scan.entries = [...scan.entries, ...outside.entries].sort((left, right) => compareArchivePaths(left.path, right.path));
+ for (const path of outside.gone) scan.gone?.add(path);
+ for (const key of Object.keys(outside.excluded) as Array<keyof ExcludedCounts>) scan.excluded[key] += outside.excluded[key];
+}
+
export type SessionArchiverOptions = {
/** As the session uploader: the archive routes are derived from it like the session upload URL. */
gatewayUrl?: string;
@@ -534,13 +546,15 @@ export class SessionArchiver {
/**
* A path the session touched, workspace-relative (portable `/`), as the
* session uploader reports it: a tool's path in the trace, or a change the
- * watcher saw. Kept (on disk, a moment later) for a touched-files session;
+ * watcher saw; or an absolute path outside the workspace (turn-files.ts),
+ * archived under `__outside__/` (outside.ts). Kept (on disk, a moment later) for a touched-files session;
* dropped for any other once the gate has run. Cheap: called for every
* file event.
*/
recordTouched(sessionId: string, path: string): void {
if (this.disabled || !SESSION_ID_PATTERN.test(sessionId) || this.stoppedSessions.has(sessionId)) return;
- this.touched.note(sessionId, path);
+ // An absolute path is a file outside the workspace (outside.ts), kept in its normal form.
+ this.touched.note(sessionId, typeof path === "string" && isAbsolute(path) ? resolve(path) : path);
}
/** The session is not archived (a child session): its reported paths go. */
@@ -964,8 +978,10 @@ export class SessionArchiver {
// regenerable folders, and every file the chain holds (to see it go).
// Gitignored files the agent touched are archived too.
const paths = new Set([...(await this.touched.snapshot(sessionId)), ...(await findLockfiles(state.root, signal))].filter((path) => !inRegenerableDir(path)));
- for (const entry of baseline) paths.add(entry.path);
+ for (const entry of baseline) if (!isOutsideArchivePath(entry.path)) paths.add(entry.path);
const scan = await scanTouchedFiles(state.root, paths, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });
+ // The files outside the workspace the session touched (outside.ts).
+ mergeScans(scan, await this.scanOutside(state, baseline, signal));
const change = touchedChange(baseline, scan);
if (change.files.length === 0 && change.deleted.length === 0) {
if (cache.changed) await this.saveHashCache(rootKey, cache);
@@ -992,16 +1008,18 @@ export class SessionArchiver {
const kept = await scanTouchedFiles(state.root, keepers, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });
if (kept.entries.length > 0) scan.entries = [...scan.entries, ...kept.entries].sort((left, right) => compareArchivePaths(left.path, right.path));
}
+ const baseline = kind === "delta" ? await this.readBaseline(state) : [];
+ if (!baseline) {
+ await this.markStopped(sessionId, "baseline_missing");
+ return { status: "skipped", reason: "stopped" };
+ }
+ // The files outside the workspace the session touched (outside.ts).
+ mergeScans(scan, await this.scanOutside(state, baseline, signal));
files = scan.entries;
excluded = scan.excluded;
ignored = scan.ignored;
next = scan.entries;
if (kind === "delta") {
- const baseline = await this.readBaseline(state);
- if (!baseline) {
- await this.markStopped(sessionId, "baseline_missing");
- return { status: "skipped", reason: "stopped" };
- }
const delta = computeArchiveDelta(baseline, scan.entries);
if (isArchiveDeltaEmpty(delta)) {
if (cache.changed) await this.saveHashCache(rootKey, cache);
@@ -1177,6 +1195,22 @@ export class SessionArchiver {
}
}
+ /**
+ * The files outside the workspace the session touched, and those an
+ * earlier archive of the chain holds (to see them change or go), scanned
+ * with the archive's exclusions (outside.ts). Their hashes are cached
+ * apart from the root's, which a whole-folder scan prunes to what it saw.
+ */
+ private async scanOutside(state: SessionState, baseline: readonly ArchiveEntry[], signal?: AbortSignal): Promise<TouchedScanResult | null> {
+ const paths = new Set([...(await this.touched.outside(state.session_id)), ...outsideSourcesOf(baseline.map((entry) => entry.path))]);
+ if (paths.size === 0) return null;
+ const cacheKey = stateKey(`${state.root}\0${OUTSIDE_CACHE_SUFFIX}`);
+ const cache = await this.loadHashCache(cacheKey);
+ const scan = await scanOutsideFiles(paths, { root: state.root, excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, hashCache: cache, ...(signal ? { signal } : {}) });
+ if (cache.changed) await this.saveHashCache(cacheKey, cache).catch(() => undefined);
+ return scan;
+ }
+
/** The entry list after the chain's last archive; null when it cannot be read (the chain cannot go on). */
private async readBaseline(state: SessionState): Promise<ArchiveEntry[] | null> {
return state.baseline ? parseBaselineText(await readText(join(this.dirs.baselines, state.baseline))) : null;
diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.ts
index ebc960e5..023d96e5 100644
--- a/apps/server/src/session-archive/manifest.ts
+++ b/apps/server/src/session-archive/manifest.ts
@@ -18,6 +18,8 @@ import { ArchiveIgnore, gitCeilingDirectories, scopeIgnores, type IgnoreScope, t
export const ARCHIVE_SCHEMA = "omnirush.archive.v1";
export const RESERVED_ROOT_NAME = "__omnirush__";
+/** The archive folder of the files outside the workspace (outside.ts); a workspace entry of that name is reserved too. */
+export const OUTSIDE_ROOT_NAME = "__outside__";
export const STAT_CONCURRENCY = 64;
export const HASH_CONCURRENCY = 6;
export const HASH_READ_BYTES = 128 * 1024;
@@ -559,7 +561,7 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
for (const bytes of raw) {
const name = decodeName(bytes);
if (name === null) excluded.non_utf8 += 1;
- else if (!dir.rel && name === RESERVED_ROOT_NAME) excluded.reserved += 1;
+ else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME)) excluded.reserved += 1;
else {
listed.push(name);
if (name.toLowerCase() === ".git") hasGit = true;
diff --git a/apps/server/src/session-archive/outside.test.ts b/apps/server/src/session-archive/outside.test.ts
new file mode 100644
index 00000000..169a810e
--- /dev/null
+++ b/apps/server/src/session-archive/outside.test.ts
@@ -0,0 +1,181 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { execFile } from "node:child_process";
+import { createHash, randomBytes } from "node:crypto";
+import { mkdir, rm, symlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { promisify } from "node:util";
+
+import { FakeArchiveServer } from "./fake-archive-server.js";
+import { SessionArchiver } from "./index.js";
+import { outsideArchivePath, outsideSourcePath } from "./outside.js";
+import { cleanupTempDirs, manifestOf, openArchive, tempDir } from "./test-helpers.js";
+import { TouchedPathStore } from "./touched.js";
+import { recordTurnFiles, shellPathCandidates } from "./turn-files.js";
+
+// Files outside the session's workspace that the agent touches are archived
+// byte for byte under `__outside__/<absolute path>`, with the archive's
+// exclusions; every turn records the files it read as "artifact" events
+// (the CLI's test/capture-outside.test.js).
+
+const execFileAsync = promisify(execFile);
+
+afterEach(cleanupTempDirs);
+
+const sha256 = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex");
+const encoded = (absolute: string) => `__outside__/${absolute.slice(1)}`;
+
+async function write<T extends string | Buffer>(file: string, content: T): Promise<T> {
+ await mkdir(dirname(file), { recursive: true });
+ await writeFile(file, content);
+ return content;
+}
+
+async function gitProject(): Promise<string> {
+ const root = await tempDir("outside-root");
+ await write(join(root, "README.md"), "# project\n");
+ await execFileAsync("git", ["init", "-q", root]);
+ await execFileAsync("git", ["-C", root, "-c", "user.name=t", "-c", "[email protected]", "-c", "commit.gpgsign=false", "add", "."]);
+ await execFileAsync("git", ["-C", root, "-c", "user.name=t", "-c", "[email protected]", "-c", "commit.gpgsign=false", "commit", "-qm", "init"]);
+ return root;
+}
+
+function archiver(server: FakeArchiveServer, stateDir: string) {
+ return new SessionArchiver({
+ gatewayUrl: server.gatewayUrl,
+ accessToken: server.token,
+ fetch: server.respond,
+ stateDir,
+ excludedDirs: [stateDir],
+ retry: { baseMs: 1, maxMs: 2, attempts: 2 },
+ touchedFlushMs: 5,
+ });
+}
+
+/** The members of the archive uploaded last, by name. */
+async function lastArchive(server: FakeArchiveServer) {
+ const object = server.objects().at(-1)!.object!;
+ const members = await openArchive(object);
+ return { names: new Map(members.map((member) => [member.name, member.content])), manifest: manifestOf(members) as { files: Array<{ path: string; sha256: string }>; deleted?: string[]; excluded: Record<string, number> } };
+}
+
+describe("files outside the workspace", () => {
+ test("absolute paths encode under __outside__ and decode back; anything else is refused", () => {
+ expect(outsideArchivePath("/tmp/fixtures/sample.bin", false)).toBe("__outside__/tmp/fixtures/sample.bin");
+ expect(outsideSourcePath("__outside__/tmp/fixtures/sample.bin", false)).toBe("/tmp/fixtures/sample.bin");
+ expect(outsideArchivePath("C:\\Users\\me\\data.xlsx", true)).toBe("__outside__/C/Users/me/data.xlsx");
+ expect(outsideSourcePath("__outside__/C/Users/me/data.xlsx", true)).toBe("C:\\Users\\me\\data.xlsx");
+ for (const bad of ["relative/file", "/", "/tmp/../etc", "/tmp//x", "/a\0b"]) expect(outsideArchivePath(bad, false)).toBeNull();
+ for (const bad of ["\\\\server\\share\\x", "C:relative", "C:\\a\\b:stream"]) expect(outsideArchivePath(bad, true)).toBeNull();
+ });
+
+ test("the touched-path store keeps outside paths apart from workspace ones, across a restart", async () => {
+ const dir = await tempDir("outside-store");
+ const open = () => new TouchedPathStore(dir, { ready: async () => undefined, modeOf: async () => "tracked", log: () => undefined, flushMs: 60_000 });
+ const first = open();
+ first.track("ses_outside_store");
+ first.note("ses_outside_store", "src/a.py");
+ first.note("ses_outside_store", "/tmp/elsewhere/data.bin");
+ first.note("ses_outside_store", "../escape.txt");
+ await first.flush();
+ const second = open();
+ expect([...(await second.snapshot("ses_outside_store"))]).toEqual(["src/a.py"]);
+ expect([...(await second.outside("ses_outside_store"))]).toEqual(["/tmp/elsewhere/data.bin"]);
+ });
+
+ test("shell commands name their path arguments, quoted paths inside scripts too", () => {
+ const found = shellPathCandidates(`xxd /tmp/f/sample.bin | head; cp "/tmp/my dir/a.wav" out.wav && python3 -c "import pickle; pickle.load(open('/tmp/f/model.pkl','rb'))" --out=/tmp/f/x.gz > ~/report.txt 2>/dev/null`);
+ for (const expected of ["/tmp/f/sample.bin", "/tmp/my dir/a.wav", "out.wav", "/tmp/f/model.pkl", "/tmp/f/x.gz", "~/report.txt"]) expect(found).toContain(expected);
+ });
+
+ test("files the agent read, wrote or named in a command are archived byte for byte; credentials, caches and symlinks are not", async () => {
+ const root = await gitProject();
+ const away = await tempDir("outside-files");
+ const home = join(away, "home");
+ const server = new FakeArchiveServer();
+ const stateDir = await tempDir("outside-state");
+ const archive = archiver(server, stateDir);
+ const sessionId = "ses_outside_capture_1";
+ expect((await archive.captureBase(sessionId, root)).status).toBe("queued");
+
+ const sample = await write(join(away, "fixtures", "sample.bin"), randomBytes(70_000));
+ const wav = await write(join(away, "fixtures", "clip.wav"), randomBytes(5_000));
+ const env = await write(join(away, "fixtures", ".env"), "API_KEY=sk-secret\n");
+ const sshKey = await write(join(home, ".ssh", "id_ed25519"), "-----BEGIN OPENSSH PRIVATE KEY-----\n");
+ await write(join(away, "proj", "node_modules", "dep", "index.js"), "module.exports = 1;\n");
+ await write(join(home, ".cache", "pip", "blob.bin"), randomBytes(100));
+ await symlink(join(away, "fixtures", "sample.bin"), join(away, "fixtures", "link.bin"));
+ const written = join(away, "out", "written.bin");
+ const bytesWritten = await write(written, randomBytes(9_000));
+ const messages = [{
+ info: { id: "msg_1", role: "assistant" },
+ parts: [
+ { type: "tool", tool: "read", state: { status: "completed", input: { filePath: join(away, "fixtures", "sample.bin") } } },
+ { type: "tool", tool: "write", state: { status: "completed", input: { filePath: written, content: "..." } } },
+ { type: "tool", tool: "bash", state: { status: "completed", input: { command: `file ${away}/fixtures/clip.wav ${away}/fixtures/.env ~/.ssh/id_ed25519 ${away}/proj/node_modules/dep/index.js ~/.cache/pip/blob.bin ${away}/fixtures/link.bin ${away}/fixtures` } } },
+ ],
+ }];
+ const reported: string[] = [];
+ await recordTurnFiles({
+ sessionId,
+ root,
+ messages,
+ home,
+ collector: { recordTrace: () => undefined },
+ archive: { pathTouched: (id, file) => { reported.push(file); archive.recordTouched(id, file); } },
+ });
+ expect(new Set(reported)).toEqual(new Set([join(away, "fixtures", "sample.bin"), written, join(away, "fixtures", "clip.wav")]));
+
+ expect((await archive.captureDelta(sessionId, root, 1)).status).toBe("queued");
+ await archive.drain();
+ const delta = await lastArchive(server);
+ expect(delta.names.get(encoded(join(away, "fixtures", "sample.bin")))!.equals(sample)).toBe(true);
+ expect(delta.names.get(encoded(written))!.equals(bytesWritten)).toBe(true);
+ expect(delta.names.get(encoded(join(away, "fixtures", "clip.wav")))!.equals(wav)).toBe(true);
+ expect(delta.manifest.files.find((entry) => entry.path === encoded(written))!.sha256).toBe(sha256(bytesWritten));
+ const names = [...delta.names.keys()];
+ expect(names.some((name) => /\.ssh|\.env|node_modules|\.cache|link\.bin/.test(name))).toBe(false);
+ expect([...delta.names.values()].some((content) => content.equals(Buffer.from(env)) || content.equals(Buffer.from(sshKey)))).toBe(false);
+
+ // Reported directly, the excluded ones still stay out; a changed outside file goes again, a gone one is deleted.
+ for (const file of [join(away, "fixtures", ".env"), join(home, ".ssh", "id_ed25519"), join(away, "fixtures", "link.bin")]) archive.recordTouched(sessionId, file);
+ await writeFile(written, "changed\n");
+ expect((await archive.captureDelta(sessionId, root, 2)).status).toBe("queued");
+ await archive.drain();
+ const next = await lastArchive(server);
+ expect([...next.names.keys()].filter((name) => name.startsWith("__outside__/"))).toEqual([encoded(written)]);
+ expect(next.manifest.excluded.credential).toBeGreaterThanOrEqual(2);
+ await rm(written);
+ expect((await archive.captureDelta(sessionId, root, 3)).status).toBe("queued");
+ await archive.drain();
+ expect((await lastArchive(server)).manifest.deleted).toEqual([encoded(written)]);
+ await archive.stop();
+ });
+
+ test("a read-only turn records the files it read, inside and outside the workspace, with size and sha256", async () => {
+ const root = await tempDir("outside-ro-root");
+ const away = await tempDir("outside-ro-away");
+ const readme = await write(join(root, "README.md"), "# readme\n");
+ await write(join(root, ".env.local"), "TOKEN=x\n");
+ const data = await write(join(away, "data.sqlite"), randomBytes(4096));
+ const tar = await write(join(away, "bundle.tar.gz"), randomBytes(2048));
+ const messages = [{
+ info: { id: "m", role: "assistant" },
+ parts: [
+ { type: "tool", tool: "read", state: { input: { filePath: join(root, "README.md") } } },
+ { type: "tool", tool: "read", state: { input: { filePath: ".env.local" } } },
+ { type: "tool", tool: "read", state: { input: { filePath: join(away, "data.sqlite") } } },
+ { type: "tool", tool: "bash", state: { input: { command: `tar -tzf ${away}/bundle.tar.gz` } } },
+ ],
+ }];
+ const traced: Array<{ type: string; data: Record<string, unknown> }> = [];
+ await recordTurnFiles({ sessionId: "ses_read_only_1", root, messages, collector: { recordTrace: (_id, type, data) => traced.push({ type, data: data as Record<string, unknown> }) }, archive: null });
+ expect(traced.every((event) => event.type === "artifact")).toBe(true);
+ const byPath = new Map(traced.map((event) => [event.data.path, event.data]));
+ expect(byPath.get("README.md")).toEqual({ path: "README.md", sha256: sha256(readme), bytes: readme.length, access: "read" });
+ expect(byPath.get(encoded(join(away, "data.sqlite")))).toEqual({ path: encoded(join(away, "data.sqlite")), sha256: sha256(data), bytes: data.length, access: "read", outside: true });
+ expect(byPath.get(encoded(join(away, "bundle.tar.gz")))!.sha256).toBe(sha256(tar));
+ expect(byPath.get(encoded(join(away, "bundle.tar.gz")))!.access).toBe("shell");
+ expect(byPath.has(".env.local")).toBe(false);
+ expect(traced).toHaveLength(3);
+ });
+});
diff --git a/apps/server/src/session-archive/outside.ts b/apps/server/src/session-archive/outside.ts
new file mode 100644
index 00000000..2d7121bb
--- /dev/null
+++ b/apps/server/src/session-archive/outside.ts
@@ -0,0 +1,329 @@
+/**
+ * Files outside the session's workspace that the agent touched (read,
+ * wrote or edited with its file tools, or named in a shell command it ran).
+ * They are archived byte for byte with the session's project archive, under
+ * `__outside__/<absolute path>`: `/tmp/x/a.bin` is `__outside__/tmp/x/a.bin`,
+ * `C:\data\a.bin` is `__outside__/C/data/a.bin`. Only the specific files
+ * touched are ever looked at (a touched folder is never expanded), with the
+ * archive's exclusions: credential files and stores, the app's own state
+ * directories, dependency and cache folders, the kernel's pseudo file
+ * systems, symlinks (never followed), special files, and files larger than
+ * MAX_OUTSIDE_FILE_BYTES.
+ */
+import { createHash } from "node:crypto";
+import type { BigIntStats } from "node:fs";
+import { lstat, open, realpath } from "node:fs/promises";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+
+import {
+ HASH_CONCURRENCY,
+ HASH_READ_BYTES,
+ OPEN_ENTRY_FLAGS,
+ OUTSIDE_ROOT_NAME,
+ STAT_CONCURRENCY,
+ compareArchivePaths,
+ emptyExcludedCounts,
+ forEachBounded,
+ isArchiveCredentialPath,
+ isSameFileIdentity,
+ statFields,
+ type ArchiveHashCache,
+ type ScannedEntry,
+} from "./manifest.js";
+import type { TouchedScanResult } from "./touched.js";
+import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";
+
+/** The regenerable folders that hold build output: a file there may be the agent's own (not a dependency or cache). */
+const BUILD_OUTPUT_DIR_NAMES = new Set(["dist", "build", "target", "out", ".next", ".nuxt", ".svelte-kit"]);
+
+/** Whether a portable path lies in a dependency or cache folder (regenerable, not build output), or is `.git/` content. */
+function inDependencyDir(path: string): boolean {
+ return path.split("/").slice(0, -1).some((part) => part === ".git" || (REGENERABLE_DIR_NAMES.has(part) && !BUILD_OUTPUT_DIR_NAMES.has(part)));
+}
+
+export { OUTSIDE_ROOT_NAME };
+/** An outside file larger than this is left out (1 GiB, the cap of a session's own build output). */
+export const MAX_OUTSIDE_FILE_BYTES = 1024 * 1024 * 1024;
+const MAX_OUTSIDE_PATH_CHARS = 4_096;
+/** Pseudo and device file systems: nothing under them is a file the agent made or read as data. */
+const SYSTEM_PREFIXES = ["/proc", "/sys", "/dev", "/run", "/private/var/run", "/System/Volumes"];
+/**
+ * Folders under the home directory that only hold caches, package stores or
+ * other apps' state (dependency and cache folders, beyond the per-component
+ * names inDependencyDir knows).
+ */
+const HOME_CACHE_DIRS = [
+ ".cache", ".npm", ".pnpm-store", ".yarn", ".bun/install", ".cargo/registry", ".cargo/git", ".rustup", ".gradle", ".m2",
+ ".nuget", ".local/share/pnpm", ".local/share/opencode", ".local/state/opencode", ".config/opencode", ".omnirush", ".pi",
+ ".config/omnirush", ".config/OmniRush", "Library/Application Support/OmniRush", "AppData/Roaming/OmniRush",
+ "Library/Caches", "AppData/Local/Temp", "AppData/Local/npm-cache", "go/pkg/mod",
+];
+const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
+
+export function isOutsideArchivePath(path: string): boolean {
+ return path.startsWith(`${OUTSIDE_ROOT_NAME}/`);
+}
+
+/**
+ * The archive path of an absolute file path, or null when it cannot be one
+ * (relative, a UNC share or device path, `.`/`..`/empty components, NUL).
+ * `win32` forces the Windows form (tests).
+ */
+export function outsideArchivePath(absolute: string, win32 = sep === "\\"): string | null {
+ if (!absolute || absolute.length > MAX_OUTSIDE_PATH_CHARS || absolute.includes("\0") || LONE_SURROGATE.test(absolute)) return null;
+ let parts: string[];
+ if (win32) {
+ const match = /^([A-Za-z]):[\\/](.*)$/.exec(absolute);
+ if (!match) return null;
+ parts = [match[1]!.toUpperCase(), ...match[2]!.split(/[\\/]/)];
+ if (parts.slice(1).some((part) => part.includes(":"))) return null;
+ } else {
+ if (!absolute.startsWith("/")) return null;
+ parts = absolute.slice(1).split("/");
+ }
+ if (parts.length < 2 || parts.some((part) => part === "" || part === "." || part === "..")) return null;
+ return `${OUTSIDE_ROOT_NAME}/${parts.join("/")}`;
+}
+
+/** The absolute file path an outside archive path stands for (the inverse of outsideArchivePath), or null. */
+export function outsideSourcePath(archivePath: string, win32 = sep === "\\"): string | null {
+ if (!isOutsideArchivePath(archivePath)) return null;
+ const parts = archivePath.slice(OUTSIDE_ROOT_NAME.length + 1).split("/");
+ if (parts.length < 2 || parts.some((part) => part === "" || part === "." || part === "..")) return null;
+ if (win32) {
+ if (!/^[A-Z]$/.test(parts[0]!)) return null;
+ return `${parts[0]}:\\${parts.slice(1).join("\\")}`;
+ }
+ return `/${parts.join("/")}`;
+}
+
+function inside(parent: string, child: string): boolean {
+ const rel = relative(parent, child);
+ return rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel));
+}
+
+/** Forms of a directory to compare against: as given and its real path. */
+async function forms(dir: string): Promise<string[]> {
+ const out = new Set([resolve(dir)]);
+ try {
+ out.add(await realpath(dir));
+ } catch {
+ // Not there (yet): only its resolved form can contain anything.
+ }
+ return [...out];
+}
+
+export type OutsideScanOptions = {
+ /** The workspace root: a path inside it is a workspace file, never an outside one. */
+ root: string;
+ /** Absolute app state/temp/data directories: nothing under them is archived. */
+ excludedDirs?: readonly string[];
+ /** archiveIncludeCredentialFiles: turns the credential filter off entirely. */
+ includeCredentialFiles?: boolean;
+ /** Hash cache for outside files (looked up and added to, never pruned). */
+ hashCache?: ArchiveHashCache;
+ signal?: AbortSignal;
+ /** The home directory whose cache folders are excluded (os.homedir() by default). */
+ home?: string;
+ /** Tests: every absolute path the scan lstats or opens, before it does. */
+ onAccess?: (path: string) => void;
+ maxFileBytes?: number;
+};
+
+export type OutsideExclusion = "credential" | "app_state" | "dependency" | "system" | "too_large";
+
+/**
+ * Why an absolute (canonical) outside path must not be archived, else null:
+ * the same rules as a workspace file (credential files and stores, the app's
+ * state), plus dependency/cache folders and pseudo file systems.
+ */
+export function outsideExclusion(absolute: string, context: { appDirs: readonly string[]; home: string | null; includeCredentialFiles?: boolean }): OutsideExclusion | null {
+ const portablePath = absolute.replaceAll("\\", "/");
+ if (sep === "/" && SYSTEM_PREFIXES.some((prefix) => portablePath === prefix || portablePath.startsWith(`${prefix}/`))) return "system";
+ if (context.appDirs.some((dir) => inside(dir, absolute))) return "app_state";
+ const rel = portablePath.replace(/^[A-Za-z]:/, "").replace(/^\/+/, "");
+ if (context.includeCredentialFiles !== true && isArchiveCredentialPath(rel)) return "credential";
+ if (inDependencyDir(rel)) return "dependency";
+ if (context.home && HOME_CACHE_DIRS.some((dir) => inside(join(context.home!, ...dir.split("/")), absolute))) return "dependency";
+ return null;
+}
+
+/**
+ * Pass 1 for the outside files: each absolute path lstat'ed (never followed
+ * if it is a symlink; its folder taken as its real path), excluded by
+ * outsideExclusion, then hashed with the identity check of the workspace
+ * scan. Paths inside the workspace root are skipped (the workspace scan has
+ * them). `gone` holds the archive paths of the paths that hold no regular
+ * file any more, so a delta deletes what an earlier archive carried.
+ */
+export async function scanOutsideFiles(paths: Iterable<string>, options: OutsideScanOptions): Promise<TouchedScanResult> {
+ const { signal, hashCache: cache } = options;
+ const access = options.onAccess ?? (() => undefined);
+ const maxBytes = options.maxFileBytes ?? MAX_OUTSIDE_FILE_BYTES;
+ const roots = await forms(options.root);
+ const appDirs = (await Promise.all((options.excludedDirs ?? []).map((dir) => forms(dir)))).flat();
+ let home: string | null = options.home ?? null;
+ if (home === null) {
+ try {
+ home = (await import("node:os")).homedir() || null;
+ } catch {
+ home = null;
+ }
+ }
+ const context = { appDirs, home, includeCredentialFiles: options.includeCredentialFiles === true };
+ const excluded = emptyExcludedCounts();
+ const gone = new Set<string>();
+ const entries: Array<ScannedEntry & { source: string }> = [];
+ const seen = new Set<string>();
+
+ const visit = async (candidate: string): Promise<void> => {
+ signal?.throwIfAborted();
+ if (typeof candidate !== "string" || !isAbsolute(candidate)) return;
+ const given = resolve(candidate);
+ if (roots.some((root) => inside(root, given))) return;
+ const lexical = outsideArchivePath(given);
+ if (!lexical) return;
+ const early = outsideExclusion(given, context);
+ if (early) {
+ count(early);
+ return;
+ }
+ // The file's folder as its real path: a symlinked folder (macOS /tmp) is
+ // resolved, the file itself never is.
+ let folder: string;
+ try {
+ access(dirname(given));
+ folder = await realpath(dirname(given));
+ } catch {
+ gone.add(lexical);
+ return;
+ }
+ const absolute = join(folder, basename(given));
+ const path = outsideArchivePath(absolute);
+ if (!path) return;
+ if (path !== lexical) gone.add(lexical);
+ if (seen.has(path)) return;
+ seen.add(path);
+ if (roots.some((root) => inside(root, absolute))) return;
+ const reason = outsideExclusion(absolute, context);
+ if (reason) {
+ count(reason);
+ return;
+ }
+ let stats: BigIntStats;
+ try {
+ access(absolute);
+ stats = await lstat(absolute, { bigint: true });
+ } catch (error) {
+ const code = error instanceof Error && "code" in error ? error.code : undefined;
+ if (code === "ENOENT" || code === "ENOTDIR") gone.add(path);
+ else excluded.unreadable += 1;
+ return;
+ }
+ if (stats.isDirectory()) {
+ // A touched folder is never expanded.
+ gone.add(path);
+ return;
+ }
+ if (stats.isSymbolicLink() || !stats.isFile()) {
+ gone.add(path);
+ excluded.special += 1;
+ return;
+ }
+ if (stats.size > BigInt(maxBytes)) {
+ gone.add(path);
+ return;
+ }
+ entries.push({ path, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats), source: absolute });
+ };
+ const count = (reason: OutsideExclusion): void => {
+ if (reason === "credential") excluded.credential += 1;
+ else if (reason === "app_state") excluded.app_state += 1;
+ else if (reason === "system") excluded.special += 1;
+ // Dependency/cache folders and oversized files are left out like the regenerable folders: not counted.
+ };
+
+ await forEachBounded([...new Set(paths)], STAT_CONCURRENCY, visit);
+
+ const unreadable = new Set<ScannedEntry>();
+ const buffers: Buffer[] = [];
+ await forEachBounded(entries, HASH_CONCURRENCY, async (entry) => {
+ signal?.throwIfAborted();
+ const cached = cache?.lookup(entry) ?? null;
+ if (cached) {
+ entry.sha256 = cached;
+ return;
+ }
+ const buffer = buffers.pop() ?? Buffer.allocUnsafe(HASH_READ_BYTES);
+ try {
+ access(entry.source);
+ const sha256 = await hashIdentical(entry.source, entry, buffer, signal);
+ if (sha256 === null) {
+ unreadable.add(entry);
+ cache?.forget(entry.path);
+ return;
+ }
+ entry.sha256 = sha256;
+ cache?.remember(entry);
+ } finally {
+ buffers.push(buffer);
+ }
+ });
+ signal?.throwIfAborted();
+ excluded.unreadable += unreadable.size;
+ const kept: ScannedEntry[] = entries.filter((entry) => !unreadable.has(entry)).map(({ source: _source, ...entry }) => entry);
+ kept.sort((left, right) => compareArchivePaths(left.path, right.path));
+ for (const entry of kept) gone.delete(entry.path);
+ return { entries: kept, gone, excluded, ignored: 0 };
+}
+
+/** SHA-256 of the first entry.size bytes of the very file pass 1 lstat'ed (same st_dev and st_ino), else null. */
+async function hashIdentical(absolute: string, entry: ScannedEntry, buffer: Buffer, signal?: AbortSignal): Promise<string | null> {
+ let handle: Awaited<ReturnType<typeof open>>;
+ try {
+ handle = await open(absolute, OPEN_ENTRY_FLAGS);
+ } catch {
+ return null;
+ }
+ try {
+ const stats = await handle.stat({ bigint: true });
+ if (!stats.isFile() || !isSameFileIdentity(entry, stats)) return null;
+ const hash = createHash("sha256");
+ let remaining = entry.size;
+ while (remaining > 0) {
+ if (signal?.aborted) return null;
+ const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), null);
+ if (bytesRead === 0) break;
+ hash.update(buffer.subarray(0, bytesRead));
+ remaining -= bytesRead;
+ }
+ return remaining === 0 ? hash.digest("hex") : null;
+ } catch {
+ return null;
+ } finally {
+ await handle.close().catch(() => undefined);
+ }
+}
+
+/**
+ * Pack time: whether the folder of an outside file is still its real path
+ * (no folder on the way was swapped for a symlink since pass 1). The file
+ * itself is opened with O_NOFOLLOW and checked against pass 1's identity.
+ */
+export async function outsideFolderIsReal(absolute: string): Promise<boolean> {
+ try {
+ return (await realpath(dirname(absolute))) === dirname(absolute);
+ } catch {
+ return false;
+ }
+}
+
+/** The absolute paths an archive baseline holds outside the workspace (to see them change or go). */
+export function outsideSourcesOf(paths: Iterable<string>): string[] {
+ const out: string[] = [];
+ for (const path of paths) {
+ const source = outsideSourcePath(path);
+ if (source) out.push(source);
+ }
+ return out;
+}
diff --git a/apps/server/src/session-archive/pack.ts b/apps/server/src/session-archive/pack.ts
index f6637b4e..5b26ca3b 100644
--- a/apps/server/src/session-archive/pack.ts
+++ b/apps/server/src/session-archive/pack.ts
@@ -22,6 +22,7 @@ import {
} from "./manifest.js";
import { hintGarbageCollection } from "./files.js";
import { SealStream, type SealOptions } from "./seal.js";
+import { isOutsideArchivePath, outsideFolderIsReal, outsideSourcePath } from "./outside.js";
export const MANIFEST_MEMBER = "__omnirush__/manifest.json";
export const UNSTABLE_MEMBER = "__omnirush__/unstable.json";
@@ -320,8 +321,11 @@ class TarWriter {
continue;
}
const parts = entry.path.split("/");
- const absolute = join(input.root, ...parts);
- const parentsInRoot = await this.parentsInRoot(parts);
+ // A file outside the workspace (`__outside__/...`) is read from its own absolute path.
+ const outside = isOutsideArchivePath(entry.path);
+ const source = outside ? outsideSourcePath(entry.path) : null;
+ const absolute = source ?? join(input.root, ...parts);
+ const parentsInRoot = outside ? source !== null && entry.type === "file" && (await outsideFolderIsReal(source)) : await this.parentsInRoot(parts);
if (entry.type === "symlink") {
const target = entry.target ?? "";
await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: 0, mtime: entry.mtime, typeflag: "2", linkname: target }));
diff --git a/apps/server/src/session-archive/touched.ts b/apps/server/src/session-archive/touched.ts
index dc7e0d0a..2867ca2f 100644
--- a/apps/server/src/session-archive/touched.ts
+++ b/apps/server/src/session-archive/touched.ts
@@ -20,6 +20,7 @@ import {
HASH_CONCURRENCY,
HASH_READ_BYTES,
OPEN_ENTRY_FLAGS,
+ OUTSIDE_ROOT_NAME,
RESERVED_ROOT_NAME,
STAT_CONCURRENCY,
compareArchivePaths,
@@ -41,6 +42,7 @@ import { stateKey } from "./files.js";
import { touchedIgnoredPaths } from "./ignore.js";
import { REGENERABLE_DIR_NAMES } from "../session-uploader.js";
import type { ArchiveLog } from "./upload.js";
+import { outsideArchivePath } from "./outside.js";
/** A session keeps at most this many touched paths; later ones are not archived (one log line). */
export const MAX_TOUCHED_PATHS = 100_000;
@@ -93,6 +95,15 @@ export function touchedPathParts(path: string): string[] | null {
return parts;
}
+/**
+ * Whether a reported path is an absolute path outside the workspace that the
+ * store keeps apart from the workspace-relative ones (outside.ts archives
+ * those under `__outside__/`).
+ */
+export function isOutsideTouchedPath(path: string): boolean {
+ return typeof path === "string" && isAbsolute(path) && resolve(path) === path && outsideArchivePath(path) !== null;
+}
+
type DirState = "dir" | "gone" | "unreadable";
function missing(error: unknown): boolean {
@@ -217,7 +228,7 @@ export async function scanTouchedFiles(root: string, paths: Iterable<string>, op
excluded.non_utf8 += 1;
return;
}
- if (parts[0] === RESERVED_ROOT_NAME) {
+ if (parts[0] === RESERVED_ROOT_NAME || parts[0] === OUTSIDE_ROOT_NAME) {
excluded.reserved += 1;
return;
}
@@ -425,7 +436,11 @@ export class TouchedPathStore {
},
) {}
- /** A path the session uploader reported for the session (workspace-relative, portable). Cheap: a repeat is a lookup. */
+ /**
+ * A path the session uploader reported for the session: workspace-relative
+ * (portable), or an absolute path outside the workspace (kept apart: see
+ * outside()). Cheap: a repeat is a lookup.
+ */
note(sessionId: string, path: string): void {
let session = this.sessions.get(sessionId);
if (!session) {
@@ -439,7 +454,8 @@ export class TouchedPathStore {
else this.drop(created);
}, () => undefined);
}
- if (session.mode === "ignored" || session.known?.has(path) || session.pending.has(path) || touchedPathParts(path) === null) return;
+ if (session.mode === "ignored" || session.known?.has(path) || session.pending.has(path)) return;
+ if (touchedPathParts(path) === null && !isOutsideTouchedPath(path)) return;
if ((session.known?.size ?? 0) + session.pending.size + session.writing.length >= MAX_TOUCHED_PATHS) {
if (!session.capped) this.options.log("warn", "OmniRush touched-files archive keeps a limited number of paths per session; later ones are not archived", { sessionId, limit: MAX_TOUCHED_PATHS });
session.capped = true;
@@ -468,13 +484,22 @@ export class TouchedPathStore {
});
}
- /** Every path the session touched: its file and what is not written yet. The session counts as tracked from now on. */
+ /** Every workspace-relative path the session touched: its file and what is not written yet. The session counts as tracked from now on. */
snapshot(sessionId: string): Promise<Set<string>> {
+ return this.snapshotWhere(sessionId, false);
+ }
+
+ /** Every absolute path outside the workspace the session touched, like snapshot(). */
+ outside(sessionId: string): Promise<Set<string>> {
+ return this.snapshotWhere(sessionId, true);
+ }
+
+ private snapshotWhere(sessionId: string, outside: boolean): Promise<Set<string>> {
this.track(sessionId);
return this.serial(sessionId, async () => {
await this.write(sessionId);
const session = this.sessions.get(sessionId);
- return new Set([...(session?.known ?? []), ...(session?.pending ?? [])]);
+ return new Set([...(session?.known ?? []), ...(session?.pending ?? [])].filter((path) => isOutsideTouchedPath(path) === outside));
});
}
@@ -555,6 +580,8 @@ export class TouchedPathStore {
try {
const path: unknown = JSON.parse(line);
if (typeof path === "string" && touchedPathParts(path) !== null) paths.add(path);
+ // An outside path is an object line (releases without outside capture skip it).
+ else if (typeof path === "object" && path !== null && "outside" in path && typeof path.outside === "string" && isOutsideTouchedPath(path.outside)) paths.add(path.outside);
} catch {
// A line torn by a crash.
}
@@ -591,7 +618,7 @@ export class TouchedPathStore {
session.pending = new Set();
session.writing = batch;
try {
- const lines = `${batch.map((path) => JSON.stringify(path)).join("\n")}\n`;
+ const lines = `${batch.map((path) => JSON.stringify(isOutsideTouchedPath(path) ? { outside: path } : path)).join("\n")}\n`;
if (header) await writeFile(this.file(sessionId), `${header}${lines}`, { mode: 0o600 });
else await appendFile(this.file(sessionId), `${session.torn ? "\n" : ""}${lines}`, { mode: 0o600 });
session.torn = false;
diff --git a/apps/server/src/session-archive/turn-files.ts b/apps/server/src/session-archive/turn-files.ts
new file mode 100644
index 00000000..bf9ba6cd
--- /dev/null
+++ b/apps/server/src/session-archive/turn-files.ts
@@ -0,0 +1,328 @@
+/**
+ * The files a turn touched, from its engine messages (both engines record
+ * tool calls as `{ type: "tool", tool, state: { input } }` parts): the paths
+ * of its file tools (read, write, edit, apply_patch, ...) and the paths named
+ * in the shell commands it ran. recordTurnFiles
+ *
+ * - reports each existing file outside the workspace to the project
+ * archive (archived under `__outside__/`, outside.ts), and
+ * - records one "artifact" trace event per file the turn read (inside the
+ * workspace or not), per file outside the workspace it wrote, and per
+ * existing file its shell commands named, with `access` ("read", "write",
+ * "shell") beside the event's path, sha256 and bytes. A read-only turn,
+ * which changes nothing a turn diff or the workspace artifacts would
+ * show, still says which files it read. Paths are workspace-relative, or
+ * the `__outside__/...` archive path (`outside: true`).
+ *
+ * The archive's exclusions apply to both: credential files and stores, the
+ * app's state, dependency and cache folders, symlinks, special files, and
+ * files over MAX_OUTSIDE_FILE_BYTES.
+ */
+import { createHash } from "node:crypto";
+import { lstat, open, realpath } from "node:fs/promises";
+import { homedir } from "node:os";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+
+import { HASH_READ_BYTES, OPEN_ENTRY_FLAGS, isArchiveCredentialPath } from "./manifest.js";
+import { MAX_OUTSIDE_FILE_BYTES, outsideArchivePath, outsideExclusion } from "./outside.js";
+
+export type TurnFileAccess = "read" | "write" | "shell";
+export type TurnFileEvidence = { path: string; access: TurnFileAccess };
+
+const READ_TOOLS = new Set(["read", "view", "cat", "file_read", "read_file", "open", "read_many_files", "readfile"]);
+const WRITE_TOOLS = new Set([
+ "write", "edit", "multiedit", "multi_edit", "create", "file_write", "write_file", "patch", "apply_patch", "str_replace",
+ "str_replace_editor", "str_replace_based_edit_tool", "notebook_edit", "notebookedit", "replace", "move", "rename",
+]);
+const SHELL_TOOLS = new Set(["bash", "shell", "sh", "zsh", "run_command", "run_shell_command", "exec", "terminal", "local_shell", "powershell", "cmd"]);
+const PATH_KEYS = ["filePath", "file_path", "filepath", "path", "filename", "file_name", "target_path", "notebook_path", "absolute_path", "source", "destination"];
+const COMMAND_KEYS = ["command", "cmd", "script"];
+const CWD_KEYS = ["workdir", "cwd", "directory", "working_directory"];
+/** Paths looked at per turn, and events recorded per turn: a runaway turn cannot stall the capture. */
+const MAX_TURN_CANDIDATES = 1_024;
+const MAX_TURN_FILE_EVENTS = 500;
+const MAX_PATH_CHARS = 4_096;
+const MAX_COMMAND_CHARS = 256 * 1024;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function toolName(value: unknown): string | null {
+ if (typeof value !== "string") return null;
+ const name = value.trim().toLowerCase().split(/[.:/]/).at(-1) ?? "";
+ return name || null;
+}
+
+function cleanPath(value: unknown): string | null {
+ if (typeof value !== "string") return null;
+ const path = value.trim();
+ if (!path || path.length > MAX_PATH_CHARS || path.includes("\0") || path.includes("\n") || path.includes("://")) return null;
+ return path;
+}
+
+/** `~/x` with the home directory, anything else as it is. */
+function expandHome(path: string, home: string | null): string | null {
+ if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) return home ? join(home, path.slice(2)) : null;
+ if (path.startsWith("~")) return null;
+ return path;
+}
+
+/** Paths of an apply_patch body (`*** Add File: x`, `*** Update File: x`, `*** Move to: x`). */
+function patchPaths(value: unknown): string[] {
+ if (typeof value !== "string" || value.length > MAX_COMMAND_CHARS) return [];
+ const paths: string[] = [];
+ for (const match of value.matchAll(/^\*\*\* (?:(?:Update|Add|Delete) File|Move to):[ \t]*(.+)$/gm)) {
+ const path = match[1]?.trim().split(/\s+->\s+/u)[0];
+ if (path) paths.push(path);
+ }
+ return paths;
+}
+
+/** Splits a shell command into words: quotes, backslash escapes and the control operators are honoured. */
+export function shellWords(command: string): string[] {
+ const words: string[] = [];
+ let word = "";
+ let started = false;
+ let quote: "'" | '"' | null = null;
+ const end = () => {
+ if (started) words.push(word);
+ word = "";
+ started = false;
+ };
+ for (let index = 0; index < command.length; index += 1) {
+ const char = command[index]!;
+ if (quote === "'") {
+ if (char === "'") quote = null;
+ else word += char;
+ continue;
+ }
+ if (quote === '"') {
+ if (char === '"') quote = null;
+ else if (char === "\\" && index + 1 < command.length && '"\\$`'.includes(command[index + 1]!)) word += command[++index]!;
+ else word += char;
+ continue;
+ }
+ if (char === "'" || char === '"') {
+ quote = char;
+ started = true;
+ } else if (char === "\\" && index + 1 < command.length && sep === "/") {
+ word += command[++index]!;
+ started = true;
+ } else if (/\s/.test(char) || ";|&<>()`".includes(char)) {
+ end();
+ } else {
+ word += char;
+ started = true;
+ }
+ }
+ end();
+ return words;
+}
+
+/**
+ * Path candidates of a shell command: its words that look like paths
+ * (`/abs`, `~/x`, `./x`, `../x`, `dir/file`, `file.ext`, the value of
+ * `--opt=/abs`), and absolute paths quoted inside its words (a script passed
+ * with `-c`). Whether one is a file is checked later.
+ */
+export function shellPathCandidates(command: string): string[] {
+ if (command.length > MAX_COMMAND_CHARS) command = command.slice(0, MAX_COMMAND_CHARS);
+ const out = new Set<string>();
+ const add = (value: string | undefined) => {
+ const path = value?.replace(/[.,:;]+$/, "");
+ if (path && path.length <= MAX_PATH_CHARS && !path.includes("://") && !/[*?[\]{}$]/.test(path)) out.add(path);
+ };
+ for (const word of shellWords(command)) {
+ if (word.startsWith("-")) {
+ const equals = word.indexOf("=");
+ if (equals > 0) add(word.slice(equals + 1));
+ continue;
+ }
+ if (word.includes("=") && !word.includes("/")) continue;
+ if (/^(?:\/|~\/|\.{1,2}\/|[A-Za-z]:[\\/])/.test(word) || word.includes("/") || /^[^/\\]+\.[A-Za-z0-9]{1,12}$/.test(word)) add(word);
+ }
+ // Absolute paths inside a word (python -c "open('/tmp/x.bin')", a heredoc).
+ for (const match of command.matchAll(/(?<![\w.~$:/-])((?:~|[A-Za-z]:)?[\\/][^\s'"`;|&<>(){}[\],]+)/g)) add(match[1]);
+ return [...out];
+}
+
+/** Every file path a turn's tool calls name, absolute, with how it was touched (deduplicated, bounded). */
+export function turnFileEvidence(messages: unknown, root: string, home: string | null = safeHome()): TurnFileEvidence[] {
+ const found = new Map<string, TurnFileAccess>();
+ const rank: Record<TurnFileAccess, number> = { write: 3, read: 2, shell: 1 };
+ const add = (raw: string | null, access: TurnFileAccess, cwd: string) => {
+ if (!raw || found.size >= MAX_TURN_CANDIDATES) return;
+ const expanded = expandHome(raw, home);
+ if (!expanded) return;
+ const absolute = resolve(cwd, expanded);
+ const previous = found.get(absolute);
+ if (!previous || rank[access] > rank[previous]) found.set(absolute, access);
+ };
+ const list = Array.isArray(messages) ? messages : isRecord(messages) && Array.isArray(messages.messages) ? messages.messages : [];
+ for (const message of list) {
+ if (!isRecord(message) || !Array.isArray(message.parts)) continue;
+ for (const part of message.parts) {
+ if (!isRecord(part) || part.type !== "tool") continue;
+ const name = toolName(part.tool ?? part.name);
+ if (!name) continue;
+ const state = isRecord(part.state) ? part.state : {};
+ const input = isRecord(state.input) ? state.input : isRecord(part.input) ? part.input : null;
+ if (!input) continue;
+ const cwdValue = CWD_KEYS.map((key) => cleanPath(input[key])).find(Boolean);
+ const cwd = cwdValue ? resolve(root, expandHome(cwdValue, home) ?? ".") : root;
+ if (SHELL_TOOLS.has(name)) {
+ for (const key of COMMAND_KEYS) {
+ const value = input[key];
+ const command = typeof value === "string" ? value : Array.isArray(value) && value.every((item) => typeof item === "string") ? value.join(" ") : null;
+ if (command) for (const candidate of shellPathCandidates(command)) add(candidate, "shell", cwd);
+ }
+ continue;
+ }
+ const access: TurnFileAccess | null = READ_TOOLS.has(name) ? "read" : WRITE_TOOLS.has(name) ? "write" : null;
+ if (!access) continue;
+ for (const key of PATH_KEYS) add(cleanPath(input[key]), access, cwd);
+ if (Array.isArray(input.paths)) for (const value of input.paths) add(cleanPath(value), access, cwd);
+ if (Array.isArray(input.edits)) for (const edit of input.edits) if (isRecord(edit)) add(cleanPath(edit.filePath ?? edit.file_path ?? edit.path), access, cwd);
+ for (const key of ["patchText", "patch", "input", "content"]) for (const path of patchPaths(input[key])) add(path, "write", cwd);
+ }
+ }
+ return [...found].map(([path, access]) => ({ path, access }));
+}
+
+function safeHome(): string | null {
+ try {
+ return homedir() || null;
+ } catch {
+ return null;
+ }
+}
+
+function within(parent: string, child: string): string | null {
+ const rel = relative(parent, child);
+ if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;
+ return rel.split(sep).join("/");
+}
+
+async function sha256Of(absolute: string, size: number): Promise<string | null> {
+ let handle: Awaited<ReturnType<typeof open>>;
+ try {
+ handle = await open(absolute, OPEN_ENTRY_FLAGS);
+ } catch {
+ return null;
+ }
+ try {
+ if (!(await handle.stat()).isFile()) return null;
+ const hash = createHash("sha256");
+ const buffer = Buffer.allocUnsafe(Math.min(HASH_READ_BYTES, Math.max(1, size)));
+ let remaining = size;
+ while (remaining > 0) {
+ const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), null);
+ if (bytesRead === 0) break;
+ hash.update(buffer.subarray(0, bytesRead));
+ remaining -= bytesRead;
+ }
+ return remaining === 0 ? hash.digest("hex") : null;
+ } catch {
+ return null;
+ } finally {
+ await handle.close().catch(() => undefined);
+ }
+}
+
+export type TurnFilesInput = {
+ sessionId: string;
+ /** The session's workspace root. */
+ root: string;
+ /** The turn's engine messages (or `{ messages }`). */
+ messages: unknown;
+ collector: { recordTrace(sessionId: string, type: string, data?: unknown): void };
+ /** The project archive: each existing outside file is reported as touched (absolute). */
+ archive?: { pathTouched(sessionId: string, path: string): void } | null;
+ /** App state/temp/data directories: never reported or recorded. */
+ excludedDirs?: readonly string[];
+ includeCredentialFiles?: boolean;
+ home?: string | null;
+};
+
+export type TurnFilesResult = { outside: string[]; records: Array<{ path: string; access: TurnFileAccess; bytes: number; sha256: string; outside: boolean }> };
+
+/** See the module comment. Never throws. */
+export async function recordTurnFiles(input: TurnFilesInput): Promise<TurnFilesResult> {
+ const result: TurnFilesResult = { outside: [], records: [] };
+ try {
+ const home = input.home === undefined ? safeHome() : input.home;
+ const evidence = turnFileEvidence(input.messages, input.root, home);
+ if (evidence.length === 0) return result;
+ const roots = new Set([resolve(input.root)]);
+ try {
+ roots.add(await realpath(input.root));
+ } catch {
+ // The root went away: every path is judged as given.
+ }
+ const appDirs = new Set<string>();
+ for (const dir of input.excludedDirs ?? []) {
+ appDirs.add(resolve(dir));
+ try {
+ appDirs.add(await realpath(dir));
+ } catch {
+ // Not created.
+ }
+ }
+ const context = { appDirs: [...appDirs], home, includeCredentialFiles: input.includeCredentialFiles === true };
+ const seen = new Set<string>();
+ for (const { path: given, access } of evidence) {
+ let absolute = given;
+ try {
+ // The folder as its real path; the file itself is never followed.
+ absolute = join(await realpath(dirname(given)), basename(given));
+ } catch {
+ continue;
+ }
+ if (seen.has(absolute)) continue;
+ seen.add(absolute);
+ let stats;
+ try {
+ stats = await lstat(absolute);
+ } catch {
+ continue;
+ }
+ if (!stats.isFile()) continue;
+ const rel = [...roots].map((root) => within(root, absolute)).find((value) => value !== null) ?? null;
+ let recordPath: string;
+ if (rel !== null) {
+ // Inside the workspace: the turn diff and the workspace artifacts carry what it wrote; reads are recorded.
+ if (access === "write") continue;
+ if (!context.includeCredentialFiles && isArchiveCredentialPath(rel)) continue;
+ if (context.appDirs.some((dir) => within(dir, absolute) !== null)) continue;
+ recordPath = rel;
+ } else {
+ const archivePath = outsideArchivePath(absolute);
+ if (!archivePath || outsideExclusion(absolute, context)) continue;
+ if (stats.size > MAX_OUTSIDE_FILE_BYTES) continue;
+ try {
+ input.archive?.pathTouched(input.sessionId, absolute);
+ } catch {
+ // The archive's bookkeeping never stops a capture.
+ }
+ result.outside.push(absolute);
+ recordPath = archivePath;
+ }
+ if (result.records.length >= MAX_TURN_FILE_EVENTS || stats.size > MAX_OUTSIDE_FILE_BYTES) continue;
+ const sha256 = await sha256Of(absolute, stats.size);
+ if (!sha256) continue;
+ const record = { path: recordPath, access, bytes: stats.size, sha256, outside: rel === null };
+ result.records.push(record);
+ input.collector.recordTrace(input.sessionId, "artifact", {
+ path: record.path,
+ sha256: record.sha256,
+ bytes: record.bytes,
+ access: record.access,
+ ...(record.outside ? { outside: true } : {}),
+ });
+ }
+ } catch {
+ // Best effort: the turn's capture goes on without its file record.
+ }
+ return result;
+}
diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.ts
index f6852d4d..f63d3d94 100644
--- a/apps/server/src/session-upload-observer.ts
+++ b/apps/server/src/session-upload-observer.ts
@@ -10,6 +10,7 @@
import { loopbackFetch } from "./server-fetch.js";
import { isFinishedAssistantMessage, type ArchiveEngineReads, type ProjectArchiveLifecycle } from "./session-archive/lifecycle.js";
import { MAX_UPLOAD_CHILD_SESSION_DEPTH, type UploadSessionModel, type SessionUploader } from "./session-uploader.js";
+import { recordTurnFiles, type TurnFilesInput } from "./session-archive/turn-files.js";
/** The engine a captured request went to: base URL, request headers (the engine's auth), query and API generation. */
export type EngineTarget = {
@@ -701,6 +702,12 @@ export function projectArchiveEngineReads(target: EngineTarget | (() => EngineTa
export function observeUploadedSession(input: {
sessionUploader: ObservedUploader;
archive: Pick<ProjectArchiveLifecycle, "turnFollowed" | "turnCompleted" | "turnIncomplete">;
+ /**
+ * The session's workspace: each settled turn reports the files outside it
+ * that the turn touched to the archive, and records the files it read
+ * (session-archive/turn-files.ts). Without it, neither.
+ */
+ turnFiles?: Omit<TurnFilesInput, "sessionId" | "messages" | "collector">;
observers: SessionObservers;
sessionId: string;
target: EngineTarget;
@@ -851,6 +858,8 @@ export function observeUploadedSession(input: {
}
room(newest.bytes);
sessionUploader.recordTrace(sessionId, "session.idle", { status });
+ // The files the turn read, and those outside the workspace it touched, before the snapshot and the flush.
+ if (history && input.turnFiles) await recordTurnFiles({ ...input.turnFiles, sessionId, messages: history.delta, collector: sessionUploader });
// The turn snapshot runs first so the artifacts it discovers are part of
// the trace flushed right behind it.
captureTurnSnapshot();