Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(process)!: retain per-file digest open failures
morluto415145d04f09
diff --git a/docs/process-capture.md b/docs/process-capture.mdindex b000994c0..316d36c99 100644--- a/docs/process-capture.md+++ b/docs/process-capture.md@@ -143,8 +143,10 @@ Every capture requires `truncation_details`, with separate accounting for:- `filesystem_before` and `filesystem_after`: file-count/depth limits,enumeration failures, whole-file hash budget and bytes successfully hashed.Each retained regular file without a digest has an aliased path, size,- remaining budget and reason: `file_bytes_budget` or- `file_changed_or_short_read`. A file too large for the remaining budget is+ remaining budget and reason: `file_bytes_budget`,+ `file_changed_or_short_read`, or `file_unavailable`. Its `system_code` is+ null unless an OS file operation failed, in which case it preserves the+ reported error code. A file too large for the remaining budget isskipped; a later smaller file can still be hashed. Hash omissions do notimply incomplete path enumeration.- `process`: sampling limit and whether sampling ended partially. Coveragediff --git a/src/domain/process/processCaptureCoverage.ts b/src/domain/process/processCaptureCoverage.tsindex a93972912..72bbb9fb1 100644--- a/src/domain/process/processCaptureCoverage.ts+++ b/src/domain/process/processCaptureCoverage.ts@@ -25,7 +25,12 @@ export const filesystemCoverageSchema = z.strictObject({path: z.string(),size_bytes: z.number().int().nonnegative(),remaining_budget_bytes: z.number().int().nonnegative(),- reason: z.enum(["file_bytes_budget", "file_changed_or_short_read"]),+ reason: z.enum([+ "file_bytes_budget",+ "file_changed_or_short_read",+ "file_unavailable",+ ]),+ system_code: z.string().nullable(),}),),});diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.tsindex fe5479689..a14cd129b 100644--- a/src/domain/process/processCaptureValidation.ts+++ b/src/domain/process/processCaptureValidation.ts@@ -200,7 +200,10 @@ const validateCoverage = (coverage.hash_omissions.every((omission) =>unhashedFiles.get(omission.path)?.size === omission.size_bytes &&- omission.remaining_budget_bytes <= coverage.hash_budget_bytes,+ omission.remaining_budget_bytes <= coverage.hash_budget_bytes &&+ (omission.reason === "file_unavailable"+ ? omission.system_code !== null+ : omission.system_code === null),), `truncation_details.${name}.hash_omissions`, "every retained regular file without a digest must have one matching omission reason");require(coverage.hashed_bytes ===filesdiff --git a/src/process/capture/FilesystemCoverage.test.ts b/src/process/capture/FilesystemCoverage.test.tsindex ee8de9fac..36ebfc6de 100644--- a/src/process/capture/FilesystemCoverage.test.ts+++ b/src/process/capture/FilesystemCoverage.test.ts@@ -1,4 +1,4 @@-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";+import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";import { join } from "node:path";import { expect, it } from "vitest";import { parseProcessScenario } from "../../domain/process/processScenario.js";@@ -81,6 +81,51 @@ it("distinguishes unavailable content hashes from completed path enumeration", a).toMatchObject({ status: "deleted", before: { sha256: null }, after: null });});+it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(+ "retains unreadable-file and sibling observations in a complete snapshot",+ async () => {+ const root = await createTestTempDirectory("rea-fs-unreadable-file-");+ const unreadable = join(root, "unreadable.txt");+ const readable = join(root, "readable.txt");+ await writeFile(unreadable, "permission denied");+ await writeFile(readable, "still observed");+ await chmod(unreadable, 0);+ const scenario = parseProcessScenario({+ executable: process.execPath,+ working_directory: root,+ filesystem_observation_paths: [root],+ });++ try {+ const snapshot = await snapshotRoots(scenario);+ expect(snapshot.files.map(({ path }) => path)).toEqual([+ "root_0:.",+ "root_0:readable.txt",+ "root_0:unreadable.txt",+ ]);+ expect(+ snapshot.files.find(({ path }) => path === "root_0:readable.txt")+ ?.sha256,+ ).not.toBeNull();+ expect(+ snapshot.files.find(({ path }) => path === "root_0:unreadable.txt"),+ ).toMatchObject({ type: "file", size: 17, sha256: null });+ expect(snapshot.coverage.hash_omissions).toContainEqual({+ path: "root_0:unreadable.txt",+ size_bytes: 17,+ remaining_budget_bytes:+ scenario.limits.file_bytes - Buffer.byteLength("still observed"),+ reason: "file_unavailable",+ system_code: "EACCES",+ });+ expect(snapshot.completeRoots).toEqual(["root_0"]);+ } finally {+ await chmod(unreadable, 0o600);+ await rm(root, { recursive: true, force: true });+ }+ },+);+it("preserves creation when the selected root was observed missing before capture", async () => {const root = await createTestTempDirectory("rea-fs-missing-root-");const file = join(root, "created.txt");diff --git a/src/process/capture/FilesystemSnapshot.test.ts b/src/process/capture/FilesystemSnapshot.test.tsindex b7e141018..16799b973 100644--- a/src/process/capture/FilesystemSnapshot.test.ts+++ b/src/process/capture/FilesystemSnapshot.test.ts@@ -44,7 +44,11 @@ it("does not hash a replacement file using the earlier path metadata", async ()await rm(path);await writeFile(path, "replacement with a different identity and size\n");- await expect(hashFile(path, expected, 1_000)).resolves.toBeNull();+ await expect(hashFile(path, expected, 1_000)).resolves.toEqual({+ state: "omitted",+ reason: "file_changed_or_short_read",+ system_code: null,+ });} finally {await rm(directory, { recursive: true, force: true });}@@ -76,7 +80,29 @@ it("rejects a same-size rewrite even when its modification time is restored", as});expect(rewritten.ctimeMs).not.toBe(expected.ctimeMs);- await expect(hashFile(path, expected, 1_000)).resolves.toBeNull();+ await expect(hashFile(path, expected, 1_000)).resolves.toMatchObject({+ state: "omitted",+ reason: "file_changed_or_short_read",+ system_code: null,+ });+ } finally {+ await rm(directory, { recursive: true, force: true });+ }+});++it("reports a file removed between metadata observation and open", async () => {+ const directory = await mkdtemp(join(tmpdir(), "rea-snapshot-open-race-"));+ const path = join(directory, "observed-file");+ try {+ await writeFile(path, "captured file\n");+ const expected = await lstat(path);+ await rm(path);++ await expect(hashFile(path, expected, 1_000)).resolves.toEqual({+ state: "omitted",+ reason: "file_unavailable",+ system_code: "ENOENT",+ });} finally {await rm(directory, { recursive: true, force: true });}@@ -116,9 +142,12 @@ it.skipIf(process.platform === "win32")(// hashFile reaches `open` synchronously before its first suspension, so// this abort is observed after the descriptor is acquired and is handled// by the `finally` close path.- controller.abort();+ const reason = Object.assign(new Error("cancelled while opening"), {+ code: "EACCES",+ });+ controller.abort(reason);- await expect(hashing).rejects.toMatchObject({ name: "AbortError" });+ await expect(hashing).rejects.toBe(reason);expect(await countOpenDescriptorsFor(path)).toBe(0);} finally {await rm(directory, { recursive: true, force: true });diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.tsindex 281151afc..17dd91174 100644--- a/src/process/capture/FilesystemSnapshot.ts+++ b/src/process/capture/FilesystemSnapshot.ts@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";import { constants as fsConstants } from "node:fs";import { lstat, open, readdir, readlink } from "node:fs/promises";import { dirname, join, relative, resolve } from "node:path";+import type { FileHandle } from "node:fs/promises";import type { ProcessScenario } from "../../domain/process/processScenario.js";import type { FilesystemCoverage } from "../../domain/process/processCaptureCoverage.js";import type { Stats } from "node:fs";@@ -29,6 +30,23 @@ const hasSameFileState = (before: Stats, after: Stats): boolean =>before.mtimeMs === after.mtimeMs &&before.ctimeMs === after.ctimeMs;+const fileUnavailableSystemCode = (cause: unknown): string | undefined => {+ if (!(cause instanceof Error) || !("code" in cause)) return undefined;+ const code = cause.code;+ return typeof code === "string" &&+ [+ "EACCES",+ "EISDIR",+ "ELOOP",+ "ENOTDIR",+ "ENOENT",+ "EPERM",+ "ESTALE",+ ].includes(code)+ ? code+ : undefined;+};+const lstatIfPresent = async (path: string): Promise<Stats | undefined> => {try {return await lstat(path);@@ -39,26 +57,51 @@ const lstatIfPresent = async (path: string): Promise<Stats | undefined> => {}};+/** Reports a complete digest or an explicit per-file reason hashing was omitted. */+export type FileHashOutcome =+ | { readonly state: "hashed"; readonly sha256: string }+ | {+ readonly state: "omitted";+ readonly reason: "file_changed_or_short_read" | "file_unavailable";+ readonly system_code: string | null;+ };+/** Hash a file only when its opened descriptor still matches the captured path state. */export const hashFile = async (path: string,expected: Stats,maxBytes: number,signal?: AbortSignal,-): Promise<string | null> => {+): Promise<FileHashOutcome> => {signal?.throwIfAborted();- const handle = await open(- path,- fsConstants.O_RDONLY |- (fsConstants.O_NOFOLLOW ?? 0) |- (fsConstants.O_NONBLOCK ?? 0),- );+ let handle: FileHandle;+ try {+ handle = await open(+ path,+ fsConstants.O_RDONLY |+ (fsConstants.O_NOFOLLOW ?? 0) |+ (fsConstants.O_NONBLOCK ?? 0),+ );+ } catch (cause: unknown) {+ signal?.throwIfAborted();+ const systemCode = fileUnavailableSystemCode(cause);+ if (systemCode === undefined) throw cause;+ return {+ state: "omitted",+ reason: "file_unavailable",+ system_code: systemCode,+ };+ }try {signal?.throwIfAborted();const stats = await handle.stat();signal?.throwIfAborted();if (!hasSameFileState(expected, stats) || stats.size > maxBytes)- return null;+ return {+ state: "omitted",+ reason: "file_changed_or_short_read",+ system_code: null,+ };const hash = createHash("sha256");const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes));let position = 0;@@ -70,14 +113,33 @@ export const hashFile = async (Math.min(buffer.length, stats.size - position),position,);- if (bytesRead === 0) return null;+ if (bytesRead === 0)+ return {+ state: "omitted",+ reason: "file_changed_or_short_read",+ system_code: null,+ };hash.update(buffer.subarray(0, bytesRead));position += bytesRead;}const after = await handle.stat();signal?.throwIfAborted();- if (!hasSameFileState(stats, after)) return null;- return hash.digest("hex");+ if (!hasSameFileState(stats, after))+ return {+ state: "omitted",+ reason: "file_changed_or_short_read",+ system_code: null,+ };+ return { state: "hashed", sha256: hash.digest("hex") };+ } catch (cause: unknown) {+ signal?.throwIfAborted();+ const systemCode = fileUnavailableSystemCode(cause);+ if (systemCode === undefined) throw cause;+ return {+ state: "omitted",+ reason: "file_unavailable",+ system_code: systemCode,+ };} finally {await handle.close();}@@ -136,22 +198,34 @@ export const snapshotRoots = async (return true;}if (stats.isFile()) {- const sha256 =- remainingBytes >= stats.size- ? await hashFile(path, stats, remainingBytes, signal)- : null;- remainingBytes -= sha256 === null ? 0 : stats.size;- if (sha256 === null) {- truncated = true;- hashOmissions.push({+ let sha256: string | null = null;+ let omission: FilesystemCoverage["hash_omissions"][number] | undefined;+ if (remainingBytes < stats.size) {+ omission = {path: `${rootAlias}:${relativePath}`,size_bytes: stats.size,remaining_budget_bytes: remainingBytes,- reason:- remainingBytes < stats.size- ? "file_bytes_budget"- : "file_changed_or_short_read",- });+ reason: "file_bytes_budget",+ system_code: null,+ };+ } else {+ const outcome = await hashFile(path, stats, remainingBytes, signal);+ if (outcome.state === "hashed") {+ sha256 = outcome.sha256;+ remainingBytes -= stats.size;+ } else {+ omission = {+ path: `${rootAlias}:${relativePath}`,+ size_bytes: stats.size,+ remaining_budget_bytes: remainingBytes,+ reason: outcome.reason,+ system_code: outcome.system_code,+ };+ }+ }+ if (omission !== undefined) {+ truncated = true;+ hashOmissions.push(omission);}entries.push({path: `${rootAlias}:${relativePath}`,diff --git a/tests/boundary/process/processTruncationAdapters.test.ts b/tests/boundary/process/processTruncationAdapters.test.tsindex 34c1ccb3a..20d41e639 100644--- a/tests/boundary/process/processTruncationAdapters.test.ts+++ b/tests/boundary/process/processTruncationAdapters.test.ts@@ -187,6 +187,7 @@ itWithCaptureCapability.each(["cli", "mcp"] as const)(size_bytes: 8,remaining_budget_bytes: 1,reason: "file_bytes_budget",+ system_code: null,},],});@@ -229,6 +230,7 @@ itWithCaptureCapability.each(["cli", "mcp"] as const)(path: "root_0:b.txt",reason: "file_bytes_budget",remaining_budget_bytes: 1,+ system_code: null,},],});diff --git a/tests/fixtures/filesystemSnapshotFifoProbe.mjs b/tests/fixtures/filesystemSnapshotFifoProbe.mjsindex de3e466b3..984d6e9f7 100644--- a/tests/fixtures/filesystemSnapshotFifoProbe.mjs+++ b/tests/fixtures/filesystemSnapshotFifoProbe.mjs@@ -18,6 +18,13 @@ execFileSync("mkfifo", [path]);// This invokes the production lstat-to-open hashing seam with a known captured// regular-file identity. The owner test bounds this child so an O_RDONLY FIFO// regression cannot hang Vitest.-const digest = await hashFile(path, expected, 1_000);-if (digest !== null) throw new Error("Expected the FIFO to be rejected");+const outcome = await hashFile(path, expected, 1_000);+if (+ outcome.state !== "omitted" ||+ outcome.reason !== "file_changed_or_short_read" ||+ outcome.system_code !== null+)+ throw new Error(+ `Expected the FIFO to be omitted as a changed file: ${JSON.stringify(outcome)}`,+ );process.stdout.write("rejected\n");