Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(process)!: retain per-file digest open failures

morluto415145d04f09
diff --git a/docs/process-capture.md b/docs/process-capture.md
index b000994c0..316d36c99 100644
--- a/docs/process-capture.md
+++ b/docs/process-capture.md
@@ -143,8 +143,10 @@ Every capture requires `truncation_details`, with separate accounting for:
- `filesystem_before` and `filesystem_after`: file-count/depth limits,
enumeration failures, whole-file hash budget and bytes successfully hashed.
Each retained regular file without a digest has an aliased path, size,
- remaining budget and reason: `file_bytes_budget` or
- `file_changed_or_short_read`. A file too large for the remaining budget is
+ remaining budget and reason: `file_bytes_budget`,
+ `file_changed_or_short_read`, or `file_unavailable`. Its `system_code` is
+ null unless an OS file operation failed, in which case it preserves the
+ reported error code. A file too large for the remaining budget is
skipped; a later smaller file can still be hashed. Hash omissions do not
imply incomplete path enumeration.
- `process`: sampling limit and whether sampling ended partially. Coverage
diff --git a/src/domain/process/processCaptureCoverage.ts b/src/domain/process/processCaptureCoverage.ts
index a93972912..72bbb9fb1 100644
--- a/src/domain/process/processCaptureCoverage.ts
+++ b/src/domain/process/processCaptureCoverage.ts
@@ -25,7 +25,12 @@ export const filesystemCoverageSchema = z.strictObject({
path: z.string(),
size_bytes: z.number().int().nonnegative(),
remaining_budget_bytes: z.number().int().nonnegative(),
- reason: z.enum(["file_bytes_budget", "file_changed_or_short_read"]),
+ reason: z.enum([
+ "file_bytes_budget",
+ "file_changed_or_short_read",
+ "file_unavailable",
+ ]),
+ system_code: z.string().nullable(),
}),
),
});
diff --git a/src/domain/process/processCaptureValidation.ts b/src/domain/process/processCaptureValidation.ts
index fe5479689..a14cd129b 100644
--- a/src/domain/process/processCaptureValidation.ts
+++ b/src/domain/process/processCaptureValidation.ts
@@ -200,7 +200,10 @@ const validateCoverage = (
coverage.hash_omissions.every(
(omission) =>
unhashedFiles.get(omission.path)?.size === omission.size_bytes &&
- omission.remaining_budget_bytes <= coverage.hash_budget_bytes,
+ omission.remaining_budget_bytes <= coverage.hash_budget_bytes &&
+ (omission.reason === "file_unavailable"
+ ? omission.system_code !== null
+ : omission.system_code === null),
), `truncation_details.${name}.hash_omissions`, "every retained regular file without a digest must have one matching omission reason");
require(coverage.hashed_bytes ===
files
diff --git a/src/process/capture/FilesystemCoverage.test.ts b/src/process/capture/FilesystemCoverage.test.ts
index ee8de9fac..36ebfc6de 100644
--- a/src/process/capture/FilesystemCoverage.test.ts
+++ b/src/process/capture/FilesystemCoverage.test.ts
@@ -1,4 +1,4 @@
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { parseProcessScenario } from "../../domain/process/processScenario.js";
@@ -81,6 +81,51 @@ it("distinguishes unavailable content hashes from completed path enumeration", a
).toMatchObject({ status: "deleted", before: { sha256: null }, after: null });
});
+it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "retains unreadable-file and sibling observations in a complete snapshot",
+ async () => {
+ const root = await createTestTempDirectory("rea-fs-unreadable-file-");
+ const unreadable = join(root, "unreadable.txt");
+ const readable = join(root, "readable.txt");
+ await writeFile(unreadable, "permission denied");
+ await writeFile(readable, "still observed");
+ await chmod(unreadable, 0);
+ const scenario = parseProcessScenario({
+ executable: process.execPath,
+ working_directory: root,
+ filesystem_observation_paths: [root],
+ });
+
+ try {
+ const snapshot = await snapshotRoots(scenario);
+ expect(snapshot.files.map(({ path }) => path)).toEqual([
+ "root_0:.",
+ "root_0:readable.txt",
+ "root_0:unreadable.txt",
+ ]);
+ expect(
+ snapshot.files.find(({ path }) => path === "root_0:readable.txt")
+ ?.sha256,
+ ).not.toBeNull();
+ expect(
+ snapshot.files.find(({ path }) => path === "root_0:unreadable.txt"),
+ ).toMatchObject({ type: "file", size: 17, sha256: null });
+ expect(snapshot.coverage.hash_omissions).toContainEqual({
+ path: "root_0:unreadable.txt",
+ size_bytes: 17,
+ remaining_budget_bytes:
+ scenario.limits.file_bytes - Buffer.byteLength("still observed"),
+ reason: "file_unavailable",
+ system_code: "EACCES",
+ });
+ expect(snapshot.completeRoots).toEqual(["root_0"]);
+ } finally {
+ await chmod(unreadable, 0o600);
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);
+
it("preserves creation when the selected root was observed missing before capture", async () => {
const root = await createTestTempDirectory("rea-fs-missing-root-");
const file = join(root, "created.txt");
diff --git a/src/process/capture/FilesystemSnapshot.test.ts b/src/process/capture/FilesystemSnapshot.test.ts
index b7e141018..16799b973 100644
--- a/src/process/capture/FilesystemSnapshot.test.ts
+++ b/src/process/capture/FilesystemSnapshot.test.ts
@@ -44,7 +44,11 @@ it("does not hash a replacement file using the earlier path metadata", async ()
await rm(path);
await writeFile(path, "replacement with a different identity and size\n");
- await expect(hashFile(path, expected, 1_000)).resolves.toBeNull();
+ await expect(hashFile(path, expected, 1_000)).resolves.toEqual({
+ state: "omitted",
+ reason: "file_changed_or_short_read",
+ system_code: null,
+ });
} finally {
await rm(directory, { recursive: true, force: true });
}
@@ -76,7 +80,29 @@ it("rejects a same-size rewrite even when its modification time is restored", as
});
expect(rewritten.ctimeMs).not.toBe(expected.ctimeMs);
- await expect(hashFile(path, expected, 1_000)).resolves.toBeNull();
+ await expect(hashFile(path, expected, 1_000)).resolves.toMatchObject({
+ state: "omitted",
+ reason: "file_changed_or_short_read",
+ system_code: null,
+ });
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
+it("reports a file removed between metadata observation and open", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "rea-snapshot-open-race-"));
+ const path = join(directory, "observed-file");
+ try {
+ await writeFile(path, "captured file\n");
+ const expected = await lstat(path);
+ await rm(path);
+
+ await expect(hashFile(path, expected, 1_000)).resolves.toEqual({
+ state: "omitted",
+ reason: "file_unavailable",
+ system_code: "ENOENT",
+ });
} finally {
await rm(directory, { recursive: true, force: true });
}
@@ -116,9 +142,12 @@ it.skipIf(process.platform === "win32")(
// hashFile reaches `open` synchronously before its first suspension, so
// this abort is observed after the descriptor is acquired and is handled
// by the `finally` close path.
- controller.abort();
+ const reason = Object.assign(new Error("cancelled while opening"), {
+ code: "EACCES",
+ });
+ controller.abort(reason);
- await expect(hashing).rejects.toMatchObject({ name: "AbortError" });
+ await expect(hashing).rejects.toBe(reason);
expect(await countOpenDescriptorsFor(path)).toBe(0);
} finally {
await rm(directory, { recursive: true, force: true });
diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts
index 281151afc..17dd91174 100644
--- a/src/process/capture/FilesystemSnapshot.ts
+++ b/src/process/capture/FilesystemSnapshot.ts
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
import { constants as fsConstants } from "node:fs";
import { lstat, open, readdir, readlink } from "node:fs/promises";
import { dirname, join, relative, resolve } from "node:path";
+import type { FileHandle } from "node:fs/promises";
import type { ProcessScenario } from "../../domain/process/processScenario.js";
import type { FilesystemCoverage } from "../../domain/process/processCaptureCoverage.js";
import type { Stats } from "node:fs";
@@ -29,6 +30,23 @@ const hasSameFileState = (before: Stats, after: Stats): boolean =>
before.mtimeMs === after.mtimeMs &&
before.ctimeMs === after.ctimeMs;
+const fileUnavailableSystemCode = (cause: unknown): string | undefined => {
+ if (!(cause instanceof Error) || !("code" in cause)) return undefined;
+ const code = cause.code;
+ return typeof code === "string" &&
+ [
+ "EACCES",
+ "EISDIR",
+ "ELOOP",
+ "ENOTDIR",
+ "ENOENT",
+ "EPERM",
+ "ESTALE",
+ ].includes(code)
+ ? code
+ : undefined;
+};
+
const lstatIfPresent = async (path: string): Promise<Stats | undefined> => {
try {
return await lstat(path);
@@ -39,26 +57,51 @@ const lstatIfPresent = async (path: string): Promise<Stats | undefined> => {
}
};
+/** Reports a complete digest or an explicit per-file reason hashing was omitted. */
+export type FileHashOutcome =
+ | { readonly state: "hashed"; readonly sha256: string }
+ | {
+ readonly state: "omitted";
+ readonly reason: "file_changed_or_short_read" | "file_unavailable";
+ readonly system_code: string | null;
+ };
+
/** Hash a file only when its opened descriptor still matches the captured path state. */
export const hashFile = async (
path: string,
expected: Stats,
maxBytes: number,
signal?: AbortSignal,
-): Promise<string | null> => {
+): Promise<FileHashOutcome> => {
signal?.throwIfAborted();
- const handle = await open(
- path,
- fsConstants.O_RDONLY |
- (fsConstants.O_NOFOLLOW ?? 0) |
- (fsConstants.O_NONBLOCK ?? 0),
- );
+ let handle: FileHandle;
+ try {
+ handle = await open(
+ path,
+ fsConstants.O_RDONLY |
+ (fsConstants.O_NOFOLLOW ?? 0) |
+ (fsConstants.O_NONBLOCK ?? 0),
+ );
+ } catch (cause: unknown) {
+ signal?.throwIfAborted();
+ const systemCode = fileUnavailableSystemCode(cause);
+ if (systemCode === undefined) throw cause;
+ return {
+ state: "omitted",
+ reason: "file_unavailable",
+ system_code: systemCode,
+ };
+ }
try {
signal?.throwIfAborted();
const stats = await handle.stat();
signal?.throwIfAborted();
if (!hasSameFileState(expected, stats) || stats.size > maxBytes)
- return null;
+ return {
+ state: "omitted",
+ reason: "file_changed_or_short_read",
+ system_code: null,
+ };
const hash = createHash("sha256");
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes));
let position = 0;
@@ -70,14 +113,33 @@ export const hashFile = async (
Math.min(buffer.length, stats.size - position),
position,
);
- if (bytesRead === 0) return null;
+ if (bytesRead === 0)
+ return {
+ state: "omitted",
+ reason: "file_changed_or_short_read",
+ system_code: null,
+ };
hash.update(buffer.subarray(0, bytesRead));
position += bytesRead;
}
const after = await handle.stat();
signal?.throwIfAborted();
- if (!hasSameFileState(stats, after)) return null;
- return hash.digest("hex");
+ if (!hasSameFileState(stats, after))
+ return {
+ state: "omitted",
+ reason: "file_changed_or_short_read",
+ system_code: null,
+ };
+ return { state: "hashed", sha256: hash.digest("hex") };
+ } catch (cause: unknown) {
+ signal?.throwIfAborted();
+ const systemCode = fileUnavailableSystemCode(cause);
+ if (systemCode === undefined) throw cause;
+ return {
+ state: "omitted",
+ reason: "file_unavailable",
+ system_code: systemCode,
+ };
} finally {
await handle.close();
}
@@ -136,22 +198,34 @@ export const snapshotRoots = async (
return true;
}
if (stats.isFile()) {
- const sha256 =
- remainingBytes >= stats.size
- ? await hashFile(path, stats, remainingBytes, signal)
- : null;
- remainingBytes -= sha256 === null ? 0 : stats.size;
- if (sha256 === null) {
- truncated = true;
- hashOmissions.push({
+ let sha256: string | null = null;
+ let omission: FilesystemCoverage["hash_omissions"][number] | undefined;
+ if (remainingBytes < stats.size) {
+ omission = {
path: `${rootAlias}:${relativePath}`,
size_bytes: stats.size,
remaining_budget_bytes: remainingBytes,
- reason:
- remainingBytes < stats.size
- ? "file_bytes_budget"
- : "file_changed_or_short_read",
- });
+ reason: "file_bytes_budget",
+ system_code: null,
+ };
+ } else {
+ const outcome = await hashFile(path, stats, remainingBytes, signal);
+ if (outcome.state === "hashed") {
+ sha256 = outcome.sha256;
+ remainingBytes -= stats.size;
+ } else {
+ omission = {
+ path: `${rootAlias}:${relativePath}`,
+ size_bytes: stats.size,
+ remaining_budget_bytes: remainingBytes,
+ reason: outcome.reason,
+ system_code: outcome.system_code,
+ };
+ }
+ }
+ if (omission !== undefined) {
+ truncated = true;
+ hashOmissions.push(omission);
}
entries.push({
path: `${rootAlias}:${relativePath}`,
diff --git a/tests/boundary/process/processTruncationAdapters.test.ts b/tests/boundary/process/processTruncationAdapters.test.ts
index 34c1ccb3a..20d41e639 100644
--- a/tests/boundary/process/processTruncationAdapters.test.ts
+++ b/tests/boundary/process/processTruncationAdapters.test.ts
@@ -187,6 +187,7 @@ itWithCaptureCapability.each(["cli", "mcp"] as const)(
size_bytes: 8,
remaining_budget_bytes: 1,
reason: "file_bytes_budget",
+ system_code: null,
},
],
});
@@ -229,6 +230,7 @@ itWithCaptureCapability.each(["cli", "mcp"] as const)(
path: "root_0:b.txt",
reason: "file_bytes_budget",
remaining_budget_bytes: 1,
+ system_code: null,
},
],
});
diff --git a/tests/fixtures/filesystemSnapshotFifoProbe.mjs b/tests/fixtures/filesystemSnapshotFifoProbe.mjs
index de3e466b3..984d6e9f7 100644
--- a/tests/fixtures/filesystemSnapshotFifoProbe.mjs
+++ b/tests/fixtures/filesystemSnapshotFifoProbe.mjs
@@ -18,6 +18,13 @@ execFileSync("mkfifo", [path]);
// This invokes the production lstat-to-open hashing seam with a known captured
// regular-file identity. The owner test bounds this child so an O_RDONLY FIFO
// regression cannot hang Vitest.
-const digest = await hashFile(path, expected, 1_000);
-if (digest !== null) throw new Error("Expected the FIFO to be rejected");
+const outcome = await hashFile(path, expected, 1_000);
+if (
+ outcome.state !== "omitted" ||
+ outcome.reason !== "file_changed_or_short_read" ||
+ outcome.system_code !== null
+)
+ throw new Error(
+ `Expected the FIFO to be omitted as a changed file: ${JSON.stringify(outcome)}`,
+ );
process.stdout.write("rejected\n");