Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(reference): preserve observed directories and unreadable links

morlutod6e4fe4a4dd8
diff --git a/src/reference/ReferenceSourceReader.test.ts b/src/reference/ReferenceSourceReader.test.ts
new file mode 100644
index 000000000..2eef9ec31
--- /dev/null
+++ b/src/reference/ReferenceSourceReader.test.ts
@@ -0,0 +1,113 @@
+import { describe, expect, it } from "vitest";
+
+import type {
+ ReferenceSourceEntry,
+ TraversalState,
+} from "./ReferenceSourceReaderTypes.js";
+import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceReader.js";
+
+const failedDirectory = (
+ path: string,
+ code: "changed" | "io" | "symlink",
+): ReferenceSourceEntry => ({
+ status: "failed",
+ kind: "directory",
+ path,
+ code,
+ message: `directory failure ${code}`,
+});
+
+const internalLink = (path: string, target: string): ReferenceSourceEntry => ({
+ status: "read",
+ kind: "symlink",
+ path,
+ target,
+ targetState: "internal",
+});
+
+const exclusions = (
+ ...items: TraversalState["exclusions"][number][]
+): TraversalState["exclusions"] => items;
+
+describe("unobserved internal symlink reconciliation", () => {
+ it("retains unreadable targets below the nearest failed directory", () => {
+ const entries: ReferenceSourceEntry[] = [
+ internalLink("alias.txt", "hidden/target.txt"),
+ failedDirectory("hidden", "io"),
+ {
+ status: "read",
+ kind: "file",
+ path: "visible.txt",
+ bytes: new Uint8Array([1]),
+ size: 1,
+ },
+ ];
+
+ reconcileUnobservedSymlinkTargets(entries);
+
+ expect(entries[0]).toMatchObject({
+ status: "failed",
+ kind: "symlink",
+ code: "io",
+ message: expect.stringContaining("directory failure io"),
+ });
+ expect(entries[2]).toMatchObject({ status: "read", path: "visible.txt" });
+ });
+
+ it("matches Unicode-equivalent failed ancestor identities", () => {
+ const entries: ReferenceSourceEntry[] = [
+ internalLink("alias.txt", "hidden/cafe\u0301/target.txt"),
+ failedDirectory("hidden/caf\u00e9", "changed"),
+ ];
+
+ reconcileUnobservedSymlinkTargets(entries);
+
+ expect(entries[0]).toMatchObject({
+ status: "failed",
+ kind: "symlink",
+ code: "changed",
+ message: expect.stringContaining("directory failure changed"),
+ });
+ });
+
+ it("stops at a complete nearer directory or an exclusion of unknown kind", () => {
+ const entries: ReferenceSourceEntry[] = [
+ internalLink("to-missing", "outer/complete/missing.txt"),
+ internalLink("to-excluded", "outer/excluded/missing.txt"),
+ failedDirectory("outer", "io"),
+ { status: "read", kind: "directory", path: "outer/complete" },
+ ];
+
+ reconcileUnobservedSymlinkTargets(
+ entries,
+ exclusions({ path: "outer/excluded", kind: "unknown" }),
+ );
+
+ expect(entries[0]).toMatchObject({
+ status: "read",
+ targetState: "internal",
+ });
+ expect(entries[1]).toMatchObject({
+ status: "read",
+ targetState: "internal",
+ });
+ });
+
+ it("keeps exact exclusions and leaves observations untouched after cancellation", () => {
+ const entries = [internalLink("alias.txt", "hidden/target.txt")];
+ reconcileUnobservedSymlinkTargets(
+ entries,
+ exclusions({ path: "hidden/target.txt", kind: "file" }),
+ );
+ expect(entries[0]).toMatchObject({ targetState: "internal" });
+
+ const cancelledEntries: ReferenceSourceEntry[] = [
+ internalLink("alias.txt", "hidden/target.txt"),
+ failedDirectory("hidden", "io"),
+ ];
+ const controller = new AbortController();
+ controller.abort();
+ reconcileUnobservedSymlinkTargets(cancelledEntries, [], controller.signal);
+ expect(cancelledEntries[0]).toMatchObject({ targetState: "internal" });
+ });
+});
diff --git a/src/reference/ReferenceSourceReader.ts b/src/reference/ReferenceSourceReader.ts
index e8e8650a1..6420e3f34 100644
--- a/src/reference/ReferenceSourceReader.ts
+++ b/src/reference/ReferenceSourceReader.ts
@@ -1,10 +1,14 @@
+import { posix } from "node:path";
+
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { createReferenceSourcePathLookup } from "../domain/referenceSourcePathIdentity.js";
import { err, ok } from "../domain/result.js";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { traverseDirectory } from "./ReferenceSourceReaderEntries.js";
import type {
ReferenceSourceReaderOptions,
+ ReferenceSourceEntry,
ReferenceSourceRead,
ReferenceSourceResult,
TraversalState,
@@ -61,10 +65,20 @@ const readSource = async (
? {}
: { shouldExclude: options.shouldExclude }),
entries: [],
+ exclusions: [],
pending: [{ path: canonicalRoot }],
bytesRead: 0,
};
const traversal = await traverse(state);
+ if (
+ !isAborted(state.signal) &&
+ (traversal.ok || traversal.error.code !== "cancelled")
+ )
+ reconcileUnobservedSymlinkTargets(
+ state.entries,
+ state.exclusions,
+ state.signal,
+ );
state.entries.sort((left, right) =>
compareUnicodeCodePoints(left.path, right.path),
);
@@ -77,6 +91,96 @@ const readSource = async (
return traversal.ok ? ok(read) : err({ ...traversal.error, partial: read });
};
+/**
+ * Resolve internal link observations against the completed reader inventory.
+ * If the closest containing directory failed before enumeration completed,
+ * the target remains unknown; a broader failed ancestor cannot override a
+ * successfully enumerated directory or a recorded exclusion.
+ */
+export const reconcileUnobservedSymlinkTargets = (
+ entries: ReferenceSourceEntry[],
+ exclusions: TraversalState["exclusions"] = [],
+ signal?: AbortSignal,
+): void => {
+ const failedDirectories = new Map(
+ entries.flatMap((entry) =>
+ entry.status === "failed" &&
+ entry.kind === "directory" &&
+ entry.code !== "cancelled"
+ ? [[entry.path, entry] as const]
+ : [],
+ ),
+ );
+ if (failedDirectories.size === 0 || isAborted(signal)) return;
+ const directoryEntries = entries.filter(
+ (entry) => entry.kind === "directory",
+ );
+ const directoryPaths = createReferenceSourcePathLookup(
+ directoryEntries.map(({ path }) => path),
+ );
+ const observedPaths = createReferenceSourcePathLookup(
+ entries.map(({ path }) => path),
+ );
+ const excludedPaths = createReferenceSourcePathLookup(
+ exclusions.map(({ path }) => path),
+ );
+ for (const [index, entry] of entries.entries()) {
+ if (isAborted(signal)) return;
+ if (
+ entry.status !== "read" ||
+ entry.kind !== "symlink" ||
+ entry.targetState !== "internal" ||
+ observedPaths.resolve(entry.target) !== undefined ||
+ excludedPaths.resolve(entry.target) !== undefined
+ )
+ continue;
+ const ancestor = nearestFailedAncestor(
+ entry.target,
+ directoryPaths,
+ failedDirectories,
+ excludedPaths,
+ signal,
+ );
+ if (ancestor === undefined) continue;
+ entries[index] = {
+ status: "failed",
+ kind: "symlink",
+ path: entry.path,
+ code: ancestor.code,
+ message: `Symbolic link target is inside a directory that could not be enumerated: ${ancestor.message}`,
+ };
+ }
+};
+
+const nearestFailedAncestor = (
+ target: string,
+ directoryPaths: ReturnType<typeof createReferenceSourcePathLookup>,
+ failedDirectories: ReadonlyMap<
+ string,
+ Extract<ReferenceSourceEntry, { status: "failed" }>
+ >,
+ excludedPaths: ReturnType<typeof createReferenceSourcePathLookup>,
+ signal: AbortSignal | undefined,
+): Extract<ReferenceSourceEntry, { status: "failed" }> | undefined => {
+ for (
+ let candidate = posix.dirname(target);
+ candidate !== "." && candidate !== "/";
+ candidate = posix.dirname(candidate)
+ ) {
+ if (isAborted(signal)) return undefined;
+ if (excludedPaths.resolve(candidate) !== undefined) return undefined;
+ const observedDirectory = directoryPaths.resolve(candidate);
+ if (observedDirectory !== undefined)
+ return failedDirectories.get(observedDirectory);
+ }
+ if (isAborted(signal)) return undefined;
+ if (excludedPaths.resolve(".") !== undefined) return undefined;
+ const observedRoot = directoryPaths.resolve(".");
+ return observedRoot === undefined
+ ? undefined
+ : failedDirectories.get(observedRoot);
+};
+
const traverse = async (
state: TraversalState,
): Promise<ReferenceSourceResult<undefined>> => {
diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts
index a7ae837e8..1ca0ce3e2 100644
--- a/src/reference/ReferenceSourceReaderEntries.ts
+++ b/src/reference/ReferenceSourceReaderEntries.ts
@@ -134,6 +134,7 @@ export const traverseDirectory = async (
readFailure,
),
);
+ scheduleObservedDirectories(state, directories);
return { ok: true, value: undefined };
}
const after = await validateDirectory(
@@ -152,6 +153,7 @@ export const traverseDirectory = async (
"Directory changed while it was read",
),
);
+ scheduleObservedDirectories(state, directories);
return { ok: true, value: undefined };
}
if (current.path !== state.root)
@@ -160,9 +162,16 @@ export const traverseDirectory = async (
kind: "directory",
path: pathFromRoot(state.root, current.path),
});
+ scheduleObservedDirectories(state, directories);
+ return { ok: true, value: undefined };
+};
+
+const scheduleObservedDirectories = (
+ state: TraversalState,
+ directories: PendingDirectory[],
+): void => {
directories.reverse();
state.pending.push(...directories);
- return { ok: true, value: undefined };
};
const directoryReadFailure = (cause: unknown): string => {
@@ -207,7 +216,10 @@ const processEntry = async (
ok: false,
error: failure("io", "Reference source exclusion check failed"),
};
- if (excluded.value) return { ok: true, value: undefined };
+ if (excluded.value) {
+ state.exclusions.push({ path, kind: "unknown" });
+ return { ok: true, value: undefined };
+ }
state.entries.push(entryFailure(path, "unknown", "io", metadata.message));
return { ok: true, value: undefined };
}
@@ -228,7 +240,10 @@ const processEntry = async (
message: "Reference source exclusion check failed",
},
};
- if (excluded.value) return { ok: true, value: undefined };
+ if (excluded.value) {
+ state.exclusions.push({ path, kind });
+ return { ok: true, value: undefined };
+ }
if (isAborted(state.signal)) return { ok: false, error: cancelled() };
if (kind === "symlink")
state.entries.push(
diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.ts
index 1fec38313..85d6d7602 100644
--- a/src/reference/ReferenceSourceReaderTypes.ts
+++ b/src/reference/ReferenceSourceReaderTypes.ts
@@ -93,6 +93,10 @@ export type TraversalState = {
kind: ReferenceSourceExclusionKind,
) => boolean;
readonly entries: ReferenceSourceEntry[];
+ readonly exclusions: Array<{
+ readonly path: string;
+ readonly kind: ReferenceSourceExclusionKind;
+ }>;
readonly pending: PendingDirectory[];
bytesRead: number;
};
diff --git a/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts b/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts
new file mode 100644
index 000000000..6bc4be707
--- /dev/null
+++ b/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts
@@ -0,0 +1,107 @@
+import type { Dir, Dirent } from "node:fs";
+import { mkdir, symlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it, vi } from "vitest";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { importReferenceSource } from "../../support/referenceSourceResourceScope.js";
+
+const { opendirMock } = vi.hoisted(() => ({ opendirMock: vi.fn() }));
+
+vi.mock("node:fs/promises", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("node:fs/promises")>()),
+ opendir: opendirMock,
+}));
+
+const directoryHandle = (items: readonly (string | Error | null)[]): Dir => {
+ let index = 0;
+ return {
+ async read() {
+ const item = items[index++];
+ if (item instanceof Error) throw item;
+ return item === null || item === undefined
+ ? null
+ : ({ name: item } as Dirent);
+ },
+ async close() {},
+ } as unknown as Dir;
+};
+
+it.skipIf(process.platform === "win32")(
+ "traverses directories lstat-observed before their parent enumeration failed",
+ async () => {
+ const root = await createTestTempDirectory(
+ "rea-queued-source-directories-",
+ );
+ const container = join(root, "container");
+ const hidden = join(container, "hidden");
+ await mkdir(hidden, { recursive: true });
+ await writeFile(join(container, "visible.txt"), "visible\n");
+ await writeFile(join(hidden, "target.txt"), "target\n");
+ await symlink("hidden/target.txt", join(container, "alias-file.txt"));
+ await symlink("hidden", join(container, "alias-directory"));
+
+ const readFailure = Object.assign(
+ new Error("injected directory read failure"),
+ { code: "EIO" },
+ );
+ opendirMock.mockImplementation(async (path: string) => {
+ if (path === root) return directoryHandle(["container", null]);
+ if (path === container)
+ return directoryHandle([
+ "visible.txt",
+ "hidden",
+ "alias-file.txt",
+ "alias-directory",
+ readFailure,
+ ]);
+ if (path === hidden) return directoryHandle(["target.txt", null]);
+ throw new Error(`Unexpected directory traversal: ${path}`);
+ });
+
+ const imported = await importReferenceSource({
+ root,
+ caller: "queued-directory-read-failure-test",
+ policy: { secretPatterns: [] },
+ });
+ if (!imported.ok) throw new Error(imported.error.message);
+ expect(imported.value.inventory_state).toBe("partial");
+ expect(imported.value.entries).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ path: "container",
+ kind: "directory",
+ tree_state: "unreadable",
+ }),
+ expect.objectContaining({
+ path: "container/hidden",
+ kind: "directory",
+ tree_state: "enumerated",
+ }),
+ expect.objectContaining({
+ path: "container/visible.txt",
+ kind: "file",
+ content_state: "hashed",
+ }),
+ expect.objectContaining({
+ path: "container/hidden/target.txt",
+ kind: "file",
+ content_state: "hashed",
+ }),
+ expect.objectContaining({
+ path: "container/alias-file.txt",
+ kind: "symlink",
+ target: "hidden/target.txt",
+ target_state: "internal",
+ }),
+ expect.objectContaining({
+ path: "container/alias-directory",
+ kind: "symlink",
+ target: "hidden",
+ target_state: "internal",
+ }),
+ ]),
+ );
+ },
+);
diff --git a/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts b/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts
index ec868052b..ae54e16ce 100644
--- a/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts
+++ b/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts
@@ -1,4 +1,4 @@
-import { chmod, mkdir, writeFile } from "node:fs/promises";
+import { chmod, mkdir, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
@@ -62,6 +62,60 @@ describe("reference source native unreadable directories", () => {
},
);
+ it.skipIf(permissionChecksUnavailable)(
+ "keeps links into an unreadable subtree uncertain while preserving readable siblings",
+ async () => {
+ const root = await createTestTempDirectory("rea-source-unreadable-link-");
+ const hidden = join(root, "hidden");
+ await mkdir(hidden);
+ await writeFile(join(root, "visible.txt"), "visible\n");
+ await writeFile(join(hidden, "target.txt"), "target\n");
+ await symlink("hidden/target.txt", join(root, "alias.txt"));
+ await chmod(hidden, 0o111);
+ try {
+ const read = await readReferenceSource(root);
+ if (!read.ok) throw new Error(read.error.message);
+ expect(read.value.entries).toContainEqual(
+ expect.objectContaining({
+ status: "failed",
+ kind: "directory",
+ path: "hidden",
+ code: "io",
+ }),
+ );
+ expect(read.value.entries).toContainEqual(
+ expect.objectContaining({
+ status: "failed",
+ kind: "symlink",
+ path: "alias.txt",
+ code: "io",
+ }),
+ );
+
+ const imported = await importTree(root);
+ if (!imported.ok) throw new Error(imported.error.message);
+ expect(imported.value.inventory_state).toBe("partial");
+ expect(imported.value.entries).toContainEqual(
+ expect.objectContaining({
+ path: "visible.txt",
+ kind: "file",
+ content_state: "hashed",
+ }),
+ );
+ expect(imported.value.entries).toContainEqual(
+ expect.objectContaining({
+ path: "alias.txt",
+ kind: "symlink",
+ target: null,
+ target_state: "unreadable",
+ }),
+ );
+ } finally {
+ await chmod(hidden, 0o700);
+ }
+ },
+ );
+
it("keeps readable and empty directories enumerated exactly once", async () => {
const root = await createTestTempDirectory("rea-source-readable-dirs-");
await mkdir(join(root, "src", "empty"), { recursive: true });