Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(reference): preserve observed directories and unreadable links
morlutod6e4fe4a4dd8
diff --git a/src/reference/ReferenceSourceReader.test.ts b/src/reference/ReferenceSourceReader.test.tsnew file mode 100644index 000000000..2eef9ec31--- /dev/null+++ b/src/reference/ReferenceSourceReader.test.ts@@ -0,0 +1,113 @@+import { describe, expect, it } from "vitest";++import type {+ ReferenceSourceEntry,+ TraversalState,+} from "./ReferenceSourceReaderTypes.js";+import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceReader.js";++const failedDirectory = (+ path: string,+ code: "changed" | "io" | "symlink",+): ReferenceSourceEntry => ({+ status: "failed",+ kind: "directory",+ path,+ code,+ message: `directory failure ${code}`,+});++const internalLink = (path: string, target: string): ReferenceSourceEntry => ({+ status: "read",+ kind: "symlink",+ path,+ target,+ targetState: "internal",+});++const exclusions = (+ ...items: TraversalState["exclusions"][number][]+): TraversalState["exclusions"] => items;++describe("unobserved internal symlink reconciliation", () => {+ it("retains unreadable targets below the nearest failed directory", () => {+ const entries: ReferenceSourceEntry[] = [+ internalLink("alias.txt", "hidden/target.txt"),+ failedDirectory("hidden", "io"),+ {+ status: "read",+ kind: "file",+ path: "visible.txt",+ bytes: new Uint8Array([1]),+ size: 1,+ },+ ];++ reconcileUnobservedSymlinkTargets(entries);++ expect(entries[0]).toMatchObject({+ status: "failed",+ kind: "symlink",+ code: "io",+ message: expect.stringContaining("directory failure io"),+ });+ expect(entries[2]).toMatchObject({ status: "read", path: "visible.txt" });+ });++ it("matches Unicode-equivalent failed ancestor identities", () => {+ const entries: ReferenceSourceEntry[] = [+ internalLink("alias.txt", "hidden/cafe\u0301/target.txt"),+ failedDirectory("hidden/caf\u00e9", "changed"),+ ];++ reconcileUnobservedSymlinkTargets(entries);++ expect(entries[0]).toMatchObject({+ status: "failed",+ kind: "symlink",+ code: "changed",+ message: expect.stringContaining("directory failure changed"),+ });+ });++ it("stops at a complete nearer directory or an exclusion of unknown kind", () => {+ const entries: ReferenceSourceEntry[] = [+ internalLink("to-missing", "outer/complete/missing.txt"),+ internalLink("to-excluded", "outer/excluded/missing.txt"),+ failedDirectory("outer", "io"),+ { status: "read", kind: "directory", path: "outer/complete" },+ ];++ reconcileUnobservedSymlinkTargets(+ entries,+ exclusions({ path: "outer/excluded", kind: "unknown" }),+ );++ expect(entries[0]).toMatchObject({+ status: "read",+ targetState: "internal",+ });+ expect(entries[1]).toMatchObject({+ status: "read",+ targetState: "internal",+ });+ });++ it("keeps exact exclusions and leaves observations untouched after cancellation", () => {+ const entries = [internalLink("alias.txt", "hidden/target.txt")];+ reconcileUnobservedSymlinkTargets(+ entries,+ exclusions({ path: "hidden/target.txt", kind: "file" }),+ );+ expect(entries[0]).toMatchObject({ targetState: "internal" });++ const cancelledEntries: ReferenceSourceEntry[] = [+ internalLink("alias.txt", "hidden/target.txt"),+ failedDirectory("hidden", "io"),+ ];+ const controller = new AbortController();+ controller.abort();+ reconcileUnobservedSymlinkTargets(cancelledEntries, [], controller.signal);+ expect(cancelledEntries[0]).toMatchObject({ targetState: "internal" });+ });+});diff --git a/src/reference/ReferenceSourceReader.ts b/src/reference/ReferenceSourceReader.tsindex e8e8650a1..6420e3f34 100644--- a/src/reference/ReferenceSourceReader.ts+++ b/src/reference/ReferenceSourceReader.ts@@ -1,10 +1,14 @@+import { posix } from "node:path";+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";+import { createReferenceSourcePathLookup } from "../domain/referenceSourcePathIdentity.js";import { err, ok } from "../domain/result.js";import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";import { traverseDirectory } from "./ReferenceSourceReaderEntries.js";import type {ReferenceSourceReaderOptions,+ ReferenceSourceEntry,ReferenceSourceRead,ReferenceSourceResult,TraversalState,@@ -61,10 +65,20 @@ const readSource = async (? {}: { shouldExclude: options.shouldExclude }),entries: [],+ exclusions: [],pending: [{ path: canonicalRoot }],bytesRead: 0,};const traversal = await traverse(state);+ if (+ !isAborted(state.signal) &&+ (traversal.ok || traversal.error.code !== "cancelled")+ )+ reconcileUnobservedSymlinkTargets(+ state.entries,+ state.exclusions,+ state.signal,+ );state.entries.sort((left, right) =>compareUnicodeCodePoints(left.path, right.path),);@@ -77,6 +91,96 @@ const readSource = async (return traversal.ok ? ok(read) : err({ ...traversal.error, partial: read });};+/**+ * Resolve internal link observations against the completed reader inventory.+ * If the closest containing directory failed before enumeration completed,+ * the target remains unknown; a broader failed ancestor cannot override a+ * successfully enumerated directory or a recorded exclusion.+ */+export const reconcileUnobservedSymlinkTargets = (+ entries: ReferenceSourceEntry[],+ exclusions: TraversalState["exclusions"] = [],+ signal?: AbortSignal,+): void => {+ const failedDirectories = new Map(+ entries.flatMap((entry) =>+ entry.status === "failed" &&+ entry.kind === "directory" &&+ entry.code !== "cancelled"+ ? [[entry.path, entry] as const]+ : [],+ ),+ );+ if (failedDirectories.size === 0 || isAborted(signal)) return;+ const directoryEntries = entries.filter(+ (entry) => entry.kind === "directory",+ );+ const directoryPaths = createReferenceSourcePathLookup(+ directoryEntries.map(({ path }) => path),+ );+ const observedPaths = createReferenceSourcePathLookup(+ entries.map(({ path }) => path),+ );+ const excludedPaths = createReferenceSourcePathLookup(+ exclusions.map(({ path }) => path),+ );+ for (const [index, entry] of entries.entries()) {+ if (isAborted(signal)) return;+ if (+ entry.status !== "read" ||+ entry.kind !== "symlink" ||+ entry.targetState !== "internal" ||+ observedPaths.resolve(entry.target) !== undefined ||+ excludedPaths.resolve(entry.target) !== undefined+ )+ continue;+ const ancestor = nearestFailedAncestor(+ entry.target,+ directoryPaths,+ failedDirectories,+ excludedPaths,+ signal,+ );+ if (ancestor === undefined) continue;+ entries[index] = {+ status: "failed",+ kind: "symlink",+ path: entry.path,+ code: ancestor.code,+ message: `Symbolic link target is inside a directory that could not be enumerated: ${ancestor.message}`,+ };+ }+};++const nearestFailedAncestor = (+ target: string,+ directoryPaths: ReturnType<typeof createReferenceSourcePathLookup>,+ failedDirectories: ReadonlyMap<+ string,+ Extract<ReferenceSourceEntry, { status: "failed" }>+ >,+ excludedPaths: ReturnType<typeof createReferenceSourcePathLookup>,+ signal: AbortSignal | undefined,+): Extract<ReferenceSourceEntry, { status: "failed" }> | undefined => {+ for (+ let candidate = posix.dirname(target);+ candidate !== "." && candidate !== "/";+ candidate = posix.dirname(candidate)+ ) {+ if (isAborted(signal)) return undefined;+ if (excludedPaths.resolve(candidate) !== undefined) return undefined;+ const observedDirectory = directoryPaths.resolve(candidate);+ if (observedDirectory !== undefined)+ return failedDirectories.get(observedDirectory);+ }+ if (isAborted(signal)) return undefined;+ if (excludedPaths.resolve(".") !== undefined) return undefined;+ const observedRoot = directoryPaths.resolve(".");+ return observedRoot === undefined+ ? undefined+ : failedDirectories.get(observedRoot);+};+const traverse = async (state: TraversalState,): Promise<ReferenceSourceResult<undefined>> => {diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.tsindex a7ae837e8..1ca0ce3e2 100644--- a/src/reference/ReferenceSourceReaderEntries.ts+++ b/src/reference/ReferenceSourceReaderEntries.ts@@ -134,6 +134,7 @@ export const traverseDirectory = async (readFailure,),);+ scheduleObservedDirectories(state, directories);return { ok: true, value: undefined };}const after = await validateDirectory(@@ -152,6 +153,7 @@ export const traverseDirectory = async ("Directory changed while it was read",),);+ scheduleObservedDirectories(state, directories);return { ok: true, value: undefined };}if (current.path !== state.root)@@ -160,9 +162,16 @@ export const traverseDirectory = async (kind: "directory",path: pathFromRoot(state.root, current.path),});+ scheduleObservedDirectories(state, directories);+ return { ok: true, value: undefined };+};++const scheduleObservedDirectories = (+ state: TraversalState,+ directories: PendingDirectory[],+): void => {directories.reverse();state.pending.push(...directories);- return { ok: true, value: undefined };};const directoryReadFailure = (cause: unknown): string => {@@ -207,7 +216,10 @@ const processEntry = async (ok: false,error: failure("io", "Reference source exclusion check failed"),};- if (excluded.value) return { ok: true, value: undefined };+ if (excluded.value) {+ state.exclusions.push({ path, kind: "unknown" });+ return { ok: true, value: undefined };+ }state.entries.push(entryFailure(path, "unknown", "io", metadata.message));return { ok: true, value: undefined };}@@ -228,7 +240,10 @@ const processEntry = async (message: "Reference source exclusion check failed",},};- if (excluded.value) return { ok: true, value: undefined };+ if (excluded.value) {+ state.exclusions.push({ path, kind });+ return { ok: true, value: undefined };+ }if (isAborted(state.signal)) return { ok: false, error: cancelled() };if (kind === "symlink")state.entries.push(diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.tsindex 1fec38313..85d6d7602 100644--- a/src/reference/ReferenceSourceReaderTypes.ts+++ b/src/reference/ReferenceSourceReaderTypes.ts@@ -93,6 +93,10 @@ export type TraversalState = {kind: ReferenceSourceExclusionKind,) => boolean;readonly entries: ReferenceSourceEntry[];+ readonly exclusions: Array<{+ readonly path: string;+ readonly kind: ReferenceSourceExclusionKind;+ }>;readonly pending: PendingDirectory[];bytesRead: number;};diff --git a/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts b/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.tsnew file mode 100644index 000000000..6bc4be707--- /dev/null+++ b/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts@@ -0,0 +1,107 @@+import type { Dir, Dirent } from "node:fs";+import { mkdir, symlink, writeFile } from "node:fs/promises";+import { join } from "node:path";++import { expect, it, vi } from "vitest";++import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { importReferenceSource } from "../../support/referenceSourceResourceScope.js";++const { opendirMock } = vi.hoisted(() => ({ opendirMock: vi.fn() }));++vi.mock("node:fs/promises", async (importOriginal) => ({+ ...(await importOriginal<typeof import("node:fs/promises")>()),+ opendir: opendirMock,+}));++const directoryHandle = (items: readonly (string | Error | null)[]): Dir => {+ let index = 0;+ return {+ async read() {+ const item = items[index++];+ if (item instanceof Error) throw item;+ return item === null || item === undefined+ ? null+ : ({ name: item } as Dirent);+ },+ async close() {},+ } as unknown as Dir;+};++it.skipIf(process.platform === "win32")(+ "traverses directories lstat-observed before their parent enumeration failed",+ async () => {+ const root = await createTestTempDirectory(+ "rea-queued-source-directories-",+ );+ const container = join(root, "container");+ const hidden = join(container, "hidden");+ await mkdir(hidden, { recursive: true });+ await writeFile(join(container, "visible.txt"), "visible\n");+ await writeFile(join(hidden, "target.txt"), "target\n");+ await symlink("hidden/target.txt", join(container, "alias-file.txt"));+ await symlink("hidden", join(container, "alias-directory"));++ const readFailure = Object.assign(+ new Error("injected directory read failure"),+ { code: "EIO" },+ );+ opendirMock.mockImplementation(async (path: string) => {+ if (path === root) return directoryHandle(["container", null]);+ if (path === container)+ return directoryHandle([+ "visible.txt",+ "hidden",+ "alias-file.txt",+ "alias-directory",+ readFailure,+ ]);+ if (path === hidden) return directoryHandle(["target.txt", null]);+ throw new Error(`Unexpected directory traversal: ${path}`);+ });++ const imported = await importReferenceSource({+ root,+ caller: "queued-directory-read-failure-test",+ policy: { secretPatterns: [] },+ });+ if (!imported.ok) throw new Error(imported.error.message);+ expect(imported.value.inventory_state).toBe("partial");+ expect(imported.value.entries).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ path: "container",+ kind: "directory",+ tree_state: "unreadable",+ }),+ expect.objectContaining({+ path: "container/hidden",+ kind: "directory",+ tree_state: "enumerated",+ }),+ expect.objectContaining({+ path: "container/visible.txt",+ kind: "file",+ content_state: "hashed",+ }),+ expect.objectContaining({+ path: "container/hidden/target.txt",+ kind: "file",+ content_state: "hashed",+ }),+ expect.objectContaining({+ path: "container/alias-file.txt",+ kind: "symlink",+ target: "hidden/target.txt",+ target_state: "internal",+ }),+ expect.objectContaining({+ path: "container/alias-directory",+ kind: "symlink",+ target: "hidden",+ target_state: "internal",+ }),+ ]),+ );+ },+);diff --git a/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts b/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.tsindex ec868052b..ae54e16ce 100644--- a/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts+++ b/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts@@ -1,4 +1,4 @@-import { chmod, mkdir, writeFile } from "node:fs/promises";+import { chmod, mkdir, symlink, writeFile } from "node:fs/promises";import { join } from "node:path";import { describe, expect, it } from "vitest";@@ -62,6 +62,60 @@ describe("reference source native unreadable directories", () => {},);+ it.skipIf(permissionChecksUnavailable)(+ "keeps links into an unreadable subtree uncertain while preserving readable siblings",+ async () => {+ const root = await createTestTempDirectory("rea-source-unreadable-link-");+ const hidden = join(root, "hidden");+ await mkdir(hidden);+ await writeFile(join(root, "visible.txt"), "visible\n");+ await writeFile(join(hidden, "target.txt"), "target\n");+ await symlink("hidden/target.txt", join(root, "alias.txt"));+ await chmod(hidden, 0o111);+ try {+ const read = await readReferenceSource(root);+ if (!read.ok) throw new Error(read.error.message);+ expect(read.value.entries).toContainEqual(+ expect.objectContaining({+ status: "failed",+ kind: "directory",+ path: "hidden",+ code: "io",+ }),+ );+ expect(read.value.entries).toContainEqual(+ expect.objectContaining({+ status: "failed",+ kind: "symlink",+ path: "alias.txt",+ code: "io",+ }),+ );++ const imported = await importTree(root);+ if (!imported.ok) throw new Error(imported.error.message);+ expect(imported.value.inventory_state).toBe("partial");+ expect(imported.value.entries).toContainEqual(+ expect.objectContaining({+ path: "visible.txt",+ kind: "file",+ content_state: "hashed",+ }),+ );+ expect(imported.value.entries).toContainEqual(+ expect.objectContaining({+ path: "alias.txt",+ kind: "symlink",+ target: null,+ target_state: "unreadable",+ }),+ );+ } finally {+ await chmod(hidden, 0o700);+ }+ },+ );+it("keeps readable and empty directories enumerated exactly once", async () => {const root = await createTestTempDirectory("rea-source-readable-dirs-");await mkdir(join(root, "src", "empty"), { recursive: true });