Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(reference)!: preserve symlink races and root failure facts
morluto6296fdf560f1
diff --git a/docs/cli.md b/docs/cli.mdindex c0f8eefff..cad2255d9 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -286,6 +286,12 @@ reported separately as configured-secret exclusions.Re-import saved graphs whose pattern-based exclusions lack `pattern` beforeusing them in source-to-bundle comparisons.+Entries that change during an import retain an unavailable-entry diagnostic+alongside readable siblings. If enumeration of an already admitted root fails,+the graph keeps its observed children and records the root failure as a+top-level limitation. That partial inventory does not establish absence among+unexamined children.+JavaScript and TypeScript import parsing requires valid UTF-8. Malformed sourcebytes retain their original hashes and sizes with a decoding diagnostic; REAdoes not infer module targets from replacement characters.diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.tsindex 84885a215..d60320182 100644--- a/src/application/ReferenceSourceImportEntries.ts+++ b/src/application/ReferenceSourceImportEntries.ts@@ -419,6 +419,10 @@ export const parseReferenceSourceEntries = (parseFailures: [],limitations: [...read.limitations],};+ if (read.rootFailure !== undefined)+ output.limitations.push(+ `Reference root enumeration is incomplete: ${read.rootFailure.message}`,+ );const packageMainInventory = readPackageMains(read);const context: SpecifierContext = {lookupPath: createReferenceSourcePathLookup(filePaths),diff --git a/src/reference/ReferenceSourceReader.test.ts b/src/reference/ReferenceSourceReader.test.tsindex 2eef9ec31..8cb73cae9 100644--- a/src/reference/ReferenceSourceReader.test.ts+++ b/src/reference/ReferenceSourceReader.test.ts@@ -30,6 +30,33 @@ const exclusions = (): TraversalState["exclusions"] => items;describe("unobserved internal symlink reconciliation", () => {+ it("uses the explicit root failure only when no closer completed directory or exclusion covers the target", () => {+ const entries: ReferenceSourceEntry[] = [+ internalLink("unobserved", "unseen/target.txt"),+ internalLink("complete", "known/target.txt"),+ internalLink("excluded", "omitted/target.txt"),+ { status: "read", kind: "directory", path: "known" },+ ];++ reconcileUnobservedSymlinkTargets(+ entries,+ exclusions({ path: "omitted", kind: "directory" }),+ undefined,+ { code: "changed", message: "Reference root changed" },+ );++ expect(entries[0]).toMatchObject({+ status: "failed",+ kind: "symlink",+ code: "changed",+ message: expect.stringContaining("Reference root changed"),+ });+ expect(entries.slice(1, 3)).toEqual([+ internalLink("complete", "known/target.txt"),+ internalLink("excluded", "omitted/target.txt"),+ ]);+ });+it("retains unreadable targets below the nearest failed directory", () => {const entries: ReferenceSourceEntry[] = [internalLink("alias.txt", "hidden/target.txt"),diff --git a/src/reference/ReferenceSourceReader.ts b/src/reference/ReferenceSourceReader.tsindex 88553bee2..4af7da9de 100644--- a/src/reference/ReferenceSourceReader.ts+++ b/src/reference/ReferenceSourceReader.ts@@ -9,6 +9,7 @@ import { traverseDirectory } from "./ReferenceSourceReaderEntries.js";import type {ReferenceSourceReaderOptions,ReferenceSourceEntry,+ ReferenceSourceRootFailure,ReferenceSourceRead,ReferenceSourceResult,TraversalState,@@ -78,12 +79,16 @@ const readSource = async (state.entries,state.exclusions,state.signal,+ state.rootFailure,);state.entries.sort((left, right) =>compareUnicodeCodePoints(left.path, right.path),);const read: ReferenceSourceRead = {root: canonicalRoot,+ ...(state.rootFailure === undefined+ ? {}+ : { rootFailure: state.rootFailure }),entries: state.entries,bytesRead: state.bytesRead,limitations: [PATH_RACE_LIMITATION],@@ -101,6 +106,7 @@ export const reconcileUnobservedSymlinkTargets = (entries: ReferenceSourceEntry[],exclusions: TraversalState["exclusions"] = [],signal?: AbortSignal,+ rootFailure?: ReferenceSourceRootFailure,): void => {const failedDirectories = new Map(entries.flatMap((entry) =>@@ -111,7 +117,11 @@ export const reconcileUnobservedSymlinkTargets = (: [],),);- if (failedDirectories.size === 0 || isAborted(signal)) return;+ if (+ (failedDirectories.size === 0 && rootFailure === undefined) ||+ isAborted(signal)+ )+ return;const directoryEntries = entries.filter((entry) => entry.kind === "directory",);@@ -140,6 +150,7 @@ export const reconcileUnobservedSymlinkTargets = (failedDirectories,excludedPaths,signal,+ rootFailure,);if (ancestor === undefined) continue;entries[index] = {@@ -161,7 +172,8 @@ const nearestFailedAncestor = (>,excludedPaths: ReturnType<typeof createReferenceSourcePathLookup>,signal: AbortSignal | undefined,-): Extract<ReferenceSourceEntry, { status: "failed" }> | undefined => {+ rootFailure: ReferenceSourceRootFailure | undefined,+): (ReferenceSourceRootFailure & { readonly path: string }) | undefined => {for (let candidate = posix.dirname(target);candidate !== "." && candidate !== "/";@@ -175,10 +187,7 @@ const nearestFailedAncestor = (}if (isAborted(signal)) return undefined;if (excludedPaths.resolve(".") !== undefined) return undefined;- const observedRoot = directoryPaths.resolve(".");- return observedRoot === undefined- ? undefined- : failedDirectories.get(observedRoot);+ return rootFailure === undefined ? undefined : { path: ".", ...rootFailure };};const traverse = async (diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.tsindex 1ca0ce3e2..c7f634168 100644--- a/src/reference/ReferenceSourceReaderEntries.ts+++ b/src/reference/ReferenceSourceReaderEntries.ts@@ -26,10 +26,29 @@ import type {PendingDirectory,ReferenceSourceEntry,ReferenceSourceExclusionKind,+ ReferenceSourceFailureCode,ReferenceSourceResult,TraversalState,} from "./ReferenceSourceReaderTypes.js";+const recordDirectoryFailure = (+ state: TraversalState,+ absolute: string,+ code: ReferenceSourceFailureCode,+ message: string,+): void => {+ if (absolute === state.root) state.rootFailure = { code, message };+ else+ state.entries.push(+ entryFailure(+ pathFromRoot(state.root, absolute),+ "directory",+ code,+ message,+ ),+ );+};+export const traverseDirectory = async (state: TraversalState,current: PendingDirectory,@@ -42,14 +61,7 @@ export const traverseDirectory = async ();if (isAborted(state.signal)) return { ok: false, error: cancelled() };if (!before.ok) {- state.entries.push(- entryFailure(- pathFromRoot(state.root, current.path),- "directory",- before.code,- before.message,- ),- );+ recordDirectoryFailure(state, current.path, before.code, before.message);return { ok: true, value: undefined };}const directories: PendingDirectory[] = [];@@ -86,14 +98,7 @@ export const traverseDirectory = async (}if (cleanup.kind === "failed") {if (result.ok && !isAborted(state.signal) && readFailure !== undefined)- state.entries.push(- entryFailure(- pathFromRoot(state.root, current.path),- "directory",- "io",- readFailure,- ),- );+ recordDirectoryFailure(state, current.path, "io", readFailure);const observation = ArtifactReaderFailure.cleanupObservation(cleanup.cause,current.path,@@ -126,14 +131,7 @@ export const traverseDirectory = async (}if (isAborted(state.signal)) return { ok: false, error: cancelled() };if (readFailure !== undefined) {- state.entries.push(- entryFailure(- pathFromRoot(state.root, current.path),- "directory",- "io",- readFailure,- ),- );+ recordDirectoryFailure(state, current.path, "io", readFailure);scheduleObservedDirectories(state, directories);return { ok: true, value: undefined };}@@ -145,13 +143,11 @@ export const traverseDirectory = async ();if (isAborted(state.signal)) return { ok: false, error: cancelled() };if (!after.ok || !sameFile(before.stats, after.stats)) {- state.entries.push(- entryFailure(- pathFromRoot(state.root, current.path),- "directory",- "changed",- "Directory changed while it was read",- ),+ recordDirectoryFailure(+ state,+ current.path,+ after.ok ? "changed" : after.code,+ after.ok ? "Directory changed while it was read" : after.message,);scheduleObservedDirectories(state, directories);return { ok: true, value: undefined };@@ -356,7 +352,17 @@ const readSymlinkTarget = async (absolute: string,path: string,): Promise<ReferenceSourceEntry> => {- const rawTarget = await readlink(absolute);+ let rawTarget: string;+ try {+ rawTarget = await readlink(absolute);+ } catch (cause: unknown) {+ // POSIX readlink reports EINVAL when the observed link became a non-link.+ // Keep this classification at the syscall, rather than hiding EINVAL from+ // unrelated metadata or target-resolution operations.+ if (cause instanceof Error && Reflect.get(cause, "code") === "EINVAL")+ return changedSymlink(path);+ throw cause;+ }const lexicalTarget = resolve(dirname(absolute), rawTarget);if (!isPathWithinRoot(root, lexicalTarget))return {diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.tsindex 85d6d7602..a9e3f470b 100644--- a/src/reference/ReferenceSourceReaderTypes.ts+++ b/src/reference/ReferenceSourceReaderTypes.ts@@ -61,8 +61,15 @@ export type ReferenceSourceEntry =readonly size?: number;};+/** A failure enumerating an already admitted root, separate from its children. */+export interface ReferenceSourceRootFailure {+ readonly code: ReferenceSourceFailureCode;+ readonly message: string;+}+export interface ReferenceSourceRead {readonly root: string;+ readonly rootFailure?: ReferenceSourceRootFailure;readonly entries: readonly ReferenceSourceEntry[];readonly bytesRead: number;/** Node exposes no portable openat traversal; pathname identity is checked around each operation. */@@ -98,6 +105,7 @@ export type TraversalState = {readonly kind: ReferenceSourceExclusionKind;}>;readonly pending: PendingDirectory[];+ rootFailure?: ReferenceSourceRootFailure;bytesRead: number;};diff --git a/tests/boundary/filesystem/referenceSourceCleanupOwnership.test.ts b/tests/boundary/filesystem/referenceSourceCleanupOwnership.test.tsindex 3ad4665bf..e471bceaf 100644--- a/tests/boundary/filesystem/referenceSourceCleanupOwnership.test.ts+++ b/tests/boundary/filesystem/referenceSourceCleanupOwnership.test.ts@@ -253,10 +253,7 @@ it("preserves a directory read failure when closing that directory also fails",bytes: Buffer.from(bytes),}),);- expect(result.error.partial?.entries).toContainEqual({- status: "failed",- kind: "directory",- path: ".",+ expect(result.error.partial?.rootFailure).toEqual({code: "io",message: expect.stringContaining(readFailure.message),});diff --git a/tests/process-global/referenceSourceSymlinkRace.test.ts b/tests/process-global/referenceSourceSymlinkRace.test.tsnew file mode 100644index 000000000..6229ab873--- /dev/null+++ b/tests/process-global/referenceSourceSymlinkRace.test.ts@@ -0,0 +1,213 @@+import { expect, it, vi } from "vitest";+import type { PathLike } from "node:fs";++const race = vi.hoisted(() => ({+ path: "",+ beforeReadlink: undefined as (() => Promise<void>) | undefined,+ actualErrno: undefined as string | undefined,+ directory: "",+ afterDirectoryClose: undefined as (() => Promise<void>) | undefined,+}));++vi.mock("node:fs/promises", async (importOriginal) => {+ const filesystem = await importOriginal<typeof import("node:fs/promises")>();+ return {+ ...filesystem,+ opendir: async (...args: Parameters<typeof filesystem.opendir>) => {+ const directory = await filesystem.opendir(...args);+ if (String(args[0]) === race.directory) {+ const close = directory.close.bind(directory);+ vi.spyOn(directory, "close").mockImplementation(async () => {+ await close();+ const hook = race.afterDirectoryClose;+ race.afterDirectoryClose = undefined;+ await hook?.();+ });+ }+ return directory;+ },+ readlink: async (+ path: PathLike,+ ...args: Parameters<typeof filesystem.readlink> extends [+ PathLike,+ ...infer R,+ ]+ ? R+ : never+ ) => {+ if (String(path) === race.path && race.beforeReadlink !== undefined) {+ const beforeReadlink = race.beforeReadlink;+ race.beforeReadlink = undefined;+ await beforeReadlink();+ }+ try {+ return await filesystem.readlink(path, ...args);+ } catch (cause: unknown) {+ if (String(path) === race.path && cause instanceof Error)+ race.actualErrno = Reflect.get(cause, "code");+ throw cause;+ }+ },+ };+});++import {+ mkdir,+ readFile,+ rename,+ unlink,+ writeFile,+ symlink,+} from "node:fs/promises";+import { join } from "node:path";+import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";+import { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js";+import { importReferenceSource } from "../../src/application/ReferenceSourceImport.js";+import { readReferenceSource } from "../../src/reference/ReferenceSourceReader.js";++it.skipIf(process.platform === "win32").each(["reader", "import"] as const)(+ "keeps sibling observations through %s when a symlink becomes a regular file before readlink",+ async (operation) => {+ const root = await createTestTempDirectory("rea-reference-symlink-race-");+ await mkdir(join(root, "nested"));+ const linkPath = join(root, "nested", "link");+ const siblingPath = join(root, "sibling.txt");+ await symlink("missing-target", linkPath);+ await writeFile(siblingPath, "sibling remains observable");+ race.path = linkPath;+ race.actualErrno = undefined;+ race.beforeReadlink = async () => {+ await unlink(linkPath);+ await writeFile(linkPath, "replacement regular file");+ };++ const resources = new ArtifactResourceScope();+ let read: Awaited<ReturnType<typeof readReferenceSource>> | undefined;+ let imported: Awaited<ReturnType<typeof importReferenceSource>> | undefined;+ let thrown: unknown;+ try {+ if (operation === "reader")+ read = await readReferenceSource(root, resources);+ else+ imported = await importReferenceSource(+ {+ root,+ caller: "symlink-replacement-test",+ policy: { secretPatterns: [] },+ },+ resources,+ );+ } catch (cause: unknown) {+ thrown = cause;+ } finally {+ race.path = "";+ race.beforeReadlink = undefined;+ await resources.close();+ }++ // The hook establishes scheduling; this assertion verifies the host's real+ // readlink syscall reports the replacement as EINVAL.+ expect(race.actualErrno).toBe("EINVAL");+ expect(thrown).toBeUndefined();+ expect(read?.ok ?? imported?.ok, JSON.stringify(imported)).toBe(true);+ if (imported?.ok) {+ expect(imported.value.inventory_state).toBe("partial");+ expect(imported.value.entries).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ path: "nested/link",+ kind: "symlink",+ target: null,+ target_state: "unreadable",+ }),+ expect.objectContaining({+ path: "sibling.txt",+ kind: "file",+ content_state: "hashed",+ sha256: expect.any(String),+ }),+ ]),+ );+ expect(+ imported.value.entries+ .find(({ path }) => path === "nested/link")+ ?.limitations.join(" "),+ ).toContain("Symbolic link changed");+ }+ if (!read?.ok) return;+ expect(read.value.rootFailure).toBeUndefined();+ expect(read.value.entries.some(({ path }) => path === ".")).toBe(false);+ expect(read.value.entries).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ status: "failed",+ kind: "symlink",+ path: "nested/link",+ code: "changed",+ }),+ expect.objectContaining({+ status: "read",+ kind: "file",+ path: "sibling.txt",+ }),+ ]),+ );+ },+);++it.skipIf(process.platform === "win32").each(["contents", "root"] as const)(+ "retains a changed %s as a graph limitation after its directory handle closes",+ async (changed) => {+ const parent = await createTestTempDirectory("rea-reference-root-race-");+ const root = join(parent, "observed");+ await mkdir(root);+ const source = join(root, "source.txt");+ await writeFile(source, "already observed source");+ race.directory = root;+ race.afterDirectoryClose = () =>+ changed === "contents"+ ? rename(source, join(root, "renamed.txt"))+ : rename(root, join(parent, "moved"));+ const resources = new ArtifactResourceScope();+ let imported: Awaited<ReturnType<typeof importReferenceSource>>;+ try {+ imported = await importReferenceSource(+ {+ root,+ caller: "root-revalidation-test",+ policy: { secretPatterns: [] },+ },+ resources,+ );+ } finally {+ race.directory = "";+ race.afterDirectoryClose = undefined;+ await resources.close();+ }+ const retainedPath =+ changed === "contents"+ ? join(root, "renamed.txt")+ : join(parent, "moved", "source.txt");+ expect(await readFile(retainedPath, "utf8")).toBe(+ "already observed source",+ );+ await expect(readFile(source)).rejects.toMatchObject({ code: "ENOENT" });+ expect(imported.ok, JSON.stringify(imported)).toBe(true);+ if (!imported.ok) return;+ expect(imported.value.inventory_state).toBe("partial");+ expect(imported.value.limitations).toContainEqual(+ expect.stringContaining(+ changed === "contents"+ ? "Reference root enumeration is incomplete: Reference root changed"+ : "Reference root enumeration is incomplete: Directory identity could not be verified: ENOENT",+ ),+ );+ expect(imported.value.entries).toEqual([+ expect.objectContaining({+ path: "source.txt",+ kind: "file",+ content_state: "hashed",+ }),+ ]);+ },+);