Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(reference): report entries beyond the path limit as I/O failures (#2716)

Kaoru0822679658b28748
diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts
index 769f8bb8c..f917f1965 100644
--- a/src/application/ReferenceSourceImport.ts
+++ b/src/application/ReferenceSourceImport.ts
@@ -12,7 +12,7 @@ import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.j
import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js";
import { readReferenceSource } from "../reference/ReferenceSourceReader.js";
import type {
- ReferenceSourceEntryKind,
+ ReferenceSourceExclusionKind,
ReferenceSourceRead,
ReferenceSourceReaderError,
} from "../reference/ReferenceSourceReaderTypes.js";
@@ -111,7 +111,8 @@ const deriveInventoryState = (
entry.limitations.length > 0 ||
(entry.kind === "file" && entry.content_state !== "hashed") ||
(entry.kind === "directory" && entry.tree_state !== "enumerated") ||
- (entry.kind === "symlink" && entry.target_state !== "internal"),
+ (entry.kind === "symlink" && entry.target_state !== "internal") ||
+ entry.kind === "unknown",
);
if (partialEntry) return "partial";
const partialRelationship = input.relationships.some(
@@ -142,11 +143,24 @@ const createShouldExclude =
exclusions: HistoricalSourceGraphInput["exclusions"],
secrets: PreparedReferenceSourceImport["secrets"],
ignored: PreparedReferenceSourceImport["ignored"],
- ): ((path: string, kind: ReferenceSourceEntryKind) => boolean) =>
+ ): ((path: string, kind: ReferenceSourceExclusionKind) => boolean) =>
(path, kind) => {
- const patternPath = kind === "directory" ? `${path}/` : path;
- const secretMatch = secrets.test(patternPath);
- if (secretMatch.ignored) {
+ // An entry of unknown kind may be a directory, so directory-only rules
+ // must still exclude it; this fails closed for caller-marked secrets.
+ const patternPaths =
+ kind === "directory"
+ ? [`${path}/`]
+ : kind === "unknown"
+ ? [path, `${path}/`]
+ : [path];
+ const firstMatch = (matcher: typeof secrets) => {
+ const results = patternPaths.map((patternPath) =>
+ matcher.test(patternPath),
+ );
+ return results.find(({ ignored }) => ignored) ?? results[0];
+ };
+ const secretMatch = firstMatch(secrets);
+ if (secretMatch?.ignored === true) {
if (!secretMatch.rule)
throw new Error(`Ignored secret path has no matching rule: ${path}`);
exclusions.push({
@@ -156,8 +170,8 @@ const createShouldExclude =
});
return true;
}
- const match = ignored.test(patternPath);
- if (!match.ignored) return false;
+ const match = firstMatch(ignored);
+ if (match?.ignored !== true) return false;
if (!match.rule)
throw new Error(`Ignored path has no matching rule: ${path}`);
const reason = match.rule.mark;
diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts
index 49c6f82f5..29dbef356 100644
--- a/src/application/ReferenceSourceImportEntries.ts
+++ b/src/application/ReferenceSourceImportEntries.ts
@@ -45,6 +45,8 @@ const entryRecoveryGuidance = (
return "This directory could not be read. Check its permissions, then try again.";
if (entry.kind === "symlink")
return "This symbolic link could not be read safely. Check the link and its permissions, then try again.";
+ if (entry.kind === "unknown")
+ return "This entry's type could not be determined. Check the parent directory and try again.";
return "This file could not be read. Check its permissions, then try again.";
};
@@ -73,6 +75,13 @@ const failedEntry = (
classifications,
limitations: [limitation],
};
+ if (entry.kind === "unknown")
+ return {
+ path: entry.path,
+ kind: "unknown",
+ classifications: ["unknown"],
+ limitations: [limitation],
+ };
return {
path: entry.path,
kind: "file",
diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts
index 1286cdde6..8fe868832 100644
--- a/src/domain/referenceSourceGraph.test.ts
+++ b/src/domain/referenceSourceGraph.test.ts
@@ -259,6 +259,30 @@ describe("historical source graph", () => {
});
describe("partial historical source graphs", () => {
+ it("keeps unknown-entry diagnostics outside the root commitment", () => {
+ const base = graphInput();
+ base.inventory_state = "partial";
+ base.entries[2] = {
+ path: "src/package.json",
+ kind: "unknown",
+ classifications: ["unknown"],
+ limitations: ["Permission denied at /private/first"],
+ };
+ const graph = createHistoricalSourceGraph(base);
+ const changedMessage = structuredClone(base);
+ changedMessage.entries[2] = {
+ path: "src/package.json",
+ kind: "unknown",
+ classifications: ["unknown"],
+ limitations: ["Permission denied at /private/second"],
+ };
+ const revised = createHistoricalSourceGraph(changedMessage);
+ expect(revised.root_sha256).toBe(graph.root_sha256);
+ expect(computeHistoricalSourceGraphSha256(revised)).not.toBe(
+ computeHistoricalSourceGraphSha256(graph),
+ );
+ });
+
it("keeps an unreadable symlink target unknown and out of root identity", () => {
const base = graphInput();
base.inventory_state = "partial";
diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts
index 197e473bf..a830e0aa0 100644
--- a/src/domain/referenceSourceGraph.ts
+++ b/src/domain/referenceSourceGraph.ts
@@ -111,6 +111,10 @@ const sourceEntrySchema = z.union([
sourceFileSchema,
sourceDirectorySchema,
sourceSymlinkSchema,
+ z.strictObject({
+ ...entryBaseShape,
+ kind: z.literal("unknown"),
+ }),
]);
const sourceRelationshipSchema = z.strictObject({
@@ -361,7 +365,8 @@ const checkCompleteInventory = (
entry.limitations.length > 0 ||
(entry.kind === "file" && entry.content_state !== "hashed") ||
(entry.kind === "directory" && entry.tree_state !== "enumerated") ||
- (entry.kind === "symlink" && entry.target_state !== "internal"),
+ (entry.kind === "symlink" && entry.target_state !== "internal") ||
+ entry.kind === "unknown",
);
const partialRelationship = graph.relationships.some(
({ parse_state, resolution }) =>
@@ -478,6 +483,11 @@ const rootCommitment = (
target: entry.target,
target_state: entry.target_state,
};
+ case "unknown":
+ return {
+ kind: entry.kind,
+ path: entry.path,
+ };
}
}),
exclusions,
diff --git a/src/domain/referenceSourceSummary.test.ts b/src/domain/referenceSourceSummary.test.ts
index a0d23b0e1..3f34a073f 100644
--- a/src/domain/referenceSourceSummary.test.ts
+++ b/src/domain/referenceSourceSummary.test.ts
@@ -81,6 +81,26 @@ describe("committed historical-source summaries", () => {
expect(graph).toEqual(original);
});
+ it("counts entries of unknown kind separately from symlinks", () => {
+ const graph = createHistoricalSourceGraph(
+ input([
+ {
+ path: "deep",
+ kind: "unknown",
+ classifications: ["unknown"],
+ limitations: ["lstat: ENAMETOOLONG"],
+ },
+ ]),
+ );
+ expect(createHistoricalSourceSummary(graph, 1).counts).toMatchObject({
+ entries: 1,
+ files: 0,
+ directories: 0,
+ symlinks: 0,
+ unknown: 1,
+ });
+ });
+
it("sums known bytes losslessly beyond the JSON safe integer range", () => {
const graph = createHistoricalSourceGraph(
input(
diff --git a/src/domain/referenceSourceSummary.ts b/src/domain/referenceSourceSummary.ts
index e7e99e3fa..55d24025d 100644
--- a/src/domain/referenceSourceSummary.ts
+++ b/src/domain/referenceSourceSummary.ts
@@ -29,6 +29,7 @@ export const createHistoricalSourceSummary = (
files: 0,
directories: 0,
symlinks: 0,
+ unknown: 0,
relationships: graph.relationships.length,
parse_failures: graph.parse_failures.length,
exclusions: graph.exclusions.length,
@@ -59,7 +60,8 @@ export const createHistoricalSourceSummary = (
} else if (entry.kind === "directory") {
counts.directories += 1;
treeStates[entry.tree_state] += 1;
- } else counts.symlinks += 1;
+ } else if (entry.kind === "symlink") counts.symlinks += 1;
+ else counts.unknown += 1;
// Canonical entries already use Unicode code-point order; filtering preserves it.
if (entry.path.split("/").length <= outlineDepth)
outline.push(outlineEntry(entry));
diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts
index e684a2b5c..8ec5e9d80 100644
--- a/src/reference/ReferenceSourceReaderEntries.ts
+++ b/src/reference/ReferenceSourceReaderEntries.ts
@@ -25,7 +25,7 @@ import {
import type {
PendingDirectory,
ReferenceSourceEntry,
- ReferenceSourceEntryKind,
+ ReferenceSourceExclusionKind,
ReferenceSourceResult,
TraversalState,
} from "./ReferenceSourceReaderTypes.js";
@@ -201,6 +201,13 @@ const processEntry = async (
const metadata = await readMetadata(absolute);
if (isAborted(state.signal)) return { ok: false, error: cancelled() };
if (!metadata.ok) {
+ const excluded = applyExclusion(state.shouldExclude, path, "unknown");
+ if (!excluded.ok)
+ return {
+ ok: false,
+ error: failure("io", "Reference source exclusion check failed"),
+ };
+ if (excluded.value) return { ok: true, value: undefined };
state.entries.push(entryFailure(path, "unknown", "io", metadata.message));
return { ok: true, value: undefined };
}
@@ -371,10 +378,10 @@ const readSymlinkTarget = async (
const applyExclusion = (
shouldExclude:
- | ((path: string, kind: ReferenceSourceEntryKind) => boolean)
+ | ((path: string, kind: ReferenceSourceExclusionKind) => boolean)
| undefined,
path: string,
- kind: ReferenceSourceEntryKind,
+ kind: ReferenceSourceExclusionKind,
): { readonly ok: true; readonly value: boolean } | { readonly ok: false } => {
try {
return { ok: true, value: shouldExclude?.(path, kind) === true };
diff --git a/src/reference/ReferenceSourceReaderErrors.ts b/src/reference/ReferenceSourceReaderErrors.ts
index 6ddd02a3f..2c826b4c9 100644
--- a/src/reference/ReferenceSourceReaderErrors.ts
+++ b/src/reference/ReferenceSourceReaderErrors.ts
@@ -29,6 +29,9 @@ const FILESYSTEM_ERROR_CODES = new Set([
"EISDIR",
"ELOOP",
"EMFILE",
+ // A traversed child can exceed the platform's absolute path limit even
+ // though the selected root and every component are valid.
+ "ENAMETOOLONG",
"ENFILE",
"ENOENT",
"ENOTDIR",
@@ -44,6 +47,8 @@ export const filesystemFailureDetail = (
): string | undefined => {
if (!(cause instanceof Error)) return undefined;
const code: unknown = Reflect.get(cause, "code");
+ if (code === "ENAMETOOLONG")
+ return `${operation}: ENAMETOOLONG (entry path exceeds the operating system path limit)`;
return typeof code === "string" && FILESYSTEM_ERROR_CODES.has(code)
? `${operation}: ${cause.message}`
: undefined;
diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.ts
index 3b0a368b5..1fec38313 100644
--- a/src/reference/ReferenceSourceReaderTypes.ts
+++ b/src/reference/ReferenceSourceReaderTypes.ts
@@ -10,11 +10,17 @@ export type ReferenceSourceEntryKind =
| "symlink"
| "other";
+/**
+ * Kind offered to exclusion predicates. `unknown` marks an entry whose
+ * metadata could not be read, so its concrete kind cannot be claimed.
+ */
+export type ReferenceSourceExclusionKind = ReferenceSourceEntryKind | "unknown";
+
export interface ReferenceSourceReaderOptions {
readonly signal?: AbortSignal;
readonly shouldExclude?: (
path: string,
- kind: ReferenceSourceEntryKind,
+ kind: ReferenceSourceExclusionKind,
) => boolean;
}
@@ -84,7 +90,7 @@ export type TraversalState = {
readonly signal?: AbortSignal;
readonly shouldExclude?: (
path: string,
- kind: ReferenceSourceEntryKind,
+ kind: ReferenceSourceExclusionKind,
) => boolean;
readonly entries: ReferenceSourceEntry[];
readonly pending: PendingDirectory[];
diff --git a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts
index dd87ea92a..82476d6db 100644
--- a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts
+++ b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts
@@ -133,3 +133,88 @@ describe("reference import native entry failure details", () => {
).toMatchObject({ kind: "file", content_state: "hashed", limitations: [] });
});
});
+
+describe("reference import paths beyond the platform limit", () => {
+ it.skipIf(process.platform === "win32")(
+ "reports an entry beyond the absolute path limit as an I/O failure",
+ async () => {
+ const root = await createTestTempDirectory("rea-import-deep-path-");
+ await writeFile(join(root, "kept.txt"), "k");
+ // Relative creation lets the tree exceed PATH_MAX (1024 on macOS, 4096
+ // on Linux) while each 200-byte component stays below NAME_MAX.
+ const deepTree = (action: "create" | "remove") =>
+ promisify(execFile)(process.execPath, [
+ "-e",
+ `const fs = require("node:fs");
+ const names = Array.from({ length: 24 }, (_, i) => "d" + String(i).padStart(2, "0") + "x".repeat(197));
+ process.chdir(${JSON.stringify(root)});
+ if (${JSON.stringify(action)} === "create") {
+ for (const name of names) { fs.mkdirSync(name); process.chdir(name); }
+ fs.writeFileSync("deep.txt", "d");
+ } else {
+ for (const name of names) process.chdir(name);
+ fs.rmSync("deep.txt");
+ for (const name of [...names].reverse()) { process.chdir(".."); fs.rmdirSync(name); }
+ }`,
+ ]);
+ await deepTree("create");
+ try {
+ const read = await readReferenceSource(root);
+ if (!read.ok) throw new Error(read.error.message);
+ expect(read.value.entries).toContainEqual(
+ expect.objectContaining({ path: "kept.txt", status: "read" }),
+ );
+ const failed = read.value.entries.find(
+ (entry) => entry.status === "failed",
+ );
+ expect(failed).toMatchObject({
+ status: "failed",
+ code: "io",
+ message: expect.stringContaining("ENAMETOOLONG"),
+ });
+ const filteredRead = await readReferenceSource(root, {
+ shouldExclude: (path, kind) =>
+ path === failed?.path && kind === "directory",
+ });
+ if (!filteredRead.ok) throw new Error(filteredRead.error.message);
+ expect(
+ filteredRead.value.entries.find(
+ (entry) => entry.path === failed?.path,
+ ),
+ ).toMatchObject({ status: "failed", kind: "unknown", code: "io" });
+ const imported = await importTree(root);
+ if (!imported.ok) throw new Error(imported.error.message);
+ expect(imported.value.entries).toContainEqual(
+ expect.objectContaining({
+ path: "kept.txt",
+ content_state: "hashed",
+ }),
+ );
+ const deep = imported.value.entries.find(
+ (entry) => entry.path === failed?.path,
+ );
+ expect(deep?.limitations.join(" ")).toContain(
+ failed?.status === "failed" ? failed.message : "missing failure",
+ );
+ // A directory-only secret rule still applies when the kind is unknown.
+ const secretPattern = `${failed?.path ?? "missing"}/`;
+ const secret = await importReferenceSource({
+ root,
+ caller: "native-entry-failure-detail-test",
+ policy: { secretPatterns: [secretPattern] },
+ });
+ if (!secret.ok) throw new Error(secret.error.message);
+ expect(
+ secret.value.entries.some(({ path }) => path === failed?.path),
+ ).toBe(false);
+ expect(secret.value.exclusions).toContainEqual({
+ path: failed?.path,
+ reason: "configured-secret",
+ pattern: secretPattern,
+ });
+ } finally {
+ await deepTree("remove");
+ }
+ },
+ );
+});