Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(reference-source): preserve raw POSIX filename identity
morluto8f9b2ab94817
diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.tsindex c7f634168..39d2b520c 100644--- a/src/reference/ReferenceSourceReaderEntries.ts+++ b/src/reference/ReferenceSourceReaderEntries.ts@@ -66,9 +66,16 @@ export const traverseDirectory = async (}const directories: PendingDirectory[] = [];let readFailure: string | undefined;+ let unrepresentableNameCount = 0;+ let firstUnrepresentableNameHex: string | undefined;let handle: Dir | undefined;try {- handle = await opendir(current.path);+ // Preserve directory-entry identity until UTF-8 validity is established.+ // Node's opendir accepts this byte-preserving encoding; @types/node's+ // OpenDirOptions currently narrows it to text encodings.+ handle = await opendir(current.path, {+ encoding: "buffer" as BufferEncoding,+ });} catch (cause: unknown) {readFailure = directoryReadFailure(cause);}@@ -86,7 +93,17 @@ export const traverseDirectory = async (break;}if (child === null) break;- result = await processEntry(state, current, child.name, directories);+ if (!Buffer.isBuffer(child.name)) {+ readFailure = "Directory entry name was not returned as raw bytes";+ break;+ }+ const name = child.name.toString("utf8");+ if (!Buffer.from(name, "utf8").equals(child.name)) {+ unrepresentableNameCount += 1;+ firstUnrepresentableNameHex ??= child.name.toString("hex");+ continue;+ }+ result = await processEntry(state, current, name, directories);if (!result.ok) break;}} finally {@@ -152,7 +169,14 @@ export const traverseDirectory = async (scheduleObservedDirectories(state, directories);return { ok: true, value: undefined };}- if (current.path !== state.root)+ if (unrepresentableNameCount > 0) {+ recordDirectoryFailure(+ state,+ current.path,+ "unsupported",+ `Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`,+ );+ } else if (current.path !== state.root)state.entries.push({status: "read",kind: "directory",diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.tsindex 0a1243a91..33af3c020 100644--- a/tests/boundary/cli/referenceSourceUtf8.test.ts+++ b/tests/boundary/cli/referenceSourceUtf8.test.ts@@ -84,6 +84,55 @@ cliTest(},);+cliTest.skipIf(process.platform !== "linux")(+ "reports unsupported raw POSIX filename bytes without substituting a Unicode sibling",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-reference-raw-name-");+ const config = await createTestTempDirectory(+ "rea-reference-raw-name-config-",+ );+ const rawName = Buffer.from([0x62, 0x61, 0x64, 0xff, 0x2e, 0x6a, 0x73]);+ const rawPath = Buffer.concat([Buffer.from(`${root}/`), rawName]);+ await Promise.all([+ writeFile(rawPath, "RAW_FILE"),+ writeFile(join(root, "bad\ufffd.js"), "UNICODE_FILE"),+ writeFile(join(root, "good.js"), "GOOD_FILE"),+ ]);++ const result = await cli.run({+ arguments: ["import-reference-source", root, "--json"],+ environment: {+ HOME: config,+ XDG_CONFIG_HOME: config,+ XDG_CACHE_HOME: config,+ },+ });+ expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe(+ 0,+ );+ const graph = historicalSourceGraphSchema.parse(result.json);+ expect(graph.inventory_state).toBe("partial");+ expect(graph.entries.map(({ path }) => path)).toEqual([+ "bad\ufffd.js",+ "good.js",+ ]);+ expect(graph.entries).toContainEqual(+ expect.objectContaining({+ path: "bad\ufffd.js",+ sha256: createHash("sha256").update("UNICODE_FILE").digest("hex"),+ size: 12,+ }),+ );+ expect(graph.limitations).toEqual(+ expect.arrayContaining([+ expect.stringContaining(+ `First raw basename bytes: ${rawName.toString("hex")}`,+ ),+ ]),+ );+ },+);+cliTest.skipIf(process.platform === "win32")("preserves historical literal-backslash entries and their exact owning import base",async ({ cli }) => {diff --git a/tests/boundary/filesystem/referenceSourceFilenameBytes.test.ts b/tests/boundary/filesystem/referenceSourceFilenameBytes.test.tsnew file mode 100644index 000000000..48c2f3f91--- /dev/null+++ b/tests/boundary/filesystem/referenceSourceFilenameBytes.test.ts@@ -0,0 +1,63 @@+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++import { expect, it } from "vitest";++import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js";+import { readReferenceSource } from "../../../src/reference/ReferenceSourceReader.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++it.skipIf(process.platform !== "linux")(+ "does not read an invalid UTF-8 directory name through its replacement-character sibling",+ async () => {+ const root = await createTestTempDirectory("rea-reference-filename-bytes-");+ const rawName = Buffer.from([0x62, 0x61, 0x64, 0xff, 0x2e, 0x6a, 0x73]);+ const rawPath = Buffer.concat([Buffer.from(`${root}/`), rawName]);+ const replacementPath = join(root, "bad\ufffd.js");+ await Promise.all([+ writeFile(rawPath, "RAW_FILE"),+ writeFile(replacementPath, "UNICODE_FILE"),+ writeFile(join(root, "good.js"), "GOOD_FILE"),+ ]);++ const resources = new ArtifactResourceScope();+ try {+ const result = await readReferenceSource(root, resources);+ if (!result.ok) throw result.error;+ expect(result.value.rootFailure).toMatchObject({+ code: "unsupported",+ message: expect.stringContaining(+ `First raw basename bytes: ${rawName.toString("hex")}`,+ ),+ });+ expect(result.value.entries).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ status: "read",+ kind: "file",+ path: "bad\ufffd.js",+ size: 12,+ bytes: Buffer.from("UNICODE_FILE"),+ }),+ expect.objectContaining({+ status: "read",+ kind: "file",+ path: "good.js",+ size: 9,+ bytes: Buffer.from("GOOD_FILE"),+ }),+ ]),+ );+ expect(+ result.value.entries.filter(+ (entry) => entry.kind === "file" && entry.path === "bad\ufffd.js",+ ),+ ).toHaveLength(1);+ expect(result.value.bytesRead).toBe(+ "UNICODE_FILE".length + "GOOD_FILE".length,+ );+ } finally {+ await resources.close();+ }+ },+);