Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(reference): report malformed UTF-8 before import parsing
Rudy Celeklif166a2ae15b0
diff --git a/docs/cli.md b/docs/cli.mdindex 87792e7c0..51a45c462 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -147,6 +147,10 @@ from observations of the current app. File names do not automatically excludefiles. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignorepatterns when you want to exclude selected paths.+JavaScript and TypeScript import parsing requires valid UTF-8. Malformed source+bytes retain their original hashes and sizes with a decoding diagnostic; REA+does not infer module targets from replacement characters.+Historical-source import requires safe no-follow file opens on Linux ormacOS. Native Windows returns `unsupported_host`; use Linux REA inside WSLor another supported host. Seediff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.tsindex 70097297a..7f9ad996a 100644--- a/src/domain/referenceSourceImportParsing.ts+++ b/src/domain/referenceSourceImportParsing.ts@@ -331,7 +331,25 @@ export const parseReferenceSourceImports = (if (language !== null && !codeSourceLanguages.has(language))return { relationships: [], parse_failures: [] };- const source = new TextDecoder("utf-8", { fatal: false }).decode(bytes);+ let source: string;+ try {+ source = new TextDecoder("utf-8", { fatal: true }).decode(bytes);+ } catch (cause: unknown) {+ // The import still records the original bytes; lossy text cannot supply+ // observed module names or relationships.+ void cause;+ return {+ relationships: [],+ parse_failures: [+ {+ path,+ parser: "utf-8",+ reason:+ "Source bytes are not valid UTF-8; import targets were not parsed.",+ },+ ],+ };+ }const { ast, reasons } = parseWithBabel(path, source, language);const parseFailures = reasons.map((reason) => ({path,diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.tsnew file mode 100644index 000000000..06093f427--- /dev/null+++ b/tests/boundary/cli/referenceSourceUtf8.test.ts@@ -0,0 +1,77 @@+import { createHash } from "node:crypto";+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++import { expect } from "vitest";++import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++cliTest(+ "retains malformed source bytes without inventing decoded import targets",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-reference-utf8-");+ const sources = new Map([+ [+ "invalid.mjs",+ Buffer.concat([+ Buffer.from('import "./caf'),+ Buffer.from([0xff]),+ Buffer.from('.js";\n'),+ ]),+ ],+ [+ "comment.ts",+ Buffer.concat([+ Buffer.from('import "./dep.js"; // '),+ Buffer.from([0xc0, 0xaf]),+ ]),+ ],+ ["valid.mjs", Buffer.from('import "./caf�.js";\n')],+ ["caf�.js", Buffer.from("export const value = 1;\n")],+ ["dep.js", Buffer.from("export const value = 2;\n")],+ ]);+ await Promise.all(+ [...sources].map(([path, bytes]) => writeFile(join(root, path), bytes)),+ );+ const result = await cli.run({+ arguments: ["import-reference-source", root, "--json"],+ environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root },+ });+ expect(result.exitCode).toBe(0);+ const graph = historicalSourceGraphSchema.parse(result.json);+ expect(graph.parse_failures).toEqual([+ {+ path: "comment.ts",+ parser: "utf-8",+ reason:+ "Source bytes are not valid UTF-8; import targets were not parsed.",+ },+ {+ path: "invalid.mjs",+ parser: "utf-8",+ reason:+ "Source bytes are not valid UTF-8; import targets were not parsed.",+ },+ ]);+ expect(graph.relationships).toEqual([+ {+ from_path: "valid.mjs",+ to: "caf�.js",+ kind: "imports",+ resolution: "internal",+ parse_state: "parsed",+ },+ ]);+ for (const [path, bytes] of sources)+ expect(graph.entries).toContainEqual(+ expect.objectContaining({+ path,+ sha256: createHash("sha256").update(bytes).digest("hex"),+ size: bytes.length,+ content_state: "hashed",+ }),+ );+ },+);