Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(reference): report malformed UTF-8 before import parsing

Rudy Celeklif166a2ae15b0
diff --git a/docs/cli.md b/docs/cli.md
index 87792e7c0..51a45c462 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -147,6 +147,10 @@ from observations of the current app. File names do not automatically exclude
files. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore
patterns when you want to exclude selected paths.
+JavaScript and TypeScript import parsing requires valid UTF-8. Malformed source
+bytes retain their original hashes and sizes with a decoding diagnostic; REA
+does not infer module targets from replacement characters.
+
Historical-source import requires safe no-follow file opens on Linux or
macOS. Native Windows returns `unsupported_host`; use Linux REA inside WSL
or another supported host. See
diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts
index 70097297a..7f9ad996a 100644
--- a/src/domain/referenceSourceImportParsing.ts
+++ b/src/domain/referenceSourceImportParsing.ts
@@ -331,7 +331,25 @@ export const parseReferenceSourceImports = (
if (language !== null && !codeSourceLanguages.has(language))
return { relationships: [], parse_failures: [] };
- const source = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+ let source: string;
+ try {
+ source = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+ } catch (cause: unknown) {
+ // The import still records the original bytes; lossy text cannot supply
+ // observed module names or relationships.
+ void cause;
+ return {
+ relationships: [],
+ parse_failures: [
+ {
+ path,
+ parser: "utf-8",
+ reason:
+ "Source bytes are not valid UTF-8; import targets were not parsed.",
+ },
+ ],
+ };
+ }
const { ast, reasons } = parseWithBabel(path, source, language);
const parseFailures = reasons.map((reason) => ({
path,
diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.ts
new file mode 100644
index 000000000..06093f427
--- /dev/null
+++ b/tests/boundary/cli/referenceSourceUtf8.test.ts
@@ -0,0 +1,77 @@
+import { createHash } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect } from "vitest";
+
+import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+cliTest(
+ "retains malformed source bytes without inventing decoded import targets",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-reference-utf8-");
+ const sources = new Map([
+ [
+ "invalid.mjs",
+ Buffer.concat([
+ Buffer.from('import "./caf'),
+ Buffer.from([0xff]),
+ Buffer.from('.js";\n'),
+ ]),
+ ],
+ [
+ "comment.ts",
+ Buffer.concat([
+ Buffer.from('import "./dep.js"; // '),
+ Buffer.from([0xc0, 0xaf]),
+ ]),
+ ],
+ ["valid.mjs", Buffer.from('import "./caf�.js";\n')],
+ ["caf�.js", Buffer.from("export const value = 1;\n")],
+ ["dep.js", Buffer.from("export const value = 2;\n")],
+ ]);
+ await Promise.all(
+ [...sources].map(([path, bytes]) => writeFile(join(root, path), bytes)),
+ );
+ const result = await cli.run({
+ arguments: ["import-reference-source", root, "--json"],
+ environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root },
+ });
+ expect(result.exitCode).toBe(0);
+ const graph = historicalSourceGraphSchema.parse(result.json);
+ expect(graph.parse_failures).toEqual([
+ {
+ path: "comment.ts",
+ parser: "utf-8",
+ reason:
+ "Source bytes are not valid UTF-8; import targets were not parsed.",
+ },
+ {
+ path: "invalid.mjs",
+ parser: "utf-8",
+ reason:
+ "Source bytes are not valid UTF-8; import targets were not parsed.",
+ },
+ ]);
+ expect(graph.relationships).toEqual([
+ {
+ from_path: "valid.mjs",
+ to: "caf�.js",
+ kind: "imports",
+ resolution: "internal",
+ parse_state: "parsed",
+ },
+ ]);
+ for (const [path, bytes] of sources)
+ expect(graph.entries).toContainEqual(
+ expect.objectContaining({
+ path,
+ sha256: createHash("sha256").update(bytes).digest("hex"),
+ size: bytes.length,
+ content_state: "hashed",
+ }),
+ );
+ },
+);