Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(reference): retain recovered parser diagnostics

Kaoru0822ed3a7f070cf0
diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts
index 2b40efcb4..0e21be174 100644
--- a/src/domain/referenceSourceImportParsing.ts
+++ b/src/domain/referenceSourceImportParsing.ts
@@ -291,8 +291,7 @@ const parseWithBabel = (
language: string | null,
): {
ast: File | undefined;
- parseState: ReferenceSourceImportParseState;
- reason?: string;
+ reasons: readonly string[];
} => {
const plugins: ParserPlugin[] = ["jsx"];
if (language === "TypeScript" || language === "TSX") {
@@ -316,12 +315,11 @@ const parseWithBabel = (
errorRecovery: true,
plugins,
});
- return { ast, parseState: "parsed" };
+ return { ast, reasons: ast.errors.map((error) => error.message) };
} catch (cause: unknown) {
return {
ast: undefined,
- parseState: "unknown",
- reason: cause instanceof Error ? cause.message : "Parse failed",
+ reasons: [cause instanceof Error ? cause.message : "Parse failed"],
};
}
};
@@ -336,18 +334,17 @@ export const parseReferenceSourceImports = (
return { relationships: [], parse_failures: [] };
const source = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
- const { ast, parseState, reason } = parseWithBabel(path, source, language);
+ const { ast, reasons } = parseWithBabel(path, source, language);
+ const parseFailures = reasons.map((reason) => ({
+ path,
+ parser: "babel",
+ reason,
+ }));
- if (ast === undefined || parseState === "unknown") {
+ if (ast === undefined) {
return {
relationships: [],
- parse_failures: [
- {
- path,
- parser: "babel",
- reason: reason ?? "Unknown parse failure",
- },
- ],
+ parse_failures: parseFailures,
};
}
@@ -356,7 +353,13 @@ export const parseReferenceSourceImports = (
extractRequireAndDynamicImports(ast.program.body, path, relationships);
return {
- relationships,
- parse_failures: [],
+ relationships: relationships.map((relationship) => ({
+ ...relationship,
+ parse_state:
+ parseFailures.length > 0 && relationship.parse_state === "parsed"
+ ? "partial"
+ : relationship.parse_state,
+ })),
+ parse_failures: parseFailures,
};
};
diff --git a/src/domain/referenceSourceImportRecovery.test.ts b/src/domain/referenceSourceImportRecovery.test.ts
new file mode 100644
index 000000000..e32973f59
--- /dev/null
+++ b/src/domain/referenceSourceImportRecovery.test.ts
@@ -0,0 +1,150 @@
+import { describe, expect, it } from "vitest";
+
+import { parseReferenceSourceImports } from "./referenceSourceImportParsing.js";
+
+const recoverySource = [
+ "const missing;",
+ "let duplicate;",
+ "let duplicate;",
+ "let duplicate;",
+].join("\n");
+
+const recoveryReasons = [
+ "Missing initializer in const declaration. (1:13)",
+ "Identifier 'duplicate' has already been declared. (3:4)",
+ "Identifier 'duplicate' has already been declared. (4:4)",
+];
+
+const moduleSource = [
+ 'import "./static.js";',
+ 'export { value } from "./named.js";',
+ 'export * from "./all.js";',
+ 'require("./common.cjs");',
+ 'require.resolve("node:fs");',
+ 'import("./lazy.js");',
+ "import(moduleName);",
+].join("\n");
+
+const moduleRelationships = [
+ ["./static.js", "imports", "internal", "parsed"],
+ ["./named.js", "imports", "internal", "parsed"],
+ ["./all.js", "imports", "internal", "parsed"],
+ ["./common.cjs", "requires", "internal", "parsed"],
+ ["node:fs", "requires", "external", "parsed"],
+ ["./lazy.js", "imports", "internal", "parsed"],
+ ["<dynamic-import>", "imports", "unknown", "partial"],
+].map(([to, kind, resolution, parse_state]) => ({
+ to,
+ kind,
+ resolution,
+ parse_state,
+}));
+
+const parseSource = (path: string, source: string, language: string | null) =>
+ parseReferenceSourceImports(path, new TextEncoder().encode(source), language);
+
+describe.each([
+ ["main.js", "JavaScript"],
+ ["main.jsx", "JSX"],
+ ["main.ts", "TypeScript"],
+ ["main.tsx", "TSX"],
+])("recovered reference imports in %s", (path, language) => {
+ it("retains every diagnostic and marks recovered relationships partial", () => {
+ const result = parseSource(
+ path,
+ `${recoverySource}\n${moduleSource}`,
+ language,
+ );
+
+ expect(result.parse_failures).toEqual(
+ recoveryReasons.map((reason) => ({ path, parser: "babel", reason })),
+ );
+ expect(result.relationships).toEqual(
+ moduleRelationships.map((relationship) => ({
+ ...relationship,
+ from_path: path,
+ parse_state: "partial",
+ })),
+ );
+ });
+
+ it("preserves valid and computed relationships without diagnostics", () => {
+ expect(parseSource(path, moduleSource, language)).toEqual({
+ relationships: moduleRelationships.map((relationship) => ({
+ ...relationship,
+ from_path: path,
+ })),
+ parse_failures: [],
+ });
+ });
+
+ it("reports recovery diagnostics when there are no relationships", () => {
+ expect(parseSource(path, recoverySource, language)).toEqual({
+ relationships: [],
+ parse_failures: recoveryReasons.map((reason) => ({
+ path,
+ parser: "babel",
+ reason,
+ })),
+ });
+ });
+
+ it("returns no relationships when parsing throws", () => {
+ expect(
+ parseSource(path, 'import "./before.js";\nconst = ;', language),
+ ).toEqual({
+ relationships: [],
+ parse_failures: [
+ { path, parser: "babel", reason: "Unexpected token (2:6)" },
+ ],
+ });
+ });
+});
+
+it.each(["main.ts", "types.d.ts", "types.d.mts", "types.d.cts"])(
+ "marks recovered TypeScript declarations partial in %s",
+ (path) => {
+ const result = parseSource(
+ path,
+ [
+ "let duplicate: number;",
+ "let duplicate: number;",
+ 'import dep = require("node:path");',
+ 'declare module "ambient-package" {}',
+ ].join("\n"),
+ "TypeScript",
+ );
+ expect(result).toEqual({
+ relationships: [
+ {
+ from_path: path,
+ to: "node:path",
+ kind: "requires",
+ resolution: "external",
+ parse_state: "partial",
+ },
+ {
+ from_path: path,
+ to: "ambient-package",
+ kind: "declares-module",
+ resolution: "unknown",
+ parse_state: "partial",
+ },
+ ],
+ parse_failures: [
+ {
+ path,
+ parser: "babel",
+ reason: "Identifier 'duplicate' has already been declared. (2:4)",
+ },
+ ],
+ });
+ },
+);
+
+it("keeps unsupported source languages outside Babel parsing", () => {
+ expect(parseSource("main.py", recoverySource, "Python")).toEqual({
+ relationships: [],
+ parse_failures: [],
+ });
+});
diff --git a/tests/boundary/filesystem/referenceSourceRecovery.test.ts b/tests/boundary/filesystem/referenceSourceRecovery.test.ts
new file mode 100644
index 000000000..6cb250f5f
--- /dev/null
+++ b/tests/boundary/filesystem/referenceSourceRecovery.test.ts
@@ -0,0 +1,143 @@
+import { createHash } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it } from "vitest";
+
+import {
+ importReferenceSource,
+ normalizeHistoricalSourceParseFailures,
+} from "../../../src/application/ReferenceSourceImport.js";
+import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+it("preserves recovered diagnostics and file evidence through filesystem import", async () => {
+ const root = await createTestTempDirectory("rea-reference-recovery-");
+ const sources = {
+ "fatal.js": 'import "./valid";\nconst = ;',
+ "recovered.js": [
+ "const missing;",
+ "let duplicate;",
+ "let duplicate;",
+ "let duplicate;",
+ 'import "./valid";',
+ 'require("node:fs");',
+ "import(moduleName);",
+ ].join("\n"),
+ "recovered.ts": [
+ "let duplicate: number;",
+ "let duplicate: number;",
+ 'import dep = require("./valid");',
+ 'declare module "ambient-package" {}',
+ ].join("\n"),
+ "valid.js": 'export * from "node:path";\n',
+ };
+ await mkdir(join(root, "src"));
+ await Promise.all(
+ Object.entries(sources).map(([path, source]) =>
+ writeFile(join(root, "src", path), source),
+ ),
+ );
+
+ const result = await importReferenceSource({
+ root,
+ caller: "reference-recovery-test",
+ policy: { secretPatterns: [] },
+ });
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ const graph = result.value;
+ expect(historicalSourceGraphSchema.safeParse(graph).success).toBe(true);
+ expect(graph.entries.map(({ path }) => path)).toEqual([
+ "src",
+ ...Object.keys(sources).map((path) => `src/${path}`),
+ ]);
+ for (const [path, source] of Object.entries(sources)) {
+ expect(graph.entries).toContainEqual(
+ expect.objectContaining({
+ path: `src/${path}`,
+ kind: "file",
+ size: Buffer.byteLength(source),
+ sha256: createHash("sha256").update(source).digest("hex"),
+ content_state: "hashed",
+ limitations: [],
+ }),
+ );
+ }
+
+ const failures = [
+ { path: "src/fatal.js", parser: "babel", reason: "Unexpected token (2:6)" },
+ {
+ path: "src/recovered.js",
+ parser: "babel",
+ reason: "Identifier 'duplicate' has already been declared. (3:4)",
+ },
+ {
+ path: "src/recovered.js",
+ parser: "babel",
+ reason: "Identifier 'duplicate' has already been declared. (4:4)",
+ },
+ {
+ path: "src/recovered.js",
+ parser: "babel",
+ reason: "Missing initializer in const declaration. (1:13)",
+ },
+ {
+ path: "src/recovered.ts",
+ parser: "babel",
+ reason: "Identifier 'duplicate' has already been declared. (2:4)",
+ },
+ ];
+ expect(graph.parse_failures).toEqual(failures);
+ expect(
+ normalizeHistoricalSourceParseFailures(
+ [...graph.parse_failures].reverse().concat(graph.parse_failures),
+ ),
+ ).toEqual(failures);
+ expect(graph.relationships).toEqual([
+ {
+ from_path: "src/recovered.js",
+ to: "<dynamic-import>",
+ kind: "imports",
+ resolution: "unknown",
+ parse_state: "partial",
+ },
+ {
+ from_path: "src/recovered.js",
+ to: "node:fs",
+ kind: "requires",
+ resolution: "external",
+ parse_state: "partial",
+ },
+ {
+ from_path: "src/recovered.js",
+ to: "src/valid.js",
+ kind: "imports",
+ resolution: "internal",
+ parse_state: "partial",
+ },
+ {
+ from_path: "src/recovered.ts",
+ to: "ambient-package",
+ kind: "declares-module",
+ resolution: "unknown",
+ parse_state: "partial",
+ },
+ {
+ from_path: "src/recovered.ts",
+ to: "src/valid.js",
+ kind: "requires",
+ resolution: "internal",
+ parse_state: "partial",
+ },
+ {
+ from_path: "src/valid.js",
+ to: "node:path",
+ kind: "imports",
+ resolution: "external",
+ parse_state: "parsed",
+ },
+ ]);
+ // The reader's pathname-race advisory already makes inventory partial.
+ expect(graph.inventory_state).toBe("partial");
+});