Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(reference): report sloppy scripts, PHP, C# and enclosing Git facts (#1770)

Martin Klein7b15762d09ac
diff --git a/docs/cli.md b/docs/cli.md
index 2d7567d06..dcd949e11 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -215,7 +215,9 @@ rea import-reference-source /absolute/path/to/source
```
The import records hashes and metadata for the supplied files, separately
-from observations of the current app. It honors the source tree's `.gitignore`
+from observations of the current app. For a directory inside a Git checkout,
+it records the HEAD of the nearest enclosing repository; whether the working
+tree matches that commit remains unknown. It honors the source tree's `.gitignore`
and applies default exclusions for common generated and dependency paths such
as `node_modules/`, `dist/`, and `*.log`. Pattern-based exclusions record the
matched pattern and whether it came from project, default, caller, or sensitive
diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts
index c7432c68d..3254d7cfb 100644
--- a/src/application/ReferenceSourceImportEntries.ts
+++ b/src/application/ReferenceSourceImportEntries.ts
@@ -109,12 +109,18 @@ const resolveInternalSpecifier = (
: { to: match, resolution: "internal" };
};
+type ReadFile = Extract<ReferenceSourceEntry, { status: "read"; kind: "file" }>;
+
const appendFile = (
- entry: Extract<ReferenceSourceEntry, { status: "read"; kind: "file" }>,
+ entry: ReadFile,
lookupPath: ReferenceSourcePathLookup,
output: ParsedReferenceSourceEntries,
): void => {
const language = detectReferenceSourceLanguage(entry.path);
+ const parsed =
+ language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language)
+ ? undefined
+ : parseReferenceSourceImports(entry.path, entry.bytes, language);
output.entries.push({
path: entry.path,
kind: "file",
@@ -123,11 +129,9 @@ const appendFile = (
language,
classifications: classifyReferenceSourcePath(entry.path),
content_state: "hashed",
- limitations: [],
+ limitations: [...(parsed?.limitations ?? [])],
});
- if (language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language))
- return;
- const parsed = parseReferenceSourceImports(entry.path, entry.bytes, language);
+ if (parsed === undefined) return;
for (const relationship of parsed.relationships) {
if (relationship.resolution !== "internal") {
output.relationships.push(relationship);
diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts
index b2a21e2b5..346ae6ebe 100644
--- a/src/application/ReferenceSourceVcsAdapter.ts
+++ b/src/application/ReferenceSourceVcsAdapter.ts
@@ -1,6 +1,6 @@
import fs from "node:fs";
import { lstat } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { dirname, join, resolve } from "node:path";
import { err, ok, type Result } from "../domain/result.js";
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
@@ -32,18 +32,24 @@ export const readReferenceSourceVcs = async (
): Promise<Result<ReferenceSourceVcsInfo, ReferenceSourceReaderError>> => {
try {
return await resources.run(async () => {
- if (isAborted(signal)) return ok(unknownVcs());
- try {
- await lstat(join(root, ".git"));
- } catch (cause: unknown) {
- return ok(
- errorCode(cause) === "ENOENT"
- ? { kind: "none", head: null, dirty: null }
- : unknownVcs(),
- );
+ // Like Git, a subdirectory belongs to its nearest enclosing repository;
+ // only a missing `.git` at every ancestor means no version control.
+ let repository = resolve(root);
+ for (;;) {
+ if (isAborted(signal)) return ok(unknownVcs());
+ try {
+ await lstat(join(repository, ".git"));
+ break;
+ } catch (cause: unknown) {
+ if (errorCode(cause) !== "ENOENT") return ok(unknownVcs());
+ }
+ const parent = dirname(repository);
+ if (parent === repository)
+ return ok({ kind: "none", head: null, dirty: null });
+ repository = parent;
}
try {
- const head = await resolveSourceHead(root, resources, signal);
+ const head = await resolveSourceHead(repository, resources, signal);
if (isAborted(signal)) return ok(unknownVcs());
return ok({ kind: "git", head, dirty: null });
} catch (cause: unknown) {
diff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.ts
index 9c380eaff..33099ee28 100644
--- a/src/cli/utilityCommands.ts
+++ b/src/cli/utilityCommands.ts
@@ -255,7 +255,7 @@ const runReferenceSourceImport = async (
caller: "rea-cli",
policy,
importer: PRODUCT_IDENTITY.packageName,
- importerVersion: null,
+ importerVersion: PRODUCT_IDENTITY.packageVersion,
},
resources,
),
diff --git a/src/domain/javascript/javascriptSourceParser.ts b/src/domain/javascript/javascriptSourceParser.ts
index 6a8016887..ce5619ead 100644
--- a/src/domain/javascript/javascriptSourceParser.ts
+++ b/src/domain/javascript/javascriptSourceParser.ts
@@ -1,4 +1,4 @@
-import { parse, type ParserPlugin } from "@babel/parser";
+import { parse, type ParserOptions, type ParserPlugin } from "@babel/parser";
/** Babel AST produced by REA's inert JavaScript parser boundary. */
export type ParsedJavaScriptSource = ReturnType<typeof parse>;
@@ -37,6 +37,67 @@ export const parserPluginsForPath = (sourcePath?: string): ParserPlugin[] => {
: ["decorators-legacy", typescript];
};
+/** `.mjs` and `.mts` are modules by extension; other paths leave the mode to syntax. */
+export const isExplicitModuleSourcePath = (path?: string): boolean =>
+ /\.m[jt]s$/iu.test(path ?? "");
+
+export interface RecoveredJavaScriptParse {
+ readonly file: ParsedJavaScriptSource;
+ /** Errors that apply only if a source of unknown mode is an ES module. */
+ readonly moduleOnlyErrors: readonly string[];
+}
+
+/**
+ * Parse a module or script with recovery. Babel tries module mode first and
+ * keeps its strict-mode errors after falling back to a script, so a script
+ * with errors is parsed again as a script. Strict errors that only the module
+ * parse reports stay visible because the source may still be a module.
+ */
+export const parseRecoveredJavaScript = (
+ source: string,
+ sourcePath: string | undefined,
+ options: Omit<ParserOptions, "sourceType" | "errorRecovery">,
+): RecoveredJavaScriptParse => {
+ // `.cjs` is CommonJS by extension, so module-only errors cannot apply.
+ if (/\.cjs$/iu.test(sourcePath ?? ""))
+ return {
+ file: parse(source, {
+ ...options,
+ sourceType: "script",
+ errorRecovery: true,
+ }),
+ moduleOnlyErrors: [],
+ };
+ if (isExplicitModuleSourcePath(sourcePath))
+ return {
+ file: parse(source, {
+ ...options,
+ sourceType: "module",
+ errorRecovery: true,
+ }),
+ moduleOnlyErrors: [],
+ };
+ const file = parse(source, {
+ ...options,
+ sourceType: "unambiguous",
+ errorRecovery: true,
+ });
+ if (file.program.sourceType !== "script" || file.errors.length === 0)
+ return { file, moduleOnlyErrors: [] };
+ const script = parse(source, {
+ ...options,
+ sourceType: "script",
+ errorRecovery: true,
+ });
+ const scriptErrors = new Set(script.errors.map((error) => error.message));
+ return {
+ file: script,
+ moduleOnlyErrors: file.errors
+ .map((error) => error.message)
+ .filter((message) => !scriptErrors.has(message)),
+ };
+};
+
/** Parse JavaScript or TypeScript once without attaching comments to AST nodes. */
export const parseJavaScriptSource = (
source: string,
diff --git a/src/domain/referenceSourceClassification.ts b/src/domain/referenceSourceClassification.ts
index c567cf06a..ea53a18a4 100644
--- a/src/domain/referenceSourceClassification.ts
+++ b/src/domain/referenceSourceClassification.ts
@@ -37,6 +37,8 @@ const CODE_EXTENSIONS = new Set([
"m",
"mm",
"sh",
+ "cs",
+ "php",
]);
const LANGUAGE_BY_EXTENSION = new Map<string, string>([
@@ -59,6 +61,8 @@ const LANGUAGE_BY_EXTENSION = new Map<string, string>([
["cc", "C++"],
["cxx", "C++"],
["swift", "Swift"],
+ ["cs", "C#"],
+ ["php", "PHP"],
["m", "Objective-C"],
["mm", "Objective-C++"],
["sh", "Shell"],
diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts
index 62ad226dd..998e368c4 100644
--- a/src/domain/referenceSourceImportParsing.ts
+++ b/src/domain/referenceSourceImportParsing.ts
@@ -1,4 +1,3 @@
-import { parse } from "@babel/parser";
import type {
CallExpression,
File,
@@ -7,7 +6,10 @@ import type {
StringLiteral,
} from "@babel/types";
import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";
-import { parserPluginsForPath } from "./javascript/javascriptSourceParser.js";
+import {
+ parseRecoveredJavaScript,
+ parserPluginsForPath,
+} from "./javascript/javascriptSourceParser.js";
import { unshadowedReferenceSourceRequireCalls } from "./referenceSourceRequireBindings.js";
import {
isCallExpression,
@@ -55,6 +57,7 @@ interface ReferenceSourceParseFailure {
export interface ReferenceSourceImportParseResult {
readonly relationships: readonly ReferenceSourceImportRelationship[];
readonly parse_failures: readonly ReferenceSourceParseFailure[];
+ readonly limitations: readonly string[];
}
const codeSourceLanguages = new Set(["JavaScript", "TypeScript", "JSX", "TSX"]);
@@ -272,6 +275,7 @@ const parseWithBabel = (
): {
ast: File | undefined;
reasons: readonly string[];
+ moduleOnlyErrors: readonly string[];
} => {
// Single parser-mode owner: path-driven plugins (dts/JSX/mts) shared with
// the semantic pipeline. `language` only gates non-JS/TS callers upstream;
@@ -280,19 +284,26 @@ const parseWithBabel = (
const plugins = parserPluginsForPath(path);
try {
- const ast = parse(source, {
- sourceType: "unambiguous",
- allowImportExportEverywhere: false,
- allowReturnOutsideFunction: true,
- allowUndeclaredExports: true,
- errorRecovery: true,
- plugins,
- });
- return { ast, reasons: ast.errors.map((error) => error.message) };
+ const { file: ast, moduleOnlyErrors } = parseRecoveredJavaScript(
+ source,
+ path,
+ {
+ allowImportExportEverywhere: false,
+ allowReturnOutsideFunction: true,
+ allowUndeclaredExports: true,
+ plugins,
+ },
+ );
+ return {
+ ast,
+ reasons: ast.errors.map((error) => error.message),
+ moduleOnlyErrors,
+ };
} catch (cause: unknown) {
return {
ast: undefined,
reasons: [cause instanceof Error ? cause.message : "Parse failed"],
+ moduleOnlyErrors: [],
};
}
};
@@ -304,7 +315,7 @@ export const parseReferenceSourceImports = (
language: string | null,
): ReferenceSourceImportParseResult => {
if (language !== null && !codeSourceLanguages.has(language))
- return { relationships: [], parse_failures: [] };
+ return { relationships: [], parse_failures: [], limitations: [] };
let source: string;
try {
@@ -320,9 +331,20 @@ export const parseReferenceSourceImports = (
"Source bytes are not valid UTF-8; import targets were not parsed.",
},
],
+ limitations: [],
};
}
- const { ast, reasons } = parseWithBabel(path, source, language);
+ const { ast, reasons, moduleOnlyErrors } = parseWithBabel(
+ path,
+ source,
+ language,
+ );
+ const limitations =
+ moduleOnlyErrors.length === 0
+ ? []
+ : [
+ `Strict-mode errors apply only if this file is an ES module: ${moduleOnlyErrors.join("; ")}`,
+ ];
const parseFailures = reasons.map((reason) => ({
path,
parser: "babel",
@@ -333,6 +355,7 @@ export const parseReferenceSourceImports = (
return {
relationships: [],
parse_failures: parseFailures,
+ limitations,
};
}
@@ -354,5 +377,6 @@ export const parseReferenceSourceImports = (
: relationship.parse_state,
})),
parse_failures: parseFailures,
+ limitations,
};
};
diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.ts
index c6df25180..f2f8d1ef5 100644
--- a/src/domain/referenceSourceRequireBindings.ts
+++ b/src/domain/referenceSourceRequireBindings.ts
@@ -7,7 +7,10 @@ import {
} from "./javascript/javascriptSemanticState.js";
import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";
import type { JavaScriptSemanticReference } from "./javascript/javascriptSemanticIr.js";
-import type { ParsedJavaScriptSource } from "./javascript/javascriptSourceParser.js";
+import {
+ isExplicitModuleSourcePath,
+ type ParsedJavaScriptSource,
+} from "./javascript/javascriptSourceParser.js";
/**
* Keep calls that resolve either to Node's CommonJS wrapper loader or to a
@@ -29,7 +32,10 @@ export const unshadowedReferenceSourceRequireCalls = (
semanticScopeId("program", file.program),
"require",
);
- const allowsUnboundRequire = !isExplicitModuleSource(path);
+ // Syntax alone cannot establish Node's runtime mode: `.js` depends on its
+ // package scope, and `.ts` may be compiled with a CommonJS module target.
+ // Keep unmarked paths runtime-unknown and preserve candidate require edges.
+ const allowsUnboundRequire = !isExplicitModuleSourcePath(path);
const hasTopLevelVarRequire =
allowsUnboundRequire && hasUninitializedTopLevelVarRequire(file.program);
const createRequireBinding = directCreateRequireBinding(
@@ -82,16 +88,6 @@ const contains = (container: t.Node, node: t.Node): boolean =>
container.start <= node.start &&
node.end <= container.end;
-const isExplicitModuleSource = (path: string): boolean => {
- const lower = path.toLowerCase();
- if (lower.endsWith(".mjs") || lower.endsWith(".mts")) return true;
- if (lower.endsWith(".cjs") || lower.endsWith(".cts")) return false;
- // Syntax alone cannot establish Node's runtime mode: `.js` depends on its
- // package scope, and `.ts` may be compiled with a CommonJS module target.
- // Keep unmarked paths runtime-unknown and preserve candidate require edges.
- return false;
-};
-
const hasUninitializedTopLevelVarRequire = (program: t.Program): boolean => {
let found = false;
let initialized = false;
diff --git a/tests/boundary/cli/referenceSourceLegacyFacts.test.ts b/tests/boundary/cli/referenceSourceLegacyFacts.test.ts
new file mode 100644
index 000000000..d254a04a0
--- /dev/null
+++ b/tests/boundary/cli/referenceSourceLegacyFacts.test.ts
@@ -0,0 +1,90 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect } from "vitest";
+
+import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js";
+import { PRODUCT_IDENTITY } from "../../../src/identity.js";
+import { execFileOutput } from "../../../src/process/ExecFileOutput.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+cliTest(
+ "imports a legacy subtree of a Git checkout with its own source facts",
+ async ({ cli }) => {
+ const repository = await createTestTempDirectory("rea-reference-legacy-");
+ const source = join(repository, "source");
+ await mkdir(source);
+ await Promise.all([
+ // `.cjs` is CommonJS, so the strict-mode rule does not apply.
+ writeFile(
+ join(source, "legacy.cjs"),
+ "var messages = {};\ndelete messages;\n",
+ ),
+ // Deleting a plain binding is valid only in a sloppy-mode script.
+ writeFile(
+ join(source, "legacy.js"),
+ "var messages = {};\ndelete messages;\n",
+ ),
+ // An explicit module keeps the strict-mode error.
+ writeFile(
+ join(source, "module.mjs"),
+ "var messages = {};\ndelete messages;\n",
+ ),
+ writeFile(join(source, "index.php"), "<?php echo 'legacy';\n"),
+ writeFile(join(source, "Program.cs"), "class Program {}\n"),
+ ]);
+ const git = (...args: string[]) =>
+ execFileOutput("git", args, { cwd: repository });
+ await git("init", "--initial-branch=main");
+ await git("add", ".");
+ await git(
+ "-c",
+ "user.name=REA fixture",
+ "-c",
+ "-c",
+ "commit.gpgsign=false",
+ "commit",
+ "-m",
+ "fixture",
+ );
+ const head = (await git("rev-parse", "HEAD")).stdout.trim();
+
+ const result = await cli.run({
+ arguments: ["import-reference-source", source, "--json"],
+ environment: {
+ HOME: repository,
+ XDG_CONFIG_HOME: repository,
+ XDG_CACHE_HOME: repository,
+ },
+ });
+
+ expect(result.exitCode).toBe(0);
+ const graph = historicalSourceGraphSchema.parse(result.json);
+ expect(graph.parse_failures).toEqual([
+ {
+ path: "module.mjs",
+ parser: "babel",
+ reason: "Deleting local variable in strict mode. (2:0)",
+ },
+ ]);
+ // Outside `.mjs`/`.mts`/`.cjs` the mode stays unknown, not forced to script.
+ const limitations = (path: string) =>
+ graph.entries.find((entry) => entry.path === path)?.limitations;
+ for (const path of ["legacy.js"])
+ expect(limitations(path)).toEqual([
+ "Strict-mode errors apply only if this file is an ES module: Deleting local variable in strict mode. (2:0)",
+ ]);
+ expect(limitations("legacy.cjs")).toEqual([]);
+ expect(graph.languages).toEqual(["C#", "JavaScript", "PHP"]);
+ for (const path of ["Program.cs", "index.php"])
+ expect(
+ graph.entries.find((entry) => entry.path === path)?.classifications,
+ ).toContain("source");
+ expect(graph.vcs).toEqual({ kind: "git", head, dirty: null });
+ expect(graph.provenance.importer_version).toBe(
+ PRODUCT_IDENTITY.packageVersion,
+ );
+ },
+);