Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(reference): report sloppy scripts, PHP, C# and enclosing Git facts (#1770)
Martin Klein7b15762d09ac
diff --git a/docs/cli.md b/docs/cli.mdindex 2d7567d06..dcd949e11 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -215,7 +215,9 @@ rea import-reference-source /absolute/path/to/source```The import records hashes and metadata for the supplied files, separately-from observations of the current app. It honors the source tree's `.gitignore`+from observations of the current app. For a directory inside a Git checkout,+it records the HEAD of the nearest enclosing repository; whether the working+tree matches that commit remains unknown. It honors the source tree's `.gitignore`and applies default exclusions for common generated and dependency paths suchas `node_modules/`, `dist/`, and `*.log`. Pattern-based exclusions record thematched pattern and whether it came from project, default, caller, or sensitivediff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.tsindex c7432c68d..3254d7cfb 100644--- a/src/application/ReferenceSourceImportEntries.ts+++ b/src/application/ReferenceSourceImportEntries.ts@@ -109,12 +109,18 @@ const resolveInternalSpecifier = (: { to: match, resolution: "internal" };};+type ReadFile = Extract<ReferenceSourceEntry, { status: "read"; kind: "file" }>;+const appendFile = (- entry: Extract<ReferenceSourceEntry, { status: "read"; kind: "file" }>,+ entry: ReadFile,lookupPath: ReferenceSourcePathLookup,output: ParsedReferenceSourceEntries,): void => {const language = detectReferenceSourceLanguage(entry.path);+ const parsed =+ language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language)+ ? undefined+ : parseReferenceSourceImports(entry.path, entry.bytes, language);output.entries.push({path: entry.path,kind: "file",@@ -123,11 +129,9 @@ const appendFile = (language,classifications: classifyReferenceSourcePath(entry.path),content_state: "hashed",- limitations: [],+ limitations: [...(parsed?.limitations ?? [])],});- if (language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language))- return;- const parsed = parseReferenceSourceImports(entry.path, entry.bytes, language);+ if (parsed === undefined) return;for (const relationship of parsed.relationships) {if (relationship.resolution !== "internal") {output.relationships.push(relationship);diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.tsindex b2a21e2b5..346ae6ebe 100644--- a/src/application/ReferenceSourceVcsAdapter.ts+++ b/src/application/ReferenceSourceVcsAdapter.ts@@ -1,6 +1,6 @@import fs from "node:fs";import { lstat } from "node:fs/promises";-import { join, resolve } from "node:path";+import { dirname, join, resolve } from "node:path";import { err, ok, type Result } from "../domain/result.js";import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";@@ -32,18 +32,24 @@ export const readReferenceSourceVcs = async (): Promise<Result<ReferenceSourceVcsInfo, ReferenceSourceReaderError>> => {try {return await resources.run(async () => {- if (isAborted(signal)) return ok(unknownVcs());- try {- await lstat(join(root, ".git"));- } catch (cause: unknown) {- return ok(- errorCode(cause) === "ENOENT"- ? { kind: "none", head: null, dirty: null }- : unknownVcs(),- );+ // Like Git, a subdirectory belongs to its nearest enclosing repository;+ // only a missing `.git` at every ancestor means no version control.+ let repository = resolve(root);+ for (;;) {+ if (isAborted(signal)) return ok(unknownVcs());+ try {+ await lstat(join(repository, ".git"));+ break;+ } catch (cause: unknown) {+ if (errorCode(cause) !== "ENOENT") return ok(unknownVcs());+ }+ const parent = dirname(repository);+ if (parent === repository)+ return ok({ kind: "none", head: null, dirty: null });+ repository = parent;}try {- const head = await resolveSourceHead(root, resources, signal);+ const head = await resolveSourceHead(repository, resources, signal);if (isAborted(signal)) return ok(unknownVcs());return ok({ kind: "git", head, dirty: null });} catch (cause: unknown) {diff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.tsindex 9c380eaff..33099ee28 100644--- a/src/cli/utilityCommands.ts+++ b/src/cli/utilityCommands.ts@@ -255,7 +255,7 @@ const runReferenceSourceImport = async (caller: "rea-cli",policy,importer: PRODUCT_IDENTITY.packageName,- importerVersion: null,+ importerVersion: PRODUCT_IDENTITY.packageVersion,},resources,),diff --git a/src/domain/javascript/javascriptSourceParser.ts b/src/domain/javascript/javascriptSourceParser.tsindex 6a8016887..ce5619ead 100644--- a/src/domain/javascript/javascriptSourceParser.ts+++ b/src/domain/javascript/javascriptSourceParser.ts@@ -1,4 +1,4 @@-import { parse, type ParserPlugin } from "@babel/parser";+import { parse, type ParserOptions, type ParserPlugin } from "@babel/parser";/** Babel AST produced by REA's inert JavaScript parser boundary. */export type ParsedJavaScriptSource = ReturnType<typeof parse>;@@ -37,6 +37,67 @@ export const parserPluginsForPath = (sourcePath?: string): ParserPlugin[] => {: ["decorators-legacy", typescript];};+/** `.mjs` and `.mts` are modules by extension; other paths leave the mode to syntax. */+export const isExplicitModuleSourcePath = (path?: string): boolean =>+ /\.m[jt]s$/iu.test(path ?? "");++export interface RecoveredJavaScriptParse {+ readonly file: ParsedJavaScriptSource;+ /** Errors that apply only if a source of unknown mode is an ES module. */+ readonly moduleOnlyErrors: readonly string[];+}++/**+ * Parse a module or script with recovery. Babel tries module mode first and+ * keeps its strict-mode errors after falling back to a script, so a script+ * with errors is parsed again as a script. Strict errors that only the module+ * parse reports stay visible because the source may still be a module.+ */+export const parseRecoveredJavaScript = (+ source: string,+ sourcePath: string | undefined,+ options: Omit<ParserOptions, "sourceType" | "errorRecovery">,+): RecoveredJavaScriptParse => {+ // `.cjs` is CommonJS by extension, so module-only errors cannot apply.+ if (/\.cjs$/iu.test(sourcePath ?? ""))+ return {+ file: parse(source, {+ ...options,+ sourceType: "script",+ errorRecovery: true,+ }),+ moduleOnlyErrors: [],+ };+ if (isExplicitModuleSourcePath(sourcePath))+ return {+ file: parse(source, {+ ...options,+ sourceType: "module",+ errorRecovery: true,+ }),+ moduleOnlyErrors: [],+ };+ const file = parse(source, {+ ...options,+ sourceType: "unambiguous",+ errorRecovery: true,+ });+ if (file.program.sourceType !== "script" || file.errors.length === 0)+ return { file, moduleOnlyErrors: [] };+ const script = parse(source, {+ ...options,+ sourceType: "script",+ errorRecovery: true,+ });+ const scriptErrors = new Set(script.errors.map((error) => error.message));+ return {+ file: script,+ moduleOnlyErrors: file.errors+ .map((error) => error.message)+ .filter((message) => !scriptErrors.has(message)),+ };+};+/** Parse JavaScript or TypeScript once without attaching comments to AST nodes. */export const parseJavaScriptSource = (source: string,diff --git a/src/domain/referenceSourceClassification.ts b/src/domain/referenceSourceClassification.tsindex c567cf06a..ea53a18a4 100644--- a/src/domain/referenceSourceClassification.ts+++ b/src/domain/referenceSourceClassification.ts@@ -37,6 +37,8 @@ const CODE_EXTENSIONS = new Set(["m","mm","sh",+ "cs",+ "php",]);const LANGUAGE_BY_EXTENSION = new Map<string, string>([@@ -59,6 +61,8 @@ const LANGUAGE_BY_EXTENSION = new Map<string, string>([["cc", "C++"],["cxx", "C++"],["swift", "Swift"],+ ["cs", "C#"],+ ["php", "PHP"],["m", "Objective-C"],["mm", "Objective-C++"],["sh", "Shell"],diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.tsindex 62ad226dd..998e368c4 100644--- a/src/domain/referenceSourceImportParsing.ts+++ b/src/domain/referenceSourceImportParsing.ts@@ -1,4 +1,3 @@-import { parse } from "@babel/parser";import type {CallExpression,File,@@ -7,7 +6,10 @@ import type {StringLiteral,} from "@babel/types";import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";-import { parserPluginsForPath } from "./javascript/javascriptSourceParser.js";+import {+ parseRecoveredJavaScript,+ parserPluginsForPath,+} from "./javascript/javascriptSourceParser.js";import { unshadowedReferenceSourceRequireCalls } from "./referenceSourceRequireBindings.js";import {isCallExpression,@@ -55,6 +57,7 @@ interface ReferenceSourceParseFailure {export interface ReferenceSourceImportParseResult {readonly relationships: readonly ReferenceSourceImportRelationship[];readonly parse_failures: readonly ReferenceSourceParseFailure[];+ readonly limitations: readonly string[];}const codeSourceLanguages = new Set(["JavaScript", "TypeScript", "JSX", "TSX"]);@@ -272,6 +275,7 @@ const parseWithBabel = (): {ast: File | undefined;reasons: readonly string[];+ moduleOnlyErrors: readonly string[];} => {// Single parser-mode owner: path-driven plugins (dts/JSX/mts) shared with// the semantic pipeline. `language` only gates non-JS/TS callers upstream;@@ -280,19 +284,26 @@ const parseWithBabel = (const plugins = parserPluginsForPath(path);try {- const ast = parse(source, {- sourceType: "unambiguous",- allowImportExportEverywhere: false,- allowReturnOutsideFunction: true,- allowUndeclaredExports: true,- errorRecovery: true,- plugins,- });- return { ast, reasons: ast.errors.map((error) => error.message) };+ const { file: ast, moduleOnlyErrors } = parseRecoveredJavaScript(+ source,+ path,+ {+ allowImportExportEverywhere: false,+ allowReturnOutsideFunction: true,+ allowUndeclaredExports: true,+ plugins,+ },+ );+ return {+ ast,+ reasons: ast.errors.map((error) => error.message),+ moduleOnlyErrors,+ };} catch (cause: unknown) {return {ast: undefined,reasons: [cause instanceof Error ? cause.message : "Parse failed"],+ moduleOnlyErrors: [],};}};@@ -304,7 +315,7 @@ export const parseReferenceSourceImports = (language: string | null,): ReferenceSourceImportParseResult => {if (language !== null && !codeSourceLanguages.has(language))- return { relationships: [], parse_failures: [] };+ return { relationships: [], parse_failures: [], limitations: [] };let source: string;try {@@ -320,9 +331,20 @@ export const parseReferenceSourceImports = ("Source bytes are not valid UTF-8; import targets were not parsed.",},],+ limitations: [],};}- const { ast, reasons } = parseWithBabel(path, source, language);+ const { ast, reasons, moduleOnlyErrors } = parseWithBabel(+ path,+ source,+ language,+ );+ const limitations =+ moduleOnlyErrors.length === 0+ ? []+ : [+ `Strict-mode errors apply only if this file is an ES module: ${moduleOnlyErrors.join("; ")}`,+ ];const parseFailures = reasons.map((reason) => ({path,parser: "babel",@@ -333,6 +355,7 @@ export const parseReferenceSourceImports = (return {relationships: [],parse_failures: parseFailures,+ limitations,};}@@ -354,5 +377,6 @@ export const parseReferenceSourceImports = (: relationship.parse_state,})),parse_failures: parseFailures,+ limitations,};};diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.tsindex c6df25180..f2f8d1ef5 100644--- a/src/domain/referenceSourceRequireBindings.ts+++ b/src/domain/referenceSourceRequireBindings.ts@@ -7,7 +7,10 @@ import {} from "./javascript/javascriptSemanticState.js";import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";import type { JavaScriptSemanticReference } from "./javascript/javascriptSemanticIr.js";-import type { ParsedJavaScriptSource } from "./javascript/javascriptSourceParser.js";+import {+ isExplicitModuleSourcePath,+ type ParsedJavaScriptSource,+} from "./javascript/javascriptSourceParser.js";/*** Keep calls that resolve either to Node's CommonJS wrapper loader or to a@@ -29,7 +32,10 @@ export const unshadowedReferenceSourceRequireCalls = (semanticScopeId("program", file.program),"require",);- const allowsUnboundRequire = !isExplicitModuleSource(path);+ // Syntax alone cannot establish Node's runtime mode: `.js` depends on its+ // package scope, and `.ts` may be compiled with a CommonJS module target.+ // Keep unmarked paths runtime-unknown and preserve candidate require edges.+ const allowsUnboundRequire = !isExplicitModuleSourcePath(path);const hasTopLevelVarRequire =allowsUnboundRequire && hasUninitializedTopLevelVarRequire(file.program);const createRequireBinding = directCreateRequireBinding(@@ -82,16 +88,6 @@ const contains = (container: t.Node, node: t.Node): boolean =>container.start <= node.start &&node.end <= container.end;-const isExplicitModuleSource = (path: string): boolean => {- const lower = path.toLowerCase();- if (lower.endsWith(".mjs") || lower.endsWith(".mts")) return true;- if (lower.endsWith(".cjs") || lower.endsWith(".cts")) return false;- // Syntax alone cannot establish Node's runtime mode: `.js` depends on its- // package scope, and `.ts` may be compiled with a CommonJS module target.- // Keep unmarked paths runtime-unknown and preserve candidate require edges.- return false;-};-const hasUninitializedTopLevelVarRequire = (program: t.Program): boolean => {let found = false;let initialized = false;diff --git a/tests/boundary/cli/referenceSourceLegacyFacts.test.ts b/tests/boundary/cli/referenceSourceLegacyFacts.test.tsnew file mode 100644index 000000000..d254a04a0--- /dev/null+++ b/tests/boundary/cli/referenceSourceLegacyFacts.test.ts@@ -0,0 +1,90 @@+import { mkdir, writeFile } from "node:fs/promises";+import { join } from "node:path";++import { expect } from "vitest";++import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js";+import { PRODUCT_IDENTITY } from "../../../src/identity.js";+import { execFileOutput } from "../../../src/process/ExecFileOutput.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++cliTest(+ "imports a legacy subtree of a Git checkout with its own source facts",+ async ({ cli }) => {+ const repository = await createTestTempDirectory("rea-reference-legacy-");+ const source = join(repository, "source");+ await mkdir(source);+ await Promise.all([+ // `.cjs` is CommonJS, so the strict-mode rule does not apply.+ writeFile(+ join(source, "legacy.cjs"),+ "var messages = {};\ndelete messages;\n",+ ),+ // Deleting a plain binding is valid only in a sloppy-mode script.+ writeFile(+ join(source, "legacy.js"),+ "var messages = {};\ndelete messages;\n",+ ),+ // An explicit module keeps the strict-mode error.+ writeFile(+ join(source, "module.mjs"),+ "var messages = {};\ndelete messages;\n",+ ),+ writeFile(join(source, "index.php"), "<?php echo 'legacy';\n"),+ writeFile(join(source, "Program.cs"), "class Program {}\n"),+ ]);+ const git = (...args: string[]) =>+ execFileOutput("git", args, { cwd: repository });+ await git("init", "--initial-branch=main");+ await git("add", ".");+ await git(+ "-c",+ "user.name=REA fixture",+ "-c",+ "[email protected]",+ "-c",+ "commit.gpgsign=false",+ "commit",+ "-m",+ "fixture",+ );+ const head = (await git("rev-parse", "HEAD")).stdout.trim();++ const result = await cli.run({+ arguments: ["import-reference-source", source, "--json"],+ environment: {+ HOME: repository,+ XDG_CONFIG_HOME: repository,+ XDG_CACHE_HOME: repository,+ },+ });++ expect(result.exitCode).toBe(0);+ const graph = historicalSourceGraphSchema.parse(result.json);+ expect(graph.parse_failures).toEqual([+ {+ path: "module.mjs",+ parser: "babel",+ reason: "Deleting local variable in strict mode. (2:0)",+ },+ ]);+ // Outside `.mjs`/`.mts`/`.cjs` the mode stays unknown, not forced to script.+ const limitations = (path: string) =>+ graph.entries.find((entry) => entry.path === path)?.limitations;+ for (const path of ["legacy.js"])+ expect(limitations(path)).toEqual([+ "Strict-mode errors apply only if this file is an ES module: Deleting local variable in strict mode. (2:0)",+ ]);+ expect(limitations("legacy.cjs")).toEqual([]);+ expect(graph.languages).toEqual(["C#", "JavaScript", "PHP"]);+ for (const path of ["Program.cs", "index.php"])+ expect(+ graph.entries.find((entry) => entry.path === path)?.classifications,+ ).toContain("source");+ expect(graph.vcs).toEqual({ kind: "git", head, dirty: null });+ expect(graph.provenance.importer_version).toBe(+ PRODUCT_IDENTITY.packageVersion,+ );+ },+);