Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): preserve CommonJS lookup and analysis provenance (#2306)
Rohit Ghumared005bd5dcd5f
diff --git a/src/application/javascript/JavaScriptApplicationEvidence.ts b/src/application/javascript/JavaScriptApplicationEvidence.tsindex b812f237e..ca20b813f 100644--- a/src/application/javascript/JavaScriptApplicationEvidence.ts+++ b/src/application/javascript/JavaScriptApplicationEvidence.ts@@ -100,4 +100,10 @@ const parameters = (...(input.source_paths === undefined? {}: { source_paths: input.source_paths }),+ ...(input.max_heap_mb === undefined+ ? {}+ : { max_heap_mb: input.max_heap_mb }),+ ...(input.analysis_timeout_ms === undefined+ ? {}+ : { analysis_timeout_ms: input.analysis_timeout_ms }),});diff --git a/src/application/javascript/JavaScriptApplicationService.ts b/src/application/javascript/JavaScriptApplicationService.tsindex ea56ce591..fa66a5521 100644--- a/src/application/javascript/JavaScriptApplicationService.ts+++ b/src/application/javascript/JavaScriptApplicationService.ts@@ -62,26 +62,7 @@ export const analyzeJavaScriptApplicationValidated = async (});await checkpointJavaScriptAnalysis(options.signal);reconstructed = await reconstructJavaScriptArtifact(- {- input_path: input.input_path,- format: input.format,- integrity_policy: input.integrity_policy,- ...(input.max_semantic_nodes === undefined- ? {}- : { max_semantic_nodes: input.max_semantic_nodes }),- ...(input.max_semantic_nodes_per_file === undefined- ? {}- : { max_semantic_nodes_per_file: input.max_semantic_nodes_per_file }),- ...(input.source_paths === undefined- ? {}- : { source_paths: input.source_paths }),- ...(input.max_heap_mb === undefined- ? {}- : { max_heap_mb: input.max_heap_mb }),- ...(input.analysis_timeout_ms === undefined- ? {}- : { analysis_timeout_ms: input.analysis_timeout_ms }),- },+ input,scope,options.signal,options.progress,diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.tsindex cb29ab2a9..88aa1508f 100644--- a/src/application/javascript/JavaScriptArtifactPathResolution.ts+++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts@@ -457,9 +457,8 @@ const bareModuleCandidate = (dependencyDirectory = posix.dirname(confined);}const source = input.files.get(input.sourcePath);- let directory = posix.dirname(input.sourcePath);let nearestNotFound: CandidateResolution | null = null;- while (true) {+ for (const directory of packageLookupDirectories(input)) {const candidate = posix.join(directory, dependencyDirectory);const metadata = readPackageMetadata(input, candidate);const continueCommonJsLookup =@@ -487,8 +486,6 @@ const bareModuleCandidate = (nearestNotFound ??= packageResolution;} else return packageResolution;}- if (directory === "." || directory === "") break;- directory = posix.dirname(directory);}if (nearestNotFound !== null) return nearestNotFound;return unresolvedCandidate("external", [@@ -496,6 +493,21 @@ const bareModuleCandidate = (]);};+function* packageLookupDirectories(+ input: ResolveArtifactPathInput,+): Generator<string> {+ let directory = posix.dirname(input.sourcePath);+ while (true) {+ if (+ input.moduleKind !== "require" ||+ posix.basename(directory) !== "node_modules"+ )+ yield directory;+ if (directory === "." || directory === "") return;+ directory = posix.dirname(directory);+ }+}+/*** Nearest same-container package.json, stopping at a `node_modules` directory* so a nested package cannot inherit an ancestor's map.@@ -960,15 +972,29 @@ const resolveCandidate = (candidate: string,packageChain: ReadonlySet<string>,emittedSourceCandidates: readonly TypeScriptEmittedSourceCandidate[] = [],-): CandidateResolution =>- resolveFileCandidates(input, fileCandidates(candidate, input.moduleKind)) ??- resolveRuntimeSourceCandidates(input, candidate, emittedSourceCandidates) ??- resolveDirectory(+): CandidateResolution => {+ if (!commonJsDirectoryRequest(input)) {+ const file =+ resolveFileCandidates(+ input,+ fileCandidates(candidate, input.moduleKind),+ ) ??+ resolveRuntimeSourceCandidates(input, candidate, emittedSourceCandidates);+ if (file !== null) return file;+ }+ return resolveDirectory(input,candidate,readPackageMetadata(input, candidate),packageChain,);+};++const commonJsDirectoryRequest = (input: ResolveArtifactPathInput): boolean =>+ input.context === "module-specifier" &&+ input.moduleKind === "require" &&+ (input.declaredPath.endsWith("/") ||+ /(?:^|\/)\.{1,2}$/u.test(input.declaredPath));const resolveDirectory = (input: ResolveArtifactPathInput,diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.tsindex bd95415c6..a5528ee10 100644--- a/src/application/javascript/JavaScriptArtifactReconstruction.ts+++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts@@ -277,15 +277,9 @@ export const reconstructJavaScriptArtifact = async (limitations: completed.graph.limitations,}),);- const { max_heap_mb, analysis_timeout_ms, ...coreInput } = input;- const parameters = {- ...coreInput,- ...(max_heap_mb === undefined ? {} : { max_heap_mb }),- ...(analysis_timeout_ms === undefined ? {} : { analysis_timeout_ms }),- };const evidence =await createOwnedJavaScriptApplicationEvidenceCooperatively(- parameters,+ input,partial,);throw interruption.retainPartialObservation(evidence);diff --git a/tests/acceptance/applications/javascriptAnalysisLimits.test.ts b/tests/acceptance/applications/javascriptAnalysisLimits.test.tsnew file mode 100644index 000000000..6f0ba8cef--- /dev/null+++ b/tests/acceptance/applications/javascriptAnalysisLimits.test.ts@@ -0,0 +1,156 @@+import type { Client } from "@modelcontextprotocol/client";+import { readFile, writeFile } from "node:fs/promises";+import { join } from "node:path";+import { expect, it } from "vitest";+import { z } from "zod";++import { parseEvidence } from "../../../src/domain/evidence.js";+import { parseEvidenceBundle } from "../../../src/domain/evidenceBundle.js";+import { parseMcpToolError } from "../../fixtures/mcpToolError.js";+import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++const cases = [+ {+ name: "both limits",+ args: ["--max-heap-mb", "128", "--analysis-timeout-ms", "30000"],+ controls: { max_heap_mb: 128, analysis_timeout_ms: 30000 },+ },+ {+ name: "heap limit alone",+ args: ["--max-heap-mb", "256"],+ controls: { max_heap_mb: 256 },+ },+ {+ name: "timeout alone",+ args: ["--analysis-timeout-ms", "40000"],+ controls: { analysis_timeout_ms: 40000 },+ },+ {+ name: "omitted limits",+ args: [],+ controls: {},+ },+];++cliTest.for(cases)(+ "preserves $name in CLI Evidence without adding ambient defaults",+ async ({ args, controls }, { cli }) => {+ const root = await createTestTempDirectory("rea-js-limit-provenance-");+ await writeFile(join(root, "main.js"), "export const observed = 7;\n");+ const response = await cli.run({+ arguments: [+ "analyze-javascript-application",+ root,+ ...args,+ "--format",+ "json",+ ],+ environment: { NODE_OPTIONS: "--max-old-space-size=512" },+ });+ expect(response.exitCode, response.stdout + response.stderr).toBe(0);+ const evidence = parseEvidence(response.json);+ expect(evidence.parameters).toEqual({+ format: "auto",+ integrity_policy: "fail",+ ...controls,+ });+ },+);++it("preserves MCP analysis limits in retained Evidence and an exported bundle round trip", async () => {+ const root = await createTestTempDirectory("rea-js-limit-mcp-");+ await writeFile(join(root, "main.js"), "export const observed = 7;\n");+ const bundleRoot = await createTestTempDirectory("rea-js-limit-bundle-");+ const path = join(bundleRoot, "evidence.json");+ const { client } = await connectLocalToolsMcp({+ NODE_OPTIONS: "--max-old-space-size=512",+ });+ const expected = new Map<string, unknown>();+ for (const { controls } of cases) {+ const response = await client.callTool({+ name: "analyze_javascript_application",+ arguments: { input_path: root, ...controls, detail: "summary" },+ });+ expect(response.isError, JSON.stringify(response.content)).not.toBe(true);+ const { parent_evidence_id } = z+ .object({+ normalized_result: z.object({ parent_evidence_id: z.string() }),+ })+ .parse(response.structuredContent).normalized_result;+ expected.set(parent_evidence_id, {+ format: "auto",+ integrity_policy: "fail",+ ...controls,+ });+ }+ expect(expected.size).toBe(cases.length);+ const retained = await getBundle(client);+ for (const [evidence_id, parameters] of expected)+ expect(+ retained.records.find((record) => record.evidence_id === evidence_id)+ ?.parameters,+ ).toEqual(parameters);+ const exported = await client.callTool({+ name: "export_evidence_bundle",+ arguments: { path },+ });+ expect(exported.isError).not.toBe(true);+ expect(parseEvidenceBundle(JSON.parse(await readFile(path, "utf8")))).toEqual(+ retained,+ );+ const closed = await client.callTool({+ name: "close_binary",+ arguments: {},+ });+ expect(closed.isError).not.toBe(true);+ expect((await getBundle(client)).records).toEqual([]);+ const imported = await client.callTool({+ name: "import_evidence_bundle",+ arguments: { path },+ });+ expect(imported.isError, JSON.stringify(imported.content)).not.toBe(true);+ expect(await getBundle(client)).toEqual(retained);+}, 60_000);++it("retains caller limits in partial Evidence after an actual analysis timeout", async () => {+ const root = await createTestTempDirectory("rea-js-limit-timeout-");+ await writeFile(join(root, "main.js"), "export const observed = 7;\n");+ const { client } = await connectLocalToolsMcp({+ NODE_OPTIONS: "--max-old-space-size=512",+ });+ const response = await client.callTool({+ name: "analyze_javascript_application",+ arguments: {+ input_path: root,+ max_heap_mb: 128,+ analysis_timeout_ms: 1,+ },+ });+ expect(parseMcpToolError(response)).toMatchObject({+ error: { code: "provider_timeout" },+ });+ const bundle = await getBundle(client);+ const partial = bundle.records.find(+ (record) => record.operation === "analyze_javascript_application",+ );+ expect(partial?.parameters).toEqual({+ format: "auto",+ integrity_policy: "fail",+ max_heap_mb: 128,+ analysis_timeout_ms: 1,+ });+ await client.ping();+}, 30_000);++async function getBundle(client: Client) {+ const response = await client.callTool({+ name: "get_evidence_bundle",+ arguments: { detail: "complete" },+ });+ expect(response.isError).not.toBe(true);+ return parseEvidenceBundle(+ z.object({ result: z.unknown() }).parse(response.structuredContent).result,+ );+}diff --git a/tests/acceptance/applications/javascriptCommonJsDirectoryLookup.test.ts b/tests/acceptance/applications/javascriptCommonJsDirectoryLookup.test.tsnew file mode 100644index 000000000..da9fcee86--- /dev/null+++ b/tests/acceptance/applications/javascriptCommonJsDirectoryLookup.test.ts@@ -0,0 +1,188 @@+import { execFile } from "node:child_process";+import { createRequire } from "node:module";+import { join, relative, sep } from "node:path";+import { fileURLToPath } from "node:url";+import { promisify } from "node:util";++import { expect } from "vitest";++import type { JavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";+import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js";+import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js";++const execute = promisify(execFile);++const logicalPath = (root: string, path: string): string =>+ relative(root, path).split(sep).join("/");++const requireSources = (+ requests: readonly (readonly [string, string, string | null])[],+): Readonly<Record<string, string>> =>+ Object.fromEntries(+ [...new Set(requests.map(([source]) => source))].map((source) => [+ source,+ requests+ .filter(([path]) => path === source)+ .map(([, specifier]) => `require(${JSON.stringify(specifier)});`)+ .join("\n"),+ ]),+ );++const expectResolution = (+ graph: JavaScriptApplicationGraph,+ source: string,+ specifier: string,+ kind: "require" | "import",+ resolvedPath: string | null,+): void => {+ const edges = graph.edges.filter(+ ({ relation, properties, evidence }) =>+ relation === "imports" &&+ properties.specifier === specifier &&+ evidence.location.available &&+ evidence.location.value.kind === "source-range" &&+ evidence.location.value.source === source,+ );+ expect(edges).toHaveLength(2);+ expect(edges.map(({ properties }) => properties)).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ kind: kind === "require" ? "require" : "static-import",+ resolved_path: resolvedPath,+ }),+ expect.objectContaining({+ module_link_kind: kind,+ resolved_path: resolvedPath,+ resolution_status: resolvedPath === null ? "not-found" : "resolved",+ }),+ ]),+ );+};++cliTest.for(["CLI", "stdio MCP"] as const)(+ "preserves CommonJS directory requests before path normalization through %s",+ async (surface, { cli }) => {+ const root = await createTestTempDirectory("rea-cjs-directory-lookup-");+ const requests = [+ ["main.cjs", "./dep/.", "dep/index.js"],+ ["main.cjs", "./dep/", "dep/index.js"],+ ["main.cjs", "./parent/child/..", "parent/index.js"],+ ["main.cjs", "./dep", "dep.js"],+ ["main.cjs", "./entry/.", "entry/dep.js"],+ ["main.cjs", "./missing/.", null],+ ["main.cjs", "directory/", "node_modules/directory/index.js"],+ ["main.cjs", "directory/.", "node_modules/directory/index.js"],+ ["main.cjs", "package/dep/.", "node_modules/package/dep/index.js"],+ ["parent/child/main.cjs", ".", "parent/child/index.js"],+ ["parent/child/main.cjs", "..", "parent/index.js"],+ ["typed.cts", "./typed.js/.", "typed.js/index.js"],+ ] as const;+ await writeFixtureFiles(root, {+ "package.json": '{"type":"commonjs"}',+ "dep.js": "module.exports = 'file';",+ "dep/index.js": "module.exports = 'directory';",+ "entry/package.json": '{"main":"./dep/."}',+ "entry/dep.js": "module.exports = 'main file';",+ "entry/dep/index.js": "module.exports = 'main directory decoy';",+ "missing.js": "module.exports = 'missing directory decoy';",+ "parent.js": "module.exports = 'parent file';",+ "parent/index.js": "module.exports = 'parent directory';",+ "parent/child.js": "module.exports = 'child file';",+ "parent/child/index.js": "module.exports = 'child directory';",+ "node_modules/directory.js": "module.exports = 'package file';",+ "node_modules/directory/index.js":+ "module.exports = 'package directory';",+ "node_modules/package/dep.js": "module.exports = 'subpath file';",+ "node_modules/package/dep/index.js":+ "module.exports = 'subpath directory';",+ "typed.ts": "module.exports = 'emitted source decoy';",+ "typed.js/index.js": "module.exports = 'typed request directory';",+ ...requireSources(requests),+ });+ for (const [source, specifier, expected] of requests) {+ const require = createRequire(join(root, source));+ if (expected === null)+ expect(() => require.resolve(specifier)).toThrow(+ expect.objectContaining({ code: "MODULE_NOT_FOUND" }),+ );+ else expect(logicalPath(root, require.resolve(specifier))).toBe(expected);+ }+ const { graph } = await analyzeJavaScriptDirectory(surface, cli, root);+ for (const [source, specifier, expected] of requests)+ expectResolution(graph, source, specifier, "require", expected);+ expect(graph.coverage.status).toBe("complete");+ },+);++cliTest.for(["CLI", "stdio MCP"] as const)(+ "preserves loader-specific node_modules ancestors through %s",+ async (surface, { cli }) => {+ const root = await createTestTempDirectory("rea-cjs-ancestor-lookup-");+ const requests = [+ [+ "node_modules/client/main.cjs",+ "shared",+ "node_modules/shared/index.js",+ ],+ [+ "node_modules/client/main.cjs",+ "@scope/shared",+ "node_modules/@scope/shared/index.js",+ ],+ [+ "node_modules/client/main.cjs",+ "private",+ "node_modules/client/node_modules/private/index.js",+ ],+ [+ "node_modules/@scope/client/main.cjs",+ "shared",+ "node_modules/shared/index.js",+ ],+ [+ "node_modules/parent/node_modules/client/main.cjs",+ "shared",+ "node_modules/parent/node_modules/shared/index.js",+ ],+ ] as const;+ const esmSource = "node_modules/client/main.mjs";+ const esmTarget = "node_modules/node_modules/shared/index.js";+ await writeFixtureFiles(root, {+ "package.json": '{"type":"commonjs"}',+ "node_modules/shared/index.js": "module.exports = 'outer dependency';",+ "node_modules/@scope/shared/index.js":+ "module.exports = 'scoped dependency';",+ "node_modules/node_modules/shared/index.js":+ "module.exports = 'ESM dependency';",+ "node_modules/node_modules/@scope/shared/index.js":+ "module.exports = 'scoped decoy';",+ "node_modules/client/node_modules/private/index.js":+ "module.exports = 'private dependency';",+ "node_modules/parent/node_modules/shared/index.js":+ "module.exports = 'nested dependency';",+ "node_modules/parent/node_modules/node_modules/shared/index.js":+ "module.exports = 'nested decoy';",+ [esmSource]:+ 'import "shared"; console.log(JSON.stringify(import.meta.resolve("shared")));',+ ...requireSources(requests),+ });+ for (const [source, specifier, expected] of requests)+ expect(+ logicalPath(root, createRequire(join(root, source)).resolve(specifier)),+ ).toBe(expected);+ const control = await execute(process.execPath, [join(root, esmSource)], {+ cwd: root,+ timeout: 5_000,+ });+ expect(+ logicalPath(root, fileURLToPath(JSON.parse(control.stdout) as string)),+ ).toBe(esmTarget);+ const { graph } = await analyzeJavaScriptDirectory(surface, cli, root);+ for (const [source, specifier, expected] of requests)+ expectResolution(graph, source, specifier, "require", expected);+ expectResolution(graph, esmSource, "shared", "import", esmTarget);+ expect(graph.coverage.status).toBe("complete");+ },+);diff --git a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts b/tests/boundary/filesystem/interfaceBuilderBundle.test.tsindex 21ec633c5..543248e63 100644--- a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts+++ b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts@@ -405,12 +405,16 @@ describe("keyed archive hierarchy coverage", () => {const resources = join(bundle, "Contents", "Resources");await mkdir(resources, { recursive: true });const archive = join(resources, "Foundation.nib");- await compile("/usr/bin/xcrun", [- "swift",- "tests/conformance/native/keyed-archive-hierarchy.swift",- archive,- "40",- ]);+ await compile(+ "/usr/bin/xcrun",+ [+ "swift",+ "tests/conformance/native/keyed-archive-hierarchy.swift",+ archive,+ "40",+ ],+ { timeout: 90_000 },+ );const analysis = await analyzeInterfaceBuilderBundle({bundlePath: bundle,@@ -431,6 +435,7 @@ describe("keyed archive hierarchy coverage", () => {),).toHaveLength(41);},+ 120_000,);});