Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): preserve evidence through deep AST analysis (#924)
morlutof826aa893fa7
diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.tsindex adbbd0771..d74d72977 100644--- a/src/browser/WebSourceMapFetcher.ts+++ b/src/browser/WebSourceMapFetcher.ts@@ -3,7 +3,8 @@ import { AnyMap, eachMapping } from "@jridgewell/trace-mapping";import { sanitizeBrowserUrl } from "../domain/browserObservation.js";import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js";-import { analyzeParsedJavaScriptSemantics } from "../domain/javascript/javascriptSemanticAnalysis.js";+import { analyzeParsedJavaScriptReferences } from "../domain/javascript/javascriptSemanticAnalysis.js";+import { traverseJavaScriptAst } from "../domain/javascript/javascriptSemanticTraversal.js";import { parseJavaScriptSource } from "../domain/javascript/javascriptSourceParser.js";import { hasValidSourceMapContents } from "../domain/sourceMapContents.js";import type {@@ -237,42 +238,42 @@ const originalModuleEdges = (// after an unrecoverable point are missing, so the edges are a subset.if (parsed.errors.length > 0) incomplete.push(source.source);let unboundRequires: ReadonlySet<string> | undefined;- t.traverseFast(parsed, (node) => {- const dependency = originalDependency(node);- if (dependency === null) return;- const { kind, specifier } = dependency;- if (kind === "require" && t.isCallExpression(node)) {- unboundRequires ??= new Set(- analyzeParsedJavaScriptSemantics(parsed)- .references.filter(- ({ name, role, resolution }) =>- name === "require" &&- role === "read" &&- resolution === "unbound",+ traverseJavaScriptAst(parsed, {+ enter: (node) => {+ const dependency = originalDependency(node);+ if (dependency === null) return;+ const { kind, specifier } = dependency;+ if (kind === "require" && t.isCallExpression(node)) {+ unboundRequires ??= new Set(+ analyzeParsedJavaScriptReferences(parsed, "require")+ .filter(+ ({ role, resolution }) =>+ role === "read" && resolution === "unbound",+ )+ .map(+ ({ location }) =>+ `${String(location.start.line)}:${String(location.start.column)}`,+ ),+ );+ const location = node.callee.loc?.start;+ if (+ location === undefined ||+ !unboundRequires.has(+ `${String(location.line)}:${String(location.column)}`,)- .map(- ({ location }) =>- `${String(location.start.line)}:${String(location.start.column)}`,- ),- );- const location = node.callee.loc?.start;- if (- location === undefined ||- !unboundRequires.has(- `${String(location.line)}:${String(location.column)}`,)- )- return;- }- const key = `${source.source}\0${kind}\0${specifier}`;- if (seen.has(key)) return;- seen.add(key);- edges.push({- from_source: source.source,- kind,- specifier,- resolved_source: resolveOriginalSource(specifier, source.source),- });+ return;+ }+ const key = `${source.source}\0${kind}\0${specifier}`;+ if (seen.has(key)) return;+ seen.add(key);+ edges.push({+ from_source: source.source,+ kind,+ specifier,+ resolved_source: resolveOriginalSource(specifier, source.source),+ });+ },});}return { edges, incomplete };diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.tsindex d73355433..b9a267fb4 100644--- a/src/domain/javascript/javascriptSemanticAnalysis.ts+++ b/src/domain/javascript/javascriptSemanticAnalysis.ts@@ -5,6 +5,7 @@ import {type JavaScriptModuleOrigin,type JavaScriptSemanticDefinition,type JavaScriptSemanticIr,+ type JavaScriptSemanticReference,} from "./javascriptSemanticIr.js";import {collectSemanticModuleLink,@@ -71,6 +72,16 @@ export const analyzeJavaScriptSemantics = (: analyzeParsedJavaScriptSemantics(file);};+/** Recover lexical references, optionally for one name, without evaluating values or provenance. */+export const analyzeParsedJavaScriptReferences = (+ file: ParsedJavaScriptSource,+ name?: string,+): readonly JavaScriptSemanticReference[] => {+ const state = createState(file.program);+ collectDefinitions(file.program, state);+ return collectSemanticReferences(file.program, state, name);+};+/** Recover semantics from an already parsed JavaScript artifact. */export const analyzeParsedJavaScriptSemantics = (file: ParsedJavaScriptSource,diff --git a/src/domain/javascript/javascriptSemanticProjection.ts b/src/domain/javascript/javascriptSemanticProjection.tsindex 578c53a13..9dcc8a7ad 100644--- a/src/domain/javascript/javascriptSemanticProjection.ts+++ b/src/domain/javascript/javascriptSemanticProjection.ts@@ -92,16 +92,22 @@ export const collectSemanticModuleLink = (else if (t.isAssignmentExpression(node)) collectCommonJsExport(node, state);};-/** Collect bounded lexical identifier references after definitions exist. */+/** Collect lexical references after definitions exist, optionally for one name. */export const collectSemanticReferences = (program: t.Program,state: JavaScriptSemanticAnalysisState,+ name?: string,): JavaScriptSemanticReference[] => {const output: JavaScriptSemanticReference[] = [];const seen = new Set<string>();traverseJavaScriptAst(program, {enter: (node, parent, readAncestors) => {- if (!t.isIdentifier(node) || parent === null) return;+ if (+ !t.isIdentifier(node) ||+ parent === null ||+ (name !== undefined && node.name !== name)+ )+ return;const role = semanticIdentifierRole(node, parent, readAncestors());if (role === null) return;// A compound assignment or update reads before it writes; emit both sodiff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.tsindex a217d45d9..681694667 100644--- a/src/domain/webBundleAnalyzer.test.ts+++ b/src/domain/webBundleAnalyzer.test.ts@@ -301,6 +301,32 @@ describe("web bundle artifact metadata", () => {});});+describe("deep captured web bundle syntax", () => {+ it("retains evidence after a parser-admitted deep property chain", () => {+ const source = `const value = root${".next".repeat(12_000)};\nimport "./last.js";\nfetch("/after");`;+ const result = analyzeCapturedWebBundle(inspection(source));+ expect(result.completeness).toMatchObject({+ status: "complete",+ parsed_scripts: 1,+ parse_failures: 0,+ });+ expect(result.observations.chunks.edges).toContainEqual(+ expect.objectContaining({+ kind: "static_import",+ specifier: "./last.js",+ resolved_url: `${origin}/assets/last.js`,+ location: expect.objectContaining({ line: 2, column: 0 }),+ }),+ );+ expect(result.observations.endpoints).toContainEqual(+ expect.objectContaining({+ value: "/after",+ location: expect.objectContaining({ line: 3, column: 0 }),+ }),+ );+ });+});+const inspection = (source: string) =>webPageInspectionSchema.parse({browser: {diff --git a/src/domain/webBundleAnalyzerInspection.ts b/src/domain/webBundleAnalyzerInspection.tsindex 3260f9b98..253885bc0 100644--- a/src/domain/webBundleAnalyzerInspection.ts+++ b/src/domain/webBundleAnalyzerInspection.ts@@ -4,6 +4,7 @@ import * as t from "@babel/types";import { sanitizeEndpointCandidate } from "./browserObservation.js";import type { WebPageInspection } from "./browserObservation.js";import type { WebBundleAnalysis } from "./webBundleAnalysis.js";+import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";import { semanticStaticPropertyName } from "./javascript/javascriptAstValues.js";import {calleeName,@@ -61,10 +62,11 @@ export const analyzeScript = (}accumulator.parsedScripts += 1;detectVendorFingerprints(script, accumulator);- t.traverseFast(file, (node) => {- accumulator.visitedNodes += 1;- inspectNode(script, node, accumulator);- return undefined;+ traverseJavaScriptAst(file, {+ enter: (node) => {+ accumulator.visitedNodes += 1;+ inspectNode(script, node, accumulator);+ },});};diff --git a/src/domain/webModuleImports.test.ts b/src/domain/webModuleImports.test.tsindex 8366b5adb..346938a4e 100644--- a/src/domain/webModuleImports.test.ts+++ b/src/domain/webModuleImports.test.ts@@ -47,6 +47,31 @@ describe("native module syntax observations", () => {});});+describe("deep native module syntax", () => {+ it("retains imports and their locations after a parser-admitted deep property chain", () => {+ const source = `const value = root${".next".repeat(12_000)};\nimport "./last.js";\nexport * from "./reexport.js";\nimport("./lazy.js");`;+ const result = collectWebModuleImports(source);+ expect(result.state).toBe("parsed");+ expect(result.diagnostics).toEqual([]);+ expect(+ result.imports.map(({ kind, specifier }) => [kind, specifier]),+ ).toEqual([+ ["static-import", "./last.js"],+ ["re-export", "./reexport.js"],+ ["dynamic-import", "./lazy.js"],+ ]);+ for (const item of result.imports)+ expect(source.slice(item.start.offset, item.end.offset)).toBe(+ item.expression,+ );+ expect(result.imports[0]?.start).toEqual({+ offset: source.indexOf('"./last.js"'),+ line: 2,+ column: 7,+ });+ });+});+const script = (url: string,kind: "page-script" | "scenario-response",diff --git a/src/domain/webModuleImports.ts b/src/domain/webModuleImports.tsindex 58dcd82d7..fdcd36e83 100644--- a/src/domain/webModuleImports.ts+++ b/src/domain/webModuleImports.ts@@ -1,5 +1,6 @@-import { traverseFast, isStringLiteral, isTemplateLiteral } from "@babel/types";+import { isStringLiteral, isTemplateLiteral } from "@babel/types";import type { Node } from "@babel/types";+import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";import { parseJavaScriptSource } from "./javascript/javascriptSourceParser.js";import type { WebModuleImport } from "./webModuleTrace.js";import type { ExportedWebScript } from "./webScriptExport.js";@@ -51,46 +52,48 @@ export const collectWebModuleImports = (},});};- traverseFast(ast, (node) => {- if (node.type === "ImportDeclaration") {- if (node.importKind === "type" || node.importKind === "typeof") return;- if (- node.specifiers.length > 0 &&- node.specifiers.every(- (s) =>- s.type === "ImportSpecifier" &&- (s.importKind === "type" || s.importKind === "typeof"),+ traverseJavaScriptAst(ast, {+ enter: (node) => {+ if (node.type === "ImportDeclaration") {+ if (node.importKind === "type" || node.importKind === "typeof") return;+ if (+ node.specifiers.length > 0 &&+ node.specifiers.every(+ (s) =>+ s.type === "ImportSpecifier" &&+ (s.importKind === "type" || s.importKind === "typeof"),+ ))- )- return;- add("static-import", node.source);- } else if (- node.type === "ExportAllDeclaration" ||- node.type === "ExportNamedDeclaration"- ) {- if (- node.exportKind === "type" ||- node.source === null ||- node.source === undefined- )- return;- if (- node.type === "ExportNamedDeclaration" &&- node.specifiers.length > 0 &&- node.specifiers.every(- (s) => s.type === "ExportSpecifier" && s.exportKind === "type",+ return;+ add("static-import", node.source);+ } else if (+ node.type === "ExportAllDeclaration" ||+ node.type === "ExportNamedDeclaration"+ ) {+ if (+ node.exportKind === "type" ||+ node.source === null ||+ node.source === undefined)+ return;+ if (+ node.type === "ExportNamedDeclaration" &&+ node.specifiers.length > 0 &&+ node.specifiers.every(+ (s) => s.type === "ExportSpecifier" && s.exportKind === "type",+ )+ )+ return;+ add("re-export", node.source);+ } else if (node.type === "ImportExpression")+ add("dynamic-import", node.source);+ else if (+ node.type === "CallExpression" &&+ node.callee.type === "Import" &&+ node.arguments[0] !== undefined)- return;- add("re-export", node.source);- } else if (node.type === "ImportExpression")- add("dynamic-import", node.source);- else if (- node.type === "CallExpression" &&- node.callee.type === "Import" &&- node.arguments[0] !== undefined- )- add("dynamic-import", node.arguments[0]);+ add("dynamic-import", node.arguments[0]);+ },});imports.sort((a, b) => a.start.offset - b.start.offset);return {diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.tsindex 568629017..6dc38c942 100644--- a/tests/boundary/browser/webSourceMapFetcher.test.ts+++ b/tests/boundary/browser/webSourceMapFetcher.test.ts@@ -658,3 +658,52 @@ describe("source-map original artifact language metadata", () => {}});});++describe("deep source-map original syntax", () => {+ it("retains valid source maps and native/CommonJS evidence after a parser-admitted deep property chain", async () => {+ const source = `const value = root${".next".repeat(12_000)};\nimport "./last.js";\nrequire("./common.js");\nfunction local(require) { require("./shadowed.js"); }`;+ const text = JSON.stringify({+ version: 3,+ names: [],+ sources: ["../src/main.js"],+ sourcesContent: [source],+ mappings: "AAAA",+ });+ const result = await fetchWebSourceMaps([request], input(), undefined, {+ fetch: async () => new Response(text),+ });+ expect(result).toMatchObject({+ status: "included",+ items: [+ {+ status: "included",+ limitation: null,+ artifact: { text },+ original_sources: [{ artifact: { text: source } }],+ original_module_edges: [+ {+ from_source: `${origin}/src/main.js`,+ kind: "static_import",+ specifier: "./last.js",+ resolved_source: `${origin}/src/last.js`,+ },+ {+ from_source: `${origin}/src/main.js`,+ kind: "require",+ specifier: "./common.js",+ resolved_source: `${origin}/src/common.js`,+ },+ ],+ mappings: [+ {+ generated_line: 1,+ generated_column: 0,+ original_line: 1,+ original_column: 0,+ },+ ],+ },+ ],+ });+ });+});