Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(browser): preserve evidence through deep AST analysis (#924)

morlutof826aa893fa7
diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts
index adbbd0771..d74d72977 100644
--- a/src/browser/WebSourceMapFetcher.ts
+++ b/src/browser/WebSourceMapFetcher.ts
@@ -3,7 +3,8 @@ import { AnyMap, eachMapping } from "@jridgewell/trace-mapping";
import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js";
-import { analyzeParsedJavaScriptSemantics } from "../domain/javascript/javascriptSemanticAnalysis.js";
+import { analyzeParsedJavaScriptReferences } from "../domain/javascript/javascriptSemanticAnalysis.js";
+import { traverseJavaScriptAst } from "../domain/javascript/javascriptSemanticTraversal.js";
import { parseJavaScriptSource } from "../domain/javascript/javascriptSourceParser.js";
import { hasValidSourceMapContents } from "../domain/sourceMapContents.js";
import type {
@@ -237,42 +238,42 @@ const originalModuleEdges = (
// after an unrecoverable point are missing, so the edges are a subset.
if (parsed.errors.length > 0) incomplete.push(source.source);
let unboundRequires: ReadonlySet<string> | undefined;
- t.traverseFast(parsed, (node) => {
- const dependency = originalDependency(node);
- if (dependency === null) return;
- const { kind, specifier } = dependency;
- if (kind === "require" && t.isCallExpression(node)) {
- unboundRequires ??= new Set(
- analyzeParsedJavaScriptSemantics(parsed)
- .references.filter(
- ({ name, role, resolution }) =>
- name === "require" &&
- role === "read" &&
- resolution === "unbound",
+ traverseJavaScriptAst(parsed, {
+ enter: (node) => {
+ const dependency = originalDependency(node);
+ if (dependency === null) return;
+ const { kind, specifier } = dependency;
+ if (kind === "require" && t.isCallExpression(node)) {
+ unboundRequires ??= new Set(
+ analyzeParsedJavaScriptReferences(parsed, "require")
+ .filter(
+ ({ role, resolution }) =>
+ role === "read" && resolution === "unbound",
+ )
+ .map(
+ ({ location }) =>
+ `${String(location.start.line)}:${String(location.start.column)}`,
+ ),
+ );
+ const location = node.callee.loc?.start;
+ if (
+ location === undefined ||
+ !unboundRequires.has(
+ `${String(location.line)}:${String(location.column)}`,
)
- .map(
- ({ location }) =>
- `${String(location.start.line)}:${String(location.start.column)}`,
- ),
- );
- const location = node.callee.loc?.start;
- if (
- location === undefined ||
- !unboundRequires.has(
- `${String(location.line)}:${String(location.column)}`,
)
- )
- return;
- }
- const key = `${source.source}\0${kind}\0${specifier}`;
- if (seen.has(key)) return;
- seen.add(key);
- edges.push({
- from_source: source.source,
- kind,
- specifier,
- resolved_source: resolveOriginalSource(specifier, source.source),
- });
+ return;
+ }
+ const key = `${source.source}\0${kind}\0${specifier}`;
+ if (seen.has(key)) return;
+ seen.add(key);
+ edges.push({
+ from_source: source.source,
+ kind,
+ specifier,
+ resolved_source: resolveOriginalSource(specifier, source.source),
+ });
+ },
});
}
return { edges, incomplete };
diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts
index d73355433..b9a267fb4 100644
--- a/src/domain/javascript/javascriptSemanticAnalysis.ts
+++ b/src/domain/javascript/javascriptSemanticAnalysis.ts
@@ -5,6 +5,7 @@ import {
type JavaScriptModuleOrigin,
type JavaScriptSemanticDefinition,
type JavaScriptSemanticIr,
+ type JavaScriptSemanticReference,
} from "./javascriptSemanticIr.js";
import {
collectSemanticModuleLink,
@@ -71,6 +72,16 @@ export const analyzeJavaScriptSemantics = (
: analyzeParsedJavaScriptSemantics(file);
};
+/** Recover lexical references, optionally for one name, without evaluating values or provenance. */
+export const analyzeParsedJavaScriptReferences = (
+ file: ParsedJavaScriptSource,
+ name?: string,
+): readonly JavaScriptSemanticReference[] => {
+ const state = createState(file.program);
+ collectDefinitions(file.program, state);
+ return collectSemanticReferences(file.program, state, name);
+};
+
/** Recover semantics from an already parsed JavaScript artifact. */
export const analyzeParsedJavaScriptSemantics = (
file: ParsedJavaScriptSource,
diff --git a/src/domain/javascript/javascriptSemanticProjection.ts b/src/domain/javascript/javascriptSemanticProjection.ts
index 578c53a13..9dcc8a7ad 100644
--- a/src/domain/javascript/javascriptSemanticProjection.ts
+++ b/src/domain/javascript/javascriptSemanticProjection.ts
@@ -92,16 +92,22 @@ export const collectSemanticModuleLink = (
else if (t.isAssignmentExpression(node)) collectCommonJsExport(node, state);
};
-/** Collect bounded lexical identifier references after definitions exist. */
+/** Collect lexical references after definitions exist, optionally for one name. */
export const collectSemanticReferences = (
program: t.Program,
state: JavaScriptSemanticAnalysisState,
+ name?: string,
): JavaScriptSemanticReference[] => {
const output: JavaScriptSemanticReference[] = [];
const seen = new Set<string>();
traverseJavaScriptAst(program, {
enter: (node, parent, readAncestors) => {
- if (!t.isIdentifier(node) || parent === null) return;
+ if (
+ !t.isIdentifier(node) ||
+ parent === null ||
+ (name !== undefined && node.name !== name)
+ )
+ return;
const role = semanticIdentifierRole(node, parent, readAncestors());
if (role === null) return;
// A compound assignment or update reads before it writes; emit both so
diff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.ts
index a217d45d9..681694667 100644
--- a/src/domain/webBundleAnalyzer.test.ts
+++ b/src/domain/webBundleAnalyzer.test.ts
@@ -301,6 +301,32 @@ describe("web bundle artifact metadata", () => {
});
});
+describe("deep captured web bundle syntax", () => {
+ it("retains evidence after a parser-admitted deep property chain", () => {
+ const source = `const value = root${".next".repeat(12_000)};\nimport "./last.js";\nfetch("/after");`;
+ const result = analyzeCapturedWebBundle(inspection(source));
+ expect(result.completeness).toMatchObject({
+ status: "complete",
+ parsed_scripts: 1,
+ parse_failures: 0,
+ });
+ expect(result.observations.chunks.edges).toContainEqual(
+ expect.objectContaining({
+ kind: "static_import",
+ specifier: "./last.js",
+ resolved_url: `${origin}/assets/last.js`,
+ location: expect.objectContaining({ line: 2, column: 0 }),
+ }),
+ );
+ expect(result.observations.endpoints).toContainEqual(
+ expect.objectContaining({
+ value: "/after",
+ location: expect.objectContaining({ line: 3, column: 0 }),
+ }),
+ );
+ });
+});
+
const inspection = (source: string) =>
webPageInspectionSchema.parse({
browser: {
diff --git a/src/domain/webBundleAnalyzerInspection.ts b/src/domain/webBundleAnalyzerInspection.ts
index 3260f9b98..253885bc0 100644
--- a/src/domain/webBundleAnalyzerInspection.ts
+++ b/src/domain/webBundleAnalyzerInspection.ts
@@ -4,6 +4,7 @@ import * as t from "@babel/types";
import { sanitizeEndpointCandidate } from "./browserObservation.js";
import type { WebPageInspection } from "./browserObservation.js";
import type { WebBundleAnalysis } from "./webBundleAnalysis.js";
+import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";
import { semanticStaticPropertyName } from "./javascript/javascriptAstValues.js";
import {
calleeName,
@@ -61,10 +62,11 @@ export const analyzeScript = (
}
accumulator.parsedScripts += 1;
detectVendorFingerprints(script, accumulator);
- t.traverseFast(file, (node) => {
- accumulator.visitedNodes += 1;
- inspectNode(script, node, accumulator);
- return undefined;
+ traverseJavaScriptAst(file, {
+ enter: (node) => {
+ accumulator.visitedNodes += 1;
+ inspectNode(script, node, accumulator);
+ },
});
};
diff --git a/src/domain/webModuleImports.test.ts b/src/domain/webModuleImports.test.ts
index 8366b5adb..346938a4e 100644
--- a/src/domain/webModuleImports.test.ts
+++ b/src/domain/webModuleImports.test.ts
@@ -47,6 +47,31 @@ describe("native module syntax observations", () => {
});
});
+describe("deep native module syntax", () => {
+ it("retains imports and their locations after a parser-admitted deep property chain", () => {
+ const source = `const value = root${".next".repeat(12_000)};\nimport "./last.js";\nexport * from "./reexport.js";\nimport("./lazy.js");`;
+ const result = collectWebModuleImports(source);
+ expect(result.state).toBe("parsed");
+ expect(result.diagnostics).toEqual([]);
+ expect(
+ result.imports.map(({ kind, specifier }) => [kind, specifier]),
+ ).toEqual([
+ ["static-import", "./last.js"],
+ ["re-export", "./reexport.js"],
+ ["dynamic-import", "./lazy.js"],
+ ]);
+ for (const item of result.imports)
+ expect(source.slice(item.start.offset, item.end.offset)).toBe(
+ item.expression,
+ );
+ expect(result.imports[0]?.start).toEqual({
+ offset: source.indexOf('"./last.js"'),
+ line: 2,
+ column: 7,
+ });
+ });
+});
+
const script = (
url: string,
kind: "page-script" | "scenario-response",
diff --git a/src/domain/webModuleImports.ts b/src/domain/webModuleImports.ts
index 58dcd82d7..fdcd36e83 100644
--- a/src/domain/webModuleImports.ts
+++ b/src/domain/webModuleImports.ts
@@ -1,5 +1,6 @@
-import { traverseFast, isStringLiteral, isTemplateLiteral } from "@babel/types";
+import { isStringLiteral, isTemplateLiteral } from "@babel/types";
import type { Node } from "@babel/types";
+import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";
import { parseJavaScriptSource } from "./javascript/javascriptSourceParser.js";
import type { WebModuleImport } from "./webModuleTrace.js";
import type { ExportedWebScript } from "./webScriptExport.js";
@@ -51,46 +52,48 @@ export const collectWebModuleImports = (
},
});
};
- traverseFast(ast, (node) => {
- if (node.type === "ImportDeclaration") {
- if (node.importKind === "type" || node.importKind === "typeof") return;
- if (
- node.specifiers.length > 0 &&
- node.specifiers.every(
- (s) =>
- s.type === "ImportSpecifier" &&
- (s.importKind === "type" || s.importKind === "typeof"),
+ traverseJavaScriptAst(ast, {
+ enter: (node) => {
+ if (node.type === "ImportDeclaration") {
+ if (node.importKind === "type" || node.importKind === "typeof") return;
+ if (
+ node.specifiers.length > 0 &&
+ node.specifiers.every(
+ (s) =>
+ s.type === "ImportSpecifier" &&
+ (s.importKind === "type" || s.importKind === "typeof"),
+ )
)
- )
- return;
- add("static-import", node.source);
- } else if (
- node.type === "ExportAllDeclaration" ||
- node.type === "ExportNamedDeclaration"
- ) {
- if (
- node.exportKind === "type" ||
- node.source === null ||
- node.source === undefined
- )
- return;
- if (
- node.type === "ExportNamedDeclaration" &&
- node.specifiers.length > 0 &&
- node.specifiers.every(
- (s) => s.type === "ExportSpecifier" && s.exportKind === "type",
+ return;
+ add("static-import", node.source);
+ } else if (
+ node.type === "ExportAllDeclaration" ||
+ node.type === "ExportNamedDeclaration"
+ ) {
+ if (
+ node.exportKind === "type" ||
+ node.source === null ||
+ node.source === undefined
)
+ return;
+ if (
+ node.type === "ExportNamedDeclaration" &&
+ node.specifiers.length > 0 &&
+ node.specifiers.every(
+ (s) => s.type === "ExportSpecifier" && s.exportKind === "type",
+ )
+ )
+ return;
+ add("re-export", node.source);
+ } else if (node.type === "ImportExpression")
+ add("dynamic-import", node.source);
+ else if (
+ node.type === "CallExpression" &&
+ node.callee.type === "Import" &&
+ node.arguments[0] !== undefined
)
- return;
- add("re-export", node.source);
- } else if (node.type === "ImportExpression")
- add("dynamic-import", node.source);
- else if (
- node.type === "CallExpression" &&
- node.callee.type === "Import" &&
- node.arguments[0] !== undefined
- )
- add("dynamic-import", node.arguments[0]);
+ add("dynamic-import", node.arguments[0]);
+ },
});
imports.sort((a, b) => a.start.offset - b.start.offset);
return {
diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts
index 568629017..6dc38c942 100644
--- a/tests/boundary/browser/webSourceMapFetcher.test.ts
+++ b/tests/boundary/browser/webSourceMapFetcher.test.ts
@@ -658,3 +658,52 @@ describe("source-map original artifact language metadata", () => {
}
});
});
+
+describe("deep source-map original syntax", () => {
+ it("retains valid source maps and native/CommonJS evidence after a parser-admitted deep property chain", async () => {
+ const source = `const value = root${".next".repeat(12_000)};\nimport "./last.js";\nrequire("./common.js");\nfunction local(require) { require("./shadowed.js"); }`;
+ const text = JSON.stringify({
+ version: 3,
+ names: [],
+ sources: ["../src/main.js"],
+ sourcesContent: [source],
+ mappings: "AAAA",
+ });
+ const result = await fetchWebSourceMaps([request], input(), undefined, {
+ fetch: async () => new Response(text),
+ });
+ expect(result).toMatchObject({
+ status: "included",
+ items: [
+ {
+ status: "included",
+ limitation: null,
+ artifact: { text },
+ original_sources: [{ artifact: { text: source } }],
+ original_module_edges: [
+ {
+ from_source: `${origin}/src/main.js`,
+ kind: "static_import",
+ specifier: "./last.js",
+ resolved_source: `${origin}/src/last.js`,
+ },
+ {
+ from_source: `${origin}/src/main.js`,
+ kind: "require",
+ specifier: "./common.js",
+ resolved_source: `${origin}/src/common.js`,
+ },
+ ],
+ mappings: [
+ {
+ generated_line: 1,
+ generated_column: 0,
+ original_line: 1,
+ original_column: 0,
+ },
+ ],
+ },
+ ],
+ });
+ });
+});