Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): yield during the largest AST traversals (#1545)

Kaoru08225d6eeb22fb3b
diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts
index 72fb67edd..fcba4cc16 100644
--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts
+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts
@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
import { setImmediate } from "node:timers/promises";
import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";
-import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";
+import { analyzeParsedJavaScriptStaticSourceSteps } from "../../domain/javascript/javascriptStaticAnalysis.js";
import {
analyzeParsedJavaScriptSemanticsSteps,
classifyParsedJavaScriptOpenReceivers,
@@ -190,7 +190,7 @@ function* analyzeArtifactFileSteps<
}
const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);
yield;
- const analysis = analyzeParsedJavaScriptStaticSource(
+ const analysis = yield* analyzeParsedJavaScriptStaticSourceSteps(
file.text.value,
parsed,
openReceiverFacts,
diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts
index 50af0f2cb..ff1dac49b 100644
--- a/src/domain/javascript/javascriptSemanticAnalysis.ts
+++ b/src/domain/javascript/javascriptSemanticAnalysis.ts
@@ -33,7 +33,7 @@ import {
} from "./javascriptSemanticState.js";
import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";
import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js";
-import { collectSemanticMemberMutations } from "./javascriptSemanticMemberMutations.js";
+import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js";
import {
collectSemanticReturns,
resolveSemanticModuleCallables,
@@ -126,8 +126,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps(
const state = createState(file.program);
collectDefinitions(file.program, state);
yield;
- collectSemanticMemberMutations(file.program, state);
- yield;
+ yield* collectSemanticMemberMutationsSteps(file.program, state);
traverseJavaScriptAst(file.program, {
enter: (node) => collectSemanticModuleLink(node, state),
});
diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts
index 9aaa71d93..7ae453914 100644
--- a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts
+++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts
@@ -5,6 +5,10 @@ import {
analyzeParsedJavaScriptSemanticsSteps,
} from "./javascriptSemanticAnalysis.js";
import { parseJavaScriptSource } from "./javascriptSourceParser.js";
+import {
+ traverseJavaScriptAst,
+ traverseJavaScriptAstSteps,
+} from "./javascriptSemanticTraversal.js";
const source = `
import { readFile } from "node:fs";
@@ -32,3 +36,26 @@ it("yields between semantic phases and returns the synchronous result (#1462)",
if (reparsed === null) throw new Error("Expected parsed source");
expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));
});
+
+it("traverses a large tree in steps with the synchronous visit order (#1462)", () => {
+ const parsed = parseJavaScriptSource(
+ Array.from(
+ { length: 400 },
+ (_, index) => `const v${index} = [${index}];`,
+ ).join("\n"),
+ "large.js",
+ );
+ if (parsed === null) throw new Error("Expected parsed source");
+ const visits = (record: string[]) => ({
+ enter: (node: { readonly type: string }) => record.push(`+${node.type}`),
+ exit: (node: { readonly type: string }) => record.push(`-${node.type}`),
+ });
+ const synchronous: string[] = [];
+ traverseJavaScriptAst(parsed.program, visits(synchronous));
+ const stepped: string[] = [];
+ const steps = traverseJavaScriptAstSteps(parsed.program, visits(stepped));
+ let yields = 0;
+ while (steps.next().done !== true) yields += 1;
+ expect(yields).toBeGreaterThan(0);
+ expect(stepped).toEqual(synchronous);
+});
diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts
index 545652a74..b30d36859 100644
--- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts
+++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts
@@ -3,7 +3,7 @@ import type * as t from "@babel/types";
import { collectJavaScriptSemanticAsyncEffects } from "./javascriptSemanticAsyncEffects.js";
import { collectJavaScriptSemanticCalls } from "./javascriptSemanticCalls.js";
import { collectJavaScriptSemanticChildProcesses } from "./javascriptSemanticChildProcesses.js";
-import { collectJavaScriptSemanticFingerprints } from "./javascriptSemanticFingerprints.js";
+import { collectJavaScriptSemanticFingerprintsSteps } from "./javascriptSemanticFingerprints.js";
import { collectJavaScriptSemanticDataEffects } from "./javascriptSemanticDataEffects.js";
import type {
JavaScriptSemanticCallable,
@@ -84,19 +84,20 @@ export function* collectJavaScriptDerivedSemanticsSteps(
callables,
);
yield;
- const functionFingerprints = collectJavaScriptSemanticFingerprints({
- state,
- callables,
- calls,
- promises: promiseOperations,
- events: asyncEffects.eventOperations,
- timers: asyncEffects.timerOperations,
- childProcessSpawns: childProcesses.childProcessSpawns,
- childProcessInteractions: childProcesses.childProcessInteractions,
- requestOperations: dataEffects.requestOperations,
- resourceOperations,
- parserPartial,
- });
+ const functionFingerprints =
+ yield* collectJavaScriptSemanticFingerprintsSteps({
+ state,
+ callables,
+ calls,
+ promises: promiseOperations,
+ events: asyncEffects.eventOperations,
+ timers: asyncEffects.timerOperations,
+ childProcessSpawns: childProcesses.childProcessSpawns,
+ childProcessInteractions: childProcesses.childProcessInteractions,
+ requestOperations: dataEffects.requestOperations,
+ resourceOperations,
+ parserPartial,
+ });
return {
...calls,
promiseOperations,
diff --git a/src/domain/javascript/javascriptSemanticFingerprints.ts b/src/domain/javascript/javascriptSemanticFingerprints.ts
index 3f7309029..3d4b828be 100644
--- a/src/domain/javascript/javascriptSemanticFingerprints.ts
+++ b/src/domain/javascript/javascriptSemanticFingerprints.ts
@@ -17,7 +17,7 @@ import type {
} from "./javascriptSemanticIr.js";
import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js";
import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js";
-import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";
+import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js";
interface SemanticFingerprintCollectionInput {
readonly state: JavaScriptSemanticAnalysisState;
@@ -33,10 +33,10 @@ interface SemanticFingerprintCollectionInput {
readonly parserPartial: boolean;
}
-/** Build bounded rename- and formatting-resistant components per callable. */
-export const collectJavaScriptSemanticFingerprints = (
+/** Build bounded rename- and formatting-resistant components per callable, yielding between callables. */
+export function* collectJavaScriptSemanticFingerprintsSteps(
input: SemanticFingerprintCollectionInput,
-): JavaScriptSemanticFunctionFingerprint[] => {
+): Generator<void, JavaScriptSemanticFunctionFingerprint[]> {
const {
state,
callables,
@@ -67,11 +67,13 @@ export const collectJavaScriptSemanticFingerprints = (
);
const networkEffectsByOwner = groupByOwner(requestOperations, ownerId);
const resourceEffectsByOwner = groupByOwner(resourceOperations, ownerId);
- return callables.flatMap((callable) => {
- if (callable.kind === "class") return [];
+ const fingerprints: JavaScriptSemanticFunctionFingerprint[] = [];
+ for (const callable of callables) {
+ if (callable.kind === "class") continue;
const node = state.callableNodesById.get(callable.callableId);
- if (node === undefined) return [];
- const syntax = fingerprintSyntax(node, callable.callableId);
+ if (node === undefined) continue;
+ yield;
+ const syntax = yield* fingerprintSyntaxSteps(node, callable.callableId);
const promiseEffects = promiseEffectsByOwner.get(callable.callableId) ?? [];
const eventEffects = eventEffectsByOwner.get(callable.callableId) ?? [];
const timerEffects = timerEffectsByOwner.get(callable.callableId) ?? [];
@@ -80,50 +82,49 @@ export const collectJavaScriptSemanticFingerprints = (
const networkEffects = networkEffectsByOwner.get(callable.callableId) ?? [];
const resourceEffects =
resourceEffectsByOwner.get(callable.callableId) ?? [];
- return [
- {
- callableId: callable.callableId,
- status:
- parserPartial || callable.returnCoverage.status !== "complete"
- ? "partial"
- : "complete",
- components: {
- parameterArity: callableParameterArity(node),
- normalizedAstSha256: digest(syntax.normalizedTokens),
- controlFlowSha256: digest(syntax.controlFlowTokens),
- relationShapeSha256: digest(
- relationShape(
- callable,
- callSitesByCaller.get(callable.callableId) ?? [],
- captureCountByCallable.get(callable.callableId) ?? 0,
- promiseEffects,
- ),
+ fingerprints.push({
+ callableId: callable.callableId,
+ status:
+ parserPartial || callable.returnCoverage.status !== "complete"
+ ? "partial"
+ : "complete",
+ components: {
+ parameterArity: callableParameterArity(node),
+ normalizedAstSha256: digest(syntax.normalizedTokens),
+ controlFlowSha256: digest(syntax.controlFlowTokens),
+ relationShapeSha256: digest(
+ relationShape(
+ callable,
+ callSitesByCaller.get(callable.callableId) ?? [],
+ captureCountByCallable.get(callable.callableId) ?? 0,
+ promiseEffects,
),
- literalSetSha256: digest(
- [...new Set(syntax.literals)].sort(compareUnicodeCodePoints),
- ),
- effects: [
- ...(isAsyncCallable(node) ? (["async"] as const) : []),
- ...(childProcessEffects.length > 0
- ? (["child-process"] as const)
- : []),
- ...(eventEffects.length > 0 ? (["event"] as const) : []),
- ...(networkEffects.length > 0 ? (["network"] as const) : []),
- ...(promiseEffects.length > 0 ? (["promise"] as const) : []),
- ...(resourceEffects.length > 0 ? (["resource"] as const) : []),
- ...(timerEffects.length > 0 ? (["timer"] as const) : []),
- ],
- },
- limitations: [
- "The fingerprint is a static candidate and does not prove behavioral equivalence.",
- ...(parserPartial
- ? ["Incomplete semantic recovery makes this fingerprint partial."]
+ ),
+ literalSetSha256: digest(
+ [...new Set(syntax.literals)].sort(compareUnicodeCodePoints),
+ ),
+ effects: [
+ ...(isAsyncCallable(node) ? (["async"] as const) : []),
+ ...(childProcessEffects.length > 0
+ ? (["child-process"] as const)
: []),
+ ...(eventEffects.length > 0 ? (["event"] as const) : []),
+ ...(networkEffects.length > 0 ? (["network"] as const) : []),
+ ...(promiseEffects.length > 0 ? (["promise"] as const) : []),
+ ...(resourceEffects.length > 0 ? (["resource"] as const) : []),
+ ...(timerEffects.length > 0 ? (["timer"] as const) : []),
],
},
- ];
- });
-};
+ limitations: [
+ "The fingerprint is a static candidate and does not prove behavioral equivalence.",
+ ...(parserPartial
+ ? ["Incomplete semantic recovery makes this fingerprint partial."]
+ : []),
+ ],
+ });
+ }
+ return fingerprints;
+}
const ownerId = <T extends { readonly ownerCallableId: string | null }>(
value: T,
@@ -161,15 +162,15 @@ interface FingerprintSyntax {
readonly literals: readonly string[];
}
-const fingerprintSyntax = (
+function* fingerprintSyntaxSteps(
root: t.Node,
targetCallableId: string,
-): FingerprintSyntax => {
+): Generator<void, FingerprintSyntax> {
const normalizedTokens: string[] = [];
const controlFlowTokens: string[] = [];
const literals: string[] = [];
const callableStack: string[] = [];
- traverseJavaScriptAst(root, {
+ yield* traverseJavaScriptAstSteps(root, {
enter: (node, parent) => {
const callableId = semanticCallableIdForNode(node);
if (callableId !== null) callableStack.push(callableId);
@@ -187,7 +188,7 @@ const fingerprintSyntax = (
},
});
return { normalizedTokens, controlFlowTokens, literals };
-};
+}
const normalizedNodeToken = (node: t.Node, parent: t.Node | null): string => {
if (t.isIdentifier(node))
diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts
index 06a335bca..12ceec71d 100644
--- a/src/domain/javascript/javascriptSemanticMemberMutations.ts
+++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts
@@ -25,7 +25,7 @@ import {
type JavaScriptSemanticPropertyPath,
} from "./javascriptSemanticPropertyPaths.js";
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
-import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";
+import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js";
import {
semanticStaticPropertyKey,
unwrapJavaScriptExpression,
@@ -34,13 +34,13 @@ import {
type PropertyPath = JavaScriptSemanticPropertyPath;
type ValueEffect = "write" | "escape";
-/** Preserve uncertainty from property writes and references exposed to calls. */
-export const collectSemanticMemberMutations = (
+/** Preserve uncertainty from property writes and references exposed to calls, yielding during traversal and fixpoint iteration. */
+export function* collectSemanticMemberMutationsSteps(
program: t.Program,
state: JavaScriptSemanticAnalysisState,
-): void => {
+): Generator<void, void> {
const parents = state.parentsByNode;
- traverseJavaScriptAst(program, {
+ yield* traverseJavaScriptAstSteps(program, {
enter: (node, parent) => {
if (parent !== null) parents.set(node, parent);
},
@@ -382,7 +382,7 @@ export const collectSemanticMemberMutations = (
);
}
};
- traverseJavaScriptAst(program, {
+ yield* traverseJavaScriptAstSteps(program, {
enter: (node, parent) => {
for (const source of semanticIterationSources(node, parent))
deferIterable({
@@ -435,7 +435,8 @@ export const collectSemanticMemberMutations = (
const effectsBefore = recordedEffects.size;
const iterablesBefore = iterableReferences.length;
const iterationBefore: boolean = arrayIterationUnknown;
- for (const reference of pendingReferences.splice(0)) {
+ for (const [index, reference] of pendingReferences.splice(0).entries()) {
+ if (index % 64 === 0) yield;
const { initializer } = reference;
if (referenceSourcesInactive(initializer, state))
pendingReferences.push(reference);
@@ -480,7 +481,8 @@ export const collectSemanticMemberMutations = (
reference.originAt,
);
}
- for (const reference of [...iterableReferences]) {
+ for (const [index, reference] of [...iterableReferences].entries()) {
+ if (index % 64 === 0) yield;
if (referenceSourcesInactive(reference, state)) continue;
if (reference.iterationOnly) {
const expanded = expandedIterationReferences(reference, state);
@@ -518,7 +520,7 @@ export const collectSemanticMemberMutations = (
// Gathering effects can evaluate a primitive projection before its object
// escapes. Final values must use the completed mutation state.
clearSemanticPrimitiveBindingValues(state);
-};
+}
interface ReferencedValue {
readonly node: t.Node;
diff --git a/src/domain/javascript/javascriptSemanticTraversal.ts b/src/domain/javascript/javascriptSemanticTraversal.ts
index 9b562a94c..16960ba8f 100644
--- a/src/domain/javascript/javascriptSemanticTraversal.ts
+++ b/src/domain/javascript/javascriptSemanticTraversal.ts
@@ -12,15 +12,28 @@ import * as t from "@babel/types";
*/
export const traverseJavaScriptAst = (
root: t.Node,
- visitor: {
- readonly enter: (
- node: t.Node,
- parent: t.Node | null,
- readAncestors: () => readonly t.Node[],
- ) => void;
- readonly exit?: (node: t.Node, parent: t.Node | null) => void;
- },
-): void => {
+ visitor: JavaScriptAstVisitor,
+): void => completeSemanticSteps(traverseJavaScriptAstSteps(root, visitor));
+
+/** Callbacks for one deterministic AST traversal. */
+export interface JavaScriptAstVisitor {
+ readonly enter: (
+ node: t.Node,
+ parent: t.Node | null,
+ readAncestors: () => readonly t.Node[],
+ ) => void;
+ readonly exit?: (node: t.Node, parent: t.Node | null) => void;
+}
+
+/**
+ * {@link traverseJavaScriptAst} with a yield every 1,024 visited nodes, so a
+ * caller can serve control messages during one large traversal. Visit order
+ * and callbacks are identical.
+ */
+export function* traverseJavaScriptAstSteps(
+ root: t.Node,
+ visitor: JavaScriptAstVisitor,
+): Generator<void, void> {
const ancestors: t.Node[] = [];
const enter = (node: t.Node, parent: t.Node | null): TraversalFrame => {
visitor.enter(node, parent, () => [...ancestors]);
@@ -28,12 +41,15 @@ export const traverseJavaScriptAst = (
return { node, parent, children: childNodes(node), nextIndex: 0 };
};
const pending = [enter(root, null)];
+ let visited = 1;
while (pending.length > 0) {
const current = pending.at(-1);
if (current === undefined) break;
const child = current.children[current.nextIndex];
if (child !== undefined) {
current.nextIndex += 1;
+ if (visited % 1024 === 0) yield;
+ visited += 1;
pending.push(enter(child, current.node));
} else {
visitor.exit?.(current.node, current.parent);
@@ -41,6 +57,16 @@ export const traverseJavaScriptAst = (
pending.pop();
}
}
+}
+
+/** Run cooperative analysis steps to completion without yielding control. */
+export const completeSemanticSteps = <Value>(
+ steps: Iterator<void, Value>,
+): Value => {
+ for (;;) {
+ const step = steps.next();
+ if (step.done === true) return step.value;
+ }
};
interface TraversalFrame {
diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts
index 524af811e..264c94182 100644
--- a/src/domain/javascript/javascriptStaticAnalysis.ts
+++ b/src/domain/javascript/javascriptStaticAnalysis.ts
@@ -1,6 +1,9 @@
import * as t from "@babel/types";
-import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";
+import {
+ completeSemanticSteps,
+ traverseJavaScriptAstSteps,
+} from "./javascriptSemanticTraversal.js";
import { inspectElectronStaticNode } from "./electronStaticAnalysis.js";
import {
@@ -57,9 +60,19 @@ export const analyzeParsedJavaScriptStaticSource = (
source: string,
file: ParsedJavaScriptSource,
openReceiverFacts?: ReadonlyMap<number, JavaScriptOpenReceiverFact>,
-): JavaScriptStaticAnalysis => {
+): JavaScriptStaticAnalysis =>
+ completeSemanticSteps(
+ analyzeParsedJavaScriptStaticSourceSteps(source, file, openReceiverFacts),
+ );
+
+/** Static analysis that yields during its source traversal; results are unchanged. */
+export function* analyzeParsedJavaScriptStaticSourceSteps(
+ source: string,
+ file: ParsedJavaScriptSource,
+ openReceiverFacts?: ReadonlyMap<number, JavaScriptOpenReceiverFact>,
+): Generator<void, JavaScriptStaticAnalysis> {
const accumulator = createJavaScriptAnalysisAccumulator(source.length);
- traverseStaticSource(file, {
+ yield* traverseStaticSourceSteps(file, {
source,
accumulator,
...(openReceiverFacts === undefined ? {} : { openReceiverFacts }),
@@ -67,20 +80,20 @@ export const analyzeParsedJavaScriptStaticSource = (
});
addSourceMapDirectives(source, file.comments ?? [], accumulator);
return finalizeStaticAnalysis(source, file, accumulator);
-};
+}
-const traverseStaticSource = (
+function* traverseStaticSourceSteps(
file: ParsedJavaScriptSource,
findings: JavaScriptFindingContext,
-): void => {
- traverseJavaScriptAst(file, {
+): Generator<void, void> {
+ yield* traverseJavaScriptAstSteps(file, {
enter: (node) => {
findings.accumulator.visitedNodes += 1;
inspectNode(node, findings);
return undefined;
},
});
-};
+}
const finalizeStaticAnalysis = (
source: string,