Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): yield during the largest AST traversals (#1545)
Kaoru08225d6eeb22fb3b
diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.tsindex 72fb67edd..fcba4cc16 100644--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";import { setImmediate } from "node:timers/promises";import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";-import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";+import { analyzeParsedJavaScriptStaticSourceSteps } from "../../domain/javascript/javascriptStaticAnalysis.js";import {analyzeParsedJavaScriptSemanticsSteps,classifyParsedJavaScriptOpenReceivers,@@ -190,7 +190,7 @@ function* analyzeArtifactFileSteps<}const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);yield;- const analysis = analyzeParsedJavaScriptStaticSource(+ const analysis = yield* analyzeParsedJavaScriptStaticSourceSteps(file.text.value,parsed,openReceiverFacts,diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.tsindex 50af0f2cb..ff1dac49b 100644--- a/src/domain/javascript/javascriptSemanticAnalysis.ts+++ b/src/domain/javascript/javascriptSemanticAnalysis.ts@@ -33,7 +33,7 @@ import {} from "./javascriptSemanticState.js";import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js";-import { collectSemanticMemberMutations } from "./javascriptSemanticMemberMutations.js";+import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js";import {collectSemanticReturns,resolveSemanticModuleCallables,@@ -126,8 +126,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps(const state = createState(file.program);collectDefinitions(file.program, state);yield;- collectSemanticMemberMutations(file.program, state);- yield;+ yield* collectSemanticMemberMutationsSteps(file.program, state);traverseJavaScriptAst(file.program, {enter: (node) => collectSemanticModuleLink(node, state),});diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.tsindex 9aaa71d93..7ae453914 100644--- a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts+++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts@@ -5,6 +5,10 @@ import {analyzeParsedJavaScriptSemanticsSteps,} from "./javascriptSemanticAnalysis.js";import { parseJavaScriptSource } from "./javascriptSourceParser.js";+import {+ traverseJavaScriptAst,+ traverseJavaScriptAstSteps,+} from "./javascriptSemanticTraversal.js";const source = `import { readFile } from "node:fs";@@ -32,3 +36,26 @@ it("yields between semantic phases and returns the synchronous result (#1462)",if (reparsed === null) throw new Error("Expected parsed source");expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));});++it("traverses a large tree in steps with the synchronous visit order (#1462)", () => {+ const parsed = parseJavaScriptSource(+ Array.from(+ { length: 400 },+ (_, index) => `const v${index} = [${index}];`,+ ).join("\n"),+ "large.js",+ );+ if (parsed === null) throw new Error("Expected parsed source");+ const visits = (record: string[]) => ({+ enter: (node: { readonly type: string }) => record.push(`+${node.type}`),+ exit: (node: { readonly type: string }) => record.push(`-${node.type}`),+ });+ const synchronous: string[] = [];+ traverseJavaScriptAst(parsed.program, visits(synchronous));+ const stepped: string[] = [];+ const steps = traverseJavaScriptAstSteps(parsed.program, visits(stepped));+ let yields = 0;+ while (steps.next().done !== true) yields += 1;+ expect(yields).toBeGreaterThan(0);+ expect(stepped).toEqual(synchronous);+});diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.tsindex 545652a74..b30d36859 100644--- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts+++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts@@ -3,7 +3,7 @@ import type * as t from "@babel/types";import { collectJavaScriptSemanticAsyncEffects } from "./javascriptSemanticAsyncEffects.js";import { collectJavaScriptSemanticCalls } from "./javascriptSemanticCalls.js";import { collectJavaScriptSemanticChildProcesses } from "./javascriptSemanticChildProcesses.js";-import { collectJavaScriptSemanticFingerprints } from "./javascriptSemanticFingerprints.js";+import { collectJavaScriptSemanticFingerprintsSteps } from "./javascriptSemanticFingerprints.js";import { collectJavaScriptSemanticDataEffects } from "./javascriptSemanticDataEffects.js";import type {JavaScriptSemanticCallable,@@ -84,19 +84,20 @@ export function* collectJavaScriptDerivedSemanticsSteps(callables,);yield;- const functionFingerprints = collectJavaScriptSemanticFingerprints({- state,- callables,- calls,- promises: promiseOperations,- events: asyncEffects.eventOperations,- timers: asyncEffects.timerOperations,- childProcessSpawns: childProcesses.childProcessSpawns,- childProcessInteractions: childProcesses.childProcessInteractions,- requestOperations: dataEffects.requestOperations,- resourceOperations,- parserPartial,- });+ const functionFingerprints =+ yield* collectJavaScriptSemanticFingerprintsSteps({+ state,+ callables,+ calls,+ promises: promiseOperations,+ events: asyncEffects.eventOperations,+ timers: asyncEffects.timerOperations,+ childProcessSpawns: childProcesses.childProcessSpawns,+ childProcessInteractions: childProcesses.childProcessInteractions,+ requestOperations: dataEffects.requestOperations,+ resourceOperations,+ parserPartial,+ });return {...calls,promiseOperations,diff --git a/src/domain/javascript/javascriptSemanticFingerprints.ts b/src/domain/javascript/javascriptSemanticFingerprints.tsindex 3f7309029..3d4b828be 100644--- a/src/domain/javascript/javascriptSemanticFingerprints.ts+++ b/src/domain/javascript/javascriptSemanticFingerprints.ts@@ -17,7 +17,7 @@ import type {} from "./javascriptSemanticIr.js";import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js";import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js";-import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";+import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js";interface SemanticFingerprintCollectionInput {readonly state: JavaScriptSemanticAnalysisState;@@ -33,10 +33,10 @@ interface SemanticFingerprintCollectionInput {readonly parserPartial: boolean;}-/** Build bounded rename- and formatting-resistant components per callable. */-export const collectJavaScriptSemanticFingerprints = (+/** Build bounded rename- and formatting-resistant components per callable, yielding between callables. */+export function* collectJavaScriptSemanticFingerprintsSteps(input: SemanticFingerprintCollectionInput,-): JavaScriptSemanticFunctionFingerprint[] => {+): Generator<void, JavaScriptSemanticFunctionFingerprint[]> {const {state,callables,@@ -67,11 +67,13 @@ export const collectJavaScriptSemanticFingerprints = ();const networkEffectsByOwner = groupByOwner(requestOperations, ownerId);const resourceEffectsByOwner = groupByOwner(resourceOperations, ownerId);- return callables.flatMap((callable) => {- if (callable.kind === "class") return [];+ const fingerprints: JavaScriptSemanticFunctionFingerprint[] = [];+ for (const callable of callables) {+ if (callable.kind === "class") continue;const node = state.callableNodesById.get(callable.callableId);- if (node === undefined) return [];- const syntax = fingerprintSyntax(node, callable.callableId);+ if (node === undefined) continue;+ yield;+ const syntax = yield* fingerprintSyntaxSteps(node, callable.callableId);const promiseEffects = promiseEffectsByOwner.get(callable.callableId) ?? [];const eventEffects = eventEffectsByOwner.get(callable.callableId) ?? [];const timerEffects = timerEffectsByOwner.get(callable.callableId) ?? [];@@ -80,50 +82,49 @@ export const collectJavaScriptSemanticFingerprints = (const networkEffects = networkEffectsByOwner.get(callable.callableId) ?? [];const resourceEffects =resourceEffectsByOwner.get(callable.callableId) ?? [];- return [- {- callableId: callable.callableId,- status:- parserPartial || callable.returnCoverage.status !== "complete"- ? "partial"- : "complete",- components: {- parameterArity: callableParameterArity(node),- normalizedAstSha256: digest(syntax.normalizedTokens),- controlFlowSha256: digest(syntax.controlFlowTokens),- relationShapeSha256: digest(- relationShape(- callable,- callSitesByCaller.get(callable.callableId) ?? [],- captureCountByCallable.get(callable.callableId) ?? 0,- promiseEffects,- ),+ fingerprints.push({+ callableId: callable.callableId,+ status:+ parserPartial || callable.returnCoverage.status !== "complete"+ ? "partial"+ : "complete",+ components: {+ parameterArity: callableParameterArity(node),+ normalizedAstSha256: digest(syntax.normalizedTokens),+ controlFlowSha256: digest(syntax.controlFlowTokens),+ relationShapeSha256: digest(+ relationShape(+ callable,+ callSitesByCaller.get(callable.callableId) ?? [],+ captureCountByCallable.get(callable.callableId) ?? 0,+ promiseEffects,),- literalSetSha256: digest(- [...new Set(syntax.literals)].sort(compareUnicodeCodePoints),- ),- effects: [- ...(isAsyncCallable(node) ? (["async"] as const) : []),- ...(childProcessEffects.length > 0- ? (["child-process"] as const)- : []),- ...(eventEffects.length > 0 ? (["event"] as const) : []),- ...(networkEffects.length > 0 ? (["network"] as const) : []),- ...(promiseEffects.length > 0 ? (["promise"] as const) : []),- ...(resourceEffects.length > 0 ? (["resource"] as const) : []),- ...(timerEffects.length > 0 ? (["timer"] as const) : []),- ],- },- limitations: [- "The fingerprint is a static candidate and does not prove behavioral equivalence.",- ...(parserPartial- ? ["Incomplete semantic recovery makes this fingerprint partial."]+ ),+ literalSetSha256: digest(+ [...new Set(syntax.literals)].sort(compareUnicodeCodePoints),+ ),+ effects: [+ ...(isAsyncCallable(node) ? (["async"] as const) : []),+ ...(childProcessEffects.length > 0+ ? (["child-process"] as const): []),+ ...(eventEffects.length > 0 ? (["event"] as const) : []),+ ...(networkEffects.length > 0 ? (["network"] as const) : []),+ ...(promiseEffects.length > 0 ? (["promise"] as const) : []),+ ...(resourceEffects.length > 0 ? (["resource"] as const) : []),+ ...(timerEffects.length > 0 ? (["timer"] as const) : []),],},- ];- });-};+ limitations: [+ "The fingerprint is a static candidate and does not prove behavioral equivalence.",+ ...(parserPartial+ ? ["Incomplete semantic recovery makes this fingerprint partial."]+ : []),+ ],+ });+ }+ return fingerprints;+}const ownerId = <T extends { readonly ownerCallableId: string | null }>(value: T,@@ -161,15 +162,15 @@ interface FingerprintSyntax {readonly literals: readonly string[];}-const fingerprintSyntax = (+function* fingerprintSyntaxSteps(root: t.Node,targetCallableId: string,-): FingerprintSyntax => {+): Generator<void, FingerprintSyntax> {const normalizedTokens: string[] = [];const controlFlowTokens: string[] = [];const literals: string[] = [];const callableStack: string[] = [];- traverseJavaScriptAst(root, {+ yield* traverseJavaScriptAstSteps(root, {enter: (node, parent) => {const callableId = semanticCallableIdForNode(node);if (callableId !== null) callableStack.push(callableId);@@ -187,7 +188,7 @@ const fingerprintSyntax = (},});return { normalizedTokens, controlFlowTokens, literals };-};+}const normalizedNodeToken = (node: t.Node, parent: t.Node | null): string => {if (t.isIdentifier(node))diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.tsindex 06a335bca..12ceec71d 100644--- a/src/domain/javascript/javascriptSemanticMemberMutations.ts+++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts@@ -25,7 +25,7 @@ import {type JavaScriptSemanticPropertyPath,} from "./javascriptSemanticPropertyPaths.js";import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";-import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";+import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js";import {semanticStaticPropertyKey,unwrapJavaScriptExpression,@@ -34,13 +34,13 @@ import {type PropertyPath = JavaScriptSemanticPropertyPath;type ValueEffect = "write" | "escape";-/** Preserve uncertainty from property writes and references exposed to calls. */-export const collectSemanticMemberMutations = (+/** Preserve uncertainty from property writes and references exposed to calls, yielding during traversal and fixpoint iteration. */+export function* collectSemanticMemberMutationsSteps(program: t.Program,state: JavaScriptSemanticAnalysisState,-): void => {+): Generator<void, void> {const parents = state.parentsByNode;- traverseJavaScriptAst(program, {+ yield* traverseJavaScriptAstSteps(program, {enter: (node, parent) => {if (parent !== null) parents.set(node, parent);},@@ -382,7 +382,7 @@ export const collectSemanticMemberMutations = ();}};- traverseJavaScriptAst(program, {+ yield* traverseJavaScriptAstSteps(program, {enter: (node, parent) => {for (const source of semanticIterationSources(node, parent))deferIterable({@@ -435,7 +435,8 @@ export const collectSemanticMemberMutations = (const effectsBefore = recordedEffects.size;const iterablesBefore = iterableReferences.length;const iterationBefore: boolean = arrayIterationUnknown;- for (const reference of pendingReferences.splice(0)) {+ for (const [index, reference] of pendingReferences.splice(0).entries()) {+ if (index % 64 === 0) yield;const { initializer } = reference;if (referenceSourcesInactive(initializer, state))pendingReferences.push(reference);@@ -480,7 +481,8 @@ export const collectSemanticMemberMutations = (reference.originAt,);}- for (const reference of [...iterableReferences]) {+ for (const [index, reference] of [...iterableReferences].entries()) {+ if (index % 64 === 0) yield;if (referenceSourcesInactive(reference, state)) continue;if (reference.iterationOnly) {const expanded = expandedIterationReferences(reference, state);@@ -518,7 +520,7 @@ export const collectSemanticMemberMutations = (// Gathering effects can evaluate a primitive projection before its object// escapes. Final values must use the completed mutation state.clearSemanticPrimitiveBindingValues(state);-};+}interface ReferencedValue {readonly node: t.Node;diff --git a/src/domain/javascript/javascriptSemanticTraversal.ts b/src/domain/javascript/javascriptSemanticTraversal.tsindex 9b562a94c..16960ba8f 100644--- a/src/domain/javascript/javascriptSemanticTraversal.ts+++ b/src/domain/javascript/javascriptSemanticTraversal.ts@@ -12,15 +12,28 @@ import * as t from "@babel/types";*/export const traverseJavaScriptAst = (root: t.Node,- visitor: {- readonly enter: (- node: t.Node,- parent: t.Node | null,- readAncestors: () => readonly t.Node[],- ) => void;- readonly exit?: (node: t.Node, parent: t.Node | null) => void;- },-): void => {+ visitor: JavaScriptAstVisitor,+): void => completeSemanticSteps(traverseJavaScriptAstSteps(root, visitor));++/** Callbacks for one deterministic AST traversal. */+export interface JavaScriptAstVisitor {+ readonly enter: (+ node: t.Node,+ parent: t.Node | null,+ readAncestors: () => readonly t.Node[],+ ) => void;+ readonly exit?: (node: t.Node, parent: t.Node | null) => void;+}++/**+ * {@link traverseJavaScriptAst} with a yield every 1,024 visited nodes, so a+ * caller can serve control messages during one large traversal. Visit order+ * and callbacks are identical.+ */+export function* traverseJavaScriptAstSteps(+ root: t.Node,+ visitor: JavaScriptAstVisitor,+): Generator<void, void> {const ancestors: t.Node[] = [];const enter = (node: t.Node, parent: t.Node | null): TraversalFrame => {visitor.enter(node, parent, () => [...ancestors]);@@ -28,12 +41,15 @@ export const traverseJavaScriptAst = (return { node, parent, children: childNodes(node), nextIndex: 0 };};const pending = [enter(root, null)];+ let visited = 1;while (pending.length > 0) {const current = pending.at(-1);if (current === undefined) break;const child = current.children[current.nextIndex];if (child !== undefined) {current.nextIndex += 1;+ if (visited % 1024 === 0) yield;+ visited += 1;pending.push(enter(child, current.node));} else {visitor.exit?.(current.node, current.parent);@@ -41,6 +57,16 @@ export const traverseJavaScriptAst = (pending.pop();}}+}++/** Run cooperative analysis steps to completion without yielding control. */+export const completeSemanticSteps = <Value>(+ steps: Iterator<void, Value>,+): Value => {+ for (;;) {+ const step = steps.next();+ if (step.done === true) return step.value;+ }};interface TraversalFrame {diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.tsindex 524af811e..264c94182 100644--- a/src/domain/javascript/javascriptStaticAnalysis.ts+++ b/src/domain/javascript/javascriptStaticAnalysis.ts@@ -1,6 +1,9 @@import * as t from "@babel/types";-import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";+import {+ completeSemanticSteps,+ traverseJavaScriptAstSteps,+} from "./javascriptSemanticTraversal.js";import { inspectElectronStaticNode } from "./electronStaticAnalysis.js";import {@@ -57,9 +60,19 @@ export const analyzeParsedJavaScriptStaticSource = (source: string,file: ParsedJavaScriptSource,openReceiverFacts?: ReadonlyMap<number, JavaScriptOpenReceiverFact>,-): JavaScriptStaticAnalysis => {+): JavaScriptStaticAnalysis =>+ completeSemanticSteps(+ analyzeParsedJavaScriptStaticSourceSteps(source, file, openReceiverFacts),+ );++/** Static analysis that yields during its source traversal; results are unchanged. */+export function* analyzeParsedJavaScriptStaticSourceSteps(+ source: string,+ file: ParsedJavaScriptSource,+ openReceiverFacts?: ReadonlyMap<number, JavaScriptOpenReceiverFact>,+): Generator<void, JavaScriptStaticAnalysis> {const accumulator = createJavaScriptAnalysisAccumulator(source.length);- traverseStaticSource(file, {+ yield* traverseStaticSourceSteps(file, {source,accumulator,...(openReceiverFacts === undefined ? {} : { openReceiverFacts }),@@ -67,20 +80,20 @@ export const analyzeParsedJavaScriptStaticSource = (});addSourceMapDirectives(source, file.comments ?? [], accumulator);return finalizeStaticAnalysis(source, file, accumulator);-};+}-const traverseStaticSource = (+function* traverseStaticSourceSteps(file: ParsedJavaScriptSource,findings: JavaScriptFindingContext,-): void => {- traverseJavaScriptAst(file, {+): Generator<void, void> {+ yield* traverseJavaScriptAstSteps(file, {enter: (node) => {findings.accumulator.visitedNodes += 1;inspectNode(node, findings);return undefined;},});-};+}const finalizeStaticAnalysis = (source: string,