Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): yield inside semantic analysis and large projection passes (#1542)
Kaoru0822ca7ef69c0cee
diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.tsindex 3fac39dbb..72fb67edd 100644--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts@@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises";import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";import {- analyzeParsedJavaScriptSemantics,+ analyzeParsedJavaScriptSemanticsSteps,classifyParsedJavaScriptOpenReceivers,} from "../../domain/javascript/javascriptSemanticAnalysis.js";import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";@@ -189,6 +189,7 @@ function* analyzeArtifactFileSteps<return;}const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);+ yield;const analysis = analyzeParsedJavaScriptStaticSource(file.text.value,parsed,@@ -198,7 +199,7 @@ function* analyzeArtifactFileSteps<yield;const semantics =analysis.parse_status === "complete" || analysis.parse_status === "partial"- ? analyzeParsedJavaScriptSemantics(parsed)+ ? yield* analyzeParsedJavaScriptSemanticsSteps(parsed): null;const projected =semantics === nulldiff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.tsindex 770930de7..a0cc8ccf8 100644--- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts+++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts@@ -287,38 +287,50 @@ function* projectFileSteps(});if (moduleNode === null) return [];state.roots.add(moduleNode.node_id);- const callableNodes = new Map(- ir.callables.flatMap((callable) => {- const node = retainNode(state, file, {- kind: "function",- roleKey: `callable:${callable.callableId}`,- location: callable.location,- label: callable.name,- functionNodeId: null,- // The label is display text; keep the exact name, which may be "".- properties: { name: callable.name },- });- return node === null ? [] : [[callable.callableId, node] as const];- }),- );+ // Node creation keeps source order, which the per-file node budget relies+ // on; yields between nodes do not change it.+ const callableNodes = new Map<string, JavaScriptSemanticGraphNode>();+ for (const [index, callable] of ir.callables.entries()) {+ if (index % 64 === 0) yield;+ const node = retainNode(state, file, {+ kind: "function",+ roleKey: `callable:${callable.callableId}`,+ location: callable.location,+ label: callable.name,+ functionNodeId: null,+ // The label is display text; keep the exact name, which may be "".+ properties: { name: callable.name },+ });+ if (node !== null) callableNodes.set(callable.callableId, node);+ }const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes);- const bindingNodes = new Map(- ir.bindings.flatMap((binding) => {- const location = binding.definitions[0]?.location ?? null;- const kind = binding.kind === "parameter" ? "parameter" : "binding";- const owner = callableOwnerAt(location);- const node = retainNode(state, file, {- kind,- roleKey: `binding:${binding.bindingId}`,- location,- label: binding.name,- functionNodeId: owner?.node_id ?? null,- });- return node === null ? [] : [[binding.bindingId, node] as const];- }),+ const bindingNodes = new Map<string, JavaScriptSemanticGraphNode>();+ for (const [index, binding] of ir.bindings.entries()) {+ if (index % 64 === 0) yield;+ const location = binding.definitions[0]?.location ?? null;+ const kind = binding.kind === "parameter" ? "parameter" : "binding";+ const owner = callableOwnerAt(location);+ const node = retainNode(state, file, {+ kind,+ roleKey: `binding:${binding.bindingId}`,+ location,+ label: binding.name,+ functionNodeId: owner?.node_id ?? null,+ });+ if (node !== null) bindingNodes.set(binding.bindingId, node);+ }+ const returnSiteNodes = yield* createReturnSiteNodes(+ file,+ ir,+ callableNodes,+ state,+ );+ const callSiteNodes = yield* createCallSiteNodes(+ file,+ ir,+ callableNodes,+ state,);- const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state);- const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state);const definitions: Omit<FileContext, "referenceNodesWithin"> = {file,ir,@@ -339,8 +351,7 @@ function* projectFileSteps(callableOwnerAt,callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),};- yield;- projectDefinitionsAndReferences(definitions);+ yield* projectDefinitionsAndReferences(definitions);const context: FileContext = {...definitions,referenceNodesWithin: createSemanticNodeRangeLookup(@@ -351,7 +362,8 @@ function* projectFileSteps(// keeps projection order and results unchanged.for (const project of FILE_PROJECTION_PASSES) {yield;- project(context);+ const steps = project(context);+ if (steps !== undefined) while (steps.next().done !== true) yield;}yield;return projectSemanticFunctionFingerprints(@@ -362,14 +374,15 @@ function* projectFileSteps();}-const createReturnSiteNodes = (+function* createReturnSiteNodes(file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,state: BuilderState,-): Map<string, JavaScriptSemanticGraphNode> => {+): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {const result = new Map<string, JavaScriptSemanticGraphNode>();- for (const callable of ir.callables) {+ for (const [index, callable] of ir.callables.entries()) {+ if (index % 64 === 0) yield;const owner = callables.get(callable.callableId);if (owner === undefined) continue;for (const site of callable.returnSites) {@@ -385,16 +398,17 @@ const createReturnSiteNodes = (}}return result;-};+}-const createCallSiteNodes = (+function* createCallSiteNodes(file: JavaScriptArtifactFile,ir: JavaScriptSemanticIr,callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,state: BuilderState,-): Map<string, JavaScriptSemanticGraphNode> => {+): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {const result = new Map<string, JavaScriptSemanticGraphNode>();- for (const call of ir.callSites) {+ for (const [index, call] of ir.callSites.entries()) {+ if (index % 64 === 0) yield;const owner =call.callerCallableId === null? null@@ -409,12 +423,13 @@ const createCallSiteNodes = (if (node !== null) result.set(call.callSiteId, node);}return result;-};+}-const projectDefinitionsAndReferences = (+function* projectDefinitionsAndReferences(context: Omit<FileContext, "referenceNodesWithin">,-): void => {- for (const binding of context.ir.bindings) {+): Generator<void, void> {+ for (const [bindingIndex, binding] of context.ir.bindings.entries()) {+ if (bindingIndex % 64 === 0) yield;const bindingNode = context.bindingNodes.get(binding.bindingId);if (bindingNode === undefined) continue;for (const [index, definition] of binding.definitions.entries()) {@@ -434,6 +449,7 @@ const projectDefinitionsAndReferences = (}}for (const [index, reference] of context.ir.references.entries()) {+ if (index % 64 === 0) yield;const expression = retainNode(context.state, context.file, {kind: "expression",roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`,@@ -464,7 +480,7 @@ const projectDefinitionsAndReferences = (resolution: "resolved",});}-};+}const projectCalls = (context: FileContext): void => {for (const call of context.ir.callSites) {@@ -551,7 +567,9 @@ const createArgumentNode = (});/** Per-file projection passes, run in this order with a yield between each. */-const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [+const FILE_PROJECTION_PASSES: readonly ((+ context: FileContext,+) => Iterator<void, void> | void)[] = [projectSemanticValues,projectSemanticObjects,projectCalls,diff --git a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.tsindex be801ab86..e98526160 100644--- a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts+++ b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts@@ -240,10 +240,11 @@ export const projectSemanticClosureCaptures = (};/** Retain bounded unresolved dynamic-call, dynamic-property and dynamic-scope frontiers. */-export const projectSemanticFrontiers = (+export function* projectSemanticFrontiers(context: SemanticFlowProjectionContext,-): void => {- for (const frontier of context.ir.frontiers) {+): Generator<void, void> {+ for (const [index, frontier] of context.ir.frontiers.entries()) {+ if (index % 64 === 0) yield;// Dynamic environments leave identifier reads and writes unresolved.const family =frontier.kind === "dynamic-call"@@ -275,4 +276,4 @@ export const projectSemanticFrontiers = ();addSemanticGraphUnknown(context.state, unknown);}-};+}diff --git a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.tsindex 1b71b2784..337899f83 100644--- a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts+++ b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts@@ -20,13 +20,14 @@ import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFloimport { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js";/** Project static object reads, writes, spreads, and destructuring. */-export const projectSemanticObjects = (+export function* projectSemanticObjects(context: SemanticFlowProjectionContext,-): void => {+): Generator<void, void> {const values = new Map(context.ir.bindings.map((binding) => [binding.bindingId, binding.value]),);- for (const operation of context.ir.objectOperations) {+ for (const [index, operation] of context.ir.objectOperations.entries()) {+ if (index % 64 === 0) yield;const occurrence = addObjectOccurrence(context, operation);const target = objectTarget(context, values, operation);const ends = objectRelationEnds(context, operation, occurrence, target);@@ -43,7 +44,7 @@ export const projectSemanticObjects = (if (!resolved)addObjectUnknown(context, operation, target.slot ?? occurrence, target);}-};+}const OBJECT_RELATIONS = {read: "reads-property",diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.tsindex 4b5ad0f72..50af0f2cb 100644--- a/src/domain/javascript/javascriptSemanticAnalysis.ts+++ b/src/domain/javascript/javascriptSemanticAnalysis.ts@@ -38,7 +38,7 @@ import {collectSemanticReturns,resolveSemanticModuleCallables,} from "./javascriptSemanticReturns.js";-import { collectJavaScriptDerivedSemantics } from "./javascriptSemanticDerivedAnalysis.js";+import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js";import { range } from "./javascriptStaticAnalysisHelpers.js";import { propertyName } from "./javascriptAstValues.js";import { semanticCoverage } from "./javascriptSemanticCoverage.js";@@ -107,18 +107,40 @@ export const analyzeParsedJavaScriptReferences = (export const analyzeParsedJavaScriptSemantics = (file: ParsedJavaScriptSource,): JavaScriptSemanticIr => {+ const steps = analyzeParsedJavaScriptSemanticsSteps(file);+ for (;;) {+ const step = steps.next();+ if (step.done === true) return step.value;+ }+};++/**+ * Recover semantics from a parsed artifact with a yield between analysis+ * phases, so a caller can serve control messages within one large file.+ * Each phase completes before the next starts; results equal the synchronous+ * form.+ */+export function* analyzeParsedJavaScriptSemanticsSteps(+ file: ParsedJavaScriptSource,+): Generator<void, JavaScriptSemanticIr> {const state = createState(file.program);collectDefinitions(file.program, state);+ yield;collectSemanticMemberMutations(file.program, state);+ yield;traverseJavaScriptAst(file.program, {enter: (node) => collectSemanticModuleLink(node, state),});+ yield;const references = collectSemanticReferences(file.program, state);+ yield;const parserPartial = file.errors.length > 0;const bindings = immutableSemanticBindings(state);+ yield;const callables = collectSemanticReturns(file.program, state, parserPartial);+ yield;const moduleLinks = resolveSemanticModuleCallables(state, callables);- const derived = collectJavaScriptDerivedSemantics(+ const derived = yield* collectJavaScriptDerivedSemanticsSteps(file.program,state,callables,@@ -154,7 +176,7 @@ export const analyzeParsedJavaScriptSemantics = ("Cross-function mutation and dynamic property resolution remain unknown.",],};-};+}/** Receiver identity facts for the overloaded `.open` syntax only. */export type JavaScriptOpenReceiverFact =diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.tsnew file mode 100644index 000000000..9aaa71d93--- /dev/null+++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts@@ -0,0 +1,34 @@+import { expect, it } from "vitest";++import {+ analyzeParsedJavaScriptSemantics,+ analyzeParsedJavaScriptSemanticsSteps,+} from "./javascriptSemanticAnalysis.js";+import { parseJavaScriptSource } from "./javascriptSourceParser.js";++const source = `+import { readFile } from "node:fs";+const shared = { token: "TOKEN" };+export function load(path) {+ return new Promise((resolve) => setTimeout(() => resolve(readFile(path)), 1));+}+export const box = { get() { return shared; } };+box.get().value = process.env.MODE ?? "default";+`;++it("yields between semantic phases and returns the synchronous result (#1462)", () => {+ const parsed = parseJavaScriptSource(source, "main.js");+ if (parsed === null) throw new Error("Expected parsed source");+ const steps = analyzeParsedJavaScriptSemanticsSteps(parsed);+ let yields = 0;+ let step = steps.next();+ while (step.done !== true) {+ yields += 1;+ step = steps.next();+ }+ // Every top-level phase and derived collector is a separate step.+ expect(yields).toBeGreaterThanOrEqual(13);+ const reparsed = parseJavaScriptSource(source, "main.js");+ if (reparsed === null) throw new Error("Expected parsed source");+ expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));+});diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.tsindex d3e630ac9..545652a74 100644--- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts+++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts@@ -36,43 +36,54 @@ type DerivedSemanticAnalysis = Pick<>;/** Compose bounded semantic passes after lexical definitions are available. */-export const collectJavaScriptDerivedSemantics = (+/**+ * Collect derived semantics with a yield between collectors. Each collector+ * completes before the next starts, so results equal the synchronous form.+ */+export function* collectJavaScriptDerivedSemanticsSteps(program: t.Program,state: JavaScriptSemanticAnalysisState,callables: readonly JavaScriptSemanticCallable[],parserPartial: boolean,-): DerivedSemanticAnalysis => {+): Generator<void, DerivedSemanticAnalysis> {const calls = collectJavaScriptSemanticCalls(program, state, callables);+ yield;const promiseOperations = collectJavaScriptSemanticPromises(program,state,callables,);+ yield;const asyncEffects = collectJavaScriptSemanticAsyncEffects(program,state,callables,);+ yield;const childProcesses = collectJavaScriptSemanticChildProcesses(program,state,callables,);+ yield;const dataEffects = collectJavaScriptSemanticDataEffects(program,state,callables,);+ yield;const resourceOperations = collectJavaScriptSemanticResources(program,state,callables,);+ yield;const objectOperations = collectJavaScriptSemanticObjects(program,state,callables,);+ yield;const functionFingerprints = collectJavaScriptSemanticFingerprints({state,callables,@@ -96,4 +107,4 @@ export const collectJavaScriptDerivedSemantics = (objectOperations,functionFingerprints,};-};+}